zillabyte 0.0.6

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -0,0 +1,369 @@
+require "rubygems"
+require "json"
+
+module Zillabyte
+
+  module Storm
+
+    module Protocol
+
+      class << self
+        attr_accessor :mode, :pending_commands, :pipe_name, :pipe_to_java, :pipe_from_java
+      end
+
+      self.pending_commands = []
+
+      def setup_pipes
+        pipe_name = Storm::Protocol.pipe_name
+        if(pipe_name and File.exist?(pipe_name+".in"))
+          Storm::Protocol.pipe_to_java = File.open("#{pipe_name}.in","w+")
+        else
+          Storm::Protocol.pipe_to_java = $stdout
+        end
+
+        if(pipe_name and File.exist?(pipe_name+".out"))
+          Storm::Protocol.pipe_from_java = File.open("#{pipe_name}.out","r+")
+        else
+          Storm::Protocol.pipe_from_java = $stdin
+        end
+      end
+
+      def read_message
+        msg = ""
+        loop do
+          line = Storm::Protocol.pipe_from_java.readline.chomp#STDIN.readline.chomp
+          break if line == "end"
+          msg << line
+          msg << "\n"
+        end
+
+        encoding_options = {
+            :invalid           => :replace,  # Replace invalid byte sequences
+            :undef             => :replace,  # Replace anything not defined in ASCII
+            :replace           => ''        # Use a blank for those replacements
+          }
+        msg.encode! Encoding.find('ASCII'), encoding_options
+
+        JSON.parse msg.chomp
+      end
+
+      def read_command
+        Storm::Protocol.pending_commands.shift ||
+          begin
+            msg = read_message
+            while msg.is_a? Array
+              Storm::Protocol.pending_taskids.push(msg)
+              msg = read_message
+            end
+            msg
+          rescue JSON::ParserError => e
+            fail "error: #{e}"
+            {}
+          end
+      end
+
+      def send_msg_to_parent(msg)
+        # log msg
+        # log msg.to_json
+        Storm::Protocol.pipe_to_java.write(msg.to_json+"\n")
+        Storm::Protocol.pipe_to_java.write("end\n")
+        Storm::Protocol.pipe_to_java.flush
+      end
+
+      def send_pid(heartbeat_dir)
+        pid = Process.pid
+        send_msg_to_parent({'pid' => pid})
+        File.open("#{heartbeat_dir}/#{pid}", "w").close
+      end
+
+      def emit(*args)
+
+        stream = nil
+        tuple = nil
+        meta = {}
+
+        if (args.size == 1)
+          tuple = args[0]
+        elsif(args.size == 2)
+          stream = args[0]
+          tuple = args[1]
+        else
+          stream = args[0]
+          tuple = args[1]
+          meta = args[2]
+        end
+
+        m = {:command => :emit, :tuple => tuple, :stream => stream, :meta => meta}
+        send_msg_to_parent m
+      end
+
+      def ack(tup)
+        send_msg_to_parent :command => :ack, :id => tup.id
+      end
+
+      def done()
+        send_msg_to_parent :command => :done #, :id => tup.id
+        # read_task_ids
+      end
+
+      def fail(msg)
+        send_msg_to_parent :command => :fail, :msg => msg.to_s
+      end
+
+      def log(msg)
+        send_msg_to_parent :command => :log, :msg => msg.to_s
+      end
+
+      def handshake
+        setup_info = read_message
+        send_pid setup_info['pidDir']
+        [setup_info['conf'], setup_info['context']]
+      end
+    end
+
+
+    class Spout
+      include Storm::Protocol
+
+      def open(conf, context); end
+
+      def next_batch; end
+
+      def ack(id); end
+
+      def run(pipe_name)
+        Storm::Protocol.mode = 'spout'
+        Storm::Protocol.pipe_name = pipe_name
+        setup_pipes
+        open(*handshake)
+
+        begin
+          while true
+            msg = read_command
+            case msg['command']
+            when 'next'
+              next_batch
+              done
+            when 'ack'
+              ack(msg['id'])
+            when 'fail'
+              fail(msg['id'])
+            end
+          end
+        rescue Exception => e
+          fail 'Exception in spout: ' + e.message + ' - ' + e.backtrace.join('\n')
+        end
+      end
+    end
+
+    class Each
+      include Storm::Protocol
+
+      def prepare(conf, context); end
+
+      def execute(tuple); end
+
+      def run(pipe_name)
+        Storm::Protocol.mode = 'each'
+        Storm::Protocol.pipe_name = pipe_name
+        setup_pipes
+        prepare(*handshake)
+
+        begin
+          while true
+            t = nil
+            begin
+              t = Tuple.from_hash(read_command)
+              if(!t)
+                next
+              end
+              execute t
+            rescue Exception => e
+              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
+            ensure
+              done()
+            end
+          end
+        rescue Exception => e
+          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
+        end
+      end
+    end
+
+    class AggregateStart
+      include Storm::Protocol
+
+      def start(tuple); end
+
+      def run(pipe_name)
+        Storm::Protocol.mode = 'agg_start'
+        Storm::Protocol.pipe_name = pipe_name
+        setup_pipes
+        handshake
+
+        begin
+          while true
+            t = nil
+            begin
+              t = Tuple.from_hash(read_command)
+              if(!t)
+                next
+              end
+              start t
+            rescue Exception => e
+              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
+            ensure
+              done()
+            end
+          end
+        rescue Exception => e
+          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
+        end
+      end
+    end
+
+    class AggregateAggregate
+      include Storm::Protocol
+
+      def aggregate(tuple1, tuple2); end
+
+      def run(pipe_name)
+        Storm::Protocol.mode = 'agg_aggregate'
+        Storm::Protocol.pipe_name = pipe_name
+        setup_pipes
+        handshake
+
+        begin
+          while true
+            t = nil
+            begin
+              t1 = Tuple.from_hash(read_command)
+              if(!t1)
+                next
+              end
+              t2 = Tuple.from_hash(read_command)
+              if(!t2)
+                next
+              end
+              aggregate t1, t2
+            rescue Exception => e
+              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
+            ensure
+              done()
+            end
+          end
+        rescue Exception => e
+          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
+        end
+      end
+    end
+
+    class AggregateComplete
+      include Storm::Protocol
+
+      def complete(tuple); end
+
+      def run(pipe_name)
+        Storm::Protocol.mode = 'agg_complete'
+        Storm::Protocol.pipe_name = pipe_name
+        setup_pipes
+        handshake
+
+        begin
+          while true
+            t = nil
+            begin
+              t = Tuple.from_hash(read_command)
+              if(!t)
+                next
+              end
+              complete t
+            rescue Exception => e
+              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
+            ensure
+              done()
+            end
+          end
+        rescue Exception => e
+          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
+        end
+      end
+    end
+  end
+
+  module Harness
+
+    class SpoutController < Storm::Spout
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+      end
+
+      def open(*args)
+        @harness._prepare.call(self, *args) if @harness._prepare
+      end
+
+      def next_batch
+        @harness._next_batch.call(self)
+      end
+
+    end
+
+    class EachController < Storm::Each
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+      end
+
+      def execute(*args)
+        @harness._execute.call(self, *args)
+      end
+
+      def prepare(*args)
+        @harness._prepare.call(self, *args) if @harness._prepare
+      end
+
+    end
+
+    class AggregateStartController < Storm::AggregateStart
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+      end
+
+      def start(*args)
+        @harness._start.call(self, *args)
+      end
+
+    end
+
+    class AggregateAggregateController < Storm::AggregateAggregate
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+      end
+
+      def aggregate(*args)
+        @harness._aggregate.call(self, *args)
+      end
+
+    end
+
+    class AggregateCompleteController < Storm::AggregateComplete
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+      end
+
+      def complete(*args)
+        @harness._complete.call(self, *args)
+      end
+
+    end
+  end
+end

data/ruby/lib/zillabyte/harness/simple_function.rb

@@ -0,0 +1,131 @@
+require 'optparse'
+
+class Zillabyte::Harness::SimpleFunction
+  attr_accessor :_nodes, :_relation, :_matches, :_emits, :_prepare, :_execute, :_name, :_info_file, :_options
+
+  def self.build()
+    h = Zillabyte::Harness::SimpleFunction.new()
+    yield(h)
+    h._name = h._name || Dir.pwd.split("/")[-1]
+    Zillabyte::Harness::Helper.check_name("simple_function", h._name, {})
+    Zillabyte::Harness::Helper.check_emits("simple_function", h._emits, {})
+    generic_emits = h.get_generic_emits
+    
+    h._nodes = []
+    h._options = Zillabyte::Harness::Helper.opt_parser()
+
+    if(h._options[:command] == :info)
+      h._info_file = File.open(h._options[:file],"w+")
+      hash = {"language" => "ruby", "name" => h._name}
+      Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
+    end
+
+    h.build_spout()
+    fn = h.build_each(generic_emits)
+    h.build_sink()   
+
+    if(h._options[:command] == :execute and h._options[:name] == h._name)
+      pipe_name = h._options[:pipe]
+      c = Zillabyte::Harness::EachController.new(fn, progress = Zillabyte::Common::Progress.new)
+      c.run(pipe_name)
+    end
+
+    h
+  end
+
+  def name(v)
+    @_name = v
+  end
+
+  def matches(v, options = {})
+    case v
+    when String
+      @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
+    when Array
+      @_matches = v
+    end
+  end
+
+  def emits(v)
+    @_emits = v
+  end
+  
+  def prepare(&block)
+    @_prepare = block
+  end
+
+  def execute(&block)
+    @_execute = block
+  end
+  
+  def get_generic_emits()
+    generic_emits = []
+    @_emits.each do |relation|
+      temit = []
+      relation[1].each do |column|
+        column.each do |col, type|
+          temit << col
+        end
+      end
+      generic_emits << [relation[0], temit]
+    end
+    generic_emits
+  end
+
+  def build_spout()
+    h = Zillabyte::Harness::Spout.new(false)
+    h._matches = @_matches if @_matches
+    h._relation = @_relation if @_relation
+    @_nodes << h
+    if(@_options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type}
+      if(h._relation)
+        info_hash["relation"] = h._relation
+      elsif(h._matches)
+        info_hash["matches"] = h._matches
+      end
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+    end
+  end
+
+  def build_each(generic_emits)
+    h = Zillabyte::Harness::Each.new()
+    h._name = @_name
+    h._emits = generic_emits
+    h._prepare = @_prepare
+    h._execute = @_execute
+    @_nodes << h
+    if(@_options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+    end
+    h
+  end
+
+  def build_sink()
+    # Construct the sink...
+    n_sinks = @_emits.length
+    @_emits.each do |emit|
+      h = Zillabyte::Harness::Sink.new()
+      h._name = emit[0]
+      columns = emit[1]
+      columns.each do |col|
+        col.each do |cname, ctype|
+          h.column(cname, ctype)
+        end
+      end
+      if(n_sinks > 1)
+        h._consumes = h._name
+      end
+      @_nodes << h
+      if(@_options[:command] == :info)
+        info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
+        if(h._consumes)
+          info_hash["consumes"] = h._consumes
+        end
+        Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+      end
+    end
+  end
+
+end