zillabyte 0.0.16 → 0.0.17

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -5,10 +5,13 @@ module Zillabyte
 
   module Storm
 
+    class UserEndCycleException < Exception
+    end
+
     module Protocol
 
       class << self
-        attr_accessor :mode, :pending_commands, :pipe_name, :pipe_to_java, :pipe_from_java
+        attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :pipe_name, :pipe_to_java, :pipe_from_java
       end
 
       self.pending_commands = []
@@ -85,18 +88,46 @@ module Zillabyte
         if (args.size == 1)
           tuple = args[0]
         elsif(args.size == 2)
-          stream = args[0]
-          tuple = args[1]
+          if(args[0].instance_of?(String))
+            stream = args[0]
+            tuple = args[1]
+          elsif(args[0].instance_of?(Hash))
+            tuple = args[0]
+            meta = args[1]
+          end
         else
           stream = args[0]
           tuple = args[1]
           meta = args[2]
         end
 
+        if !stream.nil? and !Storm::Protocol.emits.member?(stream)
+          fail "Exception in #{Storm::Protocol.mode}: " + "\"#{stream}\" is not a declared stream. Please either remove it from \"emit\" (if there's only one output stream) or declare it in an \"emits\" clause.\n"
+        end
+        if stream.nil?
+          if Storm::Protocol.emits.size == 1
+            stream = Storm::Protocol.emits.first
+          else
+            fail "Exception in #{Storm::Protocol.mode}: " + "No stream specified in \"emit\" but multiple streams declared in \"emits\". Please specify which stream to emit to in \"emit\"."
+          end
+        end
+
         m = {:command => :emit, :tuple => tuple, :stream => stream, :meta => meta}
         send_msg_to_parent m
       end
 
+      def end_cycle()
+        if !Storm::Protocol.mode == "source"
+          fail "Exception in #{Storm::Protocol.mode}: " + "\"end_cycle\" is only valid in \"source\" operations."
+        else
+          if Storm::Protocol.end_cycle_policy == :explicit
+            raise UserEndCycleException, "cycle ended"
+      #    else
+      #      fail "Exception in source: " + "\"end_cycle\" received but \"end_cycle_policy\" was not set to \"explicit\"."
+          end
+        end
+      end
+
       def ack(tup)
         send_msg_to_parent :command => :ack, :id => tup.id
       end
@@ -125,9 +156,9 @@ module Zillabyte
     class Source
       include Storm::Protocol
 
-      def open(conf, context); end
+      def begin_cycle(conf, context); end
 
-      def next_batch; end
+      def next_tuple; end
 
       def ack(id); end
 
@@ -135,19 +166,24 @@ module Zillabyte
         Storm::Protocol.mode = 'source'
         Storm::Protocol.pipe_name = pipe_name
         setup_pipes
-        open(*handshake)
+        handshake
 
         while true
           begin
             msg = read_command
             case msg['command']
+            when 'begin_cycle'
+              begin_cycle
             when 'next'
-              next_batch
+              next_tuple
             when 'ack'
               ack(msg['id'])
             when 'fail'
               fail(msg['id'])
             end
+          rescue UserEndCycleException => e
+            m = {:command => :end_cycle}
+            send_msg_to_parent m
           rescue SignalException => e
             raise
           rescue Exception => e
@@ -181,7 +217,7 @@ module Zillabyte
             raise
           rescue Exception => e
             fail 'Exception in each: ' + e.message + ' - ' + e.backtrace.join('\n')
-            break
+            # We may recover from this, but let JVM decide
           end
           done
         end
@@ -229,18 +265,16 @@ module Zillabyte
       def initialize(harness, progress)
         @harness = harness
         @progress = progress
+        Storm::Protocol.emits = harness._emits
+        Storm::Protocol.end_cycle_policy = harness._end_cycle_policy
       end
 
-      def open(*args)
-        self.instance_exec(*args, &@harness._prepare) if @harness._prepare
-        #@harness._prepare.call(self, *args) if @harness._prepare
+      def begin_cycle(*args)
+        self.instance_exec(*args, &@harness._begin_cycle) if @harness._begin_cycle
       end
 
-      def next_batch
-        #@harness._next_batch.call(self)
-        # Executes the block @harness._next_batch in the context of the class self (i.e. exposes all methods of self to
-        # the block).
-        self.instance_eval &@harness._next_batch
+      def next_tuple
+        self.instance_eval &@harness._next_tuple
       end
 
     end
@@ -250,18 +284,15 @@ module Zillabyte
       def initialize(harness, progress)
         @harness = harness
         @progress = progress
+        Storm::Protocol.emits = harness._emits
       end
 
       def execute(*args)
-        #@harness._execute.call(self, *args)
-        # Executes the block @harness._next_batch in the context of the class self (i.e. exposes all methods of self to
-        # the block). Also allows you to pass in arguments to the block in *args.
         self.instance_exec *args, &@harness._execute
       end
 
       def prepare(*args)
         self.instance_exec(*args, &@harness._prepare) if @harness._prepare
-        #@harness._prepare.call(self, *args) if @harness._prepare
       end
 
     end
@@ -271,20 +302,18 @@ module Zillabyte
       def initialize(harness, progress)
         @harness = harness
         @progress = progress
+        Storm::Protocol.emits = harness._emits
       end
 
       def begin_group(*args)
-        #@harness._begin_group.call(*args)
         self.instance_exec *args, &@harness._begin_group
       end
 
       def aggregate(*args)
-        #@harness._aggregate.call(*args)
         self.instance_exec *args, &@harness._aggregate
       end
 
       def end_group(*args)
-        #@harness._end_group.call(self, *args)
         self.instance_exec *args, &@harness._end_group
       end
 

data/ruby/lib/zillabyte/harness/source.rb

@@ -1,10 +1,10 @@
 class Zillabyte::Harness::Source
-  attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_prepare, :_next_batch, :_flow_flag
+  attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_end_cycle_policy, :_begin_cycle, :_next_tuple
 
-  def initialize(flow_flag)
+  def initialize()
     @_name = "source_"+Zillabyte::Harness::Counter.get()
     @_type = 'source'
-    @_flow_flag = flow_flag
+    @_end_cycle_policy = :null_emit
   end
 
   def matches(v, options = {})
@@ -20,29 +20,23 @@ class Zillabyte::Harness::Source
     @_name = v
   end
 
-  def emits(v)
-    if(@_flow_flag)
-      @_emits = v
-    else
-      @_emits = []
-      v.each do |relation|
-        temit = []
-        relation[1].each do |column|
-          column.each do |col, type|
-            temit << col
-          end
-        end
-        @_emits << [relation[0], temit]
-      end
+  def emits(*v)
+    @_emits = *v
+  end
+
+  def end_cycle_policy(v)
+    allowed_policies = [:null_emit, :explicit]
+    if(allowed_policies.member?(v))
+      @_end_cycle_policy = v
     end
   end
 
-  def prepare(&block)
-    @_prepare = block
+  def begin_cycle(&block)
+    @_begin_cycle = block
   end
 
-  def next_batch(&block)
-    @_next_batch = block
+  def next_tuple(&block)
+    @_next_tuple = block
   end
 
 end

data/ruby/lib/zillabyte/harness/stream.rb

@@ -0,0 +1,86 @@
+require "json"
+
+class Zillabyte::Harness::Stream
+  attr_accessor :_name, :_app, :_previous_node_name
+
+  def initialize(name, app, previous_node_name)
+    @_name = name 
+    @_app = app
+    @_previous_node_name = previous_node_name
+  end
+
+  def each(&block)
+    h = Zillabyte::Harness::Each.new()
+    # Does the block take 0 arguments? If so it's not just an execute block.
+    if(block.arity == 0)
+      h.instance_eval(&block)
+    # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
+    else
+      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      h._has_emit = true
+      h._execute = block
+    end
+    Zillabyte::Harness::Helper.check_name("each", h._name, {})
+    #Zillabyte::Harness::Helper.check_emits("each", h._emits, @_streams) || @_branched
+    @_app._nodes << h
+    if(@_app._options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
+      info_hash["emits"] = h._emits
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      pipe_name = @_app._options[:pipe]
+      c = Zillabyte::Harness::EachController.new(h, progress = Zillabyte::Common::Progress.new)
+      c.run(pipe_name)
+    end
+
+    output_streams = []
+    h._emits.each do |stream|
+      output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
+    end
+    output_streams = output_streams[0] if output_streams.size == 1
+    output_streams
+  end
+
+  def aggregate(&block)
+    h = Zillabyte::Harness::Aggregate.new()
+    #yield(h)
+    h.instance_eval(&block)
+    Zillabyte::Harness::Helper.check_name("aggregate", h._name, @_names)
+    if(@_branched)
+      Zillabyte::Harness::Helper.check_consumes(h, @_streams)
+    else
+      if(h._consumes)
+        h._consumes = nil
+      end
+    end
+    Zillabyte::Harness::Helper.check_group_by("aggregate", h, @_nodes, @_streams)
+    @_branched = Zillabyte::Harness::Helper.check_emits("aggregate", h._emits, @_streams) || @_branched
+    @_nodes << h
+    if(@_options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "group_by" => h._group_by, "emits" => h._emits}
+      if(h._consumes)
+        info_hash["consumes"] = h._consumes
+      end
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+    elsif(@_options[:command] == :execute)
+      pipe_name = @_options[:pipe]
+      if(@_options[:name] == h._name)
+        c = Zillabyte::Harness::AggregateController.new(h, Zillabyte::Common::Progress.new)
+        c.run(pipe_name)
+      end
+    end
+  end
+
+  def sink(&block)
+    h = Zillabyte::Harness::Sink.new()
+    #yield(h)
+    h.instance_eval(&block)
+#    Zillabyte::Harness::Helper.check_sink(h, @_nodes)
+    @_app._nodes << h
+    if(@_app._options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name}
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+    end
+  end
+  
+end

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.0.16" unless defined?(VERSION)
+  VERSION = "0.0.17" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.17
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-01 00:00:00.000000000 Z
+date: 2014-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.10
+        version: 0.0.17
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.10
+        version: 0.0.17
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -61,17 +61,15 @@ extra_rdoc_files: []
 files:
 - ruby/lib/zillabyte/common/progress.rb
 - ruby/lib/zillabyte/harness/aggregate.rb
+- ruby/lib/zillabyte/harness/app.rb
 - ruby/lib/zillabyte/harness/counter.rb
 - ruby/lib/zillabyte/harness/each.rb
 - ruby/lib/zillabyte/harness/groupby.rb
 - ruby/lib/zillabyte/harness/helper.rb
 - ruby/lib/zillabyte/harness/live_delegator.rb
-- ruby/lib/zillabyte/harness/simple_aggregate.rb
-- ruby/lib/zillabyte/harness/simple_function.rb
-- ruby/lib/zillabyte/harness/simple_source.rb
 - ruby/lib/zillabyte/harness/sink.rb
 - ruby/lib/zillabyte/harness/source.rb
-- ruby/lib/zillabyte/harness/topology.rb
+- ruby/lib/zillabyte/harness/stream.rb
 - ruby/lib/zillabyte/harness/tuple.rb
 - ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/version.rb
@@ -97,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.1.10
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte Gem

data/ruby/lib/zillabyte/harness/simple_aggregate.rb

@@ -1,144 +0,0 @@
-require 'optparse'
-
-class Zillabyte::Harness::SimpleAggregate
-  attr_accessor :_nodes, :_relation, :_matches, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group, :_name, :_info_file, :_options
-
-  def self.build(*args, &block)
-    h = Zillabyte::Harness::SimpleAggregate.new()
-    #yield(h)
-    h.instance_eval(&block)
-    h._name = h._name
-    Zillabyte::Harness::Helper.check_name("simple_aggregate", h._name, {})
-    Zillabyte::Harness::Helper.check_emits("simple_aggregate", h._emits, {})
-    generic_emits = h.get_generic_emits
-    
-    h._nodes = []
-    h._options = Zillabyte::Harness::Helper.opt_parser()
-
-    if(h._options[:command] == :info)
-      h._info_file = File.open(h._options[:file],"w+")
-      hash = {"language" => "ruby", "name" => h._name}
-      Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
-    end
-
-    h.build_source()
-    fn = h.build_aggregate(generic_emits)
-    h.build_sink()   
-
-    if(h._options[:command] == :execute and h._options[:name] == h._name)
-      pipe_name = h._options[:pipe]
-      c = Zillabyte::Harness::AggregateController.new(fn, progress = Zillabyte::Common::Progress.new)
-      c.run(pipe_name)
-    end
-
-    h
-  end
-
-  def name(v)
-    @_name = v
-  end
-
-  def matches(v, options = {})
-    case v
-    when String
-      @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
-    when Array
-      @_matches = v
-    end
-  end
-
-  def group_by(v)
-    @_group_by = v
-  end
-
-  def emits(v)
-    @_emits = v
-  end
-  
-  def begin_group(&block)
-    @_begin_group = block
-  end
-
-  def aggregate(&block)
-    @_aggregate = block
-  end
-  
-  def end_group(&block)
-    @_end_group = block
-  end
-
-  def get_generic_emits()
-    generic_emits = []
-    @_emits.each do |relation|
-      temit = []
-      relation[1].each do |column|
-        column.each do |col, type|
-          temit << col
-        end
-      end
-      generic_emits << [relation[0], temit]
-    end
-    generic_emits
-  end
-
-  def build_source()
-    h = Zillabyte::Harness::Source.new(false)
-    h._matches = @_matches if @_matches
-    h._relation = @_relation if @_relation
-    Zillabyte::Harness::Helper.check_source("simple_aggregate", h)
-    @_nodes << h
-    if(@_options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type}
-      if(h._relation)
-        info_hash["relation"] = h._relation
-      elsif(h._matches)
-        info_hash["matches"] = h._matches
-      end
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
-    end
-  end
-
-  def build_aggregate(generic_emits)
-    h = Zillabyte::Harness::Aggregate.new()
-    h._name = @_name
-    h._emits = generic_emits
-    h._group_by = @_group_by
-    h._begin_group = @_begin_group
-    h._aggregate = @_aggregate
-    h._end_group = @_end_group
-    Zillabyte::Harness::Helper.check_group_by("simple_aggregate", h, @_nodes, {})
-    @_nodes << h
-    if(@_options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits, "group_by" => h._group_by}
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
-    end
-    h
-  end
-
-  def build_sink()
-    # Construct the sink...
-    n_sinks = @_emits.length
-    @_emits.each do |emit|
-      h = Zillabyte::Harness::Sink.new()
-      h._name = emit[0]
-      columns = emit[1]
-      columns.each do |col|
-        col.each do |cname, ctype|
-          h.column(cname, ctype)
-        end
-      end
-      if(n_sinks > 1)
-        h._consumes = h._name
-      end
-      @_nodes << h
-      if(@_options[:command] == :info)
-        info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
-        if(h._consumes)
-          info_hash["consumes"] = h._consumes
-        end
-        Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
-      end
-    end
-  end
-
-end