zillabyte 0.0.11 → 0.0.12

data/ruby/lib/zillabyte/harness/aggregate.rb

@@ -1,5 +1,5 @@
 class Zillabyte::Harness::Aggregate
-  attr_accessor :_name, :_type, :_emits, :_start, :_aggregate, :_complete
+  attr_accessor :_name, :_type, :_consumes, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
 
   def initialize()
     @_name = "aggregate_"+Zillabyte::Harness::Counter.get()
@@ -10,20 +10,28 @@ class Zillabyte::Harness::Aggregate
     @_name = v
   end
   
+  def consumes(v)
+    @_consumes = v
+  end
+
+  def group_by(v)
+    @_group_by = v
+  end
+
   def emits(v)
     @_emits = v
   end
 
-  def start(&block)
-    @_start = block
+  def begin_group(&block)
+    @_begin_group = block
   end
 
   def aggregate(&block)
     @_aggregate = block
   end
 
-  def complete(&block)
-    @_complete = block
+  def end_group(&block)
+    @_end_group = block
   end
 
 end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -50,7 +50,7 @@ class Zillabyte::Harness::Helper
 
   def self.check_emits(operation, emits, streams)
     ee = "Error in \"#{operation}\" at \"emits\": \n\t "
-    if(operation == "simple_function" or operation == "simple_spout")
+    if(operation.include?("simple"))
       pp = @@_print_check_simple_function_emits
       nn = "relation"
     else
@@ -93,8 +93,8 @@ class Zillabyte::Harness::Helper
         Zillabyte::Harness::Helper.print_error(msg)
       end
 
-      if(operation == "simple_function" or operation == "simple_spout")
-        Zillabyte::Harness::Helper.check_simple_function_emits(e)
+      if(operation.include?("simple"))
+        Zillabyte::Harness::Helper.check_simple_function_emits(operation, e)
       else
         if(!e[1].instance_of?(Array))
           msg = "#{ee}Field names must be an ARRAY of STRINGS in stream #{e[0]}. #{pp}"
@@ -117,17 +117,18 @@ class Zillabyte::Harness::Helper
   end
 
   def self.check_consumes(h, streams)
+    ee = "Error in \"#{h._type}\" at \"consumes\": \n\t "
     if(h._type == "each") 
-      ee = "Error in \"each\" at \"consumes\": \n\t "
       pp = @@_print_check_each_consumes
+    elsif(h._type == "aggregate")
+      pp = @@_print_check_aggregate_consumes
     elsif(h._type == "sink")
-      ee = "Error in \"sink\" at \"consumes\": \n\t "
       pp = @@_print_check_sink
     end
 
     consumes = h._consumes
     if(!consumes)
-      msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\" or \"spout\" emitted multiple streams. #{pp}"
+      msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\", \"spout\" or \"aggregate\" emitted multiple streams. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(!consumes.instance_of?(String) or consumes == "")
@@ -144,6 +145,40 @@ class Zillabyte::Harness::Helper
     end
   end
 
+  def self.check_group_by(operation, h, nodes, streams)
+    ee = "Error in \"#{operation}\" at \"group_by\": \n\t "
+    pp = ""
+    if(operation == "aggregate")
+      pp = @@_print_check_group_by
+    end
+
+    group_by = h._group_by
+    if(!group_by.instance_of?(Array) or group_by == [])
+      msg = "#{ee}Group_by must be an ARRAY with at least one element! #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    if(operation == "aggregate")
+# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from spout!
+      group_by.each do |f|
+        if(h._consumes)
+          fields = streams[h._consumes]
+          if(!fields.include?(f))
+            msg = "#{ee}Group_by field not in consumed stream \"#{h._consumes}\". #{pp}"
+            Zillabyte::Harness::Helper.print_error(msg)
+          end
+        else
+          previous_emits = nodes.last._emits[0] #if no "consumes", then operation consumes the sole stream from the previous operation
+          if(!previous_emits[1].include?(f))
+            msg = "#{ee}Group_by field not in \"emits\" of previous relation \"#{previous_emits[0]}\". #{pp}"
+            Zillabyte::Harness::Helper.print_error(msg)
+          end
+        end
+      end
+    end
+      
+  end
+
   def self.check_sink(sink, nodes)
     ee = "Error in \"sink\": \n\t "
     pp = @@_print_check_sink
@@ -187,8 +222,8 @@ class Zillabyte::Harness::Helper
     if(operation == "sink")
       ee = "Error in \"sink\" at \"column\": \n\t "
       pp = @@_print_check_sink
-    elsif(operation == "simple_function")
-      ee = "Error in \"simple_function\" at \"emits\": \n\t "
+    elsif(operation.include?("simple"))
+      ee = "Error in \"#{operation}\" at \"emits\": \n\t "
       pp = @@_print_check_simple_function_emits
     end
 
@@ -228,8 +263,8 @@ class Zillabyte::Harness::Helper
     end
   end
 
-  def self.check_simple_function_emits(emits)
-    ee = "Error in \"simple_function\" at \"emits\": \n\t "
+  def self.check_simple_function_emits(operation, emits)
+    ee = "Error in \"#{operation}\" at \"emits\": \n\t "
     pp = @@_print_check_simple_function_emits
 
     name = emits[0]
@@ -250,7 +285,7 @@ class Zillabyte::Harness::Helper
       end
       colkey = colkeys[0]
       colval = col[colkey]
-      Zillabyte::Harness::Helper.check_sink_column_format("simple_function",colkey,colval,name)
+      Zillabyte::Harness::Helper.check_sink_column_format(operation,colkey,colval,name)
     end
   end
   
@@ -277,6 +312,12 @@ class Zillabyte::Harness::Helper
  \t           [ \"stream_2\", [ \"field_21\", \"field_22\", ... ] ] ] .
  - Stream and field names must all be non-empty STRINGS."
 
+  @@_print_check_group_by = "\n
+\"Group_by\" Syntax:
+ - \"Group_by\" must be a non-empty ARRAY.
+ - Each element of \"group_by\" must be a STRING corresponding to a field emitted by the previous operation or the stream
+consumed by the aggregation function."
+
   @@_print_check_simple_function_emits = "\n
 \"Emits\" Syntax:
  - \"Emits\" must be a non-empty ARRAY.
@@ -294,15 +335,15 @@ class Zillabyte::Harness::Helper
      Single stream:
  \t flow.sink do |h|
  \t \t h.name \"name_of_relation\"
- \t \t h.columns \"field_1\" :type_1
- \t \t h.columns \"field_2\" :type_2 ...
+ \t \t h.column \"field_1\" :type_1
+ \t \t h.column \"field_2\" :type_2 ...
  \t end
      Multiple streams:
  \t flow.sink do |h|
  \t \t h.name \"relation_name\"
  \t \t h.consumes \"stream_consumed\"
- \t \t h.columns \"field_1\" :type_1
- \t \t h.columns \"field_2\" :type_2 ...
+ \t \t h.column \"field_1\" :type_1
+ \t \t h.column \"field_2\" :type_2 ...
  \t end
  - \"Sink\" relation \"name\" must be specified as a non-empty STRING!
  - Field names must be non-empty STRINGS.
@@ -321,6 +362,20 @@ class Zillabyte::Harness::Helper
  \t \t h.execute ...
  \t end
  - If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
-     * \"Consumes\" is the name of a stream emitted by a preceding \"each\" or \"spout\" which the current \"each\" operates on."
+     * \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"spout\" or \"aggregate\" which the current \"each\" operates on."
+
+  @@_print_check_aggregate_consumes = "\n
+\"Aggregate\" Syntax for multiple streams:
+ \t flow.aggregate do |h|
+ \t \t h.name \"name\",  => optional
+ \t \t h.group_by [\"field_1\", \"field_2\", ...],
+ \t \t h.emits emits,
+ \t \t h.consumes \"consumed_stream\"
+ \t \t h.begin_group ...
+ \t \t h.aggregate ...
+ \t \t h.end_group ...
+ \t end
+ - If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
+     * \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"spout\" or \"aggregate\" which the current \"aggregate\" operates on."
 
 end

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -190,102 +190,32 @@ module Zillabyte
       end
     end
 
-    class AggregateStart
+    class Aggregate
       include Storm::Protocol
 
-      def start(tuple); end
-
-      def run(pipe_name)
-        Storm::Protocol.mode = 'agg_start'
-        Storm::Protocol.pipe_name = pipe_name
-        setup_pipes
-        handshake
-
-        begin
-          while true
-            t = nil
-            begin
-              t = Tuple.from_hash(read_command)
-              if(!t)
-                next
-              end
-              start t
-            rescue Exception => e
-              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
-            ensure
-              done()
-            end
-          end
-        rescue Exception => e
-          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
-        end
-      end
-    end
-
-    class AggregateAggregate
-      include Storm::Protocol
-
-      def aggregate(tuple1, tuple2); end
-
       def run(pipe_name)
-        Storm::Protocol.mode = 'agg_aggregate'
+        Storm::Protocol.mode = 'aggregation'
         Storm::Protocol.pipe_name = pipe_name
         setup_pipes
         handshake
 
         begin
           while true
-            t = nil
-            begin
-              t1 = Tuple.from_hash(read_command)
-              if(!t1)
-                next
-              end
-              t2 = Tuple.from_hash(read_command)
-              if(!t2)
-                next
-              end
-              aggregate t1, t2
-            rescue Exception => e
-              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
-            ensure
-              done()
-            end
-          end
-        rescue Exception => e
-          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
-        end
-      end
-    end
-
-    class AggregateComplete
-      include Storm::Protocol
-
-      def complete(tuple); end
-
-      def run(pipe_name)
-        Storm::Protocol.mode = 'agg_complete'
-        Storm::Protocol.pipe_name = pipe_name
-        setup_pipes
-        handshake
-
-        begin
-          while true
-            t = nil
-            begin
-              t = Tuple.from_hash(read_command)
-              if(!t)
-                next
-              end
-              complete t
-            rescue Exception => e
-              fail 'Exception in bolt (A): ' + e.message + ' - ' + e.backtrace.join('\n')
-            ensure
-              done()
+            msg = read_command
+            case msg['command']
+            when 'begin_group'
+              t = Tuple.from_hash(msg)
+              begin_group t
+            when 'aggregate'
+              t = Tuple.from_hash(msg)
+              aggregate t
+            when 'end_group'
+              end_group
             end
+            done
           end
         rescue Exception => e
-          fail 'Exception in bolt (B): ' + e.message + ' - ' + e.backtrace.join('\n')
+          fail "Exception in #{msg["command"]}: " + e.message + ' - ' + e.backtrace.join('\n')
         end
       end
     end
@@ -327,43 +257,26 @@ module Zillabyte
 
     end
 
-    class AggregateStartController < Storm::AggregateStart
+    class AggregateController < Storm::Aggregate
 
       def initialize(harness, progress)
         @harness = harness
         @progress = progress
       end
 
-      def start(*args)
-        @harness._start.call(self, *args)
-      end
-
-    end
-
-    class AggregateAggregateController < Storm::AggregateAggregate
-
-      def initialize(harness, progress)
-        @harness = harness
-        @progress = progress
+      def begin_group(*args)
+        @harness._begin_group.call(*args)
       end
 
       def aggregate(*args)
-        @harness._aggregate.call(self, *args)
-      end
-
-    end
-
-    class AggregateCompleteController < Storm::AggregateComplete
-
-      def initialize(harness, progress)
-        @harness = harness
-        @progress = progress
+        @harness._aggregate.call(*args)
       end
 
-      def complete(*args)
-        @harness._complete.call(self, *args)
+      def end_group(*args)
+        @harness._end_group.call(self, *args)
       end
 
     end
+
   end
 end

data/ruby/lib/zillabyte/harness/simple_aggregate.rb

@@ -0,0 +1,142 @@
+require 'optparse'
+
+class Zillabyte::Harness::SimpleAggregate
+  attr_accessor :_nodes, :_relation, :_matches, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group, :_name, :_info_file, :_options
+
+  def self.build()
+    h = Zillabyte::Harness::SimpleAggregate.new()
+    yield(h)
+    h._name = h._name || Dir.pwd.split("/")[-1]
+    Zillabyte::Harness::Helper.check_name("simple_aggregate", h._name, {})
+    Zillabyte::Harness::Helper.check_emits("simple_aggregate", h._emits, {})
+    generic_emits = h.get_generic_emits
+    
+    h._nodes = []
+    h._options = Zillabyte::Harness::Helper.opt_parser()
+
+    if(h._options[:command] == :info)
+      h._info_file = File.open(h._options[:file],"w+")
+      hash = {"language" => "ruby", "name" => h._name}
+      Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
+    end
+
+    h.build_spout()
+    fn = h.build_aggregate(generic_emits)
+    h.build_sink()   
+
+    if(h._options[:command] == :execute and h._options[:name] == h._name)
+      pipe_name = h._options[:pipe]
+      c = Zillabyte::Harness::AggregateController.new(fn, progress = Zillabyte::Common::Progress.new)
+      c.run(pipe_name)
+    end
+
+    h
+  end
+
+  def name(v)
+    @_name = v
+  end
+
+  def matches(v, options = {})
+    case v
+    when String
+      @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
+    when Array
+      @_matches = v
+    end
+  end
+
+  def group_by(v)
+    @_group_by = v
+  end
+
+  def emits(v)
+    @_emits = v
+  end
+  
+  def begin_group(&block)
+    @_begin_group = block
+  end
+
+  def aggregate(&block)
+    @_aggregate = block
+  end
+  
+  def end_group(&block)
+    @_end_group = block
+  end
+
+  def get_generic_emits()
+    generic_emits = []
+    @_emits.each do |relation|
+      temit = []
+      relation[1].each do |column|
+        column.each do |col, type|
+          temit << col
+        end
+      end
+      generic_emits << [relation[0], temit]
+    end
+    generic_emits
+  end
+
+  def build_spout()
+    h = Zillabyte::Harness::Spout.new(false)
+    h._matches = @_matches if @_matches
+    h._relation = @_relation if @_relation
+    @_nodes << h
+    if(@_options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type}
+      if(h._relation)
+        info_hash["relation"] = h._relation
+      elsif(h._matches)
+        info_hash["matches"] = h._matches
+      end
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+    end
+  end
+
+  def build_aggregate(generic_emits)
+    h = Zillabyte::Harness::Aggregate.new()
+    h._name = @_name
+    h._emits = generic_emits
+    h._group_by = @_group_by
+    h._begin_group = @_begin_group
+    h._aggregate = @_aggregate
+    h._end_group = @_end_group
+    Zillabyte::Harness::Helper.check_group_by("simple_aggregate", h, @_nodes, {})
+    @_nodes << h
+    if(@_options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits, "group_by" => h._group_by}
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+    end
+    h
+  end
+
+  def build_sink()
+    # Construct the sink...
+    n_sinks = @_emits.length
+    @_emits.each do |emit|
+      h = Zillabyte::Harness::Sink.new()
+      h._name = emit[0]
+      columns = emit[1]
+      columns.each do |col|
+        col.each do |cname, ctype|
+          h.column(cname, ctype)
+        end
+      end
+      if(n_sinks > 1)
+        h._consumes = h._name
+      end
+      @_nodes << h
+      if(@_options[:command] == :info)
+        info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
+        if(h._consumes)
+          info_hash["consumes"] = h._consumes
+        end
+        Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+      end
+    end
+  end
+
+end

data/ruby/lib/zillabyte/harness/topology.rb

@@ -78,31 +78,30 @@ class Zillabyte::Harness::Topology
     end
   end
 
-  def group_by(fields)
-    h = Zillabyte::Harness::GroupBy.new(fields)
-    @_nodes << h
-  end
-
   def aggregate
     h = Zillabyte::Harness::Aggregate.new()
     yield(h)
     Zillabyte::Harness::Helper.check_name("aggregate", h._name, @_names)
+    if(@_branched)
+      Zillabyte::Harness::Helper.check_consumes(h, @_streams)
+    else
+      if(h._consumes)
+        h._consumes = nil
+      end
+    end
+    Zillabyte::Harness::Helper.check_group_by("aggregate", h, @_nodes, @_streams)
     @_branched = Zillabyte::Harness::Helper.check_emits("aggregate", h._emits, @_streams) || @_branched
     @_nodes << h
     if(@_options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
+      info_hash = {"name" => h._name, "type" => h._type, "group_by" => h._group_by, "emits" => h._emits}
       if(h._consumes)
         info_hash["consumes"] = h._consumes
       end
       Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
     elsif(@_options[:command] == :execute)
       pipe_name = @_options[:pipe]
-      if(@_options[:name] == h._name+"_start")
-        c = Zillabyte::Harness::AggregateStartController.new(h, Zillabyte::Common::Progress.new)
-      elsif(@_options[:name] == h._name+"_aggregate")
-        c = Zillabyte::Harness::AggregateAggregateController.new(h, Zillabyte::Common::Progress.new)
-      elsif(@_options[:name] == h._name+"_complete")
-        c = Zillabyte::Harness::AggregateCompleteController.new(h, Zillabyte::Common::Progress.new)
+      if(@_options[:name] == h._name)
+        c = Zillabyte::Harness::AggregateController.new(h, Zillabyte::Common::Progress.new)
       end
       c.run(pipe_name)
     end

data/ruby/lib/zillabyte/harness/tuple.rb

@@ -7,6 +7,7 @@ class Tuple
   
   def [](name)
     maybe_build_alias_hash()
+p name
     values[ @_alias_hash[name] || name ]
   end
   

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.0.11" unless defined?(VERSION)
+  VERSION = "0.0.12" unless defined?(VERSION)
 end

data/ruby/lib/zillabyte/version.rb~

@@ -0,0 +1,3 @@
+module Zillabyte
+  VERSION = "0.0.11" unless defined?(VERSION)
+end

data/ruby/lib/zillabyte.rb

@@ -16,5 +16,8 @@ module Zillabyte
     Zillabyte::Harness::SimpleSpout.build(*args, &block)
   end
 
+  def self.simple_aggregate(*args, &block)
+    Zillabyte::Harness::SimpleAggregate.build(*args, &block)
+  end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-15 00:00:00.000000000 Z
+date: 2014-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -73,6 +73,7 @@ files:
 - ruby/lib/zillabyte/harness/groupby.rb
 - ruby/lib/zillabyte/harness/helper.rb
 - ruby/lib/zillabyte/harness/live_delegator.rb
+- ruby/lib/zillabyte/harness/simple_aggregate.rb
 - ruby/lib/zillabyte/harness/simple_function.rb
 - ruby/lib/zillabyte/harness/simple_spout.rb
 - ruby/lib/zillabyte/harness/sink.rb
@@ -81,6 +82,7 @@ files:
 - ruby/lib/zillabyte/harness/tuple.rb
 - ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/version.rb
+- ruby/lib/zillabyte/version.rb~
 - ruby/lib/zillabyte.rb
 - ruby/README.md
 homepage: http://www.zillabyte.com