zillabyte 0.9.36 → 0.9.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7231040a821d67441f1e33bdb7ea719ea37111d
-  data.tar.gz: ca2c97df4729f8f6c251e8712e27beb2cb35e51a
+  metadata.gz: 8ee2c261c664c1674df0530fcf41f5b909998822
+  data.tar.gz: 06196fc228374cb2eb014c91b4b563dbe1f5f5e8
 SHA512:
-  metadata.gz: 1c4e2fcc1793f57db5bd85015a47401566d2c0a7b96cfd7220f3dc2135fbe02cb613d3c34a57f6ce9ee9fa280c18a09d7d35609d5ea0029afda1b17c2d8e52ad
-  data.tar.gz: ca468ce6b88861174acbe45440bec83b8f9156d44b66992d9be2a186d0b09762c675733b15aa2b31760b40020b4570ce3675d91114d11609821884515d1ca9ef
+  metadata.gz: df3498fec8e7c6aec0e146d7614ff0b41d8c9d8386c2c08237995352b142e13c642680bd6d4250b4aab6a112de57181a6890373f52dcb515277dfbc29c3e3bff
+  data.tar.gz: 0daa4f3157c1349ce402ce53848e06d0b70267fe5782ffdee2bad4883fe8d149ad829900b1bd6ae41b214923e23c5f2b768d94d9d04c8f609d60700f013d89ad

data/ruby/lib/zillabyte/harness.rb

@@ -3,6 +3,9 @@ module Zillabyte
   module Harness
 
     def self.load
+      require File.join(File.dirname(__FILE__), "harness", "app.rb")
+      require File.join(File.dirname(__FILE__), "harness", "base.rb")
+      require File.join(File.dirname(__FILE__), "harness", "common_node.rb")
       Dir[File.join(File.dirname(__FILE__), "harness", "*.rb")].sort.each do |file|
         require file
       end

data/ruby/lib/zillabyte/harness/clump.rb

@@ -1,34 +1,110 @@
+# OPERATION Clump
+# TAGLINE 
+# Clumps tuples together in the stream for faster micro-batch processing
+# DESCRIPTION 
+# The `clump` operation will collect a small 'clump' of tuples before passing it on to your code. This allows you to 
+# easily handle situations that require small bursts of computation.  For example, suppose you are accessing a third
+# party database and wish to query it in bursts of 100 commands at a time. 
+#  
+# RUBY_SYNTAX 
+# # Simplified syntax:
+#  
+#   stream.clump(:size => 100) do |tuples|
+#     # tuples is an array
+#     |=block=|
+#   end
+#  
+# # Custom clump:
+#   stream.clump do
+#     name "name"                          # the name of the operation
+#     size 100                             # the target size of the clump
+#     emits "stream_1", "stream_2"         # optional for single output stream
+#     output_format :replace               # :replace or :merge, optional, defaults to :replace
+#     prepare |=block=|                    # optional if no initialization needed
+#     execute |=block=|
+#   end
+#  
+# RUBY_NOTES 
+# The `size` parameter is optional.  It is the target size of the clump.  The system is guaranteed to not exceed `size`, but there are no guarantees about smaller clumps. If no `size` is given, then 100 is default.
+# 
+# The simplified syntax can be used if a prepare block and other customizations are not needed.
+# 
+#   * **Note:** this syntax only works for a single output stream.
+#  
+# The allowed output formats are :replace and :merge.
+#
+#   * **:replace** : discards the input tuple values and only emits the specified values. This is the default.
+#   * **:merge** : re-emits the input tuple values along with the specified values.
+#  
+# The "prepare" and "execute" blocks can be in do...end format or {} format.
+#
+#   * **"prepare"** block : where any setup is done to prepare for tuple processing in the "execute" block.
+#   * **"execute"** block : where the tuples are actually processed. It must take in a single argument (the "tuple").
+# 
+# RUBY_EXAMPLE 
+#  
+# # Sink data into an external sql database 
+# stream.clump(:size => 50) do |tuples|
+#   sql = "INSERT INTO table VALUES #{tuples.map{|t| t['value']}.join(',')}"
+#   @my_sql_service.execute(sql)
+# end
+#   
+
 class Zillabyte::Harness::Clump
-  attr_accessor :_name, :_type, :_emits, :_output_format, :_parallelism, :_prepare, :_execute
+  attr_accessor :_app, :_node, :_options
 
-  def initialize()
-    @_name = "clump_"+Zillabyte::Harness::Counter.get()
-    @_type = "clump"
-    @_output_format = :replace
-  end
-  
-  def name(v)
-    @_name = v
-  end
+  class Node < Zillabyte::Harness::CommonNode
+    attr_accessor :_output_format, :_execute, :_size
+
+    def initialize(v, options = {})
+      @_name = v[0] || "clump_"+Zillabyte::Harness::Counter.get()
+      @_type = "clump"
+      @_size = options[:size] || 100
+      @_output_format = :replace
+      self._emits = options[:emits]
+    end
   
-  def emits(*v)
-    @_emits = *v
-  end
+    def output_format(v)
+      @_output_format = v
+    end
 
-  def output_format(v)
-    @_output_format = v
+    def execute(&block)
+      @_execute = block
+    end
+    
+    def size(size)
+      @_size = size
+    end
   end
 
-  def parallelism(v)
-    @_parallelism = v
+  def initialize(app, *args)
+    @_app = app
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
   end
 
-  def prepare(&block)
-    @_prepare = block
-  end
+  def build_node(&block)
+    @_node = Node.new(@_args, @_options)
+    # Does the block take 0 arguments? If so it's not just an execute block.
+    if(block.arity == 0)
+      @_node.instance_eval(&block)
+      Zillabyte::Harness::Helper.check_name("clump", @_node._name, @_app._names)
+      if @_node._emits
+        Zillabyte::Harness::Helper.check_emits("clump", @_node._emits, @_app._streams)
+      else
+        @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      end
+    # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
+    else
+      @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]
+      @_node._execute = block
+    end
 
-  def execute(&block)
-    @_execute = block
+    Zillabyte::Harness::Helper.check_each(@_node)
   end
 
+  def run_operation
+    c = Zillabyte::Harness::EachController.new(@_node, Zillabyte::Common::Progress.new)
+    c.run(@_app._options)
+  end
 end
+

data/ruby/lib/zillabyte/harness/helper.rb

@@ -2,7 +2,6 @@ require 'optparse'
 
 class Zillabyte::Harness::Helper
 
-  META_NAMES = ["id", "confidence", "since", "source"]
   ALLOWED_OUTPUT_FORMATS = [:replace, :merge]
   ALLOWED_END_CYCLE_POLICIES = [:null_emit, :explicit, :infinite]
   ALLOWED_TYPES = [:string, :integer, :float, :double, :boolean, :array, :map]
@@ -342,10 +341,6 @@ class Zillabyte::Harness::Helper
       msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
-    if(cname =~ /^v[0-9]+$/i or META_NAMES.member?(cname.downcase))
-      msg = "#{ee}\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your field something else. #{pp}"
-      Zillabyte::Harness::Helper.print_error(msg)
-    end
     if(!ctype.instance_of?(Symbol))
       msg = "#{ee}Field data types must be SYMBOLS in \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -109,7 +109,6 @@ module Zillabyte
 
         stream = nil
         tuple = {}
-        meta = {}
 
         if (args.size == 1)
           tuple = args[0]
@@ -119,16 +118,13 @@ module Zillabyte
             tuple = args[1]
           elsif(args[0].instance_of?(Hash))
             tuple = args[0]
-            meta = args[1]
           end
         else
           stream = args[0]
           tuple = args[1]
-          meta = args[2]
         end
         
         if tuple.is_a?(Tuple)
-          meta = tuple.meta || {} if meta == {}
           tuple = tuple.values
         end
 
@@ -140,7 +136,7 @@ module Zillabyte
           end
         end
 
-        m = {:command => :emit, :tuple => tuple, :stream => stream, :meta => meta}
+        m = {:command => :emit, :tuple => tuple, :stream => stream}
         send_msg_to_parent m
       end
 
@@ -273,6 +269,11 @@ module Zillabyte
             if(m)
               if m['command'] && m['command'] == 'prepare'
                 prepare()
+              elsif m['tuples']
+                t_ary = m['tuples'].map do |t|
+                  Tuple.new(m)
+                end
+                execute t_ary
               else
                 t = Tuple.from_hash(m)
                 if(t)

data/ruby/lib/zillabyte/harness/operation_handler.rb

@@ -34,6 +34,8 @@ class Zillabyte::Harness::OperationHandler
       @_operation = Zillabyte::Harness::ComponentInput.new(@_app, *args)
     when "component_output"
       @_operation = Zillabyte::Harness::ComponentOutput.new(@_app, *args)
+    when "clump"
+      @_operation = Zillabyte::Harness::Clump.new(@_app, *args)
     else
       throw "Unknown type"
     end

data/ruby/lib/zillabyte/harness/stream.rb

@@ -181,16 +181,27 @@ class Zillabyte::Harness::Stream
   end
   
   # Groups together N arbitrary tuples (usefull for mini-batch processing)
-  def clump(*args)
+  def clump(*args, &block)
     op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
-    op.build_jvm_operation("clump", *args)
+    op.build_multilang_operation("clump", *args, &block)
       .add_operation_properties_to_info(:name, :type)
-      .add_input_args_to_info_as(:clump)
       .create_arc_info_from_stream(self)
       .handle_operation
       .get_output_streams
   end
   
+  def group_by(*args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_multilang_operation("group_by", *args, &block)
+      .add_operation_properties_to_info(:name, :type, :group_by)
+      .add_optional_operation_properties_to_info(:parallelism)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
+  end
+  
+  
+  
   # Renames fields..
   def rename(rename_map={})
     op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)

data/ruby/lib/zillabyte/harness/tuple.rb

@@ -36,10 +36,6 @@ class Tuple
       @hash.keys
   end
   
-  def meta
-    @hash["meta"]
-  end
-  
   def to_s
     values.to_s
   end
@@ -93,14 +89,6 @@ class Tuple
       fail "Missing tuple field in emitted JSON object?: #{e.message}\nJSON: #{hash}\nStacktrace:#{e.backtrace.join('\n\t')}"
       return nil
     end
-    begin
-      hash.fetch("meta")
-    rescue SignalException => e
-      raise
-    rescue Exception => e
-      fail "Missing meta field in emitted JSON object?:  #{e.message}\nJSON: #{hash}\nStacktrace:#{e.backtrace.join('\n\t')}"
-      return nil
-    end
     Tuple.new(hash)
   end
 end

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.9.36" unless defined?(VERSION)
+  VERSION = "0.9.37" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.9.36
+  version: 0.9.37
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-22 00:00:00.000000000 Z
+date: 2014-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.36
+        version: 0.9.37
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.36
+        version: 0.9.37
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com