zillabyte 0.1.29 → 0.1.31

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: c56762d0640d1c5508e38e60e1355135782355c5
-  data.tar.gz: 8e9c679dd7253bd675dcd4a5211341dba9ebbf8b
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZGI5MDc1YmU2MzlmNzUxZDBhMzJiMjgxNzVhOGJjMTMxNDUwOGIxMw==
+  data.tar.gz: !binary |-
+    NzE0NzI4OTU0YzQ4YjVhNjQyMDljOTgyMGY3MjJiNmRhMzMyNzI0Ng==
 SHA512:
-  metadata.gz: b4272038da2c89843ffcd7605de3b9d7bf3360b5e71d303be565edab0a616660ad54c5376179e5a9bcb7d887c73d8751b0d79c2b19e6ca6b9e14b05634c5b00c
-  data.tar.gz: 2a22c7f7f0b2ded8a77d1c8ce032b8ca8bbf1c9b43269798235fdeb1d61abe27d3eb80fe67663be3b03db51e084b2607f67060e936233f82a9415c435d789485
+  metadata.gz: !binary |-
+    MTMyMDU4OGRiMjA1ZjIxMTk3MjllNTZkNDA1MDVlOTQyZmM5NzQ2NTdhNWEw
+    YTg3OTU2ZGEwNzY2ZmUzMjI5ZGQwMjJhY2ZkYTcwYWI3MjIyOWQ2N2Q3MzQ3
+    MzY3ZmQ0NzYyY2FhNmNjOWZhMmRiMzNiNmI4ZGRkMTRiZTgxOGI=
+  data.tar.gz: !binary |-
+    N2E5NDEyM2QwYzViODU5ODU1Mzg1YTRlMjVlMDU2YmJiYjAzN2MxMjI0OTRl
+    ZTQ4OTQ4NWQwMjU4ZWExZjc4YjY1MDllYTMyNzc2ZDVkODkwZTA0YTU3MWI1
+    Y2ZkYmZiNjMyMGQxNmI4MTdlMmY5N2FkNDM3OTQ5MzU4NmJiYTU=

data/ruby/lib/zillabyte/harness/clump.rb

@@ -0,0 +1,34 @@
+class Zillabyte::Harness::Clump
+  attr_accessor :_name, :_type, :_emits, :_output_format, :_parallelism, :_prepare, :_execute
+
+  def initialize()
+    @_name = "clump_"+Zillabyte::Harness::Counter.get()
+    @_type = "clump"
+    @_output_format = :replace
+  end
+  
+  def name(v)
+    @_name = v
+  end
+  
+  def emits(*v)
+    @_emits = *v
+  end
+
+  def output_format(v)
+    @_output_format = v
+  end
+
+  def parallelism(v)
+    @_parallelism = v
+  end
+
+  def prepare(&block)
+    @_prepare = block
+  end
+
+  def execute(&block)
+    @_execute = block
+  end
+
+end

data/ruby/lib/zillabyte/harness/component.rb

@@ -17,6 +17,7 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
   def inputs(*args, &block)
     h = Zillabyte::Harness::ComponentSource.new
     h.instance_eval(&block)
+    h._name ||= "comp_input.#{Zillabyte::Harness::Counter.get()}"
     Zillabyte::Harness::Helper.check_name("inputs", h._name, @_names)
     Zillabyte::Harness::Helper.check_component_source(h)
 

data/ruby/lib/zillabyte/harness/component_stream.rb

@@ -2,9 +2,10 @@ require 'zillabyte/harness/stream'
 
 class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
 
-  def sink(*args, &block)
+  def outputs(*args, &block)
     h = Zillabyte::Harness::ComponentSink.new
     h.instance_eval(&block)
+    h._name ||= "comp_output.#{Zillabyte::Harness::Counter.get()}"
     Zillabyte::Harness::Helper.check_name("outputs", h._name, @_app._names)
     Zillabyte::Harness::Helper.check_component_sink(h)
 
@@ -15,6 +16,6 @@ class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
       Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
     end
   end
-  alias :outputs :sink
+
 
 end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -55,7 +55,7 @@ class Zillabyte::Harness::Helper
     ee = "Error in \"#{operation}\" at \"name\": \n\t "
 
     # \w matchs [a-zA-Z0-9_]
-    if(!(name.is_a?(String) or name.is_a?(Symbol)) or (name =~ /^\w+$/).nil?)
+    if(!(name.is_a?(String) or name.is_a?(Symbol)) or (name =~ /^[\w\.]+$/).nil?)
       msg = "#{ee}\"Name\" must be a non-empty STRING with only alphanumeric and underscore characters at \"#{name}\". Apps and components must have names. The following methods must have names as well: \"sink\", \"inputs\" and \"outputs\"."
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -134,7 +134,7 @@ class Zillabyte::Harness::Helper
     end
 
     emits.each do |e|
-      if(!e.instance_of?(String) or (e =~ /^\w+$/).nil?)
+      if(!e.instance_of?(String) or (e =~ /^[\w\.]+$/).nil?)
         msg = "#{ee}An \"#{oo}\" stream name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
         Zillabyte::Harness::Helper.print_error(msg)
       end
@@ -182,6 +182,23 @@ class Zillabyte::Harness::Helper
 
     Zillabyte::Harness::Helper.check_output_format("each", each._output_format)
   end
+  
+  def self.check_clump(clump)
+    ee = "Error in \"clump\": \n\t "
+    pp = @@_print_check_clump
+
+    if clump._parallelism && clump._parallelism.to_i <= 0
+      msg = "Parallelism for clump must be positive integer" 
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    if !clump._execute
+      msg = "#{ee}A \"clump\" must contain an \"execute\" block. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    Zillabyte::Harness::Helper.check_output_format("clump", clump._output_format)
+  end
 
   def self.check_group_by(group_by)
     ee = "Error in \"group_by\": \n\t "
@@ -259,7 +276,7 @@ class Zillabyte::Harness::Helper
       Zillabyte::Harness::Helper.print_error(msg)
     end
 
-    if(!(id.is_a?(String) or id.is_a?(Symbol)) or (id =~ /^\w+$/).nil?)
+    if(!(id.is_a?(String) or id.is_a?(Symbol)) or (id =~ /^[\w\.]+$/).nil?)
       msg = "#{ee}The \"component_id\" must be a non-empty STRING with only alphanumeric and underscore characters in \"#{id}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -361,6 +378,23 @@ class Zillabyte::Harness::Helper
       Zillabyte::Harness::Helper.check_field_format("outputs", pp, cname, ctype, node._name)
     end
   end
+  
+  def self.get_non_option_args(args) 
+    if args.last.is_a?(Hash)
+      return args[0..-2]
+    else
+      return args
+    end
+  end
+  
+  def self.get_options(args)
+    if args.last.is_a?(Hash)
+      return args.last
+    else
+      return {}
+    end
+  end
+  
 
   # Test helper...
   def self.argv()
@@ -527,4 +561,32 @@ OUTPUT
       * The number of input and output streams specified must match the number listed for the component.
 OUTPUT
 
+
+  @@_print_check_clump = <<-OUTPUT
+\n\n"Clump" Syntax:
+  Simplified syntax:
+    stream.clump do |tuples|
+      tuples.each do |tuple|
+        |=block=|
+      end
+    end
+  - This is equivalent to just specifying an \"execute\" block below.
+  - Note that this syntax only works for a single output stream.
+
+  Custom clump:
+    stream.clump do
+       name "name" \t\t\t\t\t => optional
+       emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+       output_format :replace OR :merge \t\t => optional, defaults to :replace
+       prepare |=block=| \t\t\t\t => optional if no initialization needed
+       execute |=block=|
+    end
+  - The allowed output formats are :replace and :merge.
+      * :replace - discards the input tuple values and only emits the specified values. This is the default.
+      * :merge - re-emits the input tuple values along with the specified values.
+  - The "prepare" and "execute" blocks can be in do...end format or {} format.
+      * the "prepare" block is where any setup is done to prepare for tuple processing in the "execute" block.
+      * the "execute" block is where the tuples are actually processed. It must take in a single argument (the "tuple").
+OUTPUT
+
 end

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -15,7 +15,7 @@ module Zillabyte
     module Protocol
       
       READ_SOFT_TIMEOUT = 20  # Amount of time to wait until we send a 'ping' to the parent
-      DEATH_TIMEOUT = READ_SOFT_TIMEOUT * 3  # If no activity from the parent (even response to a ping) time before we kill ourself.
+      DEATH_TIMEOUT = READ_SOFT_TIMEOUT * 6  # If no activity from the parent (even response to a ping) time before we kill ourself.
 
       class << self
         attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :pipe_name, :pipe_to_java, :pipe_from_java

data/ruby/lib/zillabyte/harness/stream.rb

@@ -13,7 +13,7 @@ class Zillabyte::Harness::Stream
     @_name
   end
 
-  def each(&block)
+  def each(options = {}, &block)
     h = Zillabyte::Harness::Each.new()
     # Does the block take 0 arguments? If so it's not just an execute block.
     if(block.arity == 0)
@@ -35,7 +35,12 @@ class Zillabyte::Harness::Stream
 
     if(@_app._options[:command] == :info)
 
-      node_hash = {"name" => h._name, "type" => h._type, "output_format" => h._output_format}
+      node_hash = {
+        :name => h._name, 
+        :type => h._type, 
+        :output_format => h._output_format,
+        :config => options
+      }
       if (h._parallelism)
         node_hash["parallelism"] = h._parallelism
       end
@@ -57,7 +62,8 @@ class Zillabyte::Harness::Stream
     output_streams
   end
 
-  def filter(&block)
+
+  def filter(options = {}, &block)
     h = Zillabyte::Harness::Filter.new()
     # Does the block take 0 arguments? If so it's not just a keep block.
     if(block.arity == 0)
@@ -84,7 +90,12 @@ class Zillabyte::Harness::Stream
       end
 
       Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      arc_hash = {
+        :name => @_name, 
+        :origin => @_previous_node_name,
+        :dest => h._name,
+        :config => options
+      }
       Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
     elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
       pipe_name = @_app._options[:pipe]
@@ -95,11 +106,14 @@ class Zillabyte::Harness::Stream
     output_stream = self.class.new(h._emits[0], @_app, h._name) # filters only have one output stream
     output_stream
   end
+  
 
   def group_by(*args, &block)
     
     # Init
-    h = Zillabyte::Harness::GroupBy.new(*args)
+    group_args = Zillabyte::Harness::Helper.get_non_option_args(args)
+    options = Zillabyte::Harness::Helper.get_options(args)
+    h = Zillabyte::Harness::GroupBy.new(*group_args)
 
     # Yield 
     h.instance_eval(&block)
@@ -119,7 +133,12 @@ class Zillabyte::Harness::Stream
     if(@_app._options[:command] == :info)
       
       # Info.. 
-      node_hash = {"name" => h._name, "type" => h._type, "group_by" => h._group_by}
+      node_hash = {
+        :name => h._name, 
+        :type => h._type, 
+        :group_by => h._group_by,
+        :config => options
+      }
      if(h._parallelism)
         node_hash["parallelism"] = h._parallelism
       end
@@ -147,10 +166,11 @@ class Zillabyte::Harness::Stream
     
   end
 
-  def join_with(rhs_stream, *args, &block)
+
+  def join_with(rhs_stream, options = {}, &block)
     
     # Init
-    h = Zillabyte::Harness::Join.new(*args)
+    h = Zillabyte::Harness::Join.new(options)
 
     # Yield 
     throw "programmable joins are not supported at this time" unless block.nil?
@@ -172,11 +192,14 @@ class Zillabyte::Harness::Stream
     # Handle incoming command..
     if(@_app._options[:command] == :info)
       # Info.. 
-      node_hash = {"name" => h._name, 
-        "type" => h._type, 
-        "lhs_fields" => h._lhs_fields,
-        "rhs_fields" => h._rhs_fields,
-        "join_type" => h._join_type}
+      node_hash = {
+        :name => h._name, 
+        :type => h._type, 
+        :lhs_fields => h._lhs_fields,
+        :rhs_fields => h._rhs_fields,
+        :join_type => h._join_type,
+        :config => options
+      }
       
       Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
       arc_hash1 = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name, "left" => 1}
@@ -197,7 +220,7 @@ class Zillabyte::Harness::Stream
     
   end
 
-  def call_component(comp_id = nil, &block)
+  def call_component(comp_id = nil, options = {}, &block)
     
     h = Zillabyte::Harness::InjectedComponent.new()
     h.instance_eval(&block) if block_given?
@@ -210,7 +233,13 @@ class Zillabyte::Harness::Stream
     @_app._nodes << h
 
     if(@_app._options[:command] == :info)
-      node_hash = {"name" => h._name, "type" => h._type, "id" => h._id, "output_format" => h._output_format}
+      node_hash = {
+        :name => h._name, 
+        :type => h._type, 
+        :id => h._id, 
+        :output_format => h._output_format,
+        :config => options
+      }
       Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
       arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
       Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
@@ -228,6 +257,10 @@ class Zillabyte::Harness::Stream
     output_streams = output_streams[0] if output_streams.size == 1
     output_streams
   end
+  alias_method :executes, :call_component
+  alias_method :execute, :call_component
+  
+  
 
   def sink(&block)
     h = Zillabyte::Harness::Sink.new()
@@ -243,4 +276,213 @@ class Zillabyte::Harness::Stream
     end
   end
   
+  
+  
+  # Unique's a stream. Executed on backend
+  def unique(*args)
+    
+    # INIT
+    group_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
+    options = Zillabyte::Harness::Helper.get_options(args)
+    
+    if(@_app._options[:command] == :info)
+
+      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      node_name = "unique_#{Zillabyte::Harness::Counter.get()}"
+      node_hash = {
+        :name => node_name,
+        :type => "unique",
+        :group_fields => group_fields,
+        :config => options
+      }
+      arc_hash = {
+        :name => @_name,
+        :origin => @_previous_node_name, 
+        :dest => node_name
+      }
+      
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      return self.class.new(emits, @_app, node_name)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      throw "should not execute in multilang"
+    end
+    
+  end
+  
+  
+  
+  # Counts fields. Executed on backend
+  def count(*args)
+      
+    # Init 
+    group_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
+    options = Zillabyte::Harness::Helper.get_options(args)
+    
+    if(@_app._options[:command] == :info)
+
+      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      node_name = "count_#{Zillabyte::Harness::Counter.get()}"
+      node_hash = {
+        :name => node_name,
+        :type => "count",
+        :group_fields => group_fields,
+        :config => options
+      }
+      arc_hash = {
+        :name => @_name,
+        :origin => @_previous_node_name, 
+        :dest => node_name
+      }
+      
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      return self.class.new(emits, @_app, node_name)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      throw "should not execute in multilang"
+    end
+    
+  end
+  
+  
+  
+  
+  # Removes fields from the stream. 
+  def remove(*args)
+      
+    # Init 
+    remove_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
+    options = Zillabyte::Harness::Helper.get_options(args)
+    
+    if(@_app._options[:command] == :info)
+
+      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      node_name = "remove_#{Zillabyte::Harness::Counter.get()}"
+      node_hash = {
+        :name => node_name,
+        :type => "remove",
+        :remove => remove_fields,
+        :config => options
+      }
+      arc_hash = {
+        :name => @_name,
+        :origin => @_previous_node_name, 
+        :dest => node_name
+      }
+      
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      return self.class.new(emits, @_app, node_name)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      throw "should not execute in multilang"
+    end
+    
+  end
+  
+  
+  
+  # Retains fields from the stream and removes everything else
+  def retain(*args)
+      
+    # Init 
+    retain_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
+    options = Zillabyte::Harness::Helper.get_options(args)
+    
+    if(@_app._options[:command] == :info)
+
+      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      node_name = "retain_#{Zillabyte::Harness::Counter.get()}"
+      node_hash = {
+        :name => node_name,
+        :type => "retain",
+        :retain => retain_fields,
+        :config => options
+      }
+      arc_hash = {
+        :name => @_name,
+        :origin => @_previous_node_name, 
+        :dest => node_name
+      }
+      
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      return self.class.new(emits, @_app, node_name)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      throw "should not execute in multilang"
+    end
+    
+  end
+  
+  
+  
+  
+  
+  # Groups together N arbitrary tuples (usefull for mini-batch processing)
+  def clump(options = {})
+      
+    if(@_app._options[:command] == :info)
+
+      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      node_name = "clump_#{Zillabyte::Harness::Counter.get()}"
+      node_hash = {
+        :name => node_name,
+        :type => "clump",
+        :config => options
+      }
+      arc_hash = {
+        :name => @_name,
+        :origin => @_previous_node_name, 
+        :dest => node_name
+      }
+      
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      return self.class.new(emits, @_app, node_name)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      throw "should not execute in multilang"
+    end
+    
+  end
+  
+  
+  
+  
+  
+  # Renames fields..
+  def rename(rename_map)
+    
+    if(@_app._options[:command] == :info)
+
+      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      node_name = "rename_#{Zillabyte::Harness::Counter.get()}"
+      node_hash = {
+        :name => node_name,
+        :type => "rename",
+        :rename => rename_map
+      }
+      arc_hash = {
+        :name => @_name,
+        :origin => @_previous_node_name, 
+        :dest => node_name
+      }
+      
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      return self.class.new(emits, @_app, node_name)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      throw "should not execute in multilang"
+    end
+    
+  end
+  
+
+
+  
+  
 end

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.1.29" unless defined?(VERSION)
+  VERSION = "0.1.31" unless defined?(VERSION)
 end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.1.29
+  version: 0.1.31
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-08 00:00:00.000000000 Z
+date: 2014-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mkfifo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: zillabyte-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.29
+        version: 0.1.31
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.29
+        version: 0.1.31
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -64,6 +64,7 @@ files:
 - ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/harness/app.rb
 - ruby/lib/zillabyte/harness/base.rb
+- ruby/lib/zillabyte/harness/clump.rb
 - ruby/lib/zillabyte/harness/component.rb
 - ruby/lib/zillabyte/harness/component_sink.rb
 - ruby/lib/zillabyte/harness/component_source.rb
@@ -91,12 +92,12 @@ require_paths:
 - ruby/lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []