zillabyte 0.0.24 → 0.1.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MTcxZWMzOWZlNDI3MDU1ZDNhNjFiMWE5NTVjMjE4OTYxMzdiMWZiZg==
-  data.tar.gz: !binary |-
-    ZjZmOWY4YTM2MjcwNTllYTdjMTg4ZmFjODFhMDMxNWJkNzBhMWJlZA==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZTgwZDUxMTQ0OWY2ODJmYTkzYzllNDAwMWZkNGVmZTczNzBhNTc3NzMwZThi
-    MmQ4YjA5NTc0M2ZmZTBhYmEzODhmMzcyZGIzNTRkZGZjMDMyOGQzYWViNWQz
-    Y2U1NmU1YzI1YTZhYWZmMzVlYWQwZDEyY2EwNDVkODVlNDEzODQ=
-  data.tar.gz: !binary |-
-    M2QzZWI1YzA4YmM3ZDE3NjNlNjY5MzgwMzhlY2ZhOGE1YmM3ZGRkMmI5ZWYw
-    YWY3ODRjOGMwOWY4ZjM4YjA5Y2JkZmVjZjIxYTI2NGEwYjVhZWYwYjgyYjhk
-    Y2MzYWM2MDcxYjg0OWE2ZjgzOTMwZmViYTJlNTVlOTYwNmNkOTA=
+SHA1:
+  metadata.gz: c42be796292e1dd2943f43b64e37364fa8208073
+  data.tar.gz: 604ea5c2a29c5cf979f8e8f3a61d496cdd6d04bc
+SHA512:
+  metadata.gz: 419d310b450c14dadc0fdaefe0acf97db813953a24a4727f1308ab2051ce0f35b8458641bcbf79cca7cb49f79cf437d6bd13516ecc33c98cce9d6d6f85c74c02
+  data.tar.gz: 75da6f6f8f6c48162278c6fd21fd1614ff87f89cb9ddbb0e81fa33e525ddd3bd4e57a60cbfc85ba5ce0ed359ac953e12050a7f34b6c86dd1588a73a0b1f697eb

data/ruby/lib/zillabyte.rb

@@ -1,5 +1,6 @@
 require "active_support/core_ext"
 require "zillabyte/harness"
+require "zillabyte/version"
 require "zillabyte/common/progress"
 
 module Zillabyte
@@ -8,5 +9,10 @@ module Zillabyte
     @app = Zillabyte::Harness::App.build(name)
     @app
   end
+
+  def self.component(name=nil)
+    @component = Zillabyte::Harness::Component.build(name)
+    @component
+  end
   
 end

data/ruby/lib/zillabyte/harness/app.rb

@@ -1,19 +1,15 @@
 require "json"
+require "zillabyte/harness/base"
 
-class Zillabyte::Harness::App
-  attr_accessor :_name, :_options, :_nodes, :_streams, :_names, :_info_file
+class Zillabyte::Harness::App < Zillabyte::Harness::Base
 
   def self.build(name)
-    h = Zillabyte::Harness::App.new()
-    h._nodes = []
-    h._streams = {}
-    h._names = {}
-    h._name = name
+    h = Zillabyte::Harness::App.new name
     Zillabyte::Harness::Helper.check_name("app", h._name, {})
     h._options = Zillabyte::Harness::Helper.opt_parser()
     if(h._options[:command] == :info)
       h._info_file = File.open(h._options[:file],"w+")
-      info_hash = {"language" => "ruby", "name" => h._name}
+      info_hash = {"language" => "ruby", "name" => h._name, "flow_type" => "app", "multilang_version" => Zillabyte::VERSION}
       Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._info_file)
     end
     h
@@ -39,15 +35,15 @@ class Zillabyte::Harness::App
     end
     @_nodes << h
     if(@_options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
+      node_hash = {"name" => h._name, "type" => h._type}
       if(h._relation)
-        info_hash["relation"] = h._relation
+        node_hash["relation"] = h._relation
       elsif(h._matches)
-        info_hash["matches"] = h._matches
+        node_hash["matches"] = h._matches
       else
-        info_hash["end_cycle_policy"] = h._end_cycle_policy
+        node_hash["end_cycle_policy"] = h._end_cycle_policy
       end
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_info_file)
     elsif(@_options[:command] == :execute and @_options[:name] == h._name)
       pipe_name = @_options[:pipe]
       c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
@@ -56,7 +52,7 @@ class Zillabyte::Harness::App
 
     output_streams = []
     h._emits.each do |stream|
-      output_streams << Zillabyte::Harness::Stream.new(stream, self, nil)
+      output_streams << Zillabyte::Harness::Stream.new(stream, self, h._name)
     end
     output_streams = output_streams[0] if output_streams.size == 1
     output_streams

data/ruby/lib/zillabyte/harness/base.rb

@@ -0,0 +1,11 @@
+class Zillabyte::Harness::Base
+  attr_accessor :_name, :_options, :_nodes, :_streams, :_names, :_info_file
+
+  def initialize(name)
+    @_nodes = []
+    @_streams = {}
+    @_names = {}
+    @_name = name
+  end
+
+end

data/ruby/lib/zillabyte/harness/component.rb

@@ -0,0 +1,32 @@
+require "zillabyte/harness/base"
+
+class Zillabyte::Harness::Component < Zillabyte::Harness::Base
+
+  def self.build(name)
+    h = Zillabyte::Harness::Component.new name
+    Zillabyte::Harness::Helper.check_name("component", h._name, {})
+    h._options = Zillabyte::Harness::Helper.opt_parser()
+    if(h._options[:command] == :info)
+      h._info_file = File.open(h._options[:file],"w+")
+      info_hash = {"language" => "ruby", "name" => h._name, "flow_type" => "component", "multilang_version" => Zillabyte::VERSION}
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._info_file)
+    end
+    h
+  end
+
+  def inputs(*args, &block)
+    h = Zillabyte::Harness::ComponentSource.new
+    h.instance_eval(&block)
+    Zillabyte::Harness::Helper.check_name("inputs", h._name, @_names)
+    Zillabyte::Harness::Helper.check_component_source(h)
+
+    if(self._options[:command] == :info)
+      node_hash = {"name" => h._name, "type" => h._type, "fields" => h._fields}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, self._info_file)
+    end
+
+    # For components, each "inputs" statement only defines a single stream
+    output_stream = Zillabyte::Harness::ComponentStream.new(h._emits.first, self, h._name)
+    output_stream
+  end
+end

data/ruby/lib/zillabyte/harness/component_sink.rb

@@ -0,0 +1,26 @@
+class Zillabyte::Harness::ComponentSink
+  attr_accessor :_name, :_relation, :_type, :_columns, :_scope
+
+  def initialize()
+    @_type = 'sink'
+    @_columns = []
+  end
+  
+  def name(v)
+    @_name = v
+  end
+  
+  def relation(v)
+    @_relation = v
+  end
+  
+  def field(cname, ctype)
+    col = {cname => ctype}
+    @_columns << col
+  end
+  
+  def scope(v)
+    @_scope = v
+  end
+  
+end

data/ruby/lib/zillabyte/harness/component_source.rb

@@ -0,0 +1,17 @@
+class Zillabyte::Harness::ComponentSource
+  attr_accessor :_name, :_type, :_emits, :_fields
+
+  def initialize()
+    @_type = "source"
+    @_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+    @_fields = []
+  end
+
+  def name(v)
+    @_name = v
+  end
+
+  def field(name, type)
+    @_fields << {name => type}
+  end
+end

data/ruby/lib/zillabyte/harness/component_stream.rb

@@ -0,0 +1,20 @@
+require 'zillabyte/harness/stream'
+
+class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
+
+  def sink(*args, &block)
+    h = Zillabyte::Harness::ComponentSink.new
+    h.instance_eval(&block)
+    Zillabyte::Harness::Helper.check_name("outputs", h._name, @_app._names)
+    Zillabyte::Harness::Helper.check_component_sink(h)
+
+    if(@_app._options[:command] == :info)
+      node_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "scope" => h._scope}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+    end
+  end
+  alias :outputs :sink
+
+end

data/ruby/lib/zillabyte/harness/each.rb

@@ -1,5 +1,5 @@
 class Zillabyte::Harness::Each
-  attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
+  attr_accessor :_name, :_type, :_emits, :_prepare, :_execute
 
   def initialize()
     @_name = "each_"+Zillabyte::Harness::Counter.get()

data/ruby/lib/zillabyte/harness/filter.rb

@@ -1,5 +1,5 @@
 class Zillabyte::Harness::Filter
-  attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_keep
+  attr_accessor :_name, :_type, :_emits, :_prepare, :_keep
 
   def initialize()
     @_name = "filter_"+Zillabyte::Harness::Counter.get()
@@ -9,11 +9,11 @@ class Zillabyte::Harness::Filter
   def name(v)
     @_name = v
   end
-  
+
   def emits(*v)
     @_emits = *v
   end
-
+  
   def prepare(&block)
     @_prepare = block
   end

data/ruby/lib/zillabyte/harness/group_by.rb

@@ -1,36 +1,20 @@
 class Zillabyte::Harness::GroupBy
-  attr_accessor :_name, :_type, :_consumes, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
+  attr_accessor :_name, :_type, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
 
-  def initialize(*group_by_fields)
+  def initialize(*v)
     @_name = "group_by_"+Zillabyte::Harness::Counter.get()
     @_type = 'group_by'
-    @_group_by = group_by_fields;
+    @_group_by = *v
   end
   
   def name(v)
     @_name = v
   end
-  
-  def consumes(v)
-    @_consumes = v
-  end
 
-  def group_by(v)
-    @_group_by = v
+  def emits(*v)
+    @_emits = *v
   end
   
-  def group_by
-    if @_group_by.is_a?(Array)
-      return @_group_by  
-    else
-      return [@_group_by]  
-    end
-  end
-
-  def emits(v)
-    @_emits = v
-  end
-
   def begin_group(&block)
     @_begin_group = block
   end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -26,6 +26,14 @@ class Zillabyte::Harness::Helper
     options
   end
 
+  def self.write_node_to_file(hash, file)
+    Zillabyte::Harness::Helper.write_hash_to_file({"class" => "node", "info" => hash}, file)
+  end
+
+  def self.write_arc_to_file(hash, file)
+    Zillabyte::Harness::Helper.write_hash_to_file({"class" => "arc", "info" => hash}, file)
+  end
+
   def self.write_hash_to_file(hash, file)
     file.write(hash.to_json+"\n")
   end
@@ -44,7 +52,7 @@ class Zillabyte::Harness::Helper
 
     # \w matchs [a-zA-Z0-9_]
     if(!(name.is_a?(String) or name.is_a?(Symbol)) or (name =~ /^\w+$/).nil?)
-      msg = "#{ee}\"Name\" must be a non-empty STRING with only alphanumeric and underscore characters at \"#{name}\". Apps and simple_apps must have names!"
+      msg = "#{ee}\"Name\" must be a non-empty STRING with only alphanumeric and underscore characters at \"#{name}\". Apps and components must have names. The following methods must have names as well: \"sink\", \"inputs\" and \"outputs\"."
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(names[name] and names[name] != "app" and names[name] != "sink" and operation != "app" and operation != "sink")
@@ -94,22 +102,28 @@ class Zillabyte::Harness::Helper
   end
 
   def self.check_emits(operation, emits, streams)
-    ee = "Error in \"#{operation}\" at \"emits\": \n\t "
+    if operation == "component"
+      oo = "outputs"
+    else
+      oo = "emits"
+    end
+
+    ee = "Error in \"#{operation}\" at \"#{oo}\": \n\t "
     pp = ""
     
     if(!emits.instance_of?(Array))
-      msg = "#{ee}\"Emits\" must be a sequence of comma-separated STRINGS. #{pp}"
+      msg = "#{ee}An \"#{oo}\" must be a sequence of comma-separated STRINGS. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     n_emits = emits.length
     if(n_emits == 0)
-      msg = "#{ee}Must emit at least one stream, \"emits\" cannot be an empty. #{pp}"
+      msg = "#{ee}Must emit at least one stream, \"#{oo}\" cannot be an empty. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
 
     emits.each do |e|
       if(!e.instance_of?(String) or (e =~ /^\w+$/).nil?)
-        msg = "#{ee}\"Emits\" stream name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
+        msg = "#{ee}An \"#{oo}\" stream name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
         Zillabyte::Harness::Helper.print_error(msg)
       end
       if(streams[e])
@@ -124,8 +138,12 @@ class Zillabyte::Harness::Helper
     ee = "Error in \"filter\": \n\t "
     pp = @@_print_check_filter
 
+    if filter._emits.size != 1
+      msg = "#{ee}A \"filter\" must emit a single stream. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
     if !filter._keep
-      msg = "#{ee}A \"filter\" must contain a \"condition\" block. #{pp}"
+      msg = "#{ee}A \"filter\" must contain a \"keep\" block. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end 
   end
@@ -140,6 +158,83 @@ class Zillabyte::Harness::Helper
     end 
   end
 
+  def self.check_group_by(group_by)
+    ee = "Error in \"group_by\": \n\t "
+    pp = @@_print_check_group_by
+
+    if !group_by._group_by or group_by._group_by.empty?
+      msg = "#{ee} A \"group_by\" must contain a non-empty \"group_by\" clause. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    group_by._group_by.each do |field|
+      if(!(field.is_a?(String) or field.is_a?(Symbol)) or (field =~ /^\w+$/).nil?)
+        msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters at \"#{field}\"."
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+    end
+
+    if !group_by._begin_group
+      msg = "#{ee}A \"group_by\" must contain a \"begin_group\" block. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if !group_by._aggregate
+      msg = "#{ee}A \"group_by\" must contain an \"aggregate\" block. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if !group_by._end_group
+      msg = "#{ee}A \"group_by\" must contain an \"end_group\" block. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
+  def self.check_join(join_with)
+    ee = "Error in \"join_with\": \n\t "
+    pp = @@_print_check_join
+
+    if join_with._emits.size != 1
+      msg = "#{ee}A \"join_with\" must emit a single stream. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    lhs_fields = join_with._lhs_fields
+    rhs_fields = join_with._rhs_fields
+    if !lhs_fields or !rhs_fields
+      msg = "#{ee}The fields to join on must be specified in the format described below! #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(!(lhs_fields.is_a?(String) or lhs_fields.is_a?(Symbol)) or (lhs_fields =~ /^\w+$/).nil?)
+      msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters at \"#{lhs_fields}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(!(rhs_fields.is_a?(String) or rhs_fields.is_a?(Symbol)) or (rhs_fields =~ /^\w+$/).nil?)
+      msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters at \"#{rhs_fields}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    supported_joins = [:inner, :left, :right]
+    if !supported_joins.member? join_with._join_type
+      msg = "#{ee}The requested \"join_type\", \"#{join_with._join_type}\", is not supported. Currently we support \"#{supported_joins}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
+  def self.check_call_component(node)
+    ee = "Error in \"call_component\": \n\t "
+    pp = @@_print_check_call_component
+
+    id = node._id
+    if !id
+      msg = "#{ee} The \"component_id\" must be specified. Check \"zillabyte components\" to find the correct ID. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    # \w matchs [0-9]
+    if(!(id.is_a?(String) or id.is_a?(Symbol)) or (id =~ /^[0-9]+$/).nil? or id[0] == "0")
+      msg = "#{ee} Invalid \"component_id\". The \"component_id\" must be a number. Check \"zillabyte components\" to find the correct ID. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
   def self.check_sink(sink, nodes)
     ee = "Error in \"sink\": \n\t "
     pp = @@_print_check_sink
@@ -175,16 +270,15 @@ class Zillabyte::Harness::Helper
     columns.each do |col|
       cname = col.keys()[0]
       ctype = col[cname]
-      Zillabyte::Harness::Helper.check_sink_column_format(cname,ctype,name)
+      Zillabyte::Harness::Helper.check_field_format("sink",@@_print_check_sink,cname,ctype,name)
     end
   end
 
-  def self.check_sink_column_format(cname, ctype, relation_name)
-    ee = "Error in \"sink\" at \"column\": \n\t "
-    pp = @@_print_check_sink
+  def self.check_field_format(operation, pp, cname, ctype, relation_name)
+    ee = "Error in \"#{operation}\": \n\t "
 
     if(!(cname.is_a?(String) or cname.is_a?(Symbol)) or (cname =~ /^\w+$/).nil?)
-      msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in relation \"#{relation_name}\". #{pp}"
+      msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(cname =~ /^v[0-9]+$/i or @@_meta_names.member?(cname.downcase))
@@ -192,13 +286,47 @@ class Zillabyte::Harness::Helper
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(!ctype.instance_of?(Symbol))
-      msg = "#{ee}Field data types must be SYMBOLS in relation \"#{relation_name}\". #{pp}"
+      msg = "#{ee}Field data types must be SYMBOLS in \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(ctype != :string and ctype != :integer and ctype != :float and ctype != :double && ctype != :boolean)
-      msg = "#{ee}Invalid field data type at \"#{ctype}\" in relation \"#{relation_name}\". #{pp}"
+      msg = "#{ee}Invalid field data type at \"#{ctype}\" in \"#{relation_name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
+  def self.check_component_source(node)
+    ee = "Error in \"inputs\": \n\t "
+    pp = @@_print_check_component_source
+
+    fields = node._fields
+    if(fields.empty?)
+      msg = "#{ee}Must be at least one output field to \"#{node._name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    fields.each do |field|
+      fname = field.keys[0]
+      ftype = field[fname]
+      Zillabyte::Harness::Helper.check_field_format("inputs", pp, fname, ftype, node._name)
+    end
+  end
+
+  def self.check_component_sink(node)
+    ee = "Error in \"outputs\": \n\t "
+    pp = @@_print_check_component_sink
+
+    columns = node._columns
+    if(columns.empty?)
+      msg = "#{ee}Must be at least one output field to \"#{node._name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
+
+    columns.each do |col|
+      cname = col.keys[0]
+      ctype = col[cname]
+      Zillabyte::Harness::Helper.check_field_format("outputs", pp, cname, ctype, node._name)
+    end
   end
 
   # Test helper...
@@ -240,18 +368,27 @@ OUTPUT
   - The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
       * :null_emit - end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
       * :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
+  - The "begin_cycle" and "next_tuple" blocks can be in do...end format or {} format.
+      * the "begin_cycle" block is where any setup is done to initialize the content and quantity of tuples emitted by the "next_tuple" block.
+      * the "next_tuple" block is where the tuples are actually emitted.
 OUTPUT
 
   @@_print_check_filter = <<-OUTPUT
 \n\n"Filter" Syntax:
   Simplified syntax:
-    stream.filter { |tuple| |=condition=| }
+    stream.filter do |tuple|
+      |=block=|
+    end
+  - The condition for keeping the tuple should be specified in the block. 
+  - This is equivalent to just specifying a "keep" block below.
 
-    * Outputs a given tuple from the stream only if condition is true
   Custom filter:
-    stream.filter do |tuple|
-       |=condition=|
+    stream.filter do
+       name "name" \t\t\t => optional
+       keep |=block=|
     end
+  - The "keep" block must take in a single argument (the "tuple").
+  - Tuples will pass through if "keep" returns "True".
 OUTPUT
 
   @@_print_check_each = <<-OUTPUT
@@ -270,21 +407,80 @@ OUTPUT
        prepare |=block=| \t\t\t\t => optional if no initialization needed
        execute |=block=|
     end
+  - The "prepare" and "execute" blocks can be in do...end format or {} format.
+      * the "prepare" block is where any setup is done to prepare for tuple processing in the "execute" block.
+      * the "execute" block is where the tuples are actually processed. It must take in a single argument (the "tuple").
 OUTPUT
 
-  @@_print_check_aggregate = <<-OUTPUT
-\n\n"Aggregate" Syntax:
+  @@_print_check_group_by = <<-OUTPUT
+\n\n"Group By" Syntax:
     stream.group_by do
-       name "name"  => optional
-       group_by ["field_1", "field_2", ...]
-       emits emits
-       consumes "consumed_stream"  => optional if single stream
-       begin_group ...
-       aggregate ...
-       end_group ...
-    end
-  - If there are multiple streams, "consumes" must be specified as a non-empty STRING!
-     * "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "aggregate" operates on.
+       name "name"  \t\t\t\t\t => optional
+       group_by "field_1", "field_2", ...
+       emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+       begin_group |=block=|
+       aggregate |=block=|
+       end_group |=block=|
+    end
+  - The "begin_group", "aggregate" and "end_group" blocks can be in do...end format or {} format.
+      * the "begin_group" block is where the initial values for the aggregation are set. It must take in a single argument (the "grouping tuple", which is emitted at the beginning of each group and contains the values of the fields specified in "group_by").
+      * the "aggregate" block is where the aggregation is performed. It must take in a single argument (the "tuple").
+      * the "end_group" block is where the final aggregated value is emitted.
+OUTPUT
+
+  @@_print_check_join = <<-OUTPUT
+\n\n"Join With" Syntax:
+    lhs_stream.join_with( rhs_stream_object, options )
+  - Options should be specified as a hash. The following keys are recognized:
+      Mandatory:
+      * "on" \t\t -- specifies the fields to join on. The value must be a STRING or a length-2 ARRAY. If value = a STRING, the LH and RH join fields will both be set to this STRING. If value = a length-2 ARRAY, the LH join field will be set to array[0] and the RH join field will be set to array[1].
+
+      Optional:
+      * "type" \t -- specifies the join type. The default is "inner".
+      * "emits" \t -- specifies the stream to emit. A "join" may only emit a single stream.
+OUTPUT
+
+  @@_print_check_component_source = <<-OUTPUT
+\n\n"Inputs" Syntax:
+    component.inputs do
+       name "input_stream_name"
+       field "field_1", :type_1
+       field "field_2", :type_2
+    end
+  - "Inputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+  - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
+  - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+  - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
+OUTPUT
+
+  @@_print_check_component_sink = <<-OUTPUT
+\n\n"Outputs" Syntax:
+    component_stream.outputs do
+       name "output_stream_name"
+       field "field_1", :type_1
+       field "field_2", :type_2
+    end
+  - "Outputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+  - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
+  - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+  - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
+OUTPUT
+
+  @@_print_check_call_component = <<-OUTPUT
+\n\n"Call_component" Syntax:
+    stream.call_component do
+      component_id "component_id"
+      name "name" \t\t\t\t\t\t\t\t\t => optional
+      additional_inputs other_input_stream_object_1, other_input_stream_object_2, ... \t => blank if none
+      outputs "output_stream_name_1", "output_stream_name_2", ...
+    end
+  - The "component_id" MUST be given and correspond to the id listed in the output of "zillabyte components".
+  - To correctly stich in the component, the implicit assumptions below WILL BE USED:
+      * The "stream" that "call_component" is invoked on MUST correspond to the first listed input stream to the component.
+      * The streams specified in "additional_inputs" MUST correspond to the other listed input streams in order.
+      * Tuples emitted from the preceeding operation MUST contain the fields listed for the corresponding component input streams.
+      * The streams specified in "outputs" must correspond to the listed output streams to the component in order.
+      * The number of input and output streams specified must match the number listed for the component.
 OUTPUT
 
   @@_meta_names = ["id", "confidence", "since", "source"]

data/ruby/lib/zillabyte/harness/injected_component.rb

@@ -0,0 +1,31 @@
+class Zillabyte::Harness::InjectedComponent
+  attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
+
+  def initialize()
+    @_name = "component_"+Zillabyte::Harness::Counter.get()
+    @_type = "component"
+    @_consumes = []
+    @_output_format = "replace"
+  end
+
+  def name(v)
+    @_name = v
+  end
+
+  def component_id(v)
+    @_id = v.to_s
+  end
+
+  def additional_inputs(*v)
+    @_consumes = *v
+  end
+
+  def outputs(*v)
+    @_emits = *v
+  end
+
+  def output_format(v)
+    @_output_format = v
+  end
+
+end

data/ruby/lib/zillabyte/harness/join.rb

@@ -1,25 +1,33 @@
 class Zillabyte::Harness::Join
-  attr_accessor :_name, :_type, :_lhs_stream, :_rhs_stream, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
+  attr_accessor :_name, :_type, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
   #:_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
 
-  def initialize(lhs_stream, rhs_stream, options = {})
+  def initialize(options = {})
     @_name = "join_"+Zillabyte::Harness::Counter.get()
     @_type = 'join'
-    @_lhs_stream = lhs_stream
-    @_rhs_stream = rhs_stream
     @_join_type = :inner
-    
+
     options.symbolize_keys!
+
+    emits = options[:emits]
+    if emits
+      if emits.is_a?(String)
+        @_emits = [options[:emits]]
+      else
+        @_emits = options[:emits]
+      end
+    end
     
     # Figure out which fields to join on..
-    if options[:on].is_a?(Array) and options[:on].size == 2
-      @_lhs_fields = options[:on][0]
-      @_rhs_fields = options[:on][1]
-      
-    elsif options[:on]
-      @_lhs_fields = options[:on]
-      @_rhs_fields = options[:on]
-      
+    on = options[:on]
+    if on
+      if on.is_a?(Array) and on.size == 2
+        @_lhs_fields = on[0]
+        @_rhs_fields = on[1]
+      elsif (on.is_a?(String) or on.is_a?(Symbol))
+        @_lhs_fields = on
+        @_rhs_fields = on
+      end
     end
     
     # Different kind of join types...

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -1,5 +1,6 @@
 require "rubygems"
 require "json"
+require "base64"
 
 module Zillabyte
 
@@ -115,14 +116,14 @@ module Zillabyte
 
 
       def set_state(key, val)
-        send_msg_to_parent({'set_state' => Base64.encode64(Marshall::dump(val))})
+        send_msg_to_parent({'set_state' => Base64.encode64(Marshal.dump(val)), 'key' => key})
       end
       
       def get_state(key)
         send_msg_to_parent({'get_state' => key})
         ret = read_message()
         if ret['state']
-          return Marshall.load(Base64.decode64(ret['state']))
+          return Marshal.load(Base64.decode64(ret['state']))
         else
           return nil
         end
@@ -131,8 +132,8 @@ module Zillabyte
       def emit(*args)
 
         stream = nil
-        tuple = nil
-        meta = nil
+        tuple = {}
+        meta = {}
 
         if (args.size == 1)
           tuple = args[0]
@@ -151,7 +152,7 @@ module Zillabyte
         end
         
         if tuple.is_a?(Tuple)
-          meta = meta || tuple.meta || {}
+          meta = tuple.meta || {} if meta == {}
           tuple = tuple.values
         end
 
@@ -319,7 +320,7 @@ module Zillabyte
           begin
             t = Tuple.from_hash(read_command)
             next if(!t)
-            keep t
+            emit t if keep t
           rescue SignalException => e
             raise
           rescue Exception => e
@@ -419,9 +420,7 @@ module Zillabyte
       end
 
       def keep(*args)
-        if self.instance_exec *args, &@harness._keep
-            emit args[0]
-        end
+        self.instance_exec *args, &@harness._keep
       end
 
       def prepare(*args)

data/ruby/lib/zillabyte/harness/sink.rb

@@ -1,16 +1,11 @@
 class Zillabyte::Harness::Sink
-  attr_accessor :_name, :_relation, :_type, :_columns, :_consumes, :_scope
+  attr_accessor :_name, :_relation, :_type, :_columns, :_scope
 
   def initialize()
     @_type = 'sink'
     @_columns = []
   end
   
-  
-  def consumes(v)
-    @_consumes = v
-  end
-  
   def name(v)
     @_name = v
   end

data/ruby/lib/zillabyte/harness/stream.rb

@@ -19,7 +19,6 @@ class Zillabyte::Harness::Stream
     if(block.arity == 0)
       h.instance_eval(&block)
       Zillabyte::Harness::Helper.check_name("each", h._name, @_app._names)
-      Zillabyte::Harness::Helper.check_each(h)
       if h._emits
         Zillabyte::Harness::Helper.check_emits("each", h._emits, @_app._streams)
       else
@@ -30,11 +29,15 @@ class Zillabyte::Harness::Stream
       h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
       h._execute = block
     end
+
+    Zillabyte::Harness::Helper.check_each(h)
     @_app._nodes << h
+
     if(@_app._options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
-      info_hash["emits"] = h._emits
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+      node_hash = {"name" => h._name, "type" => h._type}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
     elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
       pipe_name = @_app._options[:pipe]
       c = Zillabyte::Harness::EachController.new(h, progress = Zillabyte::Common::Progress.new)
@@ -43,7 +46,7 @@ class Zillabyte::Harness::Stream
 
     output_streams = []
     h._emits.each do |stream|
-      output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
+      output_streams << self.class.new(stream, @_app, h._name)
     end
     output_streams = output_streams[0] if output_streams.size == 1
     output_streams
@@ -51,26 +54,36 @@ class Zillabyte::Harness::Stream
 
   def filter(&block)
     h = Zillabyte::Harness::Filter.new()
-    h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-    h._keep = block
+    # Does the block take 0 arguments? If so it's not just a keep block.
+    if(block.arity == 0)
+      h.instance_eval(&block)
+      Zillabyte::Harness::Helper.check_name("filter", h._name, @_app._names)
+      if h._emits
+        Zillabyte::Harness::Helper.check_emits("filter", h._emits, @_app._streams)
+      else
+        h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      end
+    else
+      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      h._keep = block
+    end
 
+    Zillabyte::Harness::Helper.check_filter(h)
     @_app._nodes << h
+
     if(@_app._options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
-      info_hash["emits"] = h._emits
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+      node_hash = {"name" => h._name, "type" => h._type}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
     elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
       pipe_name = @_app._options[:pipe]
       c = Zillabyte::Harness::FilterController.new(h, progress = Zillabyte::Common::Progress.new)
       c.run(pipe_name)
     end
 
-    output_streams = []
-    h._emits.each do |stream|
-      output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
-    end
-    output_streams = output_streams[0] if output_streams.size == 1
-    output_streams
+    output_stream = self.class.new(h._emits[0], @_app, h._name) # filters only have one output stream
+    output_stream
   end
 
   def group_by(*args, &block)
@@ -81,11 +94,7 @@ class Zillabyte::Harness::Stream
     # Yield 
     h.instance_eval(&block)
     Zillabyte::Harness::Helper.check_name("group_by", h._name, @_app._names)
-    
-    # Sanity checks
-    # Zillabyte::Harness::Helper.check_group_by("aggregate", h, @_nodes, @_streams)
-    # @_branched = Zillabyte::Harness::Helper.check_emits("aggregate", h._emits, @_streams) || @_branched
-    
+
     # Multiple emits?
     if h._emits
       Zillabyte::Harness::Helper.check_emits("group_by", h._emits, @_app._streams)
@@ -93,18 +102,20 @@ class Zillabyte::Harness::Stream
       h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
     end
     
-    # Add this node the list
+    Zillabyte::Harness::Helper.check_group_by(h)
     @_app._nodes << h
     
     # Handle incoming command..
     if(@_app._options[:command] == :info)
       
       # Info.. 
-      info_hash = {"name" => h._name, "type" => h._type, "group_by" => h.group_by, "emits" => h._emits, "consumes" => @_name}
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+      node_hash = {"name" => h._name, "type" => h._type, "group_by" => h._group_by}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
       
     elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      
+
       # Execute..
       pipe_name = @_app._options[:pipe]
       c = Zillabyte::Harness::GroupByController.new(h, Zillabyte::Common::Progress.new)
@@ -112,24 +123,20 @@ class Zillabyte::Harness::Stream
       
     end
     
-      
-    
     # Return the stream 
     output_streams = []
     h._emits.each do |stream|
-      output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
+      output_streams << self.class.new(stream, @_app, h._name)
     end
     output_streams = output_streams[0] if output_streams.size == 1
     output_streams
     
   end
 
-
-
   def join_with(rhs_stream, *args, &block)
     
     # Init
-    h = Zillabyte::Harness::Join.new(self, rhs_stream, *args)
+    h = Zillabyte::Harness::Join.new(*args)
 
     # Yield 
     throw "programmable joins are not supported at this time" unless block.nil?
@@ -137,33 +144,30 @@ class Zillabyte::Harness::Stream
     
     # Sanity
     Zillabyte::Harness::Helper.check_name("join", h._name, @_app._names)
-    case h._join_type
-    when :inner,:left,:right
-    when :outer
-      Zillabyte::Harness::Helper.print_error("Outer joins are currently not supported")
-    else 
-      Zillabyte::Harness::Helper.print_error("Unknown join type: #{h._join_type}")
-    end
-    
+
     # Build the node
-    emit_stream_name = "stream_"+Zillabyte::Harness::Counter.get()
-    h._emits = [emit_stream_name]
+    if h._emits
+      Zillabyte::Harness::Helper.check_emits("join", h._emits, @_app._streams)
+    else
+      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+    end
+
+    Zillabyte::Harness::Helper.check_join(h)
     @_app._nodes << h
     
     # Handle incoming command..
     if(@_app._options[:command] == :info)
       # Info.. 
-      info_hash = {
-        "name" => h._name, 
+      node_hash = {"name" => h._name, 
         "type" => h._type, 
-        "consumes_lhs" => h._lhs_stream._name,
-        "consumes_rhs" => h._rhs_stream._name, 
         "lhs_fields" => h._lhs_fields,
         "rhs_fields" => h._rhs_fields,
-        "join_type" => h._join_type,
-        "emits" => h._emits, 
-      }
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+        "join_type" => h._join_type}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash1 = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name, "left" => 1}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash1, @_app._info_file)
+      arc_hash2 = {"name" => rhs_stream._name, "origin" => rhs_stream._previous_node_name, "dest" => h._name, "right" => 1}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash2, @_app._info_file)
       
     elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
       
@@ -173,13 +177,38 @@ class Zillabyte::Harness::Stream
     end
     
     # Return the stream 
-    output_stream = Zillabyte::Harness::Stream.new(emit_stream_name, @_app, h._name)
+    output_stream = self.class.new(h._emits[0], @_app, h._name)
     return output_stream
     
   end
 
+  def call_component(&block)
+    h = Zillabyte::Harness::InjectedComponent.new()
+    h.instance_eval(&block)
+    Zillabyte::Harness::Helper.check_name("component", h._name, @_app._names)
+    Zillabyte::Harness::Helper.check_call_component(h)
+    Zillabyte::Harness::Helper.check_emits("component", h._emits, @_app._streams)
+    @_app._nodes << h
 
+    if(@_app._options[:command] == :info)
+      node_hash = {"name" => h._name, "type" => h._type, "id" => h._id, "output_format" => h._output_format}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      h._consumes.each do |stream|
+        arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => h._name}
+        Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      end
+    end
 
+    # Return the stream 
+    output_streams = []
+    h._emits.each do |stream|
+      output_streams << self.class.new(stream, @_app, h._name)
+    end
+    output_streams = output_streams[0] if output_streams.size == 1
+    output_streams
+  end
 
   def sink(&block)
     h = Zillabyte::Harness::Sink.new()
@@ -187,8 +216,10 @@ class Zillabyte::Harness::Stream
     Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
     @_app._nodes << h
     if(@_app._options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name, "scope" => h._scope}
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+      node_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "scope" => h._scope}
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
     end
   end
   

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.0.24" unless defined?(VERSION)
+  VERSION = "0.1.0" unless defined?(VERSION)
 end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.1.0
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-28 00:00:00.000000000 Z
+date: 2014-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mkfifo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: zillabyte-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.24
+        version: 0.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.24
+        version: 0.1.0
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -59,22 +59,28 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ruby/lib/zillabyte.rb
 - ruby/lib/zillabyte/common/progress.rb
+- ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/harness/app.rb
+- ruby/lib/zillabyte/harness/base.rb
+- ruby/lib/zillabyte/harness/component.rb
+- ruby/lib/zillabyte/harness/component_sink.rb
+- ruby/lib/zillabyte/harness/component_source.rb
+- ruby/lib/zillabyte/harness/component_stream.rb
 - ruby/lib/zillabyte/harness/counter.rb
 - ruby/lib/zillabyte/harness/each.rb
 - ruby/lib/zillabyte/harness/filter.rb
 - ruby/lib/zillabyte/harness/group_by.rb
 - ruby/lib/zillabyte/harness/helper.rb
+- ruby/lib/zillabyte/harness/injected_component.rb
 - ruby/lib/zillabyte/harness/join.rb
 - ruby/lib/zillabyte/harness/live_delegator.rb
 - ruby/lib/zillabyte/harness/sink.rb
 - ruby/lib/zillabyte/harness/source.rb
 - ruby/lib/zillabyte/harness/stream.rb
 - ruby/lib/zillabyte/harness/tuple.rb
-- ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/version.rb
-- ruby/lib/zillabyte.rb
 homepage: http://www.zillabyte.com
 licenses:
 - MIT
@@ -85,17 +91,17 @@ require_paths:
 - ruby/lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte Gem