zillabyte 0.0.16 → 0.0.17

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTIzYmY3MjJhM2U0M2I2OGU4ZGE3ZDk2YmI4NjhkYzM1MTY0ZjY1ZA==
+    YWU1ZGQ4YmZiNjUxNDdhZjExYWU5Y2IyMWJlNWE4MmU1MTM3YTYwNg==
   data.tar.gz: !binary |-
-    OTkyMzYzODhjZmNmNjZiYzc2ZGI3NTVjNDFlNDljMzBjNTJhOTRiOQ==
-!binary "U0hBNTEy":
+    NmVlNjAzNTIwMjRiODFjZmNmZTQzYzc0YTM0NWE4ZjFiN2QzOGUxZg==
+SHA512:
   metadata.gz: !binary |-
-    OTJiZjQwMGRiYjEzMTM4YTZlODZhZTA2N2Y1YjNiMDQ4MTY3NWNkZjNkYzM0
-    NzRkNmVkN2M3OTFmNjc1Yzc2ZDU5YTA5ZjNlODU3YjIwNzVlNTE1NGM5OTk5
-    YjQxOThlNWNmY2E0YWUwM2YzZjM4Y2JhMzY3Y2IyM2VkMjM0ZTQ=
+    NTNiMmYzMDBlNjJjMGVmYTQ3YjdkMTVjMmVmNDgyZTE1YmJiMTRhYmNmZjY3
+    YTUzODQwZDZlZWViZGQ1NTkxNmE1NjAyNmQ4Nzg4YjljZjMzZjJmODY0MDJi
+    OTdlYWYwOWRhOWRkZmU5OTY3ZmZjMTg3ZDZmMjEwZjMxMjAwNGY=
   data.tar.gz: !binary |-
-    N2IyMjlhMWVkZTdmODA4MmZhOWY4YTU0NTg3NDMwZTlhY2ZmOTMzZjAyNzU1
-    Y2E0ZTcxYzYyNmVjM2Q0ZDM5YjEyMmQ1Y2ZlOGJmY2NmZjk2MDRiMzQ1ODIx
-    OGUyOWY4MmRmODc4M2VkMTJiOWI5ZTY0MTVjYTZiYWM4MjE5MjQ=
+    OTRkMzYxYmM2YTY1MWYyNTllMTg5YjI1M2NhN2U5ZjBjMGE4MTM3MDExODdk
+    ODQzMDc4MDkwY2NlZWE0Mzk1NDVjYjAxYzMyMGZiMDc0ZDVjYjI5NDE4ZDI5
+    M2ExOWJjMWFkZTdkMmI2N2I4MWE4N2M5ZDM5ZTI2NzA3Nzc1NTk=

data/ruby/lib/zillabyte.rb

@@ -3,21 +3,9 @@ require "zillabyte/common/progress"
 
 module Zillabyte
 
-  def self.new(name=nil)
-    @topology = Zillabyte::Harness::Topology.build(name)
-    @topology
-  end
-
-  def self.simple_function(*args, &block)
-    Zillabyte::Harness::SimpleFunction.build(*args, &block)
-  end
-
-  def self.simple_source(*args, &block)
-    Zillabyte::Harness::SimpleSource.build(*args, &block)
-  end
-
-  def self.simple_aggregate(*args, &block)
-    Zillabyte::Harness::SimpleAggregate.build(*args, &block)
+  def self.app(name=nil)
+    @app = Zillabyte::Harness::App.build(name)
+    @app
   end
   
 end

data/ruby/lib/zillabyte/harness/app.rb

@@ -0,0 +1,59 @@
+require "json"
+
+class Zillabyte::Harness::App
+  attr_accessor :_name, :_options, :_nodes, :_streams, :_info_file
+
+  def self.build(name)
+    h = Zillabyte::Harness::App.new()
+    h._nodes = []
+    h._streams = {}
+    h._name = name
+    Zillabyte::Harness::Helper.check_name("new", h._name, {})
+    h._options = Zillabyte::Harness::Helper.opt_parser()
+    if(h._options[:command] == :info)
+      h._info_file = File.open(h._options[:file],"w+")
+      info_hash = {"language" => "ruby", "name" => h._name}
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._info_file)
+    end
+    h
+  end
+
+  def source(*args, &block)
+    h = Zillabyte::Harness::Source.new()
+    # Are we given a block?
+    if block_given?
+      h.instance_eval(&block)
+      Zillabyte::Harness::Helper.check_name("source", h._name, {})
+      Zillabyte::Harness::Helper.check_source("source", h)
+      if(!h._emits)
+        h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      end
+    # No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
+    else
+      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      h.matches(args[0])
+    end
+    @_nodes << h
+    if(@_options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits, "end_cycle_policy" => h._end_cycle_policy}
+      if(h._relation)
+        info_hash["relation"] = h._relation
+      elsif(h._matches)
+        info_hash["matches"] = h._matches
+      end
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
+    elsif(@_options[:command] == :execute and @_options[:name] == h._name)
+      pipe_name = @_options[:pipe]
+      c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
+      c.run(pipe_name)
+    end
+
+    output_streams = []
+    h._emits.each do |stream|
+      output_streams << Zillabyte::Harness::Stream.new(stream, self, nil)
+    end
+    output_streams = output_streams[0] if output_streams.size == 1
+    output_streams
+  end
+  
+end

data/ruby/lib/zillabyte/harness/each.rb

@@ -1,5 +1,5 @@
 class Zillabyte::Harness::Each
-  attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
+  attr_accessor :_has_emit, :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
 
   def initialize()
     @_name = "each_"+Zillabyte::Harness::Counter.get()
@@ -10,12 +10,12 @@ class Zillabyte::Harness::Each
     @_name = v
   end
   
-  def emits(v)
-    @_emits = v
+  def emits(*v)
+    @_emits = *v
   end
 
-  def consumes(v)
-    @_consumes = v
+  def emit(*v)
+    @_has_emit = true
   end
 
   def prepare(&block)

data/ruby/lib/zillabyte/harness/helper.rb

@@ -1,3 +1,5 @@
+require 'optparse'
+
 class Zillabyte::Harness::Helper
 
   def self.opt_parser()
@@ -38,7 +40,7 @@ class Zillabyte::Harness::Helper
 
     # \w matchs [a-zA-Z0-9_]
     if(!name.instance_of?(String) or (name =~ /^\w+$/).nil?)
-      msg = "#{ee}\"Name\" must be a non-empty STRING with only alphanumeric and underscore characters at \"#{name}\". Flows and simple_functions must have names!"
+      msg = "#{ee}\"Name\" must be a non-empty STRING with only alphanumeric and underscore characters at \"#{name}\". Apps and simple_apps must have names!"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(names[name] and names[name] != "new" and names[name] != "sink" and operation != "new" and operation != "sink")
@@ -51,22 +53,18 @@ class Zillabyte::Harness::Helper
 
   def self.check_source(operation, source)
     ee = "Error in \"#{operation}\": \n\t "
-    if(["simple_function", "simple_aggregate"].member?(operation))
-      pp = "\"#{operation}\" may only contain a single \"matches\" clause."
-    else
-      pp = "A \"#{operation}\" may only contain a single \"emits\" OR \"matches\" clause."
-    end
-
-    em = !source._emits.nil?
+    pp = @@_print_check_source
+    
     rm = !source._relation.nil?
     mm = !source._matches.nil?
-    if( (em and rm) or (rm and mm) or (mm and em) )
-      msg = "#{ee}#{pp}"
+    if( rm and mm )
+      msg = "#{ee}"
+      msg += "A \"#{operation}\" may contain either a \"matches\" clause or a \"relation\" clause, not both. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     if(rm or mm)
       if source._prepare or source._next_batch
-        msg = "#{ee}If \"#{operation}\" has a \"matches\" clause, it may not have a \"prepare\" or \"next_batch\" block."
+        msg = "#{ee}If \"#{operation}\" has a \"matches\\relation\" clause, it may not have a \"prepare\" or \"next_batch\" block. #{pp}"
         Zillabyte::Harness::Helper.print_error(msg)
       end
     end
@@ -74,16 +72,11 @@ class Zillabyte::Harness::Helper
 
   def self.check_emits(operation, emits, streams)
     ee = "Error in \"#{operation}\" at \"emits\": \n\t "
-    if(operation.include?("simple"))
-      pp = @@_print_check_simple_function_emits
-      nn = "relation"
-    else
-      pp = @@_print_check_emits
-      nn = "stream"
-    end
-
+    pp = @@_print_check_emits
+    nn = "stream"
+    
     if(!emits.instance_of?(Array))
-      msg = "#{ee}\"Emits\" must be an ARRAY at #{emits}. #{pp}"
+      msg = "#{ee}\"Emits\" must be a sequence of strings at #{emits}. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
     n_emits = emits.length
@@ -94,47 +87,15 @@ class Zillabyte::Harness::Helper
 
     current_op_streams = {}
     emits.each do |e|
-      if(!e.instance_of?(Array) or e.length != 2)
-        msg = "#{ee}Invalid format for \"emits\" in #{e}. #{pp}"
-        Zillabyte::Harness::Helper.print_error(msg)
-      end
-      if(!e[0].instance_of?(String) or (e[0] =~ /^\w+$/).nil?)
+      if(!e.instance_of?(String) or (e =~ /^\w+$/).nil?)
         msg = "#{ee}\"Emits\" #{nn} name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
         Zillabyte::Harness::Helper.print_error(msg)
       end
-      if(current_op_streams[e[0]])
+      if(current_op_streams[e])
         msg = "#{ee}The #{nn} \"#{e[0]}\" is listed multiple times in the same \"emits\". #{pp}"
         Zillabyte::Harness::Helper.print_error(msg)
       end
-      current_op_streams[e[0]] = 1
-      if(streams[e[0]] and streams[e[0]] != e[1])
-        msg = "#{ee}The #{nn} name \"#{e[0]}\" was previously defined with a different set of fields! #{pp}"
-        Zillabyte::Harness::Helper.print_error(msg)
-      end
-      streams[e[0]] = e[1]
-      if(e[1].length == 0)
-        msg = "#{ee}Must be at least one output field to #{nn} \"#{e[0]}\". #{pp}"
-        Zillabyte::Harness::Helper.print_error(msg)
-      end
-
-      if(operation.include?("simple"))
-        Zillabyte::Harness::Helper.check_simple_function_emits(operation, e)
-      else
-        if(!e[1].instance_of?(Array))
-          msg = "#{ee}Field names must be an ARRAY of STRINGS in stream \"#{e[0]}\". #{pp}"
-          Zillabyte::Harness::Helper.print_error(msg)
-        end
-        e[1].each do |f|
-          if(!f.instance_of?(String) or (f =~ /^\w+$/).nil?)
-            msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in stream \"#{e[0]}\". #{pp}"
-            Zillabyte::Harness::Helper.print_error(msg)
-          end
-          if(f =~ /^v[0-9]+$/i or @@meta_names.member?(f.downcase))
-            msg = "#{ee}\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your field something else. #{pp}"
-            Zillabyte::Harness::Helper.print_error(msg)
-          end
-        end
-      end
+      current_op_streams[e] = 1
     end
     if(n_emits > 1)
       return true
@@ -144,31 +105,6 @@ class Zillabyte::Harness::Helper
 
   end
 
-  def self.check_consumes(h, streams)
-    ee = "Error in \"#{h._type}\" at \"consumes\": \n\t "
-    if(h._type == "each") 
-      pp = @@_print_check_each_consumes
-    elsif(h._type == "aggregate")
-      pp = @@_print_check_aggregate_consumes
-    elsif(h._type == "sink")
-      pp = @@_print_check_sink
-    end
-
-    consumes = h._consumes
-    if(!consumes)
-      msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\", \"source\" or \"aggregate\" emitted multiple streams. #{pp}"
-      Zillabyte::Harness::Helper.print_error(msg)
-    end
-    if(!streams[consumes])
-      msg = "#{ee}The stream \"#{consumes}\" specified in \"consumes\" does not exist! #{pp}"
-      Zillabyte::Harness::Helper.print_error(msg)
-    end
-
-    if(h._type == "sink")
-      Zillabyte::Harness::Helper.check_sink_consumes(h, streams)
-    end
-  end
-
   def self.check_group_by(operation, h, nodes, streams)
     ee = "Error in \"#{operation}\" at \"group_by\": \n\t "
     pp = ""
@@ -248,14 +184,14 @@ class Zillabyte::Harness::Helper
       pp = @@_print_check_sink
     elsif(operation.include?("simple"))
       ee = "Error in \"#{operation}\" at \"emits\": \n\t "
-      pp = @@_print_check_simple_function_emits
+      pp = @@_print_check_simple_app_emits
     end
 
     if(!cname.instance_of?(String) or (cname =~ /^\w+$/).nil?)
       msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in relation \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
-    if(cname =~ /^v[0-9]+$/i or @@meta_names.member?(cname.downcase))
+    if(cname =~ /^v[0-9]+$/i or @@_meta_names.member?(cname.downcase))
       msg = "#{ee}\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your field something else. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -291,9 +227,9 @@ class Zillabyte::Harness::Helper
     end
   end
 
-  def self.check_simple_function_emits(operation, emits)
+  def self.check_simple_app_emits(operation, emits)
     ee = "Error in \"#{operation}\" at \"emits\": \n\t "
-    pp = @@_print_check_simple_function_emits
+    pp = @@_print_check_simple_app_emits
 
     name = emits[0]
     columns = emits[1]
@@ -317,7 +253,11 @@ class Zillabyte::Harness::Helper
     end
   end
   
-  
+  def self.print_simple_app_usage(msg)
+    ee = "Error in \"simple_app\": \n\t "
+    msg = "#{ee}#{msg} #{@@_print_check_simple_app} #{@@_print_check_simple_app_emits}"
+    Zillabyte::Harness::Helper.print_error(msg)
+  end
 
   # Test helper...
   def self.argv()
@@ -329,85 +269,131 @@ class Zillabyte::Harness::Helper
     @_argv = v
   end
 
-  @@_print_check_emits = "\n
-\"Emits\" Syntax:
- - \"Emits\" must be a non-empty ARRAY.
- - Each element of \"emits\" must be an ARRAY of length = 2.
+  @@_print_check_emits = <<-OUTPUT
+"Emits" Syntax:
+ - "Emits" must be a non-empty ARRAY.
+ - Each element of "emits" must be an ARRAY of length = 2.
      * The first element should be the unique stream name defined as a non-empty STRING with only alphanumeric and underscore characters.
      * The second element should be an ARRAY of field names for that stream.
      e.g.
- \t emits = [ [ \"stream_1\", [ \"field_11\", \"field_12\", ... ] ],
- \t           [ \"stream_2\", [ \"field_21\", \"field_22\", ... ] ] ] .
+     emits = [ [ "stream_1", [ "field_11", "field_12", ... ] ],
+              [ "stream_2", [ "field_21", "field_22", ... ] ] ] .
  - Stream and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
- - Field names cannot be \"v[number]\", \"id\", \"confidence\", \"since\" or \"source\" which are reserved Zillabyte names."
-
-  @@_print_check_group_by = "\n
-\"Group_by\" Syntax:
- - \"Group_by\" must be a non-empty ARRAY.
- - Each element of \"group_by\" must be a STRING corresponding to a field emitted by the previous operation or the stream
-consumed by the aggregation function."
-
-  @@_print_check_simple_function_emits = "\n
-\"Emits\" Syntax:
- - \"Emits\" must be a non-empty ARRAY.
- - Each element of \"emits\" must be an ARRAY of length = 2.
+ - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+OUTPUT
+
+ 
+  @@_print_check_group_by =<<-OUTPUT
+"Group_by" Syntax:
+ - "Group_by" must be a non-empty ARRAY.
+ - Each element of "group_by" must be a STRING corresponding to a field emitted by the previous operation or the stream
+consumed by the aggregation operation.
+OUTPUT
+
+  @@_print_check_simple_app_emits = <<-OUTPUT
+"Emits" Syntax:
+ - "Emits" must be a non-empty ARRAY.
+ - Each element of "emits" must be an ARRAY of length = 2.
      * The first element should be the unique relation name defined as a non-empty STRING with only alphanumeric and underscore characters.
      * The second element should be an ARRAY of HASHES with field names and data types for that relation. e.g.
- \t emits = [ [ \"relation_1\", [ {\"field_11\" => \"type_11\"}, {\"field_12\" => \"type_12\"}, ... ] ],
- \t           [ \"relation_2\", [ {\"field_21\" => \"type_21\"}, {\"field_22\" => \"type_22\"}, ... ] ] ] .
+    emits = [ [ "relation_1", [ {"field_11" => "type_11"}, {"field_12" => "type_12"}, ... ] ],
+              [ "relation_2", [ {"field_21" => "type_21"}, {"field_22" => "type_22"}, ... ] ] ] .
  - Relation and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
- - Field names cannot be \"v[number]\", \"id\", \"confidence\", \"since\" or \"source\" which are reserved Zillabyte names.
- - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean."
+ - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+ - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
+OUTPUT
 
-  @@_print_check_sink = "\n
-\"Sink\" Syntax:
+  @@_print_check_sink = <<-OUTPUT
+"Sink" Syntax:
  - Sinks must be specified using the following syntax:
      Single stream:
- \t flow.sink do |h|
- \t \t h.name \"name_of_relation\"
- \t \t h.column \"field_1\" :type_1
- \t \t h.column \"field_2\" :type_2 ...
- \t end
+    app.sink do
+       name "name_of_relation"
+       column "field_1" :type_1
+       column "field_2" :type_2 ...
+    end
      Multiple streams:
- \t flow.sink do |h|
- \t \t h.name \"relation_name\"
- \t \t h.consumes \"stream_consumed\"
- \t \t h.column \"field_1\" :type_1
- \t \t h.column \"field_2\" :type_2 ...
- \t end
- - \"Sink\" relation \"name\" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+    app.sink do
+       name "relation_name"
+       consumes "stream_consumed"
+       column "field_1" :type_1
+       column "field_2" :type_2 ...
+    end
+ - "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
  - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
- - Field names cannot be \"v[number]\", \"id\", \"confidence\", \"since\" or \"source\" which are reserved Zillabyte names.
+ - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
  - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
- - If there are multiple streams, \"consumes\" must be specified for each sink as a non-empty STRING!
-     * \"Consumes\" is the name of a stream emitted by an \"each\" or a \"source\" which the \"sink\" should save as a table.
-     * The columns specified in \"sink\" must match the fields emitted by the stream."
-
-  @@_print_check_each_consumes = "\n
-\"Each\" Syntax for multiple streams:
- \t flow.each do |h|
- \t \t h.name \"name\",  => optional
- \t \t h.emits emits,
- \t \t h.consumes \"consumed_stream\"
- \t \t h.prepare ...
- \t \t h.execute ...
- \t end
- - If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
-     * \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"source\" or \"aggregate\" which the current \"each\" operates on."
-
-  @@_print_check_aggregate_consumes = "\n
-\"Aggregate\" Syntax for multiple streams:
- \t flow.aggregate do |h|
- \t \t h.name \"name\",  => optional
- \t \t h.group_by [\"field_1\", \"field_2\", ...],
- \t \t h.emits emits,
- \t \t h.consumes \"consumed_stream\"
- \t \t h.begin_group ...
- \t \t h.aggregate ...
- \t \t h.end_group ...
- \t end
- - If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
-     * \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"source\" or \"aggregate\" which the current \"aggregate\" operates on."
-
-  @@meta_names=["id", "confidence", "since", "source"]
+ - If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
+     * "Consumes" is the name of a stream emitted by an "each" or a "source" which the "sink" should save as a table.
+     * The columns specified in "sink" must match the fields emitted by the stream."
+OUTPUT
+
+  @@_print_check_source = <<-OUTPUT
+"Source" Syntax:
+ - Sources must be specified using the following syntax:
+     Sourcing from a relation:
+    app.source do
+       name "name"  => optional
+       matches "sql_query or sxp"
+    end
+     Custom source:
+    app.source do
+       name "name"  => optional
+       emits emits
+       prepare ...
+       next_batch ...
+    end
+OUTPUT
+
+  @@_print_check_each = <<-OUTPUT
+"Each" Syntax:
+ - Eaches must be specified using the following syntax:
+    app.each do
+       name "name"  => optional
+       emits emits
+       consumes "consumed_stream"  => optional if single stream
+       prepare ...
+       execute ...
+    end
+ - If there are multiple streams, "consumes" must be specified as a non-empty STRING!
+     * "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "each" operates on."
+OUTPUT
+
+  @@_print_check_aggregate = <<-OUTPUT
+"Aggregate" Syntax:
+ - Aggregates must be specified using the following syntax:
+    app.aggregate do
+       name "name"  => optional
+       group_by ["field_1", "field_2", ...]
+       emits emits
+       consumes "consumed_stream"  => optional if single stream
+       begin_group ...
+       aggregate ...
+       end_group ...
+    end
+ - If there are multiple streams, "consumes" must be specified as a non-empty STRING!
+     * "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "aggregate" operates on.
+OUTPUT
+
+  @@_print_check_simple_app = <<-OUTPUT
+"Simple_app" Syntax:
+ - Simple_apps must be specified using the following syntax:
+     Sinking directly from a custom source:
+    Zillabyte.simple_app do 
+       name "name"
+       emits emits
+       prepare ...
+       next_batch ...
+    end
+     Processing and sinking data from a relation:
+    Zillabyte.simple_app do
+       name "name"
+       matches "sql_query or sxp"
+       emits emits
+       prepare ...
+       execute ...
+    end
+OUTPUT
+
+  @@_meta_names = ["id", "confidence", "since", "source"]
 end