zillabyte 0.0.6

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OTlmOGNkMGVlNzY4MTdjMThlODE3NDk3ZTUwMjEzNjIyMzBlOWZkMQ==
+  data.tar.gz: !binary |-
+    MDYwN2YwYmMyN2M2Yjg2Y2E3YzIxMWNlYjE0N2Y5ZDQ4OWNkNjczZg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MDFjMDk3NGU2NmUyYTUyNGViMTAwYWI1MjBiMzdkMzk1MThjYmIxMjM5NzE2
+    NzMwMGRiMjUyYzM2YzJhNjJhY2ZlZGE2N2ZmMDM5N2VlZTdhMDNmMmZlMDQx
+    OTI0MGM1MmUxNzVjOTM2OWM5ZWU2Y2NjYWJlNjQxNjA4NGVmZTM=
+  data.tar.gz: !binary |-
+    ZDcyOTM1YjkzZjBjYmNjZDY1MzJjOTVjZTJjMTYyNDEyZDQzMGEzYmJiZGFi
+    ZTAyYjM1NjAyOTdjNjU1NTg1MzJjY2JmNDA3YTZjYTZmNTk3ZGJhNzEyN2U3
+    NjM5MTYyMTJiOGI1YThlN2MzNWZkNWJhYTVkMTU5YjMwMzkwZWY=

data/ruby/README.md

@@ -0,0 +1,2 @@
+api.client.ruby
+===============

data/ruby/lib/zillabyte.rb

@@ -0,0 +1,20 @@
+require "zillabyte/harness"
+require "zillabyte/common/progress"
+
+module Zillabyte
+
+  def self.new(name=nil)
+    @topology = Zillabyte::Harness::Topology.build(name)
+    @topology
+  end
+
+  def self.simple_function(*args, &block)
+    Zillabyte::Harness::SimpleFunction.build(*args, &block)
+  end
+
+  def self.simple_spout(*args, &block)
+    Zillabyte::Harness::SimpleSpout.build(*args, &block)
+  end
+
+  
+end

data/ruby/lib/zillabyte/common/progress.rb

@@ -0,0 +1,17 @@
+module Zillabyte::Common
+  class Progress
+    
+    def update(*args)
+      display(*args)
+    end
+    
+    def display(*args)
+      puts *args
+    end
+    
+    def error(*args)
+      puts "error: #{args.first}"
+    end
+      
+  end
+end

data/ruby/lib/zillabyte/harness.rb

@@ -0,0 +1,16 @@
+module Zillabyte
+
+  module Harness
+
+    def self.load
+      Dir[File.join(File.dirname(__FILE__), "harness", "*.rb")].sort.each do |file|
+        require file
+      end
+    end
+  end
+end
+
+
+# Load all helpers...
+Zillabyte::Harness.load()
+

data/ruby/lib/zillabyte/harness/aggregate.rb

@@ -0,0 +1,29 @@
+class Zillabyte::Harness::Aggregate
+  attr_accessor :_name, :_type, :_emits, :_start, :_aggregate, :_complete
+
+  def initialize()
+    @_name = "aggregate_"+Zillabyte::Harness::Counter.get()
+    @_type = 'aggregate'
+  end
+  
+  def name(v)
+    @_name = v
+  end
+  
+  def emits(v)
+    @_emits = v
+  end
+
+  def start(&block)
+    @_start = block
+  end
+
+  def aggregate(&block)
+    @_aggregate = block
+  end
+
+  def complete(&block)
+    @_complete = block
+  end
+
+end

data/ruby/lib/zillabyte/harness/counter.rb

@@ -0,0 +1,11 @@
+class Zillabyte::Harness::Counter
+  @@count = 0
+  def self.get
+    @@count += 1
+    @@count.to_s
+  end
+  
+  def self.reset
+    @@count = 0
+  end
+end

data/ruby/lib/zillabyte/harness/each.rb

@@ -0,0 +1,29 @@
+class Zillabyte::Harness::Each
+  attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
+
+  def initialize()
+    @_name = "each_"+Zillabyte::Harness::Counter.get()
+    @_type = 'each'
+  end
+  
+  def name(v)
+    @_name = v
+  end
+  
+  def emits(v)
+    @_emits = v
+  end
+
+  def consumes(v)
+    @_consumes = v
+  end
+
+  def prepare(&block)
+    @_prepare = block
+  end
+
+  def execute(&block)
+    @_execute = block
+  end
+
+end

data/ruby/lib/zillabyte/harness/groupby.rb

@@ -0,0 +1,9 @@
+class Zillabyte::Harness::GroupBy
+  attr_accessor :_type, :_fields
+  
+  def initialize(fields)
+    @_type = 'group_by'
+    @_fields = fields
+  end
+  
+end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -0,0 +1,326 @@
+class Zillabyte::Harness::Helper
+
+  def self.opt_parser()
+    options = {
+      "name" => Dir.pwd.split("/")[-1]
+    }
+    OptionParser.new do |opts|
+      opts.on("--execute_live") do |v|
+        options[:command] = :execute
+      end
+      opts.on("--info") do |v|
+        options[:command] = :info
+      end
+      opts.on("--name NAME") do |v|
+        options[:name] = v
+      end
+      opts.on("--pipe PIPE") do |v|
+        options[:pipe] = v
+      end
+      opts.on("--file FNAME") do |v|
+        options[:file] = v
+      end
+    end.parse(self.argv)
+    options
+  end
+
+  def self.write_hash_to_file(hash, file)
+    file.write(hash.to_json+"\n")
+  end
+
+  def self.print_error(msg)
+    puts msg
+    exit(1)
+  end
+
+  def self.check_name(operation, name, names)
+    ee = "Error in \"#{operation}\" at \"name\": \n\t "
+
+    if(!name.instance_of?(String) or name == "")
+      msg = "#{ee}\"Name\" must be a non-empty STRING at #{name}."
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(names[name] and names[name] != "new" and names[name] != "sink" and operation != "new" and operation != "sink")
+      msg = "#{ee}The \"name\" \"#{name}\" was previously defined in a #{names[name]}!"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    names[name] = operation
+
+  end
+
+  def self.check_emits(operation, emits, streams)
+    ee = "Error in \"#{operation}\" at \"emits\": \n\t "
+    if(operation == "simple_function" or operation == "simple_spout")
+      pp = @@_print_check_simple_function_emits
+      nn = "relation"
+    else
+      pp = @@_print_check_emits
+      nn = "stream"
+    end
+
+    if(!emits.instance_of?(Array))
+      msg = "#{ee}\"Emits\" must be an ARRAY at #{emits}. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    n_emits = emits.length
+    if(n_emits == 0)
+      msg = "#{ee}Must emit at least one #{nn}, \"emits\" cannot be an empty array. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    current_op_streams = {}
+    emits.each do |e|
+      if(!e.instance_of?(Array) or e.length != 2)
+        msg = "#{ee}Invalid format for \"emits\" in #{e}. #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+      if(!e[0].instance_of?(String) or e[0] == "")
+        msg = "#{ee}\"Emits\" #{nn} name must be a non-empty STRING in #{e}. #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+      if(current_op_streams[e[0]])
+        msg = "#{ee}The #{nn} \"#{e[0]}\" is listed multiple times in the same \"emits\". #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+      current_op_streams[e[0]] = 1
+      if(streams[e[0]] and streams[e[0]] != e[1])
+        msg = "#{ee}The #{nn} name \"#{e[0]}\" was previously defined with a different set of fields! #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+      streams[e[0]] = e[1]
+      if(e[1].length == 0)
+        msg = "#{ee}Must be at least one output field to #{nn} \"#{e[0]}\". #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+
+      if(operation == "simple_function" or operation == "simple_spout")
+        Zillabyte::Harness::Helper.check_simple_function_emits(e)
+      else
+        if(!e[1].instance_of?(Array))
+          msg = "#{ee}Field names must be an ARRAY of STRINGS in stream #{e[0]}. #{pp}"
+          Zillabyte::Harness::Helper.print_error(msg)
+        end
+        e[1].each do |f|
+          if(!f.instance_of?(String) or f == "")
+            msg = "#{ee}Field names must be non-empty STRINGS in stream #{e[0]}. #{pp}"
+            Zillabyte::Harness::Helper.print_error(msg)
+          end
+        end
+      end
+    end
+    if(n_emits > 1)
+      return true
+    else
+      return false
+    end
+
+  end
+
+  def self.check_consumes(h, streams)
+    if(h._type == "each") 
+      ee = "Error in \"each\" at \"consumes\": \n\t "
+      pp = @@_print_check_each_consumes
+    elsif(h._type == "sink")
+      ee = "Error in \"sink\" at \"consumes\": \n\t "
+      pp = @@_print_check_sink
+    end
+
+    consumes = h._consumes
+    if(!consumes)
+      msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\" or \"spout\" emitted multiple streams. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(!consumes.instance_of?(String) or consumes == "")
+      msg = "#{ee}\"Consumes\" must be a non-empty STRING at #{consumes}. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(!streams[consumes])
+      msg = "#{ee}The stream \"#{consumes}\" specified in \"consumes\" does not exist! #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+
+    if(h._type == "sink")
+      Zillabyte::Harness::Helper.check_sink_consumes(h, streams)
+    end
+  end
+
+  def self.check_sink(sink, nodes)
+    ee = "Error in \"sink\": \n\t "
+    pp = @@_print_check_sink
+
+    name = sink._name
+    columns = sink._columns
+    if(!name)
+      msg = "#{ee}Relation name must be specified! #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    Zillabyte::Harness::Helper.check_name("sink", sink._name, {})
+
+    if(columns.length == 0)
+      msg = "#{ee}Must be at least one output field to relation \"#{name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    Zillabyte::Harness::Helper.check_sink_columns(sink)
+
+    nodes.each do |s|
+      if(s._type != "sink")
+        next
+      end
+      if(s._name == name and s._columns != columns)
+        msg = "#{ee}The relation \"#{name}\" has already been specified and contains a different set of fields/types. #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+    end
+  end
+
+  def self.check_sink_columns(sink)
+    name = sink._name
+    columns = sink._columns
+    columns.each do |col|
+      cname = col.keys()[0]
+      ctype = col[cname]
+      Zillabyte::Harness::Helper.check_sink_column_format("sink",cname,ctype,name)
+    end
+  end
+
+  def self.check_sink_column_format(operation, cname, ctype, relation_name)
+    if(operation == "sink")
+      ee = "Error in \"sink\" at \"column\": \n\t "
+      pp = @@_print_check_sink
+    elsif(operation == "simple_function")
+      ee = "Error in \"simple_function\" at \"emits\": \n\t "
+      pp = @@_print_check_simple_function_emits
+    end
+
+    if(!cname.instance_of?(String) or cname == "")
+      msg = "#{ee}Field names must be non-empty STRINGS in relation \"#{relation_name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(!ctype.instance_of?(Symbol))
+      msg = "#{ee}Field data types must be SYMBOLS in relation \"#{relation_name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    if(ctype != :string and ctype != :integer and ctype != :float and ctype != :double && ctype != :boolean)
+      msg = "#{ee}Invalid field data type at \"#{ctype}\" in relation \"#{relation_name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
+  def self.check_sink_consumes(sink, streams)
+    ee = "Error in \"sink\" at \"consumes\": \n\t "
+    pp = @@_print_check_sink
+
+    name = sink._name
+    columns = sink._columns
+    consumes = sink._consumes
+
+    stream_fields = streams[consumes]
+    if(stream_fields.length != columns.length)
+      msg = "#{ee}Number of columns in \"sink\" differs from number of fields in the consumed stream at relation \"#{name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    columns.each do |col|
+      col_name = col.keys()[0]
+      if(!stream_fields.include?(col_name))
+        msg = "#{ee}The column \"#{col_name}\", is not emitted by the stream \"#{consumes}\". #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+    end
+  end
+
+  def self.check_simple_function_emits(emits)
+    ee = "Error in \"simple_function\" at \"emits\": \n\t "
+    pp = @@_print_check_simple_function_emits
+
+    name = emits[0]
+    columns = emits[1]
+    if(!columns.instance_of?(Array))
+      msg = "#{ee}Field names must be an ARRAY of HASHES in relation \"#{name}\". #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+    columns.each do |col|
+      if(!col.instance_of?(Hash))
+        msg = "#{ee}Fields names must be listed in HASH format in relation \"#{name}\". #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+      colkeys = col.keys()
+      if(colkeys.length != 1)
+        msg = "#{ee}Each field must be a separate HASH with {field_name : data_type} in relation \"#{name}\". #{pp}"
+        Zillabyte::Harness::Helper.print_error(msg)
+      end
+      colkey = colkeys[0]
+      colval = col[colkey]
+      Zillabyte::Harness::Helper.check_sink_column_format("simple_function",colkey,colval,name)
+    end
+  end
+  
+  
+
+  # Test helper...
+  def self.argv()
+    @_argv || ARGV
+  end
+  
+  # Test helper..
+  def self.argv=(v)
+    @_argv = v
+  end
+
+  @@_print_check_emits = "\n
+\"Emits\" Syntax:
+ - \"Emits\" must be a non-empty ARRAY.
+ - Each element of \"emits\" must be an ARRAY of length = 2.
+     * The first element should be the unique stream name defined as a non-empty STRING.
+     * The second element should be an ARRAY of field names for that stream.
+     e.g.
+ \t emits = [ [ \"stream_1\", [ \"field_11\", \"field_12\", ... ] ],
+ \t           [ \"stream_2\", [ \"field_21\", \"field_22\", ... ] ] ] .
+ - Stream and field names must all be non-empty STRINGS."
+
+  @@_print_check_simple_function_emits = "\n
+\"Emits\" Syntax:
+ - \"Emits\" must be a non-empty ARRAY.
+ - Each element of \"emits\" must be an ARRAY of length = 2.
+     * The first element should be the unique relation name defined as a non-empty STRING.
+     * The second element should be an ARRAY of HASHES with field names and data types for that relation. e.g.
+ \t emits = [ [ \"relation_1\", [ {\"field_11\" => \"type_11\"}, {\"field_12\" => \"type_12\"}, ... ] ],
+ \t           [ \"relation_2\", [ {\"field_21\" => \"type_21\"}, {\"field_22\" => \"type_22\"}, ... ] ] ] .
+ - Relation and field names must all be non-empty STRINGS.
+ - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean."
+
+  @@_print_check_sink = "\n
+\"Sink\" Syntax:
+ - Sinks must be specified using the following syntax:
+     Single stream:
+ \t flow.sink do |h|
+ \t \t h.name \"name_of_relation\"
+ \t \t h.columns \"field_1\" :type_1
+ \t \t h.columns \"field_2\" :type_2 ...
+ \t end
+     Multiple streams:
+ \t flow.sink do |h|
+ \t \t h.name \"relation_name\"
+ \t \t h.consumes \"stream_consumed\"
+ \t \t h.columns \"field_1\" :type_1
+ \t \t h.columns \"field_2\" :type_2 ...
+ \t end
+ - \"Sink\" relation \"name\" must be specified as a non-empty STRING!
+ - Field names must be non-empty STRINGS.
+ - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
+ - If there are multiple streams, \"consumes\" must be specified for each sink as a non-empty STRING!
+     * \"Consumes\" is the name of a stream emitted by an \"each\" or a \"spout\" which the \"sink\" should save as a table.
+     * The columns specified in \"sink\" must match the fields emitted by the stream."
+
+  @@_print_check_each_consumes = "\n
+\"Each\" Syntax for multiple streams:
+ \t flow.each do |h|
+ \t \t h.name \"name\",  => optional
+ \t \t h.emits emits,
+ \t \t h.consumes \"consumed_stream\"
+ \t \t h.prepare ...
+ \t \t h.execute ...
+ \t end
+ - If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
+     * \"Consumes\" is the name of a stream emitted by a preceding \"each\" or \"spout\" which the current \"each\" operates on."
+
+end