zillabyte 0.1.43 → 0.1.44

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4a67906d45c018f5312141a05b16a199e8b7e00
-  data.tar.gz: 653312b49f906d7d04a96170257f38885d051ea3
+  metadata.gz: 52c3a818d68e3b81d1e7e60a2265278c5bc0cd7b
+  data.tar.gz: 01481ba634bd7f736c9257a54eb4e8a93c2530b0
 SHA512:
-  metadata.gz: 42530197ceae964a91870da58a0562ef82e2051e661727ca78e7c812ac728ee0206a1e01654261e8fab81c7804678d8ad6b9f7895df542bcadf4e590977cc99b
-  data.tar.gz: e4d1b3f9bdf620fe1b8f8f4e2f500348623feb56f20082ad036eeb4f5943d520ef5a3433821102678c4defc2f8189f9b9a1041aeaadbf7e5d8f201206cbdb222
+  metadata.gz: f2c233cb1332221b640a94e66a4bd080cf3feae7acd4d916d5b77aa49f3503744c0ef78eb00dd198f421e4e7fcfadf3a032d0a9452b8d275b96adc086492928d
+  data.tar.gz: 4291a6a52ae5e47e137fd55ff6fa66f2e7056d3b0dab4d91a2a117490788ad01e8bef37bfa2ee6f7e31d5c6ee5de32d4ca4b2d2d5e830d8d3fc6db3b3ba5c1fa

data/ruby/lib/zillabyte/harness/app.rb

@@ -16,47 +16,11 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
   end
 
   def source(*args, &block)
-    h = Zillabyte::Harness::Source.new()
-    # Are we given a block?
-    if block_given?
-      h.instance_eval(&block)
-      Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
-      Zillabyte::Harness::Helper.check_custom_source(h)
-      if h._emits
-        Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams)
-      else
-        h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      end
-    # No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
-    else
-      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      Zillabyte::Harness::Helper.check_source_args(args)
-      h.matches(args[0])
-    end
-    @_nodes << h
-    if(@_options[:command] == :info)
-      node_hash = {"name" => h._name, "type" => h._type}
-      if(h._relation)
-        node_hash["relation"] = h._relation
-      elsif(h._matches)
-        node_hash["matches"] = h._matches
-      else
-        node_hash["end_cycle_policy"] = h._end_cycle_policy
-      end
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_info_file)
-    elsif(@_options[:command] == :execute and @_options[:name] == h._name)
-      pipe_name = @_options[:pipe]
-      c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
-      c.run(pipe_name)
-    end
-
-    output_streams = []
-    h._emits.each do |stream|
-      output_streams << Zillabyte::Harness::Stream.new(stream, self, h._name)
-    end
-    output_streams = output_streams[0] if output_streams.size == 1
-    output_streams
-  end
-  
+    op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::Stream)
+    op.build_multilang_operation("source", *args, &block)
+      .add_operation_properties_to_info(:name, :type)
+      .add_optional_operation_properties_to_info(:relation, :matches, :end_cycle_policy)
+      .handle_operation
+      .get_output_streams
+  end 
 end

data/ruby/lib/zillabyte/harness/common_node.rb

@@ -0,0 +1,16 @@
+class Zillabyte::Harness::CommonNode
+  attr_accessor :_name, :_type, :_emits, :_parallelism
+
+  def name(v)
+    @_name = v
+  end
+
+  def emits(*v)
+    @_emits = *v
+  end
+
+  def parallelism(v)
+    @_parallelism = v
+  end
+
+end

data/ruby/lib/zillabyte/harness/component.rb

@@ -15,19 +15,10 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
   end
 
   def inputs(*args, &block)
-    h = Zillabyte::Harness::ComponentSource.new
-    h.instance_eval(&block)
-    h._name ||= "comp_input.#{Zillabyte::Harness::Counter.get()}"
-    Zillabyte::Harness::Helper.check_name("inputs", h._name, @_names)
-    Zillabyte::Harness::Helper.check_component_source(h)
-
-    if(self._options[:command] == :info)
-      node_hash = {"name" => h._name, "type" => h._type, "fields" => h._fields}
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, self._info_file)
-    end
-
-    # For components, each "inputs" statement only defines a single stream
-    output_stream = Zillabyte::Harness::ComponentStream.new(h._emits.first, self, h._name)
-    output_stream
+    op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::ComponentStream)
+    op.build_multilang_operation("component_source", *args, &block)
+      .add_operation_properties_to_info(:name, :type, :fields)
+      .handle_operation
+      .get_output_streams
   end
 end

data/ruby/lib/zillabyte/harness/component_sink.rb

@@ -1,26 +1,46 @@
 class Zillabyte::Harness::ComponentSink
-  attr_accessor :_name, :_relation, :_type, :_columns, :_scope
+  attr_accessor :_app, :_node
 
-  def initialize()
-    @_type = 'sink'
-    @_columns = []
-  end
+  class Node
+    attr_accessor :_name, :_relation, :_type, :_columns, :_scope
+
+    def initialize()
+      @_type = 'sink'
+      @_columns = []
+    end
   
-  def name(v)
-    @_name = v
-  end
+    def name(v)
+      @_name = v
+    end
   
-  def relation(v)
-    @_relation = v
-  end
+    def relation(v)
+      @_relation = v
+    end
   
-  def field(cname, ctype)
-    col = {cname => ctype}
-    @_columns << col
-  end
+    def field(cname, ctype)
+      col = {cname => ctype}
+      @_columns << col
+    end
   
-  def scope(v)
-    @_scope = v
+    def scope(v)
+      @_scope = v
+    end
+  end
+
+  def initialize(app, *args)
+    @_app = app
+  end
+
+  def build_node(&block)
+    @_node = Node.new
+    @_node.instance_eval(&block)
+    @_node._name ||= "comp_output.#{Zillabyte::Harness::Counter.get()}"
+    Zillabyte::Harness::Helper.check_name("outputs", @_node._name, @_app._names)
+    Zillabyte::Harness::Helper.check_component_sink(@_node)
+    @_node._relation ||= @_node._name
+  end
+
+  def run_operation
+    puts "Component output is not runnable locally."
   end
-  
 end

data/ruby/lib/zillabyte/harness/component_source.rb

@@ -1,17 +1,37 @@
 class Zillabyte::Harness::ComponentSource
-  attr_accessor :_name, :_type, :_emits, :_fields
+  attr_accessor :_app, :_node
 
-  def initialize()
-    @_type = "source"
-    @_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-    @_fields = []
+  class Node
+    attr_accessor :_name, :_type, :_emits, :_fields
+
+    def initialize()
+      @_type = "source"
+      @_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      @_fields = []
+    end
+
+    def name(v)
+      @_name = v
+    end
+
+    def field(name, type)
+      @_fields << {name => type}
+    end
+  end
+
+  def initialize(app, *args)
+    @_app = app
   end
 
-  def name(v)
-    @_name = v
+  def build_node(&block)
+    @_node = Node.new
+    @_node.instance_eval(&block)
+    @_node._name ||= "comp_input.#{Zillabyte::Harness::Counter.get()}"
+    Zillabyte::Harness::Helper.check_name("inputs", @_node._name, @_app._names)
+    Zillabyte::Harness::Helper.check_component_source(@_node)
   end
 
-  def field(name, type)
-    @_fields << {name => type}
+  def run_operation
+    puts "Component inputs cannot be run!"
   end
 end

data/ruby/lib/zillabyte/harness/component_stream.rb

@@ -3,19 +3,12 @@ require 'zillabyte/harness/stream'
 class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
 
   def outputs(*args, &block)
-    h = Zillabyte::Harness::ComponentSink.new
-    h.instance_eval(&block)
-    h._name ||= "comp_output.#{Zillabyte::Harness::Counter.get()}"
-    Zillabyte::Harness::Helper.check_name("outputs", h._name, @_app._names)
-    Zillabyte::Harness::Helper.check_component_sink(h)
-
-    if(@_app._options[:command] == :info)
-      node_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "scope" => h._scope}
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-    end
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_multilang_operation("component_sink", *args, &block)
+      .add_operation_properties_to_info(:name, :type, :columns, :scope)
+      .add_optional_operation_properties_to_info(:relation)
+      .create_arc_info_from_stream(self)
+      .handle_operation
   end
 
-
 end

data/ruby/lib/zillabyte/harness/each.rb

@@ -1,34 +1,56 @@
 class Zillabyte::Harness::Each
-  attr_accessor :_name, :_type, :_emits, :_output_format, :_parallelism, :_prepare, :_execute
+  attr_accessor :_app, :_node, :_options
 
-  def initialize()
-    @_name = "each_"+Zillabyte::Harness::Counter.get()
-    @_type = "each"
-    @_output_format = :replace
-  end
-  
-  def name(v)
-    @_name = v
-  end
+  class Node < Zillabyte::Harness::CommonNode
+    attr_accessor :_output_format, :_prepare, :_execute
+
+    def initialize()
+      @_name = "each_"+Zillabyte::Harness::Counter.get()
+      @_type = "each"
+      @_output_format = :replace
+    end
   
-  def emits(*v)
-    @_emits = *v
-  end
+    def output_format(v)
+      @_output_format = v
+    end
 
-  def output_format(v)
-    @_output_format = v
-  end
+    def prepare(&block)
+      @_prepare = block
+    end
 
-  def parallelism(v)
-    @_parallelism = v
+    def execute(&block)
+      @_execute = block
+    end
   end
 
-  def prepare(&block)
-    @_prepare = block
+  def initialize(app, *args)
+    @_app = app
+    @_options = Zillabyte::Harness::Helper.get_options(args)
   end
 
-  def execute(&block)
-    @_execute = block
+  def build_node(&block)
+    @_node = Node.new
+    # Does the block take 0 arguments? If so it's not just an execute block.
+    if(block.arity == 0)
+      @_node.instance_eval(&block)
+      Zillabyte::Harness::Helper.check_name("each", @_node._name, @_app._names)
+      if @_node._emits
+        Zillabyte::Harness::Helper.check_emits("each", @_node._emits, @_app._streams)
+      else
+        @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      end
+    # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
+    else
+      @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      @_node._execute = block
+    end
+
+    Zillabyte::Harness::Helper.check_each(@_node)
   end
 
+  def run_operation
+    pipe_name = @_app._options[:pipe]
+    c = Zillabyte::Harness::EachController.new(@_node, progress = Zillabyte::Common::Progress.new)
+    c.run(pipe_name)
+  end
 end

data/ruby/lib/zillabyte/harness/filter.rb

@@ -1,29 +1,52 @@
 class Zillabyte::Harness::Filter
-  attr_accessor :_name, :_type, :_emits, :_parallelism, :_prepare, :_keep
+  attr_accessor :_app, :_node, :_options
 
-  def initialize()
-    @_name = "filter_"+Zillabyte::Harness::Counter.get()
-    @_type = 'filter'
-  end
+  class Node < Zillabyte::Harness::CommonNode
+    attr_accessor :_prepare, :_keep
+
+    def initialize()
+      @_name = "filter_"+Zillabyte::Harness::Counter.get()
+      @_type = 'filter'
+    end
   
-  def name(v)
-    @_name = v
-  end
+    def prepare(&block)
+      @_prepare = block
+    end
 
-  def emits(*v)
-    @_emits = *v
+    def keep(&block)
+      @_keep = block
+    end
   end
 
-  def parallelism(v)
-    @_parallelism = v
+  def initialize(app, *args)
+    @_app = app
+    @_options = Zillabyte::Harness::Helper.get_options(args)
   end
-  
-  def prepare(&block)
-    @_prepare = block
+
+  def build_node(&block)
+    @_node = Node.new
+    # Does the block take 0 arguments? If so it's not just an execute block.
+    if(block.arity == 0)
+      @_node.instance_eval(&block)
+      Zillabyte::Harness::Helper.check_name("filter", @_node._name, @_app._names)
+      if @_node._emits
+        Zillabyte::Harness::Helper.check_emits("filter", @_node._emits, @_app._streams)
+      else
+        @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      end
+    # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
+    else
+      @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      @_node._keep = block
+    end
+
+    Zillabyte::Harness::Helper.check_filter(@_node)
   end
 
-  def keep(&block)
-    @_keep = block
+  def run_operation
+    pipe_name = @_app._options[:pipe]
+    c = Zillabyte::Harness::FilterController.new(@_node, progress = Zillabyte::Common::Progress.new)
+    c.run(pipe_name)
   end
 
 end

data/ruby/lib/zillabyte/harness/group_by.rb

@@ -1,34 +1,54 @@
 class Zillabyte::Harness::GroupBy
-  attr_accessor :_name, :_type, :_group_by, :_emits, :_parallelism, :_begin_group, :_aggregate, :_end_group
+  attr_accessor :_app, :_node, :_fields, :_options
 
-  def initialize(*v)
-    @_name = "group_by_"+Zillabyte::Harness::Counter.get()
-    @_type = 'group_by'
-    @_group_by = *v
-  end
+  class Node < Zillabyte::Harness::CommonNode
+    attr_accessor :_group_by, :_begin_group, :_aggregate, :_end_group
+
+    def initialize(v)
+      @_name = "group_by_"+Zillabyte::Harness::Counter.get()
+      @_type = 'group_by'
+      @_group_by = v
+    end
   
-  def name(v)
-    @_name = v
-  end
+    def begin_group(&block)
+      @_begin_group = block
+    end
 
-  def emits(*v)
-    @_emits = *v
-  end
+    def aggregate(&block)
+      @_aggregate = block
+    end
 
-  def parallelism(v)
-    @_parallelism = v
-  end
-  
-  def begin_group(&block)
-    @_begin_group = block
+    def end_group(&block)
+      @_end_group = block
+    end
   end
 
-  def aggregate(&block)
-    @_aggregate = block
+  def initialize(app, *args)
+    @_app = app
+    @_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
+    @_options = Zillabyte::Harness::Helper.get_options(args)
   end
 
-  def end_group(&block)
-    @_end_group = block
+  def build_node(&block)
+    @_node = Node.new(@_fields)
+
+    # Yield 
+    @_node.instance_eval(&block)
+    Zillabyte::Harness::Helper.check_name("group_by", @_node._name, @_app._names)
+
+    # Multiple emits?
+    if @_node._emits
+      Zillabyte::Harness::Helper.check_emits("group_by", @_node._emits, @_app._streams)
+    else
+      @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+    end
+
+    Zillabyte::Harness::Helper.check_group_by(@_node)
   end
 
+  def run_operation
+    pipe_name = @_app._options[:pipe]
+    c = Zillabyte::Harness::GroupByController.new(@_node, Zillabyte::Common::Progress.new)
+    c.run(pipe_name)
+  end
 end