zillabyte 0.1.43 → 0.1.44

data/ruby/lib/zillabyte/harness/stream.rb

@@ -13,476 +13,137 @@ class Zillabyte::Harness::Stream
     @_name
   end
 
-  def each(options = {}, &block)
-    h = Zillabyte::Harness::Each.new()
-    # Does the block take 0 arguments? If so it's not just an execute block.
-    if(block.arity == 0)
-      h.instance_eval(&block)
-      Zillabyte::Harness::Helper.check_name("each", h._name, @_app._names)
-      if h._emits
-        Zillabyte::Harness::Helper.check_emits("each", h._emits, @_app._streams)
-      else
-        h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      end
-    # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
-    else
-      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      h._execute = block
-    end
-
-    Zillabyte::Harness::Helper.check_each(h)
-    @_app._nodes << h
-
-    if(@_app._options[:command] == :info)
-
-      node_hash = {
-        :name => h._name, 
-        :type => h._type, 
-        :output_format => h._output_format,
-        :config => options
-      }
-      if (h._parallelism)
-        node_hash["parallelism"] = h._parallelism
-      end
-
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      pipe_name = @_app._options[:pipe]
-      c = Zillabyte::Harness::EachController.new(h, progress = Zillabyte::Common::Progress.new)
-      c.run(pipe_name)
-    end
-
-    output_streams = []
-    h._emits.each do |stream|
-      output_streams << self.class.new(stream, @_app, h._name)
-    end
-    output_streams = output_streams[0] if output_streams.size == 1
-    output_streams
+  def each(*args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_multilang_operation("each", *args, &block)
+      .add_operation_properties_to_info(:name, :type, :output_format)
+      .add_optional_operation_properties_to_info(:parallelism)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
 
-
-  def filter(options = {}, &block)
-    h = Zillabyte::Harness::Filter.new()
-    # Does the block take 0 arguments? If so it's not just a keep block.
-    if(block.arity == 0)
-      h.instance_eval(&block)
-      Zillabyte::Harness::Helper.check_name("filter", h._name, @_app._names)
-      if h._emits
-        Zillabyte::Harness::Helper.check_emits("filter", h._emits, @_app._streams)
-      else
-        h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      end
-    else
-      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      h._keep = block
-    end
-
-    Zillabyte::Harness::Helper.check_filter(h)
-    @_app._nodes << h
-
-    if(@_app._options[:command] == :info)
-
-      node_hash = {"name" => h._name, "type" => h._type}
-      if (h._parallelism)
-        node_hash["parallelism"] = h._parallelism
-      end
-
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {
-        :name => @_name, 
-        :origin => @_previous_node_name,
-        :dest => h._name,
-        :config => options
-      }
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      pipe_name = @_app._options[:pipe]
-      c = Zillabyte::Harness::FilterController.new(h, progress = Zillabyte::Common::Progress.new)
-      c.run(pipe_name)
-    end
-
-    output_stream = self.class.new(h._emits[0], @_app, h._name) # filters only have one output stream
-    output_stream
+  def filter(*args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_multilang_operation("filter", *args, &block)
+      .add_operation_properties_to_info(:name, :type)
+      .add_optional_operation_properties_to_info(:parallelism)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
 
   def group_by(*args, &block)
-    
-    # Init
-    group_args = Zillabyte::Harness::Helper.get_non_option_args(args)
-    options = Zillabyte::Harness::Helper.get_options(args)
-    h = Zillabyte::Harness::GroupBy.new(*group_args)
-
-    # Yield 
-    h.instance_eval(&block)
-    Zillabyte::Harness::Helper.check_name("group_by", h._name, @_app._names)
-
-    # Multiple emits?
-    if h._emits
-      Zillabyte::Harness::Helper.check_emits("group_by", h._emits, @_app._streams)
-    else
-      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-    end
-    
-    Zillabyte::Harness::Helper.check_group_by(h)
-    @_app._nodes << h
-    
-    # Handle incoming command..
-    if(@_app._options[:command] == :info)
-      
-      # Info.. 
-      node_hash = {
-        :name => h._name, 
-        :type => h._type, 
-        :group_by => h._group_by,
-        :config => options
-      }
-     if(h._parallelism)
-        node_hash["parallelism"] = h._parallelism
-      end
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-
-      # Execute..
-      pipe_name = @_app._options[:pipe]
-      c = Zillabyte::Harness::GroupByController.new(h, Zillabyte::Common::Progress.new)
-      c.run(pipe_name)
-      
-    end
-    
-    # Return the stream 
-    output_streams = []
-    h._emits.each do |stream|
-      output_streams << self.class.new(stream, @_app, h._name)
-    end
-    output_streams = output_streams[0] if output_streams.size == 1
-    output_streams
-    
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_multilang_operation("group_by", *args, &block)
+      .add_operation_properties_to_info(:name, :type, :group_by)
+      .add_optional_operation_properties_to_info(:parallelism)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
 
-
-  def join_with(rhs_stream, options = {}, &block)
-    
-    # Init
-    h = Zillabyte::Harness::Join.new(options)
-
-    # Yield 
-    throw "programmable joins are not supported at this time" unless block.nil?
-    # h.instance_eval(&block)
-    
-    # Sanity
-    Zillabyte::Harness::Helper.check_name("join", h._name, @_app._names)
-
-    # Build the node
-    if h._emits
-      Zillabyte::Harness::Helper.check_emits("join", h._emits, @_app._streams)
-    else
-      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-    end
-
-    Zillabyte::Harness::Helper.check_join(h)
-    @_app._nodes << h
-    
-    # Handle incoming command..
-    if(@_app._options[:command] == :info)
-      # Info.. 
-      node_hash = {
-        :name => h._name, 
-        :type => h._type, 
-        :lhs_fields => h._lhs_fields,
-        :rhs_fields => h._rhs_fields,
-        :join_type => h._join_type,
-        :config => options
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash1 = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name, "left" => 1}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash1, @_app._info_file)
-      arc_hash2 = {"name" => rhs_stream._name, "origin" => rhs_stream._previous_node_name, "dest" => h._name, "right" => 1}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash2, @_app._info_file)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      
-      # There's a problem, because joins should be not be executed in multilang
-      throw "unsupported: joins should be executed here"
-      
-    end
-    
-    # Return the stream 
-    output_stream = self.class.new(h._emits[0], @_app, h._name)
-    return output_stream
-    
+  def join_with(*args)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op = op.build_multilang_operation("join", self, *args)
+           .add_operation_properties_to_info(:name, :type, :lhs_fields, :rhs_fields, :join_type)
+    node = op.node
+    op.create_arc_info_from_stream(node._lhs_stream, :left)
+      .create_arc_info_from_stream(node._rhs_stream, :right)
+      .handle_operation
+      .get_output_streams 
   end
 
-  def call_component(comp_id = nil, options = {}, &block)
-    
-    h = Zillabyte::Harness::InjectedComponent.new()
-    h.instance_eval(&block) if block_given?
-    h._id ||= comp_id
-    h._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]  # We default to single-stream when called like this.. 
-    
-    Zillabyte::Harness::Helper.check_name("component", h._name, @_app._names)
-    Zillabyte::Harness::Helper.check_call_component(h)
-    Zillabyte::Harness::Helper.check_emits("component", h._emits, @_app._streams)
-    @_app._nodes << h
-
-    if(@_app._options[:command] == :info)
-      node_hash = {
-        :name => h._name, 
-        :type => h._type, 
-        :id => h._id, 
-        :output_format => h._output_format,
-        :config => options
-      }
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      h._consumes.each do |stream|
-        arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => h._name}
-        Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      end
+  def call_component(*args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op = op.build_multilang_operation("component", self, *args, &block)
+           .add_operation_properties_to_info(:name, :type, :id, :output_format)
+    node = op.node
+    node._consumes.each do |stream|
+      op = op.create_arc_info_from_stream(stream)
     end
-
-    # Return the stream 
-    output_streams = []
-    h._emits.each do |stream|
-      output_streams << self.class.new(stream, @_app, h._name)
-    end
-    output_streams = output_streams[0] if output_streams.size == 1
-    output_streams
+    op.handle_operation
+      .get_output_streams
   end
   alias_method :executes, :call_component
   alias_method :execute, :call_component
-  
-  
 
-  def sink(&block)
-    h = Zillabyte::Harness::Sink.new()
-    h.instance_eval(&block)
-    Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
-    @_app._nodes << h
-    if(@_app._options[:command] == :info)
-      node_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "scope" => h._scope}
-     
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-    end
+  def sink(*args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_multilang_operation("sink", *args, &block)
+      .add_operation_properties_to_info(:name, :type, :columns, :scope)
+      .add_optional_operation_properties_to_info(:relation)
+      .create_arc_info_from_stream(self)
+      .handle_operation
   end
   
   
   
   # Unique's a stream. Executed on backend
   def unique(*args)
-    
-    # INIT
-    group_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
-    options = Zillabyte::Harness::Helper.get_options(args)
-    
-    if(@_app._options[:command] == :info)
-
-      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      node_name = "unique_#{Zillabyte::Harness::Counter.get()}"
-      node_hash = {
-        :name => node_name,
-        :type => "unique",
-        :group_fields => group_fields,
-        :config => options
-      }
-      arc_hash = {
-        :name => @_name,
-        :origin => @_previous_node_name, 
-        :dest => node_name
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      return self.class.new(emits, @_app, node_name)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      throw "should not execute in multilang"
-    end
-    
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_jvm_operation("unique", *args)
+      .add_operation_properties_to_info(:name, :type)
+      .add_input_args_to_info_as(:group_fields)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
-  
-  
   # Counts fields. Executed on backend
   def count(*args)
-      
-    # Init 
-    group_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
-    options = Zillabyte::Harness::Helper.get_options(args)
-    
-    if(@_app._options[:command] == :info)
-
-      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      node_name = "count_#{Zillabyte::Harness::Counter.get()}"
-      node_hash = {
-        :name => node_name,
-        :type => "count",
-        :group_fields => group_fields,
-        :config => options
-      }
-      arc_hash = {
-        :name => @_name,
-        :origin => @_previous_node_name, 
-        :dest => node_name
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      return self.class.new(emits, @_app, node_name)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      throw "should not execute in multilang"
-    end
-    
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_jvm_operation("count", *args)
+      .add_operation_properties_to_info(:name, :type)
+      .add_input_args_to_info_as(:group_fields)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
-  
-  
-  
   # Removes fields from the stream. 
   def remove(*args)
-      
-    # Init 
-    remove_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
-    options = Zillabyte::Harness::Helper.get_options(args)
-    
-    if(@_app._options[:command] == :info)
-
-      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      node_name = "remove_#{Zillabyte::Harness::Counter.get()}"
-      node_hash = {
-        :name => node_name,
-        :type => "remove",
-        :remove => remove_fields,
-        :config => options
-      }
-      arc_hash = {
-        :name => @_name,
-        :origin => @_previous_node_name, 
-        :dest => node_name
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      return self.class.new(emits, @_app, node_name)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      throw "should not execute in multilang"
-    end
-    
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_jvm_operation("remove", *args)
+      .add_operation_properties_to_info(:name, :type)
+      .add_input_args_to_info_as(:remove)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
-  
-  
   # Retains fields from the stream and removes everything else
   def retain(*args)
-      
-    # Init 
-    retain_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
-    options = Zillabyte::Harness::Helper.get_options(args)
-    
-    if(@_app._options[:command] == :info)
-
-      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      node_name = "retain_#{Zillabyte::Harness::Counter.get()}"
-      node_hash = {
-        :name => node_name,
-        :type => "retain",
-        :retain => retain_fields,
-        :config => options
-      }
-      arc_hash = {
-        :name => @_name,
-        :origin => @_previous_node_name, 
-        :dest => node_name
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      return self.class.new(emits, @_app, node_name)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      throw "should not execute in multilang"
-    end
-    
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_jvm_operation("retain", *args)
+      .add_operation_properties_to_info(:name, :type)
+      .add_input_args_to_info_as(:retain)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
-  
-  
-  
-  
   # Groups together N arbitrary tuples (usefull for mini-batch processing)
-  def clump(options = {})
-      
-    if(@_app._options[:command] == :info)
-
-      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      node_name = "clump_#{Zillabyte::Harness::Counter.get()}"
-      node_hash = {
-        :name => node_name,
-        :type => "clump",
-        :config => options
-      }
-      arc_hash = {
-        :name => @_name,
-        :origin => @_previous_node_name, 
-        :dest => node_name
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      return self.class.new(emits, @_app, node_name)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      throw "should not execute in multilang"
-    end
-    
+  def clump(*args)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_jvm_operation("clump", *args)
+      .add_operation_properties_to_info(:name, :type)
+      .add_input_args_to_info_as(:clump)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
-  
-  
-  
-  
   # Renames fields..
-  def rename(rename_map)
-    
-    if(@_app._options[:command] == :info)
-
-      emits = ["stream_"+Zillabyte::Harness::Counter.get()]
-      node_name = "rename_#{Zillabyte::Harness::Counter.get()}"
-      node_hash = {
-        :name => node_name,
-        :type => "rename",
-        :rename => rename_map
-      }
-      arc_hash = {
-        :name => @_name,
-        :origin => @_previous_node_name, 
-        :dest => node_name
-      }
-      
-      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
-      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
-      return self.class.new(emits, @_app, node_name)
-      
-    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
-      throw "should not execute in multilang"
-    end
-    
+  def rename(*args)
+    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
+    op.build_jvm_operation("rename", *args)
+      .add_operation_properties_to_info(:name, :type)
+      .add_input_args_to_info_as(:rename, 0)
+      .create_arc_info_from_stream(self)
+      .handle_operation
+      .get_output_streams
   end
   
-
-
-  
-  
 end