zillabyte 0.1.43 → 0.1.44

data/ruby/lib/zillabyte/harness/helper.rb

@@ -384,7 +384,9 @@ class Zillabyte::Harness::Helper
   end
   
   def self.get_non_option_args(args) 
-    if args.last.is_a?(Hash)
+    if args.nil?
+      return []
+    elsif args.last.is_a?(Hash)
       return args[0..-2]
     else
       return args

data/ruby/lib/zillabyte/harness/injected_component.rb

@@ -1,31 +1,55 @@
 class Zillabyte::Harness::InjectedComponent
-  attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
+  attr_accessor :_app, :_node, :_id, :_input_stream_1
 
-  def initialize()
-    @_name = "component_"+Zillabyte::Harness::Counter.get()
-    @_type = "component"
-    @_consumes = []
-    @_output_format = :replace
-  end
+  class Node
+    attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
 
-  def name(v)
-    @_name = v
-  end
+    def initialize(input_stream_1)
+      @_name = "component_"+Zillabyte::Harness::Counter.get()
+      @_type = "component"
+      @_consumes = [input_stream_1]
+      @_output_format = :replace
+    end
 
-  def component_id(v)
-    @_id = v.to_s
-  end
+    def name(v)
+      @_name = v
+    end
+
+    def component_id(v)
+      @_id = v.to_s
+    end
   
-  def additional_inputs(*v)
-    @_consumes = *v
+    def additional_inputs(*v)
+      @_consumes.push(*v)
+    end
+
+    def outputs(*v)
+      @_emits = *v
+    end
+
+    def output_format(v)
+      @_output_format = v
+    end
   end
 
-  def outputs(*v)
-    @_emits = *v
+  def initialize(app, *args)
+    @_app = app
+    @_input_stream_1 = args[0]
+    @_id = args[1]
   end
 
-  def output_format(v)
-    @_output_format = v
+  def build_node(&block)
+    @_node = Node.new @_input_stream_1
+    @_node.instance_eval(&block) if block_given?
+    @_node._id ||= @_id
+    @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]  # We default to single-stream when called like this.. 
+
+    Zillabyte::Harness::Helper.check_name("component", @_node._name, @_app._names)
+    Zillabyte::Harness::Helper.check_call_component(@_node)
+    Zillabyte::Harness::Helper.check_emits("component", @_node._emits, @_app._streams)
   end
 
+  def run_operation
+    puts "Components can only be instantiated on our servers, sorry!"
+  end
 end

data/ruby/lib/zillabyte/harness/join.rb

@@ -1,39 +1,67 @@
 class Zillabyte::Harness::Join
-  attr_accessor :_name, :_type, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
-  #:_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
-
-  def initialize(options = {})
-    @_name = "join_"+Zillabyte::Harness::Counter.get()
-    @_type = 'join'
-    @_join_type = :inner
-
-    options.symbolize_keys!
-
-    emits = options[:emits]
-    if emits
-      if emits.is_a?(String)
-        @_emits = [options[:emits]]
-      else
-        @_emits = options[:emits]
+  attr_accessor :_app, :_node, :_args
+
+  class Node
+    attr_accessor :_name, :_type, :_lhs_stream, :_rhs_stream, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
+
+    def initialize(*args)
+      @_name = "join_"+Zillabyte::Harness::Counter.get()
+      @_type = 'join'
+      @_join_type = :inner
+      @_lhs_stream = args[0]
+      @_rhs_stream = args[1]
+      options = args[2]
+
+      options.symbolize_keys!
+
+      emits = options[:emits]
+      if emits
+        if emits.is_a?(String)
+          @_emits = [options[:emits]]
+        else
+          @_emits = options[:emits]
+        end
       end
-    end
     
-    # Figure out which fields to join on..
-    on = options[:on]
-    if on
-      if on.is_a?(Array) and on.size == 2
-        @_lhs_fields = on[0]
-        @_rhs_fields = on[1]
-      elsif (on.is_a?(String) or on.is_a?(Symbol))
-        @_lhs_fields = on
-        @_rhs_fields = on
+      # Figure out which fields to join on..
+      on = options[:on]
+      if on
+        if on.is_a?(Array) and on.size == 2
+          @_lhs_fields = on[0]
+          @_rhs_fields = on[1]
+        elsif (on.is_a?(String) or on.is_a?(Symbol))
+          @_lhs_fields = on
+          @_rhs_fields = on
+        end
       end
-    end
     
-    # Different kind of join types...
-    if options[:type]
-      @_join_type = options[:type].to_sym
+      # Different kind of join types...
+      if options[:type]
+        @_join_type = options[:type].to_sym
+      end
+    end  
+  end
+
+  def initialize(app, *args)
+    @_app = app
+    @_args = args
+  end
+
+  def build_node
+    @_node = Node.new(*@_args)
+    Zillabyte::Harness::Helper.check_name("join", @_node._name, @_app._names)
+
+    # Build the node
+    if @_node._emits
+      Zillabyte::Harness::Helper.check_emits("join", @_node._emits, @_app._streams)
+    else
+      @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
     end
-    
+
+    Zillabyte::Harness::Helper.check_join(@_node)
+  end
+
+  def run_operation
+    puts "Join operations can only be instantiated on our cluster, sorry!"
   end
 end

data/ruby/lib/zillabyte/harness/jvm_operation.rb

@@ -0,0 +1,28 @@
+class Zillabyte::Harness::JvmOperation
+  attr_accessor :_type, :_node, :_args
+
+  class Node
+    attr_accessor :_name, :_type, :_emits
+
+    def initialize(type)
+      @_name = "#{type}_"+Zillabyte::Harness::Counter.get()
+      @_type = type
+    end
+  
+  end
+
+  def initialize(type, *args)
+    @_type = type
+    @_args = args
+  end
+
+  def build_node
+    @_node = Node.new @_type
+    @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+    @_node
+  end
+
+  def run_operation
+    puts "#{@_type} operation can only be instantiated on our cluster, sorry!"
+  end
+end

data/ruby/lib/zillabyte/harness/operation_handler.rb

@@ -0,0 +1,109 @@
+class Zillabyte::Harness::OperationHandler
+  attr_accessor :_app, :_stream_class, :_operation, :_node_hash_elements, :_node_hash_pars, :_node_hash_optpars, :_arc_hash_streams
+
+  def initialize(app, stream_class)
+    @_app = app
+    @_stream_class = stream_class
+    @_node_hash_elements = {}
+    @_node_hash_pars = []
+    @_node_hash_optpars = []
+    @_arc_hash_streams = {}
+  end
+
+  def node
+    @_operation._node
+  end
+
+  def build_multilang_operation(type, *args, &block)
+    case type
+    when "source"
+      @_operation = Zillabyte::Harness::Source.new(@_app, *args)
+    when "each"
+      @_operation = Zillabyte::Harness::Each.new(@_app, *args)
+    when "filter"
+      @_operation = Zillabyte::Harness::Filter.new(@_app, *args)
+    when "group_by"
+      @_operation = Zillabyte::Harness::GroupBy.new(@_app, *args)
+    when "join"
+      @_operation = Zillabyte::Harness::Join.new(@_app, *args)
+    when "component"
+      @_operation = Zillabyte::Harness::InjectedComponent.new(@_app, *args)
+    when "sink"
+      @_operation = Zillabyte::Harness::Sink.new(@_app, *args)
+    when "component_source"
+      @_operation = Zillabyte::Harness::ComponentSource.new(@_app, *args)
+    when "component_sink"
+      @_operation = Zillabyte::Harness::ComponentSink.new(@_app, *args)
+    else
+      throw "Unknown type"
+    end
+    @_operation.build_node(&block)
+    self
+  end
+
+  def build_jvm_operation(type, *args)
+    @_operation = Zillabyte::Harness::JvmOperation.new(type, *args)
+    @_operation.build_node
+    self
+  end
+
+  def add_operation_properties_to_info(*args)
+    @_node_hash_pars.concat(args)
+    self
+  end
+  
+  def add_optional_operation_properties_to_info(*args)
+    @_node_hash_optpars.concat(args)
+    self
+  end
+
+  def add_input_args_to_info_as(key, index = nil)
+    if !index.nil?
+      @_node_hash_elements[key] = @_operation._args[index]
+    else
+      @_node_hash_elements[key] = @_operation._args
+    end
+    self
+  end
+
+  def create_arc_info_from_stream(stream, direction = nil)
+    @_arc_hash_streams[stream] = direction
+    self
+  end
+
+  def get_output_streams(&block)
+    output_streams = []
+    node._emits.each do |stream|
+      output_streams << @_stream_class.new(stream, @_app, node._name)
+    end
+    output_streams = output_streams[0] if output_streams.size == 1
+    output_streams
+  end
+
+  def handle_operation
+    @_app._nodes << node
+    if(@_app._options[:command] == :info)
+      node_hash = @_node_hash_elements
+      node_hash[:config] = @_operation._options if @_operation.respond_to? :_options # Always add options if it exists
+      # Add node parameters to node hash
+      @_node_hash_pars.each do |par|
+        node_hash[par] = node.instance_variable_get("@_#{par}")
+      end
+      # Add optional node parameters to node hash
+      @_node_hash_optpars.each do |par|
+        node_hash[par] = node.instance_variable_get("@_#{par}") if node.instance_variable_defined?("@_#{par}")
+      end
+      Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
+
+      # Create arc hashes from streams that feed into this node
+      @_arc_hash_streams.each do |stream, direction|
+        arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => node._name}
+        arc_hash[direction] = 1 if !direction.nil?
+        Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
+      end
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == node._name)
+      @_operation.run_operation
+    end
+    self
+  end
+end

data/ruby/lib/zillabyte/harness/sink.rb

@@ -1,26 +1,45 @@
 class Zillabyte::Harness::Sink
-  attr_accessor :_name, :_relation, :_type,  :_columns, :_scope
+  attr_accessor :_app, :_node, :_options
 
-  def initialize()
-    @_type = 'sink'
-    @_columns = []
-  end
+  class Node
+    attr_accessor :_name, :_relation, :_type,  :_columns, :_scope
+
+    def initialize()
+      @_type = 'sink'
+      @_columns = []
+    end
   
-  def name(v)
-    @_name = v
-  end
+    def name(v)
+      @_name = v
+    end
   
-  def relation(v)
-    @_relation = v
-  end
+    def relation(v)
+      @_relation = v
+    end
   
-  def column(cname, ctype)
-    col = {cname => ctype}
-    @_columns << col
-  end
+    def column(cname, ctype)
+      col = {cname => ctype}
+      @_columns << col
+    end
   
-  def scope(v)
-    @_scope = v
+    def scope(v)
+      @_scope = v
+    end
+  end
+
+  def initialize(app, *args)
+    @_app = app
+    @_options = Zillabyte::Harness::Helper.get_options(args)
+  end
+
+  def build_node(&block)
+    @_node = Node.new
+    @_node.instance_eval(&block)
+    Zillabyte::Harness::Helper.check_sink(@_node, @_app._nodes)
+    @_node._relation ||= @_node._name
+  end
+
+  def run_operation
+    puts "Sinks cannot be run."
   end
-  
 end

data/ruby/lib/zillabyte/harness/source.rb

@@ -1,44 +1,73 @@
 class Zillabyte::Harness::Source
-  
-  attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_end_cycle_policy, :_begin_cycle, :_next_tuple, :_prepare
+  attr_accessor :_app, :_node, :_relation
 
-  def initialize()
-    @_name = "source_"+Zillabyte::Harness::Counter.get()
-    @_type = 'source'
-    @_end_cycle_policy = :null_emit
-  end
+  class Node < Zillabyte::Harness::CommonNode 
+    attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple, :_prepare
 
-  def matches(v, options = {})
-    case v
-    when String
-      @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
-    when Array
-      @_matches = v
+    def initialize()
+      @_name = "source_"+Zillabyte::Harness::Counter.get()
+      @_type = 'source'
+      @_end_cycle_policy = :null_emit
     end
-  end
-  
-  def name(v)
-    @_name = v
-  end
 
-  def emits(*v)
-    @_emits = *v
-  end
+    def matches(v, options = {})
+      case v
+      when String
+        @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
+      when Array
+        @_matches = v
+      end
+    end
+ 
+    def parallelism(v)
+      throw "parallelism cannot be specified for sources"
+    end
+ 
+    def end_cycle_policy(v)
+      @_end_cycle_policy = v
+    end
 
-  def end_cycle_policy(v)
-    @_end_cycle_policy = v
-  end
+    def begin_cycle(&block)
+      @_begin_cycle = block
+    end
 
-  def begin_cycle(&block)
-    @_begin_cycle = block
+    def next_tuple(&block)
+      @_next_tuple = block
+    end
+  
+    def prepare(&block)
+      @_prepare = block
+    end
   end
 
-  def next_tuple(&block)
-    @_next_tuple = block
+  def initialize(app, *args)
+    @_app = app
+    @_args = args
   end
-  
-  def prepare(&block)
-    @_prepare = block
+
+  def build_node(&block)
+    @_node = Node.new
+    # Are we given a block?
+    if block_given?
+      @_node.instance_eval(&block)
+      Zillabyte::Harness::Helper.check_name("source", @_node._name, @_app._names)
+      Zillabyte::Harness::Helper.check_custom_source(@_node)
+      if @_node._emits
+        Zillabyte::Harness::Helper.check_emits("source", @_node._emits, @_app._streams)
+      else
+        @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      end
+    # No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
+    else
+      @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      Zillabyte::Harness::Helper.check_source_args(@_args)
+      @_node.matches(@_args[0])
+    end
   end
 
+  def run_operation
+    pipe_name = @_app._options[:pipe]
+    c = Zillabyte::Harness::SourceController.new(@_node, progress = Zillabyte::Common::Progress.new)
+    c.run(pipe_name)
+  end
 end