zillabyte 0.9.7 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8974f32fc265b24fccba0241176b2504994bc752
-  data.tar.gz: d8180962b759df939f680b9066bdd76a65d46314
+  metadata.gz: 17fe4f4d78abd8577811505fc480db3714eaafa5
+  data.tar.gz: 46dba9537746b0dd6f7c41825a00b3deb91d269b
 SHA512:
-  metadata.gz: da0718446a35dedbfe9d9d4a594bf6d2f5351a0f939f300473cdfc3dc43ab5560cf0b24ed45d1b8e608c3ccef5e93cf3bc5390e0d9178b8e08d578300228b156
-  data.tar.gz: 4611fcb7c8c98e60cf01c14ca7ce61bf9d8858c45ad92dd1f50b99aed145090d96c700ed4fd85179b0a89ab16caeff32cd506e1a2da507e5557343eb1484aa5e
+  metadata.gz: a926a7beeeb77de3ac37838af7fedeac12802b958d02ed453b75b9d806b74966a27831564e752bf4bd12a3c0b7100ef6cabbe67c749473f192558b1809e8267f
+  data.tar.gz: 07b329196b76990a06315fa7fe36df5fb7a63b2f696d7c1715522eebbcbc808a80c3d04badbcc797ce616e82652f5e4b6ad3536114101bb6a5708a3bce03aec8

data/ruby/lib/zillabyte/harness/app.rb

@@ -20,13 +20,43 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
     h
   end
 
+
   def source(*args, &block)
-    op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::Stream)
-    stream = op.build_multilang_operation("source", *args, &block)
-      .add_operation_properties_to_info(:name, :type)
-      .add_optional_operation_properties_to_info(:relation, :matches, :end_cycle_policy)
-      .handle_operation
-      .get_output_streams
-    return Zillabyte::Harness::StreamBuilder.new(stream)
+    _source_common(Zillabyte::Harness::Stream, *args, &block)
   end 
+  
+  
+  def source_from_redshift(*args, &block)
+    _source_common(Zillabyte::Harness::Stream, *args, &block)
+  end
+  
+  
+  def sink_to_redshift(*args, &block)
+    _source_common(Zillabyte::Harness::Stream, *args, &block)
+  end
+  
+  
+  def source_from_kinesis(*args, &block)
+    _source_common(Zillabyte::Harness::Stream, "kinesis_demo_source")
+  end
+  
+  
+  def sink_to_kinesis(*args, &block)
+    _source_common(Zillabyte::Harness::Stream, "kinesis_demo_sink")
+  end
+  
+  
+  
+  def call_component(*args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::Stream)
+    op = op.build_multilang_operation("component", self, *args, &block)
+      .add_operation_properties_to_info(:name, :type, :id, :output_format)
+    node = op.node
+    streams = op.handle_operation.get_output_streams
+    return Zillabyte::Harness::StreamBuilder.new(streams)
+  end
+  alias_method :executes, :call_component
+  alias_method :execute, :call_component
+  
+  
 end

data/ruby/lib/zillabyte/harness/base.rb

@@ -8,4 +8,32 @@ class Zillabyte::Harness::Base
     @_name = name
   end
 
+  def params
+    # Params are set by the caller (JVM) in the form of env variables.  Why env variables? Because we want to 
+    # expose the params as soon as possible, in case the user makes static-decisions (i.e. configuring the number
+    # of operaitons in a chain, etc).     
+    if @_params.nil?
+      if ENV['ZILLABYTE_PARAMS']
+        @_params = JSON.parse(ENV['ZILLABYTE_PARAMS'])
+      else
+        @_params = {}
+      end
+    end
+    return @_params
+  end
+
+
+  protected
+
+  def _source_common(stream_class, *args, &block)
+    op = Zillabyte::Harness::OperationHandler.new(self, stream_class)
+    stream = op.build_multilang_operation("source", *args, &block)
+      .add_operation_properties_to_info(:name, :type)
+      .add_optional_operation_properties_to_info(:relation, :matches, :end_cycle_policy)
+      .handle_operation
+      .get_output_streams
+    return Zillabyte::Harness::StreamBuilder.new(stream)
+  end 
+  
+
 end

data/ruby/lib/zillabyte/harness/component.rb

@@ -18,6 +18,10 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
     end
     h
   end
+  
+  def source(*args, &block)
+    _source_common(Zillabyte::Harness::ComponentStream, *args, &block)
+  end 
 
   def inputs(*args, &block)
     op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::ComponentStream)

data/ruby/lib/zillabyte/harness/injected_component.rb

@@ -1,5 +1,5 @@
 class Zillabyte::Harness::InjectedComponent
-  attr_accessor :_app, :_node, :_id, :_input_stream_1
+  attr_accessor :_app, :_node, :_id, :_input_stream_1, :_options
 
   class Node
     attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
@@ -20,7 +20,13 @@ class Zillabyte::Harness::InjectedComponent
     end
   
     def additional_inputs(*v)
-      @_consumes.push(*v)
+      v.each do |vv|
+        if vv.is_a?(Zillabyte::Harness::StreamBuilder)
+          @_consumes.push(vv._last_stream)
+        else
+          @_consumes.push(vv)
+        end
+      end
     end
 
     def outputs(*v)
@@ -34,8 +40,9 @@ class Zillabyte::Harness::InjectedComponent
 
   def initialize(app, *args)
     @_app = app
-    @_input_stream_1 = args[0]
-    @_id = args[1]
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
+    @_input_stream_1 = @_args[0]
+    @_id = @_args[1]
   end
 
   def build_node(&block)

data/ruby/lib/zillabyte/harness/sink.rb

@@ -27,9 +27,10 @@ class Zillabyte::Harness::Sink
   class Node
     attr_accessor :_name, :_relation, :_type,  :_columns, :_scope
 
-    def initialize()
+    def initialize(args, options)
       @_type = 'sink'
-      @_columns = []
+      @_columns = options[:columns] || []
+      @_name = options[:name] || args.first
     end
   
     def name(v)
@@ -52,12 +53,12 @@ class Zillabyte::Harness::Sink
 
   def initialize(app, *args)
     @_app = app
-    @_vector_options, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
   end
 
   def build_node(&block)
-    @_node = Node.new
-    @_node.instance_eval(&block)
+    @_node = Node.new(@_args, @_options)
+    @_node.instance_eval(&block) if block_given?
     Zillabyte::Harness::Helper.check_sink(@_node, @_app._nodes)
     @_node._relation ||= @_node._name
   end

data/ruby/lib/zillabyte/harness/source.rb

@@ -56,15 +56,16 @@
 #   end
 # end
 class Zillabyte::Harness::Source
-  attr_accessor :_app, :_node, :_relation
+  attr_accessor :_app, :_node, :_relation, :_options
 
   class Node < Zillabyte::Harness::CommonNode 
     attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
 
-    def initialize()
-      @_name = "source_"+Zillabyte::Harness::Counter.get()
+    def initialize(options = {})
+      @_name = options[:name] || "source_"+Zillabyte::Harness::Counter.get()
       @_type = 'source'
       @_end_cycle_policy = :null_emit
+      @_options = options
     end
 
     def matches(v, options = {})
@@ -96,11 +97,11 @@ class Zillabyte::Harness::Source
 
   def initialize(app, *args)
     @_app = app
-    @_args = args
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
   end
 
   def build_node(&block)
-    @_node = Node.new
+    @_node = Node.new(@_options)
     # Are we given a block?
     if block_given?
       @_node.instance_eval(&block)

data/ruby/lib/zillabyte/harness/stream.rb

@@ -13,6 +13,7 @@ class Zillabyte::Harness::Stream
     @_name
   end
 
+
   def each(*args, &block)
     op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
     op.build_multilang_operation("each", *args, &block)

data/ruby/lib/zillabyte/harness/stream_builder.rb

@@ -1,7 +1,14 @@
 class Zillabyte::Harness::StreamBuilder < Array
 
   def initialize(first_stream)
-    @_streams = [first_stream]
+    if first_stream.is_a?(Array)
+      @_streams = nil
+      first_stream.each_with_index do |s, i|
+        self[i] = s
+      end
+    else
+      @_streams = [first_stream]
+    end
   end
   
   # Override these methods explicitly because they are provided by 'Array'
@@ -13,10 +20,22 @@ class Zillabyte::Harness::StreamBuilder < Array
     _handle_function(:group_by, *args, &block)
   end
   
+  def count(*args, &block)
+    _handle_function(:count, *args, &block)
+  end
+  
   def map(*args, &block)
     raise "Unsupported"
   end
   
+
+    
+  # 'clones' the stream. 
+  # TODO: docs
+  def tee
+    return Zillabyte::Harness::StreamBuilder.new(@_streams.last)
+  end
+  
   
   
   def [](v)

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.9.7" unless defined?(VERSION)
+  VERSION = "0.9.9" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.9.7
+  version: 0.9.9
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-30 00:00:00.000000000 Z
+date: 2014-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.7
+        version: 0.9.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.7
+        version: 0.9.9
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com