zillabyte 0.9.9 → 0.9.10

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 17fe4f4d78abd8577811505fc480db3714eaafa5
-  data.tar.gz: 46dba9537746b0dd6f7c41825a00b3deb91d269b
-SHA512:
-  metadata.gz: a926a7beeeb77de3ac37838af7fedeac12802b958d02ed453b75b9d806b74966a27831564e752bf4bd12a3c0b7100ef6cabbe67c749473f192558b1809e8267f
-  data.tar.gz: 07b329196b76990a06315fa7fe36df5fb7a63b2f696d7c1715522eebbcbc808a80c3d04badbcc797ce616e82652f5e4b6ad3536114101bb6a5708a3bce03aec8
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NGUxNzIxMjQyMmFmY2Y1YjNhYTg2YWJmNjcwN2U5MTIwYzM3MjJhMw==
+  data.tar.gz: !binary |-
+    ZTkwYTI4NDcyMTJjNDQxMmM5Y2UzODM5ZmZiYzFmNDAxZDg0NGY4Mg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YTM3YWE0MWNhNzI5ZDkyY2Y5YTI3ODk1M2MzZmE5MTY0OWE1ZDU5ODgwOWI0
+    MDhiMmY5OTNmMWJlNzQwN2RiZGM4YmE0NTEwMzdmNTBjNTRhYWM4MDA3MmYx
+    ZmY0ODU4NjBhOTEyNWJlY2E4MzI5ZTI3MzM4YTkxMTllYmY4YjM=
+  data.tar.gz: !binary |-
+    NDQzMjQ4OTk0YTVkYzVhZGU2NDk5YzM1ZWE5ZWJiN2YxMGZiOWQ4NjRkNDVl
+    NTNjMjIwMjhkMTdlYTdkYmY3NDQzMzQwYTczZDI3Njc0YThhNjkzYzE3MTA5
+    NzRkZDY0YzY0YjJiNjU5ZjNlYWI3NGUzNDUwYTNiMzMxYWRkYTY=

data/ruby/lib/zillabyte/harness/app.rb

@@ -31,20 +31,11 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
   end
   
   
-  def sink_to_redshift(*args, &block)
-    _source_common(Zillabyte::Harness::Stream, *args, &block)
-  end
-  
-  
   def source_from_kinesis(*args, &block)
     _source_common(Zillabyte::Harness::Stream, "kinesis_demo_source")
   end
   
   
-  def sink_to_kinesis(*args, &block)
-    _source_common(Zillabyte::Harness::Stream, "kinesis_demo_sink")
-  end
-  
   
   
   def call_component(*args, &block)

data/ruby/lib/zillabyte/harness/component.rb

@@ -31,4 +31,6 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
       .get_output_streams
     return Zillabyte::Harness::StreamBuilder.new(stream)
   end
+  alias_method :input, :inputs
+  
 end

data/ruby/lib/zillabyte/harness/component_input.rb

@@ -17,6 +17,8 @@ class Zillabyte::Harness::ComponentInput
     def field(name, type)
       @_fields << {name => type}
     end
+    alias_method :column, :field
+    
   end
 
   def initialize(app, *args)

data/ruby/lib/zillabyte/harness/component_output.rb

@@ -21,6 +21,7 @@ class Zillabyte::Harness::ComponentOutput
       col = {cname => ctype}
       @_columns << col
     end
+    alias_method :column, :field
   
     def scope(v)
       @_scope = v

data/ruby/lib/zillabyte/harness/component_stream.rb

@@ -10,5 +10,6 @@ class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
       .create_arc_info_from_stream(self)
       .handle_operation
   end
+  alias_method :output, :outputs
 
 end

data/ruby/lib/zillabyte/harness/each.rb

@@ -54,10 +54,11 @@ class Zillabyte::Harness::Each
   class Node < Zillabyte::Harness::CommonNode
     attr_accessor :_output_format, :_execute
 
-    def initialize(*args)
-      @_name = args[0] || "each_"+Zillabyte::Harness::Counter.get()
+    def initialize(v, options = {})
+      @_name = v[0] || "each_"+Zillabyte::Harness::Counter.get()
       @_type = "each"
       @_output_format = :replace
+      self._emits = options[:emits]
     end
   
     def output_format(v)
@@ -71,11 +72,11 @@ class Zillabyte::Harness::Each
 
   def initialize(app, *args)
     @_app = app
-    @_vector_options, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
   end
 
   def build_node(&block)
-    @_node = Node.new(*@_vector_options)
+    @_node = Node.new(@_args, @_options)
     # Does the block take 0 arguments? If so it's not just an execute block.
     if(block.arity == 0)
       @_node.instance_eval(&block)
@@ -87,7 +88,7 @@ class Zillabyte::Harness::Each
       end
     # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
     else
-      @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+      @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]
       @_node._execute = block
     end
 

data/ruby/lib/zillabyte/harness/filter.rb

@@ -39,7 +39,7 @@ class Zillabyte::Harness::Filter
   class Node < Zillabyte::Harness::CommonNode
     attr_accessor :_keep
 
-    def initialize()
+    def initialize(args, options = {})
       @_name = "filter_"+Zillabyte::Harness::Counter.get()
       @_type = 'filter'
     end
@@ -51,11 +51,11 @@ class Zillabyte::Harness::Filter
 
   def initialize(app, *args)
     @_app = app
-    @_vector_options, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
   end
 
   def build_node(&block)
-    @_node = Node.new
+    @_node = Node.new(@_args, @_options)
     # Does the block take 0 arguments? If so it's not just an execute block.
     if(block.arity == 0)
       @_node.instance_eval(&block)

data/ruby/lib/zillabyte/harness/helper.rb

@@ -408,12 +408,12 @@ class Zillabyte::Harness::Helper
           Zillabyte::Harness::Helper.print_error(msg)
         end
       end
-      if node._name == previous_node_name
-        if node._emits.size < 2
-          msg = "#{ee}The preceding operation does not emit multiple streams. Please make sure that it emits one stream for the loop back and another stream for downstream operations. #{pp}"
-          Zillabyte::Harness::Helper.print_error(msg)
-        end
-      end
+      # if node._name == previous_node_name
+      #   if node._emits.size < 2
+      #     msg = "#{ee}The preceding operation does not emit multiple streams. Please make sure that it emits one stream for the loop back and another stream for downstream operations. #{pp}"
+      #     Zillabyte::Harness::Helper.print_error(msg)
+      #   end
+      # end
     end
     if !found
       msg = "#{ee}The specified loop-back node \"#{node_name}\" was not found in the operations preceding it. #{pp}"

data/ruby/lib/zillabyte/harness/injected_component.rb

@@ -4,11 +4,12 @@ class Zillabyte::Harness::InjectedComponent
   class Node
     attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
 
-    def initialize(input_stream_1)
+    def initialize(input_stream_1, options = {})
       @_name = "component_"+Zillabyte::Harness::Counter.get()
       @_type = "component"
       @_consumes = [input_stream_1]
       @_output_format = :replace
+      @_emits ||= options[:emits]
     end
 
     def name(v)
@@ -46,7 +47,7 @@ class Zillabyte::Harness::InjectedComponent
   end
 
   def build_node(&block)
-    @_node = Node.new @_input_stream_1
+    @_node = Node.new @_input_stream_1, @_options
     @_node.instance_eval(&block) if block_given?
     @_node._id ||= @_id
     @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]  # We default to single-stream when called like this.. 

data/ruby/lib/zillabyte/harness/jvm_operation.rb

@@ -1,23 +1,24 @@
 class Zillabyte::Harness::JvmOperation
-  attr_accessor :_type, :_node, :_args
+  attr_accessor :_type, :_node, :_args, :_options
 
   class Node
     attr_accessor :_name, :_type, :_emits
 
-    def initialize(type)
-      @_name = "#{type}_"+Zillabyte::Harness::Counter.get()
+    def initialize(type, options = {})
+      @_name = options[:name] || "#{type}_"+Zillabyte::Harness::Counter.get()
       @_type = type
+      @_options = options
     end
   
   end
 
   def initialize(type, *args)
     @_type = type
-    @_args = args
+    @_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
   end
 
   def build_node
-    @_node = Node.new @_type
+    @_node = Node.new @_type, @_options
     @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
     @_node
   end

data/ruby/lib/zillabyte/harness/operation_handler.rb

@@ -80,7 +80,7 @@ class Zillabyte::Harness::OperationHandler
     output_streams
   end
 
-  def handle_operation
+  def handle_operation()
     @_app._nodes << node
     if(@_app._options[:command] == :info)
       node_hash = @_node_hash_elements
@@ -106,4 +106,16 @@ class Zillabyte::Harness::OperationHandler
     end
     self
   end
+  
+  
+  def self.build(stream, operaiton_name, *args)
+    op = Zillabyte::Harness::OperationHandler.new(stream._app, stream.class)
+    op.build_jvm_operation(operaiton_name, *args)
+      .add_operation_properties_to_info(:name, :type)
+      .create_arc_info_from_stream(stream)
+      .handle_operation
+      .get_output_streams
+  end
+  
+  
 end

data/ruby/lib/zillabyte/harness/source.rb

@@ -61,7 +61,7 @@ class Zillabyte::Harness::Source
   class Node < Zillabyte::Harness::CommonNode 
     attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
 
-    def initialize(options = {})
+    def initialize(v, options = {})
       @_name = options[:name] || "source_"+Zillabyte::Harness::Counter.get()
       @_type = 'source'
       @_end_cycle_policy = :null_emit
@@ -101,7 +101,7 @@ class Zillabyte::Harness::Source
   end
 
   def build_node(&block)
-    @_node = Node.new(@_options)
+    @_node = Node.new(@_args, @_options)
     # Are we given a block?
     if block_given?
       @_node.instance_eval(&block)

data/ruby/lib/zillabyte/harness/stream.rb

@@ -13,7 +13,6 @@ class Zillabyte::Harness::Stream
     @_name
   end
 
-
   def each(*args, &block)
     op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
     op.build_multilang_operation("each", *args, &block)
@@ -89,7 +88,9 @@ class Zillabyte::Harness::Stream
     Zillabyte::Harness::Helper.check_loop_back(self, loop_back_node, max_iterations, @_app._nodes)
     arc_hash = {"name" => (@_options[:name] || self._name), "origin" => self._previous_node_name, "dest" => loop_back_node, "loop_back" => 1}
     arc_hash["max_iterations"] = max_iterations if max_iterations
-    Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
+    if(@_app._options[:command] == :info)
+      Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
+    end
   end
 
   def sink(*args, &block)
@@ -102,6 +103,18 @@ class Zillabyte::Harness::Stream
   end
   
   
+  def sink_to_redshift(*args, &block)
+    sink(*args, &block)
+  end
+  
+  
+  
+  def sink_to_kinesis(*args, &block)
+    sink(*args, &block)
+  end
+  
+  
+  
   
   # Unique's a stream. Executed on backend
   def unique(*args)
@@ -114,6 +127,26 @@ class Zillabyte::Harness::Stream
       .get_output_streams
   end
   
+  
+  
+  def route_by(*fields)
+    options = {}
+    options[:fields] = fields
+    Zillabyte::Harness::OperationHandler.build(self, "route_by", options)
+  end
+
+  def rate_limit(rate = 1, options = {})
+    options[:rate] = rate 
+    Zillabyte::Harness::OperationHandler.build(self, "rate_limit", options)
+  end
+  
+  def fetch_url(*args)
+    args, options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
+    self.call_component(:fetch_url, options)
+  end
+  
+  
+  
   # Counts fields. Executed on backend
   def count(*args)
     op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
@@ -159,14 +192,10 @@ class Zillabyte::Harness::Stream
   end
   
   # Renames fields..
-  def rename(*args)
-    op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
-    op.build_jvm_operation("rename", *args)
-      .add_operation_properties_to_info(:name, :type)
-      .add_input_args_to_info_as(:rename, 0)
-      .create_arc_info_from_stream(self)
-      .handle_operation
-      .get_output_streams
+  def rename(rename_map = {})
+    options = {}
+    options[:rename] = rename_map
+    Zillabyte::Harness::OperationHandler.build(self, "rename", options)
   end
   
 end

data/ruby/lib/zillabyte/harness/stream_builder.rb

@@ -11,7 +11,9 @@ class Zillabyte::Harness::StreamBuilder < Array
     end
   end
   
+  
   # Override these methods explicitly because they are provided by 'Array'
+  alias_method :super_each, :each
   def each(*args, &block)
     _handle_function(:each, *args, &block)
   end
@@ -42,7 +44,18 @@ class Zillabyte::Harness::StreamBuilder < Array
     if self.size == 0
       raise "The previous operation did not emit a split stream"
     else
-      return Zillabyte::Harness::StreamBuilder.new(super)
+      if v.is_a?(String)
+        # String based
+        super_each do |stream|
+          if stream._name == v
+            return Zillabyte::Harness::StreamBuilder.new(stream)
+          end
+        end
+        raise "Could not find a stream with name '#{v}'"
+      else
+        # Intger based
+        return Zillabyte::Harness::StreamBuilder.new(super)
+      end
     end
   end
 

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.9.9" unless defined?(VERSION)
+  VERSION = "0.9.10" unless defined?(VERSION)
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.9.9
+  version: 0.9.10
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-03 00:00:00.000000000 Z
+date: 2014-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: zillabyte-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.9
+        version: 0.9.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.9
+        version: 0.9.10
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -45,9 +45,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ruby/lib/zillabyte.rb
 - ruby/lib/zillabyte/common/progress.rb
-- ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/harness/app.rb
 - ruby/lib/zillabyte/harness/base.rb
 - ruby/lib/zillabyte/harness/clump.rb
@@ -71,7 +69,9 @@ files:
 - ruby/lib/zillabyte/harness/stream.rb
 - ruby/lib/zillabyte/harness/stream_builder.rb
 - ruby/lib/zillabyte/harness/tuple.rb
+- ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/version.rb
+- ruby/lib/zillabyte.rb
 homepage: http://www.zillabyte.com
 licenses:
 - MIT
@@ -82,17 +82,17 @@ require_paths:
 - ruby/lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.6
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte Gem