zillabyte 0.0.21 → 0.0.22

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDcwNWQ3NThlM2RmYTg4Yzc3OGM3NDhlMzQ4NWEzNzZmM2E2NDFhZQ==
+    ZGFiZGYyNGE4YTQzNzVhZWM1NTI1OTUzZDk4ZWU2ZGRmYWQxZmJhNw==
   data.tar.gz: !binary |-
-    MDEzMGEzNmI3OTNhMzNjNTIwOWE4MzY5NmU1ZGQ1NmNlN2NjMDIzNg==
+    N2QzYWNhNmY4OTUyZWIwODU3NDBiYzA1ZDgyOGIxZTZlNmY2NTlmMQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OTZiYzU5NzIyNTc3MjNlMGZkNmM5NDgxN2Q4ODkxYmZkNjVhZTkzZmUwNDI5
-    MTU4NGZmMjZhYmJhNDBlNmQ2MDI5OGFlMDkwNzU0N2Y1ZDdiYzI3MGNkNWM3
-    YTlmOTMyMGNhM2Q1YWFlMGNiMWJlNWY3MTNlNjBjMmE1ZDc4MDU=
+    MTU1NDE2ZGNiODMwYjM2NzY4NDYyMzc1OTY4ZTA2NTg2MjIzNDNiNjk2MzY4
+    YjJkODU5ZTUwY2YzZTFmZjA3MDI1MGI4YmVhZmRhMDJkZWExNzBhMzhhNTg4
+    NmExNGViNTZkYWQxM2M2MDRiYzFjNjczNWQ2NmE5YjIzYmQ3YjU=
   data.tar.gz: !binary |-
-    MTg1OWEyMWQ4OGYwMmMyOGZmZTIxMjYyYzNkOTc5ZDFmMDNmNDA0NGVjM2Jj
-    NWRmMjQ0NDFiOWE5MTBjZDNiY2NiYWVjMDBkYzRlOGVhODY4MmY3OWE0NDcw
-    ZDNiNDY2NzViZjQxNzBiNWI5ZmY5OTMwZGExOGM5ZDg5OWRjYWU=
+    NGYwODkxNGM3OTkxNGM4NTQwY2E4MzdiMjBjYzA0ZGM4YjA4ZTllNGIwYmIy
+    NmY4ZGEzNmYyZGZiN2FkNDU2YWQ0OTlmZWFjNzIyMDM1NDdiYTE2Mzk4M2M4
+    MjA0MTczNzEwY2Q0N2FlYTdjYjE2ZjllZWYwNWFmZGFjODg0MDc=

data/ruby/lib/zillabyte/harness/filter.rb

@@ -0,0 +1,25 @@
+class Zillabyte::Harness::Filter
+  attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_keep
+
+  def initialize()
+    @_name = "filter_"+Zillabyte::Harness::Counter.get()
+    @_type = 'filter'
+  end
+  
+  def name(v)
+    @_name = v
+  end
+  
+  def emits(*v)
+    @_emits = *v
+  end
+
+  def prepare(&block)
+    @_prepare = block
+  end
+
+  def keep(&block)
+    @_keep = block
+  end
+
+end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -120,6 +120,16 @@ class Zillabyte::Harness::Helper
     end
   end
 
+  def self.check_filter(filter)
+    ee = "Error in \"filter\": \n\t "
+    pp = @@_print_check_filter
+
+    if !filter._keep
+      msg = "#{ee}A \"filter\" must contain a \"condition\" block. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
+    end 
+  end
+
   def self.check_each(each)
     ee = "Error in \"each\": \n\t "
     pp = @@_print_check_each
@@ -232,6 +242,18 @@ OUTPUT
       * :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
 OUTPUT
 
+  @@_print_check_filter = <<-OUTPUT
+\n\n"Filter" Syntax:
+  Simplified syntax:
+    stream.filter { |tuple| |=condition=| }
+
+    * Outputs a given tuple from the stream only if condition is true
+  Custom filter:
+    stream.filter do |tuple|
+       |=condition=|
+    end
+OUTPUT
+
   @@_print_check_each = <<-OUTPUT
 \n\n"Each" Syntax:
   Simplified syntax:

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -283,6 +283,35 @@ module Zillabyte
       end
     end
 
+    class Filter
+      include Storm::Protocol
+
+      def prepare(conf, context); end
+
+      def keep(tuple); end
+
+      def run(pipe_name)
+        Storm::Protocol.mode = 'filter'
+        Storm::Protocol.pipe_name = pipe_name
+        setup_pipes
+        prepare(*handshake)
+
+        while true
+          begin
+            t = Tuple.from_hash(read_command)
+            next if(!t)
+            keep t
+          rescue SignalException => e
+            raise
+          rescue Exception => e
+            fail 'Exception in filter: ' + e.message + ' - ' + e.backtrace.join('\n')
+            # We may recover from this, but let JVM decide
+          end
+          done
+        end
+      end
+    end
+
     class GroupBy
       include Storm::Protocol
 
@@ -358,6 +387,27 @@ module Zillabyte
 
     end
 
+    class FilterController < Storm::Filter
+
+      def initialize(harness, progress)
+        @harness = harness
+        @progress = progress
+        Storm::Protocol.emits = harness._emits
+      end
+
+      def keep(*args)
+        if self.instance_exec *args, &@harness._keep
+            emit args[0]
+        end
+      end
+
+      def prepare(*args)
+        self.instance_exec(*args, &@harness._prepare) if @harness._prepare
+      end
+
+
+    end
+
     class GroupByController < Storm::GroupBy
 
       def initialize(harness, progress)

data/ruby/lib/zillabyte/harness/sink.rb

@@ -1,5 +1,5 @@
 class Zillabyte::Harness::Sink
-  attr_accessor :_name, :_relation, :_type, :_columns, :_consumes
+  attr_accessor :_name, :_relation, :_type, :_columns, :_consumes, :_scope
 
   def initialize()
     @_type = 'sink'
@@ -24,4 +24,8 @@ class Zillabyte::Harness::Sink
     @_columns << col
   end
   
+  def scope(v)
+    @_scope = v
+  end
+  
 end

data/ruby/lib/zillabyte/harness/stream.rb

@@ -49,6 +49,29 @@ class Zillabyte::Harness::Stream
     output_streams
   end
 
+  def filter(&block)
+    h = Zillabyte::Harness::Filter.new()
+    h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
+    h._keep = block
+
+    @_app._nodes << h
+    if(@_app._options[:command] == :info)
+      info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
+      info_hash["emits"] = h._emits
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      pipe_name = @_app._options[:pipe]
+      c = Zillabyte::Harness::FilterController.new(h, progress = Zillabyte::Common::Progress.new)
+      c.run(pipe_name)
+    end
+
+    output_streams = []
+    h._emits.each do |stream|
+      output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
+    end
+    output_streams = output_streams[0] if output_streams.size == 1
+    output_streams
+  end
 
   def group_by(*args, &block)
     
@@ -164,7 +187,7 @@ class Zillabyte::Harness::Stream
     Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
     @_app._nodes << h
     if(@_app._options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name}
+      info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name, "scope" => h._scope}
       Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
     end
   end

data/ruby/lib/zillabyte/harness/tuple.rb

@@ -6,29 +6,42 @@ class Tuple
   end
   
   def [](name)
+    name = name.to_s unless name.is_a?(String)
     maybe_build_alias_hash()
-    key = (@_alias_hash[name] || name).to_s
+    key = (@_alias_hash[name] || name)
     if values.has_key?( key )
       return values[key]
     else
-      raise ArgumentError, "The requested tuple does not contain the key: \"#{name}\". Current Tuple: #{values.to_s}"
+      raise ArgumentError, "The requested tuple does not contain the key: \"#{name}\". Current tuple contains keys: #{values.keys.to_s} with aliases #{@_alias_hash.to_s}"
     end
   end
   
   def values
     @hash["tuple"]
   end
+
+  def aliased_hash
+    out = {}
+    column_aliases.each do |col_hash|
+        concrete = col_hash["concrete_name"]
+        aliased = col_hash["alias"]
+        out[aliased] = values[concrete]
+    end
+    return out
+  end
+
+  def keys
+      @hash.keys
+  end
   
   def meta
     @hash["meta"]
   end
   
-  
   def to_s
     values.to_s
   end
   
-  
   def column_aliases
     @hash['column_aliases'] || []
   end

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.0.21" unless defined?(VERSION)
+  VERSION = "0.0.22" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.0.21
+  version: 0.0.22
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-05 00:00:00.000000000 Z
+date: 2014-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.21
+        version: 0.0.22
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.21
+        version: 0.0.22
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -63,6 +63,7 @@ files:
 - ruby/lib/zillabyte/harness/app.rb
 - ruby/lib/zillabyte/harness/counter.rb
 - ruby/lib/zillabyte/harness/each.rb
+- ruby/lib/zillabyte/harness/filter.rb
 - ruby/lib/zillabyte/harness/group_by.rb
 - ruby/lib/zillabyte/harness/helper.rb
 - ruby/lib/zillabyte/harness/join.rb
@@ -94,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.0.7
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte Gem