zillabyte 0.0.20 → 0.0.21

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MWQyNzZkNTM0MTcyZGJjYTE2NGFkNzgyZGNlYTE2NWE2Y2EyY2JkOQ==
+    NDcwNWQ3NThlM2RmYTg4Yzc3OGM3NDhlMzQ4NWEzNzZmM2E2NDFhZQ==
   data.tar.gz: !binary |-
-    MTViMmZiNjNkNGRiMDdkMTI2ZTdhZWJlOTJjNjBiY2Y2ODk2N2Y3Nw==
+    MDEzMGEzNmI3OTNhMzNjNTIwOWE4MzY5NmU1ZGQ1NmNlN2NjMDIzNg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZmZkODcyZGJhNTA4NTYyNzE2ZWUyNzdiNzU1N2QwODg4Nzc1NzA1MGNkNTA4
-    OTM5OWJlMTBiMDBkMWUzZGUzNzljNTczNmUwZDE1ODZiNjI1M2YwZDRjMWYy
-    YTRjZmY3OTBiNTUyMTI5NDc2MjIxMjdiODhiNmZiZjUxZDczYWI=
+    OTZiYzU5NzIyNTc3MjNlMGZkNmM5NDgxN2Q4ODkxYmZkNjVhZTkzZmUwNDI5
+    MTU4NGZmMjZhYmJhNDBlNmQ2MDI5OGFlMDkwNzU0N2Y1ZDdiYzI3MGNkNWM3
+    YTlmOTMyMGNhM2Q1YWFlMGNiMWJlNWY3MTNlNjBjMmE1ZDc4MDU=
   data.tar.gz: !binary |-
-    YzFiYTg2OTlmZjk4MTRlNjRiOWExMWQ1OGYxMGE2NzEwMTI0OTVkY2ExMzY3
-    OTc1MWZmODA4NzJmZGZkNTU0ODg1MTE3MTk2OTg0YWM3MTI3NTM2N2Q2ZWVi
-    NjNhZmFlNzRkYTFhODFmY2RmMGFjYjkzNjliYzIxYzQwY2IzZDU=
+    MTg1OWEyMWQ4OGYwMmMyOGZmZTIxMjYyYzNkOTc5ZDFmMDNmNDA0NGVjM2Jj
+    NWRmMjQ0NDFiOWE5MTBjZDNiY2NiYWVjMDBkYzRlOGVhODY4MmY3OWE0NDcw
+    ZDNiNDY2NzViZjQxNzBiNWI5ZmY5OTMwZGExOGM5ZDg5OWRjYWU=

data/ruby/lib/zillabyte.rb

@@ -1,3 +1,4 @@
+require "active_support/core_ext"
 require "zillabyte/harness"
 require "zillabyte/common/progress"
 

data/ruby/lib/zillabyte/harness/app.rb

@@ -9,7 +9,7 @@ class Zillabyte::Harness::App
     h._streams = {}
     h._names = {}
     h._name = name
-    Zillabyte::Harness::Helper.check_name("new", h._name, {})
+    Zillabyte::Harness::Helper.check_name("app", h._name, {})
     h._options = Zillabyte::Harness::Helper.opt_parser()
     if(h._options[:command] == :info)
       h._info_file = File.open(h._options[:file],"w+")

data/ruby/lib/zillabyte/harness/{aggregate.rb → group_by.rb}

@@ -1,9 +1,10 @@
-class Zillabyte::Harness::Aggregate
+class Zillabyte::Harness::GroupBy
   attr_accessor :_name, :_type, :_consumes, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
 
-  def initialize()
-    @_name = "aggregate_"+Zillabyte::Harness::Counter.get()
-    @_type = 'aggregate'
+  def initialize(*group_by_fields)
+    @_name = "group_by_"+Zillabyte::Harness::Counter.get()
+    @_type = 'group_by'
+    @_group_by = group_by_fields;
   end
   
   def name(v)
@@ -17,6 +18,14 @@ class Zillabyte::Harness::Aggregate
   def group_by(v)
     @_group_by = v
   end
+  
+  def group_by
+    if @_group_by.is_a?(Array)
+      return @_group_by  
+    else
+      return [@_group_by]  
+    end
+  end
 
   def emits(v)
     @_emits = v

data/ruby/lib/zillabyte/harness/helper.rb

@@ -43,11 +43,11 @@ class Zillabyte::Harness::Helper
     ee = "Error in \"#{operation}\" at \"name\": \n\t "
 
     # \w matchs [a-zA-Z0-9_]
-    if(!name.instance_of?(String) or (name =~ /^\w+$/).nil?)
+    if(!(name.is_a?(String) or name.is_a?(Symbol)) or (name =~ /^\w+$/).nil?)
       msg = "#{ee}\"Name\" must be a non-empty STRING with only alphanumeric and underscore characters at \"#{name}\". Apps and simple_apps must have names!"
       Zillabyte::Harness::Helper.print_error(msg)
     end
-    if(names[name] and names[name] != "new" and names[name] != "sink" and operation != "new" and operation != "sink")
+    if(names[name] and names[name] != "app" and names[name] != "sink" and operation != "app" and operation != "sink")
       msg = "#{ee}The \"name\" \"#{name}\" was previously defined in a #{names[name]}!"
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -61,17 +61,9 @@ class Zillabyte::Harness::Helper
     
     rm = !source._relation.nil?
     mm = !source._matches.nil?
-    if( rm or mm )
-      msg = "#{ee}"
-      msg += "A custom \"source\" may not contain a \"matches\" clause. To source from a relation, see the following. #{pp}"
-      Zillabyte::Harness::Helper.print_error(msg)
-    end
-
     if(rm or mm)
-      if source._begin_cycle or source._next_tuple
-        msg = "#{ee}If \"source\" has a \"matches\\relation\" clause, it may not have a \"begin_cycle\" or \"next_tuple\" block. #{pp}"
-        Zillabyte::Harness::Helper.print_error(msg)
-      end
+      msg = "#{ee}A custom \"source\" may not contain a \"matches\" clause. To source from a relation, see the following. #{pp}"
+      Zillabyte::Harness::Helper.print_error(msg)
     end
 
     allowed_policies = [:null_emit, :explicit]
@@ -181,7 +173,7 @@ class Zillabyte::Harness::Helper
     ee = "Error in \"sink\" at \"column\": \n\t "
     pp = @@_print_check_sink
 
-    if(!cname.instance_of?(String) or (cname =~ /^\w+$/).nil?)
+    if(!(cname.is_a?(String) or cname.is_a?(Symbol)) or (cname =~ /^\w+$/).nil?)
       msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in relation \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -220,9 +212,6 @@ class Zillabyte::Harness::Helper
   - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
   - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
   - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
-  - If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
-     * "Consumes" is the name of a stream emitted by an "each" or a "source" which the "sink" should save as a table.
-     * The columns specified in "sink" must match the fields emitted by the stream."
 OUTPUT
 
   @@_print_check_source = <<-OUTPUT
@@ -263,7 +252,7 @@ OUTPUT
 
   @@_print_check_aggregate = <<-OUTPUT
 \n\n"Aggregate" Syntax:
-    stream.aggregate do
+    stream.group_by do
        name "name"  => optional
        group_by ["field_1", "field_2", ...]
        emits emits

data/ruby/lib/zillabyte/harness/join.rb

@@ -0,0 +1,32 @@
+class Zillabyte::Harness::Join
+  attr_accessor :_name, :_type, :_lhs_stream, :_rhs_stream, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
+  #:_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
+
+  def initialize(lhs_stream, rhs_stream, options = {})
+    @_name = "join_"+Zillabyte::Harness::Counter.get()
+    @_type = 'join'
+    @_lhs_stream = lhs_stream
+    @_rhs_stream = rhs_stream
+    @_join_type = :inner
+    
+    options.symbolize_keys!
+    
+    # Figure out which fields to join on..
+    if options[:on].is_a?(Array) and options[:on].size == 2
+      @_lhs_fields = options[:on][0]
+      @_rhs_fields = options[:on][1]
+      
+    elsif options[:on]
+      @_lhs_fields = options[:on]
+      @_rhs_fields = options[:on]
+      
+    end
+    
+    # Different kind of join types...
+    if options[:type]
+      @_join_type = options[:type].to_sym
+    end
+    
+  end
+
+end

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -7,8 +7,14 @@ module Zillabyte
 
     class UserEndCycleException < Exception
     end
+    
+    class ParentDeadException < Exception
+    end
 
     module Protocol
+      
+      READ_SOFT_TIMEOUT = 20  # Amount of time to wait until we send a 'ping' to the parent
+      DEATH_TIMEOUT = READ_SOFT_TIMEOUT * 3  # If no activity from the parent (even response to a ping) time before we kill ourself.
 
       class << self
         attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :pipe_name, :pipe_to_java, :pipe_from_java
@@ -32,12 +38,38 @@ module Zillabyte
       end
 
       def read_message
-        msg = ""
+        
+        @_last_activity ||= Time.now.utc 
+        msg = []
+        
         loop do
-          line = Storm::Protocol.pipe_from_java.readline.chomp#STDIN.readline.chomp
+          
+          # Read the next line.. The IO.select magic here is a way to minimize concurrency overhead
+          results = IO.select([Storm::Protocol.pipe_from_java], nil, nil, READ_SOFT_TIMEOUT)
+          
+          # Process timeout... 
+          if results.nil?
+            
+            # Are we not getting any activity? 
+            if @_last_activity + DEATH_TIMEOUT < Time.now.utc
+              # JVM is not responding.. they are probably dead.  We should probably die too... 
+              raise ParentDeadException
+            end
+            
+            # Otherwise, let's ping JVM and see if they are still around... 
+            send_msg_to_parent :ping => Time.now.utc.to_f
+            next 
+          end
+          
+          # Else, successful read line. 
+          
+          # Mark the activity... 
+          @_last_activity = Time.now.utc 
+          
+          # String it together... 
+          line = results[0][0].readline.chomp
           break if line == "end"
           msg << line
-          msg << "\n"
         end
 
         encoding_options = {
@@ -45,17 +77,19 @@ module Zillabyte
             :undef             => :replace,  # Replace anything not defined in ASCII
             :replace           => ''        # Use a blank for those replacements
           }
-        msg.encode! Encoding.find('ASCII'), encoding_options
+        json_s = msg.join("\n").chomp.encode(Encoding.find('ASCII'), encoding_options)
 
-        JSON.parse msg.chomp
+        return JSON.parse(json_s)
+        
       end
+      
 
       def read_command
         Storm::Protocol.pending_commands.shift ||
           begin
             msg = read_message
-            while msg.is_a? Array
-              Storm::Protocol.pending_taskids.push(msg)
+            while msg.is_a?(Hash) && msg['pong']
+              # Ignore 'pong' commands (i.e. the response from our 'pings' above)
               msg = read_message
             end
             msg
@@ -83,7 +117,7 @@ module Zillabyte
 
         stream = nil
         tuple = nil
-        meta = {}
+        meta = nil
 
         if (args.size == 1)
           tuple = args[0]
@@ -100,6 +134,11 @@ module Zillabyte
           tuple = args[1]
           meta = args[2]
         end
+        
+        if tuple.is_a?(Tuple)
+          meta = meta || tuple.meta || {}
+          tuple = tuple.values
+        end
 
         if !stream.nil? and !Storm::Protocol.emits.member?(stream)
           fail "Exception in #{Storm::Protocol.mode}: " + "\"#{stream}\" is not a declared stream. Please either remove it from \"emit\" (if there's only one output stream) or declare it in an \"emits\" clause.\n"
@@ -146,9 +185,25 @@ module Zillabyte
       end
 
       def handshake
+        
+        # Get the parent's message.. 
         setup_info = read_message
+        
+        # Setup pid dir... 
         send_pid setup_info['pidDir']
-        [setup_info['conf'], setup_info['context']]
+        
+        # Misc stuff
+        if setup_info['config']
+          
+          # Testing hooks.. 
+          const_set('READ_SOFT_TIMEOUT', setup_info['config']['multilang.read.soft.timeout'].to_i)
+          const_set('DEATH_TIMEOUT', setup_info['config']['multilang.death.timeout'].to_i)
+          
+        end
+        
+        # Done 
+        nil
+        
       end
     end
 
@@ -186,6 +241,8 @@ module Zillabyte
             send_msg_to_parent m
           rescue SignalException => e
             raise
+          rescue ParentDeadException => e
+            raise
           rescue Exception => e
             fail 'Exception in source: ' + e.message + ' - ' + e.backtrace.join('\n')
             break
@@ -215,6 +272,8 @@ module Zillabyte
             execute t
           rescue SignalException => e
             raise
+          rescue ParentDeadException => e
+            raise
           rescue Exception => e
             fail 'Exception in each: ' + e.message + ' - ' + e.backtrace.join('\n')
             # We may recover from this, but let JVM decide
@@ -224,11 +283,11 @@ module Zillabyte
       end
     end
 
-    class Aggregate
+    class GroupBy
       include Storm::Protocol
 
       def run(pipe_name)
-        Storm::Protocol.mode = 'aggregation'
+        Storm::Protocol.mode = 'group_by'
         Storm::Protocol.pipe_name = pipe_name
         setup_pipes
         handshake
@@ -248,8 +307,10 @@ module Zillabyte
             end
           rescue SignalException => e
             raise
+          rescue ParentDeadException => e
+            raise
           rescue Exception => e
-            fail "Exception in aggregate: " + e.message + ' - ' + e.backtrace.join('\n')
+            fail "Exception in group_by: " + e.message + ' - ' + e.backtrace.join('\n')
             break
           end
           done
@@ -297,7 +358,7 @@ module Zillabyte
 
     end
 
-    class AggregateController < Storm::Aggregate
+    class GroupByController < Storm::GroupBy
 
       def initialize(harness, progress)
         @harness = harness

data/ruby/lib/zillabyte/harness/stream.rb

@@ -9,6 +9,10 @@ class Zillabyte::Harness::Stream
     @_previous_node_name = previous_node_name
   end
 
+  def name
+    @_name
+  end
+
   def each(&block)
     h = Zillabyte::Harness::Each.new()
     # Does the block take 0 arguments? If so it's not just an execute block.
@@ -45,36 +49,115 @@ class Zillabyte::Harness::Stream
     output_streams
   end
 
-  def aggregate(&block)
-    h = Zillabyte::Harness::Aggregate.new()
-    #yield(h)
+
+  def group_by(*args, &block)
+    
+    # Init
+    h = Zillabyte::Harness::GroupBy.new(*args)
+
+    # Yield 
     h.instance_eval(&block)
-    Zillabyte::Harness::Helper.check_name("aggregate", h._name, @_names)
-    if(@_branched)
-      Zillabyte::Harness::Helper.check_consumes(h, @_streams)
+    Zillabyte::Harness::Helper.check_name("group_by", h._name, @_app._names)
+    
+    # Sanity checks
+    # Zillabyte::Harness::Helper.check_group_by("aggregate", h, @_nodes, @_streams)
+    # @_branched = Zillabyte::Harness::Helper.check_emits("aggregate", h._emits, @_streams) || @_branched
+    
+    # Multiple emits?
+    if h._emits
+      Zillabyte::Harness::Helper.check_emits("group_by", h._emits, @_app._streams)
     else
-      if(h._consumes)
-        h._consumes = nil
-      end
+      h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
     end
-    Zillabyte::Harness::Helper.check_group_by("aggregate", h, @_nodes, @_streams)
-    @_branched = Zillabyte::Harness::Helper.check_emits("aggregate", h._emits, @_streams) || @_branched
-    @_nodes << h
-    if(@_options[:command] == :info)
-      info_hash = {"name" => h._name, "type" => h._type, "group_by" => h._group_by, "emits" => h._emits}
-      if(h._consumes)
-        info_hash["consumes"] = h._consumes
-      end
-      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
-    elsif(@_options[:command] == :execute)
-      pipe_name = @_options[:pipe]
-      if(@_options[:name] == h._name)
-        c = Zillabyte::Harness::AggregateController.new(h, Zillabyte::Common::Progress.new)
-        c.run(pipe_name)
-      end
+    
+    # Add this node the list
+    @_app._nodes << h
+    
+    # Handle incoming command..
+    if(@_app._options[:command] == :info)
+      
+      # Info.. 
+      info_hash = {"name" => h._name, "type" => h._type, "group_by" => h.group_by, "emits" => h._emits, "consumes" => @_name}
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      
+      # Execute..
+      pipe_name = @_app._options[:pipe]
+      c = Zillabyte::Harness::GroupByController.new(h, Zillabyte::Common::Progress.new)
+      c.run(pipe_name)
+      
+    end
+    
+      
+    
+    # Return the stream 
+    output_streams = []
+    h._emits.each do |stream|
+      output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
     end
+    output_streams = output_streams[0] if output_streams.size == 1
+    output_streams
+    
   end
 
+
+
+  def join_with(rhs_stream, *args, &block)
+    
+    # Init
+    h = Zillabyte::Harness::Join.new(self, rhs_stream, *args)
+
+    # Yield 
+    throw "programmable joins are not supported at this time" unless block.nil?
+    # h.instance_eval(&block)
+    
+    # Sanity
+    Zillabyte::Harness::Helper.check_name("join", h._name, @_app._names)
+    case h._join_type
+    when :inner,:left,:right
+    when :outer
+      Zillabyte::Harness::Helper.print_error("Outer joins are currently not supported")
+    else 
+      Zillabyte::Harness::Helper.print_error("Unknown join type: #{h._join_type}")
+    end
+    
+    # Build the node
+    emit_stream_name = "stream_"+Zillabyte::Harness::Counter.get()
+    h._emits = [emit_stream_name]
+    @_app._nodes << h
+    
+    # Handle incoming command..
+    if(@_app._options[:command] == :info)
+      # Info.. 
+      info_hash = {
+        "name" => h._name, 
+        "type" => h._type, 
+        "consumes_lhs" => h._lhs_stream._name,
+        "consumes_rhs" => h._rhs_stream._name, 
+        "lhs_fields" => h._lhs_fields,
+        "rhs_fields" => h._rhs_fields,
+        "join_type" => h._join_type,
+        "emits" => h._emits, 
+      }
+      Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
+      
+    elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
+      
+      # There's a problem, because joins should be not be executed in multilang
+      throw "unsupported: joins should be executed here"
+      
+    end
+    
+    # Return the stream 
+    output_stream = Zillabyte::Harness::Stream.new(emit_stream_name, @_app, h._name)
+    return output_stream
+    
+  end
+
+
+
+
   def sink(&block)
     h = Zillabyte::Harness::Sink.new()
     h.instance_eval(&block)

data/ruby/lib/zillabyte/harness/tuple.rb

@@ -7,15 +7,27 @@ class Tuple
   
   def [](name)
     maybe_build_alias_hash()
-    v = values[ @_alias_hash[name] || name ]
-    raise ArgumentError, "The requested tuple value \"#{name}\" does not exist." if v.nil?
-    return v
+    key = (@_alias_hash[name] || name).to_s
+    if values.has_key?( key )
+      return values[key]
+    else
+      raise ArgumentError, "The requested tuple does not contain the key: \"#{name}\". Current Tuple: #{values.to_s}"
+    end
   end
   
   def values
     @hash["tuple"]
   end
   
+  def meta
+    @hash["meta"]
+  end
+  
+  
+  def to_s
+    values.to_s
+  end
+  
   
   def column_aliases
     @hash['column_aliases'] || []

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.0.20" unless defined?(VERSION)
+  VERSION = "0.0.21" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.0.20
+  version: 0.0.21
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-23 00:00:00.000000000 Z
+date: 2014-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.20
+        version: 0.0.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.20
+        version: 0.0.21
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -60,12 +60,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ruby/lib/zillabyte/common/progress.rb
-- ruby/lib/zillabyte/harness/aggregate.rb
 - ruby/lib/zillabyte/harness/app.rb
 - ruby/lib/zillabyte/harness/counter.rb
 - ruby/lib/zillabyte/harness/each.rb
-- ruby/lib/zillabyte/harness/groupby.rb
+- ruby/lib/zillabyte/harness/group_by.rb
 - ruby/lib/zillabyte/harness/helper.rb
+- ruby/lib/zillabyte/harness/join.rb
 - ruby/lib/zillabyte/harness/live_delegator.rb
 - ruby/lib/zillabyte/harness/sink.rb
 - ruby/lib/zillabyte/harness/source.rb

data/ruby/lib/zillabyte/harness/groupby.rb

@@ -1,9 +0,0 @@
-class Zillabyte::Harness::GroupBy
-  attr_accessor :_type, :_fields
-  
-  def initialize(fields)
-    @_type = 'group_by'
-    @_fields = fields
-  end
-  
-end