zillabyte 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 8c70d8e1d80584425348ab8ca04755fbade0c947
-  data.tar.gz: b11745a058193b76fd458f0de1e63fe93e093295
-SHA512:
-  metadata.gz: 74adda596843678f98c809918b0baf540bd0883f8279a8e328efea515a2a70c2bb3bb9f4543fdf7b84adff15d9a3609443c5690c9091c097d41ee8c9c3c97e20
-  data.tar.gz: ff3e886dacd18bddf69b577425a53498e6119699bc6ea4c7142af52a2c77666a35b8e3db52f879d239c95c8d107a453203271b402948f51eb2676bac741d4a0a
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NTA2NGJmMzJjMjMzOTBhMDcyNDczNWIzNDc3MTgyODljMWI3YTYzMw==
+  data.tar.gz: !binary |-
+    ZWNiZDUwNGYxMGVmYjk2OTJkODJiMzlmOTFhN2JiMzQ5Zjc3NGEzOA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NTMwMzFmMzk1ZTIwY2JhMmFkM2ExNzYzMmExMzViZmFmYTcxYTU2YTk0ODUz
+    MjgyYjY5OTkwZDhmOWRiZTUyZDQxYWRhOTg0Y2IyMzVmN2U0NjY3YzlmNDU4
+    OTc4NjYyNDdjYzg4MTUyZGUzYjU5NzZkZDI0YzIzZDE5MDNhZTk=
+  data.tar.gz: !binary |-
+    YWFlNzQxMWQ5OWQxZjYzYTZiM2E4N2RmODMxMDQ4NTg4M2I0ZDA0YmVmY2Rm
+    Nzg1OGRmYzE1ZjEwMTE5ODJhZGZiZTQ4ZjE1YzhhM2FmY2FhYmU0YWZmMTM4
+    NDE4ZWQ5Mjk0Y2JkYWMyZWY0Mzk5N2JlNjAxMWM2ZTQ5NDczZmE=

data/ruby/lib/zillabyte/harness/each.rb

@@ -1,9 +1,10 @@
 class Zillabyte::Harness::Each
-  attr_accessor :_name, :_type, :_emits, :_prepare, :_execute
+  attr_accessor :_name, :_type, :_emits, :_output_format, :_prepare, :_execute
 
   def initialize()
     @_name = "each_"+Zillabyte::Harness::Counter.get()
-    @_type = 'each'
+    @_type = "each"
+    @_output_format = :replace
   end
   
   def name(v)
@@ -14,6 +15,10 @@ class Zillabyte::Harness::Each
     @_emits = *v
   end
 
+  def output_format(v)
+    @_output_format = v
+  end
+
   def prepare(&block)
     @_prepare = block
   end

data/ruby/lib/zillabyte/harness/helper.rb

@@ -2,6 +2,10 @@ require 'optparse'
 
 class Zillabyte::Harness::Helper
 
+  META_NAMES = ["id", "confidence", "since", "source"]
+  ALLOWED_OUTPUT_FORMATS = [:replace, :merge]
+  ALLOWED_END_CYCLE_POLICIES = [:null_emit, :explicit, :infinite]
+
   def self.opt_parser()
     options = {
       "name" => Dir.pwd.split("/")[-1]
@@ -63,6 +67,15 @@ class Zillabyte::Harness::Helper
 
   end
 
+  def self.check_output_format(operation, output_format)
+    ee = "Error in \"#{operation}\" at \"output_format\": \n\t "
+
+    if(!ALLOWED_OUTPUT_FORMATS.member? output_format)
+      msg = "#{ee}Invalid \"output_format\": #{output_format}. The allowed formats are :replace and :merge."
+      Zillabyte::Harness::Helper.print_error(msg)
+    end
+  end
+
   def self.check_custom_source(source)
     ee = "Error in \"source\": \n\t "
     pp = @@_print_check_source
@@ -74,8 +87,7 @@ class Zillabyte::Harness::Helper
       Zillabyte::Harness::Helper.print_error(msg)
     end
 
-    allowed_policies = [:null_emit, :explicit, :infinite]
-    if !(allowed_policies.member? source._end_cycle_policy)
+    if !(ALLOWED_END_CYCLE_POLICIES.member? source._end_cycle_policy)
       msg = "#{ee}Invalid \"end_cycle_policy\": \"#{source._end_cycle_policy}\". The allowed policies are :null_emit and :explicit. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -155,7 +167,9 @@ class Zillabyte::Harness::Helper
     if !each._execute
       msg = "#{ee}An \"each\" must contain an \"execute\" block. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
-    end 
+    end
+
+    Zillabyte::Harness::Helper.check_output_format("each", each._output_format)
   end
 
   def self.check_group_by(group_by)
@@ -224,15 +238,16 @@ class Zillabyte::Harness::Helper
 
     id = node._id
     if !id
-      msg = "#{ee} The \"component_id\" must be specified. Check \"zillabyte components\" to find the correct ID. #{pp}"
+      msg = "#{ee}The \"component_id\" must be specified. This may be the name or ID of the component. Check \"zillabyte components\" to find this information. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
 
-    # \w matchs [0-9]
-    if(!(id.is_a?(String) or id.is_a?(Symbol)) or (id =~ /^[0-9]+$/).nil? or id[0] == "0")
-      msg = "#{ee} Invalid \"component_id\". The \"component_id\" must be a number. Check \"zillabyte components\" to find the correct ID. #{pp}"
+    if(!(id.is_a?(String) or id.is_a?(Symbol)) or (id =~ /^\w+$/).nil?)
+      msg = "#{ee}The \"component_id\" must be a non-empty STRING with only alphanumeric and underscore characters in \"#{id}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
+
+    Zillabyte::Harness::Helper.check_output_format("call_component", node._output_format)
   end
 
   def self.check_sink(sink, nodes)
@@ -281,7 +296,7 @@ class Zillabyte::Harness::Helper
       msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in \"#{relation_name}\". #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
-    if(cname =~ /^v[0-9]+$/i or @@_meta_names.member?(cname.downcase))
+    if(cname =~ /^v[0-9]+$/i or META_NAMES.member?(cname.downcase))
       msg = "#{ee}\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your field something else. #{pp}"
       Zillabyte::Harness::Helper.print_error(msg)
     end
@@ -385,10 +400,13 @@ OUTPUT
   Custom filter:
     stream.filter do
        name "name" \t\t\t => optional
+       emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+       prepare |=block=| \t\t\t\t => optional if no initialization needed
        keep |=block=|
     end
-  - The "keep" block must take in a single argument (the "tuple").
-  - Tuples will pass through if "keep" returns "True".
+  - The "prepare" and "keep" blocks can be in do...end format or {} format.
+      * the "prepare" block is where any setup is done to prepare for tuple processing in the "keep" block.
+      * tuples will pass through the filter if "keep" returns "True". It must take in a single argument (the "tuple").
 OUTPUT
 
   @@_print_check_each = <<-OUTPUT
@@ -404,9 +422,13 @@ OUTPUT
     stream.each do
        name "name" \t\t\t\t\t => optional
        emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+       output_format :replace OR :merge \t\t => optional, defaults to :replace
        prepare |=block=| \t\t\t\t => optional if no initialization needed
        execute |=block=|
     end
+  - The allowed output formats are :replace and :merge.
+      * :replace - discards the input tuple values and only emits the specified values. This is the default.
+      * :merge - re-emits the input tuple values along with the specified values.
   - The "prepare" and "execute" blocks can be in do...end format or {} format.
       * the "prepare" block is where any setup is done to prepare for tuple processing in the "execute" block.
       * the "execute" block is where the tuples are actually processed. It must take in a single argument (the "tuple").
@@ -469,12 +491,16 @@ OUTPUT
   @@_print_check_call_component = <<-OUTPUT
 \n\n"Call_component" Syntax:
     stream.call_component do
-      component_id "component_id"
-      name "name" \t\t\t\t\t\t\t\t\t => optional
-      additional_inputs other_input_stream_object_1, other_input_stream_object_2, ... \t => blank if none
-      outputs "output_stream_name_1", "output_stream_name_2", ...
-    end
-  - The "component_id" MUST be given and correspond to the id listed in the output of "zillabyte components".
+       component_id "component_id"
+       name "name" \t\t\t\t\t\t\t\t\t => optional
+       additional_inputs other_input_stream_object_1, other_input_stream_object_2, ... \t => blank if none
+       outputs "output_stream_name_1", "output_stream_name_2", ...
+       output_format :replace OR :merge \t\t => optional, defaults to :replace
+    end
+  - The "component_id" MUST be given and correspond to the name or id listed in the output of "zillabyte components".
+  - The allowed output formats are :replace and :merge. Note that only linear components, i.e. those with only "each" and "filter" operations support :merge.
+      * :replace - discards the input tuple values and only emits the output values. This is the default.
+      * :merge - re-emits the input tuple values along with the output values.
   - To correctly stich in the component, the implicit assumptions below WILL BE USED:
       * The "stream" that "call_component" is invoked on MUST correspond to the first listed input stream to the component.
       * The streams specified in "additional_inputs" MUST correspond to the other listed input streams in order.
@@ -483,5 +509,4 @@ OUTPUT
       * The number of input and output streams specified must match the number listed for the component.
 OUTPUT
 
-  @@_meta_names = ["id", "confidence", "since", "source"]
 end

data/ruby/lib/zillabyte/harness/injected_component.rb

@@ -5,7 +5,7 @@ class Zillabyte::Harness::InjectedComponent
     @_name = "component_"+Zillabyte::Harness::Counter.get()
     @_type = "component"
     @_consumes = []
-    @_output_format = "replace"
+    @_output_format = :replace
   end
 
   def name(v)

data/ruby/lib/zillabyte/harness/live_delegator.rb

@@ -89,14 +89,14 @@ module Zillabyte
         Storm::Protocol.pending_commands.shift ||
           begin
             msg = read_message
-            while msg.is_a?(Hash) && msg['pong']
+            while msg && msg.is_a?(Hash) && msg['pong']
               # Ignore 'pong' commands (i.e. the response from our 'pings' above)
               msg = read_message
             end
             msg
-          rescue JSON::ParserError => e
-            fail "error: #{e}"
-            {}
+          rescue StandardError => e
+            log "message error: #{e}"
+            nil
           end
       end
 
@@ -114,21 +114,6 @@ module Zillabyte
         File.open("#{heartbeat_dir}/#{pid}", "w").close
       end
 
-
-      def set_state(key, val)
-        send_msg_to_parent({'set_state' => Base64.encode64(Marshal.dump(val)), 'key' => key})
-      end
-      
-      def get_state(key)
-        send_msg_to_parent({'get_state' => key})
-        ret = read_message()
-        if ret['state']
-          return Marshal.load(Base64.decode64(ret['state']))
-        else
-          return nil
-        end
-      end
-
       def emit(*args)
 
         stream = nil
@@ -243,7 +228,7 @@ module Zillabyte
 
         while true
           begin
-            msg = read_command
+            msg = read_command || {}
             case msg['command']
             when 'begin_cycle'
               begin_cycle
@@ -287,9 +272,13 @@ module Zillabyte
 
         while true
           begin
-            t = Tuple.from_hash(read_command)
-            next if(!t)
-            execute t
+            m = read_command()
+            if(m)
+              t = Tuple.from_hash(m)
+              if(t)
+                execute t
+              end
+            end
           rescue SignalException => e
             raise
           rescue ParentDeadException => e
@@ -318,9 +307,13 @@ module Zillabyte
 
         while true
           begin
-            t = Tuple.from_hash(read_command)
-            next if(!t)
-            emit t if keep t
+            m = read_command()
+            if m
+              t = Tuple.from_hash(m)
+              if t
+                emit(t) if keep t
+              end
+            end
           rescue SignalException => e
             raise
           rescue Exception => e
@@ -343,7 +336,7 @@ module Zillabyte
 
         while true
           begin
-            msg = read_command
+            msg = read_command || {}
             case msg['command']
             when 'begin_group'
               t = Tuple.from_hash(msg)

data/ruby/lib/zillabyte/harness/stream.rb

@@ -34,7 +34,7 @@ class Zillabyte::Harness::Stream
     @_app._nodes << h
 
     if(@_app._options[:command] == :info)
-      node_hash = {"name" => h._name, "type" => h._type}
+      node_hash = {"name" => h._name, "type" => h._type, "output_format" => h._output_format}
       Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
       arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
       Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
@@ -182,9 +182,13 @@ class Zillabyte::Harness::Stream
     
   end
 
-  def call_component(&block)
+  def call_component(comp_id = nil, &block)
+    
     h = Zillabyte::Harness::InjectedComponent.new()
-    h.instance_eval(&block)
+    h.instance_eval(&block) if block_given?
+    h._id ||= comp_id
+    h._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]  # We default to single-stream when called like this.. 
+    
     Zillabyte::Harness::Helper.check_name("component", h._name, @_app._names)
     Zillabyte::Harness::Helper.check_call_component(h)
     Zillabyte::Harness::Helper.check_emits("component", h._emits, @_app._streams)

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.1.1" unless defined?(VERSION)
+  VERSION = "0.1.3" unless defined?(VERSION)
 end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-27 00:00:00.000000000 Z
+date: 2014-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mkfifo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: zillabyte-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.3
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com
@@ -59,9 +59,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ruby/lib/zillabyte.rb
 - ruby/lib/zillabyte/common/progress.rb
-- ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/harness/app.rb
 - ruby/lib/zillabyte/harness/base.rb
 - ruby/lib/zillabyte/harness/component.rb
@@ -80,7 +78,9 @@ files:
 - ruby/lib/zillabyte/harness/source.rb
 - ruby/lib/zillabyte/harness/stream.rb
 - ruby/lib/zillabyte/harness/tuple.rb
+- ruby/lib/zillabyte/harness.rb
 - ruby/lib/zillabyte/version.rb
+- ruby/lib/zillabyte.rb
 homepage: http://www.zillabyte.com
 licenses:
 - MIT
@@ -91,17 +91,17 @@ require_paths:
 - ruby/lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.0.6
 signing_key: 
 specification_version: 4
 summary: The Official Zillabyte Gem