zillabyte 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1af0e7dd6900cca31d707e201cfbdfcc8e4fe3f3
-  data.tar.gz: 37532e5dcb0754700d339bf0c4863c7161098983
+  metadata.gz: a43d9f7b3f9e80e5258eb7aa7eadd5e737da2098
+  data.tar.gz: 9cd1867ebe50c7ed22659a79c52be31023e75648
 SHA512:
-  metadata.gz: 42a4e7ecf444f7cc461d2175d2a6a7ecb8658de9452a1c1b91a202c6060fba9e561cc766fbe0edfa941d8a14dfaabf99e333449a38ebd9dbbf938b9a5700654f
-  data.tar.gz: 137d82edc66b7950b671989b3a4631d1e39e88606442130c8ad8a5f389f0da717743725cb6e0f01160732497e04cb8eb5b39afd4b8e8928697a0709ff08a266b
+  metadata.gz: 0cf5964779b966dc2eb8ff59479cc994e14d76d56bb42738b51f2d2564609caecc5f552947daad3cea0f153d209c9efeae9f942f9cc734b8364e0a8114869f56
+  data.tar.gz: 0429f42873893ec3a0035a9c8abe63a75be5c6e2d961596445c1e8c783683f954857e1ed43f9579260fa5d02cde4a37ce2618db19c6d6b71e1556b1945ec309c

data/ruby/lib/zillabyte/harness/each.rb

@@ -1,3 +1,53 @@
+# OPERATION Each
+# HIGH_LEVEL_DESCRIPTION 
+# The each block can be thought of as a map operation that runs across multiple machines. 
+# The rows of the web_pages dataset are distributed across our system and processed. 
+# The result of the each is another stream object that will contain the emitted rows. 
+# The input argument tuple contains a single row, that can be accessed like a hash object.
+#  
+#  
+# LANGUAGE_SYNTAX 
+# Simplified syntax:
+#  
+#   stream.each do |tuple|
+#     |=block=|
+#   end
+# - This is equivalent to just specifying an \"execute\" block below.
+# - Note that this syntax only works for a single output stream.
+#  
+# Custom each:
+#   stream.each do
+#      name "name"                          # the name of the operation
+#      emits "stream_1", "stream_2"         # optional for single output stream
+#      output_format :replace               # :replace or :merge, optional, defaults to :replace
+#      prepare |=block=|                    # optional if no initialization needed
+#      execute |=block=|
+#   end
+#  
+# - The allowed output formats are :replace and :merge.
+#     * :replace - discards the input tuple values and only emits the specified values. This is the default.
+#     * :merge - re-emits the input tuple values along with the specified values.
+# - The "prepare" and "execute" blocks can be in do...end format or {} format.
+#     * the "prepare" block is where any setup is done to prepare for tuple processing in the "execute" block.
+#     * the "execute" block is where the tuples are actually processed. It must take in a single argument (the "tuple").
+#  
+# EXAMPLE 
+#  
+# # This is a simple example which emits the url of a tuple 
+# s = s.each do |tuple|
+#   emit :url => tuple["url"]
+# end
+#  
+#  
+#  
+# # Eaches can also be used for conditional emitting
+# stream = result_stream.each{ |tuple|
+#  
+#   if tuple['html'].include? "hello_world"
+#     emit :url => tuple['url'] 
+#   end
+# 
+#  }
 class Zillabyte::Harness::Each
   attr_accessor :_app, :_node, :_options
 

data/ruby/lib/zillabyte/harness/filter.rb

@@ -1,3 +1,38 @@
+# OPERATION Filter
+# HIGH_LEVEL_DESCRIPTION 
+# The filter is a conditional each. Conditional expressions can be used to determine if a
+# tuple received by the operation is emitted.
+#  
+# LANGUAGE_SYNTAX 
+#   Simplified syntax:
+#     stream.filter do |tuple|
+#       |=block=|
+#     end
+#   - The condition for keeping the tuple should be specified in the block. 
+#   - This is equivalent to just specifying a "keep" block below.
+#   Custom filter:
+#     stream.filter do
+#        name "name" \t\t\t => optional
+#        emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+#        prepare |=block=| \t\t\t\t => optional if no initialization needed
+#        keep |=block=|
+#     end
+#   - The "prepare" and "keep" blocks can be in do...end format or {} format.
+#       * the "prepare" block is where any setup is done to prepare for tuple processing in the "keep" block.
+#       * tuples will pass through the filter if "keep" returns "True". It must take in a single argument (the "tuple").
+#  
+# EXAMPLE 
+# # Filter for simple string inclusion
+# stream = stream.filter { |tuple| tuple["url"].include? "hello world" } 
+#   
+# # Custom Filter
+# stream = stream.filter do\
+#    name "hello_world_filter"
+#    emits "hello_stream"
+#    keep do |tuple|
+#      return tuple["url"].include? "url"
+#    end
+#  end
 class Zillabyte::Harness::Filter
   attr_accessor :_app, :_node, :_options
 

data/ruby/lib/zillabyte/harness/group_by.rb

@@ -1,3 +1,42 @@
+# OPERATION Group By 
+# HIGH_LEVEL_DESCRIPTION 
+# The `group_by` function can be used to implement aggregate computations
+# such as counting and summing. The following is a full example of a
+# word count operation in Zillabyte.
+# 
+# LANGUAGE_SYNTAX 
+#  stream.group_by do
+#      name "name"  \t\t\t\t\t => optional
+#      group_by "field_1", "field_2", ...
+#      emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+#      begin_group |=block=|
+#      aggregate |=block=|
+#      end_group |=block=|
+#   end
+# - The "begin_group", "aggregate" and "end_group" blocks can be in do...end format or {} format.
+#     * the "begin_group" block is where the initial values for the aggregation are set. It must take in a single argument (the "grouping tuple", which is emitted at the beginning of each group and contains the values of the fields specified in "group_by").
+#     * the "aggregate" block is where the aggregation is performed. It must take in a single argument (the "tuple").
+#     * the "end_group" block is where the final aggregated value is emitted.
+# EXAMPLE 
+# # Declare the group_by, grouping on the :word field
+# stream = stream.group_by(:word) do
+#
+#   # Save the word being grouped, initialize any state
+#   begin_group do |g_tuple|
+#     @word = g_tuple[:word]
+#     @count = 0
+#   end
+#  
+#   # In this case, simply increment the counter associated with the word
+#   aggregate do |tuple|
+#     @count += 1
+#   end
+#  
+#   # Emit the grouped word, along with its count
+#   end_group do |g_tuple|
+#     emit :word => @word, :count => @count
+#   end
+# end
 class Zillabyte::Harness::GroupBy
   attr_accessor :_app, :_node, :_fields, :_options
 

data/ruby/lib/zillabyte/harness/helper.rb

@@ -387,8 +387,9 @@ class Zillabyte::Harness::Helper
     end
   end
 
-  def self.check_loop_back(node_name, nodes)
-    ee = "Error in \"loop_back\": \n\t "
+  def self.check_loop_back(stream, node_name, nodes)
+    ee = "Error in \"loop_back\" to \"#{node_name}\": \n\t "
+    previous_node_name = stream._previous_node_name
     found = false
     nodes.each do |node|
       if node._name == node_name
@@ -398,6 +399,12 @@ class Zillabyte::Harness::Helper
           Zillabyte::Harness::Helper.print_error(msg)
         end
       end
+      if node._name == previous_node_name
+        if node._emits.size < 2
+          msg = "#{ee}The preceding operation does not emit multiple streams. Please make sure that it emits one stream for the loop back and another stream for downstream operations."
+          Zillabyte::Harness::Helper.print_error(msg)
+        end
+      end
     end
     if !found
       msg = "#{ee}The specified loop-back node \"#{node_name}\" was not found in the operations preceding it."

data/ruby/lib/zillabyte/harness/join.rb

@@ -1,3 +1,31 @@
+# OPERATION Join 
+# HIGH_LEVEL_DESCRIPTION 
+# Joins allow you to combine two streams into one in a manner similar
+# to traditional relational systems. Realtime tuple data from a variety
+# of sources can be combined into a single stream via joins on specified
+# fields. Here is an example of a `join` in Zillabyte.
+# LANGUAGE_SYNTAX 
+#   lhs_stream.join_with( rhs_stream_object, options )
+# - Options should be specified as a hash. The following keys are recognized:
+#     Mandatory:
+#     * "on" -- specifies the fields to join on. The value must be a STRING or a length-2 ARRAY. 
+#               If value = a STRING, the LH and RH join fields will both be set to this STRING. 
+#               If value = a length-2 ARRAY, the LH join field will be set to array[0] and the RH join field will be set to array[1].
+#     Optional:
+#     * "type" -- specifies the join type. The default is :left. Options are [:inner, :outer, :left, :right]
+#     * "emits" -- specifies the stream to emit. A "join" may only emit a single stream.
+# EXAMPLE 
+#  
+# # Left join on a shared field, in this case, an :owner_id
+# joined_stream = auto_stream.join_with(repairs_stream, :on => :owner_id)
+#  
+# # Join with budget_stream's :average on left, employee_stream's :number on right.
+# joined_stream = budget_stream.join_with(
+#   employee_stream, 
+#   :on => [:average, :number], 
+#   :type => :inner
+# )
+#
 class Zillabyte::Harness::Join
   attr_accessor :_app, :_node, :_args
 

data/ruby/lib/zillabyte/harness/sink.rb

@@ -1,3 +1,26 @@
+# OPERATION Sink 
+# HIGH_LEVEL_DESCRIPTION 
+# The `sink` is a passive operation that defines the schema of the rows that need to be saved. 
+# Of all the operations where a stream is consumed, only the `sink` requires a schema to be defined. 
+#  
+# LANGUAGE_SYNTAX 
+#   stream.sink do
+#      name "name_of_relation"
+#      column "field_1", :type_1
+#      column "field_2", :type_2 ...
+#   end
+# - "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+# - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
+# - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+# - Field types must be SYMBOLS. The following types are allowed [:string, :integer, :float, :double, :boolean, :array, :map]
+#  
+# EXAMPLE 
+# stream.sink{
+#     name "patent_sink"
+#     column "doc_number", :integer
+#     column "title", :string
+#     column "date", :string
+# }
 class Zillabyte::Harness::Sink
   attr_accessor :_app, :_node, :_options
 

data/ruby/lib/zillabyte/harness/source.rb

@@ -1,3 +1,60 @@
+# OPERATION Source
+# HIGH_LEVEL_DESCRIPTION 
+# A `source` is where the data for your app originates and is defined
+# on the app object. The easiest way to stream data into a Zillabyte
+# app is to use a Zillabyte dataset. A simple `source` takes in the
+# name of a dataset and produces a stream object. 
+#  
+# A `source` can also use data outside of the public datasets Zillabyte
+# provides. Any external data available on the web can be streamed
+# into a Zillabyte app. To do so, we use the expanded syntax of a
+# `source` and include the code needed to generate the `source` data
+# in the `next_tuple` block. The expanded syntax also allows for
+# additional customizations such as providing a `name` for the `source`
+# and specifying any preparatory steps such as initializing global
+# values in the `begin_cycle` block.
+#  
+# LANGUAGE_SYNTAX 
+#  
+#   Sourcing from a dataset:
+#     app.source("dataset_name")
+#   Custom source:
+#     app.source do
+#        name "name" \t\t\t\t\t => optional
+#        emits "stream_1", "stream_2", ... \t\t => optional for single output stream
+#        end_cycle_policy :null_emit OR :explicit \t => default :null_emit
+#        begin_cycle |=block=| \t\t\t\t => optional if no initialization needed
+#        next_tuple |=block=|
+#     end
+#   - The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
+#       * :null_emit - end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
+#       * :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
+#   - The "begin_cycle" and "next_tuple" blocks can be in do...end format or {} format.
+#       * the "begin_cycle" block is where any setup is done to initialize the content and quantity of tuples emitted by the "next_tuple" block.
+#       * the "next_tuple" block is where the tuples are actually emitted.
+# EXAMPLE 
+#  
+# # Source from the "homepages" relation
+# stream = app.source("homepages")
+#  
+# # Custom Source
+# app = Zillabyte.app(name = "small_cycle")
+# stream = app.source do
+#   DICTIONARY = %w(apple bananna cherry dairy eel) unless defined? DICTIONARY
+#   emits  "small_cycle_relation"
+#   end_cycle_policy :explicit
+#   begin_cycle do
+#     @count = 0
+#   end
+#  
+#   next_tuple do
+#     url = "http://#{@count}.example.com/#{@count}.html"
+#     html = "<html>#{DICTIONARY.sample}</html>"
+#     emit("small_cycle_relation", {:url => url, :html => html})
+#     @count += 1
+#     end_cycle if @count==1000
+#   end
+# end
 class Zillabyte::Harness::Source
   attr_accessor :_app, :_node, :_relation
 

data/ruby/lib/zillabyte/harness/stream.rb

@@ -73,9 +73,8 @@ class Zillabyte::Harness::Stream
   def loop_back(*args, &block)
     # This is not a real operation, just telling the stream to loop back to the previous operation
     loop_back_node = args[0]
-    Zillabyte::Harness::Helper.check_loop_back(loop_back_node, @_app._nodes)
+    Zillabyte::Harness::Helper.check_loop_back(self, loop_back_node, @_app._nodes)
     Zillabyte::Harness::Helper.write_arc_to_file({"name" => self._name, "origin" => self._previous_node_name, "dest" => loop_back_node, "loop_back" => 1}, @_app._socket)
-    self
   end
 
   def sink(*args, &block)

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.9.0" unless defined?(VERSION)
+  VERSION = "0.9.1" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-24 00:00:00.000000000 Z
+date: 2014-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: 0.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: 0.9.1
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com