zillabyte 0.9.30 → 0.9.31

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2ZkNTM0ODU4M2I4M2YxNjQzYmIwY2U5MDQ4YTk4YzlkMTVkNWJhMg==
+    ZDEwYWYzMTUyNGU4ZmZjYmIwYzZhNDdiYzg3N2VmYmRmZDMyMjUzNA==
   data.tar.gz: !binary |-
-    YTBhMTZiODM5MGUzM2U2OTFiNWViNzA1ZGJhYjFkMjc3NTg0NzZmZA==
+    YjMwYTg3NjNjNjlmOWJmMzE1ZjMwNzZjMTY5NDRlNzk4OWMxMzZiYg==
 SHA512:
   metadata.gz: !binary |-
-    MzZiNmE3ZDFjNDc3OTAwNmZjMzY5ZTEyOTA5MjIwNzI2ODg4MWMxYTViYzQ4
-    YzBiZWMwNjhjNjYwODAxNDhmOGZlNTI5NzkxODJkZGRmODRhODE3MDgxNmZk
-    ZDUxYTE3NWNjNjM0NWVlMDA2Zjg4ZDI5ZWI2ZGI1MzczNzdhYWE=
+    ZjA2OTIwZTFiNDUxZWE0OTc0N2IyZTM4ZDc4NzQwZjUzM2ExZTEwYzA1NGQ2
+    MTA5MDg5NDRhYmE0YzI1ZmE0M2I3ZTBiMWRkMTdhODBkYWJmOTE4NjdjYmU0
+    N2JhOGEzOTUxZTEzZjgwM2VjYzE2ZDk1NDM1NDlhMGU0NTgyNDA=
   data.tar.gz: !binary |-
-    ZDVhZWYyOTYyZDk1ZTE4ZmIxNTQwZDI3MjFmYjllNjljM2RiMjM0NWE4Y2Ey
-    YzI3ZWJjMzQ2MjlhZGE5NzZkZDAyNGQ3YzU2YjEzNzYyODIzMmU0MjZhYzNh
-    NWZkYTc5Y2EwZmNkMGI4ZjA3MjdiOGFlZTRkMjZlMmI1NGMyMzA=
+    MDNiM2RjMzE4ZjY1ZWQ5MmY1OTQ2YjAzMDJmNDM3MDI3MjQzMWExZGQ4MTMx
+    NzZhOWMzOGM3ZWQwMzZkMmFmZmViMmQzM2MxZjI5NjlhZDkzNDNiZGIxYTlj
+    NGIwMGY2ZjI3Y2Y5N2M3ZWM2OTE4YjllMTlmNmEwYzQ5MWQ2NTA=

data/ruby/lib/zillabyte/harness/component_input.rb

@@ -1,3 +1,37 @@
+# OPERATION Component Input 
+# TAGLINE 
+# Declare the input of a component
+# HIGH_LEVEL_DESCRIPTION 
+# Components work by declaring a “schema” of their inputs and outputs. 
+# The inputs are the expected fields they will look for within incoming tuples. 
+# In an example case, our component declares an input stream named “urls”, and will look for the “url” field of any incoming tuples.
+# 
+# LANGUAGE_SYNTAX 
+#   component.inputs do
+#     name "input_stream_name"
+#     field "field_1", :type_1
+#     field "field_2", :type_2
+#   end
+# 
+# SYNTAX_NOTES 
+#  "Inputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+#  
+#  Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
+#  
+#  Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+#  
+#  Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, :boolean, :array and :map.
+# 
+# EXAMPLE 
+# require 'zillabyte'
+#  
+# comp = Zillabyte.component("ruby_component")
+#  
+# # We declare the input to contain a "word" field
+# stream = comp.inputs do
+#   name "word_stream"
+#   field "word", :string
+# end
 class Zillabyte::Harness::ComponentInput
   attr_accessor :_app, :_node
 

data/ruby/lib/zillabyte/harness/component_output.rb

@@ -1,3 +1,28 @@
+# OPERATION Component Output
+# TAGLINE 
+# Declare the output of a component
+# HIGH_LEVEL_DESCRIPTION 
+# In a similar manner to component inputs, we declare the component output schema for the component. 
+# This determines the output of the component for other Zillabyte functions to expect. This is just like a sink in apps.
+# LANGUAGE_SYNTAX 
+#   component_stream.outputs do
+#     name "output_stream_name"
+#     field "field_1", :type_1
+#     field "field_2", :type_2
+#   end
+# SYNTAX_NOTES 
+# "Outputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+#  
+# Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
+#  
+# Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+#  
+# Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, :boolean, :array and :map.
+# EXAMPLE 
+# stream.outputs do
+#   name "plural"
+#   field "word", :string
+# end
 class Zillabyte::Harness::ComponentOutput
   attr_accessor :_app, :_node
 

data/ruby/lib/zillabyte/harness/each.rb

@@ -1,45 +1,48 @@
 # OPERATION Each
-# HIGH_LEVEL_DESCRIPTION 
+# TAGLINE 
 # The each block can be thought of as a map operation that runs across multiple machines. 
-# The rows of the web_pages dataset are distributed across our system and processed. 
-# The result of the each is another stream object that will contain the emitted rows. 
-# The input argument tuple contains a single row, that can be accessed like a hash object.
-#  
+# HIGH_LEVEL_DESCRIPTION 
+# The `each` operation is the simplest operation within Zillabyte. It is analogous to the inner
+# block of a conventional "for" loop which is used to apply the same set of manipulations to
+# each record. Similarly, Zillabyte applies the same code logic specified in the each to each 
+# tuple the operation receives.
 #  
 # LANGUAGE_SYNTAX 
-# Simplified syntax:
+# # Simplified syntax:
 #  
 #   stream.each do |tuple|
 #     |=block=|
 #   end
-# - This is equivalent to just specifying an \"execute\" block below.
-# - Note that this syntax only works for a single output stream.
 #  
-# Custom each:
+# # Custom each:
 #   stream.each do
-#      name "name"                          # the name of the operation
-#      emits "stream_1", "stream_2"         # optional for single output stream
-#      output_format :replace               # :replace or :merge, optional, defaults to :replace
-#      prepare |=block=|                    # optional if no initialization needed
-#      execute |=block=|
+#     name "name"                          # the name of the operation
+#     emits "stream_1", "stream_2"         # optional for single output stream
+#     output_format :replace               # :replace or :merge, optional, defaults to :replace
+#     prepare |=block=|                    # optional if no initialization needed
+#     execute |=block=|
 #   end
 #  
-# - The allowed output formats are :replace and :merge.
-#     * :replace - discards the input tuple values and only emits the specified values. This is the default.
-#     * :merge - re-emits the input tuple values along with the specified values.
-# - The "prepare" and "execute" blocks can be in do...end format or {} format.
-#     * the "prepare" block is where any setup is done to prepare for tuple processing in the "execute" block.
-#     * the "execute" block is where the tuples are actually processed. It must take in a single argument (the "tuple").
+# SYNTAX_NOTES 
+# The simplified syntax can be used if a prepare block and other customizations are not needed.
+# 
+#   * **Note:** this syntax only works for a single output stream.
+#  
+# The allowed output formats are :replace and :merge.
+#   * **:replace** : discards the input tuple values and only emits the specified values. This is the default.
+#   * **:merge** : re-emits the input tuple values along with the specified values.
 #  
+# The "prepare" and "execute" blocks can be in do...end format or {} format.
+#   * **"prepare"** block : where any setup is done to prepare for tuple processing in the "execute" block.
+#   * **"execute"** block : where the tuples are actually processed. It must take in a single argument (the "tuple").
+# 
 # EXAMPLE 
 #  
 # # This is a simple example which emits the url of a tuple 
 # s = s.each do |tuple|
 #   emit :url => tuple["url"]
 # end
-#  
-#  
-#  
+#   
 # # Eaches can also be used for conditional emitting
 # stream = result_stream.each{ |tuple|
 #  

data/ruby/lib/zillabyte/harness/filter.rb

@@ -1,32 +1,39 @@
 # OPERATION Filter
+# TAGLINE 
+# The filter is a conditional each. 
 # HIGH_LEVEL_DESCRIPTION 
-# The filter is a conditional each. Conditional expressions can be used to determine if a
+# Conditional expressions can be used to determine if a
 # tuple received by the operation is emitted.
 #  
 # LANGUAGE_SYNTAX 
-#   Simplified syntax:
+#  # Simplified syntax:
 #     stream.filter do |tuple|
 #       |=block=|
 #     end
-#   - The condition for keeping the tuple should be specified in the block. 
-#   - This is equivalent to just specifying a "keep" block below.
-#   Custom filter:
+#  
+# # Extended Syntax
 #     stream.filter do
-#        name "name" \t\t\t => optional
-#        emits "stream_1", "stream_2", ... \t\t => optional for single output stream
-#        prepare |=block=| \t\t\t\t => optional if no initialization needed
-#        keep |=block=|
+#       name "name"                          # optional
+#       emits "stream_1", "stream_2", ...    # optional for single output stream
+#       prepare |=block=|                    # optional if no initialization needed
+#       keep |=block=|
 #     end
-#   - The "prepare" and "keep" blocks can be in do...end format or {} format.
-#       * the "prepare" block is where any setup is done to prepare for tuple processing in the "keep" block.
-#       * tuples will pass through the filter if "keep" returns "True". It must take in a single argument (the "tuple").
 #  
+# SYNTAX_NOTES 
+# The simplified syntax can be used if a prepare block and other customizations are not needed.
+# 
+#   * **Note:** this syntax only works for a single output stream.
+# 
+# The "prepare" and "keep" blocks can be in do...end format or {} format.
+#  * **"prepare"** block : where any setup is done to prepare for tuple processing in the "keep" block.
+#  * **"keep"** block : tuples pass through the filter if "keep" returns "True". It must take in a single argument (the "tuple").
+# 
 # EXAMPLE 
 # # Filter for simple string inclusion
 # stream = stream.filter { |tuple| tuple["url"].include? "hello world" } 
-#   
+# 
 # # Custom Filter
-# stream = stream.filter do\
+# stream = stream.filter do
 #    name "hello_world_filter"
 #    emits "hello_stream"
 #    keep do |tuple|

data/ruby/lib/zillabyte/harness/group_by.rb

@@ -1,26 +1,27 @@
 # OPERATION Group By 
+# Group tuples by keys
 # HIGH_LEVEL_DESCRIPTION 
-# The `group_by` function can be used to implement aggregate computations
-# such as counting and summing. The following is a full example of a
-# word count operation in Zillabyte.
-# 
+# The `group_by` operation can be used to implement aggregation-type functions
+# such as counting and summing.
 # LANGUAGE_SYNTAX 
-#  stream.group_by do
-#      name "name"  \t\t\t\t\t => optional
-#      group_by "field_1", "field_2", ...
-#      emits "stream_1", "stream_2", ... \t\t => optional for single output stream
-#      begin_group |=block=|
-#      aggregate |=block=|
-#      end_group |=block=|
+#   stream.group_by do
+#     name "name"                           # optional
+#     group_by "field_1", "field_2", ...
+#     emits "stream_1", "stream_2", ...     # optional for single output stream
+#     begin_group |=block=|
+#     aggregate |=block=|
+#     end_group |=block=|
 #   end
-# - The "begin_group", "aggregate" and "end_group" blocks can be in do...end format or {} format.
-#     * the "begin_group" block is where the initial values for the aggregation are set. It must take in a single argument (the "grouping tuple", which is emitted at the beginning of each group and contains the values of the fields specified in "group_by").
-#     * the "aggregate" block is where the aggregation is performed. It must take in a single argument (the "tuple").
-#     * the "end_group" block is where the final aggregated value is emitted.
+# SYNTAX_NOTES 
+#  
+# The "begin_group", "aggregate" and "end_group" blocks can be in do...end format or {} format.
+#   * **"begin_group"** block : where the initial values for the aggregation are set. It must take in a single argument (the "grouping tuple", which is emitted at the beginning of each group and contains the values of the fields specified in "group_by").
+#   * **"aggregate"** block : where the aggregation is performed. It must take in a single argument (the "tuple").
+#   * **"end_group"** block : where the final aggregated value is emitted.
 # EXAMPLE 
 # # Declare the group_by, grouping on the :word field
 # stream = stream.group_by(:word) do
-#
+# 
 #   # Save the word being grouped, initialize any state
 #   begin_group do |g_tuple|
 #     @word = g_tuple[:word]

data/ruby/lib/zillabyte/harness/injected_component.rb

@@ -1,3 +1,64 @@
+# OPERATION Call Component 
+# TAGLINE 
+# Call a component
+# HIGH_LEVEL_DESCRIPTION 
+# Components can be embedded into other components or even applications. 
+# When running an application that has an embedded component, that component is 
+# joined into the same execution environment as its parent, allowing for high throughput component execution.
+# To embed a component, use the "call_component" operation
+# LANGUAGE_SYNTAX 
+#   stream.call_component do
+#     component_id "component_id"
+#     name "name"                                           # optional
+#     additional_inputs other_input_stream_object_1, ...    # blank if none
+#     outputs "output_stream_name_1", ...
+#     output_format :replace OR :merge                      # optional, defaults to :replace
+#   end 
+# SYNTAX_NOTES 
+# The "component_id" MUST be given and correspond to the name or id listed in the output of "zillabyte components".
+#  
+# The allowed output formats are :replace and :merge. Note that only linear components, i.e. those with only "each" and "filter" operations support :merge.
+#   * **:replace** : discards the input tuple values and only emits the output values. This is the default.
+#   * **:merge** : re-emits the input tuple values along with the output values.
+#  
+# To correctly stich in the component, the implicit assumptions below WILL BE USED:
+#   * The "stream" that "call_component" is invoked on MUST correspond to the first listed input stream to the component.
+#   * The streams specified in "additional_inputs" MUST correspond to the other listed input streams in order.
+#   * Tuples emitted from the preceeding operation MUST contain the fields listed for the corresponding component input streams.
+#   * The streams specified in "outputs" must correspond to the listed output streams to the component in order.
+#   * The number of input and output streams specified must match the number listed for the component.
+#  
+# EXAMPLE 
+# require 'zillabyte'
+# app = Zillabyte.app("plural_app")
+#  
+# stream = app.source do
+#  
+#   DICTIONARY = %w(apple butterfly bus) unless defined? DICTIONARY
+#   end_cycle_policy :explicit
+#  
+#   begin_cycle do
+#     @count = 0
+#   end
+#  
+#   next_tuple do
+#     emit "word" => DICTIONARY.sample
+#     @count += 1
+#     end_cycle if @count == 5
+#   end
+#  
+# end
+#  
+# # Here we call the "pluralize" component
+# stream = stream.call_component do
+#   component_id "pluralize"
+#   outputs "pluralized_words"
+# end
+#  
+# stream.sink do
+#   name "plural_words"
+#   column "word", :string
+# end
 class Zillabyte::Harness::InjectedComponent
   attr_accessor :_app, :_node, :_id, :_input_stream_1, :_options
 

data/ruby/lib/zillabyte/harness/join.rb

@@ -1,19 +1,22 @@
 # OPERATION Join 
+# Join streams together 
 # HIGH_LEVEL_DESCRIPTION 
 # Joins allow you to combine two streams into one in a manner similar
 # to traditional relational systems. Realtime tuple data from a variety
 # of sources can be combined into a single stream via joins on specified
-# fields. Here is an example of a `join` in Zillabyte.
+# fields. 
 # LANGUAGE_SYNTAX 
 #   lhs_stream.join_with( rhs_stream_object, options )
-# - Options should be specified as a hash. The following keys are recognized:
-#     Mandatory:
-#     * "on" -- specifies the fields to join on. The value must be a STRING or a length-2 ARRAY. 
-#               If value = a STRING, the LH and RH join fields will both be set to this STRING. 
-#               If value = a length-2 ARRAY, the LH join field will be set to array[0] and the RH join field will be set to array[1].
-#     Optional:
-#     * "type" -- specifies the join type. The default is :left. Options are [:inner, :outer, :left, :right]
-#     * "emits" -- specifies the stream to emit. A "join" may only emit a single stream.
+# SYNTAX_NOTES  
+# Options should be specified as a hash. The following keys are recognized:
+#  
+# Mandatory:
+# 
+#  * **"on"** : specifies the fields to join on. The value must be a STRING or a length-2 ARRAY. If value = a STRING, the LH and RH join fields will both be set to this STRING. If value = a length-2 ARRAY, the LH join field will be set to array[0] and the RH join field will be set to array[1].
+# 
+# Optional:
+#  * **"type"** : specifies the join type. The default is :left. Options are [:inner, :outer, :left, :right]
+#  * **"emits"** : specifies the stream to emit. A "join" may only emit a single stream.
 # EXAMPLE 
 #  
 # # Left join on a shared field, in this case, an :owner_id
@@ -25,7 +28,6 @@
 #   :on => [:average, :number], 
 #   :type => :inner
 # )
-#
 class Zillabyte::Harness::Join
   attr_accessor :_app, :_node, :_args
 

data/ruby/lib/zillabyte/harness/sink.rb

@@ -1,18 +1,24 @@
 # OPERATION Sink 
-# HIGH_LEVEL_DESCRIPTION 
+# TAGLINE 
 # The `sink` is a passive operation that defines the schema of the rows that need to be saved. 
-# Of all the operations where a stream is consumed, only the `sink` requires a schema to be defined. 
+# HIGH_LEVEL_DESCRIPTION 
+# The `sink` marks the end of a stream. It saves the data it receives according to the schema defined within it.
 #  
 # LANGUAGE_SYNTAX 
 #   stream.sink do
-#      name "name_of_relation"
-#      column "field_1", :type_1
-#      column "field_2", :type_2 ...
+#     name "name_of_relation"
+#     column "field_1", :type_1
+#     column "field_2", :type_2 ...
 #   end
-# - "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
-# - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
-# - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
-# - Field types must be SYMBOLS. The following types are allowed [:string, :integer, :float, :double, :boolean, :array, :map]
+#  
+# SYNTAX_NOTES 
+# "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
+#  
+# Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
+#  
+# Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
+#  
+# Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, :boolean, :array and :map.
 #  
 # EXAMPLE 
 # stream.sink{

data/ruby/lib/zillabyte/harness/source.rb

@@ -1,37 +1,46 @@
 # OPERATION Source
-# HIGH_LEVEL_DESCRIPTION 
-# A `source` is where the data for your app originates and is defined
-# on the app object. The easiest way to stream data into a Zillabyte
-# app is to use a Zillabyte dataset. A simple `source` takes in the
+# TAGLINE 
+# A source is where the data for your app originates and is defined
+# on the app object. 
+# 
+# HIGH_LEVEL_DESCRIPTION  
+# 
+# ## Sourcing From Datasets
+# The easiest way to stream data into a Zillabyte
+# app is to use a Zillabyte dataset. A simple source takes in the
 # name of a dataset and produces a stream object. 
 #  
-# A `source` can also use data outside of the public datasets Zillabyte
+# ## Custom Sources
+# A source can also use data outside of the public datasets Zillabyte
 # provides. Any external data available on the web can be streamed
 # into a Zillabyte app. To do so, we use the expanded syntax of a
-# `source` and include the code needed to generate the `source` data
-# in the `next_tuple` block. The expanded syntax also allows for
-# additional customizations such as providing a `name` for the `source`
+# source and include the code needed to generate the source data
+# in the next_tuple block. The expanded syntax also allows for
+# additional customizations such as providing a name for the source
 # and specifying any preparatory steps such as initializing global
-# values in the `begin_cycle` block.
-#  
+# values in the begin_cycle block.
+# 
 # LANGUAGE_SYNTAX 
-#  
-#   Sourcing from a dataset:
-#     app.source("dataset_name")
-#   Custom source:
-#     app.source do
-#        name "name" \t\t\t\t\t => optional
-#        emits "stream_1", "stream_2", ... \t\t => optional for single output stream
-#        end_cycle_policy :null_emit OR :explicit \t => default :null_emit
-#        begin_cycle |=block=| \t\t\t\t => optional if no initialization needed
-#        next_tuple |=block=|
-#     end
-#   - The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
-#       * :null_emit - end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
-#       * :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
-#   - The "begin_cycle" and "next_tuple" blocks can be in do...end format or {} format.
-#       * the "begin_cycle" block is where any setup is done to initialize the content and quantity of tuples emitted by the "next_tuple" block.
-#       * the "next_tuple" block is where the tuples are actually emitted.
+# #Sourcing from a dataset:
+#   app.source("dataset_name") # fetch a dataset from Zillabyte
+# 
+# #Custom source:
+#   app.source do
+#     name "name"                                 # optional
+#     emits "stream_1", "stream_2"                # optional for single output stream
+#     end_cycle_policy :null_emit OR :explicit    # default :null_emit
+#     begin_cycle |=block=|                       # optional if no initialization needed
+#     next_tuple |=block=|                        # The code to execute on each tuple fetch
+#   end
+# SYNTAX_NOTES 
+# 
+# The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
+#   * **:null_emit** : end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
+#   * **:explicit** : the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
+# 
+# The "begin_cycle" and "next_tuple" blocks can be in do...end format or {} format.
+#   * **"begin_cycle"** block : where any setup is done to initialize the content and quantity of tuples emitted by the "next_tuple" block.
+#   * **"next_tuple"** block : where the tuples are actually emitted.
 # EXAMPLE 
 #  
 # # Source from the "homepages" relation

data/ruby/lib/zillabyte/version.rb

@@ -1,3 +1,3 @@
 module Zillabyte
-  VERSION = "0.9.30" unless defined?(VERSION)
+  VERSION = "0.9.31" unless defined?(VERSION)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zillabyte
 version: !ruby/object:Gem::Version
-  version: 0.9.30
+  version: 0.9.31
 platform: ruby
 authors:
 - zillabyte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-14 00:00:00.000000000 Z
+date: 2014-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.30
+        version: 0.9.31
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.9.30
+        version: 0.9.31
 description: The Official Zillabyte Gem
 email:
 - gem@zillabyte.com