burner 1.0.0.pre.alpha.7 → 1.0.0.pre.alpha.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8cb851b5c317d3566c1b3b77c9c904e26970de8e0ce4329e2c8b83336c372b02
-  data.tar.gz: 0ad28e56c8b52ceede4ed321812e45edfaef2bcd4000428738c348e5e849092a
+  metadata.gz: d682634e7c0c8b850b98d80a33e4e44012a3f7ae058b72e3c828830d7e177a0a
+  data.tar.gz: afc1daecbec70a6ec88f19ab11ea92f93f28a7dd8bc8ea4821cb6a222db3f824
 SHA512:
-  metadata.gz: cf5bc810ffa5dba106e6538dbab2f44e6763f5ad073da2b645e46eff3e976e61f9a0d551ae4d5ad9998c5c1d87b5e2913b902901020ad77f20b4c29800321df5
-  data.tar.gz: ef83de8173fbad28c2cc07c44f101175ebe9c60d5a9275d2e7280ba1b621a8c51082504ae5d5c917916d398a7a89d06ca7416dd08ba7148ebebb87cd42dd7b05
+  metadata.gz: d75499b61c0b30b4882fb8cf10c74ea2cd0dfa8562824bf3a0a61c3311b78808eaeb5e0dfc1db1faa81400a23a998e4dd6b37ba35ad170ec2945c8708804d7c7
+  data.tar.gz: 42e5b5d6fead52793eefa7ef6f41423254e870e0fd319d7aad8b011c88c8048bbaec78cac79d0acdeed98f446777014b46e419ed86e39ea0f7fd9f888c786a4d

data/README.md

@@ -31,30 +31,30 @@ pipeline = {
   jobs: [
     {
       name: :read,
-      type: 'io/read',
+      type: 'b/io/read',
       path: '{input_file}'
     },
     {
       name: :output_id,
-      type: :echo,
+      type: 'b/echo',
       message: 'The job id is: {__id}'
     },
     {
       name: :output_value,
-      type: :echo,
+      type: 'b/echo',
       message: 'The current value is: {__value}'
     },
     {
       name: :parse,
-      type: 'deserialize/json'
+      type: 'b/deserialize/json'
     },
     {
       name: :convert,
-      type: 'serialize/yaml'
+      type: 'b/serialize/yaml'
     },
     {
       name: :write,
-      type: 'io/write',
+      type: 'b/io/write',
       path: '{output_file}'
     }
   ],
@@ -172,25 +172,25 @@ Write the following json_to_yaml_pipeline.yaml file to disk:
 ````yaml
 jobs:
   - name: read
-    type: io/read
+    type: b/io/read
     path: '{input_file}'
 
   - name: output_id
-    type: echo
+    type: b/echo
     message: 'The job id is: {__id}'
 
   - name: output_value
-    type: echo
+    type: b/echo
     message: 'The current value is: {__value}'
 
   - name: parse
-    type: deserialize/json
+    type: b/deserialize/json
 
   - name: convert
-    type: serialize/yaml
+    type: b/serialize/yaml
 
   - name: write
-    type: io/write
+    type: b/io/write
     path: '{output_file}'
 
 steps:
@@ -233,39 +233,44 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 
 #### Collection
 
-* **collection/arrays_to_objects** [mappings, register]: Convert an array of arrays to an array of objects.
-* **collection/graph** [config, key, register]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
-* **collection/objects_to_arrays** [mappings, register]: Convert an array of objects to an array of arrays.
-* **collection/shift** [amount, register]: Remove the first N number of elements from an array.
-* **collection/transform** [attributes, exclusive, separator, register]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.
-* **collection/unpivot** [pivot_set, register]: Take an array of objects and unpivot specific sets of keys into rows.  Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
-* **collection/validate** [invalid_register, join_char, message_key, register, separator, validations]: Take an array of objects, run it through each declared validator, and split the objects into two registers.  The valid objects will be split into the current register while the invalid ones will go into the invalid_register as declared.  Optional arguments, join_char and message_key, help determine the compiled error messages.  The separator option can be utilized to use dot-notation for validating keys.  See each validation's options by viewing their classes within the `lib/modeling/validations` directory.
-* **collection/values** [include_keys, register]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
+* **b/collection/arrays_to_objects** [mappings, register]: Convert an array of arrays to an array of objects.
+* **b/collection/concatenate** [from_registers, to_register]: Concatenate each from_register's value and place the newly concatenated array into the to_register.  Note: this does not do any deep copying and should be assumed it is shallow copying all objects.
+* **b/collection/graph** [config, key, register]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
+* **b/collection/objects_to_arrays** [mappings, register]: Convert an array of objects to an array of arrays.
+* **b/collection/shift** [amount, register]: Remove the first N number of elements from an array.
+* **b/collection/transform** [attributes, exclusive, separator, register]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.
+* **b/collection/unpivot** [pivot_set, register]: Take an array of objects and unpivot specific sets of keys into rows.  Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
+* **b/collection/validate** [invalid_register, join_char, message_key, register, separator, validations]: Take an array of objects, run it through each declared validator, and split the objects into two registers.  The valid objects will be split into the current register while the invalid ones will go into the invalid_register as declared.  Optional arguments, join_char and message_key, help determine the compiled error messages.  The separator option can be utilized to use dot-notation for validating keys.  See each validation's options by viewing their classes within the `lib/modeling/validations` directory.
+* **b/collection/values** [include_keys, register]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
 
 #### De-serialization
 
-* **deserialize/csv** [register]: Take a CSV string and de-serialize into object(s).  Currently it will return an array of arrays, with each nested array representing one row.
-* **deserialize/json** [register]: Treat input as a string and de-serialize it to JSON.
-* **deserialize/yaml** [register, safe]: Treat input as a string and de-serialize it to YAML.  By default it will try and [safely de-serialize](https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load) it (only using core classes).  If you wish to de-serialize it to any class type, pass in `safe: false`
+* **b/deserialize/csv** [register]: Take a CSV string and de-serialize into object(s).  Currently it will return an array of arrays, with each nested array representing one row.
+* **b/deserialize/json** [register]: Treat input as a string and de-serialize it to JSON.
+* **b/deserialize/yaml** [register, safe]: Treat input as a string and de-serialize it to YAML.  By default it will try and [safely de-serialize](https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load) it (only using core classes).  If you wish to de-serialize it to any class type, pass in `safe: false`
 
 #### IO
 
-* **io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
-* **io/read** [binary, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
-* **io/write** [binary, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
+* **b/io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
+* **b/io/read** [binary, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
+* **b/io/write** [binary, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
 
 #### Serialization
 
-* **serialize/csv** [register]: Take an array of arrays and create a CSV.
-* **serialize/json** [register]: Convert value to JSON.
-* **serialize/yaml** [register]: Convert value to YAML.
+* **b/serialize/csv** [register]: Take an array of arrays and create a CSV.
+* **b/serialize/json** [register]: Convert value to JSON.
+* **b/serialize/yaml** [register]: Convert value to YAML.
+
+#### Value
+
+* **b/value/copy** [from_register, to_register]: Copy from_register's value into the to_register.  Note: this does not do any deep copying and should be assumed it is shallow copying all objects.
+* **b/value/static** [register, value]: Set the value to any arbitrary value.
 
 #### General
 
-* **dummy** []: Do nothing
-* **echo** [message]: Write a message to the output.  The message parameter can be interpolated using  `Payload#params`.
-* **set** [register, value]: Set the value to any arbitrary value.
-* **sleep** [seconds]: Sleep the thread for X number of seconds.
+* **b/echo** [message]: Write a message to the output.  The message parameter can be interpolated using  `Payload#params`.
+* **b/nothing** []: Do nothing.
+* **b/sleep** [seconds]: Sleep the thread for X number of seconds.
 
 Notes:
 
@@ -296,17 +301,17 @@ pipeline = {
   jobs: [
     {
       name: :read,
-      type: 'io/read',
+      type: 'b/io/read',
       path: '{input_file}'
     },
     {
       name: :output_id,
-      type: :echo,
+      type: 'b/echo',
       message: 'The job id is: {__id}'
     },
     {
       name: :output_value,
-      type: :echo,
+      type: 'b/echo',
       message: 'The current value is: {__value}'
     },
     {
@@ -315,11 +320,11 @@ pipeline = {
     },
     {
       name: :convert,
-      type: 'serialize/yaml'
+      type: 'b/serialize/yaml'
     },
     {
       name: :write,
-      type: 'io/write',
+      type: 'b/io/write',
       path: '{output_file}'
     }
   ],

data/burner.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.summary     = 'Declarative and extendable processing pipeline'
 
   s.description = <<-DESCRIPTION
-    This library serves as the skeleton for a processing engine.  It allows you to organize your code into Jobs, then stitch those jobs together as steps.
+    This library serves as the backbone for a configurable processing engine.  It allows you to organize your code into jobs, then stitch those jobs together as steps.
   DESCRIPTION
 
   s.authors     = ['Matthew Ruggio']

data/lib/burner/job.rb

@@ -31,10 +31,11 @@ module Burner
     # The #perform method takes in two arguments: output (an instance of Burner::Output)
     # and payload (an instance of Burner::Payload).  Jobs can leverage output to emit
     # information to the pipeline's log(s).  The payload is utilized to pass data from job to job,
-    # with its most important attribute being #value.  The value attribute is mutable
-    # per the individual job's context (meaning of it is unknown without understanding a job's
-    # input and output value of #value.).  Therefore #value can mean anything and it is up to the
-    # engineers to clearly document the assumptions of its use.
+    # with its most important attribute being #registers.  The registers attribute is a mutable
+    # and accessible hash per the individual job's context
+    # (meaning of it is unknown without understanding a job's input and output value
+    # of #registers.).  Therefore #register key values can mean anything
+    # and it is up to consumers to clearly document the assumptions of its use.
     #
     # Returning false will short-circuit the pipeline right after the job method exits.
     # Returning anything else besides false just means "continue".
@@ -47,9 +48,15 @@ module Burner
     protected
 
     def job_string_template(expression, output, payload)
-      templatable_params = payload.params.merge(__id: output.id, __value: payload[''])
+      templatable_params = payload.params
+                                  .merge(__id: output.id)
+                                  .merge(templatable_register_values(payload))
 
       Util::StringTemplate.instance.evaluate(expression, templatable_params)
     end
+
+    def templatable_register_values(payload)
+      payload.registers.transform_keys { |key| "__#{key}_register" }
+    end
   end
 end

data/lib/burner/jobs.rb

@@ -16,25 +16,35 @@ module Burner
   class Jobs
     acts_as_hashable_factory
 
-    register 'collection/arrays_to_objects', Library::Collection::ArraysToObjects
-    register 'collection/graph',             Library::Collection::Graph
-    register 'collection/objects_to_arrays', Library::Collection::ObjectsToArrays
-    register 'collection/shift',             Library::Collection::Shift
-    register 'collection/transform',         Library::Collection::Transform
-    register 'collection/unpivot',           Library::Collection::Unpivot
-    register 'collection/values',            Library::Collection::Values
-    register 'deserialize/csv',              Library::Deserialize::Csv
-    register 'deserialize/json',             Library::Deserialize::Json
-    register 'deserialize/yaml',             Library::Deserialize::Yaml
-    register 'dummy', '',                    Library::Dummy
-    register 'echo',                         Library::Echo
-    register 'io/exist',                     Library::IO::Exist
-    register 'io/read',                      Library::IO::Read
-    register 'io/write',                     Library::IO::Write
-    register 'serialize/csv',                Library::Serialize::Csv
-    register 'serialize/json',               Library::Serialize::Json
-    register 'serialize/yaml',               Library::Serialize::Yaml
-    register 'set_value',                    Library::SetValue
-    register 'sleep',                        Library::Sleep
+    # Nothing is the default as noted by the ''.  This means if a type is omitted, nil, or blank
+    # string then the nothing job will be used.
+    register 'b/echo',                         Library::Echo
+    register 'b/nothing', '',                  Library::Nothing
+    register 'b/sleep',                        Library::Sleep
+
+    register 'b/collection/arrays_to_objects', Library::Collection::ArraysToObjects
+    register 'b/collection/concatenate',       Library::Collection::Concatenate
+    register 'b/collection/graph',             Library::Collection::Graph
+    register 'b/collection/objects_to_arrays', Library::Collection::ObjectsToArrays
+    register 'b/collection/shift',             Library::Collection::Shift
+    register 'b/collection/transform',         Library::Collection::Transform
+    register 'b/collection/unpivot',           Library::Collection::Unpivot
+    register 'b/collection/values',            Library::Collection::Values
+    register 'b/collection/validate',          Library::Collection::Validate
+
+    register 'b/deserialize/csv',              Library::Deserialize::Csv
+    register 'b/deserialize/json',             Library::Deserialize::Json
+    register 'b/deserialize/yaml',             Library::Deserialize::Yaml
+
+    register 'b/io/exist',                     Library::IO::Exist
+    register 'b/io/read',                      Library::IO::Read
+    register 'b/io/write',                     Library::IO::Write
+
+    register 'b/serialize/csv',                Library::Serialize::Csv
+    register 'b/serialize/json',               Library::Serialize::Json
+    register 'b/serialize/yaml',               Library::Serialize::Yaml
+
+    register 'b/value/copy',                   Library::Value::Copy
+    register 'b/value/static',                 Library::Value::Static
   end
 end

data/lib/burner/library.rb

@@ -9,7 +9,12 @@
 
 require_relative 'job_with_register'
 
+require_relative 'library/echo'
+require_relative 'library/nothing'
+require_relative 'library/sleep'
+
 require_relative 'library/collection/arrays_to_objects'
+require_relative 'library/collection/concatenate'
 require_relative 'library/collection/graph'
 require_relative 'library/collection/objects_to_arrays'
 require_relative 'library/collection/shift'
@@ -17,16 +22,18 @@ require_relative 'library/collection/transform'
 require_relative 'library/collection/unpivot'
 require_relative 'library/collection/validate'
 require_relative 'library/collection/values'
+
 require_relative 'library/deserialize/csv'
 require_relative 'library/deserialize/json'
 require_relative 'library/deserialize/yaml'
-require_relative 'library/dummy'
-require_relative 'library/echo'
+
 require_relative 'library/io/exist'
 require_relative 'library/io/read'
 require_relative 'library/io/write'
+
 require_relative 'library/serialize/csv'
 require_relative 'library/serialize/json'
 require_relative 'library/serialize/yaml'
-require_relative 'library/set_value'
-require_relative 'library/sleep'
+
+require_relative 'library/value/copy'
+require_relative 'library/value/static'

data/lib/burner/library/collection/arrays_to_objects.rb

@@ -14,8 +14,8 @@ module Burner
       # Burner::Modeling::KeyIndexMapping instances or hashable configurations which specifies
       # the index-to-key mappings to use.
       #
-      # Expected Payload#value input: array of arrays.
-      # Payload#value output: An array of hashes.
+      # Expected Payload[register] input: array of arrays.
+      # Payload[register] output: An array of hashes.
       #
       # An example using a configuration-first pipeline:
       #
@@ -23,14 +23,14 @@ module Burner
       #     jobs: [
       #       {
       #         name: 'set',
-      #         type: 'set_value',
+      #         type: 'b/value/static',
       #         value: [
       #           [1, 'funky']
       #         ]
       #       },
       #       {
       #         name: 'map',
-      #         type: 'collection/arrays_to_objects',
+      #         type: 'b/collection/arrays_to_objects',
       #         mappings: [
       #           { index: 0, key: 'id' },
       #           { index: 1, key: 'name' }
@@ -38,7 +38,7 @@ module Burner
       #       },
       #       {
       #         name: 'output',
-      #         type: 'echo',
+      #         type: 'b/echo',
       #         message: 'value is currently: {__value}'
       #       },
       #
@@ -47,6 +47,11 @@ module Burner
       #   }
       #
       #   Burner::Pipeline.make(config).execute
+      #
+      # Given the above example, the expected output would be:
+      #  [
+      #    { 'id' => 1, 'name' => 'funky' }
+      #  ]
       class ArraysToObjects < JobWithRegister
         attr_reader :mappings
 

data/lib/burner/library/collection/concatenate.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # Take the list of from_registers and concatenate each of their values together.
+      # Each from_value will be coerced into an array if it not an array.
+      #
+      # Expected Payload[from_register] input: array of objects.
+      # Payload[to_register] output: An array of objects.
+      class Concatenate < Job
+        attr_reader :from_registers, :to_register
+
+        def initialize(name:, from_registers: [], to_register: '')
+          super(name: name)
+
+          @from_registers = Array(from_registers)
+          @to_register    = to_register.to_s
+
+          freeze
+        end
+
+        def perform(output, payload)
+          output.detail("Concatenating registers: '#{from_registers}' to: '#{to_register}'")
+
+          payload[to_register] = from_registers.each_with_object([]) do |from_register, memo|
+            from_register_value = array(payload[from_register])
+
+            memo.concat(from_register_value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/collection/graph.rb

@@ -13,8 +13,8 @@ module Burner
       # Take an array of (denormalized) objects and create an object hierarchy from them.
       # Under the hood it uses Hashematics: https://github.com/bluemarblepayroll/hashematics.
       #
-      # Expected Payload#value input: array of objects.
-      # Payload#value output: An array of objects.
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: An array of objects.
       class Graph < JobWithRegister
         attr_reader :key, :groups
 

data/lib/burner/library/collection/objects_to_arrays.rb

@@ -15,39 +15,41 @@ module Burner
       # Burner::Modeling::KeyIndexMapping instances or hashable configurations which specifies
       # the key-to-index mappings to use.
       #
-      # Expected Payload#value input: array of hashes.
-      # Payload#value output: An array of arrays.
+      # Expected Payload[register] input: array of hashes.
+      # Payload[register] output: An array of arrays.
       #
       # An example using a configuration-first pipeline:
       #
-      #   config = {
-      #     jobs: [
-      #       {
-      #         name: 'set',
-      #         type: 'set_value',
-      #         value: [
-      #           [1, 'funky']
-      #         ]
-      #       },
-      #       {
-      #         name: 'map',
-      #         type: 'collection/objects_to_arrays',
-      #         mappings: [
-      #           { index: 0, key: 'id' },
-      #           { index: 1, key: 'name' }
-      #         ]
-      #       },
-      #       {
-      #         name: 'output',
-      #         type: 'echo',
-      #         message: 'value is currently: {__value}'
-      #       },
+      #  config = {
+      #    jobs: [
+      #      {
+      #        name: 'set',
+      #        type: 'b/value/static',
+      #        value: [
+      #          { 'id' => 1, 'name' => 'funky' }
+      #        ],
+      #        register: register
+      #      },
+      #      {
+      #        name: 'map',
+      #        type: 'b/collection/objects_to_arrays',
+      #        mappings: [
+      #          { index: 0, key: 'id' },
+      #          { index: 1, key: 'name' }
+      #        ],
+      #        register: register
+      #      },
+      #      {
+      #        name: 'output',
+      #        type: 'b/echo',
+      #        message: 'value is currently: {__value}'
+      #      },
       #
-      #     ],
-      #     steps: %w[set map output]
-      #   }
+      #    ],
+      #    steps: %w[set map output]
+      #  }
       #
-      #   Burner::Pipeline.make(config).execute
+      #  Burner::Pipeline.make(config).execute
       class ObjectsToArrays < JobWithRegister
         attr_reader :mappings
 

data/lib/burner/library/collection/shift.rb

@@ -14,8 +14,8 @@ module Burner
       # attribute.  The initial use case for this was to remove "header" rows from arrays,
       # like you would expect when parsing CSV files.
       #
-      # Expected Payload#value input: nothing.
-      # Payload#value output: An array with N beginning elements removed.
+      # Expected Payload[register] input: nothing.
+      # Payload[register] output: An array with N beginning elements removed.
       class Shift < JobWithRegister
         DEFAULT_AMOUNT = 0
 

data/lib/burner/library/collection/transform.rb

@@ -18,8 +18,8 @@ module Burner
       # For more information on the specific contract for attributes, see the
       # Burner::Modeling::Attribute class.
       #
-      # Expected Payload#value input: array of objects.
-      # Payload#value output: An array of objects.
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: An array of objects.
       class Transform < JobWithRegister
         BLANK = ''
 

data/lib/burner/library/collection/unpivot.rb

@@ -14,8 +14,8 @@ module Burner
       # Under the hood it uses HashMath's Unpivot class:
       # https://github.com/bluemarblepayroll/hash_math
       #
-      # Expected Payload#value input: array of objects.
-      # Payload#value output: An array of objects.
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: An array of objects.
       class Unpivot < JobWithRegister
         attr_reader :unpivot
 

data/lib/burner/library/collection/validate.rb

@@ -14,8 +14,9 @@ module Burner
       # of validations.  The main register will include the valid objects and the invalid_register
       # will contain the invalid objects.
       #
-      # Expected Payload#value input: array of objects.
-      # Payload#value output: An array of objects.
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: An array of objects that are valid.
+      # Payload[invalid_register] output: An array of objects that are invalid.
       class Validate < JobWithRegister
         DEFAULT_INVALID_REGISTER = 'invalid'
         DEFAULT_JOIN_CHAR        = ', '

data/lib/burner/library/collection/values.rb

@@ -14,8 +14,8 @@ module Burner
       # If include_keys is true (it is false by default), then call #keys on the first
       # object and inject that as a "header" object.
       #
-      # Expected Payload#value input: array of objects.
-      # Payload#value output: An array of arrays.
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: An array of arrays.
       class Values < JobWithRegister
         attr_reader :include_keys
 

data/lib/burner/library/deserialize/csv.rb

@@ -12,8 +12,8 @@ module Burner
     module Deserialize
       # Take a CSV string and de-serialize into object(s).
       #
-      # Expected Payload#value input: nothing.
-      # Payload#value output: an array of arrays.  Each inner array represents one data row.
+      # Expected Payload[register] input: nothing.
+      # Payload[register] output: an array of arrays.  Each inner array represents one data row.
       class Csv < JobWithRegister
         # This currently only supports returning an array of arrays, including the header row.
         # In the future this could be extended to offer more customizable options, such as

data/lib/burner/library/deserialize/json.rb

@@ -12,8 +12,8 @@ module Burner
     module Deserialize
       # Take a JSON string and deserialize into object(s).
       #
-      # Expected Payload#value input: string of JSON data.
-      # Payload#value output: anything, as specified by the JSON de-serializer.
+      # Expected Payload[register] input: string of JSON data.
+      # Payload[register] output: anything, as specified by the JSON de-serializer.
       class Json < JobWithRegister
         def perform(_output, payload)
           payload[register] = JSON.parse(payload[register])

data/lib/burner/library/deserialize/yaml.rb

@@ -15,8 +15,8 @@ module Burner
       # YAML.  If you wish to ease this restriction, for example if you have custom serialization
       # for custom classes, then you can pass in safe: false.
       #
-      # Expected Payload#value input: string of YAML data.
-      # Payload#value output: anything as specified by the YAML de-serializer.
+      # Expected Payload[register] input: string of YAML data.
+      # Payload[register]output: anything as specified by the YAML de-serializer.
       class Yaml < JobWithRegister
         attr_reader :safe
 

data/lib/burner/library/echo.rb

@@ -11,7 +11,7 @@ module Burner
   module Library
     # Output a simple message to the output.
     #
-    # Note: this does not use Payload#value.
+    # Note: this does not use Payload#registers.
     class Echo < Job
       attr_reader :message
 

data/lib/burner/library/io/exist.rb

@@ -15,7 +15,7 @@ module Burner
       # Check to see if a file exists.  If short_circuit is set to true and the file
       # does not exist then the job will return false and short circuit the pipeline.
       #
-      # Note: this does not use Payload#value.
+      # Note: this does not use Payload#registers.
       class Exist < Job
         attr_reader :path, :short_circuit
 

data/lib/burner/library/io/read.rb

@@ -14,8 +14,8 @@ module Burner
     module IO
       # Read value from disk.
       #
-      # Expected Payload#value input: nothing.
-      # Payload#value output: contents of the specified file.
+      # Expected Payload[register] input: nothing.
+      # Payload[register] output: contents of the specified file.
       class Read < Base
         attr_reader :binary
 

data/lib/burner/library/io/write.rb

@@ -14,8 +14,8 @@ module Burner
     module IO
       # Write value to disk.
       #
-      # Expected Payload#value input: anything.
-      # Payload#value output: whatever was passed in.
+      # Expected Payload[register] input: anything.
+      # Payload[register] output: whatever was passed in.
       class Write < Base
         attr_reader :binary
 

data/lib/burner/library/{dummy.rb → nothing.rb}

@@ -11,8 +11,8 @@ module Burner
   module Library
     # Do nothing.
     #
-    # Note: this does not use Payload#value.
-    class Dummy < Job
+    # Note: this does not use Payload#registers.
+    class Nothing < Job
       def perform(_output, _payload); end
     end
   end

data/lib/burner/library/serialize/csv.rb

@@ -12,8 +12,8 @@ module Burner
     module Serialize
       # Take an array of arrays and create a CSV.
       #
-      # Expected Payload#value input: array of arrays.
-      # Payload#value output: a serialized CSV string.
+      # Expected Payload[register] input: array of arrays.
+      # Payload[register] output: a serialized CSV string.
       class Csv < JobWithRegister
         def perform(_output, payload)
           payload[register] = CSV.generate(options) do |csv|

data/lib/burner/library/serialize/json.rb

@@ -12,8 +12,8 @@ module Burner
     module Serialize
       # Treat value like a Ruby object and serialize it using JSON.
       #
-      # Expected Payload#value input: anything.
-      # Payload#value output: string representing the output of the JSON serializer.
+      # Expected Payload[register] input: anything.
+      # Payload[register] output: string representing the output of the JSON serializer.
       class Json < JobWithRegister
         def perform(_output, payload)
           payload[register] = payload[register].to_json

data/lib/burner/library/serialize/yaml.rb

@@ -12,8 +12,8 @@ module Burner
     module Serialize
       # Treat value like a Ruby object and serialize it using YAML.
       #
-      # Expected Payload#value input: anything.
-      # Payload#value output: string representing the output of the YAML serializer.
+      # Expected Payload[register] input: anything.
+      # Payload[register] output: string representing the output of the YAML serializer.
       class Yaml < JobWithRegister
         def perform(_output, payload)
           payload[register] = payload[register].to_yaml

data/lib/burner/library/sleep.rb

@@ -11,7 +11,7 @@ module Burner
   module Library
     # Arbitrarily put thread to sleep for X number of seconds
     #
-    # Payload#value output: whatever value was specified in this job.
+    # Note: this does not use Payload#registers.
     class Sleep < Job
       attr_reader :seconds
 

data/lib/burner/library/value/copy.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Value
+      # Copy one value in a register to another.  Note that this does *not* perform any type of
+      # deep copy, it simply points one register's value to another.  If you decide to later mutate
+      # one register, you are *not* guaranteed to not mutate the other.
+      #
+      # Expected Payload[from_register] input: anything.
+      # Payload[to_register] output: whatever value was specified in the from_register.
+      class Copy < Job
+        attr_reader :from_register, :to_register
+
+        def initialize(name:, to_register: '', from_register: '')
+          super(name: name)
+
+          @from_register = from_register.to_s
+          @to_register   = to_register.to_s
+
+          freeze
+        end
+
+        def perform(output, payload)
+          output.detail("Copying register: '#{from_register}' to: '#{to_register}'")
+
+          payload[to_register] = payload[from_register]
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/value/static.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Value
+      # Arbitrarily set the value of a register.
+      #
+      # Expected Payload[register] input: anything.
+      # Payload[register] output: whatever value was specified in this job.
+      class Static < JobWithRegister
+        attr_reader :value
+
+        def initialize(name:, register: '', value: nil)
+          super(name: name, register: register)
+
+          @value = value
+
+          freeze
+        end
+
+        def perform(_output, payload)
+          payload[register] = value
+        end
+      end
+    end
+  end
+end

data/lib/burner/modeling/attribute.rb

@@ -10,7 +10,9 @@
 module Burner
   module Modeling
     # Defines a top-level key and the associated transformers for deriving the final value
-    # to set the key to.
+    # to set the key to.  The transformers that can be passed in can be any Realize::Transformers
+    # subclasses.  For more information, see the Realize library at:
+    # https://github.com/bluemarblepayroll/realize
     class Attribute
       acts_as_hashable
 

data/lib/burner/payload.rb

@@ -8,16 +8,22 @@
 #
 
 module Burner
-  # The input for all Job#perform methods.  The main notion of this object is its "value"
-  # attribute.  This is dynamic and weak on purpose and is subject to whatever the Job#perform
-  # methods decides it is.  This definitely adds an order-of-magnitude complexity to this whole
-  # library and lifecycle, but I am not sure there is any other way around it: trying to build
-  # a generic, open-ended object pipeline to serve almost any use case.
+  # The input for all Job#perform methods.  The main notion of this object is its 'registers'
+  # attribute.  This registers attribute is a key-indifferent hash, accessible on Payload using
+  # the brackets setter and getter methods.  This is dynamic and weak on purpose and is subject
+  # to whatever the Job#perform methods decides it is.  This definitely adds an order-of-magnitude
+  # complexity to this whole library and lifecycle, but I am not sure there is any other way
+  # around it: trying to build a generic, open-ended processing pipeline to serve almost
+  # any use case.
   #
   # The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
   # structured/additive manner.  The initial use case for this was for Burner's core IO jobs to
   # report back the files it has written in a more structured data way (as opposed to simply
   # writing some information to the output.)
+  #
+  # The 'time' attribute is important in that it should for the replaying of pipelines and jobs.
+  # Instead of having job's utilizing Time.now, Date.today, etc... they should rather opt to
+  # use this value instead.
   class Payload
     attr_reader :params,
                 :registers,

data/lib/burner/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Burner
-  VERSION = '1.0.0-alpha.7'
+  VERSION = '1.0.0-alpha.13'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: burner
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.alpha.7
+  version: 1.0.0.pre.alpha.13
 platform: ruby
 authors:
 - Matthew Ruggio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-27 00:00:00.000000000 Z
+date: 2020-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -192,8 +192,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.0
-description: "    This library serves as the skeleton for a processing engine.  It
-  allows you to organize your code into Jobs, then stitch those jobs together as steps.\n"
+description: "    This library serves as the backbone for a configurable processing
+  engine.  It allows you to organize your code into jobs, then stitch those jobs together
+  as steps.\n"
 email:
 - mruggio@bluemarblepayroll.com
 executables:
@@ -224,6 +225,7 @@ files:
 - lib/burner/jobs.rb
 - lib/burner/library.rb
 - lib/burner/library/collection/arrays_to_objects.rb
+- lib/burner/library/collection/concatenate.rb
 - lib/burner/library/collection/graph.rb
 - lib/burner/library/collection/objects_to_arrays.rb
 - lib/burner/library/collection/shift.rb
@@ -234,17 +236,18 @@ files:
 - lib/burner/library/deserialize/csv.rb
 - lib/burner/library/deserialize/json.rb
 - lib/burner/library/deserialize/yaml.rb
-- lib/burner/library/dummy.rb
 - lib/burner/library/echo.rb
 - lib/burner/library/io/base.rb
 - lib/burner/library/io/exist.rb
 - lib/burner/library/io/read.rb
 - lib/burner/library/io/write.rb
+- lib/burner/library/nothing.rb
 - lib/burner/library/serialize/csv.rb
 - lib/burner/library/serialize/json.rb
 - lib/burner/library/serialize/yaml.rb
-- lib/burner/library/set_value.rb
 - lib/burner/library/sleep.rb
+- lib/burner/library/value/copy.rb
+- lib/burner/library/value/static.rb
 - lib/burner/modeling.rb
 - lib/burner/modeling/attribute.rb
 - lib/burner/modeling/attribute_renderer.rb

data/lib/burner/library/set_value.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-#
-# Copyright (c) 2020-present, Blue Marble Payroll, LLC
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-module Burner
-  module Library
-    # Arbitrarily set value
-    #
-    # Expected Payload#value input: anything.
-    # Payload#value output: whatever value was specified in this job.
-    class SetValue < JobWithRegister
-      attr_reader :value
-
-      def initialize(name:, register: '', value: nil)
-        super(name: name, register: register)
-
-        @value = value
-
-        freeze
-      end
-
-      def perform(_output, payload)
-        payload[register] = value
-      end
-    end
-  end
-end