burner 1.0.0.pre.alpha.5 → 1.0.0.pre.alpha.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57b5b4290b72962e10ce7ea7244a1dcfb43894cff1d72ef4e0684de4cdea4a35
-  data.tar.gz: 2936c408e7ffa3e1e9883510d890329870d5c1a4c129fef1fb283103568c9508
+  metadata.gz: 73263c161bce669b99286160acb7129d33c529162da964b72fd34321cc7e3724
+  data.tar.gz: 18fd59799a73d9a8a3bf691a9dd3f8a004d442b37862bf83cbe347fbddd7e49a
 SHA512:
-  metadata.gz: f514870aa2b12cc4fc34f3952cabf8c738985dac1a6602c7f41aec3f28b83738991d5cf82d429b62d76b0f3a96b85907e51d0e06db0ce33579d1dc74dc55409e
-  data.tar.gz: 2e9ee10f91bb28eb4d79091ae7106f6da640daaf6b5ac2f77e828e81743dfd55d6baca12a0e376c3be709483fe7a94e0f840f47ed2ecb5031233f4ff9ae7db3a
+  metadata.gz: 7db0c2be13a882885b97681ec7c9fd60b6702368d673d97ba0aa484b446db0c77c9f823adf03c8d9e970b2078b43a99d2d159ba97df345124a461497718b7046
+  data.tar.gz: 509382aa7fb48e116b8b3eb145a8b8226c1e4d4282eec1aabb10dcc9f7b333b906a7887d97cee4a96ae730dd7c02a0b0d4cd4a7b06dbfcc90349cfd82767e048

data/README.md

@@ -133,23 +133,23 @@ The value of `log` should now look similar to:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - input_file: input.json
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - output_file: output.yaml
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] --------------------------------------------------------------------------------
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [1] Burner::Jobs::IO::Read::read
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [1] Burner::Library::IO::Read::read
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Reading: spec/fixtures/input.json
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [2] Burner::Jobs::Echo::output_id
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [2] Burner::Library::Echo::output_id
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - The job id is:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [3] Burner::Jobs::Echo::output_value
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [3] Burner::Library::Echo::output_value
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - The current value is:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [4] Burner::Jobs::Deserialize::Json::parse
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [4] Burner::Library::Deserialize::Json::parse
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [5] Burner::Jobs::Serialize::Yaml::convert
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [5] Burner::Library::Serialize::Yaml::convert
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [6] Burner::Jobs::Echo::output_value
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [6] Burner::Library::Echo::output_value
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - The current value is:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [7] Burner::Jobs::IO::Write::write
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [7] Burner::Library::IO::Write::write
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Writing: output.yaml
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] --------------------------------------------------------------------------------
@@ -163,7 +163,7 @@ Notes:
 
 ### Command Line Pipeline Processing
 
-This library also ships with a built-in script `exe/burner` that illustrates using the `Burner::Cli` API.  This class can take in an array of arguments (similar to a command-line) and execute a pipeline.  The first argument is the path to a YAML file with the pipeline's configuration and each subsequent argument is a param in `key=value` form.  Here is how the json-to-yaml example can utilize this interface:
+This library also ships with a built-in script `burner` that illustrates using the `Burner::Cli` API.  This class can take in an array of arguments (similar to a command-line) and execute a pipeline.  The first argument is the path to a YAML file with the pipeline's configuration and each subsequent argument is a param in `key=value` form.  Here is how the json-to-yaml example can utilize this interface:
 
 #### Create YAML Pipeline Configuration File
 
@@ -234,10 +234,10 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 #### Collection
 
 * **collection/arrays_to_objects** [mappings]: Convert an array of arrays to an array of objects.
-* **collection/graph** [config, key]: Use (Hashematics)[https://github.com/bluemarblepayroll/hashematics] to turn a flat array of objects into a deeply nested object tree.
+* **collection/graph** [config, key]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
 * **collection/objects_to_arrays** [mappings]: Convert an array of objects to an array of arrays.
 * **collection/shift** [amount]: Remove the first N number of elements from an array.
-* **collection/transform** [attributes, exclusive, separator]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses (Realize)[https://github.com/bluemarblepayroll/realize], which provides its own extendable value-transformation pipeline.
+* **collection/transform** [attributes, exclusive, separator]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.
 * **collection/unpivot** [pivot_set]: Take an array of objects and unpivot specific sets of keys into rows.  Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
 * **collection/values** [include_keys]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
 
@@ -245,7 +245,7 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 
 * **deserialize/csv** []: Take a CSV string and de-serialize into object(s).  Currently it will return an array of arrays, with each nested array representing one row.
 * **deserialize/json** []: Treat input as a string and de-serialize it to JSON.
-* **deserialize/yaml** [safe]: Treat input as a string and de-serialize it to YAML.  By default it will try and (safely de-serialize)[https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load] it (only using core classes).  If you wish to de-serialize it to any class type, pass in `safe: false`
+* **deserialize/yaml** [safe]: Treat input as a string and de-serialize it to YAML.  By default it will try and [safely de-serialize](https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load) it (only using core classes).  If you wish to de-serialize it to any class type, pass in `safe: false`
 
 #### IO
 
@@ -269,7 +269,7 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 
 ### Adding & Registering Jobs
 
-Where this library shines is when additional jobs are plugged in.  Burner uses its `Burner::Jobs` class as its class-level registry built with (acts_as_hashable)[https://github.com/bluemarblepayroll/acts_as_hashable]'s acts_as_hashable_factory directive.
+Where this library shines is when additional jobs are plugged in.  Burner uses its `Burner::Jobs` class as its class-level registry built with [acts_as_hashable](https://github.com/bluemarblepayroll/acts_as_hashable)'s acts_as_hashable_factory directive.
 
 Let's say we would like to register a job to parse a CSV:
 

data/exe/burner

@@ -10,11 +10,10 @@
 
 require 'bundler/setup'
 require 'burner'
-require 'pry'
 
 if ARGV.empty?
-  puts 'Usage: ./exe/burner package.yaml key=value key=value ...'
-  exit
+  warn('Usage: ./exe/burner package.yaml key=value key=value ...')
+  exit 2 # Do not return 1, that is reserved for hard errors.
 end
 
 # This should return exit code of 1 if it raises any hard errors.

data/lib/burner.rb

@@ -24,6 +24,8 @@ require 'yaml'
 
 # Common/Shared
 require_relative 'burner/modeling'
+require_relative 'burner/side_effects'
+require_relative 'burner/util'
 
 # Main Entrypoint(s)
 require_relative 'burner/cli'

data/lib/burner/cli.rb

@@ -20,6 +20,8 @@ module Burner
       config    = read_yaml(path)
       @pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
       @payload  = Payload.new(params: params)
+
+      freeze
     end
 
     def execute

data/lib/burner/job.rb

@@ -15,23 +15,43 @@ module Burner
   # the Burner::Jobs factory class method #register.  An example of a registration:
   #   Burner::Jobs.register('your_class', YourClass)
   class Job
+    include Util::Arrayable
     acts_as_hashable
 
-    attr_reader :name, :string_template
+    attr_reader :name
 
     def initialize(name:)
       raise ArgumentError, 'name is required' if name.to_s.empty?
 
-      @name            = name.to_s
-      @string_template = StringTemplate.instance
+      @name = name.to_s
     end
 
-    private
+    # There are only two requirements to be considered a valid Burner Job:
+    #   1. The class responds to #name
+    #   2. The class responds to #perform(output, payload)
+    #
+    # The #perform method takes in two arguments: output (an instance of Burner::Output)
+    # and payload (an instance of Burner::Payload).  Jobs can leverage output to emit
+    # information to the pipeline's log(s).  The payload is utilized to pass data from job to job,
+    # with its most important attribute being #value.  The value attribute is mutable
+    # per the individual job's context (meaning of it is unknown without understanding a job's
+    # input and output value of #value.).  Therefore #value can mean anything and it is up to the
+    # engineers to clearly document the assumptions of its use.
+    #
+    # Returning false will short-circuit the pipeline right after the job method exits.
+    # Returning anything else besides false just means "continue".
+    def perform(output, _payload)
+      output.detail("#perform not implemented for: #{self.class.name}")
+
+      nil
+    end
+
+    protected
 
     def job_string_template(expression, output, payload)
       templatable_params = payload.params.merge(__id: output.id, __value: payload.value)
 
-      string_template.evaluate(expression, templatable_params)
+      StringTemplate.instance.evaluate(expression, templatable_params)
     end
   end
 end

data/lib/burner/jobs.rb

@@ -7,27 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'job'
-require_relative 'jobs/collection/arrays_to_objects'
-require_relative 'jobs/collection/graph'
-require_relative 'jobs/collection/objects_to_arrays'
-require_relative 'jobs/collection/shift'
-require_relative 'jobs/collection/transform'
-require_relative 'jobs/collection/unpivot'
-require_relative 'jobs/collection/values'
-require_relative 'jobs/deserialize/csv'
-require_relative 'jobs/deserialize/json'
-require_relative 'jobs/deserialize/yaml'
-require_relative 'jobs/dummy'
-require_relative 'jobs/echo'
-require_relative 'jobs/io/exist'
-require_relative 'jobs/io/read'
-require_relative 'jobs/io/write'
-require_relative 'jobs/serialize/csv'
-require_relative 'jobs/serialize/json'
-require_relative 'jobs/serialize/yaml'
-require_relative 'jobs/set'
-require_relative 'jobs/sleep'
+require_relative 'library'
 
 module Burner
   # Main library of jobs.  This file contains all the basic/default jobs.  All other consumer
@@ -36,25 +16,25 @@ module Burner
   class Jobs
     acts_as_hashable_factory
 
-    register 'collection/arrays_to_objects', Collection::ArraysToObjects
-    register 'collection/graph',             Collection::Graph
-    register 'collection/objects_to_arrays', Collection::ObjectsToArrays
-    register 'collection/shift',             Collection::Shift
-    register 'collection/transform',         Collection::Transform
-    register 'collection/unpivot',           Collection::Unpivot
-    register 'collection/values',            Collection::Values
-    register 'deserialize/csv',              Deserialize::Csv
-    register 'deserialize/json',             Deserialize::Json
-    register 'deserialize/yaml',             Deserialize::Yaml
-    register 'dummy', '',                    Dummy
-    register 'echo',                         Echo
-    register 'io/exist',                     IO::Exist
-    register 'io/read',                      IO::Read
-    register 'io/write',                     IO::Write
-    register 'serialize/csv',                Serialize::Csv
-    register 'serialize/json',               Serialize::Json
-    register 'serialize/yaml',               Serialize::Yaml
-    register 'set',                          Set
-    register 'sleep',                        Sleep
+    register 'collection/arrays_to_objects', Library::Collection::ArraysToObjects
+    register 'collection/graph',             Library::Collection::Graph
+    register 'collection/objects_to_arrays', Library::Collection::ObjectsToArrays
+    register 'collection/shift',             Library::Collection::Shift
+    register 'collection/transform',         Library::Collection::Transform
+    register 'collection/unpivot',           Library::Collection::Unpivot
+    register 'collection/values',            Library::Collection::Values
+    register 'deserialize/csv',              Library::Deserialize::Csv
+    register 'deserialize/json',             Library::Deserialize::Json
+    register 'deserialize/yaml',             Library::Deserialize::Yaml
+    register 'dummy', '',                    Library::Dummy
+    register 'echo',                         Library::Echo
+    register 'io/exist',                     Library::IO::Exist
+    register 'io/read',                      Library::IO::Read
+    register 'io/write',                     Library::IO::Write
+    register 'serialize/csv',                Library::Serialize::Csv
+    register 'serialize/json',               Library::Serialize::Json
+    register 'serialize/yaml',               Library::Serialize::Yaml
+    register 'set_value',                    Library::SetValue
+    register 'sleep',                        Library::Sleep
   end
 end

data/lib/burner/library.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'job'
+require_relative 'library/collection/arrays_to_objects'
+require_relative 'library/collection/graph'
+require_relative 'library/collection/objects_to_arrays'
+require_relative 'library/collection/shift'
+require_relative 'library/collection/transform'
+require_relative 'library/collection/unpivot'
+require_relative 'library/collection/values'
+require_relative 'library/deserialize/csv'
+require_relative 'library/deserialize/json'
+require_relative 'library/deserialize/yaml'
+require_relative 'library/dummy'
+require_relative 'library/echo'
+require_relative 'library/io/exist'
+require_relative 'library/io/read'
+require_relative 'library/io/write'
+require_relative 'library/serialize/csv'
+require_relative 'library/serialize/json'
+require_relative 'library/serialize/yaml'
+require_relative 'library/set_value'
+require_relative 'library/sleep'

data/lib/burner/library/collection/arrays_to_objects.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # Convert an array of arrays to an array of objects.  Pass in an array of
+      # Burner::Modeling::KeyIndexMapping instances or hashable configurations which specifies
+      # the index-to-key mappings to use.
+      #
+      # Expected Payload#value input: array of arrays.
+      # Payload#value output: An array of hashes.
+      #
+      # An example using a configuration-first pipeline:
+      #
+      #   config = {
+      #     jobs: [
+      #       {
+      #         name: 'set',
+      #         type: 'set_value',
+      #         value: [
+      #           [1, 'funky']
+      #         ]
+      #       },
+      #       {
+      #         name: 'map',
+      #         type: 'collection/arrays_to_objects',
+      #         mappings: [
+      #           { index: 0, key: 'id' },
+      #           { index: 1, key: 'name' }
+      #         ]
+      #       },
+      #       {
+      #         name: 'output',
+      #         type: 'echo',
+      #         message: 'value is currently: {__value}'
+      #       },
+      #
+      #     ],
+      #     steps: %w[set map output]
+      #   }
+      #
+      #   Burner::Pipeline.make(config).execute
+      class ArraysToObjects < Job
+        attr_reader :mappings
+
+        def initialize(name:, mappings: [])
+          super(name: name)
+
+          @mappings = Modeling::KeyIndexMapping.array(mappings)
+
+          freeze
+        end
+
+        def perform(_output, payload)
+          payload.value = array(payload.value).map { |array| index_to_key_map(array) }
+
+          nil
+        end
+
+        private
+
+        def index_to_key_map(array)
+          mappings.each_with_object({}) do |mapping, memo|
+            memo[mapping.key] = array[mapping.index]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/burner/{jobs → library}/collection/graph.rb

@@ -8,10 +8,11 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     module Collection
       # Take an array of (denormalized) objects and create an object hierarchy from them.
       # Under the hood it uses Hashematics: https://github.com/bluemarblepayroll/hashematics.
+      #
       # Expected Payload#value input: array of objects.
       # Payload#value output: An array of objects.
       class Graph < Job
@@ -29,7 +30,7 @@ module Burner
         end
 
         def perform(output, payload)
-          graph = Hashematics::Graph.new(groups).add(payload.value || [])
+          graph = Hashematics::Graph.new(groups).add(array(payload.value))
 
           output.detail("Graphing: #{key}")
 

data/lib/burner/{jobs → library}/collection/objects_to_arrays.rb

@@ -8,18 +8,54 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     module Collection
       # Convert an array of objects to an array of arrays.  You can leverage the separator
-      # option to support key paths and nested objects.
+      # option to support key paths and nested objects.  Pass in an array of
+      # Burner::Modeling::KeyIndexMapping instances or hashable configurations which specifies
+      # the key-to-index mappings to use.
+      #
       # Expected Payload#value input: array of hashes.
       # Payload#value output: An array of arrays.
+      #
+      # An example using a configuration-first pipeline:
+      #
+      #   config = {
+      #     jobs: [
+      #       {
+      #         name: 'set',
+      #         type: 'set_value',
+      #         value: [
+      #           [1, 'funky']
+      #         ]
+      #       },
+      #       {
+      #         name: 'map',
+      #         type: 'collection/objects_to_arrays',
+      #         mappings: [
+      #           { index: 0, key: 'id' },
+      #           { index: 1, key: 'name' }
+      #         ]
+      #       },
+      #       {
+      #         name: 'output',
+      #         type: 'echo',
+      #         message: 'value is currently: {__value}'
+      #       },
+      #
+      #     ],
+      #     steps: %w[set map output]
+      #   }
+      #
+      #   Burner::Pipeline.make(config).execute
       class ObjectsToArrays < Job
         attr_reader :mappings
 
         # If you wish to support nested objects you can pass in a string to use as a
         # key path separator.  For example: if you would like to recognize dot-notation for
-        # nested hashes then set separator to '.'.
+        # nested hashes then set separator to '.'.  For more information, see the underlying
+        # library that supports this dot-notation concept:
+        #   https://github.com/bluemarblepayroll/objectable
         def initialize(name:, mappings: [], separator: '')
           super(name: name)
 
@@ -30,7 +66,7 @@ module Burner
         end
 
         def perform(_output, payload)
-          payload.value = (payload.value || []).map { |object| key_to_index_map(object) }
+          payload.value = array(payload.value).map { |object| key_to_index_map(object) }
 
           nil
         end

data/lib/burner/{jobs → library}/collection/shift.rb

@@ -8,10 +8,12 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     module Collection
       # Take an array and remove the first N elements, where N is specified by the amount
-      # attribute.
+      # attribute.  The initial use case for this was to remove "header" rows from arrays,
+      # like you would expect when parsing CSV files.
+      #
       # Expected Payload#value input: nothing.
       # Payload#value output: An array with N beginning elements removed.
       class Shift < Job
@@ -32,8 +34,7 @@ module Burner
         def perform(output, payload)
           output.detail("Shifting #{amount} entries.")
 
-          payload.value ||= []
-          payload.value.shift(amount)
+          payload.value = array(payload.value).slice(amount..-1)
 
           nil
         end