burner 1.0.0.pre.alpha.1 → 1.0.0.pre.alpha.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76ba9eda7f18a68f4c24c11d4724e2021c6f1d92ca4878fafb118e6c897c018c
-  data.tar.gz: 95e35512ece9b5f8acb912305d0609c86354977d3c8bd6e125a18db0584778fc
+  metadata.gz: 73263c161bce669b99286160acb7129d33c529162da964b72fd34321cc7e3724
+  data.tar.gz: 18fd59799a73d9a8a3bf691a9dd3f8a004d442b37862bf83cbe347fbddd7e49a
 SHA512:
-  metadata.gz: 47b53c96680edfd6ca08e637d2ab6fd738684e9c2e7e8d08ca6d52a6361b8cf3cfa55af072cfeeb41a44838e76a4474aa229fc23b86f54558603c9af718f5fa0
-  data.tar.gz: 804da88a604544877d7b4f34bafcf1079f3282ad0c34da69466b83bfbd37fb30589f1cb38b02aafa4ccfd17f9e24aabf28385d06a8b80c3d2f552d4e71d3fd7d
+  metadata.gz: 7db0c2be13a882885b97681ec7c9fd60b6702368d673d97ba0aa484b446db0c77c9f823adf03c8d9e970b2078b43a99d2d159ba97df345124a461497718b7046
+  data.tar.gz: 509382aa7fb48e116b8b3eb145a8b8226c1e4d4282eec1aabb10dcc9f7b333b906a7887d97cee4a96ae730dd7c02a0b0d4cd4a7b06dbfcc90349cfd82767e048

data/README.md

@@ -73,19 +73,21 @@ params = {
   input_file: 'input.json',
   output_file: 'output.yaml'
 }
+
+payload = Burner::Payload.new(params: params)
 ````
 
 Assuming we are running this script from a directory where an `input.json` file exists, we can then programatically process the pipeline:
 
 ````ruby
-Burner::Pipeline.make(pipeline).execute(params: params)
+Burner::Pipeline.make(pipeline).execute(payload: payload)
 ````
 
 We should now see a output.yaml file created.
 
 Some notes:
 
-* Some values are able to be string-interpolated using the provided params.  This allows for the passing runtime configuration/data into pipelines/jobs.
+* Some values are able to be string-interpolated using the provided Payload#params.  This allows for the passing runtime configuration/data into pipelines/jobs.
 * The job's ID can be accessed using the `__id` key.
 * The current job's payload value can be accessed using the `__value` key.
 * Jobs can be re-used (just like the output_id and output_value jobs).
@@ -116,8 +118,9 @@ end
 
 string_out = StringOut.new
 output     = Burner::Output.new(outs: string_out)
+payload    = Burner::Payload.new(params: params)
 
-Burner::Pipeline.make(pipeline).execute(output: output, params: params)
+Burner::Pipeline.make(pipeline).execute(output: output, payload: payload)
 
 log = string_out.read
 ````
@@ -130,23 +133,23 @@ The value of `log` should now look similar to:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - input_file: input.json
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - output_file: output.yaml
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] --------------------------------------------------------------------------------
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [1] Burner::Jobs::IO::Read::read
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [1] Burner::Library::IO::Read::read
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Reading: spec/fixtures/input.json
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [2] Burner::Jobs::Echo::output_id
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [2] Burner::Library::Echo::output_id
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - The job id is:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [3] Burner::Jobs::Echo::output_value
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [3] Burner::Library::Echo::output_value
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - The current value is:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [4] Burner::Jobs::Deserialize::Json::parse
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [4] Burner::Library::Deserialize::Json::parse
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [5] Burner::Jobs::Serialize::Yaml::convert
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [5] Burner::Library::Serialize::Yaml::convert
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [6] Burner::Jobs::Echo::output_value
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [6] Burner::Library::Echo::output_value
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - The current value is:
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
-[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [7] Burner::Jobs::IO::Write::write
+[8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [7] Burner::Library::IO::Write::write
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Writing: output.yaml
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC]   - Completed in: 0.0 second(s)
 [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] --------------------------------------------------------------------------------
@@ -160,7 +163,7 @@ Notes:
 
 ### Command Line Pipeline Processing
 
-This library also ships with a built-in script `exe/burner` that illustrates using the `Burner::Cli` API.  This class can take in an array of arguments (similar to a command-line) and execute a pipeline.  The first argument is the path to a YAML file with the pipeline's configuration and each subsequent argument is a param in `key=value` form.  Here is how the json-to-yaml example can utilize this interface:
+This library also ships with a built-in script `burner` that illustrates using the `Burner::Cli` API.  This class can take in an array of arguments (similar to a command-line) and execute a pipeline.  The first argument is the path to a YAML file with the pipeline's configuration and each subsequent argument is a param in `key=value` form.  Here is how the json-to-yaml example can utilize this interface:
 
 #### Create YAML Pipeline Configuration File
 
@@ -228,28 +231,51 @@ Burner::Cli.new(args).invoke
 
 This library only ships with very basic, rudimentary jobs that are meant to just serve as a baseline:
 
+#### Collection
+
+* **collection/arrays_to_objects** [mappings]: Convert an array of arrays to an array of objects.
+* **collection/graph** [config, key]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
+* **collection/objects_to_arrays** [mappings]: Convert an array of objects to an array of arrays.
+* **collection/shift** [amount]: Remove the first N number of elements from an array.
+* **collection/transform** [attributes, exclusive, separator]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.
+* **collection/unpivot** [pivot_set]: Take an array of objects and unpivot specific sets of keys into rows.  Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
+* **collection/values** [include_keys]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
+
+#### De-serialization
+
+* **deserialize/csv** []: Take a CSV string and de-serialize into object(s).  Currently it will return an array of arrays, with each nested array representing one row.
 * **deserialize/json** []: Treat input as a string and de-serialize it to JSON.
-* **deserialize/yaml** [safe]: Treat input as a string and de-serialize it to YAML.  By default it will try and (safely de-serialize)[https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load] it (only using core classes).  If you wish to de-serialize it to any class type, pass in `safe: false`
-* **dummy** []: Do nothing
-* **echo** [message]: Write a message to the output.  The message parameter can be interpolated using params.
-* **io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using params.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
-* **io/read** [binary, path]: Read in a local file.  The path parameter can be interpolated using params.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
-* **io/write** [binary, path]: Write to a local file.  The path parameter can be interpolated using params.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
+* **deserialize/yaml** [safe]: Treat input as a string and de-serialize it to YAML.  By default it will try and [safely de-serialize](https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load) it (only using core classes).  If you wish to de-serialize it to any class type, pass in `safe: false`
+
+#### IO
+
+* **io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
+* **io/read** [binary, path]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
+* **io/write** [binary, path]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
+
+#### Serialization
+
+* **serialize/csv** []: Take an array of arrays and create a CSV.
 * **serialize/json** []: Convert value to JSON.
 * **serialize/yaml** []: Convert value to YAML.
+
+#### General
+
+* **dummy** []: Do nothing
+* **echo** [message]: Write a message to the output.  The message parameter can be interpolated using  `Payload#params`.
 * **set** [value]: Set the value to any arbitrary value.
 * **sleep** [seconds]: Sleep the thread for X number of seconds.
 
 
 ### Adding & Registering Jobs
 
-Where this library shines is when additional jobs are plugged in.  Burner uses its `Burner::Jobs` class as its class-level registry built with (acts_as_hashable)[https://github.com/bluemarblepayroll/acts_as_hashable]'s acts_as_hashable_factory directive.
+Where this library shines is when additional jobs are plugged in.  Burner uses its `Burner::Jobs` class as its class-level registry built with [acts_as_hashable](https://github.com/bluemarblepayroll/acts_as_hashable)'s acts_as_hashable_factory directive.
 
 Let's say we would like to register a job to parse a CSV:
 
 ````ruby
 class ParseCsv < Burner::Job
-  def perform(output, payload, params)
+  def perform(output, payload)
     payload.value = CSV.parse(payload.value, headers: true).map(&:to_h)
 
     nil
@@ -309,7 +335,9 @@ params = {
   output_file: File.join(TEMP_DIR, "#{SecureRandom.uuid}.yaml")
 }
 
-Burner::Pipeline.make(pipeline).execute(output: output, params: params)
+payload = Burner::Payload.new(params: params)
+
+Burner::Pipeline.make(pipeline).execute(output: output, payload: payload)
 ````
 
 ## Contributing

data/burner.gemspec

@@ -29,7 +29,10 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.5'
 
   s.add_dependency('acts_as_hashable', '~>1.2')
+  s.add_dependency('hashematics', '~>1.1')
+  s.add_dependency('hash_math', '~>1.2')
   s.add_dependency('objectable', '~>1.0')
+  s.add_dependency('realize', '~>1.2')
   s.add_dependency('stringento', '~>2.1')
 
   s.add_development_dependency('guard-rspec', '~>4.7')

data/exe/burner

@@ -10,12 +10,11 @@
 
 require 'bundler/setup'
 require 'burner'
-require 'pry'
 
 if ARGV.empty?
-  puts 'Usage: ./exe/burner package.yaml key=value key=value ...'
-  exit
+  warn('Usage: ./exe/burner package.yaml key=value key=value ...')
+  exit 2 # Do not return 1, that is reserved for hard errors.
 end
 
 # This should return exit code of 1 if it raises any hard errors.
-Burner::Cli.new(ARGV).invoke
+Burner::Cli.new(ARGV).execute

data/lib/burner.rb

@@ -9,11 +9,23 @@
 
 require 'acts_as_hashable'
 require 'benchmark'
+require 'csv'
+require 'forwardable'
+require 'hash_math'
+require 'hashematics'
 require 'json'
 require 'objectable'
+require 'realize'
 require 'securerandom'
 require 'singleton'
 require 'stringento'
+require 'time'
 require 'yaml'
 
+# Common/Shared
+require_relative 'burner/modeling'
+require_relative 'burner/side_effects'
+require_relative 'burner/util'
+
+# Main Entrypoint(s)
 require_relative 'burner/cli'

data/lib/burner/cli.rb

@@ -12,18 +12,20 @@ require_relative 'pipeline'
 module Burner
   # Process a single string as a Pipeline.  This is mainly to back the command-line interface.
   class Cli
-    attr_reader :params, :pipeline
+    attr_reader :payload, :pipeline
 
     def initialize(args)
-      path       = args.first
-      cli_params = extract_cli_params(args)
-      config     = read_yaml(path)
-      @pipeline  = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
-      @params    = (config['params'] || {}).merge(cli_params)
+      path      = args.first
+      params    = extract_cli_params(args)
+      config    = read_yaml(path)
+      @pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
+      @payload  = Payload.new(params: params)
+
+      freeze
     end
 
-    def invoke
-      pipeline.execute(params: params)
+    def execute
+      pipeline.execute(payload: payload)
     end
 
     private

data/lib/burner/job.rb

@@ -15,21 +15,43 @@ module Burner
   # the Burner::Jobs factory class method #register.  An example of a registration:
   #   Burner::Jobs.register('your_class', YourClass)
   class Job
+    include Util::Arrayable
     acts_as_hashable
 
-    attr_reader :name, :string_template
+    attr_reader :name
 
     def initialize(name:)
       raise ArgumentError, 'name is required' if name.to_s.empty?
 
-      @name            = name.to_s
-      @string_template = StringTemplate.instance
+      @name = name.to_s
     end
 
-    private
+    # There are only two requirements to be considered a valid Burner Job:
+    #   1. The class responds to #name
+    #   2. The class responds to #perform(output, payload)
+    #
+    # The #perform method takes in two arguments: output (an instance of Burner::Output)
+    # and payload (an instance of Burner::Payload).  Jobs can leverage output to emit
+    # information to the pipeline's log(s).  The payload is utilized to pass data from job to job,
+    # with its most important attribute being #value.  The value attribute is mutable
+    # per the individual job's context (meaning of it is unknown without understanding a job's
+    # input and output value of #value.).  Therefore #value can mean anything and it is up to the
+    # engineers to clearly document the assumptions of its use.
+    #
+    # Returning false will short-circuit the pipeline right after the job method exits.
+    # Returning anything else besides false just means "continue".
+    def perform(output, _payload)
+      output.detail("#perform not implemented for: #{self.class.name}")
+
+      nil
+    end
+
+    protected
+
+    def job_string_template(expression, output, payload)
+      templatable_params = payload.params.merge(__id: output.id, __value: payload.value)
 
-    def eval_string_template(expression, input)
-      string_template.evaluate(expression, input)
+      StringTemplate.instance.evaluate(expression, templatable_params)
     end
   end
 end

data/lib/burner/jobs.rb

@@ -7,18 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'job'
-require_relative 'jobs/deserialize/json'
-require_relative 'jobs/deserialize/yaml'
-require_relative 'jobs/dummy'
-require_relative 'jobs/echo'
-require_relative 'jobs/io/exist'
-require_relative 'jobs/io/read'
-require_relative 'jobs/io/write'
-require_relative 'jobs/serialize/json'
-require_relative 'jobs/serialize/yaml'
-require_relative 'jobs/set'
-require_relative 'jobs/sleep'
+require_relative 'library'
 
 module Burner
   # Main library of jobs.  This file contains all the basic/default jobs.  All other consumer
@@ -27,16 +16,25 @@ module Burner
   class Jobs
     acts_as_hashable_factory
 
-    register 'deserialize/json', Deserialize::Json
-    register 'deserialize/yaml', Deserialize::Yaml
-    register 'dummy', '',        Dummy
-    register 'echo',             Echo
-    register 'io/exist',         IO::Exist
-    register 'io/read',          IO::Read
-    register 'io/write',         IO::Write
-    register 'serialize/json',   Serialize::Json
-    register 'serialize/yaml',   Serialize::Yaml
-    register 'set',              Set
-    register 'sleep',            Sleep
+    register 'collection/arrays_to_objects', Library::Collection::ArraysToObjects
+    register 'collection/graph',             Library::Collection::Graph
+    register 'collection/objects_to_arrays', Library::Collection::ObjectsToArrays
+    register 'collection/shift',             Library::Collection::Shift
+    register 'collection/transform',         Library::Collection::Transform
+    register 'collection/unpivot',           Library::Collection::Unpivot
+    register 'collection/values',            Library::Collection::Values
+    register 'deserialize/csv',              Library::Deserialize::Csv
+    register 'deserialize/json',             Library::Deserialize::Json
+    register 'deserialize/yaml',             Library::Deserialize::Yaml
+    register 'dummy', '',                    Library::Dummy
+    register 'echo',                         Library::Echo
+    register 'io/exist',                     Library::IO::Exist
+    register 'io/read',                      Library::IO::Read
+    register 'io/write',                     Library::IO::Write
+    register 'serialize/csv',                Library::Serialize::Csv
+    register 'serialize/json',               Library::Serialize::Json
+    register 'serialize/yaml',               Library::Serialize::Yaml
+    register 'set_value',                    Library::SetValue
+    register 'sleep',                        Library::Sleep
   end
 end

data/lib/burner/library.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'job'
+require_relative 'library/collection/arrays_to_objects'
+require_relative 'library/collection/graph'
+require_relative 'library/collection/objects_to_arrays'
+require_relative 'library/collection/shift'
+require_relative 'library/collection/transform'
+require_relative 'library/collection/unpivot'
+require_relative 'library/collection/values'
+require_relative 'library/deserialize/csv'
+require_relative 'library/deserialize/json'
+require_relative 'library/deserialize/yaml'
+require_relative 'library/dummy'
+require_relative 'library/echo'
+require_relative 'library/io/exist'
+require_relative 'library/io/read'
+require_relative 'library/io/write'
+require_relative 'library/serialize/csv'
+require_relative 'library/serialize/json'
+require_relative 'library/serialize/yaml'
+require_relative 'library/set_value'
+require_relative 'library/sleep'

data/lib/burner/library/collection/arrays_to_objects.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # Convert an array of arrays to an array of objects.  Pass in an array of
+      # Burner::Modeling::KeyIndexMapping instances or hashable configurations which specifies
+      # the index-to-key mappings to use.
+      #
+      # Expected Payload#value input: array of arrays.
+      # Payload#value output: An array of hashes.
+      #
+      # An example using a configuration-first pipeline:
+      #
+      #   config = {
+      #     jobs: [
+      #       {
+      #         name: 'set',
+      #         type: 'set_value',
+      #         value: [
+      #           [1, 'funky']
+      #         ]
+      #       },
+      #       {
+      #         name: 'map',
+      #         type: 'collection/arrays_to_objects',
+      #         mappings: [
+      #           { index: 0, key: 'id' },
+      #           { index: 1, key: 'name' }
+      #         ]
+      #       },
+      #       {
+      #         name: 'output',
+      #         type: 'echo',
+      #         message: 'value is currently: {__value}'
+      #       },
+      #
+      #     ],
+      #     steps: %w[set map output]
+      #   }
+      #
+      #   Burner::Pipeline.make(config).execute
+      class ArraysToObjects < Job
+        attr_reader :mappings
+
+        def initialize(name:, mappings: [])
+          super(name: name)
+
+          @mappings = Modeling::KeyIndexMapping.array(mappings)
+
+          freeze
+        end
+
+        def perform(_output, payload)
+          payload.value = array(payload.value).map { |array| index_to_key_map(array) }
+
+          nil
+        end
+
+        private
+
+        def index_to_key_map(array)
+          mappings.each_with_object({}) do |mapping, memo|
+            memo[mapping.key] = array[mapping.index]
+          end
+        end
+      end
+    end
+  end
+end