burner 1.0.0.pre.alpha.13 → 1.3.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d682634e7c0c8b850b98d80a33e4e44012a3f7ae058b72e3c828830d7e177a0a
-  data.tar.gz: afc1daecbec70a6ec88f19ab11ea92f93f28a7dd8bc8ea4821cb6a222db3f824
+  metadata.gz: 340ebade1c4d3cc238925cb21718571dd666c3c16b6ee60d8de73c4fe017d562
+  data.tar.gz: 33b272e7a4b4a15a8e92b5f7eccb9df7ec6d685f767ecbf8434dedcf7ed901b5
 SHA512:
-  metadata.gz: d75499b61c0b30b4882fb8cf10c74ea2cd0dfa8562824bf3a0a61c3311b78808eaeb5e0dfc1db1faa81400a23a998e4dd6b37ba35ad170ec2945c8708804d7c7
-  data.tar.gz: 42e5b5d6fead52793eefa7ef6f41423254e870e0fd319d7aad8b011c88c8048bbaec78cac79d0acdeed98f446777014b46e419ed86e39ea0f7fd9f888c786a4d
+  metadata.gz: a5d082d14302d38e2acf89ab1275004ae69f78087e40da2a68c15e25770a9ebfc7fa2cb8e9e7268d132e2ade6c3d6db9e259ab3e8254ae476fcd1d46a4f40eb2
+  data.tar.gz: c1a9bda279afa44c5cb5a41c4ac792a62af317d91985799a14609c7b6d33e5494d322ad14262281b3fead76cc6471bebd7b507e1158dea7f9278196b36ab1726

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+# 1.3.0 (TBD)
+
+Additions:
+
+* Decoupled storage: `Burner::Disks` factory, `Burner::Disks::Local` reference implementation, and `b/io/*` `disk` option for configuring IO jobs to use custom disks.
+# 1.2.0 (November 25th, 2020)
+
+#### Enhancements:
+
+* All for a pipeline to be configured with null steps.  When null, just execute all jobs in positional order.
+* Allow Collection::Transform job attributes to implicitly start from a resolve transformer.  `explicit: true` can be passed in as an option in case the desire is to begin from the record and not a specific value.
+
+#### Added Jobs:
+
+* b/collection/nested_aggregate
+# 1.1.0 (November 16, 2020)
+
+Added Jobs:
+
+* b/collection/coalesce
+* b/collection/group
+
+# 1.0.0 (November 5th, 2020)
+
+Initial version publication.
+
 # 0.0.1
 
 Shell

data/README.md

@@ -42,7 +42,7 @@ pipeline = {
     {
       name: :output_value,
       type: 'b/echo',
-      message: 'The current value is: {__value}'
+      message: 'The current value is: {__default_register}'
     },
     {
       name: :parse,
@@ -89,40 +89,22 @@ Some notes:
 
 * Some values are able to be string-interpolated using the provided Payload#params.  This allows for the passing runtime configuration/data into pipelines/jobs.
 * The job's ID can be accessed using the `__id` key.
-* The current job's payload value can be accessed using the `__value` key.
+* The current payload registers' values can be accessed using the `__<register_name>_register` key.
 * Jobs can be re-used (just like the output_id and output_value jobs).
+* If steps is nil then all jobs will execute in their declared order.
 
 ### Capturing Feedback / Output
 
 By default, output will be emitted to `$stdout`.  You can add or change listeners by passing in optional values into Pipeline#execute.  For example, say we wanted to capture the output from our json-to-yaml example:
 
 ````ruby
-class StringOut
-  def initialize
-    @io = StringIO.new
-  end
-
-  def puts(msg)
-    tap { io.write("#{msg}\n") }
-  end
-
-  def read
-    io.rewind
-    io.read
-  end
-
-  private
-
-  attr_reader :io
-end
-
-string_out = StringOut.new
-output     = Burner::Output.new(outs: string_out)
-payload    = Burner::Payload.new(params: params)
+io      = StringIO.new
+output  = Burner::Output.new(outs: io)
+payload = Burner::Payload.new(params: params)
 
 Burner::Pipeline.make(pipeline).execute(output: output, payload: payload)
 
-log = string_out.read
+log = io.string
 ````
 
 The value of `log` should now look similar to:
@@ -181,7 +163,7 @@ jobs:
 
   - name: output_value
     type: b/echo
-    message: 'The current value is: {__value}'
+    message: 'The current value is: {__default_register}'
 
   - name: parse
     type: b/deserialize/json
@@ -234,11 +216,14 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 #### Collection
 
 * **b/collection/arrays_to_objects** [mappings, register]: Convert an array of arrays to an array of objects.
+* **b/collection/coalesce** [register, grouped_register, key_mappings, keys, separator]: Merge two datasets together based on the key values of one dataset (array) with a grouped dataset (hash).
 * **b/collection/concatenate** [from_registers, to_register]: Concatenate each from_register's value and place the newly concatenated array into the to_register.  Note: this does not do any deep copying and should be assumed it is shallow copying all objects.
 * **b/collection/graph** [config, key, register]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
+* **b/collection/group** [keys, register, separator]: Take a register's value (an array of objects) and group the objects by the specified keys.
+* **b/collection/nested_aggregate** [register, key_mappings, key, separator]: Traverse a set of objects, resolving key's value for each object, optionally copying down key_mappings to the child records, then merging all the inner records together.
 * **b/collection/objects_to_arrays** [mappings, register]: Convert an array of objects to an array of arrays.
 * **b/collection/shift** [amount, register]: Remove the first N number of elements from an array.
-* **b/collection/transform** [attributes, exclusive, separator, register]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.
+* **b/collection/transform** [attributes, exclusive, separator, register]: Iterate over all objects and transform each key per the attribute transformers specifications.  If exclusive is set to false then the current object will be overridden/merged.  Separator can also be set for key path support.  This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.  If an attribute is not set with `explicit: true` then it will automatically start from the key's value from the record.  If `explicit: true` is started, then it will start from the record itself.
 * **b/collection/unpivot** [pivot_set, register]: Take an array of objects and unpivot specific sets of keys into rows.  Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
 * **b/collection/validate** [invalid_register, join_char, message_key, register, separator, validations]: Take an array of objects, run it through each declared validator, and split the objects into two registers.  The valid objects will be split into the current register while the invalid ones will go into the invalid_register as declared.  Optional arguments, join_char and message_key, help determine the compiled error messages.  The separator option can be utilized to use dot-notation for validating keys.  See each validation's options by viewing their classes within the `lib/modeling/validations` directory.
 * **b/collection/values** [include_keys, register]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
@@ -251,9 +236,11 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 
 #### IO
 
-* **b/io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
-* **b/io/read** [binary, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
-* **b/io/write** [binary, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
+By default all jobs will use the `Burner::Disks::Local` disk for its persistence.  But this is configurable by implementing and registering custom disk-based classes in the `Burner::Disks` factory.  For example: a consumer application may also want to interact with cloud-based storage providers and could leverage this as its job library instead of implementing custom jobs.
+
+* **b/io/exist** [disk, path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
+* **b/io/read** [binary, disk, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
+* **b/io/write** [binary, disk, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
 
 #### Serialization
 
@@ -278,6 +265,8 @@ Notes:
 
 ### Adding & Registering Jobs
 
+Note: Jobs have to be registered with a type in the Burner::Jobs factory.  All jobs that ship with this library are prefixed with `b/` in their type in order to provide a namespace for 'burner-specific' jobs vs. externally provided jobs.
+
 Where this library shines is when additional jobs are plugged in.  Burner uses its `Burner::Jobs` class as its class-level registry built with [acts_as_hashable](https://github.com/bluemarblepayroll/acts_as_hashable)'s acts_as_hashable_factory directive.
 
 Let's say we would like to register a job to parse a CSV:
@@ -312,7 +301,7 @@ pipeline = {
     {
       name: :output_value,
       type: 'b/echo',
-      message: 'The current value is: {__value}'
+      message: 'The current value is: {__default_register}'
     },
     {
       name: :parse,

data/burner.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.add_dependency('hashematics', '~>1.1')
   s.add_dependency('hash_math', '~>1.2')
   s.add_dependency('objectable', '~>1.0')
-  s.add_dependency('realize', '~>1.2')
+  s.add_dependency('realize', '~>1.3')
   s.add_dependency('stringento', '~>2.1')
 
   s.add_development_dependency('guard-rspec', '~>4.7')

data/lib/burner.rb

@@ -10,6 +10,7 @@
 require 'acts_as_hashable'
 require 'benchmark'
 require 'csv'
+require 'fileutils'
 require 'forwardable'
 require 'hash_math'
 require 'hashematics'
@@ -23,9 +24,16 @@ require 'time'
 require 'yaml'
 
 # Common/Shared
+require_relative 'burner/disks'
 require_relative 'burner/modeling'
 require_relative 'burner/side_effects'
 require_relative 'burner/util'
 
 # Main Entrypoint(s)
 require_relative 'burner/cli'
+
+# Top-level namespace
+module Burner
+  # All jobs that need to reference the main register should use this constant.
+  DEFAULT_REGISTER = 'default'
+end

data/lib/burner/disks.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'disks/local'
+
+module Burner
+  # A factory to register and emit instances that conform to the Disk interface with requests
+  # the instance responds to: #exist?, #read, and #write.  See an example implementation within
+  # the lib/burner/disks directory.
+  #
+  # The benefit to this pluggable disk model is a consumer application can decide which file
+  # backend to use and how to store files.  For example: an application may choose to use
+  # some cloud provider with their own file store implementation.  This can be wrapped up
+  # in a Disk class and registered here and then referenced in the Pipeline's IO jobs.
+  class Disks
+    acts_as_hashable_factory
+
+    register 'local', '', Disks::Local
+  end
+end

data/lib/burner/disks/local.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Disks
+    # Operations against the local file system.
+    class Local
+      acts_as_hashable
+
+      # Check to see if the passed in path exists within the local file system.
+      # It will not make assumptions on what the 'file' is, only that it is recognized
+      # by Ruby's File class.
+      def exist?(path)
+        File.exist?(path)
+      end
+
+      # Open and read the contents of a local file.  If binary is passed in as true then the file
+      # will be opened in binary mode.
+      def read(path, binary: false)
+        File.open(path, read_mode(binary), &:read)
+      end
+
+      # Open and write the specified data to a local file.  If binary is passed in as true then
+      # the file will be opened in binary mode.  It is important to note that if the file's
+      # directory structure will be automatically created if it does not exist.
+      def write(path, data, binary: false)
+        ensure_directory_exists(path)
+
+        File.open(path, write_mode(binary)) { |io| io.write(data) }
+
+        path
+      end
+
+      private
+
+      def ensure_directory_exists(path)
+        dirname = File.dirname(path)
+
+        return if File.exist?(dirname)
+
+        FileUtils.mkdir_p(dirname)
+
+        nil
+      end
+
+      def write_mode(binary)
+        binary ? 'wb' : 'w'
+      end
+
+      def read_mode(binary)
+        binary ? 'rb' : 'r'
+      end
+    end
+  end
+end

data/lib/burner/job_with_register.rb

@@ -15,7 +15,7 @@ module Burner
   class JobWithRegister < Job
     attr_reader :register
 
-    def initialize(name:, register: '')
+    def initialize(name:, register: DEFAULT_REGISTER)
       super(name: name)
 
       @register = register.to_s

data/lib/burner/jobs.rb

@@ -23,8 +23,11 @@ module Burner
     register 'b/sleep',                        Library::Sleep
 
     register 'b/collection/arrays_to_objects', Library::Collection::ArraysToObjects
+    register 'b/collection/coalesce',          Library::Collection::Coalesce
     register 'b/collection/concatenate',       Library::Collection::Concatenate
     register 'b/collection/graph',             Library::Collection::Graph
+    register 'b/collection/group',             Library::Collection::Group
+    register 'b/collection/nested_aggregate',  Library::Collection::NestedAggregate
     register 'b/collection/objects_to_arrays', Library::Collection::ObjectsToArrays
     register 'b/collection/shift',             Library::Collection::Shift
     register 'b/collection/transform',         Library::Collection::Transform

data/lib/burner/library.rb

@@ -14,8 +14,11 @@ require_relative 'library/nothing'
 require_relative 'library/sleep'
 
 require_relative 'library/collection/arrays_to_objects'
+require_relative 'library/collection/coalesce'
 require_relative 'library/collection/concatenate'
 require_relative 'library/collection/graph'
+require_relative 'library/collection/group'
+require_relative 'library/collection/nested_aggregate'
 require_relative 'library/collection/objects_to_arrays'
 require_relative 'library/collection/shift'
 require_relative 'library/collection/transform'

data/lib/burner/library/collection/arrays_to_objects.rb

@@ -55,7 +55,7 @@ module Burner
       class ArraysToObjects < JobWithRegister
         attr_reader :mappings
 
-        def initialize(name:, mappings: [], register: '')
+        def initialize(name:, mappings: [], register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           @mappings = Modeling::KeyIndexMapping.array(mappings)

data/lib/burner/library/collection/coalesce.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # This is generally used right after the Group job has been executed on a separate
+      # dataset in a separate register.  This job can match up specified values in its dataset
+      # with lookup values in another.  If it finds a match then it will (shallow) copy over
+      # the values into the respective dataset.
+      #
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: array of objects.
+      class Coalesce < JobWithRegister
+        attr_reader :grouped_register, :key_mappings, :keys, :resolver
+
+        def initialize(
+          name:,
+          grouped_register:,
+          key_mappings: [],
+          keys: [],
+          register: DEFAULT_REGISTER,
+          separator: ''
+        )
+          super(name: name, register: register)
+
+          @grouped_register = grouped_register.to_s
+          @key_mappings     = Modeling::KeyMapping.array(key_mappings)
+          @keys             = Array(keys)
+          @resolver         = Objectable.resolver(separator: separator.to_s)
+
+          raise ArgumentError, 'at least one key is required' if @keys.empty?
+
+          freeze
+        end
+
+        def perform(output, payload)
+          payload[register] = array(payload[register])
+          count             = payload[register].length
+
+          output.detail("Coalescing based on key(s): #{keys} for #{count} records(s)")
+
+          payload[register].each do |record|
+            key    = make_key(record)
+            lookup = find_lookup(payload, key)
+
+            key_mappings.each do |key_mapping|
+              value = resolver.get(lookup, key_mapping.from)
+
+              resolver.set(record, key_mapping.to, value)
+            end
+          end
+        end
+
+        private
+
+        def find_lookup(payload, key)
+          (payload[grouped_register] || {})[key] || {}
+        end
+
+        def make_key(record)
+          keys.map { |key| resolver.get(record, key) }
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/collection/concatenate.rb

@@ -18,7 +18,7 @@ module Burner
       class Concatenate < Job
         attr_reader :from_registers, :to_register
 
-        def initialize(name:, from_registers: [], to_register: '')
+        def initialize(name:, from_registers: [], to_register: DEFAULT_REGISTER)
           super(name: name)
 
           @from_registers = Array(from_registers)

data/lib/burner/library/collection/graph.rb

@@ -18,7 +18,12 @@ module Burner
       class Graph < JobWithRegister
         attr_reader :key, :groups
 
-        def initialize(name:, key:, config: Hashematics::Configuration.new, register: '')
+        def initialize(
+          name:,
+          key:,
+          config: Hashematics::Configuration.new,
+          register: DEFAULT_REGISTER
+        )
           super(name: name, register: register)
 
           raise ArgumentError, 'key is required' if key.to_s.empty?

data/lib/burner/library/collection/group.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # Take a register's value (an array of objects) and group the objects by the specified keys.
+      # It essentially creates a hash from an array. This is useful for creating a O(1) lookup
+      # which can then be used in conjunction with the Coalesce Job for another array of data.
+      # It is worth noting that the resulting hashes values are singular objects and not an array
+      # like Ruby's Enumerable#group_by method.
+      #
+      # An example of this specific job:
+      #
+      # input: [{ id: 1, code: 'a' }, { id: 2, code: 'b' }]
+      # keys: [:code]
+      # output: { ['a'] => { id: 1, code: 'a' }, ['b'] => { id: 2, code: 'b' } }
+      #
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: hash.
+      class Group < JobWithRegister
+        attr_reader :keys, :resolver
+
+        def initialize(
+          name:,
+          keys: [],
+          register: DEFAULT_REGISTER,
+          separator: ''
+        )
+          super(name: name, register: register)
+
+          @keys     = Array(keys)
+          @resolver = Objectable.resolver(separator: separator.to_s)
+
+          raise ArgumentError, 'at least one key is required' if @keys.empty?
+
+          freeze
+        end
+
+        def perform(output, payload)
+          payload[register] = array(payload[register])
+          count             = payload[register].length
+
+          output.detail("Grouping based on key(s): #{keys} for #{count} records(s)")
+
+          grouped_records = payload[register].each_with_object({}) do |record, memo|
+            key       = make_key(record)
+            memo[key] = record
+          end
+
+          payload[register] = grouped_records
+        end
+
+        private
+
+        def make_key(record)
+          keys.map { |key| resolver.get(record, key) }
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/collection/nested_aggregate.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # Iterate over a collection of objects, calling key on each object, then aggregating the
+      # returns of key together into one array.  This new derived array will be set as the value
+      # for the payload's register.  Leverage the key_mappings option to optionally copy down
+      # keys and values from outer to inner records.  This job is particularly useful
+      # if you have nested arrays but wish to deal with each level/depth in the aggregate.
+      #
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: array of objects.
+      class NestedAggregate < JobWithRegister
+        attr_reader :key, :key_mappings, :resolver
+
+        def initialize(name:, key:, key_mappings: [], register: DEFAULT_REGISTER, separator: '')
+          super(name: name, register: register)
+
+          raise ArgumentError, 'key is required' if key.to_s.empty?
+
+          @key          = key.to_s
+          @key_mappings = Modeling::KeyMapping.array(key_mappings)
+          @resolver     = Objectable.resolver(separator: separator.to_s)
+
+          freeze
+        end
+
+        def perform(output, payload)
+          records = array(payload[register])
+          count   = records.length
+
+          output.detail("Aggregating on key: #{key} for #{count} records(s)")
+
+          # Outer loop on parent records
+          payload[register] = records.each_with_object([]) do |record, memo|
+            inner_records = resolver.get(record, key)
+
+            # Inner loop on child records
+            array(inner_records).each do |inner_record|
+              memo << copy_key_mappings(record, inner_record)
+            end
+          end
+        end
+
+        private
+
+        def copy_key_mappings(source_record, destination_record)
+          key_mappings.each do |key_mapping|
+            value = resolver.get(source_record, key_mapping.from)
+
+            resolver.set(destination_record, key_mapping.to, value)
+          end
+
+          destination_record
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/collection/objects_to_arrays.rb

@@ -58,7 +58,7 @@ module Burner
         # nested hashes then set separator to '.'.  For more information, see the underlying
         # library that supports this dot-notation concept:
         #   https://github.com/bluemarblepayroll/objectable
-        def initialize(name:, mappings: [], register: '', separator: '')
+        def initialize(name:, mappings: [], register: DEFAULT_REGISTER, separator: '')
           super(name: name, register: register)
 
           @mappings = Modeling::KeyIndexMapping.array(mappings)

data/lib/burner/library/collection/shift.rb

@@ -23,7 +23,7 @@ module Burner
 
         attr_reader :amount
 
-        def initialize(name:, amount: DEFAULT_AMOUNT, register: '')
+        def initialize(name:, amount: DEFAULT_AMOUNT, register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           @amount = amount.to_i

data/lib/burner/library/collection/transform.rb

@@ -27,7 +27,13 @@ module Burner
                     :exclusive,
                     :resolver
 
-        def initialize(name:, attributes: [], exclusive: false, register: '', separator: BLANK)
+        def initialize(
+          name:,
+          attributes: [],
+          exclusive: false,
+          register: DEFAULT_REGISTER,
+          separator: BLANK
+        )
           super(name: name, register: register)
 
           @resolver  = Objectable.resolver(separator: separator)

data/lib/burner/library/collection/unpivot.rb

@@ -19,7 +19,11 @@ module Burner
       class Unpivot < JobWithRegister
         attr_reader :unpivot
 
-        def initialize(name:, pivot_set: HashMath::Unpivot::PivotSet.new, register: '')
+        def initialize(
+          name:,
+          pivot_set: HashMath::Unpivot::PivotSet.new,
+          register: DEFAULT_REGISTER
+        )
           super(name: name, register: register)
 
           @unpivot = HashMath::Unpivot.new(pivot_set)

data/lib/burner/library/collection/validate.rb

@@ -33,7 +33,7 @@ module Burner
           invalid_register: DEFAULT_INVALID_REGISTER,
           join_char: DEFAULT_JOIN_CHAR,
           message_key: DEFAULT_MESSAGE_KEY,
-          register: '',
+          register: DEFAULT_REGISTER,
           separator: '',
           validations: []
         )

data/lib/burner/library/collection/values.rb

@@ -19,7 +19,7 @@ module Burner
       class Values < JobWithRegister
         attr_reader :include_keys
 
-        def initialize(name:, include_keys: false, register: '')
+        def initialize(name:, include_keys: false, register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           @include_keys = include_keys || false

data/lib/burner/library/deserialize/yaml.rb

@@ -20,7 +20,7 @@ module Burner
       class Yaml < JobWithRegister
         attr_reader :safe
 
-        def initialize(name:, register: '', safe: true)
+        def initialize(name:, register: DEFAULT_REGISTER, safe: true)
           super(name: name, register: register)
 
           @safe = safe

data/lib/burner/library/io/exist.rb

@@ -7,8 +7,6 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
-
 module Burner
   module Library
     module IO
@@ -17,13 +15,14 @@ module Burner
       #
       # Note: this does not use Payload#registers.
       class Exist < Job
-        attr_reader :path, :short_circuit
+        attr_reader :disk, :path, :short_circuit
 
-        def initialize(name:, path:, short_circuit: false)
+        def initialize(name:, path:, disk: {}, short_circuit: false)
           super(name: name)
 
           raise ArgumentError, 'path is required' if path.to_s.empty?
 
+          @disk          = Disks.make(disk)
           @path          = path.to_s
           @short_circuit = short_circuit || false
         end
@@ -31,7 +30,7 @@ module Burner
         def perform(output, payload)
           compiled_path = job_string_template(path, output, payload)
 
-          exists = File.exist?(compiled_path)
+          exists = disk.exist?(compiled_path)
           verb   = exists ? 'does' : 'does not'
 
           output.detail("The path: #{compiled_path} #{verb} exist")

data/lib/burner/library/io/{base.rb → open_file_base.rb}

@@ -10,16 +10,20 @@
 module Burner
   module Library
     module IO
-      # Common configuration/code for all IO Job subclasses.
-      class Base < JobWithRegister
-        attr_reader :path
+      # Common configuration/code for all IO Job subclasses that open a file.
+      class OpenFileBase < JobWithRegister
+        attr_reader :binary, :disk, :path
 
-        def initialize(name:, path:, register: '')
+        def initialize(name:, path:, binary: false, disk: {}, register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           raise ArgumentError, 'path is required' if path.to_s.empty?
 
-          @path = path.to_s
+          @binary = binary || false
+          @disk   = Disks.make(disk)
+          @path   = path.to_s
+
+          freeze
         end
       end
     end

data/lib/burner/library/io/read.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'open_file_base'
 
 module Burner
   module Library
@@ -16,29 +16,13 @@ module Burner
       #
       # Expected Payload[register] input: nothing.
       # Payload[register] output: contents of the specified file.
-      class Read < Base
-        attr_reader :binary
-
-        def initialize(name:, path:, binary: false, register: '')
-          super(name: name, path: path, register: register)
-
-          @binary = binary || false
-
-          freeze
-        end
-
+      class Read < OpenFileBase
         def perform(output, payload)
           compiled_path = job_string_template(path, output, payload)
 
           output.detail("Reading: #{compiled_path}")
 
-          payload[register] = File.open(compiled_path, mode, &:read)
-        end
-
-        private
-
-        def mode
-          binary ? 'rb' : 'r'
+          payload[register] = disk.read(compiled_path, binary: binary)
         end
       end
     end

data/lib/burner/library/io/write.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'open_file_base'
 
 module Burner
   module Library
@@ -16,54 +16,27 @@ module Burner
       #
       # Expected Payload[register] input: anything.
       # Payload[register] output: whatever was passed in.
-      class Write < Base
-        attr_reader :binary
-
-        def initialize(name:, path:, binary: false, register: '')
-          super(name: name, path: path, register: register)
-
-          @binary = binary || false
-
-          freeze
-        end
-
+      class Write < OpenFileBase
         def perform(output, payload)
-          compiled_path = job_string_template(path, output, payload)
-
-          ensure_directory_exists(output, compiled_path)
+          logical_filename  = job_string_template(path, output, payload)
+          physical_filename = nil
 
-          output.detail("Writing: #{compiled_path}")
+          output.detail("Writing: #{logical_filename}")
 
           time_in_seconds = Benchmark.measure do
-            File.open(compiled_path, mode) { |io| io.write(payload[register]) }
+            physical_filename = disk.write(logical_filename, payload[register], binary: binary)
           end.real
 
+          output.detail("Wrote to: #{physical_filename}")
+
           side_effect = SideEffects::WrittenFile.new(
-            logical_filename: compiled_path,
-            physical_filename: compiled_path,
+            logical_filename: logical_filename,
+            physical_filename: physical_filename,
             time_in_seconds: time_in_seconds
           )
 
           payload.add_side_effect(side_effect)
         end
-
-        private
-
-        def ensure_directory_exists(output, compiled_path)
-          dirname = File.dirname(compiled_path)
-
-          return if File.exist?(dirname)
-
-          output.detail("Outer directory does not exist, creating: #{dirname}")
-
-          FileUtils.mkdir_p(dirname)
-
-          nil
-        end
-
-        def mode
-          binary ? 'wb' : 'w'
-        end
       end
     end
   end

data/lib/burner/library/value/copy.rb

@@ -12,14 +12,14 @@ module Burner
     module Value
       # Copy one value in a register to another.  Note that this does *not* perform any type of
       # deep copy, it simply points one register's value to another.  If you decide to later mutate
-      # one register, you are *not* guaranteed to not mutate the other.
+      # one register then you may mutate the other.
       #
       # Expected Payload[from_register] input: anything.
       # Payload[to_register] output: whatever value was specified in the from_register.
       class Copy < Job
         attr_reader :from_register, :to_register
 
-        def initialize(name:, to_register: '', from_register: '')
+        def initialize(name:, to_register: DEFAULT_REGISTER, from_register: DEFAULT_REGISTER)
           super(name: name)
 
           @from_register = from_register.to_s

data/lib/burner/library/value/static.rb

@@ -17,7 +17,7 @@ module Burner
       class Static < JobWithRegister
         attr_reader :value
 
-        def initialize(name:, register: '', value: nil)
+        def initialize(name:, register: DEFAULT_REGISTER, value: nil)
           super(name: name, register: register)
 
           @value = value

data/lib/burner/modeling.rb

@@ -10,4 +10,5 @@
 require_relative 'modeling/attribute'
 require_relative 'modeling/attribute_renderer'
 require_relative 'modeling/key_index_mapping'
+require_relative 'modeling/key_mapping'
 require_relative 'modeling/validations'

data/lib/burner/modeling/attribute.rb

@@ -13,19 +13,37 @@ module Burner
     # to set the key to.  The transformers that can be passed in can be any Realize::Transformers
     # subclasses.  For more information, see the Realize library at:
     # https://github.com/bluemarblepayroll/realize
+    #
+    # Note that if explicit: true is set then no transformers will be automatically injected.
+    # If explicit is not true (default) then it will have a resolve job automatically injected
+    # in the beginning of the pipeline.  This is the observed default behavior, with the
+    # exception having to be initially cross-mapped using a custom resolve transformation.
     class Attribute
       acts_as_hashable
 
+      RESOLVE_TYPE = 'r/value/resolve'
+
       attr_reader :key, :transformers
 
-      def initialize(key:, transformers: [])
+      def initialize(key:, explicit: false, transformers: [])
         raise ArgumentError, 'key is required' if key.to_s.empty?
 
         @key          = key.to_s
-        @transformers = Realize::Transformers.array(transformers)
+        @transformers = base_transformers(explicit) + Realize::Transformers.array(transformers)
 
         freeze
       end
+
+      private
+
+      # When explicit, this will return an empty array.
+      # When not explicit, this will return an array with a basic transformer that simply
+      # gets the key's value.  This establishes a good majority base case.
+      def base_transformers(explicit)
+        return [] if explicit
+
+        [Realize::Transformers.make(type: RESOLVE_TYPE, key: key)]
+      end
     end
   end
 end

data/lib/burner/modeling/key_mapping.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Modeling
+    # Generic mapping from a key to another key.  The argument 'to' is optional
+    # and if it is blank then the 'from' value will be used for the 'to' as well.
+    class KeyMapping
+      acts_as_hashable
+
+      attr_reader :from, :to
+
+      def initialize(from:, to: '')
+        raise ArgumentError, 'from is required' if from.to_s.empty?
+
+        @from = from.to_s
+        @to   = to.to_s.empty? ? @from : to.to_s
+
+        freeze
+      end
+    end
+  end
+end

data/lib/burner/modeling/validations/base.rb

@@ -27,7 +27,7 @@ module Burner
         end
 
         def message
-          @message.to_s.empty? ? "#{key}#{default_message}" : @message.to_s
+          @message.to_s.empty? ? "#{key} #{default_message}" : @message.to_s
         end
       end
     end

data/lib/burner/modeling/validations/blank.rb

@@ -16,14 +16,18 @@ module Burner
       class Blank < Base
         acts_as_hashable
 
+        BLANK_RE = /\A[[:space:]]*\z/.freeze
+
         def valid?(object, resolver)
-          resolver.get(object, key).to_s.empty?
+          value = resolver.get(object, key).to_s
+
+          value.empty? || BLANK_RE.match?(value)
         end
 
         private
 
         def default_message
-          ' must be blank'
+          'must be blank'
         end
       end
     end

data/lib/burner/modeling/validations/present.rb

@@ -7,23 +7,23 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'blank'
 
 module Burner
   module Modeling
     class Validations
       # Check if a value is present.  If it is blank (null or empty) then it is invalid.
-      class Present < Base
+      class Present < Blank
         acts_as_hashable
 
         def valid?(object_value, resolver)
-          !resolver.get(object_value, key).to_s.empty?
+          !super(object_value, resolver)
         end
 
         private
 
         def default_message
-          ' is required'
+          'is required'
         end
       end
     end

data/lib/burner/pipeline.rb

@@ -14,7 +14,8 @@ require_relative 'step'
 
 module Burner
   # The root package.  A Pipeline contains the job configurations along with the steps.  The steps
-  # referens jobs and tell you the order of the jobs to run.
+  # reference jobs and tell you the order of the jobs to run.  If steps is nil then all jobs
+  # will execute in their declared order.
   class Pipeline
     acts_as_hashable
 
@@ -23,14 +24,16 @@ module Burner
 
     attr_reader :steps
 
-    def initialize(jobs: [], steps: [])
+    def initialize(jobs: [], steps: nil)
       jobs = Jobs.array(jobs)
 
       assert_unique_job_names(jobs)
 
       jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
 
-      @steps = Array(steps).map do |step_name|
+      step_names = steps ? Array(steps) : jobs_by_name.keys
+
+      @steps = step_names.map do |step_name|
         job = jobs_by_name[step_name.to_s]
 
         raise JobNotFoundError, "#{step_name} was not declared as a job" unless job

data/lib/burner/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Burner
-  VERSION = '1.0.0-alpha.13'
+  VERSION = '1.3.0-alpha'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: burner
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.alpha.13
+  version: 1.3.0.pre.alpha
 platform: ruby
 authors:
 - Matthew Ruggio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-28 00:00:00.000000000 Z
+date: 2020-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: stringento
   requirement: !ruby/object:Gem::Requirement
@@ -220,13 +220,18 @@ files:
 - exe/burner
 - lib/burner.rb
 - lib/burner/cli.rb
+- lib/burner/disks.rb
+- lib/burner/disks/local.rb
 - lib/burner/job.rb
 - lib/burner/job_with_register.rb
 - lib/burner/jobs.rb
 - lib/burner/library.rb
 - lib/burner/library/collection/arrays_to_objects.rb
+- lib/burner/library/collection/coalesce.rb
 - lib/burner/library/collection/concatenate.rb
 - lib/burner/library/collection/graph.rb
+- lib/burner/library/collection/group.rb
+- lib/burner/library/collection/nested_aggregate.rb
 - lib/burner/library/collection/objects_to_arrays.rb
 - lib/burner/library/collection/shift.rb
 - lib/burner/library/collection/transform.rb
@@ -237,8 +242,8 @@ files:
 - lib/burner/library/deserialize/json.rb
 - lib/burner/library/deserialize/yaml.rb
 - lib/burner/library/echo.rb
-- lib/burner/library/io/base.rb
 - lib/burner/library/io/exist.rb
+- lib/burner/library/io/open_file_base.rb
 - lib/burner/library/io/read.rb
 - lib/burner/library/io/write.rb
 - lib/burner/library/nothing.rb
@@ -252,6 +257,7 @@ files:
 - lib/burner/modeling/attribute.rb
 - lib/burner/modeling/attribute_renderer.rb
 - lib/burner/modeling/key_index_mapping.rb
+- lib/burner/modeling/key_mapping.rb
 - lib/burner/modeling/validations.rb
 - lib/burner/modeling/validations/base.rb
 - lib/burner/modeling/validations/blank.rb