burner 1.6.0 → 1.9.0

data/lib/burner/library/param/from_register.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module Burner
+  module Library
+    module Param
+      # Copy a register's value into a param key.  Generally speaking you should only be
+      # mutating registers, that way the params stay true to the passed in params for the
+      # pipeline.  But this job is available in case a param needs to be updated.
+      #
+      # Expected Payload[register] input: anything.
+      # Payload.params(param_key) output: whatever value was specified in the register.
+      class FromRegister < Base
+        def perform(output, payload)
+          output.detail("Pushing value from register: #{register} to param: #{param_key}")
+
+          payload.update_param(param_key, payload[register])
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/param/to_register.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module Burner
+  module Library
+    module Param
+      # Copy a param key's value into a register.
+      #
+      # Expected Payload.param(param_key) input: anything.
+      # Payload[register] output: whatever value was specified as the param_key's value.
+      class ToRegister < Base
+        def perform(output, payload)
+          output.detail("Pushing value to register: #{register} from param: #{param_key}")
+
+          payload[register] = payload.param(param_key)
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/serialize/csv.rb

@@ -18,7 +18,7 @@ module Burner
       class Csv < JobWithRegister
         attr_reader :byte_order_mark
 
-        def initialize(name:, byte_order_mark: nil, register: DEFAULT_REGISTER)
+        def initialize(byte_order_mark: nil, name: '', register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           @byte_order_mark = Modeling::ByteOrderMark.resolve(byte_order_mark)

data/lib/burner/library/sleep.rb

@@ -15,7 +15,7 @@ module Burner
     class Sleep < Job
       attr_reader :seconds
 
-      def initialize(name:, seconds: 0)
+      def initialize(name: '', seconds: 0)
         super(name: name)
 
         @seconds = seconds.to_f

data/lib/burner/library/value/copy.rb

@@ -19,7 +19,7 @@ module Burner
       class Copy < Job
         attr_reader :from_register, :to_register
 
-        def initialize(name:, to_register: DEFAULT_REGISTER, from_register: DEFAULT_REGISTER)
+        def initialize(from_register: DEFAULT_REGISTER, name: '', to_register: DEFAULT_REGISTER)
           super(name: name)
 
           @from_register = from_register.to_s

data/lib/burner/library/value/nest.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Value
+      # This job will nest the current value within a new outer hash.  The specified key
+      # passed in will be the corresponding new hash key entry for the existing value.
+      #
+      # Expected Payload[from_register] input: anything.
+      # Payload[to_register] output: hash.
+      class Nest < JobWithRegister
+        DEFAULT_KEY = 'key'
+
+        attr_reader :key
+
+        def initialize(key: DEFAULT_KEY, name: '', register: Burner::DEFAULT_REGISTER)
+          super(name: name, register: register)
+
+          @key = key.to_s
+
+          freeze
+        end
+
+        def perform(_output, payload)
+          payload[register] = { key => payload[register] }
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/value/static.rb

@@ -17,7 +17,7 @@ module Burner
       class Static < JobWithRegister
         attr_reader :value
 
-        def initialize(name:, register: DEFAULT_REGISTER, value: nil)
+        def initialize(name: '', register: DEFAULT_REGISTER, value: nil)
           super(name: name, register: register)
 
           @value = value

data/lib/burner/library/value/transform.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Value
+      # Transform the current value of the register through a Realize::Pipeline.  This will
+      # transform the entire value, as opposed to the b/collection/transform job, which will
+      # iterate over each row/record in a dataset and transform each row/record.
+      #
+      # Expected Payload[register] input: anything.
+      # Payload[register] output: anything.
+      class Transform < JobWithRegister
+        attr_reader :pipeline
+
+        def initialize(name: '', register: DEFAULT_REGISTER, separator: '', transformers: [])
+          super(name: name, register: register)
+
+          resolver = Objectable.resolver(separator: separator)
+
+          @pipeline = Realize::Pipeline.new(transformers, resolver: resolver)
+
+          freeze
+        end
+
+        def perform(_output, payload)
+          payload[register] = pipeline.transform(payload[register], payload.time)
+        end
+      end
+    end
+  end
+end

data/lib/burner/payload.rb

@@ -7,6 +7,8 @@
 # LICENSE file in the root directory of this source tree.
 #
 
+require_relative 'data'
+
 module Burner
   # The input for all Job#perform methods.  The main notion of this object is its 'registers'
   # attribute.  This registers attribute is a key-indifferent hash, accessible on Payload using
@@ -24,10 +26,15 @@ module Burner
   # The 'time' attribute is important in that it should for the replaying of pipelines and jobs.
   # Instead of having job's utilizing Time.now, Date.today, etc... they should rather opt to
   # use this value instead.
+  #
+  # The notion of params are somewhat conflated with registers here.  Both are hashes and both
+  # serve as data stores. The difference is registers are really meant to be the shared-data
+  # repository across jobs, while params are, more or less, the input into the _pipeline_.
+  # It is debatable if mutation of the params should be allowed but the design decision was made
+  # early on to allow for params to be mutable albeit with registers being the preferred mutable
+  # store.
   class Payload
-    attr_reader :params,
-                :registers,
-                :side_effects,
+    attr_reader :side_effects,
                 :time
 
     def initialize(
@@ -36,12 +43,32 @@ module Burner
       side_effects: [],
       time: Time.now.utc
     )
-      @params       = params || {}
-      @registers    = {}
+      @params       = Data.new(params)
+      @registers    = Data.new(registers)
       @side_effects = side_effects || []
       @time         = time || Time.now.utc
+    end
 
-      add_registers(registers)
+    # Backwards compatibility.  This allows for control over the underlying data structure
+    # while still maintaining the same public API as before.
+    def params
+      @params.to_h
+    end
+
+    # Backwards compatibility.  This allows for control over the underlying data structure
+    # while still maintaining the same public API as before.
+    def registers
+      @registers.to_h
+    end
+
+    # Law of Demeter: While params is an accessible hash, it is preferred that the
+    # public class methods are used.
+    def param(key)
+      @params[key]
+    end
+
+    def update_param(key, value)
+      tap { @params[key] = value }
     end
 
     # Add a side effect of a job.  This helps to keep track of things jobs do outside of its
@@ -52,12 +79,12 @@ module Burner
 
     # Set a register's value.
     def []=(key, value)
-      set(key, value)
+      @registers[key] = value
     end
 
     # Retrieve a register's value.
     def [](key)
-      registers[key.to_s]
+      @registers[key]
     end
 
     # Set halt_pipeline to true.  This will indicate to the pipeline to stop all
@@ -71,14 +98,11 @@ module Burner
       @halt_pipeline || false
     end
 
-    private
-
-    def set(key, value)
-      registers[key.to_s] = value
-    end
+    def params_and_registers_hash
+      registers_hash = @registers.transform_keys { |key| "__#{key}_register" }
+      params_hash    = @params.to_h
 
-    def add_registers(registers)
-      (registers || {}).each { |k, v| set(k, v) }
+      params_hash.merge(registers_hash)
     end
   end
 end

data/lib/burner/pipeline.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'jobs'
+require_relative 'job_set'
 require_relative 'output'
 require_relative 'payload'
 require_relative 'step'
@@ -19,27 +19,13 @@ module Burner
   class Pipeline
     acts_as_hashable
 
-    class JobNotFoundError < StandardError; end
-    class DuplicateJobNameError < StandardError; end
-
     attr_reader :steps
 
     def initialize(jobs: [], steps: nil)
-      jobs = Jobs.array(jobs)
-
-      assert_unique_job_names(jobs)
-
-      jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
-
-      step_names = steps ? Array(steps) : jobs_by_name.keys
-
-      @steps = step_names.map do |step_name|
-        job = jobs_by_name[step_name.to_s]
-
-        raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
-
-        Step.new(job)
-      end
+      @steps = JobSet
+               .new(jobs)
+               .jobs(steps)
+               .map { |job| Step.new(job) }
     end
 
     # The main entry-point for kicking off a pipeline.
@@ -68,22 +54,8 @@ module Burner
 
     private
 
-    def assert_unique_job_names(jobs)
-      unique_job_names = Set.new
-
-      jobs.each do |job|
-        if unique_job_names.include?(job.name)
-          raise DuplicateJobNameError, "job with name: #{job.name} already declared"
-        end
-
-        unique_job_names << job.name
-      end
-
-      nil
-    end
-
     def output_params(params, output)
-      if params.keys.any?
+      if params.any?
         output.write('Parameters:')
       else
         output.write('No parameters passed in.')

data/lib/burner/util.rb

@@ -8,4 +8,5 @@
 #
 
 require_relative 'util/arrayable'
+require_relative 'util/keyable'
 require_relative 'util/string_template'

data/lib/burner/util/keyable.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Util
+    # Provide helper methods for keys.
+    module Keyable
+      def make_key(record, keys, resolver, insensitive)
+        keys.map do |key|
+          value = resolver.get(record, key)
+
+          insensitive ? value.to_s.downcase : value
+        end
+      end
+    end
+  end
+end

data/lib/burner/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Burner
-  VERSION = '1.6.0'
+  VERSION = '1.9.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: burner
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.9.0
 platform: ruby
 authors:
 - Matthew Ruggio
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-22 00:00:00.000000000 Z
+date: 2021-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -220,6 +220,7 @@ files:
 - ".gitignore"
 - ".rubocop.yml"
 - ".ruby-version"
+- ".tool-versions"
 - ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -234,9 +235,11 @@ files:
 - exe/burner
 - lib/burner.rb
 - lib/burner/cli.rb
+- lib/burner/data.rb
 - lib/burner/disks.rb
 - lib/burner/disks/local.rb
 - lib/burner/job.rb
+- lib/burner/job_set.rb
 - lib/burner/job_with_register.rb
 - lib/burner/jobs.rb
 - lib/burner/library.rb
@@ -246,7 +249,9 @@ files:
 - lib/burner/library/collection/graph.rb
 - lib/burner/library/collection/group.rb
 - lib/burner/library/collection/nested_aggregate.rb
+- lib/burner/library/collection/number.rb
 - lib/burner/library/collection/objects_to_arrays.rb
+- lib/burner/library/collection/pivot.rb
 - lib/burner/library/collection/shift.rb
 - lib/burner/library/collection/transform.rb
 - lib/burner/library/collection/unpivot.rb
@@ -264,12 +269,17 @@ files:
 - lib/burner/library/io/row_reader.rb
 - lib/burner/library/io/write.rb
 - lib/burner/library/nothing.rb
+- lib/burner/library/param/base.rb
+- lib/burner/library/param/from_register.rb
+- lib/burner/library/param/to_register.rb
 - lib/burner/library/serialize/csv.rb
 - lib/burner/library/serialize/json.rb
 - lib/burner/library/serialize/yaml.rb
 - lib/burner/library/sleep.rb
 - lib/burner/library/value/copy.rb
+- lib/burner/library/value/nest.rb
 - lib/burner/library/value/static.rb
+- lib/burner/library/value/transform.rb
 - lib/burner/modeling.rb
 - lib/burner/modeling/attribute.rb
 - lib/burner/modeling/attribute_renderer.rb
@@ -288,6 +298,7 @@ files:
 - lib/burner/step.rb
 - lib/burner/util.rb
 - lib/burner/util/arrayable.rb
+- lib/burner/util/keyable.rb
 - lib/burner/util/string_template.rb
 - lib/burner/version.rb
 homepage: https://github.com/bluemarblepayroll/burner
@@ -299,7 +310,7 @@ metadata:
   documentation_uri: https://www.rubydoc.info/gems/burner
   homepage_uri: https://github.com/bluemarblepayroll/burner
   source_code_uri: https://github.com/bluemarblepayroll/burner
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -315,7 +326,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Declarative and extendable processing pipeline
 test_files: []