burner 1.0.0.pre.alpha.1 → 1.0.0.pre.alpha.6

data/lib/burner/{jobs → library}/io/base.rb

@@ -8,7 +8,7 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     module IO
       # Common configuration/code for all IO Job subclasses.
       class Base < Job
@@ -21,12 +21,6 @@ module Burner
 
           @path = path.to_s
         end
-
-        private
-
-        def compile_path(params)
-          eval_string_template(path, params)
-        end
       end
     end
   end

data/lib/burner/{jobs → library}/io/exist.rb

@@ -10,10 +10,12 @@
 require_relative 'base'
 
 module Burner
-  class Jobs
+  module Library
     module IO
       # Check to see if a file exists.  If short_circuit is set to true and the file
       # does not exist then the job will return false and short circuit the pipeline.
+      #
+      # Note: this does not use Payload#value.
       class Exist < Base
         attr_reader :short_circuit
 
@@ -25,8 +27,8 @@ module Burner
           freeze
         end
 
-        def perform(output, _payload, params)
-          compiled_path = compile_path(params)
+        def perform(output, payload)
+          compiled_path = job_string_template(path, output, payload)
 
           exists = File.exist?(compiled_path)
           verb   = exists ? 'does' : 'does not'

data/lib/burner/{jobs → library}/io/read.rb

@@ -10,9 +10,12 @@
 require_relative 'base'
 
 module Burner
-  class Jobs
+  module Library
     module IO
       # Read value from disk.
+      #
+      # Expected Payload#value input: nothing.
+      # Payload#value output: contents of the specified file.
       class Read < Base
         attr_reader :binary
 
@@ -24,8 +27,8 @@ module Burner
           freeze
         end
 
-        def perform(output, payload, params)
-          compiled_path = compile_path(params)
+        def perform(output, payload)
+          compiled_path = job_string_template(path, output, payload)
 
           output.detail("Reading: #{compiled_path}")
 

data/lib/burner/{jobs → library}/io/write.rb

@@ -10,9 +10,12 @@
 require_relative 'base'
 
 module Burner
-  class Jobs
+  module Library
     module IO
       # Write value to disk.
+      #
+      # Expected Payload#value input: anything.
+      # Payload#value output: whatever was passed in.
       class Write < Base
         attr_reader :binary
 
@@ -24,8 +27,8 @@ module Burner
           freeze
         end
 
-        def perform(output, payload, params)
-          compiled_path = compile_path(params)
+        def perform(output, payload)
+          compiled_path = job_string_template(path, output, payload)
 
           ensure_directory_exists(output, compiled_path)
 
@@ -35,12 +38,14 @@ module Burner
             File.open(compiled_path, mode) { |io| io.write(payload.value) }
           end.real
 
-          payload.add_written_file(
+          side_effect = SideEffects::WrittenFile.new(
             logical_filename: compiled_path,
             physical_filename: compiled_path,
             time_in_seconds: time_in_seconds
           )
 
+          payload.add_side_effect(side_effect)
+
           nil
         end
 

data/lib/burner/library/serialize/csv.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Serialize
+      # Take an array of arrays and create a CSV.
+      #
+      # Expected Payload#value input: array of arrays.
+      # Payload#value output: a serialized CSV string.
+      class Csv < Job
+        def perform(_output, payload)
+          payload.value = CSV.generate(options) do |csv|
+            array(payload.value).each do |row|
+              csv << row
+            end
+          end
+
+          nil
+        end
+
+        private
+
+        def options
+          {
+            headers: false,
+            write_headers: false
+          }
+        end
+      end
+    end
+  end
+end

data/lib/burner/{jobs → library}/serialize/json.rb

@@ -8,11 +8,14 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     module Serialize
       # Treat value like a Ruby object and serialize it using JSON.
+      #
+      # Expected Payload#value input: anything.
+      # Payload#value output: string representing the output of the JSON serializer.
       class Json < Job
-        def perform(_output, payload, _params)
+        def perform(_output, payload)
           payload.value = payload.value.to_json
 
           nil

data/lib/burner/{jobs → library}/serialize/yaml.rb

@@ -8,11 +8,14 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     module Serialize
       # Treat value like a Ruby object and serialize it using YAML.
+      #
+      # Expected Payload#value input: anything.
+      # Payload#value output: string representing the output of the YAML serializer.
       class Yaml < Job
-        def perform(_output, payload, _params)
+        def perform(_output, payload)
           payload.value = payload.value.to_yaml
 
           nil

data/lib/burner/{jobs/set.rb → library/set_value.rb}

@@ -8,9 +8,12 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     # Arbitrarily set value
-    class Set < Job
+    #
+    # Expected Payload#value input: anything.
+    # Payload#value output: whatever value was specified in this job.
+    class SetValue < Job
       attr_reader :value
 
       def initialize(name:, value: nil)
@@ -21,7 +24,7 @@ module Burner
         freeze
       end
 
-      def perform(_output, payload, _params)
+      def perform(_output, payload)
         payload.value = value
 
         nil

data/lib/burner/{jobs → library}/sleep.rb

@@ -8,8 +8,10 @@
 #
 
 module Burner
-  class Jobs
+  module Library
     # Arbitrarily put thread to sleep for X number of seconds
+    #
+    # Payload#value output: whatever value was specified in this job.
     class Sleep < Job
       attr_reader :seconds
 
@@ -21,7 +23,7 @@ module Burner
         freeze
       end
 
-      def perform(output, _payload, _params)
+      def perform(output, _payload)
         output.detail("Going to sleep for #{seconds} second(s)")
 
         Kernel.sleep(seconds)

data/lib/burner/modeling.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'modeling/attribute'
+require_relative 'modeling/attribute_renderer'
+require_relative 'modeling/key_index_mapping'

data/lib/burner/modeling/attribute.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Modeling
+    # Defines a top-level key and the associated transformers for deriving the final value
+    # to set the key to.
+    class Attribute
+      acts_as_hashable
+
+      attr_reader :key, :transformers
+
+      def initialize(key:, transformers: [])
+        raise ArgumentError, 'key is required' if key.to_s.empty?
+
+        @key          = key.to_s
+        @transformers = Realize::Transformers.array(transformers)
+
+        freeze
+      end
+    end
+  end
+end

data/lib/burner/modeling/attribute_renderer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Modeling
+    # Composed of an Attribute instance and a Pipeline instance.  It knows how to
+    # render/transform an Attribute.  Since this library is data-first, these intermediary
+    # objects are necessary for non-data-centric modeling.
+    class AttributeRenderer
+      extend Forwardable
+
+      attr_reader :attribute, :pipeline
+
+      def_delegators :attribute, :key
+
+      def_delegators :pipeline, :transform
+
+      def initialize(attribute, resolver)
+        @attribute = attribute
+        @pipeline  = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
+
+        freeze
+      end
+    end
+  end
+end

data/lib/burner/modeling/key_index_mapping.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Modeling
+    # Generic relationship between a numeric index and a key.
+    class KeyIndexMapping
+      acts_as_hashable
+
+      attr_reader :index, :key
+
+      def initialize(index:, key:)
+        raise ArgumentError, 'index is required' if index.to_s.empty?
+        raise ArgumentError, 'key is required'   if key.to_s.empty?
+
+        @index = index.to_i
+        @key   = key.to_s
+
+        freeze
+      end
+    end
+  end
+end

data/lib/burner/payload.rb

@@ -7,27 +7,38 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'written_file'
-
 module Burner
   # The input for all Job#perform methods.  The main notion of this object is its "value"
   # attribute.  This is dynamic and weak on purpose and is subject to whatever the Job#perform
   # methods decides it is.  This definitely adds an order-of-magnitude complexity to this whole
   # library and lifecycle, but I am not sure there is any other way around it: trying to build
   # a generic, open-ended object pipeline to serve almost any use case.
+  #
+  # The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
+  # structured/additive manner.  The initial use case for this was for Burner's core IO jobs to
+  # report back the files it has written in a more structured data way (as opposed to simply
+  # writing some information to the output.)
   class Payload
     attr_accessor :value
 
-    attr_reader :context, :written_files
+    attr_reader :params,
+                :side_effects,
+                :time
 
-    def initialize(context: {}, value: nil, written_files: [])
-      @context       = context || {}
-      @value         = value
-      @written_files = written_files || []
+    def initialize(
+      params: {},
+      side_effects: [],
+      time: Time.now.utc,
+      value: nil
+    )
+      @params       = params || {}
+      @side_effects = side_effects || []
+      @time         = time || Time.now.utc
+      @value        = value
     end
 
-    def add_written_file(written_file)
-      tap { written_files << WrittenFile.make(written_file) }
+    def add_side_effect(side_effect)
+      tap { side_effects << side_effect }
     end
   end
 end

data/lib/burner/pipeline.rb

@@ -19,11 +19,16 @@ module Burner
     acts_as_hashable
 
     class JobNotFoundError < StandardError; end
+    class DuplicateJobNameError < StandardError; end
 
     attr_reader :steps
 
     def initialize(jobs: [], steps: [])
-      jobs_by_name = Jobs.array(jobs).map { |job| [job.name, job] }.to_h
+      jobs = Jobs.array(jobs)
+
+      assert_unique_job_names(jobs)
+
+      jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
 
       @steps = Array(steps).map do |step_name|
         job = jobs_by_name[step_name.to_s]
@@ -35,15 +40,15 @@ module Burner
     end
 
     # The main entry-point for kicking off a pipeline.
-    def execute(params: {}, output: Output.new, payload: Payload.new)
+    def execute(output: Output.new, payload: Payload.new)
       output.write("Pipeline started with #{steps.length} step(s)")
 
-      output_params(params, output)
+      output_params(payload.params, output)
       output.ruler
 
       time_in_seconds = Benchmark.measure do
         steps.each do |step|
-          return_value = step.perform(output, payload, params)
+          return_value = step.perform(output, payload)
 
           if return_value.is_a?(FalseClass)
             output.detail('Job returned false, ending pipeline.')
@@ -60,6 +65,20 @@ module Burner
 
     private
 
+    def assert_unique_job_names(jobs)
+      unique_job_names = Set.new
+
+      jobs.each do |job|
+        if unique_job_names.include?(job.name)
+          raise DuplicateJobNameError, "job with name: #{job.name} already declared"
+        end
+
+        unique_job_names << job.name
+      end
+
+      nil
+    end
+
     def output_params(params, output)
       if params.keys.any?
         output.write('Parameters:')