burner 0.0.0 → 1.0.0.pre.alpha.4

data/lib/burner/jobs/collection/transform/attribute_renderer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module Collection
+      class Transform < Job
+        # Composed of an Attribute instance and a Pipeline instance.  It knows how to
+        # render/transform an Attribute.  Since this library is data-first, these intermediary
+        # objects are necessary for non-data-centric modeling.
+        class AttributeRenderer
+          extend Forwardable
+
+          attr_reader :attribute, :pipeline
+
+          def_delegators :attribute, :key
+
+          def_delegators :pipeline, :transform
+
+          def initialize(attribute, resolver)
+            @attribute = attribute
+            @pipeline  = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
+
+            freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/collection/unpivot.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module Collection
+      # Take an array of objects and un-pivot groups of keys into rows.
+      # Under the hood it uses HashMath's Unpivot class:
+      # https://github.com/bluemarblepayroll/hash_math
+      # Expected Payload#value input: array of objects.
+      # Payload#value output: An array of objects.
+      class Unpivot < Job
+        attr_reader :unpivot
+
+        def initialize(name:, pivot_set: HashMath::Unpivot::PivotSet.new)
+          super(name: name)
+
+          @unpivot = HashMath::Unpivot.new(pivot_set)
+
+          freeze
+        end
+
+        def perform(output, payload)
+          pivot_count  = unpivot.pivot_set.pivots.length
+          key_count    = unpivot.pivot_set.pivots.map { |p| p.keys.length }.sum
+          object_count = payload.value&.length || 0
+
+          message = "#{pivot_count} Pivots, Key(s): #{key_count} key(s), #{object_count} objects(s)"
+
+          output.detail(message)
+
+          payload.value = (payload.value || []).flat_map { |object| unpivot.expand(object) }
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/deserialize/csv.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module Deserialize
+      # Take a CSV string and de-serialize into object(s).
+      # Expected Payload#value input: nothing.
+      # Payload#value output: an array of arrays.  Each inner array represents one data row.
+      class Csv < Job
+        # This currently only supports returning an array of arrays, including the header row.
+        # In the future this could be extended to offer more customizable options, such as
+        # making it return an array of hashes with the columns mapped, etc.)
+        def perform(_output, payload)
+          payload.value = CSV.new(payload.value, headers: false).to_a
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/deserialize/json.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module Deserialize
+      # Take a JSON string and deserialize into object(s).
+      class Json < Job
+        def perform(_output, payload)
+          payload.value = JSON.parse(payload.value)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/deserialize/yaml.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module Deserialize
+      # Take a YAML string and deserialize into object(s).
+      class Yaml < Job
+        attr_reader :safe
+
+        def initialize(name:, safe: true)
+          super(name: name)
+
+          @safe = safe
+
+          freeze
+        end
+
+        # The YAML cop was disabled because the consumer may want to actually load unsafe
+        # YAML, which can load pretty much any type of class instead of putting the loader
+        # in a sandbox.  By default, though, we will try and drive them towards using it
+        # in the safer alternative.
+        # rubocop:disable Security/YAMLLoad
+        def perform(output, payload)
+          output.detail('Warning: loading YAML not using safe_load.') unless safe
+
+          payload.value = safe ? YAML.safe_load(payload.value) : YAML.load(payload.value)
+
+          nil
+        end
+        # rubocop:enable Security/YAMLLoad
+      end
+    end
+  end
+end

data/lib/burner/jobs/dummy.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    # Do nothing.
+    class Dummy < Job
+      def perform(_output, _payload)
+        nil
+      end
+    end
+  end
+end

data/lib/burner/jobs/echo.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    # Output a simple message to the output.
+    class Echo < Job
+      attr_reader :message
+
+      def initialize(name:, message: '')
+        super(name: name)
+
+        @message = message.to_s
+
+        freeze
+      end
+
+      def perform(output, payload)
+        compiled_message = job_string_template(message, output, payload)
+
+        output.detail(compiled_message)
+
+        nil
+      end
+    end
+  end
+end

data/lib/burner/jobs/io/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module IO
+      # Common configuration/code for all IO Job subclasses.
+      class Base < Job
+        attr_reader :path
+
+        def initialize(name:, path:)
+          super(name: name)
+
+          raise ArgumentError, 'path is required' if path.to_s.empty?
+
+          @path = path.to_s
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/io/exist.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module Burner
+  class Jobs
+    module IO
+      # Check to see if a file exists.  If short_circuit is set to true and the file
+      # does not exist then the job will return false and short circuit the pipeline.
+      class Exist < Base
+        attr_reader :short_circuit
+
+        def initialize(name:, path:, short_circuit: false)
+          super(name: name, path: path)
+
+          @short_circuit = short_circuit || false
+
+          freeze
+        end
+
+        def perform(output, payload)
+          compiled_path = job_string_template(path, output, payload)
+
+          exists = File.exist?(compiled_path)
+          verb   = exists ? 'does' : 'does not'
+
+          output.detail("The path: #{compiled_path} #{verb} exist")
+
+          # if anything but false is returned then the pipeline will not short circuit.  So
+          # we need to make sure we explicitly return false.
+          short_circuit && !exists ? false : nil
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/io/read.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module Burner
+  class Jobs
+    module IO
+      # Read value from disk.
+      class Read < Base
+        attr_reader :binary
+
+        def initialize(name:, path:, binary: false)
+          super(name: name, path: path)
+
+          @binary = binary || false
+
+          freeze
+        end
+
+        def perform(output, payload)
+          compiled_path = job_string_template(path, output, payload)
+
+          output.detail("Reading: #{compiled_path}")
+
+          payload.value = File.open(compiled_path, mode, &:read)
+
+          nil
+        end
+
+        private
+
+        def mode
+          binary ? 'rb' : 'r'
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/io/write.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'base'
+
+module Burner
+  class Jobs
+    module IO
+      # Write value to disk.
+      class Write < Base
+        attr_reader :binary
+
+        def initialize(name:, path:, binary: false)
+          super(name: name, path: path)
+
+          @binary = binary || false
+
+          freeze
+        end
+
+        def perform(output, payload)
+          compiled_path = job_string_template(path, output, payload)
+
+          ensure_directory_exists(output, compiled_path)
+
+          output.detail("Writing: #{compiled_path}")
+
+          time_in_seconds = Benchmark.measure do
+            File.open(compiled_path, mode) { |io| io.write(payload.value) }
+          end.real
+
+          payload.add_written_file(
+            logical_filename: compiled_path,
+            physical_filename: compiled_path,
+            time_in_seconds: time_in_seconds
+          )
+
+          nil
+        end
+
+        private
+
+        def ensure_directory_exists(output, compiled_path)
+          dirname = File.dirname(compiled_path)
+
+          return if File.exist?(dirname)
+
+          output.detail("Outer directory does not exist, creating: #{dirname}")
+
+          FileUtils.mkdir_p(dirname)
+
+          nil
+        end
+
+        def mode
+          binary ? 'wb' : 'w'
+        end
+      end
+    end
+  end
+end

data/lib/burner/jobs/serialize/csv.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Jobs
+    module Serialize
+      # Take an array of arrays and create a CSV.
+      # Expected Payload#value input: array of arrays.
+      # Payload#value output: a serialized CSV string.
+      class Csv < Job
+        def perform(_output, payload)
+          payload.value = CSV.generate(options) do |csv|
+            (payload.value || []).each do |row|
+              csv << row
+            end
+          end
+
+          nil
+        end
+
+        private
+
+        def options
+          {
+            headers: false,
+            write_headers: false
+          }
+        end
+      end
+    end
+  end
+end