burner 1.2.0 → 1.5.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2c3e9af2bbfbf80cd5c5884e05b1282f52871566103baf4ca1b599fc76af8ca
-  data.tar.gz: e7c2eb8a00086e1937a4d9e39383d38a60c5ebaa5efa1d67da4e6af9741261cd
+  metadata.gz: 5cbb761b5c317ea13a380ac7c96fe48846a5194d628c425fa0305c59738ce74e
+  data.tar.gz: d8d178a3c4ae9ea142610a4f341608c29e28cc8e9ad76edd10031e0983558178
 SHA512:
-  metadata.gz: e885c7bf710f613323bbc3fc49162815632d7785aa6e753f3cd53bb04d2b53e74f92be549f75b6997d4dedce5c2dabe01fd032796a6d6e2fadfd6e84c633b168
-  data.tar.gz: e09d491b3fb7ef1b79932ac6094adedd74f806170a2a11aa1ff1eda838b6bf76cec1684dbfaba12d07a6e071062d432bcd5f3939f275830fc13fb71cabb39f8c
+  metadata.gz: bc546faf0d6feb8828287b5c9f8893d3f21e29edb2ec638a2839e67bc05f9b5de8bbfb3ad9a3459bb4557fe052ee874f917aefbf3ad5b1829fcc499f1a4f9979
+  data.tar.gz: 778df0850ef5ef87a843409694577f8a75de9a541d6d32830780eec9add582bf95af932c01ec8fe17f3d29ba87d410fce8d0ad498049ca217cff7d5cb278a623

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+
+# 1.5.0 (TBD)
+
+Added Jobs:
+
+* b/collection/zip
+# 1.4.0 (December 17th, 2020)
+
+Additions:
+
+* byte_order_mark option for b/serialize/csv job
+
+Added Jobs:
+
+* b/compress/row_reader
+* b/io/row_reader
+# 1.3.0 (December 11th, 2020)
+
+Additions:
+
+* Decoupled storage: `Burner::Disks` factory, `Burner::Disks::Local` reference implementation, and `b/io/*` `disk` option for configuring IO jobs to use custom disks.
 # 1.2.0 (November 25th, 2020)
 
 #### Enhancements:

data/README.md

@@ -42,7 +42,7 @@ pipeline = {
     {
       name: :output_value,
       type: 'b/echo',
-      message: 'The current value is: {__value}'
+      message: 'The current value is: {__default_register}'
     },
     {
       name: :parse,
@@ -89,7 +89,7 @@ Some notes:
 
 * Some values are able to be string-interpolated using the provided Payload#params.  This allows for the passing runtime configuration/data into pipelines/jobs.
 * The job's ID can be accessed using the `__id` key.
-* The current job's payload value can be accessed using the `__value` key.
+* The current payload registers' values can be accessed using the `__<register_name>_register` key.
 * Jobs can be re-used (just like the output_id and output_value jobs).
 * If steps is nil then all jobs will execute in their declared order.
 
@@ -163,7 +163,7 @@ jobs:
 
   - name: output_value
     type: b/echo
-    message: 'The current value is: {__value}'
+    message: 'The current value is: {__default_register}'
 
   - name: parse
     type: b/deserialize/json
@@ -227,6 +227,11 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 * **b/collection/unpivot** [pivot_set, register]: Take an array of objects and unpivot specific sets of keys into rows.  Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
 * **b/collection/validate** [invalid_register, join_char, message_key, register, separator, validations]: Take an array of objects, run it through each declared validator, and split the objects into two registers.  The valid objects will be split into the current register while the invalid ones will go into the invalid_register as declared.  Optional arguments, join_char and message_key, help determine the compiled error messages.  The separator option can be utilized to use dot-notation for validating keys.  See each validation's options by viewing their classes within the `lib/modeling/validations` directory.
 * **b/collection/values** [include_keys, register]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
+* **b/collection/zip** [base_register, register, with_register]: Combines `base_register` and `with_register`s' data to form one single array in `register`.  It will combine each element, positionally in each array to form the final array.  For example:  ['hello', 'bugs'] + ['world', 'bunny'] => [['hello', 'world'], ['bugs', 'bunny']]
+
+#### Compression
+
+* **b/compress/row_reader** [data_key, ignore_blank_path, ignore_blank_data, path_key, register, separator]: Iterates over an array of objects, extracts a path and data in each object, and creates a zip file.
 
 #### De-serialization
 
@@ -236,13 +241,16 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 
 #### IO
 
-* **b/io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
-* **b/io/read** [binary, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
-* **b/io/write** [binary, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
+By default all jobs will use the `Burner::Disks::Local` disk for its persistence.  But this is configurable by implementing and registering custom disk-based classes in the `Burner::Disks` factory.  For example: a consumer application may also want to interact with cloud-based storage providers and could leverage this as its job library instead of implementing custom jobs.
+
+* **b/io/exist** [disk, path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
+* **b/io/read** [binary, disk, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
+* **b/io/row_reader** [data_key, disk, ignore_blank_path, ignore_file_not_found, path_key, register, separator]: Iterates over an array of objects, extracts a filepath from a key in each object, and attempts to load the file's content for each record.  The file's content will be stored at the specified data_key. By default missing paths or files will be treated as hard errors.  If you wish to ignore these then pass in true for ignore_blank_path and/or ignore_file_not_found.
+* **b/io/write** [binary, disk, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
 
 #### Serialization
 
-* **b/serialize/csv** [register]: Take an array of arrays and create a CSV.
+* **b/serialize/csv** [byte_order_mark, register]: Take an array of arrays and create a CSV.  You can optionally pre-pend a byte order mark, see Burner::Modeling::ByteOrderMark for acceptable options.
 * **b/serialize/json** [register]: Convert value to JSON.
 * **b/serialize/yaml** [register]: Convert value to YAML.
 
@@ -299,7 +307,7 @@ pipeline = {
     {
       name: :output_value,
       type: 'b/echo',
-      message: 'The current value is: {__value}'
+      message: 'The current value is: {__default_register}'
     },
     {
       name: :parse,

data/burner.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
   s.add_dependency('hash_math', '~>1.2')
   s.add_dependency('objectable', '~>1.0')
   s.add_dependency('realize', '~>1.3')
+  s.add_dependency('rubyzip', '~>1.2')
   s.add_dependency('stringento', '~>2.1')
 
   s.add_development_dependency('guard-rspec', '~>4.7')

data/lib/burner.rb

@@ -10,6 +10,7 @@
 require 'acts_as_hashable'
 require 'benchmark'
 require 'csv'
+require 'fileutils'
 require 'forwardable'
 require 'hash_math'
 require 'hashematics'
@@ -21,8 +22,10 @@ require 'singleton'
 require 'stringento'
 require 'time'
 require 'yaml'
+require 'zip'
 
 # Common/Shared
+require_relative 'burner/disks'
 require_relative 'burner/modeling'
 require_relative 'burner/side_effects'
 require_relative 'burner/util'

data/lib/burner/disks.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'disks/local'
+
+module Burner
+  # A factory to register and emit instances that conform to the Disk interface with requests
+  # the instance responds to: #exist?, #read, and #write.  See an example implementation within
+  # the lib/burner/disks directory.
+  #
+  # The benefit to this pluggable disk model is a consumer application can decide which file
+  # backend to use and how to store files.  For example: an application may choose to use
+  # some cloud provider with their own file store implementation.  This can be wrapped up
+  # in a Disk class and registered here and then referenced in the Pipeline's IO jobs.
+  class Disks
+    acts_as_hashable_factory
+
+    register 'local', '', Disks::Local
+  end
+end

data/lib/burner/disks/local.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Disks
+    # Operations against the local file system.
+    class Local
+      acts_as_hashable
+
+      # Check to see if the passed in path exists within the local file system.
+      # It will not make assumptions on what the 'file' is, only that it is recognized
+      # by Ruby's File class.
+      def exist?(path)
+        File.exist?(path)
+      end
+
+      # Open and read the contents of a local file.  If binary is passed in as true then the file
+      # will be opened in binary mode.
+      def read(path, binary: false)
+        File.open(path, read_mode(binary), &:read)
+      end
+
+      # Open and write the specified data to a local file.  If binary is passed in as true then
+      # the file will be opened in binary mode.  It is important to note that if the file's
+      # directory structure will be automatically created if it does not exist.
+      def write(path, data, binary: false)
+        ensure_directory_exists(path)
+
+        File.open(path, write_mode(binary)) { |io| io.write(data) }
+
+        path
+      end
+
+      private
+
+      def ensure_directory_exists(path)
+        dirname = File.dirname(path)
+
+        return if File.exist?(dirname)
+
+        FileUtils.mkdir_p(dirname)
+
+        nil
+      end
+
+      def write_mode(binary)
+        binary ? 'wb' : 'w'
+      end
+
+      def read_mode(binary)
+        binary ? 'rb' : 'r'
+      end
+    end
+  end
+end

data/lib/burner/jobs.rb

@@ -34,6 +34,9 @@ module Burner
     register 'b/collection/unpivot',           Library::Collection::Unpivot
     register 'b/collection/values',            Library::Collection::Values
     register 'b/collection/validate',          Library::Collection::Validate
+    register 'b/collection/zip',               Library::Collection::Zip
+
+    register 'b/compress/row_reader',          Library::Compress::RowReader
 
     register 'b/deserialize/csv',              Library::Deserialize::Csv
     register 'b/deserialize/json',             Library::Deserialize::Json
@@ -41,6 +44,7 @@ module Burner
 
     register 'b/io/exist',                     Library::IO::Exist
     register 'b/io/read',                      Library::IO::Read
+    register 'b/io/row_reader',                Library::IO::RowReader
     register 'b/io/write',                     Library::IO::Write
 
     register 'b/serialize/csv',                Library::Serialize::Csv

data/lib/burner/library.rb

@@ -25,6 +25,9 @@ require_relative 'library/collection/transform'
 require_relative 'library/collection/unpivot'
 require_relative 'library/collection/validate'
 require_relative 'library/collection/values'
+require_relative 'library/collection/zip'
+
+require_relative 'library/compress/row_reader'
 
 require_relative 'library/deserialize/csv'
 require_relative 'library/deserialize/json'
@@ -32,6 +35,7 @@ require_relative 'library/deserialize/yaml'
 
 require_relative 'library/io/exist'
 require_relative 'library/io/read'
+require_relative 'library/io/row_reader'
 require_relative 'library/io/write'
 
 require_relative 'library/serialize/csv'

data/lib/burner/library/collection/zip.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Collection
+      # This job can take two arrays and coalesces them by index.  For example:
+      #
+      # input:
+      #   base_register: [ 'hello', 'bugs' ]
+      #   with_register: [ 'world', 'bunny' ]
+      # output:
+      #  register: [ ['hello', 'world'], ['bugs', 'bunny'] ]
+      #
+      # Expected Payload[base_register] input: array of objects.
+      # Expected Payload[with_register] input: array of objects.
+      # Payload[register] output: An array of two-dimensional arrays.
+      class Zip < JobWithRegister
+        attr_reader :base_register, :with_register
+
+        def initialize(
+          name:,
+          with_register:,
+          base_register: DEFAULT_REGISTER,
+          register: DEFAULT_REGISTER
+        )
+          super(name: name, register: register)
+
+          @base_register = base_register.to_s
+          @with_register = with_register.to_s
+
+          freeze
+        end
+
+        def perform(output, payload)
+          base_data = array(payload[base_register])
+          with_data = array(payload[with_register])
+
+          output.detail("Combining register: #{base_register} (#{base_data.length} record(s))")
+          output.detail("With register: #{with_register} (#{with_data.length} record(s))")
+
+          payload[register] = base_data.zip(with_data)
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/compress/row_reader.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Library
+    module Compress
+      # Iterates over an array of objects, extracts a path and data in each object, and
+      # creates a zip file.  By default, if a path is blank then an ArgumentError will be raised.
+      # If this is undesirable then you can set ignore_blank_path to true and the record will be
+      # skipped.  You also have the option to supress blank files being added by configuring
+      # ignore_blank_data as true.
+      #
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: compressed binary zip file contents.
+      class RowReader < JobWithRegister
+        Content = Struct.new(:path, :data)
+
+        private_constant :Content
+
+        DEFAULT_DATA_KEY = 'data'
+        DEFAULT_PATH_KEY = 'path'
+
+        attr_reader :data_key,
+                    :ignore_blank_data,
+                    :ignore_blank_path,
+                    :path_key,
+                    :resolver
+
+        def initialize(
+          name:,
+          data_key: DEFAULT_DATA_KEY,
+          ignore_blank_data: false,
+          ignore_blank_path: false,
+          path_key: DEFAULT_PATH_KEY,
+          register: DEFAULT_REGISTER,
+          separator: ''
+        )
+          super(name: name, register: register)
+
+          @data_key          = data_key.to_s
+          @ignore_blank_data = ignore_blank_data || false
+          @ignore_blank_path = ignore_blank_path || false
+          @path_key          = path_key.to_s
+          @resolver          = Objectable.resolver(separator: separator)
+
+          freeze
+        end
+
+        def perform(output, payload)
+          payload[register] = Zip::OutputStream.write_buffer do |zip|
+            array(payload[register]).each.with_index(1) do |record, index|
+              content = extract_path_and_data(record, index, output)
+
+              next unless content
+
+              zip.put_next_entry(content.path)
+              zip.write(content.data)
+            end
+          end.string
+        end
+
+        private
+
+        def extract_path_and_data(record, index, output)
+          path = strip_leading_separator(resolver.get(record, path_key))
+          data = resolver.get(record, data_key)
+
+          return if assert_and_skip_missing_path?(path, index, output)
+          return if skip_missing_data?(data, index, output)
+
+          Content.new(path, data)
+        end
+
+        def strip_leading_separator(path)
+          path.to_s.start_with?(File::SEPARATOR) ? path.to_s[1..-1] : path.to_s
+        end
+
+        def assert_and_skip_missing_path?(path, index, output)
+          if ignore_blank_path && path.to_s.empty?
+            output.detail("Skipping record #{index} because of blank path")
+            true
+          elsif path.to_s.empty?
+            raise ArgumentError, "Record #{index} is missing a path at key: #{path_key}"
+          end
+        end
+
+        def skip_missing_data?(data, index, output)
+          return false unless ignore_blank_data && data.to_s.empty?
+
+          output.detail("Skipping record #{index} because of blank data")
+          true
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/io/exist.rb

@@ -7,8 +7,6 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
-
 module Burner
   module Library
     module IO
@@ -17,13 +15,14 @@ module Burner
       #
       # Note: this does not use Payload#registers.
       class Exist < Job
-        attr_reader :path, :short_circuit
+        attr_reader :disk, :path, :short_circuit
 
-        def initialize(name:, path:, short_circuit: false)
+        def initialize(name:, path:, disk: {}, short_circuit: false)
           super(name: name)
 
           raise ArgumentError, 'path is required' if path.to_s.empty?
 
+          @disk          = Disks.make(disk)
           @path          = path.to_s
           @short_circuit = short_circuit || false
         end
@@ -31,7 +30,7 @@ module Burner
         def perform(output, payload)
           compiled_path = job_string_template(path, output, payload)
 
-          exists = File.exist?(compiled_path)
+          exists = disk.exist?(compiled_path)
           verb   = exists ? 'does' : 'does not'
 
           output.detail("The path: #{compiled_path} #{verb} exist")

data/lib/burner/library/io/{base.rb → open_file_base.rb}

@@ -10,16 +10,20 @@
 module Burner
   module Library
     module IO
-      # Common configuration/code for all IO Job subclasses.
-      class Base < JobWithRegister
-        attr_reader :path
+      # Common configuration/code for all IO Job subclasses that open a file.
+      class OpenFileBase < JobWithRegister
+        attr_reader :binary, :disk, :path
 
-        def initialize(name:, path:, register: DEFAULT_REGISTER)
+        def initialize(name:, path:, binary: false, disk: {}, register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           raise ArgumentError, 'path is required' if path.to_s.empty?
 
-          @path = path.to_s
+          @binary = binary || false
+          @disk   = Disks.make(disk)
+          @path   = path.to_s
+
+          freeze
         end
       end
     end

data/lib/burner/library/io/read.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'open_file_base'
 
 module Burner
   module Library
@@ -16,29 +16,13 @@ module Burner
       #
       # Expected Payload[register] input: nothing.
       # Payload[register] output: contents of the specified file.
-      class Read < Base
-        attr_reader :binary
-
-        def initialize(name:, path:, binary: false, register: DEFAULT_REGISTER)
-          super(name: name, path: path, register: register)
-
-          @binary = binary || false
-
-          freeze
-        end
-
+      class Read < OpenFileBase
         def perform(output, payload)
           compiled_path = job_string_template(path, output, payload)
 
           output.detail("Reading: #{compiled_path}")
 
-          payload[register] = File.open(compiled_path, mode, &:read)
-        end
-
-        private
-
-        def mode
-          binary ? 'rb' : 'r'
+          payload[register] = disk.read(compiled_path, binary: binary)
         end
       end
     end

data/lib/burner/library/io/row_reader.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'open_file_base'
+
+module Burner
+  module Library
+    module IO
+      # Iterates over an array of objects, extracts a filepath from a key in each object,
+      # and attempts to load the file's content for each record.  The file's content will be
+      # stored at the specified data_key. By default missing paths or files will be
+      # treated as hard errors.  If you wish to ignore these then pass in true for
+      # ignore_blank_path and/or ignore_file_not_found.
+      #
+      # Expected Payload[register] input: array of objects.
+      # Payload[register] output: array of objects.
+      class RowReader < JobWithRegister
+        class FileNotFoundError < StandardError; end
+
+        DEFAULT_DATA_KEY = 'data'
+        DEFAULT_PATH_KEY = 'path'
+
+        attr_reader :binary,
+                    :data_key,
+                    :disk,
+                    :ignore_blank_path,
+                    :ignore_file_not_found,
+                    :path_key,
+                    :resolver
+
+        def initialize(
+          name:,
+          binary: false,
+          data_key: DEFAULT_DATA_KEY,
+          disk: {},
+          ignore_blank_path: false,
+          ignore_file_not_found: false,
+          path_key: DEFAULT_PATH_KEY,
+          register: DEFAULT_REGISTER,
+          separator: ''
+        )
+          super(name: name, register: register)
+
+          @binary                = binary || false
+          @data_key              = data_key.to_s
+          @disk                  = Disks.make(disk)
+          @ignore_blank_path     = ignore_blank_path || false
+          @ignore_file_not_found = ignore_file_not_found || false
+          @path_key              = path_key.to_s
+          @resolver              = Objectable.resolver(separator: separator)
+
+          freeze
+        end
+
+        def perform(output, payload)
+          records = array(payload[register])
+
+          output.detail("Reading path_key: #{path_key} for #{payload[register].length} records(s)")
+          output.detail("Storing read data in: #{path_key}")
+
+          payload[register] = records.map.with_index(1) do |object, index|
+            load_data(object, index, output)
+          end
+        end
+
+        private
+
+        def assert_and_skip_missing_path?(path, index, output)
+          missing_path            = path.to_s.empty?
+          blank_path_raises_error = !ignore_blank_path
+
+          if missing_path && blank_path_raises_error
+            output.detail("Record #{index} is missing a path, raising error")
+
+            raise ArgumentError, "Record #{index} is missing a path"
+          elsif missing_path
+            output.detail("Record #{index} is missing a path")
+
+            true
+          end
+        end
+
+        def assert_and_skip_file_not_found?(path, index, output)
+          does_not_exist              = !disk.exist?(path)
+          file_not_found_raises_error = !ignore_file_not_found
+
+          if file_not_found_raises_error && does_not_exist
+            output.detail("Record #{index} path: '#{path}' does not exist, raising error")
+
+            raise FileNotFoundError, "#{path} does not exist"
+          elsif does_not_exist
+            output.detail("Record #{index} path: '#{path}' does not exist, skipping")
+
+            true
+          end
+        end
+
+        def load_data(object, index, output)
+          path = resolver.get(object, path_key)
+
+          return object if assert_and_skip_missing_path?(path, index, output)
+          return object if assert_and_skip_file_not_found?(path, index, output)
+
+          data = disk.read(path, binary: binary)
+
+          resolver.set(object, data_key, data)
+
+          object
+        end
+      end
+    end
+  end
+end

data/lib/burner/library/io/write.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'open_file_base'
 
 module Burner
   module Library
@@ -16,54 +16,27 @@ module Burner
       #
       # Expected Payload[register] input: anything.
       # Payload[register] output: whatever was passed in.
-      class Write < Base
-        attr_reader :binary
-
-        def initialize(name:, path:, binary: false, register: DEFAULT_REGISTER)
-          super(name: name, path: path, register: register)
-
-          @binary = binary || false
-
-          freeze
-        end
-
+      class Write < OpenFileBase
         def perform(output, payload)
-          compiled_path = job_string_template(path, output, payload)
-
-          ensure_directory_exists(output, compiled_path)
+          logical_filename  = job_string_template(path, output, payload)
+          physical_filename = nil
 
-          output.detail("Writing: #{compiled_path}")
+          output.detail("Writing: #{logical_filename}")
 
           time_in_seconds = Benchmark.measure do
-            File.open(compiled_path, mode) { |io| io.write(payload[register]) }
+            physical_filename = disk.write(logical_filename, payload[register], binary: binary)
           end.real
 
+          output.detail("Wrote to: #{physical_filename}")
+
           side_effect = SideEffects::WrittenFile.new(
-            logical_filename: compiled_path,
-            physical_filename: compiled_path,
+            logical_filename: logical_filename,
+            physical_filename: physical_filename,
             time_in_seconds: time_in_seconds
           )
 
           payload.add_side_effect(side_effect)
         end
-
-        private
-
-        def ensure_directory_exists(output, compiled_path)
-          dirname = File.dirname(compiled_path)
-
-          return if File.exist?(dirname)
-
-          output.detail("Outer directory does not exist, creating: #{dirname}")
-
-          FileUtils.mkdir_p(dirname)
-
-          nil
-        end
-
-        def mode
-          binary ? 'wb' : 'w'
-        end
       end
     end
   end

data/lib/burner/library/serialize/csv.rb

@@ -10,17 +10,30 @@
 module Burner
   module Library
     module Serialize
-      # Take an array of arrays and create a CSV.
+      # Take an array of arrays and create a CSV.  You can optionally pre-pend a byte order mark,
+      # see Burner::Modeling::ByteOrderMark for acceptable options.
       #
       # Expected Payload[register] input: array of arrays.
       # Payload[register] output: a serialized CSV string.
       class Csv < JobWithRegister
+        attr_reader :byte_order_mark
+
+        def initialize(name:, byte_order_mark: nil, register: DEFAULT_REGISTER)
+          super(name: name, register: register)
+
+          @byte_order_mark = Modeling::ByteOrderMark.resolve(byte_order_mark)
+
+          freeze
+        end
+
         def perform(_output, payload)
-          payload[register] = CSV.generate(options) do |csv|
+          serialized_rows = CSV.generate(options) do |csv|
             array(payload[register]).each do |row|
               csv << row
             end
           end
+
+          payload[register] = "#{byte_order_mark}#{serialized_rows}"
         end
 
         private

data/lib/burner/modeling.rb

@@ -9,6 +9,7 @@
 
 require_relative 'modeling/attribute'
 require_relative 'modeling/attribute_renderer'
+require_relative 'modeling/byte_order_mark'
 require_relative 'modeling/key_index_mapping'
 require_relative 'modeling/key_mapping'
 require_relative 'modeling/validations'

data/lib/burner/modeling/byte_order_mark.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  module Modeling
+    # Define all acceptable byte order mark values.
+    module ByteOrderMark
+      UTF_8    = "\xEF\xBB\xBF"
+      UTF_16BE = "\xFE\xFF"
+      UTF_16LE = "\xFF\xFE"
+      UTF_32BE = "\x00\x00\xFE\xFF"
+      UTF_32LE = "\xFE\xFF\x00\x00"
+
+      class << self
+        def resolve(value)
+          value ? const_get(value.to_s.upcase.to_sym) : nil
+        end
+      end
+    end
+  end
+end

data/lib/burner/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Burner
-  VERSION = '1.2.0'
+  VERSION = '1.5.0-alpha'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: burner
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.5.0.pre.alpha
 platform: ruby
 authors:
 - Matthew Ruggio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-25 00:00:00.000000000 Z
+date: 2020-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: stringento
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +234,8 @@ files:
 - exe/burner
 - lib/burner.rb
 - lib/burner/cli.rb
+- lib/burner/disks.rb
+- lib/burner/disks/local.rb
 - lib/burner/job.rb
 - lib/burner/job_with_register.rb
 - lib/burner/jobs.rb
@@ -236,13 +252,16 @@ files:
 - lib/burner/library/collection/unpivot.rb
 - lib/burner/library/collection/validate.rb
 - lib/burner/library/collection/values.rb
+- lib/burner/library/collection/zip.rb
+- lib/burner/library/compress/row_reader.rb
 - lib/burner/library/deserialize/csv.rb
 - lib/burner/library/deserialize/json.rb
 - lib/burner/library/deserialize/yaml.rb
 - lib/burner/library/echo.rb
-- lib/burner/library/io/base.rb
 - lib/burner/library/io/exist.rb
+- lib/burner/library/io/open_file_base.rb
 - lib/burner/library/io/read.rb
+- lib/burner/library/io/row_reader.rb
 - lib/burner/library/io/write.rb
 - lib/burner/library/nothing.rb
 - lib/burner/library/serialize/csv.rb
@@ -254,6 +273,7 @@ files:
 - lib/burner/modeling.rb
 - lib/burner/modeling/attribute.rb
 - lib/burner/modeling/attribute_renderer.rb
+- lib/burner/modeling/byte_order_mark.rb
 - lib/burner/modeling/key_index_mapping.rb
 - lib/burner/modeling/key_mapping.rb
 - lib/burner/modeling/validations.rb
@@ -290,9 +310,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: