burner 1.2.0 → 1.3.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2c3e9af2bbfbf80cd5c5884e05b1282f52871566103baf4ca1b599fc76af8ca
-  data.tar.gz: e7c2eb8a00086e1937a4d9e39383d38a60c5ebaa5efa1d67da4e6af9741261cd
+  metadata.gz: 340ebade1c4d3cc238925cb21718571dd666c3c16b6ee60d8de73c4fe017d562
+  data.tar.gz: 33b272e7a4b4a15a8e92b5f7eccb9df7ec6d685f767ecbf8434dedcf7ed901b5
 SHA512:
-  metadata.gz: e885c7bf710f613323bbc3fc49162815632d7785aa6e753f3cd53bb04d2b53e74f92be549f75b6997d4dedce5c2dabe01fd032796a6d6e2fadfd6e84c633b168
-  data.tar.gz: e09d491b3fb7ef1b79932ac6094adedd74f806170a2a11aa1ff1eda838b6bf76cec1684dbfaba12d07a6e071062d432bcd5f3939f275830fc13fb71cabb39f8c
+  metadata.gz: a5d082d14302d38e2acf89ab1275004ae69f78087e40da2a68c15e25770a9ebfc7fa2cb8e9e7268d132e2ade6c3d6db9e259ab3e8254ae476fcd1d46a4f40eb2
+  data.tar.gz: c1a9bda279afa44c5cb5a41c4ac792a62af317d91985799a14609c7b6d33e5494d322ad14262281b3fead76cc6471bebd7b507e1158dea7f9278196b36ab1726

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 1.3.0 (TBD)
+
+Additions:
+
+* Decoupled storage: `Burner::Disks` factory, `Burner::Disks::Local` reference implementation, and `b/io/*` `disk` option for configuring IO jobs to use custom disks.
 # 1.2.0 (November 25th, 2020)
 
 #### Enhancements:

data/README.md

@@ -42,7 +42,7 @@ pipeline = {
     {
       name: :output_value,
       type: 'b/echo',
-      message: 'The current value is: {__value}'
+      message: 'The current value is: {__default_register}'
     },
     {
       name: :parse,
@@ -89,7 +89,7 @@ Some notes:
 
 * Some values are able to be string-interpolated using the provided Payload#params.  This allows for the passing runtime configuration/data into pipelines/jobs.
 * The job's ID can be accessed using the `__id` key.
-* The current job's payload value can be accessed using the `__value` key.
+* The current payload registers' values can be accessed using the `__<register_name>_register` key.
 * Jobs can be re-used (just like the output_id and output_value jobs).
 * If steps is nil then all jobs will execute in their declared order.
 
@@ -163,7 +163,7 @@ jobs:
 
   - name: output_value
     type: b/echo
-    message: 'The current value is: {__value}'
+    message: 'The current value is: {__default_register}'
 
   - name: parse
     type: b/deserialize/json
@@ -236,9 +236,11 @@ This library only ships with very basic, rudimentary jobs that are meant to just
 
 #### IO
 
-* **b/io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
-* **b/io/read** [binary, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
-* **b/io/write** [binary, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
+By default all jobs will use the `Burner::Disks::Local` disk for its persistence.  But this is configurable by implementing and registering custom disk-based classes in the `Burner::Disks` factory.  For example: a consumer application may also want to interact with cloud-based storage providers and could leverage this as its job library instead of implementing custom jobs.
+
+* **b/io/exist** [disk, path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`.  If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
+* **b/io/read** [binary, disk, path, register]: Read in a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
+* **b/io/write** [binary, disk, path, register]: Write to a local file.  The path parameter can be interpolated using `Payload#params`.  If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
 
 #### Serialization
 
@@ -299,7 +301,7 @@ pipeline = {
     {
       name: :output_value,
       type: 'b/echo',
-      message: 'The current value is: {__value}'
+      message: 'The current value is: {__default_register}'
     },
     {
       name: :parse,

data/lib/burner.rb

@@ -10,6 +10,7 @@
 require 'acts_as_hashable'
 require 'benchmark'
 require 'csv'
+require 'fileutils'
 require 'forwardable'
 require 'hash_math'
 require 'hashematics'
@@ -23,6 +24,7 @@ require 'time'
 require 'yaml'
 
 # Common/Shared
+require_relative 'burner/disks'
 require_relative 'burner/modeling'
 require_relative 'burner/side_effects'
 require_relative 'burner/util'

data/lib/burner/disks.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require_relative 'disks/local'
+
+module Burner
+  # A factory to register and emit instances that conform to the Disk interface with requests
+  # the instance responds to: #exist?, #read, and #write.  See an example implementation within
+  # the lib/burner/disks directory.
+  #
+  # The benefit to this pluggable disk model is a consumer application can decide which file
+  # backend to use and how to store files.  For example: an application may choose to use
+  # some cloud provider with their own file store implementation.  This can be wrapped up
+  # in a Disk class and registered here and then referenced in the Pipeline's IO jobs.
+  class Disks
+    acts_as_hashable_factory
+
+    register 'local', '', Disks::Local
+  end
+end

data/lib/burner/disks/local.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2020-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+module Burner
+  class Disks
+    # Operations against the local file system.
+    class Local
+      acts_as_hashable
+
+      # Check to see if the passed in path exists within the local file system.
+      # It will not make assumptions on what the 'file' is, only that it is recognized
+      # by Ruby's File class.
+      def exist?(path)
+        File.exist?(path)
+      end
+
+      # Open and read the contents of a local file.  If binary is passed in as true then the file
+      # will be opened in binary mode.
+      def read(path, binary: false)
+        File.open(path, read_mode(binary), &:read)
+      end
+
+      # Open and write the specified data to a local file.  If binary is passed in as true then
+      # the file will be opened in binary mode.  It is important to note that if the file's
+      # directory structure will be automatically created if it does not exist.
+      def write(path, data, binary: false)
+        ensure_directory_exists(path)
+
+        File.open(path, write_mode(binary)) { |io| io.write(data) }
+
+        path
+      end
+
+      private
+
+      def ensure_directory_exists(path)
+        dirname = File.dirname(path)
+
+        return if File.exist?(dirname)
+
+        FileUtils.mkdir_p(dirname)
+
+        nil
+      end
+
+      def write_mode(binary)
+        binary ? 'wb' : 'w'
+      end
+
+      def read_mode(binary)
+        binary ? 'rb' : 'r'
+      end
+    end
+  end
+end

data/lib/burner/library/io/exist.rb

@@ -7,8 +7,6 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
-
 module Burner
   module Library
     module IO
@@ -17,13 +15,14 @@ module Burner
       #
       # Note: this does not use Payload#registers.
       class Exist < Job
-        attr_reader :path, :short_circuit
+        attr_reader :disk, :path, :short_circuit
 
-        def initialize(name:, path:, short_circuit: false)
+        def initialize(name:, path:, disk: {}, short_circuit: false)
           super(name: name)
 
           raise ArgumentError, 'path is required' if path.to_s.empty?
 
+          @disk          = Disks.make(disk)
           @path          = path.to_s
           @short_circuit = short_circuit || false
         end
@@ -31,7 +30,7 @@ module Burner
         def perform(output, payload)
           compiled_path = job_string_template(path, output, payload)
 
-          exists = File.exist?(compiled_path)
+          exists = disk.exist?(compiled_path)
           verb   = exists ? 'does' : 'does not'
 
           output.detail("The path: #{compiled_path} #{verb} exist")

data/lib/burner/library/io/{base.rb → open_file_base.rb}

@@ -10,16 +10,20 @@
 module Burner
   module Library
     module IO
-      # Common configuration/code for all IO Job subclasses.
-      class Base < JobWithRegister
-        attr_reader :path
+      # Common configuration/code for all IO Job subclasses that open a file.
+      class OpenFileBase < JobWithRegister
+        attr_reader :binary, :disk, :path
 
-        def initialize(name:, path:, register: DEFAULT_REGISTER)
+        def initialize(name:, path:, binary: false, disk: {}, register: DEFAULT_REGISTER)
           super(name: name, register: register)
 
           raise ArgumentError, 'path is required' if path.to_s.empty?
 
-          @path = path.to_s
+          @binary = binary || false
+          @disk   = Disks.make(disk)
+          @path   = path.to_s
+
+          freeze
         end
       end
     end

data/lib/burner/library/io/read.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'open_file_base'
 
 module Burner
   module Library
@@ -16,29 +16,13 @@ module Burner
       #
       # Expected Payload[register] input: nothing.
       # Payload[register] output: contents of the specified file.
-      class Read < Base
-        attr_reader :binary
-
-        def initialize(name:, path:, binary: false, register: DEFAULT_REGISTER)
-          super(name: name, path: path, register: register)
-
-          @binary = binary || false
-
-          freeze
-        end
-
+      class Read < OpenFileBase
         def perform(output, payload)
           compiled_path = job_string_template(path, output, payload)
 
           output.detail("Reading: #{compiled_path}")
 
-          payload[register] = File.open(compiled_path, mode, &:read)
-        end
-
-        private
-
-        def mode
-          binary ? 'rb' : 'r'
+          payload[register] = disk.read(compiled_path, binary: binary)
         end
       end
     end

data/lib/burner/library/io/write.rb

@@ -7,7 +7,7 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'base'
+require_relative 'open_file_base'
 
 module Burner
   module Library
@@ -16,54 +16,27 @@ module Burner
       #
       # Expected Payload[register] input: anything.
       # Payload[register] output: whatever was passed in.
-      class Write < Base
-        attr_reader :binary
-
-        def initialize(name:, path:, binary: false, register: DEFAULT_REGISTER)
-          super(name: name, path: path, register: register)
-
-          @binary = binary || false
-
-          freeze
-        end
-
+      class Write < OpenFileBase
         def perform(output, payload)
-          compiled_path = job_string_template(path, output, payload)
-
-          ensure_directory_exists(output, compiled_path)
+          logical_filename  = job_string_template(path, output, payload)
+          physical_filename = nil
 
-          output.detail("Writing: #{compiled_path}")
+          output.detail("Writing: #{logical_filename}")
 
           time_in_seconds = Benchmark.measure do
-            File.open(compiled_path, mode) { |io| io.write(payload[register]) }
+            physical_filename = disk.write(logical_filename, payload[register], binary: binary)
           end.real
 
+          output.detail("Wrote to: #{physical_filename}")
+
           side_effect = SideEffects::WrittenFile.new(
-            logical_filename: compiled_path,
-            physical_filename: compiled_path,
+            logical_filename: logical_filename,
+            physical_filename: physical_filename,
             time_in_seconds: time_in_seconds
           )
 
           payload.add_side_effect(side_effect)
         end
-
-        private
-
-        def ensure_directory_exists(output, compiled_path)
-          dirname = File.dirname(compiled_path)
-
-          return if File.exist?(dirname)
-
-          output.detail("Outer directory does not exist, creating: #{dirname}")
-
-          FileUtils.mkdir_p(dirname)
-
-          nil
-        end
-
-        def mode
-          binary ? 'wb' : 'w'
-        end
       end
     end
   end

data/lib/burner/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Burner
-  VERSION = '1.2.0'
+  VERSION = '1.3.0-alpha'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: burner
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0.pre.alpha
 platform: ruby
 authors:
 - Matthew Ruggio
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-25 00:00:00.000000000 Z
+date: 2020-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -220,6 +220,8 @@ files:
 - exe/burner
 - lib/burner.rb
 - lib/burner/cli.rb
+- lib/burner/disks.rb
+- lib/burner/disks/local.rb
 - lib/burner/job.rb
 - lib/burner/job_with_register.rb
 - lib/burner/jobs.rb
@@ -240,8 +242,8 @@ files:
 - lib/burner/library/deserialize/json.rb
 - lib/burner/library/deserialize/yaml.rb
 - lib/burner/library/echo.rb
-- lib/burner/library/io/base.rb
 - lib/burner/library/io/exist.rb
+- lib/burner/library/io/open_file_base.rb
 - lib/burner/library/io/read.rb
 - lib/burner/library/io/write.rb
 - lib/burner/library/nothing.rb
@@ -290,9 +292,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: