burner 1.0.0.pre.alpha.1 → 1.0.0.pre.alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +48 -20
  3. data/burner.gemspec +3 -0
  4. data/exe/burner +3 -4
  5. data/lib/burner.rb +12 -0
  6. data/lib/burner/cli.rb +10 -8
  7. data/lib/burner/job.rb +28 -6
  8. data/lib/burner/jobs.rb +21 -23
  9. data/lib/burner/library.rb +30 -0
  10. data/lib/burner/library/collection/arrays_to_objects.rb +77 -0
  11. data/lib/burner/library/collection/graph.rb +44 -0
  12. data/lib/burner/library/collection/objects_to_arrays.rb +90 -0
  13. data/lib/burner/library/collection/shift.rb +44 -0
  14. data/lib/burner/library/collection/transform.rb +68 -0
  15. data/lib/burner/library/collection/unpivot.rb +47 -0
  16. data/lib/burner/library/collection/values.rb +51 -0
  17. data/lib/burner/library/deserialize/csv.rb +29 -0
  18. data/lib/burner/{jobs → library}/deserialize/json.rb +5 -2
  19. data/lib/burner/{jobs → library}/deserialize/yaml.rb +9 -3
  20. data/lib/burner/{jobs → library}/dummy.rb +4 -2
  21. data/lib/burner/{jobs → library}/echo.rb +5 -3
  22. data/lib/burner/{jobs → library}/io/base.rb +1 -7
  23. data/lib/burner/{jobs → library}/io/exist.rb +5 -3
  24. data/lib/burner/{jobs → library}/io/read.rb +6 -3
  25. data/lib/burner/{jobs → library}/io/write.rb +9 -4
  26. data/lib/burner/library/serialize/csv.rb +39 -0
  27. data/lib/burner/{jobs → library}/serialize/json.rb +5 -2
  28. data/lib/burner/{jobs → library}/serialize/yaml.rb +5 -2
  29. data/lib/burner/{jobs/set.rb → library/set_value.rb} +6 -3
  30. data/lib/burner/{jobs → library}/sleep.rb +4 -2
  31. data/lib/burner/modeling.rb +12 -0
  32. data/lib/burner/modeling/attribute.rb +29 -0
  33. data/lib/burner/modeling/attribute_renderer.rb +32 -0
  34. data/lib/burner/modeling/key_index_mapping.rb +29 -0
  35. data/lib/burner/payload.rb +20 -9
  36. data/lib/burner/pipeline.rb +23 -4
  37. data/lib/burner/side_effects.rb +10 -0
  38. data/lib/burner/side_effects/written_file.rb +28 -0
  39. data/lib/burner/step.rb +2 -4
  40. data/lib/burner/string_template.rb +6 -5
  41. data/lib/burner/util.rb +10 -0
  42. data/lib/burner/util/arrayable.rb +30 -0
  43. data/lib/burner/version.rb +1 -1
  44. metadata +74 -15
  45. data/lib/burner/written_file.rb +0 -28
@@ -8,7 +8,7 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- class Jobs
11
+ module Library
12
12
  module IO
13
13
  # Common configuration/code for all IO Job subclasses.
14
14
  class Base < Job
@@ -21,12 +21,6 @@ module Burner
21
21
 
22
22
  @path = path.to_s
23
23
  end
24
-
25
- private
26
-
27
- def compile_path(params)
28
- eval_string_template(path, params)
29
- end
30
24
  end
31
25
  end
32
26
  end
@@ -10,10 +10,12 @@
10
10
  require_relative 'base'
11
11
 
12
12
  module Burner
13
- class Jobs
13
+ module Library
14
14
  module IO
15
15
  # Check to see if a file exists. If short_circuit is set to true and the file
16
16
  # does not exist then the job will return false and short circuit the pipeline.
17
+ #
18
+ # Note: this does not use Payload#value.
17
19
  class Exist < Base
18
20
  attr_reader :short_circuit
19
21
 
@@ -25,8 +27,8 @@ module Burner
25
27
  freeze
26
28
  end
27
29
 
28
- def perform(output, _payload, params)
29
- compiled_path = compile_path(params)
30
+ def perform(output, payload)
31
+ compiled_path = job_string_template(path, output, payload)
30
32
 
31
33
  exists = File.exist?(compiled_path)
32
34
  verb = exists ? 'does' : 'does not'
@@ -10,9 +10,12 @@
10
10
  require_relative 'base'
11
11
 
12
12
  module Burner
13
- class Jobs
13
+ module Library
14
14
  module IO
15
15
  # Read value from disk.
16
+ #
17
+ # Expected Payload#value input: nothing.
18
+ # Payload#value output: contents of the specified file.
16
19
  class Read < Base
17
20
  attr_reader :binary
18
21
 
@@ -24,8 +27,8 @@ module Burner
24
27
  freeze
25
28
  end
26
29
 
27
- def perform(output, payload, params)
28
- compiled_path = compile_path(params)
30
+ def perform(output, payload)
31
+ compiled_path = job_string_template(path, output, payload)
29
32
 
30
33
  output.detail("Reading: #{compiled_path}")
31
34
 
@@ -10,9 +10,12 @@
10
10
  require_relative 'base'
11
11
 
12
12
  module Burner
13
- class Jobs
13
+ module Library
14
14
  module IO
15
15
  # Write value to disk.
16
+ #
17
+ # Expected Payload#value input: anything.
18
+ # Payload#value output: whatever was passed in.
16
19
  class Write < Base
17
20
  attr_reader :binary
18
21
 
@@ -24,8 +27,8 @@ module Burner
24
27
  freeze
25
28
  end
26
29
 
27
- def perform(output, payload, params)
28
- compiled_path = compile_path(params)
30
+ def perform(output, payload)
31
+ compiled_path = job_string_template(path, output, payload)
29
32
 
30
33
  ensure_directory_exists(output, compiled_path)
31
34
 
@@ -35,12 +38,14 @@ module Burner
35
38
  File.open(compiled_path, mode) { |io| io.write(payload.value) }
36
39
  end.real
37
40
 
38
- payload.add_written_file(
41
+ side_effect = SideEffects::WrittenFile.new(
39
42
  logical_filename: compiled_path,
40
43
  physical_filename: compiled_path,
41
44
  time_in_seconds: time_in_seconds
42
45
  )
43
46
 
47
+ payload.add_side_effect(side_effect)
48
+
44
49
  nil
45
50
  end
46
51
 
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Library
12
+ module Serialize
13
+ # Take an array of arrays and create a CSV.
14
+ #
15
+ # Expected Payload#value input: array of arrays.
16
+ # Payload#value output: a serialized CSV string.
17
+ class Csv < Job
18
+ def perform(_output, payload)
19
+ payload.value = CSV.generate(options) do |csv|
20
+ array(payload.value).each do |row|
21
+ csv << row
22
+ end
23
+ end
24
+
25
+ nil
26
+ end
27
+
28
+ private
29
+
30
+ def options
31
+ {
32
+ headers: false,
33
+ write_headers: false
34
+ }
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -8,11 +8,14 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- class Jobs
11
+ module Library
12
12
  module Serialize
13
13
  # Treat value like a Ruby object and serialize it using JSON.
14
+ #
15
+ # Expected Payload#value input: anything.
16
+ # Payload#value output: string representing the output of the JSON serializer.
14
17
  class Json < Job
15
- def perform(_output, payload, _params)
18
+ def perform(_output, payload)
16
19
  payload.value = payload.value.to_json
17
20
 
18
21
  nil
@@ -8,11 +8,14 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- class Jobs
11
+ module Library
12
12
  module Serialize
13
13
  # Treat value like a Ruby object and serialize it using YAML.
14
+ #
15
+ # Expected Payload#value input: anything.
16
+ # Payload#value output: string representing the output of the YAML serializer.
14
17
  class Yaml < Job
15
- def perform(_output, payload, _params)
18
+ def perform(_output, payload)
16
19
  payload.value = payload.value.to_yaml
17
20
 
18
21
  nil
@@ -8,9 +8,12 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- class Jobs
11
+ module Library
12
12
  # Arbitrarily set value
13
- class Set < Job
13
+ #
14
+ # Expected Payload#value input: anything.
15
+ # Payload#value output: whatever value was specified in this job.
16
+ class SetValue < Job
14
17
  attr_reader :value
15
18
 
16
19
  def initialize(name:, value: nil)
@@ -21,7 +24,7 @@ module Burner
21
24
  freeze
22
25
  end
23
26
 
24
- def perform(_output, payload, _params)
27
+ def perform(_output, payload)
25
28
  payload.value = value
26
29
 
27
30
  nil
@@ -8,8 +8,10 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- class Jobs
11
+ module Library
12
12
  # Arbitrarily put thread to sleep for X number of seconds
13
+ #
14
+ # Payload#value output: whatever value was specified in this job.
13
15
  class Sleep < Job
14
16
  attr_reader :seconds
15
17
 
@@ -21,7 +23,7 @@ module Burner
21
23
  freeze
22
24
  end
23
25
 
24
- def perform(output, _payload, _params)
26
+ def perform(output, _payload)
25
27
  output.detail("Going to sleep for #{seconds} second(s)")
26
28
 
27
29
  Kernel.sleep(seconds)
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'modeling/attribute'
11
+ require_relative 'modeling/attribute_renderer'
12
+ require_relative 'modeling/key_index_mapping'
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Modeling
12
+ # Defines a top-level key and the associated transformers for deriving the final value
13
+ # to set the key to.
14
+ class Attribute
15
+ acts_as_hashable
16
+
17
+ attr_reader :key, :transformers
18
+
19
+ def initialize(key:, transformers: [])
20
+ raise ArgumentError, 'key is required' if key.to_s.empty?
21
+
22
+ @key = key.to_s
23
+ @transformers = Realize::Transformers.array(transformers)
24
+
25
+ freeze
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Modeling
12
+ # Composed of an Attribute instance and a Pipeline instance. It knows how to
13
+ # render/transform an Attribute. Since this library is data-first, these intermediary
14
+ # objects are necessary for non-data-centric modeling.
15
+ class AttributeRenderer
16
+ extend Forwardable
17
+
18
+ attr_reader :attribute, :pipeline
19
+
20
+ def_delegators :attribute, :key
21
+
22
+ def_delegators :pipeline, :transform
23
+
24
+ def initialize(attribute, resolver)
25
+ @attribute = attribute
26
+ @pipeline = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
27
+
28
+ freeze
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Modeling
12
+ # Generic relationship between a numeric index and a key.
13
+ class KeyIndexMapping
14
+ acts_as_hashable
15
+
16
+ attr_reader :index, :key
17
+
18
+ def initialize(index:, key:)
19
+ raise ArgumentError, 'index is required' if index.to_s.empty?
20
+ raise ArgumentError, 'key is required' if key.to_s.empty?
21
+
22
+ @index = index.to_i
23
+ @key = key.to_s
24
+
25
+ freeze
26
+ end
27
+ end
28
+ end
29
+ end
@@ -7,27 +7,38 @@
7
7
  # LICENSE file in the root directory of this source tree.
8
8
  #
9
9
 
10
- require_relative 'written_file'
11
-
12
10
  module Burner
13
11
  # The input for all Job#perform methods. The main notion of this object is its "value"
14
12
  # attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
15
13
  # methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
16
14
  # library and lifecycle, but I am not sure there is any other way around it: trying to build
17
15
  # a generic, open-ended object pipeline to serve almost any use case.
16
+ #
17
+ # The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
18
+ # structured/additive manner. The initial use case for this was for Burner's core IO jobs to
19
+ # report back the files it has written in a more structured data way (as opposed to simply
20
+ # writing some information to the output.)
18
21
  class Payload
19
22
  attr_accessor :value
20
23
 
21
- attr_reader :context, :written_files
24
+ attr_reader :params,
25
+ :side_effects,
26
+ :time
22
27
 
23
- def initialize(context: {}, value: nil, written_files: [])
24
- @context = context || {}
25
- @value = value
26
- @written_files = written_files || []
28
+ def initialize(
29
+ params: {},
30
+ side_effects: [],
31
+ time: Time.now.utc,
32
+ value: nil
33
+ )
34
+ @params = params || {}
35
+ @side_effects = side_effects || []
36
+ @time = time || Time.now.utc
37
+ @value = value
27
38
  end
28
39
 
29
- def add_written_file(written_file)
30
- tap { written_files << WrittenFile.make(written_file) }
40
+ def add_side_effect(side_effect)
41
+ tap { side_effects << side_effect }
31
42
  end
32
43
  end
33
44
  end
@@ -19,11 +19,16 @@ module Burner
19
19
  acts_as_hashable
20
20
 
21
21
  class JobNotFoundError < StandardError; end
22
+ class DuplicateJobNameError < StandardError; end
22
23
 
23
24
  attr_reader :steps
24
25
 
25
26
  def initialize(jobs: [], steps: [])
26
- jobs_by_name = Jobs.array(jobs).map { |job| [job.name, job] }.to_h
27
+ jobs = Jobs.array(jobs)
28
+
29
+ assert_unique_job_names(jobs)
30
+
31
+ jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
27
32
 
28
33
  @steps = Array(steps).map do |step_name|
29
34
  job = jobs_by_name[step_name.to_s]
@@ -35,15 +40,15 @@ module Burner
35
40
  end
36
41
 
37
42
  # The main entry-point for kicking off a pipeline.
38
- def execute(params: {}, output: Output.new, payload: Payload.new)
43
+ def execute(output: Output.new, payload: Payload.new)
39
44
  output.write("Pipeline started with #{steps.length} step(s)")
40
45
 
41
- output_params(params, output)
46
+ output_params(payload.params, output)
42
47
  output.ruler
43
48
 
44
49
  time_in_seconds = Benchmark.measure do
45
50
  steps.each do |step|
46
- return_value = step.perform(output, payload, params)
51
+ return_value = step.perform(output, payload)
47
52
 
48
53
  if return_value.is_a?(FalseClass)
49
54
  output.detail('Job returned false, ending pipeline.')
@@ -60,6 +65,20 @@ module Burner
60
65
 
61
66
  private
62
67
 
68
+ def assert_unique_job_names(jobs)
69
+ unique_job_names = Set.new
70
+
71
+ jobs.each do |job|
72
+ if unique_job_names.include?(job.name)
73
+ raise DuplicateJobNameError, "job with name: #{job.name} already declared"
74
+ end
75
+
76
+ unique_job_names << job.name
77
+ end
78
+
79
+ nil
80
+ end
81
+
63
82
  def output_params(params, output)
64
83
  if params.keys.any?
65
84
  output.write('Parameters:')