burner 1.0.0.pre.alpha.1 → 1.0.0.pre.alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +48 -20
- data/burner.gemspec +3 -0
- data/exe/burner +3 -4
- data/lib/burner.rb +12 -0
- data/lib/burner/cli.rb +10 -8
- data/lib/burner/job.rb +28 -6
- data/lib/burner/jobs.rb +21 -23
- data/lib/burner/library.rb +30 -0
- data/lib/burner/library/collection/arrays_to_objects.rb +77 -0
- data/lib/burner/library/collection/graph.rb +44 -0
- data/lib/burner/library/collection/objects_to_arrays.rb +90 -0
- data/lib/burner/library/collection/shift.rb +44 -0
- data/lib/burner/library/collection/transform.rb +68 -0
- data/lib/burner/library/collection/unpivot.rb +47 -0
- data/lib/burner/library/collection/values.rb +51 -0
- data/lib/burner/library/deserialize/csv.rb +29 -0
- data/lib/burner/{jobs → library}/deserialize/json.rb +5 -2
- data/lib/burner/{jobs → library}/deserialize/yaml.rb +9 -3
- data/lib/burner/{jobs → library}/dummy.rb +4 -2
- data/lib/burner/{jobs → library}/echo.rb +5 -3
- data/lib/burner/{jobs → library}/io/base.rb +1 -7
- data/lib/burner/{jobs → library}/io/exist.rb +5 -3
- data/lib/burner/{jobs → library}/io/read.rb +6 -3
- data/lib/burner/{jobs → library}/io/write.rb +9 -4
- data/lib/burner/library/serialize/csv.rb +39 -0
- data/lib/burner/{jobs → library}/serialize/json.rb +5 -2
- data/lib/burner/{jobs → library}/serialize/yaml.rb +5 -2
- data/lib/burner/{jobs/set.rb → library/set_value.rb} +6 -3
- data/lib/burner/{jobs → library}/sleep.rb +4 -2
- data/lib/burner/modeling.rb +12 -0
- data/lib/burner/modeling/attribute.rb +29 -0
- data/lib/burner/modeling/attribute_renderer.rb +32 -0
- data/lib/burner/modeling/key_index_mapping.rb +29 -0
- data/lib/burner/payload.rb +20 -9
- data/lib/burner/pipeline.rb +23 -4
- data/lib/burner/side_effects.rb +10 -0
- data/lib/burner/side_effects/written_file.rb +28 -0
- data/lib/burner/step.rb +2 -4
- data/lib/burner/string_template.rb +6 -5
- data/lib/burner/util.rb +10 -0
- data/lib/burner/util/arrayable.rb +30 -0
- data/lib/burner/version.rb +1 -1
- metadata +74 -15
- data/lib/burner/written_file.rb +0 -28
@@ -8,7 +8,7 @@
|
|
8
8
|
#
|
9
9
|
|
10
10
|
module Burner
|
11
|
-
|
11
|
+
module Library
|
12
12
|
module IO
|
13
13
|
# Common configuration/code for all IO Job subclasses.
|
14
14
|
class Base < Job
|
@@ -21,12 +21,6 @@ module Burner
|
|
21
21
|
|
22
22
|
@path = path.to_s
|
23
23
|
end
|
24
|
-
|
25
|
-
private
|
26
|
-
|
27
|
-
def compile_path(params)
|
28
|
-
eval_string_template(path, params)
|
29
|
-
end
|
30
24
|
end
|
31
25
|
end
|
32
26
|
end
|
@@ -10,10 +10,12 @@
|
|
10
10
|
require_relative 'base'
|
11
11
|
|
12
12
|
module Burner
|
13
|
-
|
13
|
+
module Library
|
14
14
|
module IO
|
15
15
|
# Check to see if a file exists. If short_circuit is set to true and the file
|
16
16
|
# does not exist then the job will return false and short circuit the pipeline.
|
17
|
+
#
|
18
|
+
# Note: this does not use Payload#value.
|
17
19
|
class Exist < Base
|
18
20
|
attr_reader :short_circuit
|
19
21
|
|
@@ -25,8 +27,8 @@ module Burner
|
|
25
27
|
freeze
|
26
28
|
end
|
27
29
|
|
28
|
-
def perform(output,
|
29
|
-
compiled_path =
|
30
|
+
def perform(output, payload)
|
31
|
+
compiled_path = job_string_template(path, output, payload)
|
30
32
|
|
31
33
|
exists = File.exist?(compiled_path)
|
32
34
|
verb = exists ? 'does' : 'does not'
|
@@ -10,9 +10,12 @@
|
|
10
10
|
require_relative 'base'
|
11
11
|
|
12
12
|
module Burner
|
13
|
-
|
13
|
+
module Library
|
14
14
|
module IO
|
15
15
|
# Read value from disk.
|
16
|
+
#
|
17
|
+
# Expected Payload#value input: nothing.
|
18
|
+
# Payload#value output: contents of the specified file.
|
16
19
|
class Read < Base
|
17
20
|
attr_reader :binary
|
18
21
|
|
@@ -24,8 +27,8 @@ module Burner
|
|
24
27
|
freeze
|
25
28
|
end
|
26
29
|
|
27
|
-
def perform(output, payload
|
28
|
-
compiled_path =
|
30
|
+
def perform(output, payload)
|
31
|
+
compiled_path = job_string_template(path, output, payload)
|
29
32
|
|
30
33
|
output.detail("Reading: #{compiled_path}")
|
31
34
|
|
@@ -10,9 +10,12 @@
|
|
10
10
|
require_relative 'base'
|
11
11
|
|
12
12
|
module Burner
|
13
|
-
|
13
|
+
module Library
|
14
14
|
module IO
|
15
15
|
# Write value to disk.
|
16
|
+
#
|
17
|
+
# Expected Payload#value input: anything.
|
18
|
+
# Payload#value output: whatever was passed in.
|
16
19
|
class Write < Base
|
17
20
|
attr_reader :binary
|
18
21
|
|
@@ -24,8 +27,8 @@ module Burner
|
|
24
27
|
freeze
|
25
28
|
end
|
26
29
|
|
27
|
-
def perform(output, payload
|
28
|
-
compiled_path =
|
30
|
+
def perform(output, payload)
|
31
|
+
compiled_path = job_string_template(path, output, payload)
|
29
32
|
|
30
33
|
ensure_directory_exists(output, compiled_path)
|
31
34
|
|
@@ -35,12 +38,14 @@ module Burner
|
|
35
38
|
File.open(compiled_path, mode) { |io| io.write(payload.value) }
|
36
39
|
end.real
|
37
40
|
|
38
|
-
|
41
|
+
side_effect = SideEffects::WrittenFile.new(
|
39
42
|
logical_filename: compiled_path,
|
40
43
|
physical_filename: compiled_path,
|
41
44
|
time_in_seconds: time_in_seconds
|
42
45
|
)
|
43
46
|
|
47
|
+
payload.add_side_effect(side_effect)
|
48
|
+
|
44
49
|
nil
|
45
50
|
end
|
46
51
|
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Library
|
12
|
+
module Serialize
|
13
|
+
# Take an array of arrays and create a CSV.
|
14
|
+
#
|
15
|
+
# Expected Payload#value input: array of arrays.
|
16
|
+
# Payload#value output: a serialized CSV string.
|
17
|
+
class Csv < Job
|
18
|
+
def perform(_output, payload)
|
19
|
+
payload.value = CSV.generate(options) do |csv|
|
20
|
+
array(payload.value).each do |row|
|
21
|
+
csv << row
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
nil
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def options
|
31
|
+
{
|
32
|
+
headers: false,
|
33
|
+
write_headers: false
|
34
|
+
}
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -8,11 +8,14 @@
|
|
8
8
|
#
|
9
9
|
|
10
10
|
module Burner
|
11
|
-
|
11
|
+
module Library
|
12
12
|
module Serialize
|
13
13
|
# Treat value like a Ruby object and serialize it using JSON.
|
14
|
+
#
|
15
|
+
# Expected Payload#value input: anything.
|
16
|
+
# Payload#value output: string representing the output of the JSON serializer.
|
14
17
|
class Json < Job
|
15
|
-
def perform(_output, payload
|
18
|
+
def perform(_output, payload)
|
16
19
|
payload.value = payload.value.to_json
|
17
20
|
|
18
21
|
nil
|
@@ -8,11 +8,14 @@
|
|
8
8
|
#
|
9
9
|
|
10
10
|
module Burner
|
11
|
-
|
11
|
+
module Library
|
12
12
|
module Serialize
|
13
13
|
# Treat value like a Ruby object and serialize it using YAML.
|
14
|
+
#
|
15
|
+
# Expected Payload#value input: anything.
|
16
|
+
# Payload#value output: string representing the output of the YAML serializer.
|
14
17
|
class Yaml < Job
|
15
|
-
def perform(_output, payload
|
18
|
+
def perform(_output, payload)
|
16
19
|
payload.value = payload.value.to_yaml
|
17
20
|
|
18
21
|
nil
|
@@ -8,9 +8,12 @@
|
|
8
8
|
#
|
9
9
|
|
10
10
|
module Burner
|
11
|
-
|
11
|
+
module Library
|
12
12
|
# Arbitrarily set value
|
13
|
-
|
13
|
+
#
|
14
|
+
# Expected Payload#value input: anything.
|
15
|
+
# Payload#value output: whatever value was specified in this job.
|
16
|
+
class SetValue < Job
|
14
17
|
attr_reader :value
|
15
18
|
|
16
19
|
def initialize(name:, value: nil)
|
@@ -21,7 +24,7 @@ module Burner
|
|
21
24
|
freeze
|
22
25
|
end
|
23
26
|
|
24
|
-
def perform(_output, payload
|
27
|
+
def perform(_output, payload)
|
25
28
|
payload.value = value
|
26
29
|
|
27
30
|
nil
|
@@ -8,8 +8,10 @@
|
|
8
8
|
#
|
9
9
|
|
10
10
|
module Burner
|
11
|
-
|
11
|
+
module Library
|
12
12
|
# Arbitrarily put thread to sleep for X number of seconds
|
13
|
+
#
|
14
|
+
# Payload#value output: whatever value was specified in this job.
|
13
15
|
class Sleep < Job
|
14
16
|
attr_reader :seconds
|
15
17
|
|
@@ -21,7 +23,7 @@ module Burner
|
|
21
23
|
freeze
|
22
24
|
end
|
23
25
|
|
24
|
-
def perform(output, _payload
|
26
|
+
def perform(output, _payload)
|
25
27
|
output.detail("Going to sleep for #{seconds} second(s)")
|
26
28
|
|
27
29
|
Kernel.sleep(seconds)
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'modeling/attribute'
|
11
|
+
require_relative 'modeling/attribute_renderer'
|
12
|
+
require_relative 'modeling/key_index_mapping'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
# Defines a top-level key and the associated transformers for deriving the final value
|
13
|
+
# to set the key to.
|
14
|
+
class Attribute
|
15
|
+
acts_as_hashable
|
16
|
+
|
17
|
+
attr_reader :key, :transformers
|
18
|
+
|
19
|
+
def initialize(key:, transformers: [])
|
20
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
21
|
+
|
22
|
+
@key = key.to_s
|
23
|
+
@transformers = Realize::Transformers.array(transformers)
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
# Composed of an Attribute instance and a Pipeline instance. It knows how to
|
13
|
+
# render/transform an Attribute. Since this library is data-first, these intermediary
|
14
|
+
# objects are necessary for non-data-centric modeling.
|
15
|
+
class AttributeRenderer
|
16
|
+
extend Forwardable
|
17
|
+
|
18
|
+
attr_reader :attribute, :pipeline
|
19
|
+
|
20
|
+
def_delegators :attribute, :key
|
21
|
+
|
22
|
+
def_delegators :pipeline, :transform
|
23
|
+
|
24
|
+
def initialize(attribute, resolver)
|
25
|
+
@attribute = attribute
|
26
|
+
@pipeline = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
# Generic relationship between a numeric index and a key.
|
13
|
+
class KeyIndexMapping
|
14
|
+
acts_as_hashable
|
15
|
+
|
16
|
+
attr_reader :index, :key
|
17
|
+
|
18
|
+
def initialize(index:, key:)
|
19
|
+
raise ArgumentError, 'index is required' if index.to_s.empty?
|
20
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
21
|
+
|
22
|
+
@index = index.to_i
|
23
|
+
@key = key.to_s
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/burner/payload.rb
CHANGED
@@ -7,27 +7,38 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
require_relative 'written_file'
|
11
|
-
|
12
10
|
module Burner
|
13
11
|
# The input for all Job#perform methods. The main notion of this object is its "value"
|
14
12
|
# attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
|
15
13
|
# methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
|
16
14
|
# library and lifecycle, but I am not sure there is any other way around it: trying to build
|
17
15
|
# a generic, open-ended object pipeline to serve almost any use case.
|
16
|
+
#
|
17
|
+
# The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
|
18
|
+
# structured/additive manner. The initial use case for this was for Burner's core IO jobs to
|
19
|
+
# report back the files it has written in a more structured data way (as opposed to simply
|
20
|
+
# writing some information to the output.)
|
18
21
|
class Payload
|
19
22
|
attr_accessor :value
|
20
23
|
|
21
|
-
attr_reader :
|
24
|
+
attr_reader :params,
|
25
|
+
:side_effects,
|
26
|
+
:time
|
22
27
|
|
23
|
-
def initialize(
|
24
|
-
|
25
|
-
|
26
|
-
|
28
|
+
def initialize(
|
29
|
+
params: {},
|
30
|
+
side_effects: [],
|
31
|
+
time: Time.now.utc,
|
32
|
+
value: nil
|
33
|
+
)
|
34
|
+
@params = params || {}
|
35
|
+
@side_effects = side_effects || []
|
36
|
+
@time = time || Time.now.utc
|
37
|
+
@value = value
|
27
38
|
end
|
28
39
|
|
29
|
-
def
|
30
|
-
tap {
|
40
|
+
def add_side_effect(side_effect)
|
41
|
+
tap { side_effects << side_effect }
|
31
42
|
end
|
32
43
|
end
|
33
44
|
end
|
data/lib/burner/pipeline.rb
CHANGED
@@ -19,11 +19,16 @@ module Burner
|
|
19
19
|
acts_as_hashable
|
20
20
|
|
21
21
|
class JobNotFoundError < StandardError; end
|
22
|
+
class DuplicateJobNameError < StandardError; end
|
22
23
|
|
23
24
|
attr_reader :steps
|
24
25
|
|
25
26
|
def initialize(jobs: [], steps: [])
|
26
|
-
|
27
|
+
jobs = Jobs.array(jobs)
|
28
|
+
|
29
|
+
assert_unique_job_names(jobs)
|
30
|
+
|
31
|
+
jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
|
27
32
|
|
28
33
|
@steps = Array(steps).map do |step_name|
|
29
34
|
job = jobs_by_name[step_name.to_s]
|
@@ -35,15 +40,15 @@ module Burner
|
|
35
40
|
end
|
36
41
|
|
37
42
|
# The main entry-point for kicking off a pipeline.
|
38
|
-
def execute(
|
43
|
+
def execute(output: Output.new, payload: Payload.new)
|
39
44
|
output.write("Pipeline started with #{steps.length} step(s)")
|
40
45
|
|
41
|
-
output_params(params, output)
|
46
|
+
output_params(payload.params, output)
|
42
47
|
output.ruler
|
43
48
|
|
44
49
|
time_in_seconds = Benchmark.measure do
|
45
50
|
steps.each do |step|
|
46
|
-
return_value = step.perform(output, payload
|
51
|
+
return_value = step.perform(output, payload)
|
47
52
|
|
48
53
|
if return_value.is_a?(FalseClass)
|
49
54
|
output.detail('Job returned false, ending pipeline.')
|
@@ -60,6 +65,20 @@ module Burner
|
|
60
65
|
|
61
66
|
private
|
62
67
|
|
68
|
+
def assert_unique_job_names(jobs)
|
69
|
+
unique_job_names = Set.new
|
70
|
+
|
71
|
+
jobs.each do |job|
|
72
|
+
if unique_job_names.include?(job.name)
|
73
|
+
raise DuplicateJobNameError, "job with name: #{job.name} already declared"
|
74
|
+
end
|
75
|
+
|
76
|
+
unique_job_names << job.name
|
77
|
+
end
|
78
|
+
|
79
|
+
nil
|
80
|
+
end
|
81
|
+
|
63
82
|
def output_params(params, output)
|
64
83
|
if params.keys.any?
|
65
84
|
output.write('Parameters:')
|