burner 0.0.0 → 1.0.0.pre.alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -3
- data/README.md +390 -2
- data/burner.gemspec +4 -1
- data/exe/burner +21 -0
- data/lib/burner.rb +19 -4
- data/lib/burner/cli.rb +45 -0
- data/lib/burner/job.rb +37 -0
- data/lib/burner/jobs.rb +58 -0
- data/lib/burner/jobs/collection/arrays_to_objects.rb +43 -0
- data/lib/burner/jobs/collection/graph.rb +43 -0
- data/lib/burner/jobs/collection/objects_to_arrays.rb +54 -0
- data/lib/burner/jobs/collection/shift.rb +43 -0
- data/lib/burner/jobs/collection/transform.rb +64 -0
- data/lib/burner/jobs/collection/transform/attribute.rb +33 -0
- data/lib/burner/jobs/collection/transform/attribute_renderer.rb +36 -0
- data/lib/burner/jobs/collection/unpivot.rb +45 -0
- data/lib/burner/jobs/deserialize/csv.rb +28 -0
- data/lib/burner/jobs/deserialize/json.rb +23 -0
- data/lib/burner/jobs/deserialize/yaml.rb +41 -0
- data/lib/burner/jobs/dummy.rb +19 -0
- data/lib/burner/jobs/echo.rb +33 -0
- data/lib/burner/jobs/io/base.rb +27 -0
- data/lib/burner/jobs/io/exist.rb +43 -0
- data/lib/burner/jobs/io/read.rb +45 -0
- data/lib/burner/jobs/io/write.rb +67 -0
- data/lib/burner/jobs/serialize/csv.rb +38 -0
- data/lib/burner/jobs/serialize/json.rb +23 -0
- data/lib/burner/jobs/serialize/yaml.rb +23 -0
- data/lib/burner/jobs/set.rb +31 -0
- data/lib/burner/jobs/sleep.rb +33 -0
- data/lib/burner/modeling.rb +10 -0
- data/lib/burner/modeling/key_index_mapping.rb +29 -0
- data/lib/burner/output.rb +66 -0
- data/lib/burner/payload.rb +41 -0
- data/lib/burner/pipeline.rb +73 -0
- data/lib/burner/step.rb +45 -0
- data/lib/burner/string_template.rb +40 -0
- data/lib/burner/version.rb +1 -1
- data/lib/burner/written_file.rb +28 -0
- metadata +82 -5
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
class Transform < Job
|
14
|
+
# Composed of an Attribute instance and a Pipeline instance. It knows how to
|
15
|
+
# render/transform an Attribute. Since this library is data-first, these intermediary
|
16
|
+
# objects are necessary for non-data-centric modeling.
|
17
|
+
class AttributeRenderer
|
18
|
+
extend Forwardable
|
19
|
+
|
20
|
+
attr_reader :attribute, :pipeline
|
21
|
+
|
22
|
+
def_delegators :attribute, :key
|
23
|
+
|
24
|
+
def_delegators :pipeline, :transform
|
25
|
+
|
26
|
+
def initialize(attribute, resolver)
|
27
|
+
@attribute = attribute
|
28
|
+
@pipeline = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
|
29
|
+
|
30
|
+
freeze
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
# Take an array of objects and un-pivot groups of keys into rows.
|
14
|
+
# Under the hood it uses HashMath's Unpivot class:
|
15
|
+
# https://github.com/bluemarblepayroll/hash_math
|
16
|
+
# Expected Payload#value input: array of objects.
|
17
|
+
# Payload#value output: An array of objects.
|
18
|
+
class Unpivot < Job
|
19
|
+
attr_reader :unpivot
|
20
|
+
|
21
|
+
def initialize(name:, pivot_set: HashMath::Unpivot::PivotSet.new)
|
22
|
+
super(name: name)
|
23
|
+
|
24
|
+
@unpivot = HashMath::Unpivot.new(pivot_set)
|
25
|
+
|
26
|
+
freeze
|
27
|
+
end
|
28
|
+
|
29
|
+
def perform(output, payload)
|
30
|
+
pivot_count = unpivot.pivot_set.pivots.length
|
31
|
+
key_count = unpivot.pivot_set.pivots.map { |p| p.keys.length }.sum
|
32
|
+
object_count = payload.value&.length || 0
|
33
|
+
|
34
|
+
message = "#{pivot_count} Pivots, Key(s): #{key_count} key(s), #{object_count} objects(s)"
|
35
|
+
|
36
|
+
output.detail(message)
|
37
|
+
|
38
|
+
payload.value = (payload.value || []).flat_map { |object| unpivot.expand(object) }
|
39
|
+
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a CSV string and de-serialize into object(s).
|
14
|
+
# Expected Payload#value input: nothing.
|
15
|
+
# Payload#value output: an array of arrays. Each inner array represents one data row.
|
16
|
+
class Csv < Job
|
17
|
+
# This currently only supports returning an array of arrays, including the header row.
|
18
|
+
# In the future this could be extended to offer more customizable options, such as
|
19
|
+
# making it return an array of hashes with the columns mapped, etc.)
|
20
|
+
def perform(_output, payload)
|
21
|
+
payload.value = CSV.new(payload.value, headers: false).to_a
|
22
|
+
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a JSON string and deserialize into object(s).
|
14
|
+
class Json < Job
|
15
|
+
def perform(_output, payload)
|
16
|
+
payload.value = JSON.parse(payload.value)
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a YAML string and deserialize into object(s).
|
14
|
+
class Yaml < Job
|
15
|
+
attr_reader :safe
|
16
|
+
|
17
|
+
def initialize(name:, safe: true)
|
18
|
+
super(name: name)
|
19
|
+
|
20
|
+
@safe = safe
|
21
|
+
|
22
|
+
freeze
|
23
|
+
end
|
24
|
+
|
25
|
+
# The YAML cop was disabled because the consumer may want to actually load unsafe
|
26
|
+
# YAML, which can load pretty much any type of class instead of putting the loader
|
27
|
+
# in a sandbox. By default, though, we will try and drive them towards using it
|
28
|
+
# in the safer alternative.
|
29
|
+
# rubocop:disable Security/YAMLLoad
|
30
|
+
def perform(output, payload)
|
31
|
+
output.detail('Warning: loading YAML not using safe_load.') unless safe
|
32
|
+
|
33
|
+
payload.value = safe ? YAML.safe_load(payload.value) : YAML.load(payload.value)
|
34
|
+
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
# rubocop:enable Security/YAMLLoad
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Do nothing.
|
13
|
+
class Dummy < Job
|
14
|
+
def perform(_output, _payload)
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Output a simple message to the output.
|
13
|
+
class Echo < Job
|
14
|
+
attr_reader :message
|
15
|
+
|
16
|
+
def initialize(name:, message: '')
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@message = message.to_s
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(output, payload)
|
25
|
+
compiled_message = job_string_template(message, output, payload)
|
26
|
+
|
27
|
+
output.detail(compiled_message)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module IO
|
13
|
+
# Common configuration/code for all IO Job subclasses.
|
14
|
+
class Base < Job
|
15
|
+
attr_reader :path
|
16
|
+
|
17
|
+
def initialize(name:, path:)
|
18
|
+
super(name: name)
|
19
|
+
|
20
|
+
raise ArgumentError, 'path is required' if path.to_s.empty?
|
21
|
+
|
22
|
+
@path = path.to_s
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Check to see if a file exists. If short_circuit is set to true and the file
|
16
|
+
# does not exist then the job will return false and short circuit the pipeline.
|
17
|
+
class Exist < Base
|
18
|
+
attr_reader :short_circuit
|
19
|
+
|
20
|
+
def initialize(name:, path:, short_circuit: false)
|
21
|
+
super(name: name, path: path)
|
22
|
+
|
23
|
+
@short_circuit = short_circuit || false
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
|
28
|
+
def perform(output, payload)
|
29
|
+
compiled_path = job_string_template(path, output, payload)
|
30
|
+
|
31
|
+
exists = File.exist?(compiled_path)
|
32
|
+
verb = exists ? 'does' : 'does not'
|
33
|
+
|
34
|
+
output.detail("The path: #{compiled_path} #{verb} exist")
|
35
|
+
|
36
|
+
# if anything but false is returned then the pipeline will not short circuit. So
|
37
|
+
# we need to make sure we explicitly return false.
|
38
|
+
short_circuit && !exists ? false : nil
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Read value from disk.
|
16
|
+
class Read < Base
|
17
|
+
attr_reader :binary
|
18
|
+
|
19
|
+
def initialize(name:, path:, binary: false)
|
20
|
+
super(name: name, path: path)
|
21
|
+
|
22
|
+
@binary = binary || false
|
23
|
+
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
|
27
|
+
def perform(output, payload)
|
28
|
+
compiled_path = job_string_template(path, output, payload)
|
29
|
+
|
30
|
+
output.detail("Reading: #{compiled_path}")
|
31
|
+
|
32
|
+
payload.value = File.open(compiled_path, mode, &:read)
|
33
|
+
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def mode
|
40
|
+
binary ? 'rb' : 'r'
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Write value to disk.
|
16
|
+
class Write < Base
|
17
|
+
attr_reader :binary
|
18
|
+
|
19
|
+
def initialize(name:, path:, binary: false)
|
20
|
+
super(name: name, path: path)
|
21
|
+
|
22
|
+
@binary = binary || false
|
23
|
+
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
|
27
|
+
def perform(output, payload)
|
28
|
+
compiled_path = job_string_template(path, output, payload)
|
29
|
+
|
30
|
+
ensure_directory_exists(output, compiled_path)
|
31
|
+
|
32
|
+
output.detail("Writing: #{compiled_path}")
|
33
|
+
|
34
|
+
time_in_seconds = Benchmark.measure do
|
35
|
+
File.open(compiled_path, mode) { |io| io.write(payload.value) }
|
36
|
+
end.real
|
37
|
+
|
38
|
+
payload.add_written_file(
|
39
|
+
logical_filename: compiled_path,
|
40
|
+
physical_filename: compiled_path,
|
41
|
+
time_in_seconds: time_in_seconds
|
42
|
+
)
|
43
|
+
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def ensure_directory_exists(output, compiled_path)
|
50
|
+
dirname = File.dirname(compiled_path)
|
51
|
+
|
52
|
+
return if File.exist?(dirname)
|
53
|
+
|
54
|
+
output.detail("Outer directory does not exist, creating: #{dirname}")
|
55
|
+
|
56
|
+
FileUtils.mkdir_p(dirname)
|
57
|
+
|
58
|
+
nil
|
59
|
+
end
|
60
|
+
|
61
|
+
def mode
|
62
|
+
binary ? 'wb' : 'w'
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Take an array of arrays and create a CSV.
|
14
|
+
# Expected Payload#value input: array of arrays.
|
15
|
+
# Payload#value output: a serialized CSV string.
|
16
|
+
class Csv < Job
|
17
|
+
def perform(_output, payload)
|
18
|
+
payload.value = CSV.generate(options) do |csv|
|
19
|
+
(payload.value || []).each do |row|
|
20
|
+
csv << row
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
nil
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def options
|
30
|
+
{
|
31
|
+
headers: false,
|
32
|
+
write_headers: false
|
33
|
+
}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|