burner 0.0.0 → 1.0.0.pre.alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -3
  3. data/README.md +390 -2
  4. data/burner.gemspec +4 -1
  5. data/exe/burner +21 -0
  6. data/lib/burner.rb +19 -4
  7. data/lib/burner/cli.rb +45 -0
  8. data/lib/burner/job.rb +37 -0
  9. data/lib/burner/jobs.rb +58 -0
  10. data/lib/burner/jobs/collection/arrays_to_objects.rb +43 -0
  11. data/lib/burner/jobs/collection/graph.rb +43 -0
  12. data/lib/burner/jobs/collection/objects_to_arrays.rb +54 -0
  13. data/lib/burner/jobs/collection/shift.rb +43 -0
  14. data/lib/burner/jobs/collection/transform.rb +64 -0
  15. data/lib/burner/jobs/collection/transform/attribute.rb +33 -0
  16. data/lib/burner/jobs/collection/transform/attribute_renderer.rb +36 -0
  17. data/lib/burner/jobs/collection/unpivot.rb +45 -0
  18. data/lib/burner/jobs/deserialize/csv.rb +28 -0
  19. data/lib/burner/jobs/deserialize/json.rb +23 -0
  20. data/lib/burner/jobs/deserialize/yaml.rb +41 -0
  21. data/lib/burner/jobs/dummy.rb +19 -0
  22. data/lib/burner/jobs/echo.rb +33 -0
  23. data/lib/burner/jobs/io/base.rb +27 -0
  24. data/lib/burner/jobs/io/exist.rb +43 -0
  25. data/lib/burner/jobs/io/read.rb +45 -0
  26. data/lib/burner/jobs/io/write.rb +67 -0
  27. data/lib/burner/jobs/serialize/csv.rb +38 -0
  28. data/lib/burner/jobs/serialize/json.rb +23 -0
  29. data/lib/burner/jobs/serialize/yaml.rb +23 -0
  30. data/lib/burner/jobs/set.rb +31 -0
  31. data/lib/burner/jobs/sleep.rb +33 -0
  32. data/lib/burner/modeling.rb +10 -0
  33. data/lib/burner/modeling/key_index_mapping.rb +29 -0
  34. data/lib/burner/output.rb +66 -0
  35. data/lib/burner/payload.rb +41 -0
  36. data/lib/burner/pipeline.rb +73 -0
  37. data/lib/burner/step.rb +45 -0
  38. data/lib/burner/string_template.rb +40 -0
  39. data/lib/burner/version.rb +1 -1
  40. data/lib/burner/written_file.rb +28 -0
  41. metadata +82 -5
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ class Transform < Job
14
+ # Composed of an Attribute instance and a Pipeline instance. It knows how to
15
+ # render/transform an Attribute. Since this library is data-first, these intermediary
16
+ # objects are necessary for non-data-centric modeling.
17
+ class AttributeRenderer
18
+ extend Forwardable
19
+
20
+ attr_reader :attribute, :pipeline
21
+
22
+ def_delegators :attribute, :key
23
+
24
+ def_delegators :pipeline, :transform
25
+
26
+ def initialize(attribute, resolver)
27
+ @attribute = attribute
28
+ @pipeline = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
29
+
30
+ freeze
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ # Take an array of objects and un-pivot groups of keys into rows.
14
+ # Under the hood it uses HashMath's Unpivot class:
15
+ # https://github.com/bluemarblepayroll/hash_math
16
+ # Expected Payload#value input: array of objects.
17
+ # Payload#value output: An array of objects.
18
+ class Unpivot < Job
19
+ attr_reader :unpivot
20
+
21
+ def initialize(name:, pivot_set: HashMath::Unpivot::PivotSet.new)
22
+ super(name: name)
23
+
24
+ @unpivot = HashMath::Unpivot.new(pivot_set)
25
+
26
+ freeze
27
+ end
28
+
29
+ def perform(output, payload)
30
+ pivot_count = unpivot.pivot_set.pivots.length
31
+ key_count = unpivot.pivot_set.pivots.map { |p| p.keys.length }.sum
32
+ object_count = payload.value&.length || 0
33
+
34
+ message = "#{pivot_count} Pivots, Key(s): #{key_count} key(s), #{object_count} objects(s)"
35
+
36
+ output.detail(message)
37
+
38
+ payload.value = (payload.value || []).flat_map { |object| unpivot.expand(object) }
39
+
40
+ nil
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Deserialize
13
+ # Take a CSV string and de-serialize into object(s).
14
+ # Expected Payload#value input: nothing.
15
+ # Payload#value output: an array of arrays. Each inner array represents one data row.
16
+ class Csv < Job
17
+ # This currently only supports returning an array of arrays, including the header row.
18
+ # In the future this could be extended to offer more customizable options, such as
19
+ # making it return an array of hashes with the columns mapped, etc.)
20
+ def perform(_output, payload)
21
+ payload.value = CSV.new(payload.value, headers: false).to_a
22
+
23
+ nil
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Deserialize
13
+ # Take a JSON string and deserialize into object(s).
14
+ class Json < Job
15
+ def perform(_output, payload)
16
+ payload.value = JSON.parse(payload.value)
17
+
18
+ nil
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Deserialize
13
+ # Take a YAML string and deserialize into object(s).
14
+ class Yaml < Job
15
+ attr_reader :safe
16
+
17
+ def initialize(name:, safe: true)
18
+ super(name: name)
19
+
20
+ @safe = safe
21
+
22
+ freeze
23
+ end
24
+
25
+ # The YAML cop was disabled because the consumer may want to actually load unsafe
26
+ # YAML, which can load pretty much any type of class instead of putting the loader
27
+ # in a sandbox. By default, though, we will try and drive them towards using it
28
+ # in the safer alternative.
29
+ # rubocop:disable Security/YAMLLoad
30
+ def perform(output, payload)
31
+ output.detail('Warning: loading YAML not using safe_load.') unless safe
32
+
33
+ payload.value = safe ? YAML.safe_load(payload.value) : YAML.load(payload.value)
34
+
35
+ nil
36
+ end
37
+ # rubocop:enable Security/YAMLLoad
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ # Do nothing.
13
+ class Dummy < Job
14
+ def perform(_output, _payload)
15
+ nil
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ # Output a simple message to the output.
13
+ class Echo < Job
14
+ attr_reader :message
15
+
16
+ def initialize(name:, message: '')
17
+ super(name: name)
18
+
19
+ @message = message.to_s
20
+
21
+ freeze
22
+ end
23
+
24
+ def perform(output, payload)
25
+ compiled_message = job_string_template(message, output, payload)
26
+
27
+ output.detail(compiled_message)
28
+
29
+ nil
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module IO
13
+ # Common configuration/code for all IO Job subclasses.
14
+ class Base < Job
15
+ attr_reader :path
16
+
17
+ def initialize(name:, path:)
18
+ super(name: name)
19
+
20
+ raise ArgumentError, 'path is required' if path.to_s.empty?
21
+
22
+ @path = path.to_s
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'base'
11
+
12
+ module Burner
13
+ class Jobs
14
+ module IO
15
+ # Check to see if a file exists. If short_circuit is set to true and the file
16
+ # does not exist then the job will return false and short circuit the pipeline.
17
+ class Exist < Base
18
+ attr_reader :short_circuit
19
+
20
+ def initialize(name:, path:, short_circuit: false)
21
+ super(name: name, path: path)
22
+
23
+ @short_circuit = short_circuit || false
24
+
25
+ freeze
26
+ end
27
+
28
+ def perform(output, payload)
29
+ compiled_path = job_string_template(path, output, payload)
30
+
31
+ exists = File.exist?(compiled_path)
32
+ verb = exists ? 'does' : 'does not'
33
+
34
+ output.detail("The path: #{compiled_path} #{verb} exist")
35
+
36
+ # if anything but false is returned then the pipeline will not short circuit. So
37
+ # we need to make sure we explicitly return false.
38
+ short_circuit && !exists ? false : nil
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'base'
11
+
12
+ module Burner
13
+ class Jobs
14
+ module IO
15
+ # Read value from disk.
16
+ class Read < Base
17
+ attr_reader :binary
18
+
19
+ def initialize(name:, path:, binary: false)
20
+ super(name: name, path: path)
21
+
22
+ @binary = binary || false
23
+
24
+ freeze
25
+ end
26
+
27
+ def perform(output, payload)
28
+ compiled_path = job_string_template(path, output, payload)
29
+
30
+ output.detail("Reading: #{compiled_path}")
31
+
32
+ payload.value = File.open(compiled_path, mode, &:read)
33
+
34
+ nil
35
+ end
36
+
37
+ private
38
+
39
+ def mode
40
+ binary ? 'rb' : 'r'
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'base'
11
+
12
+ module Burner
13
+ class Jobs
14
+ module IO
15
+ # Write value to disk.
16
+ class Write < Base
17
+ attr_reader :binary
18
+
19
+ def initialize(name:, path:, binary: false)
20
+ super(name: name, path: path)
21
+
22
+ @binary = binary || false
23
+
24
+ freeze
25
+ end
26
+
27
+ def perform(output, payload)
28
+ compiled_path = job_string_template(path, output, payload)
29
+
30
+ ensure_directory_exists(output, compiled_path)
31
+
32
+ output.detail("Writing: #{compiled_path}")
33
+
34
+ time_in_seconds = Benchmark.measure do
35
+ File.open(compiled_path, mode) { |io| io.write(payload.value) }
36
+ end.real
37
+
38
+ payload.add_written_file(
39
+ logical_filename: compiled_path,
40
+ physical_filename: compiled_path,
41
+ time_in_seconds: time_in_seconds
42
+ )
43
+
44
+ nil
45
+ end
46
+
47
+ private
48
+
49
+ def ensure_directory_exists(output, compiled_path)
50
+ dirname = File.dirname(compiled_path)
51
+
52
+ return if File.exist?(dirname)
53
+
54
+ output.detail("Outer directory does not exist, creating: #{dirname}")
55
+
56
+ FileUtils.mkdir_p(dirname)
57
+
58
+ nil
59
+ end
60
+
61
+ def mode
62
+ binary ? 'wb' : 'w'
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Serialize
13
+ # Take an array of arrays and create a CSV.
14
+ # Expected Payload#value input: array of arrays.
15
+ # Payload#value output: a serialized CSV string.
16
+ class Csv < Job
17
+ def perform(_output, payload)
18
+ payload.value = CSV.generate(options) do |csv|
19
+ (payload.value || []).each do |row|
20
+ csv << row
21
+ end
22
+ end
23
+
24
+ nil
25
+ end
26
+
27
+ private
28
+
29
+ def options
30
+ {
31
+ headers: false,
32
+ write_headers: false
33
+ }
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end