burner 0.0.0 → 1.0.0.pre.alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -3
  3. data/README.md +390 -2
  4. data/burner.gemspec +4 -1
  5. data/exe/burner +21 -0
  6. data/lib/burner.rb +19 -4
  7. data/lib/burner/cli.rb +45 -0
  8. data/lib/burner/job.rb +37 -0
  9. data/lib/burner/jobs.rb +58 -0
  10. data/lib/burner/jobs/collection/arrays_to_objects.rb +43 -0
  11. data/lib/burner/jobs/collection/graph.rb +43 -0
  12. data/lib/burner/jobs/collection/objects_to_arrays.rb +54 -0
  13. data/lib/burner/jobs/collection/shift.rb +43 -0
  14. data/lib/burner/jobs/collection/transform.rb +64 -0
  15. data/lib/burner/jobs/collection/transform/attribute.rb +33 -0
  16. data/lib/burner/jobs/collection/transform/attribute_renderer.rb +36 -0
  17. data/lib/burner/jobs/collection/unpivot.rb +45 -0
  18. data/lib/burner/jobs/deserialize/csv.rb +28 -0
  19. data/lib/burner/jobs/deserialize/json.rb +23 -0
  20. data/lib/burner/jobs/deserialize/yaml.rb +41 -0
  21. data/lib/burner/jobs/dummy.rb +19 -0
  22. data/lib/burner/jobs/echo.rb +33 -0
  23. data/lib/burner/jobs/io/base.rb +27 -0
  24. data/lib/burner/jobs/io/exist.rb +43 -0
  25. data/lib/burner/jobs/io/read.rb +45 -0
  26. data/lib/burner/jobs/io/write.rb +67 -0
  27. data/lib/burner/jobs/serialize/csv.rb +38 -0
  28. data/lib/burner/jobs/serialize/json.rb +23 -0
  29. data/lib/burner/jobs/serialize/yaml.rb +23 -0
  30. data/lib/burner/jobs/set.rb +31 -0
  31. data/lib/burner/jobs/sleep.rb +33 -0
  32. data/lib/burner/modeling.rb +10 -0
  33. data/lib/burner/modeling/key_index_mapping.rb +29 -0
  34. data/lib/burner/output.rb +66 -0
  35. data/lib/burner/payload.rb +41 -0
  36. data/lib/burner/pipeline.rb +73 -0
  37. data/lib/burner/step.rb +45 -0
  38. data/lib/burner/string_template.rb +40 -0
  39. data/lib/burner/version.rb +1 -1
  40. data/lib/burner/written_file.rb +28 -0
  41. metadata +82 -5
@@ -7,8 +7,23 @@
7
7
  # LICENSE file in the root directory of this source tree.
8
8
  #
9
9
 
10
- require_relative 'burner/cli'
10
+ require 'acts_as_hashable'
11
+ require 'benchmark'
12
+ require 'csv'
13
+ require 'forwardable'
14
+ require 'hash_math'
15
+ require 'hashematics'
16
+ require 'json'
17
+ require 'objectable'
18
+ require 'realize'
19
+ require 'securerandom'
20
+ require 'singleton'
21
+ require 'stringento'
22
+ require 'time'
23
+ require 'yaml'
24
+
25
+ # Common/Shared
26
+ require_relative 'burner/modeling'
11
27
 
12
- # Placeholder
13
- module Burner
14
- end
28
+ # Main Entrypoint(s)
29
+ require_relative 'burner/cli'
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'pipeline'
11
+
12
+ module Burner
13
+ # Process a single string as a Pipeline. This is mainly to back the command-line interface.
14
+ class Cli
15
+ attr_reader :payload, :pipeline
16
+
17
+ def initialize(args)
18
+ path = args.first
19
+ params = extract_cli_params(args)
20
+ config = read_yaml(path)
21
+ @pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
22
+ @payload = Payload.new(params: params)
23
+ end
24
+
25
+ def execute
26
+ pipeline.execute(payload: payload)
27
+ end
28
+
29
+ private
30
+
31
+ def read_yaml(path)
32
+ yaml = IO.read(path)
33
+
34
+ YAML.safe_load(yaml)
35
+ end
36
+
37
+ def extract_cli_params(args)
38
+ args[1..-1].each_with_object({}) do |arg, memo|
39
+ parts = arg.to_s.split('=')
40
+
41
+ memo[parts.first] = parts.last
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'string_template'
11
+
12
+ module Burner
13
+ # Abstract base class for all job subclasses. The only public method a subclass needs to
14
+ # implement #perform(params, payload, reporter) and then you can register it for use using
15
+ # the Burner::Jobs factory class method #register. An example of a registration:
16
+ # Burner::Jobs.register('your_class', YourClass)
17
+ class Job
18
+ acts_as_hashable
19
+
20
+ attr_reader :name, :string_template
21
+
22
+ def initialize(name:)
23
+ raise ArgumentError, 'name is required' if name.to_s.empty?
24
+
25
+ @name = name.to_s
26
+ @string_template = StringTemplate.instance
27
+ end
28
+
29
+ private
30
+
31
+ def job_string_template(expression, output, payload)
32
+ templatable_params = payload.params.merge(__id: output.id, __value: payload.value)
33
+
34
+ string_template.evaluate(expression, templatable_params)
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'job'
11
+ require_relative 'jobs/collection/arrays_to_objects'
12
+ require_relative 'jobs/collection/graph'
13
+ require_relative 'jobs/collection/objects_to_arrays'
14
+ require_relative 'jobs/collection/shift'
15
+ require_relative 'jobs/collection/transform'
16
+ require_relative 'jobs/collection/unpivot'
17
+ require_relative 'jobs/deserialize/csv'
18
+ require_relative 'jobs/deserialize/json'
19
+ require_relative 'jobs/deserialize/yaml'
20
+ require_relative 'jobs/dummy'
21
+ require_relative 'jobs/echo'
22
+ require_relative 'jobs/io/exist'
23
+ require_relative 'jobs/io/read'
24
+ require_relative 'jobs/io/write'
25
+ require_relative 'jobs/serialize/csv'
26
+ require_relative 'jobs/serialize/json'
27
+ require_relative 'jobs/serialize/yaml'
28
+ require_relative 'jobs/set'
29
+ require_relative 'jobs/sleep'
30
+
31
+ module Burner
32
+ # Main library of jobs. This file contains all the basic/default jobs. All other consumer
33
+ # libraries/applications can register their own, for example:
34
+ # Burner::Jobs.register('your_class', YourClass)
35
+ class Jobs
36
+ acts_as_hashable_factory
37
+
38
+ register 'collection/arrays_to_objects', Collection::ArraysToObjects
39
+ register 'collection/graph', Collection::Graph
40
+ register 'collection/objects_to_arrays', Collection::ObjectsToArrays
41
+ register 'collection/shift', Collection::Shift
42
+ register 'collection/transform', Collection::Transform
43
+ register 'collection/unpivot', Collection::Unpivot
44
+ register 'deserialize/csv', Deserialize::Csv
45
+ register 'deserialize/json', Deserialize::Json
46
+ register 'deserialize/yaml', Deserialize::Yaml
47
+ register 'dummy', '', Dummy
48
+ register 'echo', Echo
49
+ register 'io/exist', IO::Exist
50
+ register 'io/read', IO::Read
51
+ register 'io/write', IO::Write
52
+ register 'serialize/csv', Serialize::Csv
53
+ register 'serialize/json', Serialize::Json
54
+ register 'serialize/yaml', Serialize::Yaml
55
+ register 'set', Set
56
+ register 'sleep', Sleep
57
+ end
58
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ # Convert an array of arrays to an array of objects.
14
+ # Expected Payload#value input: array of arrays.
15
+ # Payload#value output: An array of hashes.
16
+ class ArraysToObjects < Job
17
+ attr_reader :mappings
18
+
19
+ def initialize(name:, mappings: [])
20
+ super(name: name)
21
+
22
+ @mappings = Modeling::KeyIndexMapping.array(mappings)
23
+
24
+ freeze
25
+ end
26
+
27
+ def perform(_output, payload)
28
+ payload.value = (payload.value || []).map { |array| index_to_key_map(array) }
29
+
30
+ nil
31
+ end
32
+
33
+ private
34
+
35
+ def index_to_key_map(array)
36
+ mappings.each_with_object({}) do |mapping, memo|
37
+ memo[mapping.key] = array[mapping.index]
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ # Take an array of (denormalized) objects and create an object hierarchy from them.
14
+ # Under the hood it uses Hashematics: https://github.com/bluemarblepayroll/hashematics.
15
+ # Expected Payload#value input: array of objects.
16
+ # Payload#value output: An array of objects.
17
+ class Graph < Job
18
+ attr_reader :key, :groups
19
+
20
+ def initialize(name:, key:, config: Hashematics::Configuration.new)
21
+ super(name: name)
22
+
23
+ raise ArgumentError, 'key is required' if key.to_s.empty?
24
+
25
+ @groups = Hashematics::Configuration.new(config).groups
26
+ @key = key.to_s
27
+
28
+ freeze
29
+ end
30
+
31
+ def perform(output, payload)
32
+ graph = Hashematics::Graph.new(groups).add(payload.value || [])
33
+
34
+ output.detail("Graphing: #{key}")
35
+
36
+ payload.value = graph.data(key)
37
+
38
+ nil
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ # Convert an array of objects to an array of arrays. You can leverage the separator
14
+ # option to support key paths and nested objects.
15
+ # Expected Payload#value input: array of hashes.
16
+ # Payload#value output: An array of arrays.
17
+ class ObjectsToArrays < Job
18
+ attr_reader :mappings
19
+
20
+ # If you wish to support nested objects you can pass in a string to use as a
21
+ # key path separator. For example: if you would like to recognize dot-notation for
22
+ # nested hashes then set separator to '.'.
23
+ def initialize(name:, mappings: [], separator: '')
24
+ super(name: name)
25
+
26
+ @mappings = Modeling::KeyIndexMapping.array(mappings)
27
+ @resolver = Objectable.resolver(separator: separator.to_s)
28
+
29
+ freeze
30
+ end
31
+
32
+ def perform(_output, payload)
33
+ payload.value = (payload.value || []).map { |object| key_to_index_map(object) }
34
+
35
+ nil
36
+ end
37
+
38
+ private
39
+
40
+ attr_reader :resolver
41
+
42
+ def key_to_index_map(object)
43
+ mappings.each_with_object(prototype_array) do |mapping, memo|
44
+ memo[mapping.index] = resolver.get(object, mapping.key)
45
+ end
46
+ end
47
+
48
+ def prototype_array
49
+ Array.new(mappings.length)
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ # Take an array and remove the first N elements, where N is specified by the amount
14
+ # attribute.
15
+ # Expected Payload#value input: nothing.
16
+ # Payload#value output: An array with N beginning elements removed.
17
+ class Shift < Job
18
+ DEFAULT_AMOUNT = 0
19
+
20
+ private_constant :DEFAULT_AMOUNT
21
+
22
+ attr_reader :amount
23
+
24
+ def initialize(name:, amount: DEFAULT_AMOUNT)
25
+ super(name: name)
26
+
27
+ @amount = amount.to_i
28
+
29
+ freeze
30
+ end
31
+
32
+ def perform(output, payload)
33
+ output.detail("Shifting #{amount} entries.")
34
+
35
+ payload.value ||= []
36
+ payload.value.shift(amount)
37
+
38
+ nil
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'transform/attribute'
11
+ require_relative 'transform/attribute_renderer'
12
+
13
+ module Burner
14
+ class Jobs
15
+ module Collection
16
+ # Iterate over all objects and return a new set of transformed objects. Under the hood
17
+ # this uses the Realize library: https://github.com/bluemarblepayroll/realize
18
+ # Expected Payload#value input: array of objects.
19
+ # Payload#value output: An array of objects.
20
+ class Transform < Job
21
+ BLANK = ''
22
+
23
+ attr_reader :attribute_renderers,
24
+ :exclusive,
25
+ :resolver
26
+
27
+ def initialize(name:, attributes: [], exclusive: false, separator: BLANK)
28
+ super(name: name)
29
+
30
+ @resolver = Objectable.resolver(separator: separator)
31
+ @exclusive = exclusive || false
32
+
33
+ @attribute_renderers = Attribute.array(attributes)
34
+ .map { |a| AttributeRenderer.new(a, resolver) }
35
+
36
+ freeze
37
+ end
38
+
39
+ def perform(output, payload)
40
+ payload.value = (payload.value || []).map { |row| transform(row, payload.time) }
41
+
42
+ attr_count = attribute_renderers.length
43
+ row_count = payload.value.length
44
+
45
+ output.detail("Transformed #{attr_count} attributes(s) for #{row_count} row(s)")
46
+
47
+ nil
48
+ end
49
+
50
+ private
51
+
52
+ def transform(row, time)
53
+ outgoing_row = exclusive ? {} : row
54
+
55
+ attribute_renderers.each_with_object(outgoing_row) do |attribute_renderer, memo|
56
+ value = attribute_renderer.transform(row, time)
57
+
58
+ resolver.set(memo, attribute_renderer.key, value)
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Collection
13
+ class Transform < Job
14
+ # Defines a top-level key and the associated transformers for deriving the final value
15
+ # to set the key to.
16
+ class Attribute
17
+ acts_as_hashable
18
+
19
+ attr_reader :key, :transformers
20
+
21
+ def initialize(key:, transformers: [])
22
+ raise ArgumentError, 'key is required' if key.to_s.empty?
23
+
24
+ @key = key.to_s
25
+ @transformers = Realize::Transformers.array(transformers)
26
+
27
+ freeze
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end