burner 0.0.0 → 1.0.0.pre.alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -3
- data/README.md +390 -2
- data/burner.gemspec +4 -1
- data/exe/burner +21 -0
- data/lib/burner.rb +19 -4
- data/lib/burner/cli.rb +45 -0
- data/lib/burner/job.rb +37 -0
- data/lib/burner/jobs.rb +58 -0
- data/lib/burner/jobs/collection/arrays_to_objects.rb +43 -0
- data/lib/burner/jobs/collection/graph.rb +43 -0
- data/lib/burner/jobs/collection/objects_to_arrays.rb +54 -0
- data/lib/burner/jobs/collection/shift.rb +43 -0
- data/lib/burner/jobs/collection/transform.rb +64 -0
- data/lib/burner/jobs/collection/transform/attribute.rb +33 -0
- data/lib/burner/jobs/collection/transform/attribute_renderer.rb +36 -0
- data/lib/burner/jobs/collection/unpivot.rb +45 -0
- data/lib/burner/jobs/deserialize/csv.rb +28 -0
- data/lib/burner/jobs/deserialize/json.rb +23 -0
- data/lib/burner/jobs/deserialize/yaml.rb +41 -0
- data/lib/burner/jobs/dummy.rb +19 -0
- data/lib/burner/jobs/echo.rb +33 -0
- data/lib/burner/jobs/io/base.rb +27 -0
- data/lib/burner/jobs/io/exist.rb +43 -0
- data/lib/burner/jobs/io/read.rb +45 -0
- data/lib/burner/jobs/io/write.rb +67 -0
- data/lib/burner/jobs/serialize/csv.rb +38 -0
- data/lib/burner/jobs/serialize/json.rb +23 -0
- data/lib/burner/jobs/serialize/yaml.rb +23 -0
- data/lib/burner/jobs/set.rb +31 -0
- data/lib/burner/jobs/sleep.rb +33 -0
- data/lib/burner/modeling.rb +10 -0
- data/lib/burner/modeling/key_index_mapping.rb +29 -0
- data/lib/burner/output.rb +66 -0
- data/lib/burner/payload.rb +41 -0
- data/lib/burner/pipeline.rb +73 -0
- data/lib/burner/step.rb +45 -0
- data/lib/burner/string_template.rb +40 -0
- data/lib/burner/version.rb +1 -1
- data/lib/burner/written_file.rb +28 -0
- metadata +82 -5
data/lib/burner.rb
CHANGED
@@ -7,8 +7,23 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
|
10
|
+
require 'acts_as_hashable'
|
11
|
+
require 'benchmark'
|
12
|
+
require 'csv'
|
13
|
+
require 'forwardable'
|
14
|
+
require 'hash_math'
|
15
|
+
require 'hashematics'
|
16
|
+
require 'json'
|
17
|
+
require 'objectable'
|
18
|
+
require 'realize'
|
19
|
+
require 'securerandom'
|
20
|
+
require 'singleton'
|
21
|
+
require 'stringento'
|
22
|
+
require 'time'
|
23
|
+
require 'yaml'
|
24
|
+
|
25
|
+
# Common/Shared
|
26
|
+
require_relative 'burner/modeling'
|
11
27
|
|
12
|
-
#
|
13
|
-
|
14
|
-
end
|
28
|
+
# Main Entrypoint(s)
|
29
|
+
require_relative 'burner/cli'
|
data/lib/burner/cli.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'pipeline'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# Process a single string as a Pipeline. This is mainly to back the command-line interface.
|
14
|
+
class Cli
|
15
|
+
attr_reader :payload, :pipeline
|
16
|
+
|
17
|
+
def initialize(args)
|
18
|
+
path = args.first
|
19
|
+
params = extract_cli_params(args)
|
20
|
+
config = read_yaml(path)
|
21
|
+
@pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
|
22
|
+
@payload = Payload.new(params: params)
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute
|
26
|
+
pipeline.execute(payload: payload)
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def read_yaml(path)
|
32
|
+
yaml = IO.read(path)
|
33
|
+
|
34
|
+
YAML.safe_load(yaml)
|
35
|
+
end
|
36
|
+
|
37
|
+
def extract_cli_params(args)
|
38
|
+
args[1..-1].each_with_object({}) do |arg, memo|
|
39
|
+
parts = arg.to_s.split('=')
|
40
|
+
|
41
|
+
memo[parts.first] = parts.last
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/burner/job.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'string_template'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# Abstract base class for all job subclasses. The only public method a subclass needs to
|
14
|
+
# implement #perform(params, payload, reporter) and then you can register it for use using
|
15
|
+
# the Burner::Jobs factory class method #register. An example of a registration:
|
16
|
+
# Burner::Jobs.register('your_class', YourClass)
|
17
|
+
class Job
|
18
|
+
acts_as_hashable
|
19
|
+
|
20
|
+
attr_reader :name, :string_template
|
21
|
+
|
22
|
+
def initialize(name:)
|
23
|
+
raise ArgumentError, 'name is required' if name.to_s.empty?
|
24
|
+
|
25
|
+
@name = name.to_s
|
26
|
+
@string_template = StringTemplate.instance
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def job_string_template(expression, output, payload)
|
32
|
+
templatable_params = payload.params.merge(__id: output.id, __value: payload.value)
|
33
|
+
|
34
|
+
string_template.evaluate(expression, templatable_params)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/burner/jobs.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'job'
|
11
|
+
require_relative 'jobs/collection/arrays_to_objects'
|
12
|
+
require_relative 'jobs/collection/graph'
|
13
|
+
require_relative 'jobs/collection/objects_to_arrays'
|
14
|
+
require_relative 'jobs/collection/shift'
|
15
|
+
require_relative 'jobs/collection/transform'
|
16
|
+
require_relative 'jobs/collection/unpivot'
|
17
|
+
require_relative 'jobs/deserialize/csv'
|
18
|
+
require_relative 'jobs/deserialize/json'
|
19
|
+
require_relative 'jobs/deserialize/yaml'
|
20
|
+
require_relative 'jobs/dummy'
|
21
|
+
require_relative 'jobs/echo'
|
22
|
+
require_relative 'jobs/io/exist'
|
23
|
+
require_relative 'jobs/io/read'
|
24
|
+
require_relative 'jobs/io/write'
|
25
|
+
require_relative 'jobs/serialize/csv'
|
26
|
+
require_relative 'jobs/serialize/json'
|
27
|
+
require_relative 'jobs/serialize/yaml'
|
28
|
+
require_relative 'jobs/set'
|
29
|
+
require_relative 'jobs/sleep'
|
30
|
+
|
31
|
+
module Burner
|
32
|
+
# Main library of jobs. This file contains all the basic/default jobs. All other consumer
|
33
|
+
# libraries/applications can register their own, for example:
|
34
|
+
# Burner::Jobs.register('your_class', YourClass)
|
35
|
+
class Jobs
|
36
|
+
acts_as_hashable_factory
|
37
|
+
|
38
|
+
register 'collection/arrays_to_objects', Collection::ArraysToObjects
|
39
|
+
register 'collection/graph', Collection::Graph
|
40
|
+
register 'collection/objects_to_arrays', Collection::ObjectsToArrays
|
41
|
+
register 'collection/shift', Collection::Shift
|
42
|
+
register 'collection/transform', Collection::Transform
|
43
|
+
register 'collection/unpivot', Collection::Unpivot
|
44
|
+
register 'deserialize/csv', Deserialize::Csv
|
45
|
+
register 'deserialize/json', Deserialize::Json
|
46
|
+
register 'deserialize/yaml', Deserialize::Yaml
|
47
|
+
register 'dummy', '', Dummy
|
48
|
+
register 'echo', Echo
|
49
|
+
register 'io/exist', IO::Exist
|
50
|
+
register 'io/read', IO::Read
|
51
|
+
register 'io/write', IO::Write
|
52
|
+
register 'serialize/csv', Serialize::Csv
|
53
|
+
register 'serialize/json', Serialize::Json
|
54
|
+
register 'serialize/yaml', Serialize::Yaml
|
55
|
+
register 'set', Set
|
56
|
+
register 'sleep', Sleep
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
# Convert an array of arrays to an array of objects.
|
14
|
+
# Expected Payload#value input: array of arrays.
|
15
|
+
# Payload#value output: An array of hashes.
|
16
|
+
class ArraysToObjects < Job
|
17
|
+
attr_reader :mappings
|
18
|
+
|
19
|
+
def initialize(name:, mappings: [])
|
20
|
+
super(name: name)
|
21
|
+
|
22
|
+
@mappings = Modeling::KeyIndexMapping.array(mappings)
|
23
|
+
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
|
27
|
+
def perform(_output, payload)
|
28
|
+
payload.value = (payload.value || []).map { |array| index_to_key_map(array) }
|
29
|
+
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def index_to_key_map(array)
|
36
|
+
mappings.each_with_object({}) do |mapping, memo|
|
37
|
+
memo[mapping.key] = array[mapping.index]
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
# Take an array of (denormalized) objects and create an object hierarchy from them.
|
14
|
+
# Under the hood it uses Hashematics: https://github.com/bluemarblepayroll/hashematics.
|
15
|
+
# Expected Payload#value input: array of objects.
|
16
|
+
# Payload#value output: An array of objects.
|
17
|
+
class Graph < Job
|
18
|
+
attr_reader :key, :groups
|
19
|
+
|
20
|
+
def initialize(name:, key:, config: Hashematics::Configuration.new)
|
21
|
+
super(name: name)
|
22
|
+
|
23
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
24
|
+
|
25
|
+
@groups = Hashematics::Configuration.new(config).groups
|
26
|
+
@key = key.to_s
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
def perform(output, payload)
|
32
|
+
graph = Hashematics::Graph.new(groups).add(payload.value || [])
|
33
|
+
|
34
|
+
output.detail("Graphing: #{key}")
|
35
|
+
|
36
|
+
payload.value = graph.data(key)
|
37
|
+
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
# Convert an array of objects to an array of arrays. You can leverage the separator
|
14
|
+
# option to support key paths and nested objects.
|
15
|
+
# Expected Payload#value input: array of hashes.
|
16
|
+
# Payload#value output: An array of arrays.
|
17
|
+
class ObjectsToArrays < Job
|
18
|
+
attr_reader :mappings
|
19
|
+
|
20
|
+
# If you wish to support nested objects you can pass in a string to use as a
|
21
|
+
# key path separator. For example: if you would like to recognize dot-notation for
|
22
|
+
# nested hashes then set separator to '.'.
|
23
|
+
def initialize(name:, mappings: [], separator: '')
|
24
|
+
super(name: name)
|
25
|
+
|
26
|
+
@mappings = Modeling::KeyIndexMapping.array(mappings)
|
27
|
+
@resolver = Objectable.resolver(separator: separator.to_s)
|
28
|
+
|
29
|
+
freeze
|
30
|
+
end
|
31
|
+
|
32
|
+
def perform(_output, payload)
|
33
|
+
payload.value = (payload.value || []).map { |object| key_to_index_map(object) }
|
34
|
+
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
attr_reader :resolver
|
41
|
+
|
42
|
+
def key_to_index_map(object)
|
43
|
+
mappings.each_with_object(prototype_array) do |mapping, memo|
|
44
|
+
memo[mapping.index] = resolver.get(object, mapping.key)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def prototype_array
|
49
|
+
Array.new(mappings.length)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
# Take an array and remove the first N elements, where N is specified by the amount
|
14
|
+
# attribute.
|
15
|
+
# Expected Payload#value input: nothing.
|
16
|
+
# Payload#value output: An array with N beginning elements removed.
|
17
|
+
class Shift < Job
|
18
|
+
DEFAULT_AMOUNT = 0
|
19
|
+
|
20
|
+
private_constant :DEFAULT_AMOUNT
|
21
|
+
|
22
|
+
attr_reader :amount
|
23
|
+
|
24
|
+
def initialize(name:, amount: DEFAULT_AMOUNT)
|
25
|
+
super(name: name)
|
26
|
+
|
27
|
+
@amount = amount.to_i
|
28
|
+
|
29
|
+
freeze
|
30
|
+
end
|
31
|
+
|
32
|
+
def perform(output, payload)
|
33
|
+
output.detail("Shifting #{amount} entries.")
|
34
|
+
|
35
|
+
payload.value ||= []
|
36
|
+
payload.value.shift(amount)
|
37
|
+
|
38
|
+
nil
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'transform/attribute'
|
11
|
+
require_relative 'transform/attribute_renderer'
|
12
|
+
|
13
|
+
module Burner
|
14
|
+
class Jobs
|
15
|
+
module Collection
|
16
|
+
# Iterate over all objects and return a new set of transformed objects. Under the hood
|
17
|
+
# this uses the Realize library: https://github.com/bluemarblepayroll/realize
|
18
|
+
# Expected Payload#value input: array of objects.
|
19
|
+
# Payload#value output: An array of objects.
|
20
|
+
class Transform < Job
|
21
|
+
BLANK = ''
|
22
|
+
|
23
|
+
attr_reader :attribute_renderers,
|
24
|
+
:exclusive,
|
25
|
+
:resolver
|
26
|
+
|
27
|
+
def initialize(name:, attributes: [], exclusive: false, separator: BLANK)
|
28
|
+
super(name: name)
|
29
|
+
|
30
|
+
@resolver = Objectable.resolver(separator: separator)
|
31
|
+
@exclusive = exclusive || false
|
32
|
+
|
33
|
+
@attribute_renderers = Attribute.array(attributes)
|
34
|
+
.map { |a| AttributeRenderer.new(a, resolver) }
|
35
|
+
|
36
|
+
freeze
|
37
|
+
end
|
38
|
+
|
39
|
+
def perform(output, payload)
|
40
|
+
payload.value = (payload.value || []).map { |row| transform(row, payload.time) }
|
41
|
+
|
42
|
+
attr_count = attribute_renderers.length
|
43
|
+
row_count = payload.value.length
|
44
|
+
|
45
|
+
output.detail("Transformed #{attr_count} attributes(s) for #{row_count} row(s)")
|
46
|
+
|
47
|
+
nil
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def transform(row, time)
|
53
|
+
outgoing_row = exclusive ? {} : row
|
54
|
+
|
55
|
+
attribute_renderers.each_with_object(outgoing_row) do |attribute_renderer, memo|
|
56
|
+
value = attribute_renderer.transform(row, time)
|
57
|
+
|
58
|
+
resolver.set(memo, attribute_renderer.key, value)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Collection
|
13
|
+
class Transform < Job
|
14
|
+
# Defines a top-level key and the associated transformers for deriving the final value
|
15
|
+
# to set the key to.
|
16
|
+
class Attribute
|
17
|
+
acts_as_hashable
|
18
|
+
|
19
|
+
attr_reader :key, :transformers
|
20
|
+
|
21
|
+
def initialize(key:, transformers: [])
|
22
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
23
|
+
|
24
|
+
@key = key.to_s
|
25
|
+
@transformers = Realize::Transformers.array(transformers)
|
26
|
+
|
27
|
+
freeze
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|