burner 1.5.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/CHANGELOG.md +36 -0
- data/README.md +59 -3
- data/lib/burner/data.rb +46 -0
- data/lib/burner/job.rb +2 -10
- data/lib/burner/job_set.rb +64 -0
- data/lib/burner/job_with_register.rb +8 -1
- data/lib/burner/jobs.rb +6 -0
- data/lib/burner/library.rb +6 -0
- data/lib/burner/library/collection/arrays_to_objects.rb +1 -1
- data/lib/burner/library/collection/coalesce.rb +14 -9
- data/lib/burner/library/collection/concatenate.rb +1 -1
- data/lib/burner/library/collection/graph.rb +1 -1
- data/lib/burner/library/collection/group.rb +15 -11
- data/lib/burner/library/collection/nested_aggregate.rb +1 -1
- data/lib/burner/library/collection/number.rb +51 -0
- data/lib/burner/library/collection/objects_to_arrays.rb +1 -1
- data/lib/burner/library/collection/shift.rb +1 -1
- data/lib/burner/library/collection/transform.rb +1 -1
- data/lib/burner/library/collection/unpivot.rb +1 -1
- data/lib/burner/library/collection/validate.rb +1 -1
- data/lib/burner/library/collection/values.rb +1 -1
- data/lib/burner/library/collection/zip.rb +1 -1
- data/lib/burner/library/compress/row_reader.rb +1 -1
- data/lib/burner/library/deserialize/yaml.rb +1 -1
- data/lib/burner/library/echo.rb +1 -1
- data/lib/burner/library/io/exist.rb +1 -1
- data/lib/burner/library/io/open_file_base.rb +1 -1
- data/lib/burner/library/io/row_reader.rb +1 -1
- data/lib/burner/library/io/write.rb +28 -1
- data/lib/burner/library/param/base.rb +29 -0
- data/lib/burner/library/param/from_register.rb +30 -0
- data/lib/burner/library/param/to_register.rb +28 -0
- data/lib/burner/library/serialize/csv.rb +1 -1
- data/lib/burner/library/sleep.rb +1 -1
- data/lib/burner/library/value/copy.rb +1 -1
- data/lib/burner/library/value/nest.rb +37 -0
- data/lib/burner/library/value/static.rb +1 -1
- data/lib/burner/library/value/transform.rb +38 -0
- data/lib/burner/payload.rb +39 -15
- data/lib/burner/pipeline.rb +6 -34
- data/lib/burner/util.rb +1 -0
- data/lib/burner/util/keyable.rb +23 -0
- data/lib/burner/version.rb +1 -1
- metadata +15 -5
@@ -16,6 +16,12 @@ module Burner
|
|
16
16
|
# It is worth noting that the resulting hashes values are singular objects and not an array
|
17
17
|
# like Ruby's Enumerable#group_by method.
|
18
18
|
#
|
19
|
+
# If the insensitive option is set as true then each key's value will be coerced as
|
20
|
+
# a lowercase string. This can help provide two types of insensitivity: case and type
|
21
|
+
# insensitivity. This may be appropriate in some places but not others. If any other
|
22
|
+
# value coercion is needed then another option would be to first transform the records
|
23
|
+
# before grouping them.
|
24
|
+
#
|
19
25
|
# An example of this specific job:
|
20
26
|
#
|
21
27
|
# input: [{ id: 1, code: 'a' }, { id: 2, code: 'b' }]
|
@@ -25,18 +31,22 @@ module Burner
|
|
25
31
|
# Expected Payload[register] input: array of objects.
|
26
32
|
# Payload[register] output: hash.
|
27
33
|
class Group < JobWithRegister
|
28
|
-
|
34
|
+
include Util::Keyable
|
35
|
+
|
36
|
+
attr_reader :insensitive, :keys, :resolver
|
29
37
|
|
30
38
|
def initialize(
|
31
|
-
|
39
|
+
insensitive: false,
|
32
40
|
keys: [],
|
41
|
+
name: '',
|
33
42
|
register: DEFAULT_REGISTER,
|
34
43
|
separator: ''
|
35
44
|
)
|
36
45
|
super(name: name, register: register)
|
37
46
|
|
38
|
-
@
|
39
|
-
@
|
47
|
+
@insensitive = insensitive || false
|
48
|
+
@keys = Array(keys)
|
49
|
+
@resolver = Objectable.resolver(separator: separator.to_s)
|
40
50
|
|
41
51
|
raise ArgumentError, 'at least one key is required' if @keys.empty?
|
42
52
|
|
@@ -50,18 +60,12 @@ module Burner
|
|
50
60
|
output.detail("Grouping based on key(s): #{keys} for #{count} records(s)")
|
51
61
|
|
52
62
|
grouped_records = payload[register].each_with_object({}) do |record, memo|
|
53
|
-
key = make_key(record)
|
63
|
+
key = make_key(record, keys, resolver, insensitive)
|
54
64
|
memo[key] = record
|
55
65
|
end
|
56
66
|
|
57
67
|
payload[register] = grouped_records
|
58
68
|
end
|
59
|
-
|
60
|
-
private
|
61
|
-
|
62
|
-
def make_key(record)
|
63
|
-
keys.map { |key| resolver.get(record, key) }
|
64
|
-
end
|
65
69
|
end
|
66
70
|
end
|
67
71
|
end
|
@@ -21,7 +21,7 @@ module Burner
|
|
21
21
|
class NestedAggregate < JobWithRegister
|
22
22
|
attr_reader :key, :key_mappings, :resolver
|
23
23
|
|
24
|
-
def initialize(
|
24
|
+
def initialize(key:, key_mappings: [], name: '', register: DEFAULT_REGISTER, separator: '')
|
25
25
|
super(name: name, register: register)
|
26
26
|
|
27
27
|
raise ArgumentError, 'key is required' if key.to_s.empty?
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Library
|
12
|
+
module Collection
|
13
|
+
# This job can iterate over a set of records and sequence them (set the specified key to
|
14
|
+
# a sequential index value.)
|
15
|
+
#
|
16
|
+
# Expected Payload[register] input: array of objects.
|
17
|
+
# Payload[register] output: array of objects.
|
18
|
+
class Number < JobWithRegister
|
19
|
+
BLANK = ''
|
20
|
+
DEFAULT_KEY = 'number'
|
21
|
+
DEFAULT_START_AT = 1
|
22
|
+
|
23
|
+
attr_reader :key, :resolver, :start_at
|
24
|
+
|
25
|
+
def initialize(
|
26
|
+
key: DEFAULT_KEY,
|
27
|
+
name: BLANK,
|
28
|
+
register: Burner::DEFAULT_REGISTER,
|
29
|
+
separator: BLANK,
|
30
|
+
start_at: DEFAULT_START_AT
|
31
|
+
)
|
32
|
+
super(name: name, register: register)
|
33
|
+
|
34
|
+
@key = key.to_s
|
35
|
+
@resolver = Objectable.resolver(separator: separator)
|
36
|
+
@start_at = start_at.to_i
|
37
|
+
|
38
|
+
freeze
|
39
|
+
end
|
40
|
+
|
41
|
+
def perform(output, payload)
|
42
|
+
output.detail("Setting '#{key}' for each record with values starting at #{start_at}")
|
43
|
+
|
44
|
+
ensure_array(payload).each.with_index(start_at) do |record, index|
|
45
|
+
resolver.set(record, key, index)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -58,7 +58,7 @@ module Burner
|
|
58
58
|
# nested hashes then set separator to '.'. For more information, see the underlying
|
59
59
|
# library that supports this dot-notation concept:
|
60
60
|
# https://github.com/bluemarblepayroll/objectable
|
61
|
-
def initialize(
|
61
|
+
def initialize(mappings: [], name: '', register: DEFAULT_REGISTER, separator: '')
|
62
62
|
super(name: name, register: register)
|
63
63
|
|
64
64
|
@mappings = Modeling::KeyIndexMapping.array(mappings)
|
@@ -23,7 +23,7 @@ module Burner
|
|
23
23
|
|
24
24
|
attr_reader :amount
|
25
25
|
|
26
|
-
def initialize(
|
26
|
+
def initialize(amount: DEFAULT_AMOUNT, name: '', register: DEFAULT_REGISTER)
|
27
27
|
super(name: name, register: register)
|
28
28
|
|
29
29
|
@amount = amount.to_i
|
@@ -29,10 +29,10 @@ module Burner
|
|
29
29
|
:validations
|
30
30
|
|
31
31
|
def initialize(
|
32
|
-
name:,
|
33
32
|
invalid_register: DEFAULT_INVALID_REGISTER,
|
34
33
|
join_char: DEFAULT_JOIN_CHAR,
|
35
34
|
message_key: DEFAULT_MESSAGE_KEY,
|
35
|
+
name: '',
|
36
36
|
register: DEFAULT_REGISTER,
|
37
37
|
separator: '',
|
38
38
|
validations: []
|
@@ -19,7 +19,7 @@ module Burner
|
|
19
19
|
class Values < JobWithRegister
|
20
20
|
attr_reader :include_keys
|
21
21
|
|
22
|
-
def initialize(
|
22
|
+
def initialize(include_keys: false, name: '', register: DEFAULT_REGISTER)
|
23
23
|
super(name: name, register: register)
|
24
24
|
|
25
25
|
@include_keys = include_keys || false
|
@@ -20,7 +20,7 @@ module Burner
|
|
20
20
|
class Yaml < JobWithRegister
|
21
21
|
attr_reader :safe
|
22
22
|
|
23
|
-
def initialize(name
|
23
|
+
def initialize(name: '', register: DEFAULT_REGISTER, safe: true)
|
24
24
|
super(name: name, register: register)
|
25
25
|
|
26
26
|
@safe = safe
|
data/lib/burner/library/echo.rb
CHANGED
@@ -17,7 +17,7 @@ module Burner
|
|
17
17
|
class Exist < Job
|
18
18
|
attr_reader :disk, :path, :short_circuit
|
19
19
|
|
20
|
-
def initialize(
|
20
|
+
def initialize(path:, disk: {}, name: '', short_circuit: false)
|
21
21
|
super(name: name)
|
22
22
|
|
23
23
|
raise ArgumentError, 'path is required' if path.to_s.empty?
|
@@ -14,7 +14,7 @@ module Burner
|
|
14
14
|
class OpenFileBase < JobWithRegister
|
15
15
|
attr_reader :binary, :disk, :path
|
16
16
|
|
17
|
-
def initialize(
|
17
|
+
def initialize(path:, binary: false, disk: {}, name: '', register: DEFAULT_REGISTER)
|
18
18
|
super(name: name, register: register)
|
19
19
|
|
20
20
|
raise ArgumentError, 'path is required' if path.to_s.empty?
|
@@ -35,12 +35,12 @@ module Burner
|
|
35
35
|
:resolver
|
36
36
|
|
37
37
|
def initialize(
|
38
|
-
name:,
|
39
38
|
binary: false,
|
40
39
|
data_key: DEFAULT_DATA_KEY,
|
41
40
|
disk: {},
|
42
41
|
ignore_blank_path: false,
|
43
42
|
ignore_file_not_found: false,
|
43
|
+
name: '',
|
44
44
|
path_key: DEFAULT_PATH_KEY,
|
45
45
|
register: DEFAULT_REGISTER,
|
46
46
|
separator: ''
|
@@ -12,11 +12,34 @@ require_relative 'open_file_base'
|
|
12
12
|
module Burner
|
13
13
|
module Library
|
14
14
|
module IO
|
15
|
-
# Write value to disk.
|
15
|
+
# Write value to disk. By default, written files are also logged as WrittenFile
|
16
|
+
# instances to the Payload#side_effects array. You can pass in
|
17
|
+
# supress_side_effect: true to disable this behavior.
|
16
18
|
#
|
17
19
|
# Expected Payload[register] input: anything.
|
18
20
|
# Payload[register] output: whatever was passed in.
|
19
21
|
class Write < OpenFileBase
|
22
|
+
attr_reader :supress_side_effect
|
23
|
+
|
24
|
+
def initialize(
|
25
|
+
path:,
|
26
|
+
binary: false,
|
27
|
+
disk: {},
|
28
|
+
name: '',
|
29
|
+
register: DEFAULT_REGISTER,
|
30
|
+
supress_side_effect: false
|
31
|
+
)
|
32
|
+
@supress_side_effect = supress_side_effect || false
|
33
|
+
|
34
|
+
super(
|
35
|
+
binary: binary,
|
36
|
+
disk: disk,
|
37
|
+
name: name,
|
38
|
+
path: path,
|
39
|
+
register: register
|
40
|
+
)
|
41
|
+
end
|
42
|
+
|
20
43
|
def perform(output, payload)
|
21
44
|
logical_filename = job_string_template(path, output, payload)
|
22
45
|
physical_filename = nil
|
@@ -29,6 +52,10 @@ module Burner
|
|
29
52
|
|
30
53
|
output.detail("Wrote to: #{physical_filename}")
|
31
54
|
|
55
|
+
return if supress_side_effect
|
56
|
+
|
57
|
+
output.detail("Saving to side effects: #{logical_filename}")
|
58
|
+
|
32
59
|
side_effect = SideEffects::WrittenFile.new(
|
33
60
|
logical_filename: logical_filename,
|
34
61
|
physical_filename: physical_filename,
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Library
|
12
|
+
module Param
|
13
|
+
# Common logic shared across Param job subclasses.
|
14
|
+
class Base < JobWithRegister
|
15
|
+
BLANK = ''
|
16
|
+
|
17
|
+
attr_reader :param_key
|
18
|
+
|
19
|
+
def initialize(name: BLANK, param_key: BLANK, register: DEFAULT_REGISTER)
|
20
|
+
super(name: name, register: register)
|
21
|
+
|
22
|
+
@param_key = param_key.to_s
|
23
|
+
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
module Library
|
14
|
+
module Param
|
15
|
+
# Copy a register's value into a param key. Generally speaking you should only be
|
16
|
+
# mutating registers, that way the params stay true to the passed in params for the
|
17
|
+
# pipeline. But this job is available in case a param needs to be updated.
|
18
|
+
#
|
19
|
+
# Expected Payload[register] input: anything.
|
20
|
+
# Payload.params(param_key) output: whatever value was specified in the register.
|
21
|
+
class FromRegister < Base
|
22
|
+
def perform(output, payload)
|
23
|
+
output.detail("Pushing value from register: #{register} to param: #{param_key}")
|
24
|
+
|
25
|
+
payload.update_param(param_key, payload[register])
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
module Library
|
14
|
+
module Param
|
15
|
+
# Copy a param key's value into a register.
|
16
|
+
#
|
17
|
+
# Expected Payload.param(param_key) input: anything.
|
18
|
+
# Payload[register] output: whatever value was specified as the param_key's value.
|
19
|
+
class ToRegister < Base
|
20
|
+
def perform(output, payload)
|
21
|
+
output.detail("Pushing value to register: #{register} from param: #{param_key}")
|
22
|
+
|
23
|
+
payload[register] = payload.param(param_key)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -18,7 +18,7 @@ module Burner
|
|
18
18
|
class Csv < JobWithRegister
|
19
19
|
attr_reader :byte_order_mark
|
20
20
|
|
21
|
-
def initialize(
|
21
|
+
def initialize(byte_order_mark: nil, name: '', register: DEFAULT_REGISTER)
|
22
22
|
super(name: name, register: register)
|
23
23
|
|
24
24
|
@byte_order_mark = Modeling::ByteOrderMark.resolve(byte_order_mark)
|
data/lib/burner/library/sleep.rb
CHANGED
@@ -19,7 +19,7 @@ module Burner
|
|
19
19
|
class Copy < Job
|
20
20
|
attr_reader :from_register, :to_register
|
21
21
|
|
22
|
-
def initialize(
|
22
|
+
def initialize(from_register: DEFAULT_REGISTER, name: '', to_register: DEFAULT_REGISTER)
|
23
23
|
super(name: name)
|
24
24
|
|
25
25
|
@from_register = from_register.to_s
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Library
|
12
|
+
module Value
|
13
|
+
# This job will nest the current value within a new outer hash. The specified key
|
14
|
+
# passed in will be the corresponding new hash key entry for the existing value.
|
15
|
+
#
|
16
|
+
# Expected Payload[from_register] input: anything.
|
17
|
+
# Payload[to_register] output: hash.
|
18
|
+
class Nest < JobWithRegister
|
19
|
+
DEFAULT_KEY = 'key'
|
20
|
+
|
21
|
+
attr_reader :key
|
22
|
+
|
23
|
+
def initialize(key: DEFAULT_KEY, name: '', register: Burner::DEFAULT_REGISTER)
|
24
|
+
super(name: name, register: register)
|
25
|
+
|
26
|
+
@key = key.to_s
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
def perform(_output, payload)
|
32
|
+
payload[register] = { key => payload[register] }
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|