burner 1.5.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/CHANGELOG.md +36 -0
- data/README.md +59 -3
- data/lib/burner/data.rb +46 -0
- data/lib/burner/job.rb +2 -10
- data/lib/burner/job_set.rb +64 -0
- data/lib/burner/job_with_register.rb +8 -1
- data/lib/burner/jobs.rb +6 -0
- data/lib/burner/library.rb +6 -0
- data/lib/burner/library/collection/arrays_to_objects.rb +1 -1
- data/lib/burner/library/collection/coalesce.rb +14 -9
- data/lib/burner/library/collection/concatenate.rb +1 -1
- data/lib/burner/library/collection/graph.rb +1 -1
- data/lib/burner/library/collection/group.rb +15 -11
- data/lib/burner/library/collection/nested_aggregate.rb +1 -1
- data/lib/burner/library/collection/number.rb +51 -0
- data/lib/burner/library/collection/objects_to_arrays.rb +1 -1
- data/lib/burner/library/collection/shift.rb +1 -1
- data/lib/burner/library/collection/transform.rb +1 -1
- data/lib/burner/library/collection/unpivot.rb +1 -1
- data/lib/burner/library/collection/validate.rb +1 -1
- data/lib/burner/library/collection/values.rb +1 -1
- data/lib/burner/library/collection/zip.rb +1 -1
- data/lib/burner/library/compress/row_reader.rb +1 -1
- data/lib/burner/library/deserialize/yaml.rb +1 -1
- data/lib/burner/library/echo.rb +1 -1
- data/lib/burner/library/io/exist.rb +1 -1
- data/lib/burner/library/io/open_file_base.rb +1 -1
- data/lib/burner/library/io/row_reader.rb +1 -1
- data/lib/burner/library/io/write.rb +28 -1
- data/lib/burner/library/param/base.rb +29 -0
- data/lib/burner/library/param/from_register.rb +30 -0
- data/lib/burner/library/param/to_register.rb +28 -0
- data/lib/burner/library/serialize/csv.rb +1 -1
- data/lib/burner/library/sleep.rb +1 -1
- data/lib/burner/library/value/copy.rb +1 -1
- data/lib/burner/library/value/nest.rb +37 -0
- data/lib/burner/library/value/static.rb +1 -1
- data/lib/burner/library/value/transform.rb +38 -0
- data/lib/burner/payload.rb +39 -15
- data/lib/burner/pipeline.rb +6 -34
- data/lib/burner/util.rb +1 -0
- data/lib/burner/util/keyable.rb +23 -0
- data/lib/burner/version.rb +1 -1
- metadata +15 -5
@@ -17,7 +17,7 @@ module Burner
|
|
17
17
|
class Static < JobWithRegister
|
18
18
|
attr_reader :value
|
19
19
|
|
20
|
-
def initialize(name
|
20
|
+
def initialize(name: '', register: DEFAULT_REGISTER, value: nil)
|
21
21
|
super(name: name, register: register)
|
22
22
|
|
23
23
|
@value = value
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Library
|
12
|
+
module Value
|
13
|
+
# Transform the current value of the register through a Realize::Pipeline. This will
|
14
|
+
# transform the entire value, as opposed to the b/collection/transform job, which will
|
15
|
+
# iterate over each row/record in a dataset and transform each row/record.
|
16
|
+
#
|
17
|
+
# Expected Payload[register] input: anything.
|
18
|
+
# Payload[register] output: anything.
|
19
|
+
class Transform < JobWithRegister
|
20
|
+
attr_reader :pipeline
|
21
|
+
|
22
|
+
def initialize(name: '', register: DEFAULT_REGISTER, separator: '', transformers: [])
|
23
|
+
super(name: name, register: register)
|
24
|
+
|
25
|
+
resolver = Objectable.resolver(separator: separator)
|
26
|
+
|
27
|
+
@pipeline = Realize::Pipeline.new(transformers, resolver: resolver)
|
28
|
+
|
29
|
+
freeze
|
30
|
+
end
|
31
|
+
|
32
|
+
def perform(_output, payload)
|
33
|
+
payload[register] = pipeline.transform(payload[register], payload.time)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/burner/payload.rb
CHANGED
@@ -7,6 +7,8 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
+
require_relative 'data'
|
11
|
+
|
10
12
|
module Burner
|
11
13
|
# The input for all Job#perform methods. The main notion of this object is its 'registers'
|
12
14
|
# attribute. This registers attribute is a key-indifferent hash, accessible on Payload using
|
@@ -24,10 +26,15 @@ module Burner
|
|
24
26
|
# The 'time' attribute is important in that it should for the replaying of pipelines and jobs.
|
25
27
|
# Instead of having job's utilizing Time.now, Date.today, etc... they should rather opt to
|
26
28
|
# use this value instead.
|
29
|
+
#
|
30
|
+
# The notion of params are somewhat conflated with registers here. Both are hashes and both
|
31
|
+
# serve as data stores. The difference is registers are really meant to be the shared-data
|
32
|
+
# repository across jobs, while params are, more or less, the input into the _pipeline_.
|
33
|
+
# It is debatable if mutation of the params should be allowed but the design decision was made
|
34
|
+
# early on to allow for params to be mutable albeit with registers being the preferred mutable
|
35
|
+
# store.
|
27
36
|
class Payload
|
28
|
-
attr_reader :
|
29
|
-
:registers,
|
30
|
-
:side_effects,
|
37
|
+
attr_reader :side_effects,
|
31
38
|
:time
|
32
39
|
|
33
40
|
def initialize(
|
@@ -36,12 +43,32 @@ module Burner
|
|
36
43
|
side_effects: [],
|
37
44
|
time: Time.now.utc
|
38
45
|
)
|
39
|
-
@params = params
|
40
|
-
@registers =
|
46
|
+
@params = Data.new(params)
|
47
|
+
@registers = Data.new(registers)
|
41
48
|
@side_effects = side_effects || []
|
42
49
|
@time = time || Time.now.utc
|
50
|
+
end
|
43
51
|
|
44
|
-
|
52
|
+
# Backwards compatibility. This allows for control over the underlying data structure
|
53
|
+
# while still maintaining the same public API as before.
|
54
|
+
def params
|
55
|
+
@params.to_h
|
56
|
+
end
|
57
|
+
|
58
|
+
# Backwards compatibility. This allows for control over the underlying data structure
|
59
|
+
# while still maintaining the same public API as before.
|
60
|
+
def registers
|
61
|
+
@registers.to_h
|
62
|
+
end
|
63
|
+
|
64
|
+
# Law of Demeter: While params is an accessible hash, it is preferred that the
|
65
|
+
# public class methods are used.
|
66
|
+
def param(key)
|
67
|
+
@params[key]
|
68
|
+
end
|
69
|
+
|
70
|
+
def update_param(key, value)
|
71
|
+
tap { @params[key] = value }
|
45
72
|
end
|
46
73
|
|
47
74
|
# Add a side effect of a job. This helps to keep track of things jobs do outside of its
|
@@ -52,12 +79,12 @@ module Burner
|
|
52
79
|
|
53
80
|
# Set a register's value.
|
54
81
|
def []=(key, value)
|
55
|
-
|
82
|
+
@registers[key] = value
|
56
83
|
end
|
57
84
|
|
58
85
|
# Retrieve a register's value.
|
59
86
|
def [](key)
|
60
|
-
registers[key
|
87
|
+
@registers[key]
|
61
88
|
end
|
62
89
|
|
63
90
|
# Set halt_pipeline to true. This will indicate to the pipeline to stop all
|
@@ -71,14 +98,11 @@ module Burner
|
|
71
98
|
@halt_pipeline || false
|
72
99
|
end
|
73
100
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
registers[key.to_s] = value
|
78
|
-
end
|
101
|
+
def params_and_registers_hash
|
102
|
+
registers_hash = @registers.transform_keys { |key| "__#{key}_register" }
|
103
|
+
params_hash = @params.to_h
|
79
104
|
|
80
|
-
|
81
|
-
(registers || {}).each { |k, v| set(k, v) }
|
105
|
+
params_hash.merge(registers_hash)
|
82
106
|
end
|
83
107
|
end
|
84
108
|
end
|
data/lib/burner/pipeline.rb
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
require_relative '
|
10
|
+
require_relative 'job_set'
|
11
11
|
require_relative 'output'
|
12
12
|
require_relative 'payload'
|
13
13
|
require_relative 'step'
|
@@ -19,27 +19,13 @@ module Burner
|
|
19
19
|
class Pipeline
|
20
20
|
acts_as_hashable
|
21
21
|
|
22
|
-
class JobNotFoundError < StandardError; end
|
23
|
-
class DuplicateJobNameError < StandardError; end
|
24
|
-
|
25
22
|
attr_reader :steps
|
26
23
|
|
27
24
|
def initialize(jobs: [], steps: nil)
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
|
33
|
-
|
34
|
-
step_names = steps ? Array(steps) : jobs_by_name.keys
|
35
|
-
|
36
|
-
@steps = step_names.map do |step_name|
|
37
|
-
job = jobs_by_name[step_name.to_s]
|
38
|
-
|
39
|
-
raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
|
40
|
-
|
41
|
-
Step.new(job)
|
42
|
-
end
|
25
|
+
@steps = JobSet
|
26
|
+
.new(jobs)
|
27
|
+
.jobs(steps)
|
28
|
+
.map { |job| Step.new(job) }
|
43
29
|
end
|
44
30
|
|
45
31
|
# The main entry-point for kicking off a pipeline.
|
@@ -68,22 +54,8 @@ module Burner
|
|
68
54
|
|
69
55
|
private
|
70
56
|
|
71
|
-
def assert_unique_job_names(jobs)
|
72
|
-
unique_job_names = Set.new
|
73
|
-
|
74
|
-
jobs.each do |job|
|
75
|
-
if unique_job_names.include?(job.name)
|
76
|
-
raise DuplicateJobNameError, "job with name: #{job.name} already declared"
|
77
|
-
end
|
78
|
-
|
79
|
-
unique_job_names << job.name
|
80
|
-
end
|
81
|
-
|
82
|
-
nil
|
83
|
-
end
|
84
|
-
|
85
57
|
def output_params(params, output)
|
86
|
-
if params.
|
58
|
+
if params.any?
|
87
59
|
output.write('Parameters:')
|
88
60
|
else
|
89
61
|
output.write('No parameters passed in.')
|
data/lib/burner/util.rb
CHANGED
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Util
|
12
|
+
# Provide helper methods for keys.
|
13
|
+
module Keyable
|
14
|
+
def make_key(record, keys, resolver, insensitive)
|
15
|
+
keys.map do |key|
|
16
|
+
value = resolver.get(record, key)
|
17
|
+
|
18
|
+
insensitive ? value.to_s.downcase : value
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/burner/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: burner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: acts_as_hashable
|
@@ -220,6 +220,7 @@ files:
|
|
220
220
|
- ".gitignore"
|
221
221
|
- ".rubocop.yml"
|
222
222
|
- ".ruby-version"
|
223
|
+
- ".tool-versions"
|
223
224
|
- ".travis.yml"
|
224
225
|
- CHANGELOG.md
|
225
226
|
- CODE_OF_CONDUCT.md
|
@@ -234,9 +235,11 @@ files:
|
|
234
235
|
- exe/burner
|
235
236
|
- lib/burner.rb
|
236
237
|
- lib/burner/cli.rb
|
238
|
+
- lib/burner/data.rb
|
237
239
|
- lib/burner/disks.rb
|
238
240
|
- lib/burner/disks/local.rb
|
239
241
|
- lib/burner/job.rb
|
242
|
+
- lib/burner/job_set.rb
|
240
243
|
- lib/burner/job_with_register.rb
|
241
244
|
- lib/burner/jobs.rb
|
242
245
|
- lib/burner/library.rb
|
@@ -246,6 +249,7 @@ files:
|
|
246
249
|
- lib/burner/library/collection/graph.rb
|
247
250
|
- lib/burner/library/collection/group.rb
|
248
251
|
- lib/burner/library/collection/nested_aggregate.rb
|
252
|
+
- lib/burner/library/collection/number.rb
|
249
253
|
- lib/burner/library/collection/objects_to_arrays.rb
|
250
254
|
- lib/burner/library/collection/shift.rb
|
251
255
|
- lib/burner/library/collection/transform.rb
|
@@ -264,12 +268,17 @@ files:
|
|
264
268
|
- lib/burner/library/io/row_reader.rb
|
265
269
|
- lib/burner/library/io/write.rb
|
266
270
|
- lib/burner/library/nothing.rb
|
271
|
+
- lib/burner/library/param/base.rb
|
272
|
+
- lib/burner/library/param/from_register.rb
|
273
|
+
- lib/burner/library/param/to_register.rb
|
267
274
|
- lib/burner/library/serialize/csv.rb
|
268
275
|
- lib/burner/library/serialize/json.rb
|
269
276
|
- lib/burner/library/serialize/yaml.rb
|
270
277
|
- lib/burner/library/sleep.rb
|
271
278
|
- lib/burner/library/value/copy.rb
|
279
|
+
- lib/burner/library/value/nest.rb
|
272
280
|
- lib/burner/library/value/static.rb
|
281
|
+
- lib/burner/library/value/transform.rb
|
273
282
|
- lib/burner/modeling.rb
|
274
283
|
- lib/burner/modeling/attribute.rb
|
275
284
|
- lib/burner/modeling/attribute_renderer.rb
|
@@ -288,6 +297,7 @@ files:
|
|
288
297
|
- lib/burner/step.rb
|
289
298
|
- lib/burner/util.rb
|
290
299
|
- lib/burner/util/arrayable.rb
|
300
|
+
- lib/burner/util/keyable.rb
|
291
301
|
- lib/burner/util/string_template.rb
|
292
302
|
- lib/burner/version.rb
|
293
303
|
homepage: https://github.com/bluemarblepayroll/burner
|
@@ -299,7 +309,7 @@ metadata:
|
|
299
309
|
documentation_uri: https://www.rubydoc.info/gems/burner
|
300
310
|
homepage_uri: https://github.com/bluemarblepayroll/burner
|
301
311
|
source_code_uri: https://github.com/bluemarblepayroll/burner
|
302
|
-
post_install_message:
|
312
|
+
post_install_message:
|
303
313
|
rdoc_options: []
|
304
314
|
require_paths:
|
305
315
|
- lib
|
@@ -315,7 +325,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
315
325
|
version: '0'
|
316
326
|
requirements: []
|
317
327
|
rubygems_version: 3.0.3
|
318
|
-
signing_key:
|
328
|
+
signing_key:
|
319
329
|
specification_version: 4
|
320
330
|
summary: Declarative and extendable processing pipeline
|
321
331
|
test_files: []
|