burner 1.5.0 → 1.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/CHANGELOG.md +36 -0
- data/README.md +59 -3
- data/lib/burner/data.rb +46 -0
- data/lib/burner/job.rb +2 -10
- data/lib/burner/job_set.rb +64 -0
- data/lib/burner/job_with_register.rb +8 -1
- data/lib/burner/jobs.rb +6 -0
- data/lib/burner/library.rb +6 -0
- data/lib/burner/library/collection/arrays_to_objects.rb +1 -1
- data/lib/burner/library/collection/coalesce.rb +14 -9
- data/lib/burner/library/collection/concatenate.rb +1 -1
- data/lib/burner/library/collection/graph.rb +1 -1
- data/lib/burner/library/collection/group.rb +15 -11
- data/lib/burner/library/collection/nested_aggregate.rb +1 -1
- data/lib/burner/library/collection/number.rb +51 -0
- data/lib/burner/library/collection/objects_to_arrays.rb +1 -1
- data/lib/burner/library/collection/shift.rb +1 -1
- data/lib/burner/library/collection/transform.rb +1 -1
- data/lib/burner/library/collection/unpivot.rb +1 -1
- data/lib/burner/library/collection/validate.rb +1 -1
- data/lib/burner/library/collection/values.rb +1 -1
- data/lib/burner/library/collection/zip.rb +1 -1
- data/lib/burner/library/compress/row_reader.rb +1 -1
- data/lib/burner/library/deserialize/yaml.rb +1 -1
- data/lib/burner/library/echo.rb +1 -1
- data/lib/burner/library/io/exist.rb +1 -1
- data/lib/burner/library/io/open_file_base.rb +1 -1
- data/lib/burner/library/io/row_reader.rb +1 -1
- data/lib/burner/library/io/write.rb +28 -1
- data/lib/burner/library/param/base.rb +29 -0
- data/lib/burner/library/param/from_register.rb +30 -0
- data/lib/burner/library/param/to_register.rb +28 -0
- data/lib/burner/library/serialize/csv.rb +1 -1
- data/lib/burner/library/sleep.rb +1 -1
- data/lib/burner/library/value/copy.rb +1 -1
- data/lib/burner/library/value/nest.rb +37 -0
- data/lib/burner/library/value/static.rb +1 -1
- data/lib/burner/library/value/transform.rb +38 -0
- data/lib/burner/payload.rb +39 -15
- data/lib/burner/pipeline.rb +6 -34
- data/lib/burner/util.rb +1 -0
- data/lib/burner/util/keyable.rb +23 -0
- data/lib/burner/version.rb +1 -1
- metadata +15 -5
@@ -17,7 +17,7 @@ module Burner
|
|
17
17
|
class Static < JobWithRegister
|
18
18
|
attr_reader :value
|
19
19
|
|
20
|
-
def initialize(name
|
20
|
+
def initialize(name: '', register: DEFAULT_REGISTER, value: nil)
|
21
21
|
super(name: name, register: register)
|
22
22
|
|
23
23
|
@value = value
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Library
|
12
|
+
module Value
|
13
|
+
# Transform the current value of the register through a Realize::Pipeline. This will
|
14
|
+
# transform the entire value, as opposed to the b/collection/transform job, which will
|
15
|
+
# iterate over each row/record in a dataset and transform each row/record.
|
16
|
+
#
|
17
|
+
# Expected Payload[register] input: anything.
|
18
|
+
# Payload[register] output: anything.
|
19
|
+
class Transform < JobWithRegister
|
20
|
+
attr_reader :pipeline
|
21
|
+
|
22
|
+
def initialize(name: '', register: DEFAULT_REGISTER, separator: '', transformers: [])
|
23
|
+
super(name: name, register: register)
|
24
|
+
|
25
|
+
resolver = Objectable.resolver(separator: separator)
|
26
|
+
|
27
|
+
@pipeline = Realize::Pipeline.new(transformers, resolver: resolver)
|
28
|
+
|
29
|
+
freeze
|
30
|
+
end
|
31
|
+
|
32
|
+
def perform(_output, payload)
|
33
|
+
payload[register] = pipeline.transform(payload[register], payload.time)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/burner/payload.rb
CHANGED
@@ -7,6 +7,8 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
+
require_relative 'data'
|
11
|
+
|
10
12
|
module Burner
|
11
13
|
# The input for all Job#perform methods. The main notion of this object is its 'registers'
|
12
14
|
# attribute. This registers attribute is a key-indifferent hash, accessible on Payload using
|
@@ -24,10 +26,15 @@ module Burner
|
|
24
26
|
# The 'time' attribute is important in that it should for the replaying of pipelines and jobs.
|
25
27
|
# Instead of having job's utilizing Time.now, Date.today, etc... they should rather opt to
|
26
28
|
# use this value instead.
|
29
|
+
#
|
30
|
+
# The notion of params are somewhat conflated with registers here. Both are hashes and both
|
31
|
+
# serve as data stores. The difference is registers are really meant to be the shared-data
|
32
|
+
# repository across jobs, while params are, more or less, the input into the _pipeline_.
|
33
|
+
# It is debatable if mutation of the params should be allowed but the design decision was made
|
34
|
+
# early on to allow for params to be mutable albeit with registers being the preferred mutable
|
35
|
+
# store.
|
27
36
|
class Payload
|
28
|
-
attr_reader :
|
29
|
-
:registers,
|
30
|
-
:side_effects,
|
37
|
+
attr_reader :side_effects,
|
31
38
|
:time
|
32
39
|
|
33
40
|
def initialize(
|
@@ -36,12 +43,32 @@ module Burner
|
|
36
43
|
side_effects: [],
|
37
44
|
time: Time.now.utc
|
38
45
|
)
|
39
|
-
@params = params
|
40
|
-
@registers =
|
46
|
+
@params = Data.new(params)
|
47
|
+
@registers = Data.new(registers)
|
41
48
|
@side_effects = side_effects || []
|
42
49
|
@time = time || Time.now.utc
|
50
|
+
end
|
43
51
|
|
44
|
-
|
52
|
+
# Backwards compatibility. This allows for control over the underlying data structure
|
53
|
+
# while still maintaining the same public API as before.
|
54
|
+
def params
|
55
|
+
@params.to_h
|
56
|
+
end
|
57
|
+
|
58
|
+
# Backwards compatibility. This allows for control over the underlying data structure
|
59
|
+
# while still maintaining the same public API as before.
|
60
|
+
def registers
|
61
|
+
@registers.to_h
|
62
|
+
end
|
63
|
+
|
64
|
+
# Law of Demeter: While params is an accessible hash, it is preferred that the
|
65
|
+
# public class methods are used.
|
66
|
+
def param(key)
|
67
|
+
@params[key]
|
68
|
+
end
|
69
|
+
|
70
|
+
def update_param(key, value)
|
71
|
+
tap { @params[key] = value }
|
45
72
|
end
|
46
73
|
|
47
74
|
# Add a side effect of a job. This helps to keep track of things jobs do outside of its
|
@@ -52,12 +79,12 @@ module Burner
|
|
52
79
|
|
53
80
|
# Set a register's value.
|
54
81
|
def []=(key, value)
|
55
|
-
|
82
|
+
@registers[key] = value
|
56
83
|
end
|
57
84
|
|
58
85
|
# Retrieve a register's value.
|
59
86
|
def [](key)
|
60
|
-
registers[key
|
87
|
+
@registers[key]
|
61
88
|
end
|
62
89
|
|
63
90
|
# Set halt_pipeline to true. This will indicate to the pipeline to stop all
|
@@ -71,14 +98,11 @@ module Burner
|
|
71
98
|
@halt_pipeline || false
|
72
99
|
end
|
73
100
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
registers[key.to_s] = value
|
78
|
-
end
|
101
|
+
def params_and_registers_hash
|
102
|
+
registers_hash = @registers.transform_keys { |key| "__#{key}_register" }
|
103
|
+
params_hash = @params.to_h
|
79
104
|
|
80
|
-
|
81
|
-
(registers || {}).each { |k, v| set(k, v) }
|
105
|
+
params_hash.merge(registers_hash)
|
82
106
|
end
|
83
107
|
end
|
84
108
|
end
|
data/lib/burner/pipeline.rb
CHANGED
@@ -7,7 +7,7 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
require_relative '
|
10
|
+
require_relative 'job_set'
|
11
11
|
require_relative 'output'
|
12
12
|
require_relative 'payload'
|
13
13
|
require_relative 'step'
|
@@ -19,27 +19,13 @@ module Burner
|
|
19
19
|
class Pipeline
|
20
20
|
acts_as_hashable
|
21
21
|
|
22
|
-
class JobNotFoundError < StandardError; end
|
23
|
-
class DuplicateJobNameError < StandardError; end
|
24
|
-
|
25
22
|
attr_reader :steps
|
26
23
|
|
27
24
|
def initialize(jobs: [], steps: nil)
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
|
33
|
-
|
34
|
-
step_names = steps ? Array(steps) : jobs_by_name.keys
|
35
|
-
|
36
|
-
@steps = step_names.map do |step_name|
|
37
|
-
job = jobs_by_name[step_name.to_s]
|
38
|
-
|
39
|
-
raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
|
40
|
-
|
41
|
-
Step.new(job)
|
42
|
-
end
|
25
|
+
@steps = JobSet
|
26
|
+
.new(jobs)
|
27
|
+
.jobs(steps)
|
28
|
+
.map { |job| Step.new(job) }
|
43
29
|
end
|
44
30
|
|
45
31
|
# The main entry-point for kicking off a pipeline.
|
@@ -68,22 +54,8 @@ module Burner
|
|
68
54
|
|
69
55
|
private
|
70
56
|
|
71
|
-
def assert_unique_job_names(jobs)
|
72
|
-
unique_job_names = Set.new
|
73
|
-
|
74
|
-
jobs.each do |job|
|
75
|
-
if unique_job_names.include?(job.name)
|
76
|
-
raise DuplicateJobNameError, "job with name: #{job.name} already declared"
|
77
|
-
end
|
78
|
-
|
79
|
-
unique_job_names << job.name
|
80
|
-
end
|
81
|
-
|
82
|
-
nil
|
83
|
-
end
|
84
|
-
|
85
57
|
def output_params(params, output)
|
86
|
-
if params.
|
58
|
+
if params.any?
|
87
59
|
output.write('Parameters:')
|
88
60
|
else
|
89
61
|
output.write('No parameters passed in.')
|
data/lib/burner/util.rb
CHANGED
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Util
|
12
|
+
# Provide helper methods for keys.
|
13
|
+
module Keyable
|
14
|
+
def make_key(record, keys, resolver, insensitive)
|
15
|
+
keys.map do |key|
|
16
|
+
value = resolver.get(record, key)
|
17
|
+
|
18
|
+
insensitive ? value.to_s.downcase : value
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/burner/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: burner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: acts_as_hashable
|
@@ -220,6 +220,7 @@ files:
|
|
220
220
|
- ".gitignore"
|
221
221
|
- ".rubocop.yml"
|
222
222
|
- ".ruby-version"
|
223
|
+
- ".tool-versions"
|
223
224
|
- ".travis.yml"
|
224
225
|
- CHANGELOG.md
|
225
226
|
- CODE_OF_CONDUCT.md
|
@@ -234,9 +235,11 @@ files:
|
|
234
235
|
- exe/burner
|
235
236
|
- lib/burner.rb
|
236
237
|
- lib/burner/cli.rb
|
238
|
+
- lib/burner/data.rb
|
237
239
|
- lib/burner/disks.rb
|
238
240
|
- lib/burner/disks/local.rb
|
239
241
|
- lib/burner/job.rb
|
242
|
+
- lib/burner/job_set.rb
|
240
243
|
- lib/burner/job_with_register.rb
|
241
244
|
- lib/burner/jobs.rb
|
242
245
|
- lib/burner/library.rb
|
@@ -246,6 +249,7 @@ files:
|
|
246
249
|
- lib/burner/library/collection/graph.rb
|
247
250
|
- lib/burner/library/collection/group.rb
|
248
251
|
- lib/burner/library/collection/nested_aggregate.rb
|
252
|
+
- lib/burner/library/collection/number.rb
|
249
253
|
- lib/burner/library/collection/objects_to_arrays.rb
|
250
254
|
- lib/burner/library/collection/shift.rb
|
251
255
|
- lib/burner/library/collection/transform.rb
|
@@ -264,12 +268,17 @@ files:
|
|
264
268
|
- lib/burner/library/io/row_reader.rb
|
265
269
|
- lib/burner/library/io/write.rb
|
266
270
|
- lib/burner/library/nothing.rb
|
271
|
+
- lib/burner/library/param/base.rb
|
272
|
+
- lib/burner/library/param/from_register.rb
|
273
|
+
- lib/burner/library/param/to_register.rb
|
267
274
|
- lib/burner/library/serialize/csv.rb
|
268
275
|
- lib/burner/library/serialize/json.rb
|
269
276
|
- lib/burner/library/serialize/yaml.rb
|
270
277
|
- lib/burner/library/sleep.rb
|
271
278
|
- lib/burner/library/value/copy.rb
|
279
|
+
- lib/burner/library/value/nest.rb
|
272
280
|
- lib/burner/library/value/static.rb
|
281
|
+
- lib/burner/library/value/transform.rb
|
273
282
|
- lib/burner/modeling.rb
|
274
283
|
- lib/burner/modeling/attribute.rb
|
275
284
|
- lib/burner/modeling/attribute_renderer.rb
|
@@ -288,6 +297,7 @@ files:
|
|
288
297
|
- lib/burner/step.rb
|
289
298
|
- lib/burner/util.rb
|
290
299
|
- lib/burner/util/arrayable.rb
|
300
|
+
- lib/burner/util/keyable.rb
|
291
301
|
- lib/burner/util/string_template.rb
|
292
302
|
- lib/burner/version.rb
|
293
303
|
homepage: https://github.com/bluemarblepayroll/burner
|
@@ -299,7 +309,7 @@ metadata:
|
|
299
309
|
documentation_uri: https://www.rubydoc.info/gems/burner
|
300
310
|
homepage_uri: https://github.com/bluemarblepayroll/burner
|
301
311
|
source_code_uri: https://github.com/bluemarblepayroll/burner
|
302
|
-
post_install_message:
|
312
|
+
post_install_message:
|
303
313
|
rdoc_options: []
|
304
314
|
require_paths:
|
305
315
|
- lib
|
@@ -315,7 +325,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
315
325
|
version: '0'
|
316
326
|
requirements: []
|
317
327
|
rubygems_version: 3.0.3
|
318
|
-
signing_key:
|
328
|
+
signing_key:
|
319
329
|
specification_version: 4
|
320
330
|
summary: Declarative and extendable processing pipeline
|
321
331
|
test_files: []
|