burner 1.0.0.pre.alpha.5 → 1.0.0.pre.alpha.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -0
- data/README.md +52 -48
- data/burner.gemspec +1 -1
- data/exe/burner +2 -3
- data/lib/burner.rb +2 -0
- data/lib/burner/cli.rb +2 -0
- data/lib/burner/job.rb +27 -9
- data/lib/burner/job_with_register.rb +24 -0
- data/lib/burner/jobs.rb +28 -41
- data/lib/burner/library.rb +32 -0
- data/lib/burner/library/collection/arrays_to_objects.rb +75 -0
- data/lib/burner/{jobs → library}/collection/graph.rb +7 -8
- data/lib/burner/library/collection/objects_to_arrays.rb +88 -0
- data/lib/burner/{jobs → library}/collection/shift.rb +8 -9
- data/lib/burner/{jobs → library}/collection/transform.rb +17 -15
- data/lib/burner/{jobs → library}/collection/unpivot.rb +17 -9
- data/lib/burner/library/collection/validate.rb +89 -0
- data/lib/burner/{jobs → library}/collection/values.rb +9 -10
- data/lib/burner/{jobs → library}/deserialize/csv.rb +4 -5
- data/lib/burner/{jobs → library}/deserialize/json.rb +6 -5
- data/lib/burner/{jobs → library}/deserialize/yaml.rb +13 -7
- data/lib/burner/{jobs → library}/dummy.rb +4 -4
- data/lib/burner/{jobs → library}/echo.rb +3 -3
- data/lib/burner/{jobs → library}/io/base.rb +4 -4
- data/lib/burner/{jobs → library}/io/exist.rb +11 -9
- data/lib/burner/{jobs → library}/io/read.rb +7 -6
- data/lib/burner/{jobs → library}/io/write.rb +9 -6
- data/lib/burner/{jobs → library}/serialize/csv.rb +5 -6
- data/lib/burner/{jobs → library}/serialize/json.rb +6 -5
- data/lib/burner/{jobs → library}/serialize/yaml.rb +6 -5
- data/lib/burner/{jobs/set.rb → library/set_value.rb} +8 -7
- data/lib/burner/{jobs → library}/sleep.rb +3 -3
- data/lib/burner/modeling.rb +3 -0
- data/lib/burner/modeling/attribute.rb +29 -0
- data/lib/burner/modeling/attribute_renderer.rb +32 -0
- data/lib/burner/modeling/validations.rb +23 -0
- data/lib/burner/modeling/validations/base.rb +35 -0
- data/lib/burner/modeling/validations/blank.rb +31 -0
- data/lib/burner/modeling/validations/present.rb +31 -0
- data/lib/burner/payload.rb +52 -15
- data/lib/burner/pipeline.rb +23 -4
- data/lib/burner/side_effects.rb +10 -0
- data/lib/burner/side_effects/written_file.rb +28 -0
- data/lib/burner/step.rb +1 -5
- data/lib/burner/util.rb +11 -0
- data/lib/burner/util/arrayable.rb +30 -0
- data/lib/burner/util/string_template.rb +42 -0
- data/lib/burner/version.rb +1 -1
- metadata +40 -29
- data/lib/burner/jobs/collection/arrays_to_objects.rb +0 -43
- data/lib/burner/jobs/collection/objects_to_arrays.rb +0 -54
- data/lib/burner/jobs/collection/transform/attribute.rb +0 -33
- data/lib/burner/jobs/collection/transform/attribute_renderer.rb +0 -36
- data/lib/burner/string_template.rb +0 -40
- data/lib/burner/written_file.rb +0 -28
data/lib/burner/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: burner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.pre.alpha.
|
4
|
+
version: 1.0.0.pre.alpha.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-10-
|
11
|
+
date: 2020-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: acts_as_hashable
|
@@ -192,8 +192,9 @@ dependencies:
|
|
192
192
|
- - "~>"
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: 0.7.0
|
195
|
-
description: " This library serves as the
|
196
|
-
allows you to organize your code into
|
195
|
+
description: " This library serves as the backbone for a configurable processing
|
196
|
+
engine. It allows you to organize your code into jobs, then stitch those jobs together
|
197
|
+
as steps.\n"
|
197
198
|
email:
|
198
199
|
- mruggio@bluemarblepayroll.com
|
199
200
|
executables:
|
@@ -220,39 +221,49 @@ files:
|
|
220
221
|
- lib/burner.rb
|
221
222
|
- lib/burner/cli.rb
|
222
223
|
- lib/burner/job.rb
|
224
|
+
- lib/burner/job_with_register.rb
|
223
225
|
- lib/burner/jobs.rb
|
224
|
-
- lib/burner/
|
225
|
-
- lib/burner/
|
226
|
-
- lib/burner/
|
227
|
-
- lib/burner/
|
228
|
-
- lib/burner/
|
229
|
-
- lib/burner/
|
230
|
-
- lib/burner/
|
231
|
-
- lib/burner/
|
232
|
-
- lib/burner/
|
233
|
-
- lib/burner/
|
234
|
-
- lib/burner/
|
235
|
-
- lib/burner/
|
236
|
-
- lib/burner/
|
237
|
-
- lib/burner/
|
238
|
-
- lib/burner/
|
239
|
-
- lib/burner/
|
240
|
-
- lib/burner/
|
241
|
-
- lib/burner/
|
242
|
-
- lib/burner/
|
243
|
-
- lib/burner/
|
244
|
-
- lib/burner/
|
245
|
-
- lib/burner/
|
246
|
-
- lib/burner/
|
226
|
+
- lib/burner/library.rb
|
227
|
+
- lib/burner/library/collection/arrays_to_objects.rb
|
228
|
+
- lib/burner/library/collection/graph.rb
|
229
|
+
- lib/burner/library/collection/objects_to_arrays.rb
|
230
|
+
- lib/burner/library/collection/shift.rb
|
231
|
+
- lib/burner/library/collection/transform.rb
|
232
|
+
- lib/burner/library/collection/unpivot.rb
|
233
|
+
- lib/burner/library/collection/validate.rb
|
234
|
+
- lib/burner/library/collection/values.rb
|
235
|
+
- lib/burner/library/deserialize/csv.rb
|
236
|
+
- lib/burner/library/deserialize/json.rb
|
237
|
+
- lib/burner/library/deserialize/yaml.rb
|
238
|
+
- lib/burner/library/dummy.rb
|
239
|
+
- lib/burner/library/echo.rb
|
240
|
+
- lib/burner/library/io/base.rb
|
241
|
+
- lib/burner/library/io/exist.rb
|
242
|
+
- lib/burner/library/io/read.rb
|
243
|
+
- lib/burner/library/io/write.rb
|
244
|
+
- lib/burner/library/serialize/csv.rb
|
245
|
+
- lib/burner/library/serialize/json.rb
|
246
|
+
- lib/burner/library/serialize/yaml.rb
|
247
|
+
- lib/burner/library/set_value.rb
|
248
|
+
- lib/burner/library/sleep.rb
|
247
249
|
- lib/burner/modeling.rb
|
250
|
+
- lib/burner/modeling/attribute.rb
|
251
|
+
- lib/burner/modeling/attribute_renderer.rb
|
248
252
|
- lib/burner/modeling/key_index_mapping.rb
|
253
|
+
- lib/burner/modeling/validations.rb
|
254
|
+
- lib/burner/modeling/validations/base.rb
|
255
|
+
- lib/burner/modeling/validations/blank.rb
|
256
|
+
- lib/burner/modeling/validations/present.rb
|
249
257
|
- lib/burner/output.rb
|
250
258
|
- lib/burner/payload.rb
|
251
259
|
- lib/burner/pipeline.rb
|
260
|
+
- lib/burner/side_effects.rb
|
261
|
+
- lib/burner/side_effects/written_file.rb
|
252
262
|
- lib/burner/step.rb
|
253
|
-
- lib/burner/
|
263
|
+
- lib/burner/util.rb
|
264
|
+
- lib/burner/util/arrayable.rb
|
265
|
+
- lib/burner/util/string_template.rb
|
254
266
|
- lib/burner/version.rb
|
255
|
-
- lib/burner/written_file.rb
|
256
267
|
homepage: https://github.com/bluemarblepayroll/burner
|
257
268
|
licenses:
|
258
269
|
- MIT
|
@@ -1,43 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module Burner
|
11
|
-
class Jobs
|
12
|
-
module Collection
|
13
|
-
# Convert an array of arrays to an array of objects.
|
14
|
-
# Expected Payload#value input: array of arrays.
|
15
|
-
# Payload#value output: An array of hashes.
|
16
|
-
class ArraysToObjects < Job
|
17
|
-
attr_reader :mappings
|
18
|
-
|
19
|
-
def initialize(name:, mappings: [])
|
20
|
-
super(name: name)
|
21
|
-
|
22
|
-
@mappings = Modeling::KeyIndexMapping.array(mappings)
|
23
|
-
|
24
|
-
freeze
|
25
|
-
end
|
26
|
-
|
27
|
-
def perform(_output, payload)
|
28
|
-
payload.value = (payload.value || []).map { |array| index_to_key_map(array) }
|
29
|
-
|
30
|
-
nil
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
|
-
|
35
|
-
def index_to_key_map(array)
|
36
|
-
mappings.each_with_object({}) do |mapping, memo|
|
37
|
-
memo[mapping.key] = array[mapping.index]
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module Burner
|
11
|
-
class Jobs
|
12
|
-
module Collection
|
13
|
-
# Convert an array of objects to an array of arrays. You can leverage the separator
|
14
|
-
# option to support key paths and nested objects.
|
15
|
-
# Expected Payload#value input: array of hashes.
|
16
|
-
# Payload#value output: An array of arrays.
|
17
|
-
class ObjectsToArrays < Job
|
18
|
-
attr_reader :mappings
|
19
|
-
|
20
|
-
# If you wish to support nested objects you can pass in a string to use as a
|
21
|
-
# key path separator. For example: if you would like to recognize dot-notation for
|
22
|
-
# nested hashes then set separator to '.'.
|
23
|
-
def initialize(name:, mappings: [], separator: '')
|
24
|
-
super(name: name)
|
25
|
-
|
26
|
-
@mappings = Modeling::KeyIndexMapping.array(mappings)
|
27
|
-
@resolver = Objectable.resolver(separator: separator.to_s)
|
28
|
-
|
29
|
-
freeze
|
30
|
-
end
|
31
|
-
|
32
|
-
def perform(_output, payload)
|
33
|
-
payload.value = (payload.value || []).map { |object| key_to_index_map(object) }
|
34
|
-
|
35
|
-
nil
|
36
|
-
end
|
37
|
-
|
38
|
-
private
|
39
|
-
|
40
|
-
attr_reader :resolver
|
41
|
-
|
42
|
-
def key_to_index_map(object)
|
43
|
-
mappings.each_with_object(prototype_array) do |mapping, memo|
|
44
|
-
memo[mapping.index] = resolver.get(object, mapping.key)
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
def prototype_array
|
49
|
-
Array.new(mappings.length)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,33 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module Burner
|
11
|
-
class Jobs
|
12
|
-
module Collection
|
13
|
-
class Transform < Job
|
14
|
-
# Defines a top-level key and the associated transformers for deriving the final value
|
15
|
-
# to set the key to.
|
16
|
-
class Attribute
|
17
|
-
acts_as_hashable
|
18
|
-
|
19
|
-
attr_reader :key, :transformers
|
20
|
-
|
21
|
-
def initialize(key:, transformers: [])
|
22
|
-
raise ArgumentError, 'key is required' if key.to_s.empty?
|
23
|
-
|
24
|
-
@key = key.to_s
|
25
|
-
@transformers = Realize::Transformers.array(transformers)
|
26
|
-
|
27
|
-
freeze
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module Burner
|
11
|
-
class Jobs
|
12
|
-
module Collection
|
13
|
-
class Transform < Job
|
14
|
-
# Composed of an Attribute instance and a Pipeline instance. It knows how to
|
15
|
-
# render/transform an Attribute. Since this library is data-first, these intermediary
|
16
|
-
# objects are necessary for non-data-centric modeling.
|
17
|
-
class AttributeRenderer
|
18
|
-
extend Forwardable
|
19
|
-
|
20
|
-
attr_reader :attribute, :pipeline
|
21
|
-
|
22
|
-
def_delegators :attribute, :key
|
23
|
-
|
24
|
-
def_delegators :pipeline, :transform
|
25
|
-
|
26
|
-
def initialize(attribute, resolver)
|
27
|
-
@attribute = attribute
|
28
|
-
@pipeline = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
|
29
|
-
|
30
|
-
freeze
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
@@ -1,40 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module Burner
|
11
|
-
# Can take in a string and an object and use the object for formatting string interpolations
|
12
|
-
# using tokens of form: {attribute_name}. This templating class does not understand nested
|
13
|
-
# structures, so input should be a flat object/hash in the form of key-value pairs. A benefit of
|
14
|
-
# using Objectable for resolution is that it can understand almost any type of
|
15
|
-
# object: Hash, Struct, OpenStruct, custom objects, etc.
|
16
|
-
# For more information see underlying libraries:
|
17
|
-
# * Stringento: https://github.com/bluemarblepayroll/stringento
|
18
|
-
# * Objectable: https://github.com/bluemarblepayroll/objectable
|
19
|
-
class StringTemplate
|
20
|
-
include Singleton
|
21
|
-
|
22
|
-
attr_reader :resolver
|
23
|
-
|
24
|
-
def initialize
|
25
|
-
@resolver = Objectable.resolver(separator: '')
|
26
|
-
|
27
|
-
freeze
|
28
|
-
end
|
29
|
-
|
30
|
-
# For general consumption
|
31
|
-
def evaluate(expression, input)
|
32
|
-
Stringento.evaluate(expression, input, resolver: self)
|
33
|
-
end
|
34
|
-
|
35
|
-
# For Stringento consumption
|
36
|
-
def resolve(value, input)
|
37
|
-
resolver.get(input, value)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
data/lib/burner/written_file.rb
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
#
|
4
|
-
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
-
#
|
6
|
-
# This source code is licensed under the MIT license found in the
|
7
|
-
# LICENSE file in the root directory of this source tree.
|
8
|
-
#
|
9
|
-
|
10
|
-
module Burner
|
11
|
-
# Describes a file that was generated by a Job. If a Job emits a file, it should also add the
|
12
|
-
# file details to the Payload#written_files array using the Payload#add_written_file method.
|
13
|
-
class WrittenFile
|
14
|
-
acts_as_hashable
|
15
|
-
|
16
|
-
attr_reader :logical_filename,
|
17
|
-
:physical_filename,
|
18
|
-
:time_in_seconds
|
19
|
-
|
20
|
-
def initialize(logical_filename:, physical_filename:, time_in_seconds:)
|
21
|
-
@logical_filename = logical_filename.to_s
|
22
|
-
@physical_filename = physical_filename.to_s
|
23
|
-
@time_in_seconds = time_in_seconds.to_f
|
24
|
-
|
25
|
-
freeze
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|