burner 1.0.0.pre.alpha.2 → 1.0.0.pre.alpha.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -0
- data/README.md +57 -25
- data/burner.gemspec +3 -0
- data/exe/burner +2 -3
- data/lib/burner.rb +11 -0
- data/lib/burner/cli.rb +11 -9
- data/lib/burner/job.rb +29 -9
- data/lib/burner/job_with_register.rb +24 -0
- data/lib/burner/jobs.rb +21 -23
- data/lib/burner/library.rb +32 -0
- data/lib/burner/library/collection/arrays_to_objects.rb +75 -0
- data/lib/burner/library/collection/graph.rb +42 -0
- data/lib/burner/library/collection/objects_to_arrays.rb +88 -0
- data/lib/burner/library/collection/shift.rb +42 -0
- data/lib/burner/library/collection/transform.rb +66 -0
- data/lib/burner/library/collection/unpivot.rb +53 -0
- data/lib/burner/library/collection/validate.rb +89 -0
- data/lib/burner/library/collection/values.rb +49 -0
- data/lib/burner/library/deserialize/csv.rb +27 -0
- data/lib/burner/{jobs → library}/deserialize/json.rb +7 -6
- data/lib/burner/{jobs → library}/deserialize/yaml.rb +14 -8
- data/lib/burner/{jobs → library}/dummy.rb +4 -4
- data/lib/burner/{jobs → library}/echo.rb +5 -5
- data/lib/burner/{jobs → library}/io/base.rb +4 -10
- data/lib/burner/{jobs → library}/io/exist.rb +13 -11
- data/lib/burner/{jobs → library}/io/read.rb +9 -8
- data/lib/burner/{jobs → library}/io/write.rb +11 -8
- data/lib/burner/library/serialize/csv.rb +37 -0
- data/lib/burner/{jobs → library}/serialize/json.rb +7 -6
- data/lib/burner/{jobs → library}/serialize/yaml.rb +7 -6
- data/lib/burner/{jobs/set.rb → library/set_value.rb} +9 -8
- data/lib/burner/{jobs → library}/sleep.rb +4 -4
- data/lib/burner/modeling.rb +13 -0
- data/lib/burner/modeling/attribute.rb +29 -0
- data/lib/burner/modeling/attribute_renderer.rb +32 -0
- data/lib/burner/modeling/key_index_mapping.rb +29 -0
- data/lib/burner/modeling/validations.rb +23 -0
- data/lib/burner/modeling/validations/base.rb +35 -0
- data/lib/burner/modeling/validations/blank.rb +31 -0
- data/lib/burner/modeling/validations/present.rb +31 -0
- data/lib/burner/payload.rb +55 -10
- data/lib/burner/pipeline.rb +25 -6
- data/lib/burner/side_effects.rb +10 -0
- data/lib/burner/side_effects/written_file.rb +28 -0
- data/lib/burner/step.rb +2 -8
- data/lib/burner/util.rb +11 -0
- data/lib/burner/util/arrayable.rb +30 -0
- data/lib/burner/util/string_template.rb +42 -0
- data/lib/burner/version.rb +1 -1
- metadata +81 -16
- data/lib/burner/string_template.rb +0 -40
- data/lib/burner/written_file.rb +0 -28
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
# Composed of an Attribute instance and a Pipeline instance. It knows how to
|
13
|
+
# render/transform an Attribute. Since this library is data-first, these intermediary
|
14
|
+
# objects are necessary for non-data-centric modeling.
|
15
|
+
class AttributeRenderer
|
16
|
+
extend Forwardable
|
17
|
+
|
18
|
+
attr_reader :attribute, :pipeline
|
19
|
+
|
20
|
+
def_delegators :attribute, :key
|
21
|
+
|
22
|
+
def_delegators :pipeline, :transform
|
23
|
+
|
24
|
+
def initialize(attribute, resolver)
|
25
|
+
@attribute = attribute
|
26
|
+
@pipeline = Realize::Pipeline.new(attribute.transformers, resolver: resolver)
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
# Generic relationship between a numeric index and a key.
|
13
|
+
class KeyIndexMapping
|
14
|
+
acts_as_hashable
|
15
|
+
|
16
|
+
attr_reader :index, :key
|
17
|
+
|
18
|
+
def initialize(index:, key:)
|
19
|
+
raise ArgumentError, 'index is required' if index.to_s.empty?
|
20
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
21
|
+
|
22
|
+
@index = index.to_i
|
23
|
+
@key = key.to_s
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'validations/blank'
|
11
|
+
require_relative 'validations/present'
|
12
|
+
|
13
|
+
module Burner
|
14
|
+
module Modeling
|
15
|
+
# Factory for building sub-classes that can validate an individual object and field value.
|
16
|
+
class Validations
|
17
|
+
acts_as_hashable_factory
|
18
|
+
|
19
|
+
register 'blank', Validations::Blank
|
20
|
+
register 'present', Validations::Present
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
class Validations
|
13
|
+
# Common logic shared among all Validation subclasses.
|
14
|
+
# This class is an abstract class, make sure to implement:
|
15
|
+
# - #valid?(object, resolver)
|
16
|
+
# - #default_message
|
17
|
+
class Base
|
18
|
+
acts_as_hashable
|
19
|
+
|
20
|
+
attr_reader :key
|
21
|
+
|
22
|
+
def initialize(key:, message: '')
|
23
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
24
|
+
|
25
|
+
@key = key.to_s
|
26
|
+
@message = message.to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
def message
|
30
|
+
@message.to_s.empty? ? "#{key}#{default_message}" : @message.to_s
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
module Modeling
|
14
|
+
class Validations
|
15
|
+
# Check if a value is blank, if it is not blank then it is not valid.
|
16
|
+
class Blank < Base
|
17
|
+
acts_as_hashable
|
18
|
+
|
19
|
+
def valid?(object, resolver)
|
20
|
+
resolver.get(object, key).to_s.empty?
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def default_message
|
26
|
+
' must be blank'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
module Modeling
|
14
|
+
class Validations
|
15
|
+
# Check if a value is present. If it is blank (null or empty) then it is invalid.
|
16
|
+
class Present < Base
|
17
|
+
acts_as_hashable
|
18
|
+
|
19
|
+
def valid?(object_value, resolver)
|
20
|
+
!resolver.get(object_value, key).to_s.empty?
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def default_message
|
26
|
+
' is required'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/burner/payload.rb
CHANGED
@@ -7,27 +7,72 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
require_relative 'written_file'
|
11
|
-
|
12
10
|
module Burner
|
13
11
|
# The input for all Job#perform methods. The main notion of this object is its "value"
|
14
12
|
# attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
|
15
13
|
# methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
|
16
14
|
# library and lifecycle, but I am not sure there is any other way around it: trying to build
|
17
15
|
# a generic, open-ended object pipeline to serve almost any use case.
|
16
|
+
#
|
17
|
+
# The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
|
18
|
+
# structured/additive manner. The initial use case for this was for Burner's core IO jobs to
|
19
|
+
# report back the files it has written in a more structured data way (as opposed to simply
|
20
|
+
# writing some information to the output.)
|
18
21
|
class Payload
|
19
|
-
|
22
|
+
attr_reader :params,
|
23
|
+
:registers,
|
24
|
+
:side_effects,
|
25
|
+
:time
|
26
|
+
|
27
|
+
def initialize(
|
28
|
+
params: {},
|
29
|
+
registers: {},
|
30
|
+
side_effects: [],
|
31
|
+
time: Time.now.utc
|
32
|
+
)
|
33
|
+
@params = params || {}
|
34
|
+
@registers = {}
|
35
|
+
@side_effects = side_effects || []
|
36
|
+
@time = time || Time.now.utc
|
37
|
+
|
38
|
+
add_registers(registers)
|
39
|
+
end
|
40
|
+
|
41
|
+
# Add a side effect of a job. This helps to keep track of things jobs do outside of its
|
42
|
+
# register mutations.
|
43
|
+
def add_side_effect(side_effect)
|
44
|
+
tap { side_effects << side_effect }
|
45
|
+
end
|
46
|
+
|
47
|
+
# Set a register's value.
|
48
|
+
def []=(key, value)
|
49
|
+
set(key, value)
|
50
|
+
end
|
51
|
+
|
52
|
+
# Retrieve a register's value.
|
53
|
+
def [](key)
|
54
|
+
registers[key.to_s]
|
55
|
+
end
|
56
|
+
|
57
|
+
# Set halt_pipeline to true. This will indicate to the pipeline to stop all
|
58
|
+
# subsequent processing.
|
59
|
+
def halt_pipeline
|
60
|
+
@halt_pipeline = true
|
61
|
+
end
|
62
|
+
|
63
|
+
# Check and see if halt_pipeline was called.
|
64
|
+
def halt_pipeline?
|
65
|
+
@halt_pipeline || false
|
66
|
+
end
|
20
67
|
|
21
|
-
|
68
|
+
private
|
22
69
|
|
23
|
-
def
|
24
|
-
|
25
|
-
@value = value
|
26
|
-
@written_files = written_files || []
|
70
|
+
def set(key, value)
|
71
|
+
registers[key.to_s] = value
|
27
72
|
end
|
28
73
|
|
29
|
-
def
|
30
|
-
|
74
|
+
def add_registers(registers)
|
75
|
+
(registers || {}).each { |k, v| set(k, v) }
|
31
76
|
end
|
32
77
|
end
|
33
78
|
end
|
data/lib/burner/pipeline.rb
CHANGED
@@ -19,11 +19,16 @@ module Burner
|
|
19
19
|
acts_as_hashable
|
20
20
|
|
21
21
|
class JobNotFoundError < StandardError; end
|
22
|
+
class DuplicateJobNameError < StandardError; end
|
22
23
|
|
23
24
|
attr_reader :steps
|
24
25
|
|
25
26
|
def initialize(jobs: [], steps: [])
|
26
|
-
|
27
|
+
jobs = Jobs.array(jobs)
|
28
|
+
|
29
|
+
assert_unique_job_names(jobs)
|
30
|
+
|
31
|
+
jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
|
27
32
|
|
28
33
|
@steps = Array(steps).map do |step_name|
|
29
34
|
job = jobs_by_name[step_name.to_s]
|
@@ -35,18 +40,18 @@ module Burner
|
|
35
40
|
end
|
36
41
|
|
37
42
|
# The main entry-point for kicking off a pipeline.
|
38
|
-
def execute(output: Output.new,
|
43
|
+
def execute(output: Output.new, payload: Payload.new)
|
39
44
|
output.write("Pipeline started with #{steps.length} step(s)")
|
40
45
|
|
41
|
-
output_params(params, output)
|
46
|
+
output_params(payload.params, output)
|
42
47
|
output.ruler
|
43
48
|
|
44
49
|
time_in_seconds = Benchmark.measure do
|
45
50
|
steps.each do |step|
|
46
|
-
|
51
|
+
step.perform(output, payload)
|
47
52
|
|
48
|
-
if
|
49
|
-
output.detail('
|
53
|
+
if payload.halt_pipeline?
|
54
|
+
output.detail('Payload was halted, ending pipeline.')
|
50
55
|
break
|
51
56
|
end
|
52
57
|
end
|
@@ -60,6 +65,20 @@ module Burner
|
|
60
65
|
|
61
66
|
private
|
62
67
|
|
68
|
+
def assert_unique_job_names(jobs)
|
69
|
+
unique_job_names = Set.new
|
70
|
+
|
71
|
+
jobs.each do |job|
|
72
|
+
if unique_job_names.include?(job.name)
|
73
|
+
raise DuplicateJobNameError, "job with name: #{job.name} already declared"
|
74
|
+
end
|
75
|
+
|
76
|
+
unique_job_names << job.name
|
77
|
+
end
|
78
|
+
|
79
|
+
nil
|
80
|
+
end
|
81
|
+
|
63
82
|
def output_params(params, output)
|
64
83
|
if params.keys.any?
|
65
84
|
output.write('Parameters:')
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'side_effects/written_file'
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module SideEffects
|
12
|
+
# Describes a file that was generated by a Job. If a Job emits a file, it should also add the
|
13
|
+
# file details to the Payload#side_effects array using the Payload#add_side_effect method.
|
14
|
+
class WrittenFile
|
15
|
+
attr_reader :logical_filename,
|
16
|
+
:physical_filename,
|
17
|
+
:time_in_seconds
|
18
|
+
|
19
|
+
def initialize(logical_filename:, physical_filename:, time_in_seconds:)
|
20
|
+
@logical_filename = logical_filename.to_s
|
21
|
+
@physical_filename = physical_filename.to_s
|
22
|
+
@time_in_seconds = time_in_seconds.to_f
|
23
|
+
|
24
|
+
freeze
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/burner/step.rb
CHANGED
@@ -28,20 +28,14 @@ module Burner
|
|
28
28
|
freeze
|
29
29
|
end
|
30
30
|
|
31
|
-
def perform(output, payload
|
32
|
-
return_value = nil
|
33
|
-
|
31
|
+
def perform(output, payload)
|
34
32
|
output.title("#{job.class.name}#{SEPARATOR}#{job.name}")
|
35
33
|
|
36
34
|
time_in_seconds = Benchmark.measure do
|
37
|
-
|
38
|
-
|
39
|
-
return_value = job.perform(output, payload, job_params)
|
35
|
+
job.perform(output, payload)
|
40
36
|
end.real.round(3)
|
41
37
|
|
42
38
|
output.complete(time_in_seconds)
|
43
|
-
|
44
|
-
return_value
|
45
39
|
end
|
46
40
|
end
|
47
41
|
end
|
data/lib/burner/util.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'util/arrayable'
|
11
|
+
require_relative 'util/string_template'
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Util
|
12
|
+
# Provide helper methods for dealing with Arrays.
|
13
|
+
module Arrayable
|
14
|
+
# Since Ruby's Kernel#Array will properly call #to_a for scalar Hash objects, this could
|
15
|
+
# return something funky in the context of this library. In this library, Hash instances
|
16
|
+
# are typically viewed as an atomic key-value-based "object". This library likes to deal
|
17
|
+
# with object-like things, treating Hash, OpenStruct, Struct, or Object subclasses as
|
18
|
+
# basically the same thing. In this vein, this library leverages Objectable to help
|
19
|
+
# unify access data from objects. See the Objectable library for more information:
|
20
|
+
# https://github.com/bluemarblepayroll/objectable
|
21
|
+
def array(value)
|
22
|
+
if value.is_a?(Hash)
|
23
|
+
[value]
|
24
|
+
else
|
25
|
+
Array(value)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Util
|
12
|
+
# Can take in a string and an object and use the object for formatting string interpolations
|
13
|
+
# using tokens of form: {attribute_name}. This templating class does not understand nested
|
14
|
+
# structures, so input should be a flat object/hash in the form of key-value pairs.
|
15
|
+
# A benefit of using Objectable for resolution is that it can understand almost any type of
|
16
|
+
# object: Hash, Struct, OpenStruct, custom objects, etc.
|
17
|
+
# For more information see underlying libraries:
|
18
|
+
# * Stringento: https://github.com/bluemarblepayroll/stringento
|
19
|
+
# * Objectable: https://github.com/bluemarblepayroll/objectable
|
20
|
+
class StringTemplate
|
21
|
+
include Singleton
|
22
|
+
|
23
|
+
attr_reader :resolver
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
@resolver = Objectable.resolver(separator: '')
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
# For general consumption
|
32
|
+
def evaluate(expression, input)
|
33
|
+
Stringento.evaluate(expression, input, resolver: self)
|
34
|
+
end
|
35
|
+
|
36
|
+
# For Stringento consumption
|
37
|
+
def resolve(value, input)
|
38
|
+
resolver.get(input, value)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|