burner 1.0.0.pre.alpha.6 → 1.0.0.pre.alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -0
- data/README.md +43 -39
- data/burner.gemspec +1 -1
- data/lib/burner/job.rb +15 -10
- data/lib/burner/job_with_register.rb +24 -0
- data/lib/burner/jobs.rb +27 -20
- data/lib/burner/library.rb +11 -5
- data/lib/burner/library/collection/arrays_to_objects.rb +14 -11
- data/lib/burner/library/collection/graph.rb +7 -9
- data/lib/burner/library/collection/objects_to_arrays.rb +34 -34
- data/lib/burner/library/collection/shift.rb +6 -8
- data/lib/burner/library/collection/transform.rb +7 -9
- data/lib/burner/library/collection/unpivot.rb +17 -11
- data/lib/burner/library/collection/validate.rb +90 -0
- data/lib/burner/library/collection/values.rb +9 -11
- data/lib/burner/library/deserialize/csv.rb +4 -6
- data/lib/burner/library/deserialize/json.rb +4 -6
- data/lib/burner/library/deserialize/yaml.rb +7 -7
- data/lib/burner/library/echo.rb +1 -3
- data/lib/burner/library/io/base.rb +3 -3
- data/lib/burner/library/io/exist.rb +9 -9
- data/lib/burner/library/io/read.rb +5 -7
- data/lib/burner/library/io/write.rb +5 -7
- data/lib/burner/library/{dummy.rb → nothing.rb} +3 -5
- data/lib/burner/library/serialize/csv.rb +5 -7
- data/lib/burner/library/serialize/json.rb +4 -6
- data/lib/burner/library/serialize/yaml.rb +4 -6
- data/lib/burner/library/set_value.rb +6 -8
- data/lib/burner/library/sleep.rb +1 -3
- data/lib/burner/modeling.rb +1 -0
- data/lib/burner/modeling/attribute.rb +3 -1
- data/lib/burner/modeling/validations.rb +23 -0
- data/lib/burner/modeling/validations/base.rb +35 -0
- data/lib/burner/modeling/validations/blank.rb +31 -0
- data/lib/burner/modeling/validations/present.rb +31 -0
- data/lib/burner/payload.rb +50 -10
- data/lib/burner/pipeline.rb +3 -3
- data/lib/burner/step.rb +1 -5
- data/lib/burner/util.rb +1 -0
- data/lib/burner/util/string_template.rb +42 -0
- data/lib/burner/version.rb +1 -1
- metadata +13 -6
- data/lib/burner/string_template.rb +0 -40
@@ -15,16 +15,17 @@ module Burner
|
|
15
15
|
# Check to see if a file exists. If short_circuit is set to true and the file
|
16
16
|
# does not exist then the job will return false and short circuit the pipeline.
|
17
17
|
#
|
18
|
-
# Note: this does not use Payload#
|
19
|
-
class Exist <
|
20
|
-
attr_reader :short_circuit
|
18
|
+
# Note: this does not use Payload#registers.
|
19
|
+
class Exist < Job
|
20
|
+
attr_reader :path, :short_circuit
|
21
21
|
|
22
22
|
def initialize(name:, path:, short_circuit: false)
|
23
|
-
super(name: name
|
23
|
+
super(name: name)
|
24
24
|
|
25
|
-
|
25
|
+
raise ArgumentError, 'path is required' if path.to_s.empty?
|
26
26
|
|
27
|
-
|
27
|
+
@path = path.to_s
|
28
|
+
@short_circuit = short_circuit || false
|
28
29
|
end
|
29
30
|
|
30
31
|
def perform(output, payload)
|
@@ -35,9 +36,8 @@ module Burner
|
|
35
36
|
|
36
37
|
output.detail("The path: #{compiled_path} #{verb} exist")
|
37
38
|
|
38
|
-
# if anything but false is returned then the pipeline will not short circuit.
|
39
|
-
|
40
|
-
short_circuit && !exists ? false : nil
|
39
|
+
# if anything but false is returned then the pipeline will not short circuit.
|
40
|
+
payload.halt_pipeline if short_circuit && !exists
|
41
41
|
end
|
42
42
|
end
|
43
43
|
end
|
@@ -14,13 +14,13 @@ module Burner
|
|
14
14
|
module IO
|
15
15
|
# Read value from disk.
|
16
16
|
#
|
17
|
-
# Expected Payload
|
18
|
-
# Payload
|
17
|
+
# Expected Payload[register] input: nothing.
|
18
|
+
# Payload[register] output: contents of the specified file.
|
19
19
|
class Read < Base
|
20
20
|
attr_reader :binary
|
21
21
|
|
22
|
-
def initialize(name:, path:, binary: false)
|
23
|
-
super(name: name, path: path)
|
22
|
+
def initialize(name:, path:, binary: false, register: '')
|
23
|
+
super(name: name, path: path, register: register)
|
24
24
|
|
25
25
|
@binary = binary || false
|
26
26
|
|
@@ -32,9 +32,7 @@ module Burner
|
|
32
32
|
|
33
33
|
output.detail("Reading: #{compiled_path}")
|
34
34
|
|
35
|
-
payload
|
36
|
-
|
37
|
-
nil
|
35
|
+
payload[register] = File.open(compiled_path, mode, &:read)
|
38
36
|
end
|
39
37
|
|
40
38
|
private
|
@@ -14,13 +14,13 @@ module Burner
|
|
14
14
|
module IO
|
15
15
|
# Write value to disk.
|
16
16
|
#
|
17
|
-
# Expected Payload
|
18
|
-
# Payload
|
17
|
+
# Expected Payload[register] input: anything.
|
18
|
+
# Payload[register] output: whatever was passed in.
|
19
19
|
class Write < Base
|
20
20
|
attr_reader :binary
|
21
21
|
|
22
|
-
def initialize(name:, path:, binary: false)
|
23
|
-
super(name: name, path: path)
|
22
|
+
def initialize(name:, path:, binary: false, register: '')
|
23
|
+
super(name: name, path: path, register: register)
|
24
24
|
|
25
25
|
@binary = binary || false
|
26
26
|
|
@@ -35,7 +35,7 @@ module Burner
|
|
35
35
|
output.detail("Writing: #{compiled_path}")
|
36
36
|
|
37
37
|
time_in_seconds = Benchmark.measure do
|
38
|
-
File.open(compiled_path, mode) { |io| io.write(payload
|
38
|
+
File.open(compiled_path, mode) { |io| io.write(payload[register]) }
|
39
39
|
end.real
|
40
40
|
|
41
41
|
side_effect = SideEffects::WrittenFile.new(
|
@@ -45,8 +45,6 @@ module Burner
|
|
45
45
|
)
|
46
46
|
|
47
47
|
payload.add_side_effect(side_effect)
|
48
|
-
|
49
|
-
nil
|
50
48
|
end
|
51
49
|
|
52
50
|
private
|
@@ -11,11 +11,9 @@ module Burner
|
|
11
11
|
module Library
|
12
12
|
# Do nothing.
|
13
13
|
#
|
14
|
-
# Note: this does not use Payload#
|
15
|
-
class
|
16
|
-
def perform(_output, _payload)
|
17
|
-
nil
|
18
|
-
end
|
14
|
+
# Note: this does not use Payload#registers.
|
15
|
+
class Nothing < Job
|
16
|
+
def perform(_output, _payload); end
|
19
17
|
end
|
20
18
|
end
|
21
19
|
end
|
@@ -12,17 +12,15 @@ module Burner
|
|
12
12
|
module Serialize
|
13
13
|
# Take an array of arrays and create a CSV.
|
14
14
|
#
|
15
|
-
# Expected Payload
|
16
|
-
# Payload
|
17
|
-
class Csv <
|
15
|
+
# Expected Payload[register] input: array of arrays.
|
16
|
+
# Payload[register] output: a serialized CSV string.
|
17
|
+
class Csv < JobWithRegister
|
18
18
|
def perform(_output, payload)
|
19
|
-
payload
|
20
|
-
array(payload
|
19
|
+
payload[register] = CSV.generate(options) do |csv|
|
20
|
+
array(payload[register]).each do |row|
|
21
21
|
csv << row
|
22
22
|
end
|
23
23
|
end
|
24
|
-
|
25
|
-
nil
|
26
24
|
end
|
27
25
|
|
28
26
|
private
|
@@ -12,13 +12,11 @@ module Burner
|
|
12
12
|
module Serialize
|
13
13
|
# Treat value like a Ruby object and serialize it using JSON.
|
14
14
|
#
|
15
|
-
# Expected Payload
|
16
|
-
# Payload
|
17
|
-
class Json <
|
15
|
+
# Expected Payload[register] input: anything.
|
16
|
+
# Payload[register] output: string representing the output of the JSON serializer.
|
17
|
+
class Json < JobWithRegister
|
18
18
|
def perform(_output, payload)
|
19
|
-
payload
|
20
|
-
|
21
|
-
nil
|
19
|
+
payload[register] = payload[register].to_json
|
22
20
|
end
|
23
21
|
end
|
24
22
|
end
|
@@ -12,13 +12,11 @@ module Burner
|
|
12
12
|
module Serialize
|
13
13
|
# Treat value like a Ruby object and serialize it using YAML.
|
14
14
|
#
|
15
|
-
# Expected Payload
|
16
|
-
# Payload
|
17
|
-
class Yaml <
|
15
|
+
# Expected Payload[register] input: anything.
|
16
|
+
# Payload[register] output: string representing the output of the YAML serializer.
|
17
|
+
class Yaml < JobWithRegister
|
18
18
|
def perform(_output, payload)
|
19
|
-
payload
|
20
|
-
|
21
|
-
nil
|
19
|
+
payload[register] = payload[register].to_yaml
|
22
20
|
end
|
23
21
|
end
|
24
22
|
end
|
@@ -11,13 +11,13 @@ module Burner
|
|
11
11
|
module Library
|
12
12
|
# Arbitrarily set value
|
13
13
|
#
|
14
|
-
# Expected Payload
|
15
|
-
# Payload
|
16
|
-
class SetValue <
|
14
|
+
# Expected Payload[register] input: anything.
|
15
|
+
# Payload[register] output: whatever value was specified in this job.
|
16
|
+
class SetValue < JobWithRegister
|
17
17
|
attr_reader :value
|
18
18
|
|
19
|
-
def initialize(name:, value: nil)
|
20
|
-
super(name: name)
|
19
|
+
def initialize(name:, register: '', value: nil)
|
20
|
+
super(name: name, register: register)
|
21
21
|
|
22
22
|
@value = value
|
23
23
|
|
@@ -25,9 +25,7 @@ module Burner
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def perform(_output, payload)
|
28
|
-
payload
|
29
|
-
|
30
|
-
nil
|
28
|
+
payload[register] = value
|
31
29
|
end
|
32
30
|
end
|
33
31
|
end
|
data/lib/burner/library/sleep.rb
CHANGED
@@ -11,7 +11,7 @@ module Burner
|
|
11
11
|
module Library
|
12
12
|
# Arbitrarily put thread to sleep for X number of seconds
|
13
13
|
#
|
14
|
-
#
|
14
|
+
# Note: this does not use Payload#registers.
|
15
15
|
class Sleep < Job
|
16
16
|
attr_reader :seconds
|
17
17
|
|
@@ -27,8 +27,6 @@ module Burner
|
|
27
27
|
output.detail("Going to sleep for #{seconds} second(s)")
|
28
28
|
|
29
29
|
Kernel.sleep(seconds)
|
30
|
-
|
31
|
-
nil
|
32
30
|
end
|
33
31
|
end
|
34
32
|
end
|
data/lib/burner/modeling.rb
CHANGED
@@ -10,7 +10,9 @@
|
|
10
10
|
module Burner
|
11
11
|
module Modeling
|
12
12
|
# Defines a top-level key and the associated transformers for deriving the final value
|
13
|
-
# to set the key to.
|
13
|
+
# to set the key to. The transformers that can be passed in can be any Realize::Transformers
|
14
|
+
# subclasses. For more information, see the Realize library at:
|
15
|
+
# https://github.com/bluemarblepayroll/realize
|
14
16
|
class Attribute
|
15
17
|
acts_as_hashable
|
16
18
|
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'validations/blank'
|
11
|
+
require_relative 'validations/present'
|
12
|
+
|
13
|
+
module Burner
|
14
|
+
module Modeling
|
15
|
+
# Factory for building sub-classes that can validate an individual object and field value.
|
16
|
+
class Validations
|
17
|
+
acts_as_hashable_factory
|
18
|
+
|
19
|
+
register 'blank', Validations::Blank
|
20
|
+
register 'present', Validations::Present
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
class Validations
|
13
|
+
# Common logic shared among all Validation subclasses.
|
14
|
+
# This class is an abstract class, make sure to implement:
|
15
|
+
# - #valid?(object, resolver)
|
16
|
+
# - #default_message
|
17
|
+
class Base
|
18
|
+
acts_as_hashable
|
19
|
+
|
20
|
+
attr_reader :key
|
21
|
+
|
22
|
+
def initialize(key:, message: '')
|
23
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
24
|
+
|
25
|
+
@key = key.to_s
|
26
|
+
@message = message.to_s
|
27
|
+
end
|
28
|
+
|
29
|
+
def message
|
30
|
+
@message.to_s.empty? ? "#{key}#{default_message}" : @message.to_s
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
module Modeling
|
14
|
+
class Validations
|
15
|
+
# Check if a value is blank, if it is not blank then it is not valid.
|
16
|
+
class Blank < Base
|
17
|
+
acts_as_hashable
|
18
|
+
|
19
|
+
def valid?(object, resolver)
|
20
|
+
resolver.get(object, key).to_s.empty?
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def default_message
|
26
|
+
' must be blank'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
module Modeling
|
14
|
+
class Validations
|
15
|
+
# Check if a value is present. If it is blank (null or empty) then it is invalid.
|
16
|
+
class Present < Base
|
17
|
+
acts_as_hashable
|
18
|
+
|
19
|
+
def valid?(object_value, resolver)
|
20
|
+
!resolver.get(object_value, key).to_s.empty?
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def default_message
|
26
|
+
' is required'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/burner/payload.rb
CHANGED
@@ -8,37 +8,77 @@
|
|
8
8
|
#
|
9
9
|
|
10
10
|
module Burner
|
11
|
-
# The input for all Job#perform methods. The main notion of this object is its
|
12
|
-
# attribute. This
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
11
|
+
# The input for all Job#perform methods. The main notion of this object is its 'registers'
|
12
|
+
# attribute. This registers attribute is a key-indifferent hash, accessible on Payload using
|
13
|
+
# the brackets setter and getter methods. This is dynamic and weak on purpose and is subject
|
14
|
+
# to whatever the Job#perform methods decides it is. This definitely adds an order-of-magnitude
|
15
|
+
# complexity to this whole library and lifecycle, but I am not sure there is any other way
|
16
|
+
# around it: trying to build a generic, open-ended processing pipeline to serve almost
|
17
|
+
# any use case.
|
16
18
|
#
|
17
19
|
# The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
|
18
20
|
# structured/additive manner. The initial use case for this was for Burner's core IO jobs to
|
19
21
|
# report back the files it has written in a more structured data way (as opposed to simply
|
20
22
|
# writing some information to the output.)
|
23
|
+
#
|
24
|
+
# The 'time' attribute is important in that it should for the replaying of pipelines and jobs.
|
25
|
+
# Instead of having job's utilizing Time.now, Date.today, etc... they should rather opt to
|
26
|
+
# use this value instead.
|
21
27
|
class Payload
|
22
|
-
attr_accessor :value
|
23
|
-
|
24
28
|
attr_reader :params,
|
29
|
+
:registers,
|
25
30
|
:side_effects,
|
26
31
|
:time
|
27
32
|
|
28
33
|
def initialize(
|
29
34
|
params: {},
|
35
|
+
registers: {},
|
30
36
|
side_effects: [],
|
31
|
-
time: Time.now.utc
|
32
|
-
value: nil
|
37
|
+
time: Time.now.utc
|
33
38
|
)
|
34
39
|
@params = params || {}
|
40
|
+
@registers = {}
|
35
41
|
@side_effects = side_effects || []
|
36
42
|
@time = time || Time.now.utc
|
37
|
-
|
43
|
+
|
44
|
+
add_registers(registers)
|
38
45
|
end
|
39
46
|
|
47
|
+
# Add a side effect of a job. This helps to keep track of things jobs do outside of its
|
48
|
+
# register mutations.
|
40
49
|
def add_side_effect(side_effect)
|
41
50
|
tap { side_effects << side_effect }
|
42
51
|
end
|
52
|
+
|
53
|
+
# Set a register's value.
|
54
|
+
def []=(key, value)
|
55
|
+
set(key, value)
|
56
|
+
end
|
57
|
+
|
58
|
+
# Retrieve a register's value.
|
59
|
+
def [](key)
|
60
|
+
registers[key.to_s]
|
61
|
+
end
|
62
|
+
|
63
|
+
# Set halt_pipeline to true. This will indicate to the pipeline to stop all
|
64
|
+
# subsequent processing.
|
65
|
+
def halt_pipeline
|
66
|
+
@halt_pipeline = true
|
67
|
+
end
|
68
|
+
|
69
|
+
# Check and see if halt_pipeline was called.
|
70
|
+
def halt_pipeline?
|
71
|
+
@halt_pipeline || false
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def set(key, value)
|
77
|
+
registers[key.to_s] = value
|
78
|
+
end
|
79
|
+
|
80
|
+
def add_registers(registers)
|
81
|
+
(registers || {}).each { |k, v| set(k, v) }
|
82
|
+
end
|
43
83
|
end
|
44
84
|
end
|