burner 1.0.0.pre.alpha.6 → 1.0.0.pre.alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -0
  3. data/README.md +43 -39
  4. data/burner.gemspec +1 -1
  5. data/lib/burner/job.rb +15 -10
  6. data/lib/burner/job_with_register.rb +24 -0
  7. data/lib/burner/jobs.rb +27 -20
  8. data/lib/burner/library.rb +11 -5
  9. data/lib/burner/library/collection/arrays_to_objects.rb +14 -11
  10. data/lib/burner/library/collection/graph.rb +7 -9
  11. data/lib/burner/library/collection/objects_to_arrays.rb +34 -34
  12. data/lib/burner/library/collection/shift.rb +6 -8
  13. data/lib/burner/library/collection/transform.rb +7 -9
  14. data/lib/burner/library/collection/unpivot.rb +17 -11
  15. data/lib/burner/library/collection/validate.rb +90 -0
  16. data/lib/burner/library/collection/values.rb +9 -11
  17. data/lib/burner/library/deserialize/csv.rb +4 -6
  18. data/lib/burner/library/deserialize/json.rb +4 -6
  19. data/lib/burner/library/deserialize/yaml.rb +7 -7
  20. data/lib/burner/library/echo.rb +1 -3
  21. data/lib/burner/library/io/base.rb +3 -3
  22. data/lib/burner/library/io/exist.rb +9 -9
  23. data/lib/burner/library/io/read.rb +5 -7
  24. data/lib/burner/library/io/write.rb +5 -7
  25. data/lib/burner/library/{dummy.rb → nothing.rb} +3 -5
  26. data/lib/burner/library/serialize/csv.rb +5 -7
  27. data/lib/burner/library/serialize/json.rb +4 -6
  28. data/lib/burner/library/serialize/yaml.rb +4 -6
  29. data/lib/burner/library/set_value.rb +6 -8
  30. data/lib/burner/library/sleep.rb +1 -3
  31. data/lib/burner/modeling.rb +1 -0
  32. data/lib/burner/modeling/attribute.rb +3 -1
  33. data/lib/burner/modeling/validations.rb +23 -0
  34. data/lib/burner/modeling/validations/base.rb +35 -0
  35. data/lib/burner/modeling/validations/blank.rb +31 -0
  36. data/lib/burner/modeling/validations/present.rb +31 -0
  37. data/lib/burner/payload.rb +50 -10
  38. data/lib/burner/pipeline.rb +3 -3
  39. data/lib/burner/step.rb +1 -5
  40. data/lib/burner/util.rb +1 -0
  41. data/lib/burner/util/string_template.rb +42 -0
  42. data/lib/burner/version.rb +1 -1
  43. metadata +13 -6
  44. data/lib/burner/string_template.rb +0 -40
@@ -15,16 +15,17 @@ module Burner
15
15
  # Check to see if a file exists. If short_circuit is set to true and the file
16
16
  # does not exist then the job will return false and short circuit the pipeline.
17
17
  #
18
- # Note: this does not use Payload#value.
19
- class Exist < Base
20
- attr_reader :short_circuit
18
+ # Note: this does not use Payload#registers.
19
+ class Exist < Job
20
+ attr_reader :path, :short_circuit
21
21
 
22
22
  def initialize(name:, path:, short_circuit: false)
23
- super(name: name, path: path)
23
+ super(name: name)
24
24
 
25
- @short_circuit = short_circuit || false
25
+ raise ArgumentError, 'path is required' if path.to_s.empty?
26
26
 
27
- freeze
27
+ @path = path.to_s
28
+ @short_circuit = short_circuit || false
28
29
  end
29
30
 
30
31
  def perform(output, payload)
@@ -35,9 +36,8 @@ module Burner
35
36
 
36
37
  output.detail("The path: #{compiled_path} #{verb} exist")
37
38
 
38
- # if anything but false is returned then the pipeline will not short circuit. So
39
- # we need to make sure we explicitly return false.
40
- short_circuit && !exists ? false : nil
39
+ # if anything but false is returned then the pipeline will not short circuit.
40
+ payload.halt_pipeline if short_circuit && !exists
41
41
  end
42
42
  end
43
43
  end
@@ -14,13 +14,13 @@ module Burner
14
14
  module IO
15
15
  # Read value from disk.
16
16
  #
17
- # Expected Payload#value input: nothing.
18
- # Payload#value output: contents of the specified file.
17
+ # Expected Payload[register] input: nothing.
18
+ # Payload[register] output: contents of the specified file.
19
19
  class Read < Base
20
20
  attr_reader :binary
21
21
 
22
- def initialize(name:, path:, binary: false)
23
- super(name: name, path: path)
22
+ def initialize(name:, path:, binary: false, register: '')
23
+ super(name: name, path: path, register: register)
24
24
 
25
25
  @binary = binary || false
26
26
 
@@ -32,9 +32,7 @@ module Burner
32
32
 
33
33
  output.detail("Reading: #{compiled_path}")
34
34
 
35
- payload.value = File.open(compiled_path, mode, &:read)
36
-
37
- nil
35
+ payload[register] = File.open(compiled_path, mode, &:read)
38
36
  end
39
37
 
40
38
  private
@@ -14,13 +14,13 @@ module Burner
14
14
  module IO
15
15
  # Write value to disk.
16
16
  #
17
- # Expected Payload#value input: anything.
18
- # Payload#value output: whatever was passed in.
17
+ # Expected Payload[register] input: anything.
18
+ # Payload[register] output: whatever was passed in.
19
19
  class Write < Base
20
20
  attr_reader :binary
21
21
 
22
- def initialize(name:, path:, binary: false)
23
- super(name: name, path: path)
22
+ def initialize(name:, path:, binary: false, register: '')
23
+ super(name: name, path: path, register: register)
24
24
 
25
25
  @binary = binary || false
26
26
 
@@ -35,7 +35,7 @@ module Burner
35
35
  output.detail("Writing: #{compiled_path}")
36
36
 
37
37
  time_in_seconds = Benchmark.measure do
38
- File.open(compiled_path, mode) { |io| io.write(payload.value) }
38
+ File.open(compiled_path, mode) { |io| io.write(payload[register]) }
39
39
  end.real
40
40
 
41
41
  side_effect = SideEffects::WrittenFile.new(
@@ -45,8 +45,6 @@ module Burner
45
45
  )
46
46
 
47
47
  payload.add_side_effect(side_effect)
48
-
49
- nil
50
48
  end
51
49
 
52
50
  private
@@ -11,11 +11,9 @@ module Burner
11
11
  module Library
12
12
  # Do nothing.
13
13
  #
14
- # Note: this does not use Payload#value.
15
- class Dummy < Job
16
- def perform(_output, _payload)
17
- nil
18
- end
14
+ # Note: this does not use Payload#registers.
15
+ class Nothing < Job
16
+ def perform(_output, _payload); end
19
17
  end
20
18
  end
21
19
  end
@@ -12,17 +12,15 @@ module Burner
12
12
  module Serialize
13
13
  # Take an array of arrays and create a CSV.
14
14
  #
15
- # Expected Payload#value input: array of arrays.
16
- # Payload#value output: a serialized CSV string.
17
- class Csv < Job
15
+ # Expected Payload[register] input: array of arrays.
16
+ # Payload[register] output: a serialized CSV string.
17
+ class Csv < JobWithRegister
18
18
  def perform(_output, payload)
19
- payload.value = CSV.generate(options) do |csv|
20
- array(payload.value).each do |row|
19
+ payload[register] = CSV.generate(options) do |csv|
20
+ array(payload[register]).each do |row|
21
21
  csv << row
22
22
  end
23
23
  end
24
-
25
- nil
26
24
  end
27
25
 
28
26
  private
@@ -12,13 +12,11 @@ module Burner
12
12
  module Serialize
13
13
  # Treat value like a Ruby object and serialize it using JSON.
14
14
  #
15
- # Expected Payload#value input: anything.
16
- # Payload#value output: string representing the output of the JSON serializer.
17
- class Json < Job
15
+ # Expected Payload[register] input: anything.
16
+ # Payload[register] output: string representing the output of the JSON serializer.
17
+ class Json < JobWithRegister
18
18
  def perform(_output, payload)
19
- payload.value = payload.value.to_json
20
-
21
- nil
19
+ payload[register] = payload[register].to_json
22
20
  end
23
21
  end
24
22
  end
@@ -12,13 +12,11 @@ module Burner
12
12
  module Serialize
13
13
  # Treat value like a Ruby object and serialize it using YAML.
14
14
  #
15
- # Expected Payload#value input: anything.
16
- # Payload#value output: string representing the output of the YAML serializer.
17
- class Yaml < Job
15
+ # Expected Payload[register] input: anything.
16
+ # Payload[register] output: string representing the output of the YAML serializer.
17
+ class Yaml < JobWithRegister
18
18
  def perform(_output, payload)
19
- payload.value = payload.value.to_yaml
20
-
21
- nil
19
+ payload[register] = payload[register].to_yaml
22
20
  end
23
21
  end
24
22
  end
@@ -11,13 +11,13 @@ module Burner
11
11
  module Library
12
12
  # Arbitrarily set value
13
13
  #
14
- # Expected Payload#value input: anything.
15
- # Payload#value output: whatever value was specified in this job.
16
- class SetValue < Job
14
+ # Expected Payload[register] input: anything.
15
+ # Payload[register] output: whatever value was specified in this job.
16
+ class SetValue < JobWithRegister
17
17
  attr_reader :value
18
18
 
19
- def initialize(name:, value: nil)
20
- super(name: name)
19
+ def initialize(name:, register: '', value: nil)
20
+ super(name: name, register: register)
21
21
 
22
22
  @value = value
23
23
 
@@ -25,9 +25,7 @@ module Burner
25
25
  end
26
26
 
27
27
  def perform(_output, payload)
28
- payload.value = value
29
-
30
- nil
28
+ payload[register] = value
31
29
  end
32
30
  end
33
31
  end
@@ -11,7 +11,7 @@ module Burner
11
11
  module Library
12
12
  # Arbitrarily put thread to sleep for X number of seconds
13
13
  #
14
- # Payload#value output: whatever value was specified in this job.
14
+ # Note: this does not use Payload#registers.
15
15
  class Sleep < Job
16
16
  attr_reader :seconds
17
17
 
@@ -27,8 +27,6 @@ module Burner
27
27
  output.detail("Going to sleep for #{seconds} second(s)")
28
28
 
29
29
  Kernel.sleep(seconds)
30
-
31
- nil
32
30
  end
33
31
  end
34
32
  end
@@ -10,3 +10,4 @@
10
10
  require_relative 'modeling/attribute'
11
11
  require_relative 'modeling/attribute_renderer'
12
12
  require_relative 'modeling/key_index_mapping'
13
+ require_relative 'modeling/validations'
@@ -10,7 +10,9 @@
10
10
  module Burner
11
11
  module Modeling
12
12
  # Defines a top-level key and the associated transformers for deriving the final value
13
- # to set the key to.
13
+ # to set the key to. The transformers that can be passed in can be any Realize::Transformers
14
+ # subclasses. For more information, see the Realize library at:
15
+ # https://github.com/bluemarblepayroll/realize
14
16
  class Attribute
15
17
  acts_as_hashable
16
18
 
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'validations/blank'
11
+ require_relative 'validations/present'
12
+
13
+ module Burner
14
+ module Modeling
15
+ # Factory for building sub-classes that can validate an individual object and field value.
16
+ class Validations
17
+ acts_as_hashable_factory
18
+
19
+ register 'blank', Validations::Blank
20
+ register 'present', Validations::Present
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Modeling
12
+ class Validations
13
+ # Common logic shared among all Validation subclasses.
14
+ # This class is an abstract class, make sure to implement:
15
+ # - #valid?(object, resolver)
16
+ # - #default_message
17
+ class Base
18
+ acts_as_hashable
19
+
20
+ attr_reader :key
21
+
22
+ def initialize(key:, message: '')
23
+ raise ArgumentError, 'key is required' if key.to_s.empty?
24
+
25
+ @key = key.to_s
26
+ @message = message.to_s
27
+ end
28
+
29
+ def message
30
+ @message.to_s.empty? ? "#{key}#{default_message}" : @message.to_s
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'base'
11
+
12
+ module Burner
13
+ module Modeling
14
+ class Validations
15
+ # Check if a value is blank, if it is not blank then it is not valid.
16
+ class Blank < Base
17
+ acts_as_hashable
18
+
19
+ def valid?(object, resolver)
20
+ resolver.get(object, key).to_s.empty?
21
+ end
22
+
23
+ private
24
+
25
+ def default_message
26
+ ' must be blank'
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'base'
11
+
12
+ module Burner
13
+ module Modeling
14
+ class Validations
15
+ # Check if a value is present. If it is blank (null or empty) then it is invalid.
16
+ class Present < Base
17
+ acts_as_hashable
18
+
19
+ def valid?(object_value, resolver)
20
+ !resolver.get(object_value, key).to_s.empty?
21
+ end
22
+
23
+ private
24
+
25
+ def default_message
26
+ ' is required'
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -8,37 +8,77 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- # The input for all Job#perform methods. The main notion of this object is its "value"
12
- # attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
13
- # methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
14
- # library and lifecycle, but I am not sure there is any other way around it: trying to build
15
- # a generic, open-ended object pipeline to serve almost any use case.
11
+ # The input for all Job#perform methods. The main notion of this object is its 'registers'
12
+ # attribute. This registers attribute is a key-indifferent hash, accessible on Payload using
13
+ # the brackets setter and getter methods. This is dynamic and weak on purpose and is subject
14
+ # to whatever the Job#perform methods decides it is. This definitely adds an order-of-magnitude
15
+ # complexity to this whole library and lifecycle, but I am not sure there is any other way
16
+ # around it: trying to build a generic, open-ended processing pipeline to serve almost
17
+ # any use case.
16
18
  #
17
19
  # The side_effects attribute can also be utilized as a way for jobs to emit any data in a more
18
20
  # structured/additive manner. The initial use case for this was for Burner's core IO jobs to
19
21
  # report back the files it has written in a more structured data way (as opposed to simply
20
22
  # writing some information to the output.)
23
+ #
24
+ # The 'time' attribute is important in that it should for the replaying of pipelines and jobs.
25
+ # Instead of having job's utilizing Time.now, Date.today, etc... they should rather opt to
26
+ # use this value instead.
21
27
  class Payload
22
- attr_accessor :value
23
-
24
28
  attr_reader :params,
29
+ :registers,
25
30
  :side_effects,
26
31
  :time
27
32
 
28
33
  def initialize(
29
34
  params: {},
35
+ registers: {},
30
36
  side_effects: [],
31
- time: Time.now.utc,
32
- value: nil
37
+ time: Time.now.utc
33
38
  )
34
39
  @params = params || {}
40
+ @registers = {}
35
41
  @side_effects = side_effects || []
36
42
  @time = time || Time.now.utc
37
- @value = value
43
+
44
+ add_registers(registers)
38
45
  end
39
46
 
47
+ # Add a side effect of a job. This helps to keep track of things jobs do outside of its
48
+ # register mutations.
40
49
  def add_side_effect(side_effect)
41
50
  tap { side_effects << side_effect }
42
51
  end
52
+
53
+ # Set a register's value.
54
+ def []=(key, value)
55
+ set(key, value)
56
+ end
57
+
58
+ # Retrieve a register's value.
59
+ def [](key)
60
+ registers[key.to_s]
61
+ end
62
+
63
+ # Set halt_pipeline to true. This will indicate to the pipeline to stop all
64
+ # subsequent processing.
65
+ def halt_pipeline
66
+ @halt_pipeline = true
67
+ end
68
+
69
+ # Check and see if halt_pipeline was called.
70
+ def halt_pipeline?
71
+ @halt_pipeline || false
72
+ end
73
+
74
+ private
75
+
76
+ def set(key, value)
77
+ registers[key.to_s] = value
78
+ end
79
+
80
+ def add_registers(registers)
81
+ (registers || {}).each { |k, v| set(k, v) }
82
+ end
43
83
  end
44
84
  end