burner 1.6.0 → 1.7.0.pre.alpha

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +12 -0
  3. data/README.md +53 -2
  4. data/lib/burner/job.rb +1 -3
  5. data/lib/burner/job_set.rb +64 -0
  6. data/lib/burner/job_with_register.rb +8 -1
  7. data/lib/burner/jobs.rb +3 -0
  8. data/lib/burner/library.rb +3 -0
  9. data/lib/burner/library/collection/arrays_to_objects.rb +1 -1
  10. data/lib/burner/library/collection/coalesce.rb +14 -9
  11. data/lib/burner/library/collection/concatenate.rb +1 -1
  12. data/lib/burner/library/collection/graph.rb +1 -1
  13. data/lib/burner/library/collection/group.rb +15 -11
  14. data/lib/burner/library/collection/nested_aggregate.rb +1 -1
  15. data/lib/burner/library/collection/number.rb +51 -0
  16. data/lib/burner/library/collection/objects_to_arrays.rb +1 -1
  17. data/lib/burner/library/collection/shift.rb +1 -1
  18. data/lib/burner/library/collection/transform.rb +1 -1
  19. data/lib/burner/library/collection/unpivot.rb +1 -1
  20. data/lib/burner/library/collection/validate.rb +1 -1
  21. data/lib/burner/library/collection/values.rb +1 -1
  22. data/lib/burner/library/collection/zip.rb +1 -1
  23. data/lib/burner/library/compress/row_reader.rb +1 -1
  24. data/lib/burner/library/deserialize/yaml.rb +1 -1
  25. data/lib/burner/library/echo.rb +1 -1
  26. data/lib/burner/library/io/exist.rb +1 -1
  27. data/lib/burner/library/io/open_file_base.rb +1 -1
  28. data/lib/burner/library/io/row_reader.rb +1 -1
  29. data/lib/burner/library/io/write.rb +1 -1
  30. data/lib/burner/library/serialize/csv.rb +1 -1
  31. data/lib/burner/library/sleep.rb +1 -1
  32. data/lib/burner/library/value/copy.rb +1 -1
  33. data/lib/burner/library/value/nest.rb +37 -0
  34. data/lib/burner/library/value/static.rb +1 -1
  35. data/lib/burner/library/value/transform.rb +38 -0
  36. data/lib/burner/pipeline.rb +5 -33
  37. data/lib/burner/util.rb +1 -0
  38. data/lib/burner/util/keyable.rb +23 -0
  39. data/lib/burner/version.rb +1 -1
  40. metadata +9 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: aa6bd0e3067707c53c82cd82816d3320ebe687ba071fa6bbbf97707035b5f46f
4
- data.tar.gz: d104558bc6dcd8e4dd7186e7f5365294d3e421ff3e7fc13fe28765d3f7a69b7d
3
+ metadata.gz: 015d18494cd63b8440f061092912d8c70d2b9327d4983f2e2465f5ccc0fbc971
4
+ data.tar.gz: d1fc16e3a9aed39dc9a910e178fd8dd6adabfd0f69320c525b7ef6b6b092541b
5
5
  SHA512:
6
- metadata.gz: 47ff0ccaba50a39709a13b84f6727876a480dd8f4c6c420a9e374396db30fa9c74e64d62a32c4daf452320eca6266b662f3710d7f86262fbc9507391752a5d9c
7
- data.tar.gz: d0900276a49941b883fddd34f1fdd78d126ca076e790cf53b7d72b94a7d24892142013710a53a7b314a270a1853ffcb8e51731868c5c7c47461322e38fd24315
6
+ metadata.gz: 3429b4a33464567987939d911a8265854f2f35788a9fadeb294d7209ab5e09c2f4b1bd0bba77c93ada6c315ad0fae57079a6843bb67fb2bc49cce284939146ab
7
+ data.tar.gz: 4f31f6728c709dac0af4168122cba9d63409e8e018d8fb8140a0a1d29baad893b0a022ada878f535ef08f36b28697996c6d14b1a253416230be284c2b4e60b7b
@@ -1,4 +1,16 @@
1
1
 
2
+ # 1.7.0 (TBD)
3
+
4
+ Added Jobs:
5
+
6
+ * b/collection/number
7
+ * b/value/nest
8
+ * b/value/transform
9
+
10
+ Enhanced Jobs:
11
+
12
+ * b/collection/coalesce and b/collection/group now support the notion of case and type-insensitivity (insensitive option).
13
+
2
14
  # 1.6.0 (December 22nd, 2020)
3
15
 
4
16
  Additions:
data/README.md CHANGED
@@ -93,6 +93,54 @@ Some notes:
93
93
  * Jobs can be re-used (just like the output_id and output_value jobs).
94
94
  * If steps is nil then all jobs will execute in their declared order.
95
95
 
96
+ ### Omitting Job Names and Steps
97
+
98
+ Job names are optional, but steps can only correspond to named jobs. This means if steps is declared then anonymous jobs will have no way to be executed. Here is the same pipeline as above, but without job names and steps:
99
+
100
+ ````ruby
101
+ pipeline = {
102
+ jobs: [
103
+ {
104
+ type: 'b/io/read',
105
+ path: '{input_file}'
106
+ },
107
+ {
108
+ type: 'b/echo',
109
+ message: 'The job id is: {__id}'
110
+ },
111
+ {
112
+ type: 'b/echo',
113
+ message: 'The current value is: {__default_register}'
114
+ },
115
+ {
116
+ type: 'b/deserialize/json'
117
+ },
118
+ {
119
+ type: 'b/serialize/yaml'
120
+ },
121
+ {
122
+ type: 'b/echo',
123
+ message: 'The current value is: {__default_register}'
124
+ },
125
+ {
126
+ type: 'b/io/write',
127
+ path: '{output_file}'
128
+ }
129
+ ]
130
+ }
131
+
132
+ params = {
133
+ input_file: 'input.json',
134
+ output_file: 'output.yaml'
135
+ }
136
+
137
+ payload = Burner::Payload.new(params: params)
138
+
139
+ Burner::Pipeline.make(pipeline).execute(payload: payload)
140
+ ````
141
+
142
+ Like everything in software, there are trade-offs to the above two equivalent pipelines. The former (one with steps and job names) has less jobs but is more verbose. The latter (without steps and job names) has more jobs but reads terser. Names also can aid in self-documenting your code/configuration so it may be a good idea to enforce at least names are used.
143
+
96
144
  ### Capturing Feedback / Output
97
145
 
98
146
  By default, output will be emitted to `$stdout`. You can add or change listeners by passing in optional values into Pipeline#execute. For example, say we wanted to capture the output from our json-to-yaml example:
@@ -216,11 +264,12 @@ This library only ships with very basic, rudimentary jobs that are meant to just
216
264
  #### Collection
217
265
 
218
266
  * **b/collection/arrays_to_objects** [mappings, register]: Convert an array of arrays to an array of objects.
219
- * **b/collection/coalesce** [register, grouped_register, key_mappings, keys, separator]: Merge two datasets together based on the key values of one dataset (array) with a grouped dataset (hash).
267
+ * **b/collection/coalesce** [grouped_register, insensitive, key_mappings, keys, register, separator]: Merge two datasets together based on the key values of one dataset (array) with a grouped dataset (hash). If insensitive (false by default) is true then each key's value will be converted/coerced to a lowercase string.
220
268
  * **b/collection/concatenate** [from_registers, to_register]: Concatenate each from_register's value and place the newly concatenated array into the to_register. Note: this does not do any deep copying and should be assumed it is shallow copying all objects.
221
269
  * **b/collection/graph** [config, key, register]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
222
- * **b/collection/group** [keys, register, separator]: Take a register's value (an array of objects) and group the objects by the specified keys.
270
+ * **b/collection/group** [insensitive, keys, register, separator]: Take a register's value (an array of objects) and group the objects by the specified keys. If insensitive (false by default) is true then each key's value will be converted/coerced to a lowercase string.
223
271
  * **b/collection/nested_aggregate** [register, key_mappings, key, separator]: Traverse a set of objects, resolving key's value for each object, optionally copying down key_mappings to the child records, then merging all the inner records together.
272
+ * **b/collection/number** [key, register, separator, start_at]: This job can iterate over a set of records and sequence them (set the specified key to a sequential index value.)
224
273
  * **b/collection/objects_to_arrays** [mappings, register]: Convert an array of objects to an array of arrays.
225
274
  * **b/collection/shift** [amount, register]: Remove the first N number of elements from an array.
226
275
  * **b/collection/transform** [attributes, exclusive, separator, register]: Iterate over all objects and transform each key per the attribute transformers specifications. If exclusive is set to false then the current object will be overridden/merged. Separator can also be set for key path support. This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline. If an attribute is not set with `explicit: true` then it will automatically start from the key's value from the record. If `explicit: true` is started, then it will start from the record itself.
@@ -257,7 +306,9 @@ By default all jobs will use the `Burner::Disks::Local` disk for its persistence
257
306
  #### Value
258
307
 
259
308
  * **b/value/copy** [from_register, to_register]: Copy from_register's value into the to_register. Note: this does not do any deep copying and should be assumed it is shallow copying all objects.
309
+ * **b/value/nest** [key, register]: This job will nest the current value within a new outer hash. The specified key passed in will be the corresponding new hash key entry for the existing value.
260
310
  * **b/value/static** [register, value]: Set the value to any arbitrary value.
311
+ * **b/value/transform** [register, separator, transformers]: Transform the current value of the register through a Realize::Pipeline. This will transform the entire value, as opposed to the b/collection/transform job, which will iterate over each row/record in a dataset and transform each row/record.
261
312
 
262
313
  #### General
263
314
 
@@ -18,9 +18,7 @@ module Burner
18
18
 
19
19
  attr_reader :name
20
20
 
21
- def initialize(name:)
22
- raise ArgumentError, 'name is required' if name.to_s.empty?
23
-
21
+ def initialize(name: '')
24
22
  @name = name.to_s
25
23
  end
26
24
 
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'jobs'
11
+
12
+ module Burner
13
+ # This class understands how jobs fit together as a cohesive unit. It does not know how to
14
+ # use them, but it knows how to group them together in a logical manner following some simple
15
+ # rules, such as:
16
+ # - Jobs in a set should have unique names (unless the name is blank)
17
+ # - Subsets of jobs can be extracted, by name, in constant time.
18
+ class JobSet
19
+ class DuplicateJobNameError < StandardError; end
20
+ class JobNotFoundError < StandardError; end
21
+
22
+ def initialize(jobs = [])
23
+ @jobs = Jobs.array(jobs).freeze
24
+
25
+ assert_unique_job_names
26
+ end
27
+
28
+ def jobs(names = nil)
29
+ return @jobs unless names
30
+
31
+ Array(names).map do |name|
32
+ job = named_jobs_by_name[name.to_s]
33
+
34
+ raise JobNotFoundError, "#{name} was not declared as a job" unless job
35
+
36
+ job
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def named_jobs_by_name
43
+ @named_jobs_by_name ||= named_jobs.each_with_object({}) { |job, memo| memo[job.name] = job }
44
+ end
45
+
46
+ def named_jobs
47
+ @named_jobs ||= @jobs.reject { |job| job.name == '' }
48
+ end
49
+
50
+ def assert_unique_job_names
51
+ unique_job_names = Set.new
52
+
53
+ named_jobs.each do |job|
54
+ if unique_job_names.include?(job.name)
55
+ raise DuplicateJobNameError, "job with name: #{job.name} already declared"
56
+ end
57
+
58
+ unique_job_names << job.name
59
+ end
60
+
61
+ nil
62
+ end
63
+ end
64
+ end
@@ -15,10 +15,17 @@ module Burner
15
15
  class JobWithRegister < Job
16
16
  attr_reader :register
17
17
 
18
- def initialize(name:, register: DEFAULT_REGISTER)
18
+ def initialize(name: '', register: DEFAULT_REGISTER)
19
19
  super(name: name)
20
20
 
21
21
  @register = register.to_s
22
22
  end
23
+
24
+ protected
25
+
26
+ # Helper method that knows how to ensure the register is an array.
27
+ def ensure_array(payload)
28
+ payload[register] = array(payload[register])
29
+ end
23
30
  end
24
31
  end
@@ -28,6 +28,7 @@ module Burner
28
28
  register 'b/collection/graph', Library::Collection::Graph
29
29
  register 'b/collection/group', Library::Collection::Group
30
30
  register 'b/collection/nested_aggregate', Library::Collection::NestedAggregate
31
+ register 'b/collection/number', Library::Collection::Number
31
32
  register 'b/collection/objects_to_arrays', Library::Collection::ObjectsToArrays
32
33
  register 'b/collection/shift', Library::Collection::Shift
33
34
  register 'b/collection/transform', Library::Collection::Transform
@@ -52,6 +53,8 @@ module Burner
52
53
  register 'b/serialize/yaml', Library::Serialize::Yaml
53
54
 
54
55
  register 'b/value/copy', Library::Value::Copy
56
+ register 'b/value/nest', Library::Value::Nest
55
57
  register 'b/value/static', Library::Value::Static
58
+ register 'b/value/transform', Library::Value::Transform
56
59
  end
57
60
  end
@@ -19,6 +19,7 @@ require_relative 'library/collection/concatenate'
19
19
  require_relative 'library/collection/graph'
20
20
  require_relative 'library/collection/group'
21
21
  require_relative 'library/collection/nested_aggregate'
22
+ require_relative 'library/collection/number'
22
23
  require_relative 'library/collection/objects_to_arrays'
23
24
  require_relative 'library/collection/shift'
24
25
  require_relative 'library/collection/transform'
@@ -43,4 +44,6 @@ require_relative 'library/serialize/json'
43
44
  require_relative 'library/serialize/yaml'
44
45
 
45
46
  require_relative 'library/value/copy'
47
+ require_relative 'library/value/nest'
46
48
  require_relative 'library/value/static'
49
+ require_relative 'library/value/transform'
@@ -55,7 +55,7 @@ module Burner
55
55
  class ArraysToObjects < JobWithRegister
56
56
  attr_reader :mappings
57
57
 
58
- def initialize(name:, mappings: [], register: DEFAULT_REGISTER)
58
+ def initialize(mappings: [], name: '', register: DEFAULT_REGISTER)
59
59
  super(name: name, register: register)
60
60
 
61
61
  @mappings = Modeling::KeyIndexMapping.array(mappings)
@@ -18,19 +18,27 @@ module Burner
18
18
  # Expected Payload[register] input: array of objects.
19
19
  # Payload[register] output: array of objects.
20
20
  class Coalesce < JobWithRegister
21
- attr_reader :grouped_register, :key_mappings, :keys, :resolver
21
+ include Util::Keyable
22
+
23
+ attr_reader :grouped_register,
24
+ :insensitive,
25
+ :key_mappings,
26
+ :keys,
27
+ :resolver
22
28
 
23
29
  def initialize(
24
- name:,
25
30
  grouped_register:,
31
+ insensitive: false,
26
32
  key_mappings: [],
27
33
  keys: [],
34
+ name: '',
28
35
  register: DEFAULT_REGISTER,
29
36
  separator: ''
30
37
  )
31
38
  super(name: name, register: register)
32
39
 
33
40
  @grouped_register = grouped_register.to_s
41
+ @insensitive = insensitive || false
34
42
  @key_mappings = Modeling::KeyMapping.array(key_mappings)
35
43
  @keys = Array(keys)
36
44
  @resolver = Objectable.resolver(separator: separator.to_s)
@@ -41,13 +49,14 @@ module Burner
41
49
  end
42
50
 
43
51
  def perform(output, payload)
44
- payload[register] = array(payload[register])
45
- count = payload[register].length
52
+ ensure_array(payload)
53
+
54
+ count = payload[register].length
46
55
 
47
56
  output.detail("Coalescing based on key(s): #{keys} for #{count} records(s)")
48
57
 
49
58
  payload[register].each do |record|
50
- key = make_key(record)
59
+ key = make_key(record, keys, resolver, insensitive)
51
60
  lookup = find_lookup(payload, key)
52
61
 
53
62
  key_mappings.each do |key_mapping|
@@ -63,10 +72,6 @@ module Burner
63
72
  def find_lookup(payload, key)
64
73
  (payload[grouped_register] || {})[key] || {}
65
74
  end
66
-
67
- def make_key(record)
68
- keys.map { |key| resolver.get(record, key) }
69
- end
70
75
  end
71
76
  end
72
77
  end
@@ -18,7 +18,7 @@ module Burner
18
18
  class Concatenate < Job
19
19
  attr_reader :from_registers, :to_register
20
20
 
21
- def initialize(name:, from_registers: [], to_register: DEFAULT_REGISTER)
21
+ def initialize(from_registers: [], name: '', to_register: DEFAULT_REGISTER)
22
22
  super(name: name)
23
23
 
24
24
  @from_registers = Array(from_registers)
@@ -19,9 +19,9 @@ module Burner
19
19
  attr_reader :key, :groups
20
20
 
21
21
  def initialize(
22
- name:,
23
22
  key:,
24
23
  config: Hashematics::Configuration.new,
24
+ name: '',
25
25
  register: DEFAULT_REGISTER
26
26
  )
27
27
  super(name: name, register: register)
@@ -16,6 +16,12 @@ module Burner
16
16
  # It is worth noting that the resulting hashes values are singular objects and not an array
17
17
  # like Ruby's Enumerable#group_by method.
18
18
  #
19
+ # If the insensitive option is set as true then each key's value will be coerced as
20
+ # a lowercase string. This can help provide two types of insensitivity: case and type
21
+ # insensitivity. This may be appropriate in some places but not others. If any other
22
+ # value coercion is needed then another option would be to first transform the records
23
+ # before grouping them.
24
+ #
19
25
  # An example of this specific job:
20
26
  #
21
27
  # input: [{ id: 1, code: 'a' }, { id: 2, code: 'b' }]
@@ -25,18 +31,22 @@ module Burner
25
31
  # Expected Payload[register] input: array of objects.
26
32
  # Payload[register] output: hash.
27
33
  class Group < JobWithRegister
28
- attr_reader :keys, :resolver
34
+ include Util::Keyable
35
+
36
+ attr_reader :insensitive, :keys, :resolver
29
37
 
30
38
  def initialize(
31
- name:,
39
+ insensitive: false,
32
40
  keys: [],
41
+ name: '',
33
42
  register: DEFAULT_REGISTER,
34
43
  separator: ''
35
44
  )
36
45
  super(name: name, register: register)
37
46
 
38
- @keys = Array(keys)
39
- @resolver = Objectable.resolver(separator: separator.to_s)
47
+ @insensitive = insensitive || false
48
+ @keys = Array(keys)
49
+ @resolver = Objectable.resolver(separator: separator.to_s)
40
50
 
41
51
  raise ArgumentError, 'at least one key is required' if @keys.empty?
42
52
 
@@ -50,18 +60,12 @@ module Burner
50
60
  output.detail("Grouping based on key(s): #{keys} for #{count} records(s)")
51
61
 
52
62
  grouped_records = payload[register].each_with_object({}) do |record, memo|
53
- key = make_key(record)
63
+ key = make_key(record, keys, resolver, insensitive)
54
64
  memo[key] = record
55
65
  end
56
66
 
57
67
  payload[register] = grouped_records
58
68
  end
59
-
60
- private
61
-
62
- def make_key(record)
63
- keys.map { |key| resolver.get(record, key) }
64
- end
65
69
  end
66
70
  end
67
71
  end
@@ -21,7 +21,7 @@ module Burner
21
21
  class NestedAggregate < JobWithRegister
22
22
  attr_reader :key, :key_mappings, :resolver
23
23
 
24
- def initialize(name:, key:, key_mappings: [], register: DEFAULT_REGISTER, separator: '')
24
+ def initialize(key:, key_mappings: [], name: '', register: DEFAULT_REGISTER, separator: '')
25
25
  super(name: name, register: register)
26
26
 
27
27
  raise ArgumentError, 'key is required' if key.to_s.empty?
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Library
12
+ module Collection
13
+ # This job can iterate over a set of records and sequence them (set the specified key to
14
+ # a sequential index value.)
15
+ #
16
+ # Expected Payload[register] input: array of objects.
17
+ # Payload[register] output: array of objects.
18
+ class Number < JobWithRegister
19
+ BLANK = ''
20
+ DEFAULT_KEY = 'number'
21
+ DEFAULT_START_AT = 1
22
+
23
+ attr_reader :key, :resolver, :start_at
24
+
25
+ def initialize(
26
+ key: DEFAULT_KEY,
27
+ name: BLANK,
28
+ register: Burner::DEFAULT_REGISTER,
29
+ separator: BLANK,
30
+ start_at: DEFAULT_START_AT
31
+ )
32
+ super(name: name, register: register)
33
+
34
+ @key = key.to_s
35
+ @resolver = Objectable.resolver(separator: separator)
36
+ @start_at = start_at.to_i
37
+
38
+ freeze
39
+ end
40
+
41
+ def perform(output, payload)
42
+ output.detail("Setting '#{key}' for each record with values starting at #{start_at}")
43
+
44
+ ensure_array(payload).each.with_index(start_at) do |record, index|
45
+ resolver.set(record, key, index)
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -58,7 +58,7 @@ module Burner
58
58
  # nested hashes then set separator to '.'. For more information, see the underlying
59
59
  # library that supports this dot-notation concept:
60
60
  # https://github.com/bluemarblepayroll/objectable
61
- def initialize(name:, mappings: [], register: DEFAULT_REGISTER, separator: '')
61
+ def initialize(mappings: [], name: '', register: DEFAULT_REGISTER, separator: '')
62
62
  super(name: name, register: register)
63
63
 
64
64
  @mappings = Modeling::KeyIndexMapping.array(mappings)
@@ -23,7 +23,7 @@ module Burner
23
23
 
24
24
  attr_reader :amount
25
25
 
26
- def initialize(name:, amount: DEFAULT_AMOUNT, register: DEFAULT_REGISTER)
26
+ def initialize(amount: DEFAULT_AMOUNT, name: '', register: DEFAULT_REGISTER)
27
27
  super(name: name, register: register)
28
28
 
29
29
  @amount = amount.to_i
@@ -28,9 +28,9 @@ module Burner
28
28
  :resolver
29
29
 
30
30
  def initialize(
31
- name:,
32
31
  attributes: [],
33
32
  exclusive: false,
33
+ name: '',
34
34
  register: DEFAULT_REGISTER,
35
35
  separator: BLANK
36
36
  )
@@ -20,7 +20,7 @@ module Burner
20
20
  attr_reader :unpivot
21
21
 
22
22
  def initialize(
23
- name:,
23
+ name: '',
24
24
  pivot_set: HashMath::Unpivot::PivotSet.new,
25
25
  register: DEFAULT_REGISTER
26
26
  )
@@ -29,10 +29,10 @@ module Burner
29
29
  :validations
30
30
 
31
31
  def initialize(
32
- name:,
33
32
  invalid_register: DEFAULT_INVALID_REGISTER,
34
33
  join_char: DEFAULT_JOIN_CHAR,
35
34
  message_key: DEFAULT_MESSAGE_KEY,
35
+ name: '',
36
36
  register: DEFAULT_REGISTER,
37
37
  separator: '',
38
38
  validations: []
@@ -19,7 +19,7 @@ module Burner
19
19
  class Values < JobWithRegister
20
20
  attr_reader :include_keys
21
21
 
22
- def initialize(name:, include_keys: false, register: DEFAULT_REGISTER)
22
+ def initialize(include_keys: false, name: '', register: DEFAULT_REGISTER)
23
23
  super(name: name, register: register)
24
24
 
25
25
  @include_keys = include_keys || false
@@ -25,9 +25,9 @@ module Burner
25
25
  attr_reader :base_register, :with_register
26
26
 
27
27
  def initialize(
28
- name:,
29
28
  with_register:,
30
29
  base_register: DEFAULT_REGISTER,
30
+ name: '',
31
31
  register: DEFAULT_REGISTER
32
32
  )
33
33
  super(name: name, register: register)
@@ -33,10 +33,10 @@ module Burner
33
33
  :resolver
34
34
 
35
35
  def initialize(
36
- name:,
37
36
  data_key: DEFAULT_DATA_KEY,
38
37
  ignore_blank_data: false,
39
38
  ignore_blank_path: false,
39
+ name: '',
40
40
  path_key: DEFAULT_PATH_KEY,
41
41
  register: DEFAULT_REGISTER,
42
42
  separator: ''
@@ -20,7 +20,7 @@ module Burner
20
20
  class Yaml < JobWithRegister
21
21
  attr_reader :safe
22
22
 
23
- def initialize(name:, register: DEFAULT_REGISTER, safe: true)
23
+ def initialize(name: '', register: DEFAULT_REGISTER, safe: true)
24
24
  super(name: name, register: register)
25
25
 
26
26
  @safe = safe
@@ -15,7 +15,7 @@ module Burner
15
15
  class Echo < Job
16
16
  attr_reader :message
17
17
 
18
- def initialize(name:, message: '')
18
+ def initialize(message: '', name: '')
19
19
  super(name: name)
20
20
 
21
21
  @message = message.to_s
@@ -17,7 +17,7 @@ module Burner
17
17
  class Exist < Job
18
18
  attr_reader :disk, :path, :short_circuit
19
19
 
20
- def initialize(name:, path:, disk: {}, short_circuit: false)
20
+ def initialize(path:, disk: {}, name: '', short_circuit: false)
21
21
  super(name: name)
22
22
 
23
23
  raise ArgumentError, 'path is required' if path.to_s.empty?
@@ -14,7 +14,7 @@ module Burner
14
14
  class OpenFileBase < JobWithRegister
15
15
  attr_reader :binary, :disk, :path
16
16
 
17
- def initialize(name:, path:, binary: false, disk: {}, register: DEFAULT_REGISTER)
17
+ def initialize(path:, binary: false, disk: {}, name: '', register: DEFAULT_REGISTER)
18
18
  super(name: name, register: register)
19
19
 
20
20
  raise ArgumentError, 'path is required' if path.to_s.empty?
@@ -35,12 +35,12 @@ module Burner
35
35
  :resolver
36
36
 
37
37
  def initialize(
38
- name:,
39
38
  binary: false,
40
39
  data_key: DEFAULT_DATA_KEY,
41
40
  disk: {},
42
41
  ignore_blank_path: false,
43
42
  ignore_file_not_found: false,
43
+ name: '',
44
44
  path_key: DEFAULT_PATH_KEY,
45
45
  register: DEFAULT_REGISTER,
46
46
  separator: ''
@@ -22,10 +22,10 @@ module Burner
22
22
  attr_reader :supress_side_effect
23
23
 
24
24
  def initialize(
25
- name:,
26
25
  path:,
27
26
  binary: false,
28
27
  disk: {},
28
+ name: '',
29
29
  register: DEFAULT_REGISTER,
30
30
  supress_side_effect: false
31
31
  )
@@ -18,7 +18,7 @@ module Burner
18
18
  class Csv < JobWithRegister
19
19
  attr_reader :byte_order_mark
20
20
 
21
- def initialize(name:, byte_order_mark: nil, register: DEFAULT_REGISTER)
21
+ def initialize(byte_order_mark: nil, name: '', register: DEFAULT_REGISTER)
22
22
  super(name: name, register: register)
23
23
 
24
24
  @byte_order_mark = Modeling::ByteOrderMark.resolve(byte_order_mark)
@@ -15,7 +15,7 @@ module Burner
15
15
  class Sleep < Job
16
16
  attr_reader :seconds
17
17
 
18
- def initialize(name:, seconds: 0)
18
+ def initialize(name: '', seconds: 0)
19
19
  super(name: name)
20
20
 
21
21
  @seconds = seconds.to_f
@@ -19,7 +19,7 @@ module Burner
19
19
  class Copy < Job
20
20
  attr_reader :from_register, :to_register
21
21
 
22
- def initialize(name:, to_register: DEFAULT_REGISTER, from_register: DEFAULT_REGISTER)
22
+ def initialize(from_register: DEFAULT_REGISTER, name: '', to_register: DEFAULT_REGISTER)
23
23
  super(name: name)
24
24
 
25
25
  @from_register = from_register.to_s
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Library
12
+ module Value
13
+ # This job will nest the current value within a new outer hash. The specified key
14
+ # passed in will be the corresponding new hash key entry for the existing value.
15
+ #
16
+ # Expected Payload[from_register] input: anything.
17
+ # Payload[to_register] output: hash.
18
+ class Nest < JobWithRegister
19
+ DEFAULT_KEY = 'key'
20
+
21
+ attr_reader :key
22
+
23
+ def initialize(key: DEFAULT_KEY, name: '', register: Burner::DEFAULT_REGISTER)
24
+ super(name: name, register: register)
25
+
26
+ @key = key.to_s
27
+
28
+ freeze
29
+ end
30
+
31
+ def perform(_output, payload)
32
+ payload[register] = { key => payload[register] }
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -17,7 +17,7 @@ module Burner
17
17
  class Static < JobWithRegister
18
18
  attr_reader :value
19
19
 
20
- def initialize(name:, register: DEFAULT_REGISTER, value: nil)
20
+ def initialize(name: '', register: DEFAULT_REGISTER, value: nil)
21
21
  super(name: name, register: register)
22
22
 
23
23
  @value = value
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Library
12
+ module Value
13
+ # Transform the current value of the register through a Realize::Pipeline. This will
14
+ # transform the entire value, as opposed to the b/collection/transform job, which will
15
+ # iterate over each row/record in a dataset and transform each row/record.
16
+ #
17
+ # Expected Payload[register] input: anything.
18
+ # Payload[register] output: anything.
19
+ class Transform < JobWithRegister
20
+ attr_reader :pipeline
21
+
22
+ def initialize(name: '', register: DEFAULT_REGISTER, separator: '', transformers: [])
23
+ super(name: name, register: register)
24
+
25
+ resolver = Objectable.resolver(separator: separator)
26
+
27
+ @pipeline = Realize::Pipeline.new(transformers, resolver: resolver)
28
+
29
+ freeze
30
+ end
31
+
32
+ def perform(_output, payload)
33
+ payload[register] = pipeline.transform(payload[register], payload.time)
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -7,7 +7,7 @@
7
7
  # LICENSE file in the root directory of this source tree.
8
8
  #
9
9
 
10
- require_relative 'jobs'
10
+ require_relative 'job_set'
11
11
  require_relative 'output'
12
12
  require_relative 'payload'
13
13
  require_relative 'step'
@@ -19,27 +19,13 @@ module Burner
19
19
  class Pipeline
20
20
  acts_as_hashable
21
21
 
22
- class JobNotFoundError < StandardError; end
23
- class DuplicateJobNameError < StandardError; end
24
-
25
22
  attr_reader :steps
26
23
 
27
24
  def initialize(jobs: [], steps: nil)
28
- jobs = Jobs.array(jobs)
29
-
30
- assert_unique_job_names(jobs)
31
-
32
- jobs_by_name = jobs.map { |job| [job.name, job] }.to_h
33
-
34
- step_names = steps ? Array(steps) : jobs_by_name.keys
35
-
36
- @steps = step_names.map do |step_name|
37
- job = jobs_by_name[step_name.to_s]
38
-
39
- raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
40
-
41
- Step.new(job)
42
- end
25
+ @steps = JobSet
26
+ .new(jobs)
27
+ .jobs(steps)
28
+ .map { |job| Step.new(job) }
43
29
  end
44
30
 
45
31
  # The main entry-point for kicking off a pipeline.
@@ -68,20 +54,6 @@ module Burner
68
54
 
69
55
  private
70
56
 
71
- def assert_unique_job_names(jobs)
72
- unique_job_names = Set.new
73
-
74
- jobs.each do |job|
75
- if unique_job_names.include?(job.name)
76
- raise DuplicateJobNameError, "job with name: #{job.name} already declared"
77
- end
78
-
79
- unique_job_names << job.name
80
- end
81
-
82
- nil
83
- end
84
-
85
57
  def output_params(params, output)
86
58
  if params.keys.any?
87
59
  output.write('Parameters:')
@@ -8,4 +8,5 @@
8
8
  #
9
9
 
10
10
  require_relative 'util/arrayable'
11
+ require_relative 'util/keyable'
11
12
  require_relative 'util/string_template'
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Util
12
+ # Provide helper methods for keys.
13
+ module Keyable
14
+ def make_key(record, keys, resolver, insensitive)
15
+ keys.map do |key|
16
+ value = resolver.get(record, key)
17
+
18
+ insensitive ? value.to_s.downcase : value
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -8,5 +8,5 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- VERSION = '1.6.0'
11
+ VERSION = '1.7.0-alpha'
12
12
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: burner
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.0
4
+ version: 1.7.0.pre.alpha
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matthew Ruggio
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-12-22 00:00:00.000000000 Z
11
+ date: 2021-01-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: acts_as_hashable
@@ -237,6 +237,7 @@ files:
237
237
  - lib/burner/disks.rb
238
238
  - lib/burner/disks/local.rb
239
239
  - lib/burner/job.rb
240
+ - lib/burner/job_set.rb
240
241
  - lib/burner/job_with_register.rb
241
242
  - lib/burner/jobs.rb
242
243
  - lib/burner/library.rb
@@ -246,6 +247,7 @@ files:
246
247
  - lib/burner/library/collection/graph.rb
247
248
  - lib/burner/library/collection/group.rb
248
249
  - lib/burner/library/collection/nested_aggregate.rb
250
+ - lib/burner/library/collection/number.rb
249
251
  - lib/burner/library/collection/objects_to_arrays.rb
250
252
  - lib/burner/library/collection/shift.rb
251
253
  - lib/burner/library/collection/transform.rb
@@ -269,7 +271,9 @@ files:
269
271
  - lib/burner/library/serialize/yaml.rb
270
272
  - lib/burner/library/sleep.rb
271
273
  - lib/burner/library/value/copy.rb
274
+ - lib/burner/library/value/nest.rb
272
275
  - lib/burner/library/value/static.rb
276
+ - lib/burner/library/value/transform.rb
273
277
  - lib/burner/modeling.rb
274
278
  - lib/burner/modeling/attribute.rb
275
279
  - lib/burner/modeling/attribute_renderer.rb
@@ -288,6 +292,7 @@ files:
288
292
  - lib/burner/step.rb
289
293
  - lib/burner/util.rb
290
294
  - lib/burner/util/arrayable.rb
295
+ - lib/burner/util/keyable.rb
291
296
  - lib/burner/util/string_template.rb
292
297
  - lib/burner/version.rb
293
298
  homepage: https://github.com/bluemarblepayroll/burner
@@ -310,9 +315,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
310
315
  version: '2.5'
311
316
  required_rubygems_version: !ruby/object:Gem::Requirement
312
317
  requirements:
313
- - - ">="
318
+ - - ">"
314
319
  - !ruby/object:Gem::Version
315
- version: '0'
320
+ version: 1.3.1
316
321
  requirements: []
317
322
  rubygems_version: 3.0.3
318
323
  signing_key: