burner 1.0.0.pre.alpha.5 → 1.0.0.pre.alpha.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +2 -0
  3. data/README.md +52 -48
  4. data/burner.gemspec +1 -1
  5. data/exe/burner +2 -3
  6. data/lib/burner.rb +2 -0
  7. data/lib/burner/cli.rb +2 -0
  8. data/lib/burner/job.rb +27 -9
  9. data/lib/burner/job_with_register.rb +24 -0
  10. data/lib/burner/jobs.rb +28 -41
  11. data/lib/burner/library.rb +32 -0
  12. data/lib/burner/library/collection/arrays_to_objects.rb +75 -0
  13. data/lib/burner/{jobs → library}/collection/graph.rb +7 -8
  14. data/lib/burner/library/collection/objects_to_arrays.rb +88 -0
  15. data/lib/burner/{jobs → library}/collection/shift.rb +8 -9
  16. data/lib/burner/{jobs → library}/collection/transform.rb +17 -15
  17. data/lib/burner/{jobs → library}/collection/unpivot.rb +17 -9
  18. data/lib/burner/library/collection/validate.rb +89 -0
  19. data/lib/burner/{jobs → library}/collection/values.rb +9 -10
  20. data/lib/burner/{jobs → library}/deserialize/csv.rb +4 -5
  21. data/lib/burner/{jobs → library}/deserialize/json.rb +6 -5
  22. data/lib/burner/{jobs → library}/deserialize/yaml.rb +13 -7
  23. data/lib/burner/{jobs → library}/dummy.rb +4 -4
  24. data/lib/burner/{jobs → library}/echo.rb +3 -3
  25. data/lib/burner/{jobs → library}/io/base.rb +4 -4
  26. data/lib/burner/{jobs → library}/io/exist.rb +11 -9
  27. data/lib/burner/{jobs → library}/io/read.rb +7 -6
  28. data/lib/burner/{jobs → library}/io/write.rb +9 -6
  29. data/lib/burner/{jobs → library}/serialize/csv.rb +5 -6
  30. data/lib/burner/{jobs → library}/serialize/json.rb +6 -5
  31. data/lib/burner/{jobs → library}/serialize/yaml.rb +6 -5
  32. data/lib/burner/{jobs/set.rb → library/set_value.rb} +8 -7
  33. data/lib/burner/{jobs → library}/sleep.rb +3 -3
  34. data/lib/burner/modeling.rb +3 -0
  35. data/lib/burner/modeling/attribute.rb +29 -0
  36. data/lib/burner/modeling/attribute_renderer.rb +32 -0
  37. data/lib/burner/modeling/validations.rb +23 -0
  38. data/lib/burner/modeling/validations/base.rb +35 -0
  39. data/lib/burner/modeling/validations/blank.rb +31 -0
  40. data/lib/burner/modeling/validations/present.rb +31 -0
  41. data/lib/burner/payload.rb +52 -15
  42. data/lib/burner/pipeline.rb +23 -4
  43. data/lib/burner/side_effects.rb +10 -0
  44. data/lib/burner/side_effects/written_file.rb +28 -0
  45. data/lib/burner/step.rb +1 -5
  46. data/lib/burner/util.rb +11 -0
  47. data/lib/burner/util/arrayable.rb +30 -0
  48. data/lib/burner/util/string_template.rb +42 -0
  49. data/lib/burner/version.rb +1 -1
  50. metadata +40 -29
  51. data/lib/burner/jobs/collection/arrays_to_objects.rb +0 -43
  52. data/lib/burner/jobs/collection/objects_to_arrays.rb +0 -54
  53. data/lib/burner/jobs/collection/transform/attribute.rb +0 -33
  54. data/lib/burner/jobs/collection/transform/attribute_renderer.rb +0 -36
  55. data/lib/burner/string_template.rb +0 -40
  56. data/lib/burner/written_file.rb +0 -28
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 57b5b4290b72962e10ce7ea7244a1dcfb43894cff1d72ef4e0684de4cdea4a35
4
- data.tar.gz: 2936c408e7ffa3e1e9883510d890329870d5c1a4c129fef1fb283103568c9508
3
+ metadata.gz: 070f757d857aa8f1e07dd5cdaea5bdd261b050ef343c5f16f2692c6a5ba85eb1
4
+ data.tar.gz: 719a284fde5030e3e92fbaebd8c1f91f4445703710b6c00c9e2acf68c108a853
5
5
  SHA512:
6
- metadata.gz: f514870aa2b12cc4fc34f3952cabf8c738985dac1a6602c7f41aec3f28b83738991d5cf82d429b62d76b0f3a96b85907e51d0e06db0ce33579d1dc74dc55409e
7
- data.tar.gz: 2e9ee10f91bb28eb4d79091ae7106f6da640daaf6b5ac2f77e828e81743dfd55d6baca12a0e376c3be709483fe7a94e0f840f47ed2ecb5031233f4ff9ae7db3a
6
+ metadata.gz: bd8938f87808a5c71a64839bc4d6cbc05bdfbb3f8df98a4aba92bbbb1a6163527ff5ba702525ad972a952faf05e35eda39adafe916540b94c775e2a96e71fcc0
7
+ data.tar.gz: debd08acef4aa54f88d796a0e217a83477af63d3a82bb7f07123d2360d8b40787e621831ea9d0d58cd39b088684ac6219f23c5aed49d274e39be360cdb16af1a
@@ -31,3 +31,5 @@ Style/TrailingCommaInHashLiteral:
31
31
  Style/TrailingCommaInArrayLiteral:
32
32
  Enabled: false
33
33
 
34
+ Metrics/ParameterLists:
35
+ CountKeywordArgs: false
data/README.md CHANGED
@@ -31,30 +31,30 @@ pipeline = {
31
31
  jobs: [
32
32
  {
33
33
  name: :read,
34
- type: 'io/read',
34
+ type: 'b/io/read',
35
35
  path: '{input_file}'
36
36
  },
37
37
  {
38
38
  name: :output_id,
39
- type: :echo,
39
+ type: 'b/echo',
40
40
  message: 'The job id is: {__id}'
41
41
  },
42
42
  {
43
43
  name: :output_value,
44
- type: :echo,
44
+ type: 'b/echo',
45
45
  message: 'The current value is: {__value}'
46
46
  },
47
47
  {
48
48
  name: :parse,
49
- type: 'deserialize/json'
49
+ type: 'b/deserialize/json'
50
50
  },
51
51
  {
52
52
  name: :convert,
53
- type: 'serialize/yaml'
53
+ type: 'b/serialize/yaml'
54
54
  },
55
55
  {
56
56
  name: :write,
57
- type: 'io/write',
57
+ type: 'b/io/write',
58
58
  path: '{output_file}'
59
59
  }
60
60
  ],
@@ -133,23 +133,23 @@ The value of `log` should now look similar to:
133
133
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - input_file: input.json
134
134
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - output_file: output.yaml
135
135
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] --------------------------------------------------------------------------------
136
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [1] Burner::Jobs::IO::Read::read
136
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [1] Burner::Library::IO::Read::read
137
137
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Reading: spec/fixtures/input.json
138
138
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
139
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [2] Burner::Jobs::Echo::output_id
139
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [2] Burner::Library::Echo::output_id
140
140
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - The job id is:
141
141
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
142
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [3] Burner::Jobs::Echo::output_value
142
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [3] Burner::Library::Echo::output_value
143
143
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - The current value is:
144
144
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
145
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [4] Burner::Jobs::Deserialize::Json::parse
145
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [4] Burner::Library::Deserialize::Json::parse
146
146
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
147
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [5] Burner::Jobs::Serialize::Yaml::convert
147
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [5] Burner::Library::Serialize::Yaml::convert
148
148
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
149
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [6] Burner::Jobs::Echo::output_value
149
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [6] Burner::Library::Echo::output_value
150
150
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - The current value is:
151
151
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
152
- [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [7] Burner::Jobs::IO::Write::write
152
+ [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] [7] Burner::Library::IO::Write::write
153
153
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Writing: output.yaml
154
154
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] - Completed in: 0.0 second(s)
155
155
  [8bdc394e-7047-4a1a-87ed-6c54ed690ed5 | 2020-10-14 13:49:59 UTC] --------------------------------------------------------------------------------
@@ -163,7 +163,7 @@ Notes:
163
163
 
164
164
  ### Command Line Pipeline Processing
165
165
 
166
- This library also ships with a built-in script `exe/burner` that illustrates using the `Burner::Cli` API. This class can take in an array of arguments (similar to a command-line) and execute a pipeline. The first argument is the path to a YAML file with the pipeline's configuration and each subsequent argument is a param in `key=value` form. Here is how the json-to-yaml example can utilize this interface:
166
+ This library also ships with a built-in script `burner` that illustrates using the `Burner::Cli` API. This class can take in an array of arguments (similar to a command-line) and execute a pipeline. The first argument is the path to a YAML file with the pipeline's configuration and each subsequent argument is a param in `key=value` form. Here is how the json-to-yaml example can utilize this interface:
167
167
 
168
168
  #### Create YAML Pipeline Configuration File
169
169
 
@@ -172,25 +172,25 @@ Write the following json_to_yaml_pipeline.yaml file to disk:
172
172
  ````yaml
173
173
  jobs:
174
174
  - name: read
175
- type: io/read
175
+ type: b/io/read
176
176
  path: '{input_file}'
177
177
 
178
178
  - name: output_id
179
- type: echo
179
+ type: b/echo
180
180
  message: 'The job id is: {__id}'
181
181
 
182
182
  - name: output_value
183
- type: echo
183
+ type: b/echo
184
184
  message: 'The current value is: {__value}'
185
185
 
186
186
  - name: parse
187
- type: deserialize/json
187
+ type: b/deserialize/json
188
188
 
189
189
  - name: convert
190
- type: serialize/yaml
190
+ type: b/serialize/yaml
191
191
 
192
192
  - name: write
193
- type: io/write
193
+ type: b/io/write
194
194
  path: '{output_file}'
195
195
 
196
196
  steps:
@@ -233,50 +233,54 @@ This library only ships with very basic, rudimentary jobs that are meant to just
233
233
 
234
234
  #### Collection
235
235
 
236
- * **collection/arrays_to_objects** [mappings]: Convert an array of arrays to an array of objects.
237
- * **collection/graph** [config, key]: Use (Hashematics)[https://github.com/bluemarblepayroll/hashematics] to turn a flat array of objects into a deeply nested object tree.
238
- * **collection/objects_to_arrays** [mappings]: Convert an array of objects to an array of arrays.
239
- * **collection/shift** [amount]: Remove the first N number of elements from an array.
240
- * **collection/transform** [attributes, exclusive, separator]: Iterate over all objects and transform each key per the attribute transformers specifications. If exclusive is set to false then the current object will be overridden/merged. Separator can also be set for key path support. This job uses (Realize)[https://github.com/bluemarblepayroll/realize], which provides its own extendable value-transformation pipeline.
241
- * **collection/unpivot** [pivot_set]: Take an array of objects and unpivot specific sets of keys into rows. Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
242
- * **collection/values** [include_keys]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
236
+ * **b/collection/arrays_to_objects** [mappings, register]: Convert an array of arrays to an array of objects.
237
+ * **b/collection/graph** [config, key, register]: Use [Hashematics](https://github.com/bluemarblepayroll/hashematics) to turn a flat array of objects into a deeply nested object tree.
238
+ * **b/collection/objects_to_arrays** [mappings, register]: Convert an array of objects to an array of arrays.
239
+ * **b/collection/shift** [amount, register]: Remove the first N number of elements from an array.
240
+ * **b/collection/transform** [attributes, exclusive, separator, register]: Iterate over all objects and transform each key per the attribute transformers specifications. If exclusive is set to false then the current object will be overridden/merged. Separator can also be set for key path support. This job uses [Realize](https://github.com/bluemarblepayroll/realize), which provides its own extendable value-transformation pipeline.
241
+ * **b/collection/unpivot** [pivot_set, register]: Take an array of objects and unpivot specific sets of keys into rows. Under the hood it uses [HashMath's Unpivot class](https://github.com/bluemarblepayroll/hash_math#unpivot-hash-key-coalescence-and-row-extrapolation).
242
+ * **b/collection/validate** [invalid_register, join_char, message_key, register, separator, validations]: Take an array of objects, run it through each declared validator, and split the objects into two registers. The valid objects will be split into the current register while the invalid ones will go into the invalid_register as declared. Optional arguments, join_char and message_key, help determine the compiled error messages. The separator option can be utilized to use dot-notation for validating keys. See each validation's options by viewing their classes within the `lib/modeling/validations` directory.
243
+ * **b/collection/values** [include_keys, register]: Take an array of objects and call `#values` on each object. If include_keys is true (it is false by default), then call `#keys` on the first object and inject that as a "header" object.
243
244
 
244
245
  #### De-serialization
245
246
 
246
- * **deserialize/csv** []: Take a CSV string and de-serialize into object(s). Currently it will return an array of arrays, with each nested array representing one row.
247
- * **deserialize/json** []: Treat input as a string and de-serialize it to JSON.
248
- * **deserialize/yaml** [safe]: Treat input as a string and de-serialize it to YAML. By default it will try and (safely de-serialize)[https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load] it (only using core classes). If you wish to de-serialize it to any class type, pass in `safe: false`
247
+ * **b/deserialize/csv** [register]: Take a CSV string and de-serialize into object(s). Currently it will return an array of arrays, with each nested array representing one row.
248
+ * **b/deserialize/json** [register]: Treat input as a string and de-serialize it to JSON.
249
+ * **b/deserialize/yaml** [register, safe]: Treat input as a string and de-serialize it to YAML. By default it will try and [safely de-serialize](https://ruby-doc.org/stdlib-2.6.1/libdoc/psych/rdoc/Psych.html#method-c-safe_load) it (only using core classes). If you wish to de-serialize it to any class type, pass in `safe: false`
249
250
 
250
251
  #### IO
251
252
 
252
- * **io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`. If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
253
- * **io/read** [binary, path]: Read in a local file. The path parameter can be interpolated using `Payload#params`. If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
254
- * **io/write** [binary, path]: Write to a local file. The path parameter can be interpolated using `Payload#params`. If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
253
+ * **b/io/exist** [path, short_circuit]: Check to see if a file exists. The path parameter can be interpolated using `Payload#params`. If short_circuit was set to true (defaults to false) and the file does not exist then the pipeline will be short-circuited.
254
+ * **b/io/read** [binary, path, register]: Read in a local file. The path parameter can be interpolated using `Payload#params`. If the contents are binary, pass in `binary: true` to open it up in binary+read mode.
255
+ * **b/io/write** [binary, path, register]: Write to a local file. The path parameter can be interpolated using `Payload#params`. If the contents are binary, pass in `binary: true` to open it up in binary+write mode.
255
256
 
256
257
  #### Serialization
257
258
 
258
- * **serialize/csv** []: Take an array of arrays and create a CSV.
259
- * **serialize/json** []: Convert value to JSON.
260
- * **serialize/yaml** []: Convert value to YAML.
259
+ * **b/serialize/csv** [register]: Take an array of arrays and create a CSV.
260
+ * **b/serialize/json** [register]: Convert value to JSON.
261
+ * **b/serialize/yaml** [register]: Convert value to YAML.
261
262
 
262
263
  #### General
263
264
 
264
- * **dummy** []: Do nothing
265
- * **echo** [message]: Write a message to the output. The message parameter can be interpolated using `Payload#params`.
266
- * **set** [value]: Set the value to any arbitrary value.
267
- * **sleep** [seconds]: Sleep the thread for X number of seconds.
265
+ * **b/dummy** []: Do nothing
266
+ * **b/echo** [message]: Write a message to the output. The message parameter can be interpolated using `Payload#params`.
267
+ * **b/set** [register, value]: Set the value to any arbitrary value.
268
+ * **b/sleep** [seconds]: Sleep the thread for X number of seconds.
268
269
 
270
+ Notes:
271
+
272
+ * If you see that a job accepts a 'register' attribute/argument, that indicates a job will access and/or mutate the payload. The register indicates which part of the payload the job will interact with. This allows jobs to be placed into 'lanes'. If register is not specified, then the default register is used.
269
273
 
270
274
  ### Adding & Registering Jobs
271
275
 
272
- Where this library shines is when additional jobs are plugged in. Burner uses its `Burner::Jobs` class as its class-level registry built with (acts_as_hashable)[https://github.com/bluemarblepayroll/acts_as_hashable]'s acts_as_hashable_factory directive.
276
+ Where this library shines is when additional jobs are plugged in. Burner uses its `Burner::Jobs` class as its class-level registry built with [acts_as_hashable](https://github.com/bluemarblepayroll/acts_as_hashable)'s acts_as_hashable_factory directive.
273
277
 
274
278
  Let's say we would like to register a job to parse a CSV:
275
279
 
276
280
  ````ruby
277
- class ParseCsv < Burner::Job
281
+ class ParseCsv < Burner::JobWithRegister
278
282
  def perform(output, payload)
279
- payload.value = CSV.parse(payload.value, headers: true).map(&:to_h)
283
+ payload[register] = CSV.parse(payload[register], headers: true).map(&:to_h)
280
284
 
281
285
  nil
282
286
  end
@@ -292,17 +296,17 @@ pipeline = {
292
296
  jobs: [
293
297
  {
294
298
  name: :read,
295
- type: 'io/read',
299
+ type: 'b/io/read',
296
300
  path: '{input_file}'
297
301
  },
298
302
  {
299
303
  name: :output_id,
300
- type: :echo,
304
+ type: 'b/echo',
301
305
  message: 'The job id is: {__id}'
302
306
  },
303
307
  {
304
308
  name: :output_value,
305
- type: :echo,
309
+ type: 'b/echo',
306
310
  message: 'The current value is: {__value}'
307
311
  },
308
312
  {
@@ -311,11 +315,11 @@ pipeline = {
311
315
  },
312
316
  {
313
317
  name: :convert,
314
- type: 'serialize/yaml'
318
+ type: 'b/serialize/yaml'
315
319
  },
316
320
  {
317
321
  name: :write,
318
- type: 'io/write',
322
+ type: 'b/io/write',
319
323
  path: '{output_file}'
320
324
  }
321
325
  ],
@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
8
8
  s.summary = 'Declarative and extendable processing pipeline'
9
9
 
10
10
  s.description = <<-DESCRIPTION
11
- This library serves as the skeleton for a processing engine. It allows you to organize your code into Jobs, then stitch those jobs together as steps.
11
+ This library serves as the backbone for a configurable processing engine. It allows you to organize your code into jobs, then stitch those jobs together as steps.
12
12
  DESCRIPTION
13
13
 
14
14
  s.authors = ['Matthew Ruggio']
data/exe/burner CHANGED
@@ -10,11 +10,10 @@
10
10
 
11
11
  require 'bundler/setup'
12
12
  require 'burner'
13
- require 'pry'
14
13
 
15
14
  if ARGV.empty?
16
- puts 'Usage: ./exe/burner package.yaml key=value key=value ...'
17
- exit
15
+ warn('Usage: ./exe/burner package.yaml key=value key=value ...')
16
+ exit 2 # Do not return 1, that is reserved for hard errors.
18
17
  end
19
18
 
20
19
  # This should return exit code of 1 if it raises any hard errors.
@@ -24,6 +24,8 @@ require 'yaml'
24
24
 
25
25
  # Common/Shared
26
26
  require_relative 'burner/modeling'
27
+ require_relative 'burner/side_effects'
28
+ require_relative 'burner/util'
27
29
 
28
30
  # Main Entrypoint(s)
29
31
  require_relative 'burner/cli'
@@ -20,6 +20,8 @@ module Burner
20
20
  config = read_yaml(path)
21
21
  @pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
22
22
  @payload = Payload.new(params: params)
23
+
24
+ freeze
23
25
  end
24
26
 
25
27
  def execute
@@ -7,31 +7,49 @@
7
7
  # LICENSE file in the root directory of this source tree.
8
8
  #
9
9
 
10
- require_relative 'string_template'
11
-
12
10
  module Burner
13
11
  # Abstract base class for all job subclasses. The only public method a subclass needs to
14
- # implement #perform(params, payload, reporter) and then you can register it for use using
12
+ # implement #perform(output, payload) and then you can register it for use using
15
13
  # the Burner::Jobs factory class method #register. An example of a registration:
16
14
  # Burner::Jobs.register('your_class', YourClass)
17
15
  class Job
16
+ include Util::Arrayable
18
17
  acts_as_hashable
19
18
 
20
- attr_reader :name, :string_template
19
+ attr_reader :name
21
20
 
22
21
  def initialize(name:)
23
22
  raise ArgumentError, 'name is required' if name.to_s.empty?
24
23
 
25
- @name = name.to_s
26
- @string_template = StringTemplate.instance
24
+ @name = name.to_s
25
+ end
26
+
27
+ # There are only a few requirements to be considered a valid Burner Job:
28
+ # 1. The class responds to #name
29
+ # 2. The class responds to #perform(output, payload)
30
+ #
31
+ # The #perform method takes in two arguments: output (an instance of Burner::Output)
32
+ # and payload (an instance of Burner::Payload). Jobs can leverage output to emit
33
+ # information to the pipeline's log(s). The payload is utilized to pass data from job to job,
34
+ # with its most important attribute being #value. The value attribute is mutable
35
+ # per the individual job's context (meaning of it is unknown without understanding a job's
36
+ # input and output value of #value.). Therefore #value can mean anything and it is up to the
37
+ # engineers to clearly document the assumptions of its use.
38
+ #
39
+ # Returning false will short-circuit the pipeline right after the job method exits.
40
+ # Returning anything else besides false just means "continue".
41
+ def perform(output, _payload)
42
+ output.detail("#perform not implemented for: #{self.class.name}")
43
+
44
+ nil
27
45
  end
28
46
 
29
- private
47
+ protected
30
48
 
31
49
  def job_string_template(expression, output, payload)
32
- templatable_params = payload.params.merge(__id: output.id, __value: payload.value)
50
+ templatable_params = payload.params.merge(__id: output.id, __value: payload[''])
33
51
 
34
- string_template.evaluate(expression, templatable_params)
52
+ Util::StringTemplate.instance.evaluate(expression, templatable_params)
35
53
  end
36
54
  end
37
55
  end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'job'
11
+
12
+ module Burner
13
+ # Add on a register attribute to the configuration for a job. This indicates that a job
14
+ # either accesses and/or mutates the payload's registers.
15
+ class JobWithRegister < Job
16
+ attr_reader :register
17
+
18
+ def initialize(name:, register: '')
19
+ super(name: name)
20
+
21
+ @register = register.to_s
22
+ end
23
+ end
24
+ end
@@ -7,27 +7,7 @@
7
7
  # LICENSE file in the root directory of this source tree.
8
8
  #
9
9
 
10
- require_relative 'job'
11
- require_relative 'jobs/collection/arrays_to_objects'
12
- require_relative 'jobs/collection/graph'
13
- require_relative 'jobs/collection/objects_to_arrays'
14
- require_relative 'jobs/collection/shift'
15
- require_relative 'jobs/collection/transform'
16
- require_relative 'jobs/collection/unpivot'
17
- require_relative 'jobs/collection/values'
18
- require_relative 'jobs/deserialize/csv'
19
- require_relative 'jobs/deserialize/json'
20
- require_relative 'jobs/deserialize/yaml'
21
- require_relative 'jobs/dummy'
22
- require_relative 'jobs/echo'
23
- require_relative 'jobs/io/exist'
24
- require_relative 'jobs/io/read'
25
- require_relative 'jobs/io/write'
26
- require_relative 'jobs/serialize/csv'
27
- require_relative 'jobs/serialize/json'
28
- require_relative 'jobs/serialize/yaml'
29
- require_relative 'jobs/set'
30
- require_relative 'jobs/sleep'
10
+ require_relative 'library'
31
11
 
32
12
  module Burner
33
13
  # Main library of jobs. This file contains all the basic/default jobs. All other consumer
@@ -36,25 +16,32 @@ module Burner
36
16
  class Jobs
37
17
  acts_as_hashable_factory
38
18
 
39
- register 'collection/arrays_to_objects', Collection::ArraysToObjects
40
- register 'collection/graph', Collection::Graph
41
- register 'collection/objects_to_arrays', Collection::ObjectsToArrays
42
- register 'collection/shift', Collection::Shift
43
- register 'collection/transform', Collection::Transform
44
- register 'collection/unpivot', Collection::Unpivot
45
- register 'collection/values', Collection::Values
46
- register 'deserialize/csv', Deserialize::Csv
47
- register 'deserialize/json', Deserialize::Json
48
- register 'deserialize/yaml', Deserialize::Yaml
49
- register 'dummy', '', Dummy
50
- register 'echo', Echo
51
- register 'io/exist', IO::Exist
52
- register 'io/read', IO::Read
53
- register 'io/write', IO::Write
54
- register 'serialize/csv', Serialize::Csv
55
- register 'serialize/json', Serialize::Json
56
- register 'serialize/yaml', Serialize::Yaml
57
- register 'set', Set
58
- register 'sleep', Sleep
19
+ # Dummy is the default as noted by the ''. This means if a type is omitted, nil, or blank
20
+ # string then the dummy job will be used.
21
+ register 'b/dummy', '', Library::Dummy
22
+ register 'b/echo', Library::Echo
23
+ register 'b/set_value', Library::SetValue
24
+ register 'b/sleep', Library::Sleep
25
+
26
+ register 'b/collection/arrays_to_objects', Library::Collection::ArraysToObjects
27
+ register 'b/collection/graph', Library::Collection::Graph
28
+ register 'b/collection/objects_to_arrays', Library::Collection::ObjectsToArrays
29
+ register 'b/collection/shift', Library::Collection::Shift
30
+ register 'b/collection/transform', Library::Collection::Transform
31
+ register 'b/collection/unpivot', Library::Collection::Unpivot
32
+ register 'b/collection/values', Library::Collection::Values
33
+ register 'b/collection/validate', Library::Collection::Validate
34
+
35
+ register 'b/deserialize/csv', Library::Deserialize::Csv
36
+ register 'b/deserialize/json', Library::Deserialize::Json
37
+ register 'b/deserialize/yaml', Library::Deserialize::Yaml
38
+
39
+ register 'b/io/exist', Library::IO::Exist
40
+ register 'b/io/read', Library::IO::Read
41
+ register 'b/io/write', Library::IO::Write
42
+
43
+ register 'b/serialize/csv', Library::Serialize::Csv
44
+ register 'b/serialize/json', Library::Serialize::Json
45
+ register 'b/serialize/yaml', Library::Serialize::Yaml
59
46
  end
60
47
  end