burner 0.0.0 → 1.0.0.pre.alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -3
  3. data/README.md +390 -2
  4. data/burner.gemspec +4 -1
  5. data/exe/burner +21 -0
  6. data/lib/burner.rb +19 -4
  7. data/lib/burner/cli.rb +45 -0
  8. data/lib/burner/job.rb +37 -0
  9. data/lib/burner/jobs.rb +58 -0
  10. data/lib/burner/jobs/collection/arrays_to_objects.rb +43 -0
  11. data/lib/burner/jobs/collection/graph.rb +43 -0
  12. data/lib/burner/jobs/collection/objects_to_arrays.rb +54 -0
  13. data/lib/burner/jobs/collection/shift.rb +43 -0
  14. data/lib/burner/jobs/collection/transform.rb +64 -0
  15. data/lib/burner/jobs/collection/transform/attribute.rb +33 -0
  16. data/lib/burner/jobs/collection/transform/attribute_renderer.rb +36 -0
  17. data/lib/burner/jobs/collection/unpivot.rb +45 -0
  18. data/lib/burner/jobs/deserialize/csv.rb +28 -0
  19. data/lib/burner/jobs/deserialize/json.rb +23 -0
  20. data/lib/burner/jobs/deserialize/yaml.rb +41 -0
  21. data/lib/burner/jobs/dummy.rb +19 -0
  22. data/lib/burner/jobs/echo.rb +33 -0
  23. data/lib/burner/jobs/io/base.rb +27 -0
  24. data/lib/burner/jobs/io/exist.rb +43 -0
  25. data/lib/burner/jobs/io/read.rb +45 -0
  26. data/lib/burner/jobs/io/write.rb +67 -0
  27. data/lib/burner/jobs/serialize/csv.rb +38 -0
  28. data/lib/burner/jobs/serialize/json.rb +23 -0
  29. data/lib/burner/jobs/serialize/yaml.rb +23 -0
  30. data/lib/burner/jobs/set.rb +31 -0
  31. data/lib/burner/jobs/sleep.rb +33 -0
  32. data/lib/burner/modeling.rb +10 -0
  33. data/lib/burner/modeling/key_index_mapping.rb +29 -0
  34. data/lib/burner/output.rb +66 -0
  35. data/lib/burner/payload.rb +41 -0
  36. data/lib/burner/pipeline.rb +73 -0
  37. data/lib/burner/step.rb +45 -0
  38. data/lib/burner/string_template.rb +40 -0
  39. data/lib/burner/version.rb +1 -1
  40. data/lib/burner/written_file.rb +28 -0
  41. metadata +82 -5
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Serialize
13
+ # Treat value like a Ruby object and serialize it using JSON.
14
+ class Json < Job
15
+ def perform(_output, payload)
16
+ payload.value = payload.value.to_json
17
+
18
+ nil
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ module Serialize
13
+ # Treat value like a Ruby object and serialize it using YAML.
14
+ class Yaml < Job
15
+ def perform(_output, payload)
16
+ payload.value = payload.value.to_yaml
17
+
18
+ nil
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ # Arbitrarily set value
13
+ class Set < Job
14
+ attr_reader :value
15
+
16
+ def initialize(name:, value: nil)
17
+ super(name: name)
18
+
19
+ @value = value
20
+
21
+ freeze
22
+ end
23
+
24
+ def perform(_output, payload)
25
+ payload.value = value
26
+
27
+ nil
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ class Jobs
12
+ # Arbitrarily put thread to sleep for X number of seconds
13
+ class Sleep < Job
14
+ attr_reader :seconds
15
+
16
+ def initialize(name:, seconds: 0)
17
+ super(name: name)
18
+
19
+ @seconds = seconds.to_f
20
+
21
+ freeze
22
+ end
23
+
24
+ def perform(output, _payload)
25
+ output.detail("Going to sleep for #{seconds} second(s)")
26
+
27
+ Kernel.sleep(seconds)
28
+
29
+ nil
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'modeling/key_index_mapping'
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ module Modeling
12
+ # Generic relationship between a numeric index and a key.
13
+ class KeyIndexMapping
14
+ acts_as_hashable
15
+
16
+ attr_reader :index, :key
17
+
18
+ def initialize(index:, key:)
19
+ raise ArgumentError, 'index is required' if index.to_s.empty?
20
+ raise ArgumentError, 'key is required' if key.to_s.empty?
21
+
22
+ @index = index.to_i
23
+ @key = key.to_s
24
+
25
+ freeze
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,66 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ # A Pipeline execution can write main output, which is really an outline to whats happening,
12
+ # step-by-step. This is not meant to replace or be true logging, but serve more as a summary
13
+ # for each of the jobs. Each job can decide what it would like to include in this summary
14
+ # and leverage an instance of this class for inclusion of that information.
15
+ class Output
16
+ RULER_LENGTH = 80
17
+
18
+ attr_reader :id, :job_count, :outs
19
+
20
+ def initialize(id: SecureRandom.uuid, outs: [$stdout])
21
+ @id = id
22
+ @outs = Array(outs)
23
+ @job_count = 1
24
+ end
25
+
26
+ def ruler
27
+ write('-' * RULER_LENGTH)
28
+
29
+ self
30
+ end
31
+
32
+ def title(message)
33
+ write("[#{job_count}] #{message}")
34
+
35
+ @job_count += 1
36
+
37
+ self
38
+ end
39
+
40
+ def detail(message)
41
+ write(" - #{message}")
42
+
43
+ self
44
+ end
45
+
46
+ def write(message)
47
+ raw("[#{id} | #{Time.now.utc}] #{message}")
48
+
49
+ self
50
+ end
51
+
52
+ def complete(time_in_seconds)
53
+ detail("Completed in: #{time_in_seconds.round(3)} second(s)")
54
+
55
+ self
56
+ end
57
+
58
+ private
59
+
60
+ def raw(message)
61
+ outs.each { |out| out.puts(message) }
62
+
63
+ self
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'written_file'
11
+
12
+ module Burner
13
+ # The input for all Job#perform methods. The main notion of this object is its "value"
14
+ # attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
15
+ # methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
16
+ # library and lifecycle, but I am not sure there is any other way around it: trying to build
17
+ # a generic, open-ended object pipeline to serve almost any use case.
18
+ class Payload
19
+ attr_accessor :value
20
+
21
+ attr_reader :params,
22
+ :time,
23
+ :written_files
24
+
25
+ def initialize(
26
+ params: {},
27
+ time: Time.now.utc,
28
+ value: nil,
29
+ written_files: []
30
+ )
31
+ @params = params || {}
32
+ @time = time || Time.now.utc
33
+ @value = value
34
+ @written_files = written_files || []
35
+ end
36
+
37
+ def add_written_file(written_file)
38
+ tap { written_files << WrittenFile.make(written_file) }
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ require_relative 'jobs'
11
+ require_relative 'output'
12
+ require_relative 'payload'
13
+ require_relative 'step'
14
+
15
+ module Burner
16
+ # The root package. A Pipeline contains the job configurations along with the steps. The steps
17
+ # referens jobs and tell you the order of the jobs to run.
18
+ class Pipeline
19
+ acts_as_hashable
20
+
21
+ class JobNotFoundError < StandardError; end
22
+
23
+ attr_reader :steps
24
+
25
+ def initialize(jobs: [], steps: [])
26
+ jobs_by_name = Jobs.array(jobs).map { |job| [job.name, job] }.to_h
27
+
28
+ @steps = Array(steps).map do |step_name|
29
+ job = jobs_by_name[step_name.to_s]
30
+
31
+ raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
32
+
33
+ Step.new(job)
34
+ end
35
+ end
36
+
37
+ # The main entry-point for kicking off a pipeline.
38
+ def execute(output: Output.new, payload: Payload.new)
39
+ output.write("Pipeline started with #{steps.length} step(s)")
40
+
41
+ output_params(payload.params, output)
42
+ output.ruler
43
+
44
+ time_in_seconds = Benchmark.measure do
45
+ steps.each do |step|
46
+ return_value = step.perform(output, payload)
47
+
48
+ if return_value.is_a?(FalseClass)
49
+ output.detail('Job returned false, ending pipeline.')
50
+ break
51
+ end
52
+ end
53
+ end.real.round(3)
54
+
55
+ output.ruler
56
+ output.write("Pipeline ended, took #{time_in_seconds} second(s) to complete")
57
+
58
+ payload
59
+ end
60
+
61
+ private
62
+
63
+ def output_params(params, output)
64
+ if params.keys.any?
65
+ output.write('Parameters:')
66
+ else
67
+ output.write('No parameters passed in.')
68
+ end
69
+
70
+ params.each { |key, value| output.detail("#{key}: #{value}") }
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ # A wrapper to execute a job (in the context of a Pipeline.)
12
+ class Step
13
+ extend Forwardable
14
+
15
+ SEPARATOR = '::'
16
+
17
+ private_constant :SEPARATOR
18
+
19
+ attr_reader :job
20
+
21
+ def_delegators :job, :name
22
+
23
+ def initialize(job)
24
+ raise ArgumentError, 'job is required' unless job
25
+
26
+ @job = job
27
+
28
+ freeze
29
+ end
30
+
31
+ def perform(output, payload)
32
+ return_value = nil
33
+
34
+ output.title("#{job.class.name}#{SEPARATOR}#{job.name}")
35
+
36
+ time_in_seconds = Benchmark.measure do
37
+ return_value = job.perform(output, payload)
38
+ end.real.round(3)
39
+
40
+ output.complete(time_in_seconds)
41
+
42
+ return_value
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ # Can take in a string and an object and use the object for formatting string interpolations
12
+ # using tokens of form: {attribute_name}. This templating class does not understand nested
13
+ # structures, so input should be a flat object/hash in the form of key-value pairs. A benefit of
14
+ # using Objectable for resolution is that it can understand almost any type of
15
+ # object: Hash, Struct, OpenStruct, custom objects, etc.
16
+ # For more information see underlying libraries:
17
+ # * Stringento: https://github.com/bluemarblepayroll/stringento
18
+ # * Objectable: https://github.com/bluemarblepayroll/objectable
19
+ class StringTemplate
20
+ include Singleton
21
+
22
+ attr_reader :resolver
23
+
24
+ def initialize
25
+ @resolver = Objectable.resolver(separator: '')
26
+
27
+ freeze
28
+ end
29
+
30
+ # For general consumption
31
+ def evaluate(expression, input)
32
+ Stringento.evaluate(expression, input, resolver: self)
33
+ end
34
+
35
+ # For Stringento consumption
36
+ def resolve(value, input)
37
+ resolver.get(input, value)
38
+ end
39
+ end
40
+ end
@@ -8,5 +8,5 @@
8
8
  #
9
9
 
10
10
  module Burner
11
- VERSION = '0.0.0'
11
+ VERSION = '1.0.0-alpha.4'
12
12
  end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ #
4
+ # Copyright (c) 2020-present, Blue Marble Payroll, LLC
5
+ #
6
+ # This source code is licensed under the MIT license found in the
7
+ # LICENSE file in the root directory of this source tree.
8
+ #
9
+
10
+ module Burner
11
+ # Describes a file that was generated by a Job. If a Job emits a file, it should also add the
12
+ # file details to the Payload#written_files array using the Payload#add_written_file method.
13
+ class WrittenFile
14
+ acts_as_hashable
15
+
16
+ attr_reader :logical_filename,
17
+ :physical_filename,
18
+ :time_in_seconds
19
+
20
+ def initialize(logical_filename:, physical_filename:, time_in_seconds:)
21
+ @logical_filename = logical_filename.to_s
22
+ @physical_filename = physical_filename.to_s
23
+ @time_in_seconds = time_in_seconds.to_f
24
+
25
+ freeze
26
+ end
27
+ end
28
+ end