burner 0.0.0 → 1.0.0.pre.alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -3
- data/README.md +390 -2
- data/burner.gemspec +4 -1
- data/exe/burner +21 -0
- data/lib/burner.rb +19 -4
- data/lib/burner/cli.rb +45 -0
- data/lib/burner/job.rb +37 -0
- data/lib/burner/jobs.rb +58 -0
- data/lib/burner/jobs/collection/arrays_to_objects.rb +43 -0
- data/lib/burner/jobs/collection/graph.rb +43 -0
- data/lib/burner/jobs/collection/objects_to_arrays.rb +54 -0
- data/lib/burner/jobs/collection/shift.rb +43 -0
- data/lib/burner/jobs/collection/transform.rb +64 -0
- data/lib/burner/jobs/collection/transform/attribute.rb +33 -0
- data/lib/burner/jobs/collection/transform/attribute_renderer.rb +36 -0
- data/lib/burner/jobs/collection/unpivot.rb +45 -0
- data/lib/burner/jobs/deserialize/csv.rb +28 -0
- data/lib/burner/jobs/deserialize/json.rb +23 -0
- data/lib/burner/jobs/deserialize/yaml.rb +41 -0
- data/lib/burner/jobs/dummy.rb +19 -0
- data/lib/burner/jobs/echo.rb +33 -0
- data/lib/burner/jobs/io/base.rb +27 -0
- data/lib/burner/jobs/io/exist.rb +43 -0
- data/lib/burner/jobs/io/read.rb +45 -0
- data/lib/burner/jobs/io/write.rb +67 -0
- data/lib/burner/jobs/serialize/csv.rb +38 -0
- data/lib/burner/jobs/serialize/json.rb +23 -0
- data/lib/burner/jobs/serialize/yaml.rb +23 -0
- data/lib/burner/jobs/set.rb +31 -0
- data/lib/burner/jobs/sleep.rb +33 -0
- data/lib/burner/modeling.rb +10 -0
- data/lib/burner/modeling/key_index_mapping.rb +29 -0
- data/lib/burner/output.rb +66 -0
- data/lib/burner/payload.rb +41 -0
- data/lib/burner/pipeline.rb +73 -0
- data/lib/burner/step.rb +45 -0
- data/lib/burner/string_template.rb +40 -0
- data/lib/burner/version.rb +1 -1
- data/lib/burner/written_file.rb +28 -0
- metadata +82 -5
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Treat value like a Ruby object and serialize it using JSON.
|
14
|
+
class Json < Job
|
15
|
+
def perform(_output, payload)
|
16
|
+
payload.value = payload.value.to_json
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Treat value like a Ruby object and serialize it using YAML.
|
14
|
+
class Yaml < Job
|
15
|
+
def perform(_output, payload)
|
16
|
+
payload.value = payload.value.to_yaml
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Arbitrarily set value
|
13
|
+
class Set < Job
|
14
|
+
attr_reader :value
|
15
|
+
|
16
|
+
def initialize(name:, value: nil)
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@value = value
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(_output, payload)
|
25
|
+
payload.value = value
|
26
|
+
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Arbitrarily put thread to sleep for X number of seconds
|
13
|
+
class Sleep < Job
|
14
|
+
attr_reader :seconds
|
15
|
+
|
16
|
+
def initialize(name:, seconds: 0)
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@seconds = seconds.to_f
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(output, _payload)
|
25
|
+
output.detail("Going to sleep for #{seconds} second(s)")
|
26
|
+
|
27
|
+
Kernel.sleep(seconds)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'modeling/key_index_mapping'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
module Modeling
|
12
|
+
# Generic relationship between a numeric index and a key.
|
13
|
+
class KeyIndexMapping
|
14
|
+
acts_as_hashable
|
15
|
+
|
16
|
+
attr_reader :index, :key
|
17
|
+
|
18
|
+
def initialize(index:, key:)
|
19
|
+
raise ArgumentError, 'index is required' if index.to_s.empty?
|
20
|
+
raise ArgumentError, 'key is required' if key.to_s.empty?
|
21
|
+
|
22
|
+
@index = index.to_i
|
23
|
+
@key = key.to_s
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# A Pipeline execution can write main output, which is really an outline to whats happening,
|
12
|
+
# step-by-step. This is not meant to replace or be true logging, but serve more as a summary
|
13
|
+
# for each of the jobs. Each job can decide what it would like to include in this summary
|
14
|
+
# and leverage an instance of this class for inclusion of that information.
|
15
|
+
class Output
|
16
|
+
RULER_LENGTH = 80
|
17
|
+
|
18
|
+
attr_reader :id, :job_count, :outs
|
19
|
+
|
20
|
+
def initialize(id: SecureRandom.uuid, outs: [$stdout])
|
21
|
+
@id = id
|
22
|
+
@outs = Array(outs)
|
23
|
+
@job_count = 1
|
24
|
+
end
|
25
|
+
|
26
|
+
def ruler
|
27
|
+
write('-' * RULER_LENGTH)
|
28
|
+
|
29
|
+
self
|
30
|
+
end
|
31
|
+
|
32
|
+
def title(message)
|
33
|
+
write("[#{job_count}] #{message}")
|
34
|
+
|
35
|
+
@job_count += 1
|
36
|
+
|
37
|
+
self
|
38
|
+
end
|
39
|
+
|
40
|
+
def detail(message)
|
41
|
+
write(" - #{message}")
|
42
|
+
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
def write(message)
|
47
|
+
raw("[#{id} | #{Time.now.utc}] #{message}")
|
48
|
+
|
49
|
+
self
|
50
|
+
end
|
51
|
+
|
52
|
+
def complete(time_in_seconds)
|
53
|
+
detail("Completed in: #{time_in_seconds.round(3)} second(s)")
|
54
|
+
|
55
|
+
self
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def raw(message)
|
61
|
+
outs.each { |out| out.puts(message) }
|
62
|
+
|
63
|
+
self
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'written_file'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# The input for all Job#perform methods. The main notion of this object is its "value"
|
14
|
+
# attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
|
15
|
+
# methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
|
16
|
+
# library and lifecycle, but I am not sure there is any other way around it: trying to build
|
17
|
+
# a generic, open-ended object pipeline to serve almost any use case.
|
18
|
+
class Payload
|
19
|
+
attr_accessor :value
|
20
|
+
|
21
|
+
attr_reader :params,
|
22
|
+
:time,
|
23
|
+
:written_files
|
24
|
+
|
25
|
+
def initialize(
|
26
|
+
params: {},
|
27
|
+
time: Time.now.utc,
|
28
|
+
value: nil,
|
29
|
+
written_files: []
|
30
|
+
)
|
31
|
+
@params = params || {}
|
32
|
+
@time = time || Time.now.utc
|
33
|
+
@value = value
|
34
|
+
@written_files = written_files || []
|
35
|
+
end
|
36
|
+
|
37
|
+
def add_written_file(written_file)
|
38
|
+
tap { written_files << WrittenFile.make(written_file) }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'jobs'
|
11
|
+
require_relative 'output'
|
12
|
+
require_relative 'payload'
|
13
|
+
require_relative 'step'
|
14
|
+
|
15
|
+
module Burner
|
16
|
+
# The root package. A Pipeline contains the job configurations along with the steps. The steps
|
17
|
+
# referens jobs and tell you the order of the jobs to run.
|
18
|
+
class Pipeline
|
19
|
+
acts_as_hashable
|
20
|
+
|
21
|
+
class JobNotFoundError < StandardError; end
|
22
|
+
|
23
|
+
attr_reader :steps
|
24
|
+
|
25
|
+
def initialize(jobs: [], steps: [])
|
26
|
+
jobs_by_name = Jobs.array(jobs).map { |job| [job.name, job] }.to_h
|
27
|
+
|
28
|
+
@steps = Array(steps).map do |step_name|
|
29
|
+
job = jobs_by_name[step_name.to_s]
|
30
|
+
|
31
|
+
raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
|
32
|
+
|
33
|
+
Step.new(job)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# The main entry-point for kicking off a pipeline.
|
38
|
+
def execute(output: Output.new, payload: Payload.new)
|
39
|
+
output.write("Pipeline started with #{steps.length} step(s)")
|
40
|
+
|
41
|
+
output_params(payload.params, output)
|
42
|
+
output.ruler
|
43
|
+
|
44
|
+
time_in_seconds = Benchmark.measure do
|
45
|
+
steps.each do |step|
|
46
|
+
return_value = step.perform(output, payload)
|
47
|
+
|
48
|
+
if return_value.is_a?(FalseClass)
|
49
|
+
output.detail('Job returned false, ending pipeline.')
|
50
|
+
break
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end.real.round(3)
|
54
|
+
|
55
|
+
output.ruler
|
56
|
+
output.write("Pipeline ended, took #{time_in_seconds} second(s) to complete")
|
57
|
+
|
58
|
+
payload
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def output_params(params, output)
|
64
|
+
if params.keys.any?
|
65
|
+
output.write('Parameters:')
|
66
|
+
else
|
67
|
+
output.write('No parameters passed in.')
|
68
|
+
end
|
69
|
+
|
70
|
+
params.each { |key, value| output.detail("#{key}: #{value}") }
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
data/lib/burner/step.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# A wrapper to execute a job (in the context of a Pipeline.)
|
12
|
+
class Step
|
13
|
+
extend Forwardable
|
14
|
+
|
15
|
+
SEPARATOR = '::'
|
16
|
+
|
17
|
+
private_constant :SEPARATOR
|
18
|
+
|
19
|
+
attr_reader :job
|
20
|
+
|
21
|
+
def_delegators :job, :name
|
22
|
+
|
23
|
+
def initialize(job)
|
24
|
+
raise ArgumentError, 'job is required' unless job
|
25
|
+
|
26
|
+
@job = job
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
def perform(output, payload)
|
32
|
+
return_value = nil
|
33
|
+
|
34
|
+
output.title("#{job.class.name}#{SEPARATOR}#{job.name}")
|
35
|
+
|
36
|
+
time_in_seconds = Benchmark.measure do
|
37
|
+
return_value = job.perform(output, payload)
|
38
|
+
end.real.round(3)
|
39
|
+
|
40
|
+
output.complete(time_in_seconds)
|
41
|
+
|
42
|
+
return_value
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# Can take in a string and an object and use the object for formatting string interpolations
|
12
|
+
# using tokens of form: {attribute_name}. This templating class does not understand nested
|
13
|
+
# structures, so input should be a flat object/hash in the form of key-value pairs. A benefit of
|
14
|
+
# using Objectable for resolution is that it can understand almost any type of
|
15
|
+
# object: Hash, Struct, OpenStruct, custom objects, etc.
|
16
|
+
# For more information see underlying libraries:
|
17
|
+
# * Stringento: https://github.com/bluemarblepayroll/stringento
|
18
|
+
# * Objectable: https://github.com/bluemarblepayroll/objectable
|
19
|
+
class StringTemplate
|
20
|
+
include Singleton
|
21
|
+
|
22
|
+
attr_reader :resolver
|
23
|
+
|
24
|
+
def initialize
|
25
|
+
@resolver = Objectable.resolver(separator: '')
|
26
|
+
|
27
|
+
freeze
|
28
|
+
end
|
29
|
+
|
30
|
+
# For general consumption
|
31
|
+
def evaluate(expression, input)
|
32
|
+
Stringento.evaluate(expression, input, resolver: self)
|
33
|
+
end
|
34
|
+
|
35
|
+
# For Stringento consumption
|
36
|
+
def resolve(value, input)
|
37
|
+
resolver.get(input, value)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/burner/version.rb
CHANGED
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# Describes a file that was generated by a Job. If a Job emits a file, it should also add the
|
12
|
+
# file details to the Payload#written_files array using the Payload#add_written_file method.
|
13
|
+
class WrittenFile
|
14
|
+
acts_as_hashable
|
15
|
+
|
16
|
+
attr_reader :logical_filename,
|
17
|
+
:physical_filename,
|
18
|
+
:time_in_seconds
|
19
|
+
|
20
|
+
def initialize(logical_filename:, physical_filename:, time_in_seconds:)
|
21
|
+
@logical_filename = logical_filename.to_s
|
22
|
+
@physical_filename = physical_filename.to_s
|
23
|
+
@time_in_seconds = time_in_seconds.to_f
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|