burner 0.0.0 → 1.0.0.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -3
- data/README.md +73 -2
- data/burner.gemspec +1 -1
- data/exe/burner +21 -0
- data/lib/burner.rb +9 -4
- data/lib/burner/cli.rb +45 -0
- data/lib/burner/job.rb +35 -0
- data/lib/burner/jobs.rb +40 -0
- data/lib/burner/jobs/deserialize/json.rb +23 -0
- data/lib/burner/jobs/deserialize/yaml.rb +41 -0
- data/lib/burner/jobs/dummy.rb +19 -0
- data/lib/burner/jobs/echo.rb +33 -0
- data/lib/burner/jobs/io/base.rb +40 -0
- data/lib/burner/jobs/io/read.rb +35 -0
- data/lib/burner/jobs/io/write.rb +45 -0
- data/lib/burner/jobs/serialize/json.rb +23 -0
- data/lib/burner/jobs/serialize/yaml.rb +23 -0
- data/lib/burner/jobs/set.rb +31 -0
- data/lib/burner/jobs/sleep.rb +33 -0
- data/lib/burner/output.rb +66 -0
- data/lib/burner/payload.rb +26 -0
- data/lib/burner/pipeline.rb +66 -0
- data/lib/burner/step.rb +43 -0
- data/lib/burner/string_template.rb +39 -0
- data/lib/burner/version.rb +1 -1
- metadata +26 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 920f3b0f0027e21b30da75f86335d75d2d351f49c8b68662de7226ade43474c6
|
4
|
+
data.tar.gz: b7f4189e37023f1d627ea5222df0f397a24a69c7ce3721b13ce714d2e274ee94
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d488edf32a95e5da64190c512f365d76e2414d16712cee9b56a400bd62e11ed7824a316a95923af15f0ecac3390e4be359d2e31ffc33fb13129309c4efce9403
|
7
|
+
data.tar.gz: 8640cfdb2fa4241fb2a493c2ac2aed62e08e1826446907f4b16d043276fffee656c787d2079545fe03d52b8f45756240a523ad878fc89063884868c3f94bfecd
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,76 @@
|
|
1
1
|
# Burner
|
2
2
|
|
3
|
-
|
3
|
+
[](https://badge.fury.io/rb/burner) [](https://travis-ci.org/bluemarblepayroll/burner) [](https://codeclimate.com/github/bluemarblepayroll/burner/maintainability) [](https://codeclimate.com/github/bluemarblepayroll/burner/test_coverage) [](https://opensource.org/licenses/MIT)
|
4
4
|
|
5
|
-
|
5
|
+
TODO
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
To install through Rubygems:
|
10
|
+
|
11
|
+
````bash
|
12
|
+
gem install burner
|
13
|
+
````
|
14
|
+
|
15
|
+
You can also add this to your Gemfile:
|
16
|
+
|
17
|
+
````bash
|
18
|
+
bundle add burner
|
19
|
+
````
|
20
|
+
|
21
|
+
## Examples
|
22
|
+
|
23
|
+
TODO
|
24
|
+
|
25
|
+
## Contributing
|
26
|
+
|
27
|
+
### Development Environment Configuration
|
28
|
+
|
29
|
+
Basic steps to take to get this repository compiling:
|
30
|
+
|
31
|
+
1. Install [Ruby](https://www.ruby-lang.org/en/documentation/installation/) (check burner.gemspec for versions supported)
|
32
|
+
2. Install bundler (gem install bundler)
|
33
|
+
3. Clone the repository (git clone git@github.com:bluemarblepayroll/burner.git)
|
34
|
+
4. Navigate to the root folder (cd burner)
|
35
|
+
5. Install dependencies (bundle)
|
36
|
+
|
37
|
+
### Running Tests
|
38
|
+
|
39
|
+
To execute the test suite run:
|
40
|
+
|
41
|
+
````bash
|
42
|
+
bundle exec rspec spec --format documentation
|
43
|
+
````
|
44
|
+
|
45
|
+
Alternatively, you can have Guard watch for changes:
|
46
|
+
|
47
|
+
````bash
|
48
|
+
bundle exec guard
|
49
|
+
````
|
50
|
+
|
51
|
+
Also, do not forget to run Rubocop:
|
52
|
+
|
53
|
+
````bash
|
54
|
+
bundle exec rubocop
|
55
|
+
````
|
56
|
+
|
57
|
+
### Publishing
|
58
|
+
|
59
|
+
Note: ensure you have proper authorization before trying to publish new versions.
|
60
|
+
|
61
|
+
After code changes have successfully gone through the Pull Request review process then the following steps should be followed for publishing new versions:
|
62
|
+
|
63
|
+
1. Merge Pull Request into master
|
64
|
+
2. Update `lib/burner/version.rb` using [semantic versioning](https://semver.org/)
|
65
|
+
3. Install dependencies: `bundle`
|
66
|
+
4. Update `CHANGELOG.md` with release notes
|
67
|
+
5. Commit & push master to remote and ensure CI builds master successfully
|
68
|
+
6. Run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
69
|
+
|
70
|
+
## Code of Conduct
|
71
|
+
|
72
|
+
Everyone interacting in this codebase, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/bluemarblepayroll/burner/blob/master/CODE_OF_CONDUCT.md).
|
73
|
+
|
74
|
+
## License
|
75
|
+
|
76
|
+
This project is MIT Licensed.
|
data/burner.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.email = ['mruggio@bluemarblepayroll.com']
|
16
16
|
s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
17
|
s.bindir = 'exe'
|
18
|
-
s.executables = %w[]
|
18
|
+
s.executables = %w[burner]
|
19
19
|
s.homepage = 'https://github.com/bluemarblepayroll/burner'
|
20
20
|
s.license = 'MIT'
|
21
21
|
s.metadata = {
|
data/exe/burner
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
#
|
5
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
6
|
+
#
|
7
|
+
# This source code is licensed under the MIT license found in the
|
8
|
+
# LICENSE file in the root directory of this source tree.
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'bundler/setup'
|
12
|
+
require 'burner'
|
13
|
+
require 'pry'
|
14
|
+
|
15
|
+
if ARGV.empty?
|
16
|
+
puts 'Usage: ./exe/burner package.yaml key=value key=value ...'
|
17
|
+
exit
|
18
|
+
end
|
19
|
+
|
20
|
+
# This should return exit code of 1 if it raises any hard errors.
|
21
|
+
Burner::Cli.new(ARGV).execute
|
data/lib/burner.rb
CHANGED
@@ -7,8 +7,13 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
|
10
|
+
require 'acts_as_hashable'
|
11
|
+
require 'benchmark'
|
12
|
+
require 'json'
|
13
|
+
require 'objectable'
|
14
|
+
require 'securerandom'
|
15
|
+
require 'singleton'
|
16
|
+
require 'stringento'
|
17
|
+
require 'yaml'
|
11
18
|
|
12
|
-
|
13
|
-
module Burner
|
14
|
-
end
|
19
|
+
require_relative 'burner/cli'
|
data/lib/burner/cli.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'pipeline'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# Process a single string as a Pipeline. This is mainly to back the command-line interface.
|
14
|
+
class Cli
|
15
|
+
attr_reader :params, :pipeline
|
16
|
+
|
17
|
+
def initialize(args)
|
18
|
+
path = args.first
|
19
|
+
cli_params = extract_cli_params(args)
|
20
|
+
config = read_yaml(path)
|
21
|
+
@pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
|
22
|
+
@params = (config['params'] || {}).merge(cli_params)
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute
|
26
|
+
pipeline.execute(params: params)
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def read_yaml(path)
|
32
|
+
yaml = IO.read(path)
|
33
|
+
|
34
|
+
YAML.safe_load(yaml)
|
35
|
+
end
|
36
|
+
|
37
|
+
def extract_cli_params(args)
|
38
|
+
args[1..-1].each_with_object({}) do |arg, memo|
|
39
|
+
parts = arg.to_s.split('=')
|
40
|
+
|
41
|
+
memo[parts.first] = parts.last
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/burner/job.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'string_template'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# Abstract base class for all job subclasses. The only public method a subclass needs to
|
14
|
+
# implement #perform(params, payload, reporter) and then you can register it for use using
|
15
|
+
# the Burner::Jobs factory class method #register. An example of a registration:
|
16
|
+
# Burner::Jobs.register('your_class', YourClass)
|
17
|
+
class Job
|
18
|
+
acts_as_hashable
|
19
|
+
|
20
|
+
attr_reader :name, :string_template
|
21
|
+
|
22
|
+
def initialize(name:)
|
23
|
+
raise ArgumentError, 'name is required' if name.to_s.empty?
|
24
|
+
|
25
|
+
@name = name.to_s
|
26
|
+
@string_template = StringTemplate.instance
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def eval_string_template(expression, input)
|
32
|
+
string_template.evaluate(expression, input)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/burner/jobs.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'job'
|
11
|
+
require_relative 'jobs/deserialize/json'
|
12
|
+
require_relative 'jobs/deserialize/yaml'
|
13
|
+
require_relative 'jobs/dummy'
|
14
|
+
require_relative 'jobs/echo'
|
15
|
+
require_relative 'jobs/io/read'
|
16
|
+
require_relative 'jobs/io/write'
|
17
|
+
require_relative 'jobs/serialize/json'
|
18
|
+
require_relative 'jobs/serialize/yaml'
|
19
|
+
require_relative 'jobs/set'
|
20
|
+
require_relative 'jobs/sleep'
|
21
|
+
|
22
|
+
module Burner
|
23
|
+
# Main library of jobs. This file contains all the basic/default jobs. All other consumer
|
24
|
+
# libraries/applications can register their own, for example:
|
25
|
+
# Burner::Jobs.register('your_class', YourClass)
|
26
|
+
class Jobs
|
27
|
+
acts_as_hashable_factory
|
28
|
+
|
29
|
+
register 'deserialize/json', Deserialize::Json
|
30
|
+
register 'deserialize/yaml', Deserialize::Yaml
|
31
|
+
register 'dummy', '', Dummy
|
32
|
+
register 'echo', Echo
|
33
|
+
register 'io/read', IO::Read
|
34
|
+
register 'io/write', IO::Write
|
35
|
+
register 'serialize/json', Serialize::Json
|
36
|
+
register 'serialize/yaml', Serialize::Yaml
|
37
|
+
register 'set', Set
|
38
|
+
register 'sleep', Sleep
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a JSON string and deserialize into object(s).
|
14
|
+
class Json < Job
|
15
|
+
def perform(_output, payload, _params)
|
16
|
+
payload.value = JSON.parse(payload.value)
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a YAML string and deserialize into object(s).
|
14
|
+
class Yaml < Job
|
15
|
+
attr_reader :safe
|
16
|
+
|
17
|
+
def initialize(name:, safe: true)
|
18
|
+
super(name: name)
|
19
|
+
|
20
|
+
@safe = safe
|
21
|
+
|
22
|
+
freeze
|
23
|
+
end
|
24
|
+
|
25
|
+
# The YAML cop was disabled because the consumer may want to actually load unsafe
|
26
|
+
# YAML, which can load pretty much any type of class instead of putting the loader
|
27
|
+
# in a sandbox. By default, though, we will try and drive them towards using it
|
28
|
+
# in the safer alternative.
|
29
|
+
# rubocop:disable Security/YAMLLoad
|
30
|
+
def perform(output, payload, _params)
|
31
|
+
output.detail('Warning: loading YAML not using safe_load.') unless safe
|
32
|
+
|
33
|
+
payload.value = safe ? YAML.safe_load(payload.value) : YAML.load(payload.value)
|
34
|
+
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
# rubocop:enable Security/YAMLLoad
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Do nothing.
|
13
|
+
class Dummy < Job
|
14
|
+
def perform(_output, _payload, _params)
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Output a simple message to the output.
|
13
|
+
class Echo < Job
|
14
|
+
attr_reader :message
|
15
|
+
|
16
|
+
def initialize(name:, message: '')
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@message = message.to_s
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(output, _payload, params)
|
25
|
+
compiled_message = eval_string_template(message, params)
|
26
|
+
|
27
|
+
output.detail(compiled_message)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module IO
|
13
|
+
# Common configuration/code for all IO Job subclasses.
|
14
|
+
class Base < Job
|
15
|
+
attr_reader :binary, :path
|
16
|
+
|
17
|
+
def initialize(name:, path:, binary: false)
|
18
|
+
super(name: name)
|
19
|
+
|
20
|
+
raise ArgumentError, 'path is required' if path.to_s.empty?
|
21
|
+
|
22
|
+
@path = path.to_s
|
23
|
+
@binary = binary || false
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def compile_path(params)
|
31
|
+
eval_string_template(path, params)
|
32
|
+
end
|
33
|
+
|
34
|
+
def mode
|
35
|
+
binary ? 'wb' : 'w'
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Read value from disk.
|
16
|
+
class Read < Base
|
17
|
+
def perform(output, payload, params)
|
18
|
+
compiled_path = compile_path(params)
|
19
|
+
|
20
|
+
output.detail("Reading: #{compiled_path}")
|
21
|
+
|
22
|
+
payload.value = File.open(compiled_path, mode, &:read)
|
23
|
+
|
24
|
+
nil
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def mode
|
30
|
+
binary ? 'rb' : 'r'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Write value to disk.
|
16
|
+
class Write < Base
|
17
|
+
def perform(output, payload, params)
|
18
|
+
compiled_path = compile_path(params)
|
19
|
+
|
20
|
+
ensure_directory_exists(output, compiled_path)
|
21
|
+
|
22
|
+
output.detail("Writing: #{compiled_path}")
|
23
|
+
|
24
|
+
File.open(compiled_path, mode) { |io| io.write(payload.value) }
|
25
|
+
|
26
|
+
nil
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def ensure_directory_exists(output, compiled_path)
|
32
|
+
dirname = File.dirname(compiled_path)
|
33
|
+
|
34
|
+
return if File.exist?(dirname)
|
35
|
+
|
36
|
+
output.detail("Outer directory does not exist, creating: #{dirname}")
|
37
|
+
|
38
|
+
FileUtils.mkdir_p(dirname)
|
39
|
+
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Treat value like a Ruby object and serialize it using JSON.
|
14
|
+
class Json < Job
|
15
|
+
def perform(_output, payload, _params)
|
16
|
+
payload.value = payload.value.to_json
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Treat value like a Ruby object and serialize it using YAML.
|
14
|
+
class Yaml < Job
|
15
|
+
def perform(_output, payload, _params)
|
16
|
+
payload.value = payload.value.to_yaml
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Arbitrarily set value
|
13
|
+
class Set < Job
|
14
|
+
attr_reader :value
|
15
|
+
|
16
|
+
def initialize(name:, value: nil)
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@value = value
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(_output, payload, _params)
|
25
|
+
payload.value = value
|
26
|
+
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Arbitrarily put thread to sleep for X number of seconds
|
13
|
+
class Sleep < Job
|
14
|
+
attr_reader :seconds
|
15
|
+
|
16
|
+
def initialize(name:, seconds: 0)
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@seconds = seconds.to_f
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(output, _payload, _params)
|
25
|
+
output.detail("Going to sleep for #{seconds} second(s)")
|
26
|
+
|
27
|
+
Kernel.sleep(seconds)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# A Pipeline execution can write main output, which is really an outline to whats happening,
|
12
|
+
# step-by-step. This is not meant to replace or be true logging, but serve more as a summary
|
13
|
+
# for each of the jobs. Each job can decide what it would like to include in this summary
|
14
|
+
# and leverage an instance of this class for inclusion of that information.
|
15
|
+
class Output
|
16
|
+
RULER_LENGTH = 80
|
17
|
+
|
18
|
+
attr_reader :id, :job_count, :outs
|
19
|
+
|
20
|
+
def initialize(id: SecureRandom.uuid, outs: [$stdout])
|
21
|
+
@id = id
|
22
|
+
@outs = Array(outs)
|
23
|
+
@job_count = 1
|
24
|
+
end
|
25
|
+
|
26
|
+
def ruler
|
27
|
+
write('-' * RULER_LENGTH)
|
28
|
+
|
29
|
+
self
|
30
|
+
end
|
31
|
+
|
32
|
+
def title(message)
|
33
|
+
write("[#{job_count}] #{message}")
|
34
|
+
|
35
|
+
@job_count += 1
|
36
|
+
|
37
|
+
self
|
38
|
+
end
|
39
|
+
|
40
|
+
def detail(message)
|
41
|
+
write(" - #{message}")
|
42
|
+
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
def write(message)
|
47
|
+
raw("[#{id} | #{Time.now.utc}] #{message}")
|
48
|
+
|
49
|
+
self
|
50
|
+
end
|
51
|
+
|
52
|
+
def complete(time_in_seconds)
|
53
|
+
detail("Completed in: #{time_in_seconds.round(3)} second(s)")
|
54
|
+
|
55
|
+
self
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def raw(message)
|
61
|
+
outs.each { |out| out.puts(message) }
|
62
|
+
|
63
|
+
self
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# The input for all Job#perform methods. The main notion of this object is its "value"
|
12
|
+
# attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
|
13
|
+
# methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
|
14
|
+
# library and lifecycle, but I am not sure there is any other way around it: trying to build
|
15
|
+
# a generic, open-ended object pipeline to serve almost any use case.
|
16
|
+
class Payload
|
17
|
+
attr_accessor :value
|
18
|
+
|
19
|
+
attr_reader :context
|
20
|
+
|
21
|
+
def initialize(context: {}, value: nil)
|
22
|
+
@context = context || {}
|
23
|
+
@value = value
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'jobs'
|
11
|
+
require_relative 'output'
|
12
|
+
require_relative 'payload'
|
13
|
+
require_relative 'step'
|
14
|
+
|
15
|
+
module Burner
|
16
|
+
# The root package. A Pipeline contains the job configurations along with the steps. The steps
|
17
|
+
# referens jobs and tell you the order of the jobs to run.
|
18
|
+
class Pipeline
|
19
|
+
acts_as_hashable
|
20
|
+
|
21
|
+
class JobNotFoundError < StandardError; end
|
22
|
+
|
23
|
+
attr_reader :steps
|
24
|
+
|
25
|
+
def initialize(jobs: [], steps: [])
|
26
|
+
jobs_by_name = Jobs.array(jobs).map { |job| [job.name, job] }.to_h
|
27
|
+
|
28
|
+
@steps = Array(steps).map do |step_name|
|
29
|
+
job = jobs_by_name[step_name.to_s]
|
30
|
+
|
31
|
+
raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
|
32
|
+
|
33
|
+
Step.new(job)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# The main entry-point for kicking off a pipeline.
|
38
|
+
def execute(params: {}, output: Output.new, payload: Payload.new)
|
39
|
+
output.write("Pipeline started with #{steps.length} step(s)")
|
40
|
+
|
41
|
+
output_params(params, output)
|
42
|
+
output.ruler
|
43
|
+
|
44
|
+
time_in_seconds = Benchmark.measure do
|
45
|
+
steps.each { |step| step.perform(output, payload, params) }
|
46
|
+
end.real.round(3)
|
47
|
+
|
48
|
+
output.ruler
|
49
|
+
output.write("Pipeline ended, took #{time_in_seconds} second(s) to complete")
|
50
|
+
|
51
|
+
payload
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def output_params(params, output)
|
57
|
+
if params.keys.any?
|
58
|
+
output.write('Parameters:')
|
59
|
+
else
|
60
|
+
output.write('No parameters passed in.')
|
61
|
+
end
|
62
|
+
|
63
|
+
params.each { |key, value| output.detail("#{key}: #{value}") }
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/burner/step.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# A wrapper to execute a job (in the context of a Pipeline.)
|
12
|
+
class Step
|
13
|
+
extend Forwardable
|
14
|
+
|
15
|
+
SEPARATOR = '::'
|
16
|
+
|
17
|
+
private_constant :SEPARATOR
|
18
|
+
|
19
|
+
attr_reader :job
|
20
|
+
|
21
|
+
def_delegators :job, :name
|
22
|
+
|
23
|
+
def initialize(job)
|
24
|
+
raise ArgumentError, 'job is required' unless job
|
25
|
+
|
26
|
+
@job = job
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
def perform(output, payload, params)
|
32
|
+
output.title("#{job.class.name}#{SEPARATOR}#{job.name}")
|
33
|
+
|
34
|
+
time_in_seconds = Benchmark.measure do
|
35
|
+
job.perform(output, payload, params)
|
36
|
+
end.real.round(3)
|
37
|
+
|
38
|
+
output.complete(time_in_seconds)
|
39
|
+
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# Can take in a string and an object and use the object for formatting string interpolations
|
12
|
+
# using tokens of form: {attribute_name}. It can also understand dot-notation for nested
|
13
|
+
# objects using the Objectable library. Another benefit of using Objectable for resolution
|
14
|
+
# is that it can understand almost any type of object: Hash, Struct, OpenStruct, custom
|
15
|
+
# objects, etc. For more information see underlying libraries:
|
16
|
+
# * Stringento: https://github.com/bluemarblepayroll/stringento
|
17
|
+
# * Objectable: https://github.com/bluemarblepayroll/objectable
|
18
|
+
class StringTemplate
|
19
|
+
include Singleton
|
20
|
+
|
21
|
+
attr_reader :resolver
|
22
|
+
|
23
|
+
def initialize
|
24
|
+
@resolver = Objectable.resolver
|
25
|
+
|
26
|
+
freeze
|
27
|
+
end
|
28
|
+
|
29
|
+
# For general consumption
|
30
|
+
def evaluate(expression, input)
|
31
|
+
Stringento.evaluate(expression, input, resolver: self)
|
32
|
+
end
|
33
|
+
|
34
|
+
# For Stringento consumption
|
35
|
+
def resolve(value, input)
|
36
|
+
resolver.get(input, value)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/burner/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: burner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 1.0.0.pre.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-10-
|
11
|
+
date: 2020-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: acts_as_hashable
|
@@ -154,7 +154,8 @@ description: " This library serves as the skeleton for a processing engine.
|
|
154
154
|
allows you to organize your code into Jobs, then stitch those jobs together as steps.\n"
|
155
155
|
email:
|
156
156
|
- mruggio@bluemarblepayroll.com
|
157
|
-
executables:
|
157
|
+
executables:
|
158
|
+
- burner
|
158
159
|
extensions: []
|
159
160
|
extra_rdoc_files: []
|
160
161
|
files:
|
@@ -173,7 +174,27 @@ files:
|
|
173
174
|
- bin/console
|
174
175
|
- burner.gemspec
|
175
176
|
- exe/.gitkeep
|
177
|
+
- exe/burner
|
176
178
|
- lib/burner.rb
|
179
|
+
- lib/burner/cli.rb
|
180
|
+
- lib/burner/job.rb
|
181
|
+
- lib/burner/jobs.rb
|
182
|
+
- lib/burner/jobs/deserialize/json.rb
|
183
|
+
- lib/burner/jobs/deserialize/yaml.rb
|
184
|
+
- lib/burner/jobs/dummy.rb
|
185
|
+
- lib/burner/jobs/echo.rb
|
186
|
+
- lib/burner/jobs/io/base.rb
|
187
|
+
- lib/burner/jobs/io/read.rb
|
188
|
+
- lib/burner/jobs/io/write.rb
|
189
|
+
- lib/burner/jobs/serialize/json.rb
|
190
|
+
- lib/burner/jobs/serialize/yaml.rb
|
191
|
+
- lib/burner/jobs/set.rb
|
192
|
+
- lib/burner/jobs/sleep.rb
|
193
|
+
- lib/burner/output.rb
|
194
|
+
- lib/burner/payload.rb
|
195
|
+
- lib/burner/pipeline.rb
|
196
|
+
- lib/burner/step.rb
|
197
|
+
- lib/burner/string_template.rb
|
177
198
|
- lib/burner/version.rb
|
178
199
|
homepage: https://github.com/bluemarblepayroll/burner
|
179
200
|
licenses:
|
@@ -195,9 +216,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
195
216
|
version: '2.5'
|
196
217
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
197
218
|
requirements:
|
198
|
-
- - "
|
219
|
+
- - ">"
|
199
220
|
- !ruby/object:Gem::Version
|
200
|
-
version:
|
221
|
+
version: 1.3.1
|
201
222
|
requirements: []
|
202
223
|
rubygems_version: 3.0.3
|
203
224
|
signing_key:
|