burner 0.0.0 → 1.0.0.pre.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +0 -3
- data/README.md +73 -2
- data/burner.gemspec +1 -1
- data/exe/burner +21 -0
- data/lib/burner.rb +9 -4
- data/lib/burner/cli.rb +45 -0
- data/lib/burner/job.rb +35 -0
- data/lib/burner/jobs.rb +40 -0
- data/lib/burner/jobs/deserialize/json.rb +23 -0
- data/lib/burner/jobs/deserialize/yaml.rb +41 -0
- data/lib/burner/jobs/dummy.rb +19 -0
- data/lib/burner/jobs/echo.rb +33 -0
- data/lib/burner/jobs/io/base.rb +40 -0
- data/lib/burner/jobs/io/read.rb +35 -0
- data/lib/burner/jobs/io/write.rb +45 -0
- data/lib/burner/jobs/serialize/json.rb +23 -0
- data/lib/burner/jobs/serialize/yaml.rb +23 -0
- data/lib/burner/jobs/set.rb +31 -0
- data/lib/burner/jobs/sleep.rb +33 -0
- data/lib/burner/output.rb +66 -0
- data/lib/burner/payload.rb +26 -0
- data/lib/burner/pipeline.rb +66 -0
- data/lib/burner/step.rb +43 -0
- data/lib/burner/string_template.rb +39 -0
- data/lib/burner/version.rb +1 -1
- metadata +26 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 920f3b0f0027e21b30da75f86335d75d2d351f49c8b68662de7226ade43474c6
|
4
|
+
data.tar.gz: b7f4189e37023f1d627ea5222df0f397a24a69c7ce3721b13ce714d2e274ee94
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d488edf32a95e5da64190c512f365d76e2414d16712cee9b56a400bd62e11ed7824a316a95923af15f0ecac3390e4be359d2e31ffc33fb13129309c4efce9403
|
7
|
+
data.tar.gz: 8640cfdb2fa4241fb2a493c2ac2aed62e08e1826446907f4b16d043276fffee656c787d2079545fe03d52b8f45756240a523ad878fc89063884868c3f94bfecd
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,76 @@
|
|
1
1
|
# Burner
|
2
2
|
|
3
|
-
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/burner.svg)](https://badge.fury.io/rb/burner) [![Build Status](https://travis-ci.org/bluemarblepayroll/burner.svg?branch=master)](https://travis-ci.org/bluemarblepayroll/burner) [![Maintainability](https://api.codeclimate.com/v1/badges/dbc3757929b67504f6ca/maintainability)](https://codeclimate.com/github/bluemarblepayroll/burner/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/dbc3757929b67504f6ca/test_coverage)](https://codeclimate.com/github/bluemarblepayroll/burner/test_coverage) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
|
4
4
|
|
5
|
-
|
5
|
+
TODO
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
To install through Rubygems:
|
10
|
+
|
11
|
+
````bash
|
12
|
+
gem install burner
|
13
|
+
````
|
14
|
+
|
15
|
+
You can also add this to your Gemfile:
|
16
|
+
|
17
|
+
````bash
|
18
|
+
bundle add burner
|
19
|
+
````
|
20
|
+
|
21
|
+
## Examples
|
22
|
+
|
23
|
+
TODO
|
24
|
+
|
25
|
+
## Contributing
|
26
|
+
|
27
|
+
### Development Environment Configuration
|
28
|
+
|
29
|
+
Basic steps to take to get this repository compiling:
|
30
|
+
|
31
|
+
1. Install [Ruby](https://www.ruby-lang.org/en/documentation/installation/) (check burner.gemspec for versions supported)
|
32
|
+
2. Install bundler (gem install bundler)
|
33
|
+
3. Clone the repository (git clone git@github.com:bluemarblepayroll/burner.git)
|
34
|
+
4. Navigate to the root folder (cd burner)
|
35
|
+
5. Install dependencies (bundle)
|
36
|
+
|
37
|
+
### Running Tests
|
38
|
+
|
39
|
+
To execute the test suite run:
|
40
|
+
|
41
|
+
````bash
|
42
|
+
bundle exec rspec spec --format documentation
|
43
|
+
````
|
44
|
+
|
45
|
+
Alternatively, you can have Guard watch for changes:
|
46
|
+
|
47
|
+
````bash
|
48
|
+
bundle exec guard
|
49
|
+
````
|
50
|
+
|
51
|
+
Also, do not forget to run Rubocop:
|
52
|
+
|
53
|
+
````bash
|
54
|
+
bundle exec rubocop
|
55
|
+
````
|
56
|
+
|
57
|
+
### Publishing
|
58
|
+
|
59
|
+
Note: ensure you have proper authorization before trying to publish new versions.
|
60
|
+
|
61
|
+
After code changes have successfully gone through the Pull Request review process then the following steps should be followed for publishing new versions:
|
62
|
+
|
63
|
+
1. Merge Pull Request into master
|
64
|
+
2. Update `lib/burner/version.rb` using [semantic versioning](https://semver.org/)
|
65
|
+
3. Install dependencies: `bundle`
|
66
|
+
4. Update `CHANGELOG.md` with release notes
|
67
|
+
5. Commit & push master to remote and ensure CI builds master successfully
|
68
|
+
6. Run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
69
|
+
|
70
|
+
## Code of Conduct
|
71
|
+
|
72
|
+
Everyone interacting in this codebase, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/bluemarblepayroll/burner/blob/master/CODE_OF_CONDUCT.md).
|
73
|
+
|
74
|
+
## License
|
75
|
+
|
76
|
+
This project is MIT Licensed.
|
data/burner.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.email = ['mruggio@bluemarblepayroll.com']
|
16
16
|
s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
17
|
s.bindir = 'exe'
|
18
|
-
s.executables = %w[]
|
18
|
+
s.executables = %w[burner]
|
19
19
|
s.homepage = 'https://github.com/bluemarblepayroll/burner'
|
20
20
|
s.license = 'MIT'
|
21
21
|
s.metadata = {
|
data/exe/burner
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
#
|
5
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
6
|
+
#
|
7
|
+
# This source code is licensed under the MIT license found in the
|
8
|
+
# LICENSE file in the root directory of this source tree.
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'bundler/setup'
|
12
|
+
require 'burner'
|
13
|
+
require 'pry'
|
14
|
+
|
15
|
+
if ARGV.empty?
|
16
|
+
puts 'Usage: ./exe/burner package.yaml key=value key=value ...'
|
17
|
+
exit
|
18
|
+
end
|
19
|
+
|
20
|
+
# This should return exit code of 1 if it raises any hard errors.
|
21
|
+
Burner::Cli.new(ARGV).execute
|
data/lib/burner.rb
CHANGED
@@ -7,8 +7,13 @@
|
|
7
7
|
# LICENSE file in the root directory of this source tree.
|
8
8
|
#
|
9
9
|
|
10
|
-
|
10
|
+
require 'acts_as_hashable'
|
11
|
+
require 'benchmark'
|
12
|
+
require 'json'
|
13
|
+
require 'objectable'
|
14
|
+
require 'securerandom'
|
15
|
+
require 'singleton'
|
16
|
+
require 'stringento'
|
17
|
+
require 'yaml'
|
11
18
|
|
12
|
-
|
13
|
-
module Burner
|
14
|
-
end
|
19
|
+
require_relative 'burner/cli'
|
data/lib/burner/cli.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'pipeline'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# Process a single string as a Pipeline. This is mainly to back the command-line interface.
|
14
|
+
class Cli
|
15
|
+
attr_reader :params, :pipeline
|
16
|
+
|
17
|
+
def initialize(args)
|
18
|
+
path = args.first
|
19
|
+
cli_params = extract_cli_params(args)
|
20
|
+
config = read_yaml(path)
|
21
|
+
@pipeline = Burner::Pipeline.make(jobs: config['jobs'], steps: config['steps'])
|
22
|
+
@params = (config['params'] || {}).merge(cli_params)
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute
|
26
|
+
pipeline.execute(params: params)
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def read_yaml(path)
|
32
|
+
yaml = IO.read(path)
|
33
|
+
|
34
|
+
YAML.safe_load(yaml)
|
35
|
+
end
|
36
|
+
|
37
|
+
def extract_cli_params(args)
|
38
|
+
args[1..-1].each_with_object({}) do |arg, memo|
|
39
|
+
parts = arg.to_s.split('=')
|
40
|
+
|
41
|
+
memo[parts.first] = parts.last
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/lib/burner/job.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'string_template'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
# Abstract base class for all job subclasses. The only public method a subclass needs to
|
14
|
+
# implement #perform(params, payload, reporter) and then you can register it for use using
|
15
|
+
# the Burner::Jobs factory class method #register. An example of a registration:
|
16
|
+
# Burner::Jobs.register('your_class', YourClass)
|
17
|
+
class Job
|
18
|
+
acts_as_hashable
|
19
|
+
|
20
|
+
attr_reader :name, :string_template
|
21
|
+
|
22
|
+
def initialize(name:)
|
23
|
+
raise ArgumentError, 'name is required' if name.to_s.empty?
|
24
|
+
|
25
|
+
@name = name.to_s
|
26
|
+
@string_template = StringTemplate.instance
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def eval_string_template(expression, input)
|
32
|
+
string_template.evaluate(expression, input)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/burner/jobs.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'job'
|
11
|
+
require_relative 'jobs/deserialize/json'
|
12
|
+
require_relative 'jobs/deserialize/yaml'
|
13
|
+
require_relative 'jobs/dummy'
|
14
|
+
require_relative 'jobs/echo'
|
15
|
+
require_relative 'jobs/io/read'
|
16
|
+
require_relative 'jobs/io/write'
|
17
|
+
require_relative 'jobs/serialize/json'
|
18
|
+
require_relative 'jobs/serialize/yaml'
|
19
|
+
require_relative 'jobs/set'
|
20
|
+
require_relative 'jobs/sleep'
|
21
|
+
|
22
|
+
module Burner
|
23
|
+
# Main library of jobs. This file contains all the basic/default jobs. All other consumer
|
24
|
+
# libraries/applications can register their own, for example:
|
25
|
+
# Burner::Jobs.register('your_class', YourClass)
|
26
|
+
class Jobs
|
27
|
+
acts_as_hashable_factory
|
28
|
+
|
29
|
+
register 'deserialize/json', Deserialize::Json
|
30
|
+
register 'deserialize/yaml', Deserialize::Yaml
|
31
|
+
register 'dummy', '', Dummy
|
32
|
+
register 'echo', Echo
|
33
|
+
register 'io/read', IO::Read
|
34
|
+
register 'io/write', IO::Write
|
35
|
+
register 'serialize/json', Serialize::Json
|
36
|
+
register 'serialize/yaml', Serialize::Yaml
|
37
|
+
register 'set', Set
|
38
|
+
register 'sleep', Sleep
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a JSON string and deserialize into object(s).
|
14
|
+
class Json < Job
|
15
|
+
def perform(_output, payload, _params)
|
16
|
+
payload.value = JSON.parse(payload.value)
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Deserialize
|
13
|
+
# Take a YAML string and deserialize into object(s).
|
14
|
+
class Yaml < Job
|
15
|
+
attr_reader :safe
|
16
|
+
|
17
|
+
def initialize(name:, safe: true)
|
18
|
+
super(name: name)
|
19
|
+
|
20
|
+
@safe = safe
|
21
|
+
|
22
|
+
freeze
|
23
|
+
end
|
24
|
+
|
25
|
+
# The YAML cop was disabled because the consumer may want to actually load unsafe
|
26
|
+
# YAML, which can load pretty much any type of class instead of putting the loader
|
27
|
+
# in a sandbox. By default, though, we will try and drive them towards using it
|
28
|
+
# in the safer alternative.
|
29
|
+
# rubocop:disable Security/YAMLLoad
|
30
|
+
def perform(output, payload, _params)
|
31
|
+
output.detail('Warning: loading YAML not using safe_load.') unless safe
|
32
|
+
|
33
|
+
payload.value = safe ? YAML.safe_load(payload.value) : YAML.load(payload.value)
|
34
|
+
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
# rubocop:enable Security/YAMLLoad
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Do nothing.
|
13
|
+
class Dummy < Job
|
14
|
+
def perform(_output, _payload, _params)
|
15
|
+
nil
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Output a simple message to the output.
|
13
|
+
class Echo < Job
|
14
|
+
attr_reader :message
|
15
|
+
|
16
|
+
def initialize(name:, message: '')
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@message = message.to_s
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(output, _payload, params)
|
25
|
+
compiled_message = eval_string_template(message, params)
|
26
|
+
|
27
|
+
output.detail(compiled_message)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module IO
|
13
|
+
# Common configuration/code for all IO Job subclasses.
|
14
|
+
class Base < Job
|
15
|
+
attr_reader :binary, :path
|
16
|
+
|
17
|
+
def initialize(name:, path:, binary: false)
|
18
|
+
super(name: name)
|
19
|
+
|
20
|
+
raise ArgumentError, 'path is required' if path.to_s.empty?
|
21
|
+
|
22
|
+
@path = path.to_s
|
23
|
+
@binary = binary || false
|
24
|
+
|
25
|
+
freeze
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def compile_path(params)
|
31
|
+
eval_string_template(path, params)
|
32
|
+
end
|
33
|
+
|
34
|
+
def mode
|
35
|
+
binary ? 'wb' : 'w'
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Read value from disk.
|
16
|
+
class Read < Base
|
17
|
+
def perform(output, payload, params)
|
18
|
+
compiled_path = compile_path(params)
|
19
|
+
|
20
|
+
output.detail("Reading: #{compiled_path}")
|
21
|
+
|
22
|
+
payload.value = File.open(compiled_path, mode, &:read)
|
23
|
+
|
24
|
+
nil
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def mode
|
30
|
+
binary ? 'rb' : 'r'
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'base'
|
11
|
+
|
12
|
+
module Burner
|
13
|
+
class Jobs
|
14
|
+
module IO
|
15
|
+
# Write value to disk.
|
16
|
+
class Write < Base
|
17
|
+
def perform(output, payload, params)
|
18
|
+
compiled_path = compile_path(params)
|
19
|
+
|
20
|
+
ensure_directory_exists(output, compiled_path)
|
21
|
+
|
22
|
+
output.detail("Writing: #{compiled_path}")
|
23
|
+
|
24
|
+
File.open(compiled_path, mode) { |io| io.write(payload.value) }
|
25
|
+
|
26
|
+
nil
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def ensure_directory_exists(output, compiled_path)
|
32
|
+
dirname = File.dirname(compiled_path)
|
33
|
+
|
34
|
+
return if File.exist?(dirname)
|
35
|
+
|
36
|
+
output.detail("Outer directory does not exist, creating: #{dirname}")
|
37
|
+
|
38
|
+
FileUtils.mkdir_p(dirname)
|
39
|
+
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Treat value like a Ruby object and serialize it using JSON.
|
14
|
+
class Json < Job
|
15
|
+
def perform(_output, payload, _params)
|
16
|
+
payload.value = payload.value.to_json
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
module Serialize
|
13
|
+
# Treat value like a Ruby object and serialize it using YAML.
|
14
|
+
class Yaml < Job
|
15
|
+
def perform(_output, payload, _params)
|
16
|
+
payload.value = payload.value.to_yaml
|
17
|
+
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Arbitrarily set value
|
13
|
+
class Set < Job
|
14
|
+
attr_reader :value
|
15
|
+
|
16
|
+
def initialize(name:, value: nil)
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@value = value
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(_output, payload, _params)
|
25
|
+
payload.value = value
|
26
|
+
|
27
|
+
nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
class Jobs
|
12
|
+
# Arbitrarily put thread to sleep for X number of seconds
|
13
|
+
class Sleep < Job
|
14
|
+
attr_reader :seconds
|
15
|
+
|
16
|
+
def initialize(name:, seconds: 0)
|
17
|
+
super(name: name)
|
18
|
+
|
19
|
+
@seconds = seconds.to_f
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(output, _payload, _params)
|
25
|
+
output.detail("Going to sleep for #{seconds} second(s)")
|
26
|
+
|
27
|
+
Kernel.sleep(seconds)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# A Pipeline execution can write main output, which is really an outline to whats happening,
|
12
|
+
# step-by-step. This is not meant to replace or be true logging, but serve more as a summary
|
13
|
+
# for each of the jobs. Each job can decide what it would like to include in this summary
|
14
|
+
# and leverage an instance of this class for inclusion of that information.
|
15
|
+
class Output
|
16
|
+
RULER_LENGTH = 80
|
17
|
+
|
18
|
+
attr_reader :id, :job_count, :outs
|
19
|
+
|
20
|
+
def initialize(id: SecureRandom.uuid, outs: [$stdout])
|
21
|
+
@id = id
|
22
|
+
@outs = Array(outs)
|
23
|
+
@job_count = 1
|
24
|
+
end
|
25
|
+
|
26
|
+
def ruler
|
27
|
+
write('-' * RULER_LENGTH)
|
28
|
+
|
29
|
+
self
|
30
|
+
end
|
31
|
+
|
32
|
+
def title(message)
|
33
|
+
write("[#{job_count}] #{message}")
|
34
|
+
|
35
|
+
@job_count += 1
|
36
|
+
|
37
|
+
self
|
38
|
+
end
|
39
|
+
|
40
|
+
def detail(message)
|
41
|
+
write(" - #{message}")
|
42
|
+
|
43
|
+
self
|
44
|
+
end
|
45
|
+
|
46
|
+
def write(message)
|
47
|
+
raw("[#{id} | #{Time.now.utc}] #{message}")
|
48
|
+
|
49
|
+
self
|
50
|
+
end
|
51
|
+
|
52
|
+
def complete(time_in_seconds)
|
53
|
+
detail("Completed in: #{time_in_seconds.round(3)} second(s)")
|
54
|
+
|
55
|
+
self
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def raw(message)
|
61
|
+
outs.each { |out| out.puts(message) }
|
62
|
+
|
63
|
+
self
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# The input for all Job#perform methods. The main notion of this object is its "value"
|
12
|
+
# attribute. This is dynamic and weak on purpose and is subject to whatever the Job#perform
|
13
|
+
# methods decides it is. This definitely adds an order-of-magnitude complexity to this whole
|
14
|
+
# library and lifecycle, but I am not sure there is any other way around it: trying to build
|
15
|
+
# a generic, open-ended object pipeline to serve almost any use case.
|
16
|
+
class Payload
|
17
|
+
attr_accessor :value
|
18
|
+
|
19
|
+
attr_reader :context
|
20
|
+
|
21
|
+
def initialize(context: {}, value: nil)
|
22
|
+
@context = context || {}
|
23
|
+
@value = value
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require_relative 'jobs'
|
11
|
+
require_relative 'output'
|
12
|
+
require_relative 'payload'
|
13
|
+
require_relative 'step'
|
14
|
+
|
15
|
+
module Burner
|
16
|
+
# The root package. A Pipeline contains the job configurations along with the steps. The steps
|
17
|
+
# referens jobs and tell you the order of the jobs to run.
|
18
|
+
class Pipeline
|
19
|
+
acts_as_hashable
|
20
|
+
|
21
|
+
class JobNotFoundError < StandardError; end
|
22
|
+
|
23
|
+
attr_reader :steps
|
24
|
+
|
25
|
+
def initialize(jobs: [], steps: [])
|
26
|
+
jobs_by_name = Jobs.array(jobs).map { |job| [job.name, job] }.to_h
|
27
|
+
|
28
|
+
@steps = Array(steps).map do |step_name|
|
29
|
+
job = jobs_by_name[step_name.to_s]
|
30
|
+
|
31
|
+
raise JobNotFoundError, "#{step_name} was not declared as a job" unless job
|
32
|
+
|
33
|
+
Step.new(job)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# The main entry-point for kicking off a pipeline.
|
38
|
+
def execute(params: {}, output: Output.new, payload: Payload.new)
|
39
|
+
output.write("Pipeline started with #{steps.length} step(s)")
|
40
|
+
|
41
|
+
output_params(params, output)
|
42
|
+
output.ruler
|
43
|
+
|
44
|
+
time_in_seconds = Benchmark.measure do
|
45
|
+
steps.each { |step| step.perform(output, payload, params) }
|
46
|
+
end.real.round(3)
|
47
|
+
|
48
|
+
output.ruler
|
49
|
+
output.write("Pipeline ended, took #{time_in_seconds} second(s) to complete")
|
50
|
+
|
51
|
+
payload
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def output_params(params, output)
|
57
|
+
if params.keys.any?
|
58
|
+
output.write('Parameters:')
|
59
|
+
else
|
60
|
+
output.write('No parameters passed in.')
|
61
|
+
end
|
62
|
+
|
63
|
+
params.each { |key, value| output.detail("#{key}: #{value}") }
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/burner/step.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# A wrapper to execute a job (in the context of a Pipeline.)
|
12
|
+
class Step
|
13
|
+
extend Forwardable
|
14
|
+
|
15
|
+
SEPARATOR = '::'
|
16
|
+
|
17
|
+
private_constant :SEPARATOR
|
18
|
+
|
19
|
+
attr_reader :job
|
20
|
+
|
21
|
+
def_delegators :job, :name
|
22
|
+
|
23
|
+
def initialize(job)
|
24
|
+
raise ArgumentError, 'job is required' unless job
|
25
|
+
|
26
|
+
@job = job
|
27
|
+
|
28
|
+
freeze
|
29
|
+
end
|
30
|
+
|
31
|
+
def perform(output, payload, params)
|
32
|
+
output.title("#{job.class.name}#{SEPARATOR}#{job.name}")
|
33
|
+
|
34
|
+
time_in_seconds = Benchmark.measure do
|
35
|
+
job.perform(output, payload, params)
|
36
|
+
end.real.round(3)
|
37
|
+
|
38
|
+
output.complete(time_in_seconds)
|
39
|
+
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Burner
|
11
|
+
# Can take in a string and an object and use the object for formatting string interpolations
|
12
|
+
# using tokens of form: {attribute_name}. It can also understand dot-notation for nested
|
13
|
+
# objects using the Objectable library. Another benefit of using Objectable for resolution
|
14
|
+
# is that it can understand almost any type of object: Hash, Struct, OpenStruct, custom
|
15
|
+
# objects, etc. For more information see underlying libraries:
|
16
|
+
# * Stringento: https://github.com/bluemarblepayroll/stringento
|
17
|
+
# * Objectable: https://github.com/bluemarblepayroll/objectable
|
18
|
+
class StringTemplate
|
19
|
+
include Singleton
|
20
|
+
|
21
|
+
attr_reader :resolver
|
22
|
+
|
23
|
+
def initialize
|
24
|
+
@resolver = Objectable.resolver
|
25
|
+
|
26
|
+
freeze
|
27
|
+
end
|
28
|
+
|
29
|
+
# For general consumption
|
30
|
+
def evaluate(expression, input)
|
31
|
+
Stringento.evaluate(expression, input, resolver: self)
|
32
|
+
end
|
33
|
+
|
34
|
+
# For Stringento consumption
|
35
|
+
def resolve(value, input)
|
36
|
+
resolver.get(input, value)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/burner/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: burner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 1.0.0.pre.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-10-
|
11
|
+
date: 2020-10-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: acts_as_hashable
|
@@ -154,7 +154,8 @@ description: " This library serves as the skeleton for a processing engine.
|
|
154
154
|
allows you to organize your code into Jobs, then stitch those jobs together as steps.\n"
|
155
155
|
email:
|
156
156
|
- mruggio@bluemarblepayroll.com
|
157
|
-
executables:
|
157
|
+
executables:
|
158
|
+
- burner
|
158
159
|
extensions: []
|
159
160
|
extra_rdoc_files: []
|
160
161
|
files:
|
@@ -173,7 +174,27 @@ files:
|
|
173
174
|
- bin/console
|
174
175
|
- burner.gemspec
|
175
176
|
- exe/.gitkeep
|
177
|
+
- exe/burner
|
176
178
|
- lib/burner.rb
|
179
|
+
- lib/burner/cli.rb
|
180
|
+
- lib/burner/job.rb
|
181
|
+
- lib/burner/jobs.rb
|
182
|
+
- lib/burner/jobs/deserialize/json.rb
|
183
|
+
- lib/burner/jobs/deserialize/yaml.rb
|
184
|
+
- lib/burner/jobs/dummy.rb
|
185
|
+
- lib/burner/jobs/echo.rb
|
186
|
+
- lib/burner/jobs/io/base.rb
|
187
|
+
- lib/burner/jobs/io/read.rb
|
188
|
+
- lib/burner/jobs/io/write.rb
|
189
|
+
- lib/burner/jobs/serialize/json.rb
|
190
|
+
- lib/burner/jobs/serialize/yaml.rb
|
191
|
+
- lib/burner/jobs/set.rb
|
192
|
+
- lib/burner/jobs/sleep.rb
|
193
|
+
- lib/burner/output.rb
|
194
|
+
- lib/burner/payload.rb
|
195
|
+
- lib/burner/pipeline.rb
|
196
|
+
- lib/burner/step.rb
|
197
|
+
- lib/burner/string_template.rb
|
177
198
|
- lib/burner/version.rb
|
178
199
|
homepage: https://github.com/bluemarblepayroll/burner
|
179
200
|
licenses:
|
@@ -195,9 +216,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
195
216
|
version: '2.5'
|
196
217
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
197
218
|
requirements:
|
198
|
-
- - "
|
219
|
+
- - ">"
|
199
220
|
- !ruby/object:Gem::Version
|
200
|
-
version:
|
221
|
+
version: 1.3.1
|
201
222
|
requirements: []
|
202
223
|
rubygems_version: 3.0.3
|
203
224
|
signing_key:
|