pdi 2.0.0.pre.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.editorconfig +8 -0
- data/.gitignore +6 -0
- data/.rubocop.yml +25 -0
- data/.ruby-version +1 -0
- data/.travis.yml +25 -0
- data/CHANGELOG.md +19 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +5 -0
- data/Guardfile +16 -0
- data/LICENSE +7 -0
- data/README.md +137 -0
- data/Rakefile +17 -0
- data/bin/console +18 -0
- data/lib/pdi.rb +17 -0
- data/lib/pdi/executor.rb +54 -0
- data/lib/pdi/executor/result.rb +31 -0
- data/lib/pdi/executor/status.rb +27 -0
- data/lib/pdi/spoon.rb +106 -0
- data/lib/pdi/spoon/kitchen_error.rb +36 -0
- data/lib/pdi/spoon/options.rb +82 -0
- data/lib/pdi/spoon/options/arg.rb +68 -0
- data/lib/pdi/spoon/options/level.rb +23 -0
- data/lib/pdi/spoon/options/param.rb +29 -0
- data/lib/pdi/spoon/pan_error.rb +37 -0
- data/lib/pdi/spoon/parser.rb +25 -0
- data/lib/pdi/spoon/result.rb +25 -0
- data/lib/pdi/version.rb +12 -0
- data/pdi.gemspec +33 -0
- data/spec/mocks/spoon/return_code.sh +14 -0
- data/spec/mocks/spoon/sleep.sh +10 -0
- data/spec/mocks/spoon/version.sh +10 -0
- data/spec/pdi/executor_spec.rb +52 -0
- data/spec/pdi/spoon/kitchen_error_spec.rb +42 -0
- data/spec/pdi/spoon/options/arg_spec.rb +73 -0
- data/spec/pdi/spoon/options/param_spec.rb +39 -0
- data/spec/pdi/spoon/options_spec.rb +102 -0
- data/spec/pdi/spoon/pan_error_spec.rb +42 -0
- data/spec/pdi/spoon_spec.rb +175 -0
- data/spec/spec_helper.rb +22 -0
- metadata +207 -0
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Pdi
|
11
|
+
class Spoon
|
12
|
+
# This class knows how to parse feedback from Pentaho and make sense of it.
|
13
|
+
class Parser
|
14
|
+
NEW_LINE = "\n"
|
15
|
+
|
16
|
+
def version(string)
|
17
|
+
string.to_s
|
18
|
+
.chomp
|
19
|
+
.split(NEW_LINE)
|
20
|
+
.last
|
21
|
+
.to_s
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Pdi
|
11
|
+
class Spoon
|
12
|
+
# General return object for wrapping up a execution call result (execution) and
|
13
|
+
# a usable result (value)
|
14
|
+
class Result
|
15
|
+
attr_reader :execution, :value
|
16
|
+
|
17
|
+
def initialize(execution, value)
|
18
|
+
@execution = execution
|
19
|
+
@value = value
|
20
|
+
|
21
|
+
freeze
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/pdi/version.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2020-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
module Pdi
|
11
|
+
VERSION = '2.0.0-alpha'
|
12
|
+
end
|
data/pdi.gemspec
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require './lib/pdi/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = 'pdi'
|
7
|
+
s.version = Pdi::VERSION
|
8
|
+
s.summary = 'Ruby wrapper for invoking Pentaho Data Integration'
|
9
|
+
|
10
|
+
s.description = <<-DESCRIPTION
|
11
|
+
Pentaho Data Integration allows for the creation of ETL transformation and jobs. This library allows those ETL tasks to be executed from Ruby.
|
12
|
+
DESCRIPTION
|
13
|
+
|
14
|
+
s.authors = ['Matthew Ruggio']
|
15
|
+
s.email = ['mruggio@bluemarblepayroll.com']
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
19
|
+
s.homepage = 'https://github.com/bluemarblepayroll/pdi'
|
20
|
+
s.license = 'MIT'
|
21
|
+
|
22
|
+
s.required_ruby_version = '>= 2.3.8'
|
23
|
+
|
24
|
+
s.add_dependency('acts_as_hashable', '~>1')
|
25
|
+
|
26
|
+
s.add_development_dependency('guard-rspec', '~>4.7')
|
27
|
+
s.add_development_dependency('pry', '~>0')
|
28
|
+
s.add_development_dependency('rake', '~> 13')
|
29
|
+
s.add_development_dependency('rspec')
|
30
|
+
s.add_development_dependency('rubocop', '~>0.79.0')
|
31
|
+
s.add_development_dependency('simplecov', '~>0.17.0')
|
32
|
+
s.add_development_dependency('simplecov-console', '~>0.6.0')
|
33
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2018-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'spec_helper'
|
11
|
+
|
12
|
+
describe Pdi::Executor do
|
13
|
+
let(:script) { File.join('spec', 'mocks', 'spoon', 'sleep.sh') }
|
14
|
+
let(:one_second) { 1 }
|
15
|
+
let(:return_code) { 33 }
|
16
|
+
|
17
|
+
describe '#run' do
|
18
|
+
context 'with a timeout' do
|
19
|
+
# do not make these too high, bugs could cause the entire script to still be executed without
|
20
|
+
# killing it so these high values could draw out results.
|
21
|
+
let(:thirty_seconds) { 30 }
|
22
|
+
|
23
|
+
subject { described_class.new(timeout_in_seconds: one_second) }
|
24
|
+
|
25
|
+
# This will run a script that will take 30 seconds to process, but by limiting the
|
26
|
+
# timeout using the #run argument, it should raise an error after one second.
|
27
|
+
it 'times out and kills process after 5 seconds' do
|
28
|
+
args = [script, thirty_seconds]
|
29
|
+
|
30
|
+
expect { subject.run(args) }.to raise_error(Timeout::Error)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'without a timeout' do
|
35
|
+
it 'returns right exit status as code' do
|
36
|
+
args = [script, one_second, return_code]
|
37
|
+
|
38
|
+
result = subject.run(args)
|
39
|
+
|
40
|
+
expect(result.code).to eq(return_code)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'returns right standard output and error as out' do
|
44
|
+
args = [script, one_second, return_code]
|
45
|
+
|
46
|
+
result = subject.run(args)
|
47
|
+
|
48
|
+
expect(result.out).to eq("std_out\nerr_out\nafter_sleep\n")
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2018-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'spec_helper'
|
11
|
+
|
12
|
+
describe Pdi::Spoon::KitchenError do
|
13
|
+
describe 'initialization' do
|
14
|
+
[1, 2, 7, 8, 9].each do |code|
|
15
|
+
specify "code #{code} should have message" do
|
16
|
+
result = Pdi::Executor::Result.new(
|
17
|
+
args: [],
|
18
|
+
status: {
|
19
|
+
code: code,
|
20
|
+
pid: 123
|
21
|
+
}
|
22
|
+
)
|
23
|
+
|
24
|
+
expect(described_class.new(result).message).not_to eq('Unknown')
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
[-1, 0, 3, 4, 5, 6, 10, 11].each do |code|
|
29
|
+
specify "code #{code} should not have message" do
|
30
|
+
result = Pdi::Executor::Result.new(
|
31
|
+
args: [],
|
32
|
+
status: {
|
33
|
+
code: code,
|
34
|
+
pid: 123
|
35
|
+
}
|
36
|
+
)
|
37
|
+
|
38
|
+
expect(described_class.new(result).message).to eq('Unknown')
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2018-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'spec_helper'
|
11
|
+
|
12
|
+
describe Pdi::Spoon::Options::Arg do
|
13
|
+
let(:key) { :param }
|
14
|
+
let(:value) { 'v1' }
|
15
|
+
let(:value_with_space) { 'v 1' }
|
16
|
+
|
17
|
+
describe '#initialize' do
|
18
|
+
subject { described_class.new(key, value) }
|
19
|
+
|
20
|
+
it 'requires a key' do
|
21
|
+
expect { described_class.new('') }.to raise_error(ArgumentError)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'sets key' do
|
25
|
+
expect(subject.key).to eq(key)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'sets value' do
|
29
|
+
expect(subject.value).to eq(value)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
describe '#to_s' do
|
34
|
+
it 'works without a value' do
|
35
|
+
subject = described_class.new(key)
|
36
|
+
expected = "-#{key}"
|
37
|
+
|
38
|
+
expect(subject.to_s).to eq(expected)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'works with a value (with no space)' do
|
42
|
+
subject = described_class.new(key, value)
|
43
|
+
expected = "-#{key}:#{value}"
|
44
|
+
|
45
|
+
expect(subject.to_s).to eq(expected)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'works with a value (with a space)' do
|
49
|
+
subject = described_class.new(key, value_with_space)
|
50
|
+
expected = "\"-#{key}:#{value_with_space}\""
|
51
|
+
|
52
|
+
expect(subject.to_s).to eq(expected)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
describe 'equality' do
|
57
|
+
let(:subject2) { described_class.new(key, value) }
|
58
|
+
|
59
|
+
subject { described_class.new(key, value) }
|
60
|
+
|
61
|
+
specify '#hash compares key and value' do
|
62
|
+
expect(subject.hash).to eq(subject2.hash)
|
63
|
+
end
|
64
|
+
|
65
|
+
specify '#== compares #to_s values' do
|
66
|
+
expect(subject).to eq(subject2)
|
67
|
+
end
|
68
|
+
|
69
|
+
specify '#eql? compares #to_s values' do
|
70
|
+
expect(subject).to eql(subject2)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2018-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'spec_helper'
|
11
|
+
|
12
|
+
describe Pdi::Spoon::Options::Param do
|
13
|
+
let(:key) { 'k1' }
|
14
|
+
let(:value) { 'v1' }
|
15
|
+
let(:value_with_space) { 'v 1' }
|
16
|
+
|
17
|
+
describe '#to_s' do
|
18
|
+
it 'works without a value' do
|
19
|
+
subject = described_class.new(key)
|
20
|
+
expected = "-param:#{key}="
|
21
|
+
|
22
|
+
expect(subject.to_s).to eq(expected)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'works with a value (with no space)' do
|
26
|
+
subject = described_class.new(key, value)
|
27
|
+
expected = "-param:#{key}=#{value}"
|
28
|
+
|
29
|
+
expect(subject.to_s).to eq(expected)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'works with a value (with a space)' do
|
33
|
+
subject = described_class.new(key, value_with_space)
|
34
|
+
expected = "\"-param:#{key}=#{value_with_space}\""
|
35
|
+
|
36
|
+
expect(subject.to_s).to eq(expected)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Copyright (c) 2018-present, Blue Marble Payroll, LLC
|
5
|
+
#
|
6
|
+
# This source code is licensed under the MIT license found in the
|
7
|
+
# LICENSE file in the root directory of this source tree.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'spec_helper'
|
11
|
+
|
12
|
+
describe Pdi::Spoon::Options do
|
13
|
+
describe '#to_args' do
|
14
|
+
let(:params) do
|
15
|
+
{
|
16
|
+
key1: 'value1',
|
17
|
+
key2: 'value2',
|
18
|
+
key3: 'value with spaces'
|
19
|
+
}
|
20
|
+
end
|
21
|
+
|
22
|
+
let(:repository) { 'some repo' }
|
23
|
+
let(:name) { 'some_transformation' }
|
24
|
+
|
25
|
+
subject do
|
26
|
+
described_class.new(
|
27
|
+
params: params,
|
28
|
+
repository: repository,
|
29
|
+
name: name,
|
30
|
+
type: type
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'when type is transformation' do
|
35
|
+
let(:type) { 'transformation' }
|
36
|
+
|
37
|
+
it 'wraps any args with spaces inside double quotes' do
|
38
|
+
args = subject.to_args
|
39
|
+
args_with_spaces = args.select { |a| a.to_s.include?(' ') }
|
40
|
+
|
41
|
+
raise ArgumentError, 'no examples to assert!' if args_with_spaces.empty?
|
42
|
+
|
43
|
+
args_with_spaces.each do |arg|
|
44
|
+
actual = arg.to_s
|
45
|
+
|
46
|
+
expect(actual).to start_with('"')
|
47
|
+
expect(actual).to end_with('"')
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'includes each option' do
|
52
|
+
expected = [
|
53
|
+
'"-rep:some repo"',
|
54
|
+
'-trans:some_transformation',
|
55
|
+
'-level:Basic',
|
56
|
+
'-param:key1=value1',
|
57
|
+
'-param:key2=value2',
|
58
|
+
'"-param:key3=value with spaces"'
|
59
|
+
]
|
60
|
+
|
61
|
+
expected_set = Set[*expected]
|
62
|
+
actual_set = Set[*subject.to_args.map(&:to_s)]
|
63
|
+
|
64
|
+
expect(actual_set).to eq(expected_set)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'when type is job' do
|
69
|
+
let(:type) { 'job' }
|
70
|
+
|
71
|
+
it 'wraps any args with spaces inside double quotes' do
|
72
|
+
args = subject.to_args
|
73
|
+
args_with_spaces = args.select { |a| a.to_s.include?(' ') }
|
74
|
+
|
75
|
+
raise ArgumentError, 'no examples to assert!' if args_with_spaces.empty?
|
76
|
+
|
77
|
+
args_with_spaces.each do |arg|
|
78
|
+
actual = arg.to_s
|
79
|
+
|
80
|
+
expect(actual).to start_with('"')
|
81
|
+
expect(actual).to end_with('"')
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
it 'includes each option' do
|
86
|
+
expected = [
|
87
|
+
'"-rep:some repo"',
|
88
|
+
'-job:some_transformation',
|
89
|
+
'-level:Basic',
|
90
|
+
'-param:key1=value1',
|
91
|
+
'-param:key2=value2',
|
92
|
+
'"-param:key3=value with spaces"'
|
93
|
+
]
|
94
|
+
|
95
|
+
expected_set = Set[*expected]
|
96
|
+
actual_set = Set[*subject.to_args.map(&:to_s)]
|
97
|
+
|
98
|
+
expect(actual_set).to eq(expected_set)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|