pdi 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -2
- data/.ruby-version +1 -1
- data/.travis.yml +3 -4
- data/CHANGELOG.md +6 -0
- data/README.md +9 -1
- data/lib/pdi/executor.rb +38 -24
- data/lib/pdi/spoon.rb +9 -4
- data/lib/pdi/version.rb +1 -1
- data/pdi.gemspec +2 -2
- data/spec/pdi/executor_spec.rb +13 -1
- data/spec/pdi/spoon_spec.rb +13 -8
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7aad69dc7048e9bc7b4c017d42b2343bca49c98718d1ba3d44981bb4434562f4
|
4
|
+
data.tar.gz: 5405f5f819e2ae116de4831d5a609b32cbf0d08b9117992f3126ca0eefc39e67
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e3fbfc5cb28c486153acda733620c63c21ccb7a7521cb8534de0c153364ff01d12f0af0fcd1a495eeaa8c146430faf002f3c9bd06d1e1ce6a9b35df12af6f76
|
7
|
+
data.tar.gz: c220cf297f8b8a3377fc1f6287e236efdb1549867680bdf6e52a3e9b905e6e717e4420df093d1be537dcb768540a8040e23bac3eaea12aa49cfbb1282c2ef8ff
|
data/.rubocop.yml
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.6.
|
1
|
+
2.6.6
|
data/.travis.yml
CHANGED
@@ -4,10 +4,9 @@ env:
|
|
4
4
|
language: ruby
|
5
5
|
rvm:
|
6
6
|
# Build on the latest stable of all supported Rubies (https://www.ruby-lang.org/en/downloads/):
|
7
|
-
- 2.
|
8
|
-
- 2.
|
9
|
-
- 2.
|
10
|
-
- 2.6.5
|
7
|
+
- 2.5.8
|
8
|
+
- 2.6.6
|
9
|
+
- 2.7.1
|
11
10
|
cache: bundler
|
12
11
|
before_script:
|
13
12
|
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -70,13 +70,21 @@ result = spoon.run(options)
|
|
70
70
|
|
71
71
|
`Spoon#run` will return:
|
72
72
|
|
73
|
-
* `Pdi::
|
73
|
+
* `Pdi::Executor::Result` upon a successful run.
|
74
74
|
* If a non-zero exit code was returned then a `Pdi::Spoon::PanError` or `Pdi::Spoon::KitchenError` will be raised.
|
75
75
|
|
76
76
|
You can access the raw command line results by tapping into the execution attribute of the result or error object.
|
77
77
|
|
78
78
|
Note: Not all options are currently supported. See PDI's official references for [Pan](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/000) and [Kitchen](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/010) to see all options.
|
79
79
|
|
80
|
+
### Output
|
81
|
+
|
82
|
+
There are two ways to see the output of a `Pdi::Spoon` run. First, the output is available when a run completes through `Pdi::Executor::Result#out`. It is also possible to get the output throughout the run by passing a block to run. For example:
|
83
|
+
|
84
|
+
````ruby
|
85
|
+
spoon.run(options) { |output| print output }
|
86
|
+
````
|
87
|
+
|
80
88
|
## Contributing
|
81
89
|
|
82
90
|
### Development Environment Configuration
|
data/lib/pdi/executor.rb
CHANGED
@@ -21,34 +21,48 @@ module Pdi
|
|
21
21
|
freeze
|
22
22
|
end
|
23
23
|
|
24
|
-
def run(args)
|
24
|
+
def run(args, &streaming_reader)
|
25
25
|
args = Array(args).map(&:to_s)
|
26
26
|
|
27
27
|
IO.popen(args, err: %i[child out]) do |io|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
28
|
+
io_read = read(io, &streaming_reader)
|
29
|
+
io.close
|
30
|
+
status = $CHILD_STATUS
|
31
|
+
|
32
|
+
Result.new(
|
33
|
+
args: args,
|
34
|
+
status: {
|
35
|
+
code: status.exitstatus,
|
36
|
+
out: io_read,
|
37
|
+
pid: status.pid
|
38
|
+
}
|
39
|
+
)
|
40
|
+
rescue Timeout::Error => e
|
41
|
+
Process.kill(9, io.pid)
|
42
|
+
raise e
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def read(io, &streaming_reader)
|
49
|
+
if timeout_in_seconds
|
50
|
+
Timeout.timeout(timeout_in_seconds) { streamed_read(io, &streaming_reader) }
|
51
|
+
else
|
52
|
+
streamed_read(io, &streaming_reader)
|
51
53
|
end
|
52
54
|
end
|
55
|
+
|
56
|
+
def streamed_read(io, &streaming_reader)
|
57
|
+
return io.read unless block_given?
|
58
|
+
|
59
|
+
read = ''
|
60
|
+
io.each_line do |line_read|
|
61
|
+
streaming_reader.call(line_read)
|
62
|
+
read += line_read
|
63
|
+
end
|
64
|
+
|
65
|
+
read
|
66
|
+
end
|
53
67
|
end
|
54
68
|
end
|
data/lib/pdi/spoon.rb
CHANGED
@@ -60,13 +60,18 @@ module Pdi
|
|
60
60
|
Result.new(result, version_line)
|
61
61
|
end
|
62
62
|
|
63
|
-
# Returns
|
64
|
-
# a PanError (transformation) or KitchenError (job).
|
65
|
-
|
63
|
+
# Returns a <tt>Pdi::Executor::Result</tt> instance when PDI returns error
|
64
|
+
# code 0 or else raises a PanError (transformation) or KitchenError (job).
|
65
|
+
#
|
66
|
+
# An optional block may be passed in so that the output of a run is
|
67
|
+
# available in a streaming manner during the run. This block takes one
|
68
|
+
# parameter which is the current chunk of output and is called repeatedly
|
69
|
+
# throughout the run.
|
70
|
+
def run(options, &streaming_reader)
|
66
71
|
options = Options.make(options)
|
67
72
|
all_args = run_args(options)
|
68
73
|
|
69
|
-
executor.run(all_args).tap do |result|
|
74
|
+
executor.run(all_args, &streaming_reader).tap do |result|
|
70
75
|
raise(error_constant(options), result) if result.code != 0
|
71
76
|
end
|
72
77
|
end
|
data/lib/pdi/version.rb
CHANGED
data/pdi.gemspec
CHANGED
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
11
11
|
Pentaho Data Integration allows for the creation of ETL transformation and jobs. This library allows those ETL tasks to be executed from Ruby.
|
12
12
|
DESCRIPTION
|
13
13
|
|
14
|
-
s.authors = ['Matthew Ruggio']
|
14
|
+
s.authors = ['Matthew Ruggio', 'Ryan Gerry']
|
15
15
|
s.email = ['mruggio@bluemarblepayroll.com']
|
16
16
|
s.files = `git ls-files`.split("\n")
|
17
17
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.homepage = 'https://github.com/bluemarblepayroll/pdi'
|
20
20
|
s.license = 'MIT'
|
21
21
|
|
22
|
-
s.required_ruby_version = '>= 2.
|
22
|
+
s.required_ruby_version = '>= 2.5.8'
|
23
23
|
|
24
24
|
s.add_dependency('acts_as_hashable', '~>1')
|
25
25
|
|
data/spec/pdi/executor_spec.rb
CHANGED
@@ -13,6 +13,7 @@ describe Pdi::Executor do
|
|
13
13
|
let(:script) { File.join('spec', 'mocks', 'spoon', 'sleep.sh') }
|
14
14
|
let(:one_second) { 1 }
|
15
15
|
let(:return_code) { 33 }
|
16
|
+
let(:script_output) { "std_out\nerr_out\nafter_sleep\n" }
|
16
17
|
|
17
18
|
describe '#run' do
|
18
19
|
context 'with a timeout' do
|
@@ -45,7 +46,18 @@ describe Pdi::Executor do
|
|
45
46
|
|
46
47
|
result = subject.run(args)
|
47
48
|
|
48
|
-
expect(result.out).to eq
|
49
|
+
expect(result.out).to eq script_output
|
50
|
+
end
|
51
|
+
|
52
|
+
describe 'streaming output' do
|
53
|
+
it 'accepts a block and passes in the Pentaho output and populates result output' do
|
54
|
+
found_output = ''
|
55
|
+
|
56
|
+
result = subject.run([script, 0, return_code]) { |output| found_output += output }
|
57
|
+
|
58
|
+
expect(found_output).to eq script_output
|
59
|
+
expect(result.out).to eq script_output
|
60
|
+
end
|
49
61
|
end
|
50
62
|
end
|
51
63
|
end
|
data/spec/pdi/spoon_spec.rb
CHANGED
@@ -25,6 +25,8 @@ describe Pdi::Spoon do
|
|
25
25
|
end
|
26
26
|
|
27
27
|
describe '#run' do
|
28
|
+
let(:script_output) { "output to stdout\noutput to sterr\n" }
|
29
|
+
|
28
30
|
context 'transformations' do
|
29
31
|
let(:options) do
|
30
32
|
{
|
@@ -43,19 +45,22 @@ describe Pdi::Spoon do
|
|
43
45
|
end
|
44
46
|
|
45
47
|
context 'when code is 0' do
|
46
|
-
|
47
|
-
subject = described_class.new(
|
48
|
-
args: 0,
|
49
|
-
dir: dir,
|
50
|
-
kitchen: script,
|
51
|
-
pan: script
|
52
|
-
)
|
48
|
+
let(:subject) { described_class.new(args: 0, dir: dir, kitchen: script, pan: script) }
|
53
49
|
|
50
|
+
it 'returns correct stdout, stderr and code' do
|
54
51
|
result = subject.run(options)
|
55
52
|
|
56
|
-
expect(result.out).to
|
53
|
+
expect(result.out).to eq script_output
|
57
54
|
expect(result.code).to eq(0)
|
58
55
|
end
|
56
|
+
|
57
|
+
it 'allows for streaming output when passed a block' do
|
58
|
+
found_output = ''
|
59
|
+
|
60
|
+
subject.run(options) { |output| found_output += output }
|
61
|
+
|
62
|
+
expect(found_output).to eq script_output
|
63
|
+
end
|
59
64
|
end
|
60
65
|
|
61
66
|
[1, 2, 3, 7, 8, 9].each do |code|
|
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pdi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matthew Ruggio
|
8
|
+
- Ryan Gerry
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2020-
|
12
|
+
date: 2020-06-09 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: acts_as_hashable
|
@@ -182,7 +183,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
182
183
|
requirements:
|
183
184
|
- - ">="
|
184
185
|
- !ruby/object:Gem::Version
|
185
|
-
version: 2.
|
186
|
+
version: 2.5.8
|
186
187
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
187
188
|
requirements:
|
188
189
|
- - ">="
|