pdi 1.0.0.pre.alpha.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e653f6aed9fd26cce60e4f84cfdf662e0be847f40828142dd28fcc6118150cc7
-  data.tar.gz: 6ab7d8da5c628dc867d0607bcb3a3db6a5b4c030f07d91adc3f47516a2b3350f
+  metadata.gz: 7aad69dc7048e9bc7b4c017d42b2343bca49c98718d1ba3d44981bb4434562f4
+  data.tar.gz: 5405f5f819e2ae116de4831d5a609b32cbf0d08b9117992f3126ca0eefc39e67
 SHA512:
-  metadata.gz: 2e2688a153fea7a256e80a9e682b6488e71923d2752e32f37c23a39c7899f33faf2ffd99c91b5770e4e4279dd0af0ea9f3bdd6d52057b9780d6f798f7a44fedc
-  data.tar.gz: 66f618cc1203d313091cd849403501c127907a719f2b6503792cffe606f02062d0f2f31e179a45395e4460c47245de5461c1b78496208ad5ddf68437f7889558
+  metadata.gz: 2e3fbfc5cb28c486153acda733620c63c21ccb7a7521cb8534de0c153364ff01d12f0af0fcd1a495eeaa8c146430faf002f3c9bd06d1e1ce6a9b35df12af6f76
+  data.tar.gz: c220cf297f8b8a3377fc1f6287e236efdb1549867680bdf6e52a3e9b905e6e717e4420df093d1be537dcb768540a8040e23bac3eaea12aa49cfbb1282c2ef8ff

data/.rubocop.yml

@@ -1,4 +1,4 @@
-Metrics/LineLength:
+Layout/LineLength:
   Max: 100
   Exclude:
     - pdi.gemspec
@@ -13,10 +13,10 @@ Metrics/BlockLength:
     - define
 
 Metrics/MethodLength:
-  Max: 25
+  Max: 17
 
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
 
 Metrics/AbcSize:
   Max: 16

data/.ruby-version

@@ -1 +1 @@
-2.6.5
+2.6.6

data/.travis.yml

@@ -4,10 +4,9 @@ env:
 language: ruby
 rvm:
   # Build on the latest stable of all supported Rubies (https://www.ruby-lang.org/en/downloads/):
-  - 2.3.8
-  - 2.4.6
-  - 2.5.5
-  - 2.6.5
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 cache: bundler
 before_script:
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter

data/CHANGELOG.md

@@ -0,0 +1,25 @@
+# 2.1.0 (June 3rd, 2020)
+
+Enhancements:
+
+* Support for streamed command output.
+
+# 2.0.0 (May 11th, 2020)
+
+Breaking Changes:
+
+* Standard error and output have been combined into one stream (out).  Early feedback indicated that reading both at the same time was preferable.
+
+Enhancements:
+
+* Added optional `timeout_in_seconds` argument to Pdi::Spoon#initialize.
+
+# 1.0.1 (February 19th, 2020)
+
+Fixes:
+
+* It now properly calls kitchen for jobs (it was previously calling pan.)
+
+# 1.0.0 (February 19th, 2020)
+
+Initial release.

data/README.md

@@ -1,4 +1,4 @@
-# Pdi
+# PDI
 
 ---
 
@@ -36,17 +36,21 @@ Pull Requests are welcome for:
 
 ## Examples
 
-All examples assume Pdi has been installed to your home directory: `~/data-integration`.
+All examples assume PDI has been installed to your home directory: `~/data-integration`.
 
 ### Creating a Spoon Instance
 
-`Pdi::Spoon` is the common interface you will use when interacting with Pdi.  It will use Pan and Kitchen for executing Spoon commands.
+`Pdi::Spoon` is the common interface you will use when interacting with PDI.  It will use Pan and Kitchen for executing Spoon commands.
 
 ```ruby
-spoon = Pdi::Spoon.new(dir: `~/data-integration`)
+spoon = Pdi::Spoon.new(dir: '~/data-integration')
 ```
 
-Note: You can also override the names of the scripts using the `kitchen` and `pan` constructor keyword arguments.  The defaults are `kitchen.sh` and `pan.sh`, respectively.
+Notes:
+
+* You can also override the names of the scripts using the `kitchen` and `pan` constructor keyword arguments.  The defaults are `kitchen.sh` and `pan.sh`, respectively.
+* For other command line arguments that are not supported first-class in the Options objects below you can utilize the `args` argument when instantiating a `Spoon` instance.
+* Another optional argument is `timeout_in_seconds`.  It is set to `nil` by default which means there is no timeout.  If set it will ensure the sub-process runs within a given window.  If it times out the sub-process will be terminated and a Timeout::Error will be raised.
 
 ### Executing a Job/Transformation
 
@@ -66,12 +70,20 @@ result = spoon.run(options)
 
 `Spoon#run` will return:
 
-* `Pdi::Spoon::Result` upon a successful run.
+* `Pdi::Executor::Result` upon a successful run.
 * If a non-zero exit code was returned then a `Pdi::Spoon::PanError` or `Pdi::Spoon::KitchenError` will be raised.
 
 You can access the raw command line results by tapping into the execution attribute of the result or error object.
 
-Note: Not all options are currently supported.  See Pdi's official references for [Pan](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/000) and [Kitchen](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/010) to see all options.
+Note: Not all options are currently supported.  See PDI's official references for [Pan](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/000) and [Kitchen](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/010) to see all options.
+
+### Output
+
+There are two ways to see the output of a `Pdi::Spoon` run.  First, the output is available when a run completes through `Pdi::Executor::Result#out`. It is also possible to get the output throughout the run by passing a block to run. For example:
+
+````ruby
+spoon.run(options) { |output| print output }
+````
 
 ## Contributing
 

data/lib/pdi.rb

@@ -8,8 +8,10 @@
 #
 
 require 'acts_as_hashable'
+require 'English'
 require 'forwardable'
 require 'open3'
+require 'timeout'
 
 require_relative 'pdi/executor'
 require_relative 'pdi/spoon'

data/lib/pdi/executor.rb

@@ -13,20 +13,56 @@ module Pdi
   # This class is the library's "metal" layer, the one which actually makes the system call and
   # interacts with the operating system (through Ruby's standard library.)
   class Executor
-    def run(args)
+    attr_reader :timeout_in_seconds
+
+    def initialize(timeout_in_seconds: nil)
+      @timeout_in_seconds = timeout_in_seconds
+
+      freeze
+    end
+
+    def run(args, &streaming_reader)
       args = Array(args).map(&:to_s)
 
-      out, err, status = Open3.capture3(*args)
-
-      Result.new(
-        args: args,
-        status: {
-          code: status.exitstatus,
-          out: out,
-          err: err,
-          pid: status.pid
-        }
-      )
+      IO.popen(args, err: %i[child out]) do |io|
+        io_read = read(io, &streaming_reader)
+        io.close
+        status = $CHILD_STATUS
+
+        Result.new(
+          args: args,
+          status: {
+            code: status.exitstatus,
+            out: io_read,
+            pid: status.pid
+          }
+        )
+      rescue Timeout::Error => e
+        Process.kill(9, io.pid)
+        raise e
+      end
+    end
+
+    private
+
+    def read(io, &streaming_reader)
+      if timeout_in_seconds
+        Timeout.timeout(timeout_in_seconds) { streamed_read(io, &streaming_reader) }
+      else
+        streamed_read(io, &streaming_reader)
+      end
+    end
+
+    def streamed_read(io, &streaming_reader)
+      return io.read unless block_given?
+
+      read = ''
+      io.each_line do |line_read|
+        streaming_reader.call(line_read)
+        read += line_read
+      end
+
+      read
     end
   end
 end

data/lib/pdi/executor/result.rb

@@ -18,7 +18,7 @@ module Pdi
 
       attr_reader :args, :status
 
-      def_delegators :status, :code, :out, :err, :pid
+      def_delegators :status, :code, :out, :pid
 
       def initialize(args:, status: {})
         @args   = args

data/lib/pdi/executor/status.rb

@@ -13,12 +13,11 @@ module Pdi
     class Status
       acts_as_hashable
 
-      attr_reader :code, :out, :err, :pid
+      attr_reader :code, :out, :pid
 
-      def initialize(code:, out: '', err: '', pid:)
+      def initialize(code:, out: '', pid:)
         @code = code
         @out  = out
-        @err  = err
         @pid  = pid
 
         freeze

data/lib/pdi/spoon.rb

@@ -7,33 +7,41 @@
 # LICENSE file in the root directory of this source tree.
 #
 
+require_relative 'spoon/kitchen_error'
 require_relative 'spoon/options'
+require_relative 'spoon/pan_error'
 require_relative 'spoon/parser'
 require_relative 'spoon/result'
 
 module Pdi
-  # This class is the main wrapper for Pdi's pan and kitchen scripts.
+  # This class is the main wrapper for PDI's pan and kitchen scripts.
   class Spoon
     DEFAULT_KITCHEN = 'kitchen.sh'
     DEFAULT_PAN     = 'pan.sh'
 
-    attr_reader :args, :dir, :kitchen, :pan
+    TYPES_TO_ERRORS = {
+      Options::Type::JOB => KitchenError,
+      Options::Type::TRANSFORMATION => PanError
+    }.freeze
+
+    attr_reader :args, :dir, :executor, :kitchen, :pan
 
     def initialize(
       args: [],
       dir:,
       kitchen: DEFAULT_KITCHEN,
-      pan: DEFAULT_PAN
+      pan: DEFAULT_PAN,
+      timeout_in_seconds: nil
     )
       assert_required(:dir, dir)
       assert_required(:kitchen, kitchen)
       assert_required(:pan, pan)
 
       @args           = Array(args)
-      @dir            = dir.to_s
+      @dir            = File.expand_path(dir.to_s)
       @kitchen        = kitchen.to_s
       @pan            = pan.to_s
-      @executor       = Executor.new
+      @executor       = Executor.new(timeout_in_seconds: timeout_in_seconds)
       @parser         = Parser.new
 
       freeze
@@ -52,20 +60,41 @@ module Pdi
       Result.new(result, version_line)
     end
 
-    # Returns an Executor::Result instance when PDI returns error code 0 or else raises
-    # a PanError (transformation) or KitchenError (job).
-    def run(options)
-      options    = Options.make(options)
-      final_args = [pan_path] + args + options.to_args
-
-      executor.run(final_args).tap do |result|
-        raise(options.error_constant, result) if result.code != 0
+    # Returns a <tt>Pdi::Executor::Result</tt> instance when PDI returns error
+    # code 0 or else raises a PanError (transformation) or KitchenError (job).
+    #
+    # An optional block may be passed in so that the output of a run is
+    # available in a streaming manner during the run. This block takes one
+    # parameter which is the current chunk of output and is called repeatedly
+    # throughout the run.
+    def run(options, &streaming_reader)
+      options  = Options.make(options)
+      all_args = run_args(options)
+
+      executor.run(all_args, &streaming_reader).tap do |result|
+        raise(error_constant(options), result) if result.code != 0
       end
     end
 
     private
 
-    attr_reader :executor, :parser
+    attr_reader :parser
+
+    def error_constant(options)
+      TYPES_TO_ERRORS.fetch(options.type)
+    end
+
+    def run_args(options)
+      [script_path(options.type)] + args + options.to_args
+    end
+
+    def script_path(options_type)
+      if options_type == Options::Type::JOB
+        kitchen_path
+      elsif options_type == Options::Type::TRANSFORMATION
+        pan_path
+      end
+    end
 
     def kitchen_path
       File.join(dir, kitchen)

data/lib/pdi/spoon/kitchen_error.rb

@@ -11,6 +11,8 @@ module Pdi
   class Spoon
     # This class subclasses Ruby's StandardError and provides a mapping to custom Kitchen
     # specific error codes to messages.
+    # You can find the list of errors in Pentaho's documentation site, for example:
+    # https://help.pentaho.com/Documentation/8.0/Products/Data_Integration/Command_Line_Tools
     class KitchenError < StandardError
       MESSAGES = {
         '1' => 'Errors occurred during processing',

data/lib/pdi/spoon/options.rb

@@ -7,10 +7,8 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'kitchen_error'
 require_relative 'options/level'
 require_relative 'options/param'
-require_relative 'pan_error'
 
 module Pdi
   class Spoon
@@ -23,11 +21,6 @@ module Pdi
         TRANSFORMATION = :transformation
       end
 
-      TYPES_TO_ERRORS = {
-        Type::JOB => KitchenError,
-        Type::TRANSFORMATION => PanError
-      }.freeze
-
       TYPES_TO_KEYS = {
         Type::JOB => Arg::Key::JOB,
         Type::TRANSFORMATION => Arg::Key::TRANS
@@ -63,10 +56,6 @@ module Pdi
         base_args + param_args
       end
 
-      def error_constant
-        TYPES_TO_ERRORS.fetch(type)
-      end
-
       private
 
       def key

data/lib/pdi/spoon/pan_error.rb

@@ -11,6 +11,8 @@ module Pdi
   class Spoon
     # This class subclasses Ruby's StandardError and provides a mapping to custom Pan
     # specific error codes to messages.
+    # You can find the list of errors in Pentaho's documentation site, for example:
+    # https://help.pentaho.com/Documentation/8.0/Products/Data_Integration/Command_Line_Tools
     class PanError < StandardError
       MESSAGES = {
         '1' => 'Errors occurred during processing',

data/lib/pdi/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Pdi
-  VERSION = '1.0.0-alpha.1'
+  VERSION = '2.1.0'
 end

data/pdi.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
     Pentaho Data Integration allows for the creation of ETL transformation and jobs.  This library allows those ETL tasks to be executed from Ruby.
   DESCRIPTION
 
-  s.authors     = ['Matthew Ruggio']
+  s.authors     = ['Matthew Ruggio', 'Ryan Gerry']
   s.email       = ['mruggio@bluemarblepayroll.com']
   s.files       = `git ls-files`.split("\n")
   s.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.homepage    = 'https://github.com/bluemarblepayroll/pdi'
   s.license     = 'MIT'
 
-  s.required_ruby_version = '>= 2.3.8'
+  s.required_ruby_version = '>= 2.5.8'
 
   s.add_dependency('acts_as_hashable', '~>1')
 

data/spec/mocks/spoon/sleep.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo 'std_out'
+echo 'err_out' >&2
+
+sleep $1
+
+echo 'after_sleep'
+
+exit $2

data/spec/pdi/executor_spec.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2018-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe Pdi::Executor do
+  let(:script)      { File.join('spec', 'mocks', 'spoon', 'sleep.sh') }
+  let(:one_second)  { 1 }
+  let(:return_code) { 33 }
+  let(:script_output) { "std_out\nerr_out\nafter_sleep\n" }
+
+  describe '#run' do
+    context 'with a timeout' do
+      # do not make these too high, bugs could cause the entire script to still be executed without
+      # killing it so these high values could draw out results.
+      let(:thirty_seconds) { 30 }
+
+      subject { described_class.new(timeout_in_seconds: one_second) }
+
+      # This will run a script that will take 30 seconds to process, but by limiting the
+      # timeout using the #run argument, it should raise an error after one second.
+      it 'times out and kills process after 1 second' do
+        args = [script, thirty_seconds]
+
+        expect { subject.run(args) }.to raise_error(Timeout::Error)
+      end
+    end
+
+    context 'without a timeout' do
+      it 'returns right exit status as code' do
+        args = [script, one_second, return_code]
+
+        result = subject.run(args)
+
+        expect(result.code).to eq(return_code)
+      end
+
+      it 'returns right standard output and error as out' do
+        args = [script, one_second, return_code]
+
+        result = subject.run(args)
+
+        expect(result.out).to eq script_output
+      end
+
+      describe 'streaming output' do
+        it 'accepts a block and passes in the Pentaho output and populates result output' do
+          found_output = ''
+
+          result = subject.run([script, 0, return_code]) { |output| found_output += output }
+
+          expect(found_output).to eq script_output
+          expect(result.out).to eq script_output
+        end
+      end
+    end
+  end
+end

data/spec/pdi/spoon/kitchen_error_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2018-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe Pdi::Spoon::KitchenError do
+  describe 'initialization' do
+    [1, 2, 7, 8, 9].each do |code|
+      specify "code #{code} should have message" do
+        result = Pdi::Executor::Result.new(
+          args: [],
+          status: {
+            code: code,
+            pid: 123
+          }
+        )
+
+        expect(described_class.new(result).message).not_to eq('Unknown')
+      end
+    end
+
+    [-1, 0, 3, 4, 5, 6, 10, 11].each do |code|
+      specify "code #{code} should not have message" do
+        result = Pdi::Executor::Result.new(
+          args: [],
+          status: {
+            code: code,
+            pid: 123
+          }
+        )
+
+        expect(described_class.new(result).message).to eq('Unknown')
+      end
+    end
+  end
+end

data/spec/pdi/spoon_spec.rb

@@ -14,7 +14,19 @@ describe Pdi::Spoon do
   let(:mocks_dir) { %w[spec mocks spoon] }
   let(:dir)       { File.join(*mocks_dir) }
 
+  describe '#initialize' do
+    it 'sets Executor#timeout_in_seconds' do
+      timeout_in_seconds = 987
+      subject            = described_class.new(dir: dir, timeout_in_seconds: timeout_in_seconds)
+      executor           = subject.executor
+
+      expect(executor.timeout_in_seconds).to eq(timeout_in_seconds)
+    end
+  end
+
   describe '#run' do
+    let(:script_output) { "output to stdout\noutput to sterr\n" }
+
     context 'transformations' do
       let(:options) do
         {
@@ -24,21 +36,31 @@ describe Pdi::Spoon do
         }
       end
 
+      it 'will call pan script' do
+        subject = described_class.new(args: 0, dir: dir, pan: script)
+
+        result = subject.run(options)
+
+        expect(result.args.first).to include(script)
+      end
+
       context 'when code is 0' do
-        it 'returns correct stdout, stderr and code' do
-          subject = described_class.new(
-            args: 0,
-            dir: dir,
-            kitchen: script,
-            pan: script
-          )
+        let(:subject) { described_class.new(args: 0, dir: dir, kitchen: script, pan: script) }
 
+        it 'returns correct stdout, stderr and code' do
           result = subject.run(options)
 
-          expect(result.out).to  eq("output to stdout\n")
-          expect(result.err).to  eq("output to sterr\n")
+          expect(result.out).to eq script_output
           expect(result.code).to eq(0)
         end
+
+        it 'allows for streaming output when passed a block' do
+          found_output = ''
+
+          subject.run(options) { |output| found_output += output }
+
+          expect(found_output).to eq script_output
+        end
       end
 
       [1, 2, 3, 7, 8, 9].each do |code|
@@ -70,6 +92,14 @@ describe Pdi::Spoon do
         }
       end
 
+      it 'will call kitchen script' do
+        subject = described_class.new(args: 0, dir: dir, kitchen: script)
+
+        result = subject.run(options)
+
+        expect(result.args.first).to include(script)
+      end
+
       context 'when code is 0' do
         it 'returns correct stdout, stderr and code' do
           subject = described_class.new(
@@ -81,8 +111,7 @@ describe Pdi::Spoon do
 
           result = subject.run(options)
 
-          expect(result.out).to  eq("output to stdout\n")
-          expect(result.err).to  eq("output to sterr\n")
+          expect(result.out).to  eq("output to stdout\noutput to sterr\n")
           expect(result.code).to eq(0)
         end
       end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: pdi
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.alpha.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Matthew Ruggio
+- Ryan Gerry
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-18 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -160,11 +161,14 @@ files:
 - lib/pdi/version.rb
 - pdi.gemspec
 - spec/mocks/spoon/return_code.sh
+- spec/mocks/spoon/sleep.sh
 - spec/mocks/spoon/version.sh
-- spec/pdi/spoon/error_spec.rb
+- spec/pdi/executor_spec.rb
+- spec/pdi/spoon/kitchen_error_spec.rb
 - spec/pdi/spoon/options/arg_spec.rb
 - spec/pdi/spoon/options/param_spec.rb
 - spec/pdi/spoon/options_spec.rb
+- spec/pdi/spoon/pan_error_spec.rb
 - spec/pdi/spoon_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/bluemarblepayroll/pdi
@@ -179,12 +183,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.8
+      version: 2.5.8
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.3
 signing_key: 
@@ -192,10 +196,13 @@ specification_version: 4
 summary: Ruby wrapper for invoking Pentaho Data Integration
 test_files:
 - spec/mocks/spoon/return_code.sh
+- spec/mocks/spoon/sleep.sh
 - spec/mocks/spoon/version.sh
-- spec/pdi/spoon/error_spec.rb
+- spec/pdi/executor_spec.rb
+- spec/pdi/spoon/kitchen_error_spec.rb
 - spec/pdi/spoon/options/arg_spec.rb
 - spec/pdi/spoon/options/param_spec.rb
 - spec/pdi/spoon/options_spec.rb
+- spec/pdi/spoon/pan_error_spec.rb
 - spec/pdi/spoon_spec.rb
 - spec/spec_helper.rb