pdi 1.0.0.pre.alpha.1 → 2.0.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e653f6aed9fd26cce60e4f84cfdf662e0be847f40828142dd28fcc6118150cc7
-  data.tar.gz: 6ab7d8da5c628dc867d0607bcb3a3db6a5b4c030f07d91adc3f47516a2b3350f
+  metadata.gz: 3adae07a9c634b7075a87a47c810c0d4e3c90137ee85a08e01e09038baf990e1
+  data.tar.gz: 9f7ecf0e06ace02def29a7e710ea87d5ed5e2af82282a1dba935fa3628116533
 SHA512:
-  metadata.gz: 2e2688a153fea7a256e80a9e682b6488e71923d2752e32f37c23a39c7899f33faf2ffd99c91b5770e4e4279dd0af0ea9f3bdd6d52057b9780d6f798f7a44fedc
-  data.tar.gz: 66f618cc1203d313091cd849403501c127907a719f2b6503792cffe606f02062d0f2f31e179a45395e4460c47245de5461c1b78496208ad5ddf68437f7889558
+  metadata.gz: 5104d4d97f12a70e7f2d492dee6f8f870c388fea8091bc26d60fd87efb826331479a5efa851de7cbeb278d48aad9368d12cbad7bbb675b9cd9a20baee50f087e
+  data.tar.gz: 3ff6caa7e64806a1f03f918b7a6692e25c7493aa6280c0f9edabc99dd937e8890d76f3e8e0f060584964b993791c990621dc3cd8c908f91b5e81a25ac3e9f5ff

data/.rubocop.yml

@@ -1,4 +1,4 @@
-Metrics/LineLength:
+Layout/LineLength:
   Max: 100
   Exclude:
     - pdi.gemspec

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# 2.0.0 (May 7th, 2020)
+
+Breaking Changes:
+
+* Standard error and output have been combined into one stream (out).  Early feedback indicated that reading both at the same time was preferable.
+
+Enhancements:
+
+* Added optional `timeout_in_seconds` argument to Pdi::Spoon#initialize.
+
+# 1.0.1 (February 19th, 2020)
+
+Fixes:
+
+* It now properly calls kitchen for jobs (it was previously calling pan.)
+
+# 1.0.0 (February 19th, 2020)
+
+Initial release.

data/README.md

@@ -1,4 +1,4 @@
-# Pdi
+# PDI
 
 ---
 
@@ -36,17 +36,21 @@ Pull Requests are welcome for:
 
 ## Examples
 
-All examples assume Pdi has been installed to your home directory: `~/data-integration`.
+All examples assume PDI has been installed to your home directory: `~/data-integration`.
 
 ### Creating a Spoon Instance
 
-`Pdi::Spoon` is the common interface you will use when interacting with Pdi.  It will use Pan and Kitchen for executing Spoon commands.
+`Pdi::Spoon` is the common interface you will use when interacting with PDI.  It will use Pan and Kitchen for executing Spoon commands.
 
 ```ruby
-spoon = Pdi::Spoon.new(dir: `~/data-integration`)
+spoon = Pdi::Spoon.new(dir: '~/data-integration')
 ```
 
-Note: You can also override the names of the scripts using the `kitchen` and `pan` constructor keyword arguments.  The defaults are `kitchen.sh` and `pan.sh`, respectively.
+Notes:
+
+* You can also override the names of the scripts using the `kitchen` and `pan` constructor keyword arguments.  The defaults are `kitchen.sh` and `pan.sh`, respectively.
+* For other command line arguments that are not supported first-class in the Options objects below you can utilize the `args` argument when instantiating a `Spoon` instance.
+* Another optional argument is `timeout_in_seconds`.  If set it will ensure the sub-process runs within a given window.  If it times out the sub-process will be terminated and a Timeout::Error will be raised.
 
 ### Executing a Job/Transformation
 
@@ -71,7 +75,7 @@ result = spoon.run(options)
 
 You can access the raw command line results by tapping into the execution attribute of the result or error object.
 
-Note: Not all options are currently supported.  See Pdi's official references for [Pan](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/000) and [Kitchen](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/010) to see all options.
+Note: Not all options are currently supported.  See PDI's official references for [Pan](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/000) and [Kitchen](https://help.pentaho.com/Documentation/6.1/0L0/0Y0/070/010) to see all options.
 
 ## Contributing
 

data/lib/pdi.rb

@@ -8,8 +8,10 @@
 #
 
 require 'acts_as_hashable'
+require 'English'
 require 'forwardable'
 require 'open3'
+require 'timeout'
 
 require_relative 'pdi/executor'
 require_relative 'pdi/spoon'

data/lib/pdi/executor.rb

@@ -13,20 +13,42 @@ module Pdi
   # This class is the library's "metal" layer, the one which actually makes the system call and
   # interacts with the operating system (through Ruby's standard library.)
   class Executor
+    attr_reader :timeout_in_seconds
+
+    def initialize(timeout_in_seconds: nil)
+      @timeout_in_seconds = timeout_in_seconds
+
+      freeze
+    end
+
     def run(args)
       args = Array(args).map(&:to_s)
 
-      out, err, status = Open3.capture3(*args)
-
-      Result.new(
-        args: args,
-        status: {
-          code: status.exitstatus,
-          out: out,
-          err: err,
-          pid: status.pid
-        }
-      )
+      IO.popen(args, err: %i[child out]) do |io|
+        begin
+          io_read =
+            if timeout_in_seconds
+              Timeout.timeout(timeout_in_seconds) { io.read }
+            else
+              io.read
+            end
+
+          io.close
+          status = $CHILD_STATUS
+
+          Result.new(
+            args: args,
+            status: {
+              code: status.exitstatus,
+              out: io_read,
+              pid: status.pid
+            }
+          )
+        rescue Timeout::Error => e
+          Process.kill(9, io.pid)
+          raise e
+        end
+      end
     end
   end
 end

data/lib/pdi/executor/result.rb

@@ -18,7 +18,7 @@ module Pdi
 
       attr_reader :args, :status
 
-      def_delegators :status, :code, :out, :err, :pid
+      def_delegators :status, :code, :out, :pid
 
       def initialize(args:, status: {})
         @args   = args

data/lib/pdi/executor/status.rb

@@ -13,12 +13,11 @@ module Pdi
     class Status
       acts_as_hashable
 
-      attr_reader :code, :out, :err, :pid
+      attr_reader :code, :out, :pid
 
-      def initialize(code:, out: '', err: '', pid:)
+      def initialize(code:, out: '', pid:)
         @code = code
         @out  = out
-        @err  = err
         @pid  = pid
 
         freeze

data/lib/pdi/spoon.rb

@@ -7,33 +7,41 @@
 # LICENSE file in the root directory of this source tree.
 #
 
+require_relative 'spoon/kitchen_error'
 require_relative 'spoon/options'
+require_relative 'spoon/pan_error'
 require_relative 'spoon/parser'
 require_relative 'spoon/result'
 
 module Pdi
-  # This class is the main wrapper for Pdi's pan and kitchen scripts.
+  # This class is the main wrapper for PDI's pan and kitchen scripts.
   class Spoon
     DEFAULT_KITCHEN = 'kitchen.sh'
     DEFAULT_PAN     = 'pan.sh'
 
+    TYPES_TO_ERRORS = {
+      Options::Type::JOB => KitchenError,
+      Options::Type::TRANSFORMATION => PanError
+    }.freeze
+
     attr_reader :args, :dir, :kitchen, :pan
 
     def initialize(
       args: [],
       dir:,
       kitchen: DEFAULT_KITCHEN,
-      pan: DEFAULT_PAN
+      pan: DEFAULT_PAN,
+      timeout_in_seconds: nil
     )
       assert_required(:dir, dir)
       assert_required(:kitchen, kitchen)
       assert_required(:pan, pan)
 
       @args           = Array(args)
-      @dir            = dir.to_s
+      @dir            = File.expand_path(dir.to_s)
       @kitchen        = kitchen.to_s
       @pan            = pan.to_s
-      @executor       = Executor.new
+      @executor       = Executor.new(timeout_in_seconds: timeout_in_seconds)
       @parser         = Parser.new
 
       freeze
@@ -55,11 +63,11 @@ module Pdi
     # Returns an Executor::Result instance when PDI returns error code 0 or else raises
     # a PanError (transformation) or KitchenError (job).
     def run(options)
-      options    = Options.make(options)
-      final_args = [pan_path] + args + options.to_args
+      options  = Options.make(options)
+      all_args = run_args(options)
 
-      executor.run(final_args).tap do |result|
-        raise(options.error_constant, result) if result.code != 0
+      executor.run(all_args).tap do |result|
+        raise(error_constant(options), result) if result.code != 0
       end
     end
 
@@ -67,6 +75,22 @@ module Pdi
 
     attr_reader :executor, :parser
 
+    def error_constant(options)
+      TYPES_TO_ERRORS.fetch(options.type)
+    end
+
+    def run_args(options)
+      [script_path(options.type)] + args + options.to_args
+    end
+
+    def script_path(options_type)
+      if options_type == Options::Type::JOB
+        kitchen_path
+      elsif options_type == Options::Type::TRANSFORMATION
+        pan_path
+      end
+    end
+
     def kitchen_path
       File.join(dir, kitchen)
     end

data/lib/pdi/spoon/kitchen_error.rb

@@ -11,6 +11,8 @@ module Pdi
   class Spoon
     # This class subclasses Ruby's StandardError and provides a mapping to custom Kitchen
     # specific error codes to messages.
+    # You can find the list of errors in Pentaho's documentation site, for example:
+    # https://help.pentaho.com/Documentation/8.0/Products/Data_Integration/Command_Line_Tools
     class KitchenError < StandardError
       MESSAGES = {
         '1' => 'Errors occurred during processing',

data/lib/pdi/spoon/options.rb

@@ -7,10 +7,8 @@
 # LICENSE file in the root directory of this source tree.
 #
 
-require_relative 'kitchen_error'
 require_relative 'options/level'
 require_relative 'options/param'
-require_relative 'pan_error'
 
 module Pdi
   class Spoon
@@ -23,11 +21,6 @@ module Pdi
         TRANSFORMATION = :transformation
       end
 
-      TYPES_TO_ERRORS = {
-        Type::JOB => KitchenError,
-        Type::TRANSFORMATION => PanError
-      }.freeze
-
       TYPES_TO_KEYS = {
         Type::JOB => Arg::Key::JOB,
         Type::TRANSFORMATION => Arg::Key::TRANS
@@ -63,10 +56,6 @@ module Pdi
         base_args + param_args
       end
 
-      def error_constant
-        TYPES_TO_ERRORS.fetch(type)
-      end
-
       private
 
       def key

data/lib/pdi/spoon/pan_error.rb

@@ -11,6 +11,8 @@ module Pdi
   class Spoon
     # This class subclasses Ruby's StandardError and provides a mapping to custom Pan
     # specific error codes to messages.
+    # You can find the list of errors in Pentaho's documentation site, for example:
+    # https://help.pentaho.com/Documentation/8.0/Products/Data_Integration/Command_Line_Tools
     class PanError < StandardError
       MESSAGES = {
         '1' => 'Errors occurred during processing',

data/lib/pdi/version.rb

@@ -8,5 +8,5 @@
 #
 
 module Pdi
-  VERSION = '1.0.0-alpha.1'
+  VERSION = '2.0.0-alpha'
 end

data/spec/mocks/spoon/sleep.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo 'std_out'
+echo 'err_out' >&2
+
+sleep $1
+
+echo 'after_sleep'
+
+exit $2

data/spec/pdi/executor_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2018-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe Pdi::Executor do
+  let(:script)      { File.join('spec', 'mocks', 'spoon', 'sleep.sh') }
+  let(:one_second)  { 1 }
+  let(:return_code) { 33 }
+
+  describe '#run' do
+    context 'with a timeout' do
+      # do not make these too high, bugs could cause the entire script to still be executed without
+      # killing it so these high values could draw out results.
+      let(:thirty_seconds) { 30 }
+
+      subject { described_class.new(timeout_in_seconds: one_second) }
+
+      # This will run a script that will take 30 seconds to process, but by limiting the
+      # timeout using the #run argument, it should raise an error after one second.
+      it 'times out and kills process after 5 seconds' do
+        args = [script, thirty_seconds]
+
+        expect { subject.run(args) }.to raise_error(Timeout::Error)
+      end
+    end
+
+    context 'without a timeout' do
+      it 'returns right exit status as code' do
+        args = [script, one_second, return_code]
+
+        result = subject.run(args)
+
+        expect(result.code).to eq(return_code)
+      end
+
+      it 'returns right standard output and error as out' do
+        args = [script, one_second, return_code]
+
+        result = subject.run(args)
+
+        expect(result.out).to eq("std_out\nerr_out\nafter_sleep\n")
+      end
+    end
+  end
+end

data/spec/pdi/spoon/kitchen_error_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+#
+# Copyright (c) 2018-present, Blue Marble Payroll, LLC
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+require 'spec_helper'
+
+describe Pdi::Spoon::KitchenError do
+  describe 'initialization' do
+    [1, 2, 7, 8, 9].each do |code|
+      specify "code #{code} should have message" do
+        result = Pdi::Executor::Result.new(
+          args: [],
+          status: {
+            code: code,
+            pid: 123
+          }
+        )
+
+        expect(described_class.new(result).message).not_to eq('Unknown')
+      end
+    end
+
+    [-1, 0, 3, 4, 5, 6, 10, 11].each do |code|
+      specify "code #{code} should not have message" do
+        result = Pdi::Executor::Result.new(
+          args: [],
+          status: {
+            code: code,
+            pid: 123
+          }
+        )
+
+        expect(described_class.new(result).message).to eq('Unknown')
+      end
+    end
+  end
+end

data/spec/pdi/spoon_spec.rb

@@ -14,6 +14,21 @@ describe Pdi::Spoon do
   let(:mocks_dir) { %w[spec mocks spoon] }
   let(:dir)       { File.join(*mocks_dir) }
 
+  describe '#initialize' do
+    it 'sets executor' do
+      timeout_in_seconds = 987
+
+      subject = described_class.new(dir: dir, timeout_in_seconds: timeout_in_seconds)
+
+      # Private/internal testing is not recommended, but I really wanted to ensure
+      # this class is properly configuring the Executor instance, that way I can rely
+      # mainly on the Executor unit tests instead of integration tests at this level.
+      executor = subject.send('executor')
+
+      expect(executor.timeout_in_seconds).to eq(timeout_in_seconds)
+    end
+  end
+
   describe '#run' do
     context 'transformations' do
       let(:options) do
@@ -24,6 +39,14 @@ describe Pdi::Spoon do
         }
       end
 
+      it 'will call pan script' do
+        subject = described_class.new(args: 0, dir: dir, pan: script)
+
+        result = subject.run(options)
+
+        expect(result.args.first).to include(script)
+      end
+
       context 'when code is 0' do
         it 'returns correct stdout, stderr and code' do
           subject = described_class.new(
@@ -35,8 +58,7 @@ describe Pdi::Spoon do
 
           result = subject.run(options)
 
-          expect(result.out).to  eq("output to stdout\n")
-          expect(result.err).to  eq("output to sterr\n")
+          expect(result.out).to  eq("output to stdout\noutput to sterr\n")
           expect(result.code).to eq(0)
         end
       end
@@ -70,6 +92,14 @@ describe Pdi::Spoon do
         }
       end
 
+      it 'will call kitchen script' do
+        subject = described_class.new(args: 0, dir: dir, kitchen: script)
+
+        result = subject.run(options)
+
+        expect(result.args.first).to include(script)
+      end
+
       context 'when code is 0' do
         it 'returns correct stdout, stderr and code' do
           subject = described_class.new(
@@ -81,8 +111,7 @@ describe Pdi::Spoon do
 
           result = subject.run(options)
 
-          expect(result.out).to  eq("output to stdout\n")
-          expect(result.err).to  eq("output to sterr\n")
+          expect(result.out).to  eq("output to stdout\noutput to sterr\n")
           expect(result.code).to eq(0)
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pdi
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.alpha.1
+  version: 2.0.0.pre.alpha
 platform: ruby
 authors:
 - Matthew Ruggio
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-18 00:00:00.000000000 Z
+date: 2020-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: acts_as_hashable
@@ -160,11 +160,14 @@ files:
 - lib/pdi/version.rb
 - pdi.gemspec
 - spec/mocks/spoon/return_code.sh
+- spec/mocks/spoon/sleep.sh
 - spec/mocks/spoon/version.sh
-- spec/pdi/spoon/error_spec.rb
+- spec/pdi/executor_spec.rb
+- spec/pdi/spoon/kitchen_error_spec.rb
 - spec/pdi/spoon/options/arg_spec.rb
 - spec/pdi/spoon/options/param_spec.rb
 - spec/pdi/spoon/options_spec.rb
+- spec/pdi/spoon/pan_error_spec.rb
 - spec/pdi/spoon_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/bluemarblepayroll/pdi
@@ -192,10 +195,13 @@ specification_version: 4
 summary: Ruby wrapper for invoking Pentaho Data Integration
 test_files:
 - spec/mocks/spoon/return_code.sh
+- spec/mocks/spoon/sleep.sh
 - spec/mocks/spoon/version.sh
-- spec/pdi/spoon/error_spec.rb
+- spec/pdi/executor_spec.rb
+- spec/pdi/spoon/kitchen_error_spec.rb
 - spec/pdi/spoon/options/arg_spec.rb
 - spec/pdi/spoon/options/param_spec.rb
 - spec/pdi/spoon/options_spec.rb
+- spec/pdi/spoon/pan_error_spec.rb
 - spec/pdi/spoon_spec.rb
 - spec/spec_helper.rb