octocatalog-diff 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86467134c489b13251760bd9ffd23caced1c5222
-  data.tar.gz: 8260b43c4e844b2224dd491018feb8a1da65c4ed
+  metadata.gz: 75c0856d744b0c909315a69aceb6e2d5e416991b
+  data.tar.gz: 127ac1617cc437a73d716ff68a0eea0623e62c8b
 SHA512:
-  metadata.gz: fedd421520a95d4147878d50510d7f2f149cfcba0933c75d7f88c5f3efbeb45f6105b9c5bab53cbc46cf7234db1987cba0b9bdc327cd0b418ce8f69365723282
-  data.tar.gz: 1bc344f71b92ef5154253e4b115f7916e59a9ee49c21c7344adb80ad7dd97d2f164fb951016e8a22d95ba564eab1dbc27a9c1b0cff8a014d327bfde881fb5d21
+  metadata.gz: a4a873ed14e553d8362267e88fdac5eeaf46e5929275601871048959805f93515af9f8d8152115e49f1c15ddf17fc38fa70158b1e001be71d20a0646f05c422e
+  data.tar.gz: 7c3f9a881d1e2b9d8669c96d6ffbb9953a25b8ff2c63f2a9c3151201c09be9ee95c6182b040370b08a54720c830e98f701cbe3f903c7b876ae80ec140b08a3b0

data/.version

@@ -1 +1 @@
-1.1.0
+1.2.0

data/doc/CHANGELOG.md

@@ -8,6 +8,16 @@
 </tr>
 </thead><tbody>
 <tr valign=top>
+<td>1.2.0</td>
+<td>2017-05-18</td>
+<td>
+<li><a href="https://github.com/github/octocatalog-diff/pull/112">#112</a>: Split arguments added for ENC</li>
+<li><a href="https://github.com/github/octocatalog-diff/pull/113">#113</a>: (Enhancement) Override facts and ENC parameters using regular expressions</li>
+<li><a href="https://github.com/github/octocatalog-diff/pull/103">#111</a>: Simplify parallel processing to solve some intermittent failures</li>
+<li><a href="https://github.com/github/octocatalog-diff/pull/110">#110</a>: Ruby 2.4 compatibility</li>
+</td>
+</tr>
+<tr valign=top>
 <td>1.1.0</td>
 <td>2017-05-08</td>
 <td>

data/doc/advanced-override-facts.md

@@ -65,3 +65,16 @@ The following data types in parentheses are supported:
 | `(json)` | Treat the input as a JSON string (calls `JSON.parse` in ruby) |
 | `(boolean)` | Treat the input as a boolean -- it must be `true` or `false`, case-insensitive |
 | `(nil)` | Ignore any characters after `(nil)` and deletes the fact if the fact exists |
+
+## Regular expressions
+
+If you wish to match multiple facts by pattern, specify the regular expression in place of the key name. For example:
+
+```
+octocatalog-diff -n some-node.example.com -f master -t master \
+  --to-fact-override /^ipaddress/=10.11.12.13
+```
+
+In this example, `$::ipaddress`, `$::ipaddress_eth0`, `$::ipaddress_bond0`, and any other facts starting with "ipaddress" would be overridden. However, a fact named `$::additional_ipaddress` would not be overridden, because it does not match the regular expression.
+
+Please note that you cannot *add* a fact with a regular expression -- when using regular expressions you can only modify or delete facts.

data/doc/dev/integration-tests.md

@@ -22,7 +22,7 @@ describe 'whatever behavior' do
     # @result[:logs] is a String containing everything printed to STDERR (Logger)
     # @result[:output] is a String containing everything printed to STDOUT
     # @result[:diffs] is an Array of differences
-    # @result[:exitcode] is a Fixnum representing the exit code: 0 = no changes, 1 = failure, 2 = success, with changes
+    # @result[:exitcode] is an Integer representing the exit code: 0 = no changes, 1 = failure, 2 = success, with changes
     # @result[:exception] contains any exception that was thrown
   end
 
@@ -46,7 +46,7 @@ describe 'whatever behavior' do
     # @result[:logs] is a String containing everything printed to STDERR (Logger)
     # @result[:output] is a String containing everything printed to STDOUT
     # @result[:diffs] is an Array of differences
-    # @result[:exitcode] is a Fixnum representing the exit code: 0 = no changes, 1 = failure, 2 = success, with changes
+    # @result[:exitcode] is an Integer representing the exit code: 0 = no changes, 1 = failure, 2 = success, with changes
     # @result[:exception] contains any exception that was thrown
   end
 

data/doc/requirements.md

@@ -2,10 +2,10 @@
 
 To run `octocatalog-diff` you will need these basics:
 
-- Ruby 2.0 or higher
+- Ruby 2.0 through 2.4 (we test octocatalog-diff with Ruby 2.0, 2.1, 2.2, 2.3, and 2.4)
 - Mac OS, Linux, or other Unix-line operating system (Windows is not supported)
 - Ability to install gems, e.g. with [rbenv](https://github.com/rbenv/rbenv) or [rvm](https://rvm.io/), or root privileges to install into the system Ruby
-- Puppet agent for [Linux](https://docs.puppet.com/puppet/latest/reference/install_linux.html) or [Mac OS X](https://docs.puppet.com/puppet/latest/reference/install_osx.html), or installed as a gem
+- Puppet agent for [Linux](https://docs.puppet.com/puppet/latest/reference/install_linux.html) or [Mac OS X](https://docs.puppet.com/puppet/latest/reference/install_osx.html), or installed as a gem (we support Puppet 3.8.7 and all versions of Puppet 4.x)
 
 We recommend that you also have the following to get the most out of `octocatalog-diff`, but these are not absolute requirements:
 

data/lib/octocatalog-diff/api/v1/override.rb

@@ -13,7 +13,8 @@ module OctocatalogDiff
         # Constructor: Accepts a key and value.
         # @param input [Hash] Must contain :key and :value
         def initialize(input)
-          @key = input.fetch(:key)
+          key = input.fetch(:key)
+          @key = key =~ %r{\A/(.+)/\Z} ? Regexp.new(Regexp.last_match(1)) : key
           @value = parsed_value(input.fetch(:value))
         end
 
@@ -29,6 +30,7 @@ module OctocatalogDiff
           # If input is not a string, we can still construct the object if the key is given.
           # That input would come directly from code and not from the command line, since inputs
           # from the command line are always strings.
+          # Also support regular expressions for the key name, if delimited by //.
           if key.nil? && input.is_a?(String)
             unless input.include?('=')
               raise ArgumentError, "Fact override '#{input}' is not in 'key=(data type)value' format"
@@ -72,9 +74,9 @@ module OctocatalogDiff
           return value if datatype == 'string'
           return parse_json(value) if datatype == 'json'
           return nil if datatype == 'nil'
-          if datatype == 'fixnum'
+          if datatype == 'fixnum' || datatype == 'integer'
             return Regexp.last_match(1).to_i if value =~ /^(-?\d+)$/
-            raise ArgumentError, "Illegal fixnum '#{value}'"
+            raise ArgumentError, "Illegal integer '#{value}'"
           end
           if datatype == 'float'
             return Regexp.last_match(1).to_f if value =~ /^(-?\d*\.\d+)$/

data/lib/octocatalog-diff/bootstrap.rb

@@ -10,7 +10,7 @@ module OctocatalogDiff
     # @param options [Hash] Options hash:
     #        :path [String] => Directory to bootstrap
     #        :bootstrap_script [String] => Bootstrap script, relative to directory
-    # @return [Hash] => [Fixnum] :status_code, [String] :output
+    # @return [Hash] => [Integer] :status_code, [String] :output
     def self.bootstrap(options = {})
       # Options validation
       unless options[:path].is_a?(String)

data/lib/octocatalog-diff/catalog-diff/differ.rb

@@ -530,7 +530,7 @@ module OctocatalogDiff
 
           # Added a new key that points to some kind of data structure that we know how
           # to handle.
-          if obj[1] =~ /^(.+)\f([^\f]+)$/ && [String, Fixnum, Float, TrueClass, FalseClass, Array, Hash].include?(obj[2].class)
+          if obj[1] =~ /^(.+)\f([^\f]+)$/ && [String, Integer, Float, TrueClass, FalseClass, Array, Hash].include?(obj[2].class)
             hashdiff_add_remove.add(obj[1])
             next
           end

data/lib/octocatalog-diff/catalog-diff/display/text.rb

@@ -255,7 +255,7 @@ module OctocatalogDiff
         # Get the diff of two long strings. Call the 'diffy' gem for this.
         # @param string1 [String] First string (-)
         # @param string2 [String] Second string (+)
-        # @param depth [Fixnum] Depth, for correct indentation
+        # @param depth [Integer] Depth, for correct indentation
         # @return Array<String> Displayable result
         def self.diff_two_strings_with_diffy(string1, string2, depth)
           # Single line strings?
@@ -324,8 +324,8 @@ module OctocatalogDiff
         # Get the diff of two hashes. Call the 'diffy' gem for this.
         # @param hash1 [Hash] First hash (-)
         # @param hash1 [Hash] Second hash (+)
-        # @param depth [Fixnum] Depth, for correct indentation
-        # @param limit [Fixnum] Maximum string length
+        # @param depth [Integer] Depth, for correct indentation
+        # @param limit [Integer] Maximum string length
         # @param strip_diff [Boolean] Strip leading +/-/" "
         # @return [Array<String>] Displayable result
         def self.diff_two_hashes_with_diffy(opts = {})
@@ -358,7 +358,7 @@ module OctocatalogDiff
         end
 
         # Special case: addition only, no truncation
-        # @param depth [Fixnum] Depth, for correct indentation
+        # @param depth [Integer] Depth, for correct indentation
         # @param hash [Hash] Added object
         # @return [Array<String>] Displayable result
         def self.addition_only_no_truncation(depth, hash)
@@ -383,7 +383,7 @@ module OctocatalogDiff
 
         # Limit length of a string
         # @param str [String] String
-        # @param limit [Fixnum] Limit (0=unlimited)
+        # @param limit [Integer] Limit (0=unlimited)
         # @return [String] Truncated string
         def self.truncate_string(str, limit)
           return str if limit.nil? || str.length <= limit
@@ -392,7 +392,7 @@ module OctocatalogDiff
 
         # Get the diff between two hashes. This is recursive-aware.
         # @param obj [diff object] diff object
-        # @param depth [Fixnum] Depth of nesting, used for indentation
+        # @param depth [Integer] Depth of nesting, used for indentation
         # @return Array<String> Printable diff outputs
         def self.hash_diff(obj, depth, key_in, nested = false)
           result = []
@@ -417,7 +417,7 @@ module OctocatalogDiff
         end
 
         # Get the diff between two arbitrary objects
-        # @param depth [Fixnum] Depth of nesting, used for indentation
+        # @param depth [Integer] Depth of nesting, used for indentation
         # @param old_obj [?] Old object
         # @param new_obj [?] New object
         # @return Array<String> Diff output
@@ -432,12 +432,31 @@ module OctocatalogDiff
 
         # Utility Method!
         # Indent a given text string with a certain number of spaces
-        # @param spaces [Fixnum] Number of spaces
+        # @param spaces [Integer] Number of spaces
         # @param text [String] Text
         def self.left_pad(spaces, text = '')
           [' ' * spaces, text].join('')
         end
 
+        # Utility Method!
+        # Harmonize equivalent class names for comparison purposes.
+        # @param class_name [String] Class name as input
+        # @return [String] Class name as output
+        def self.class_name_for_diffy(class_name)
+          return 'Integer' if class_name == 'Fixnum'
+          class_name
+        end
+
+        # Utility Method!
+        # `is_a?(class)` only allows one method, but this uses an array
+        # @param object [?] Object to consider
+        # @param classes [Array] Classes to determine if object is a member of
+        # @return [Boolean] True if object is_a any of the classes, false otherwise
+        def self.object_is_any_of?(object, classes)
+          classes.each { |clazz| return true if object.is_a? clazz }
+          false
+        end
+
         # Utility Method!
         # Given an arbitrary object, convert it into a string for use by 'diffy'.
         # This basically exists so we can do something prettier than just calling .inspect or .to_s
@@ -445,10 +464,10 @@ module OctocatalogDiff
         # @param obj [?] Object to be stringified
         # @return [String] String representation of object for diffy
         def self.stringify_for_diffy(obj)
-          return JSON.pretty_generate(obj) if [Hash, Array].include?(obj.class)
+          return JSON.pretty_generate(obj) if object_is_any_of?(obj, [Hash, Array])
           return '""' if obj.is_a?(String) && obj == ''
-          return obj if [String, Fixnum, Float].include?(obj.class)
-          "#{obj.class}: #{obj.inspect}"
+          return obj if object_is_any_of?(obj, [String, Fixnum, Integer, Float])
+          "#{class_name_for_diffy(obj.class)}: #{obj.inspect}"
         end
 
         # Utility Method!
@@ -512,8 +531,8 @@ module OctocatalogDiff
           return ['""', 'undef'] if obj2.nil?
 
           # If one is an integer and the other is a string
-          return [obj1, "\"#{obj2}\""] if obj1.is_a?(Fixnum) && obj2.is_a?(String)
-          return ["\"#{obj1}\"", obj2] if obj1.is_a?(String) && obj2.is_a?(Fixnum)
+          return [obj1, "\"#{obj2}\""] if obj1.is_a?(Integer) && obj2.is_a?(String)
+          return ["\"#{obj1}\"", obj2] if obj1.is_a?(String) && obj2.is_a?(Integer)
 
           # True and false
           return [obj1, "\"#{obj2}\""] if obj1.is_a?(TrueClass) && obj2.is_a?(String)

data/lib/octocatalog-diff/catalog-util/bootstrap.rb

@@ -74,8 +74,11 @@ module OctocatalogDiff
           if result.status
             logger.debug("Success bootstrap_directory for #{result.args[:tag]}")
           else
+            # Believed to be a bug condition, since error should have already been raised if this happens.
+            # :nocov:
             errmsg = "Failed bootstrap_directory for #{result.args[:tag]}: #{result.exception.class} #{result.exception.message}"
             raise OctocatalogDiff::Errors::BootstrapError, errmsg
+            # :nocov:
           end
         end
       end

data/lib/octocatalog-diff/catalog-util/builddir.rb

@@ -21,7 +21,7 @@ module OctocatalogDiff
       # Constructor
       # Options for constructor:
       # :puppetdb_url [String] PuppetDB Server URLs
-      # :puppetdb_server_url_timeout [Fixnum] Timeout (seconds) for puppetdb.conf
+      # :puppetdb_server_url_timeout [Integer] Timeout (seconds) for puppetdb.conf
       # :facts [OctocatalogDiff::Facts] Facts object
       # :fact_file [String] File from which to read facts
       # :node [String] Node name
@@ -99,14 +99,14 @@ module OctocatalogDiff
 
       # Install puppetdb.conf file in temporary directory
       # @param server_urls [String] String for server_urls in puppetdb.conf
-      # @param server_url_timeout [Fixnum] Value for server_url_timeout in puppetdb.conf
+      # @param server_url_timeout [Integer] Value for server_url_timeout in puppetdb.conf
       def install_puppetdb_conf(logger, server_urls, server_url_timeout = 30)
         unless server_urls.is_a?(String)
           raise ArgumentError, "server_urls must be a string, got a: #{server_urls.class}"
         end
 
         server_url_timeout ||= 30 # If called with nil argument, supply default
-        unless server_url_timeout.is_a?(Fixnum)
+        unless server_url_timeout.is_a?(Integer)
           raise ArgumentError, "server_url_timeout must be a fixnum, got a: #{server_url_timeout.class}"
         end
 
@@ -163,9 +163,12 @@ module OctocatalogDiff
 
         if options[:fact_override].is_a?(Array)
           options[:fact_override].each do |override|
-            old_value = facts.fact(override.key)
-            facts.override(override.key, override.value)
-            logger.debug("Override #{override.key} from #{old_value.inspect} to #{override.value.inspect}")
+            keys = override.key.is_a?(Regexp) ? facts.matching(override.key) : [override.key]
+            keys.each do |key|
+              old_value = facts.fact(key)
+              facts.override(key, override.value)
+              logger.debug("Override #{key} from #{old_value.inspect} to #{override.value.inspect}")
+            end
           end
         end
 

data/lib/octocatalog-diff/catalog-util/command.rb

@@ -68,7 +68,8 @@ module OctocatalogDiff
         # enc?
         if @options[:enc]
           raise Errno::ENOENT, "Did not find ENC as expected at #{@options[:enc]}" unless File.file?(@options[:enc])
-          cmdline << "--node_terminus=exec --external_nodes=#{Shellwords.escape(@options[:enc])}"
+          cmdline << '--node_terminus=exec'
+          cmdline << "--external_nodes=#{Shellwords.escape(@options[:enc])}"
         end
 
         # Future parser?
@@ -167,7 +168,7 @@ module OctocatalogDiff
       # the index.
       # @param cmdline [Array] Existing command line
       # @param key [String] Key to look up
-      # @return [Fixnum] Index of where key is defined (nil if undefined)
+      # @return [Integer] Index of where key is defined (nil if undefined)
       def key_position(cmdline, key)
         cmdline.index { |x| x == "--#{key}" || x =~ /\A--#{key}=/ }
       end

data/lib/octocatalog-diff/catalog-util/enc.rb

@@ -65,8 +65,11 @@ module OctocatalogDiff
         return unless @options[:enc_override].is_a?(Array) && @options[:enc_override].any?
         content_structure = YAML.load(content)
         @options[:enc_override].each do |x|
-          merge_enc_param(content_structure, x.key, x.value)
-          logger.debug "ENC override: #{x.key} #{x.value.nil? ? 'DELETED' : '= ' + x.value.inspect}"
+          keys = x.key.is_a?(Regexp) ? content_structure.keys.select { |y| x.key.match(y) } : [x.key]
+          keys.each do |key|
+            merge_enc_param(content_structure, key, x.value)
+            logger.debug "ENC override: #{key} #{x.value.nil? ? 'DELETED' : '= ' + x.value.inspect}"
+          end
         end
         @content = content_structure.to_yaml
       end

data/lib/octocatalog-diff/catalog-util/fileresources.rb

@@ -86,7 +86,12 @@ module OctocatalogDiff
         resources.map! do |resource|
           if resource_convertible?(resource)
             path = file_path(resource['parameters']['source'], modulepaths)
-            raise Errno::ENOENT, "Unable to resolve '#{resource['parameters']['source']}'!" if path.nil?
+            if path.nil?
+              # Pass this through as a wrapped exception, because it's more likely to be something wrong
+              # in the catalog itself than it is to be a broken setup of octocatalog-diff.
+              message = "Errno::ENOENT: Unable to resolve '#{resource['parameters']['source']}'!"
+              raise OctocatalogDiff::Errors::CatalogError, message
+            end
 
             if File.file?(path)
               # If the file is found, read its content. If the content is all ASCII, substitute it into

data/lib/octocatalog-diff/catalog/computed.rb

@@ -22,7 +22,7 @@ module OctocatalogDiff
       # @param :node [String] REQUIRED: Node name
       # @param :basedir [String] Directory in which to compile the catalog
       # @param :pass_env_vars [Array<String>] Environment variables to pass when compiling catalog
-      # @param :retry_failed_catalog [Fixnum] Number of retries if a catalog compilation fails
+      # @param :retry_failed_catalog [Integer] Number of retries if a catalog compilation fails
       # @param :tag [String] For display purposes, the catalog being compiled
       # @param :puppet_binary [String] Full path to Puppet
       # @param :puppet_version [String] Puppet version (optional; if not supplied, it is calculated)

data/lib/octocatalog-diff/catalog/puppetdb.rb

@@ -15,7 +15,7 @@ module OctocatalogDiff
 
       # Constructor - See OctocatalogDiff::PuppetDB for additional parameters
       # @param :node [String] Node name
-      # @param :retry [Fixnum] Number of retries, if fetch fails
+      # @param :retry [Integer] Number of retries, if fetch fails
       def initialize(options)
         raise ArgumentError, 'Hash of options must be passed to OctocatalogDiff::Catalog::PuppetDB' unless options.is_a?(Hash)
         raise ArgumentError, 'node must be a non-empty string' unless options[:node].is_a?(String) && options[:node] != ''

data/lib/octocatalog-diff/catalog/puppetmaster.rb

@@ -22,17 +22,17 @@ module OctocatalogDiff
 
       # Constructor
       # @param :node [String] Node name
-      # @param :retry_failed_catalog [Fixnum] Number of retries, if fetch fails
+      # @param :retry_failed_catalog [Integer] Number of retries, if fetch fails
       # @param :branch [String] Environment to fetch from Puppet Master
       # @param :puppet_master [String] Puppet server and port number (assumed to be DEFAULT_PUPPET_PORT_NUMBER if not given)
-      # @param :puppet_master_api_version [Fixnum] Puppet server API (default DEFAULT_PUPPET_SERVER_API)
+      # @param :puppet_master_api_version [Integer] Puppet server API (default DEFAULT_PUPPET_SERVER_API)
       # @param :puppet_master_ssl_ca [String] Path to file used to sign puppet master's certificate
       # @param :puppet_master_ssl_verify [Boolean] Override the CA verification setting guessed from parameters
       # @param :puppet_master_ssl_client_pem [String] PEM-encoded client key and certificate
       # @param :puppet_master_ssl_client_p12 [String] pkcs12-encoded client key and certificate
       # @param :puppet_master_ssl_client_password [String] Path to file containing password for SSL client key (any format)
       # @param :puppet_master_ssl_client_auth [Boolean] Override the client-auth that is guessed from parameters
-      # @param :timeout [Fixnum] Connection timeout for Puppet master (default=PUPPET_MASTER_TIMEOUT seconds)
+      # @param :timeout [Integer] Connection timeout for Puppet master (default=PUPPET_MASTER_TIMEOUT seconds)
       def initialize(options)
         raise ArgumentError, 'Hash of options must be passed to OctocatalogDiff::Catalog::PuppetMaster' unless options.is_a?(Hash)
         raise ArgumentError, 'node must be a non-empty string' unless options[:node].is_a?(String) && options[:node] != ''

data/lib/octocatalog-diff/cli.rb

@@ -50,7 +50,7 @@ module OctocatalogDiff
     # @param argv [Array] Use specified arguments (defaults to ARGV)
     # @param logger [Logger] Logger object
     # @param opts [Hash] Additional options
-    # @return [Fixnum] Exit code: 0=no diffs, 1=something went wrong, 2=worked but there are diffs
+    # @return [Integer] Exit code: 0=no diffs, 1=something went wrong, 2=worked but there are diffs
     def self.cli(argv = ARGV, logger = Logger.new(STDERR), opts = {})
       # Save a copy of argv to print out later in debugging
       argv_save = argv.dup

data/lib/octocatalog-diff/cli/options/puppet_master_timeout.rb

@@ -14,7 +14,7 @@ OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_timeout) do
       cli_name: 'puppet-master-timeout',
       option_name: 'puppet_master_timeout',
       desc: 'Puppet Master catalog retrieval timeout in seconds',
-      validator: ->(x) { x.to_i > 0 || raise(ArgumentError, 'Specify timeout as a integer greater than 0') },
+      validator: ->(x) { x.to_i > 0 || raise(ArgumentError, 'Specify timeout as an integer greater than 0') },
       translator: ->(x) { x.to_i }
     )
   end

data/lib/octocatalog-diff/facts.rb

@@ -120,5 +120,12 @@ module OctocatalogDiff
         @facts['values'][key] = value
       end
     end
+
+    # Find all facts matching a particular pattern
+    # @param regex [Regexp] Regular expression to match
+    # @return [Array<String>] Facts that match the regexp
+    def matching(regex)
+      @facts['values'].keys.select { |fact| regex.match(fact) }
+    end
   end
 end

data/lib/octocatalog-diff/facts/puppetdb.rb

@@ -17,7 +17,7 @@ module OctocatalogDiff
 
       # Retrieve facts from PuppetDB for a specified node.
       # @param :puppetdb_url [String|Array] => URL to PuppetDB
-      # @param :retry [Fixnum] => Retry after timeout (default 0 retries, can be more)
+      # @param :retry [Integer] => Retry after timeout (default 0 retries, can be more)
       # @param node [String] Node name. (REQUIRED for PuppetDB fact source)
       # @return [Hash] Facts
       def self.fact_retriever(options = {}, node)

data/lib/octocatalog-diff/puppetdb.rb

@@ -38,7 +38,7 @@ module OctocatalogDiff
     # Supported arguments:
     # @param :puppetdb_url [String or Array<String>] PuppetDB URL(s) to try in random order
     # @param :puppetdb_host [String] PuppetDB hostname, when constructing a URL
-    # @param :puppetdb_port [Fixnum] Port number, defaults to 8080 (non-SSL) or 8081 (SSL)
+    # @param :puppetdb_port [Integer] Port number, defaults to 8080 (non-SSL) or 8081 (SSL)
     # @param :puppetdb_ssl [Boolean] defaults to true, because you should use SSL
     # @param :puppetdb_ssl_ca [String] Path to file containing CA certificate
     # @param :puppetdb_ssl_verify [Boolean] Override the CA verification setting guessed from parameters
@@ -46,7 +46,7 @@ module OctocatalogDiff
     # @param :puppetdb_ssl_client_p12 [String] pkcs12-encoded client key and certificate
     # @param :puppetdb_ssl_client_password [String] Path to file containing password for SSL client key (any format)
     # @param :puppetdb_ssl_client_auth [Boolean] Override the client-auth that is guessed from parameters
-    # @param :timeout [Fixnum] Connection timeout for PuppetDB (default=10)
+    # @param :timeout [Integer] Connection timeout for PuppetDB (default=10)
     def initialize(options = {})
       @connections =
         if options.key?(:puppetdb_url)
@@ -149,7 +149,7 @@ module OctocatalogDiff
 
     # Parse a URL to determine hostname, port number, and whether or not SSL is used.
     # @param url [String] URL to parse
-    # @return [Hash] { ssl: true/false, host: <String>, port: <Fixnum> }
+    # @return [Hash] { ssl: true/false, host: <String>, port: <Integer> }
     def parse_url(url)
       uri = URI(url)
       raise ArgumentError, "URL #{url} has invalid scheme" unless uri.scheme =~ /^https?$/

data/lib/octocatalog-diff/util/catalogs.rb

@@ -99,6 +99,15 @@ module OctocatalogDiff
           # :nocov:
         end
 
+        # If catalogs failed to compile, report that. Prefer to display an actual failure message rather
+        # than a generic incomplete parallel task message if there is a more specific message present.
+        failures = parallel_catalogs.reject(&:status)
+        if failures.any?
+          f = failures.reject { |r| r.exception.is_a?(OctocatalogDiff::Util::Parallel::IncompleteTask) }.first
+          f ||= failures.first
+          raise f.exception
+        end
+
         # Construct result hash. Will eventually be in the format
         # { :from => OctocatalogDiff::Catalog, :to => OctocatalogDiff::Catalog }
 
@@ -203,10 +212,12 @@ module OctocatalogDiff
           end
         else
           # Something unhandled went wrong, and an exception was thrown. Reveal a generic message.
+          # :nocov:
           msg = parallel_catalog_obj.exception.message
           message = "Catalog for '#{key}' (#{branch}) failed to compile with #{parallel_catalog_obj.exception.class}: #{msg}"
           message += "\n" + parallel_catalog_obj.exception.backtrace.map { |x| "   #{x}" }.join("\n") if @options[:debug]
           raise OctocatalogDiff::Errors::CatalogError, message
+          # :nocov:
         end
       end
 
@@ -220,22 +231,25 @@ module OctocatalogDiff
         time_start = Time.now
         catalog.build(logger)
         time_it_took = Time.now - time_start
-        retries_str = " retries = #{catalog.retries}" if catalog.retries.is_a?(Fixnum)
+        retries_str = " retries = #{catalog.retries}" if catalog.retries.is_a?(Integer)
         time_str = "in #{time_it_took} seconds#{retries_str}"
         status_str = catalog.valid? ? 'successfully built' : 'failed'
         logger.debug "Catalog for #{opts[:branch]} #{status_str} with #{catalog.builder} #{time_str}"
         catalog
       end
 
-      # Validate a catalog in the parallel execution
+      # The catalog validator method can indicate failure one of two ways:
+      # - Raise an exception (this is preferred, since it gives a specific error message)
+      # - Return false (supported but discouraged, since it only surfaces a generic error)
       # @param catalog [OctocatalogDiff::Catalog] Catalog object
       # @param logger [Logger] Logger object (presently unused)
       # @param args [Hash] Additional arguments set specifically for validator
-      # @return [Boolean] true if catalog is valid, false otherwise
+      # @return [Boolean] Return true if catalog is valid, false otherwise
       def catalog_validator(catalog = nil, _logger = @logger, args = {})
-        return false unless catalog.is_a?(OctocatalogDiff::Catalog)
-        catalog.validate_references if args[:task] == :to
-        catalog.valid?
+        raise ArgumentError, "Expects a catalog, got #{catalog.class}" unless catalog.is_a?(OctocatalogDiff::Catalog)
+        raise OctocatalogDiff::Errors::CatalogError, "Catalog failed: #{catalog.error_message}" unless catalog.valid?
+        catalog.validate_references if args[:task] == :to # Raises exception for broken references
+        true
       end
     end
   end

data/lib/octocatalog-diff/util/httparty.rb

@@ -118,12 +118,20 @@ module OctocatalogDiff
           else
             raise ArgumentError, 'SSL client auth enabled but no client keypair specified'
           end
-          if result[:pem]
-            result[:pem_password] = options[:ssl_client_password] if options[:ssl_client_password]
-            # Make sure there's not a password required, or that if the password is given, it is correct.
-            # We do not want to wait on STDIN.
-            # This will raise OpenSSL::PKey::RSAError if the key needs a password.
-            OpenSSL::PKey::RSA.new(result[:pem], result[:pem_password] || '')
+
+          # Make sure there's not a password required, or that if the password is given, it is correct.
+          # This will raise OpenSSL::PKey::RSAError if the key needs a password.
+          if result[:pem] && options[:ssl_client_password]
+            result[:pem_password] = options[:ssl_client_password]
+            _trash = OpenSSL::PKey::RSA.new(result[:pem], result[:pem_password])
+          elsif result[:pem]
+            # Ruby 2.4 requires a minimum password length of 4. If no password is needed for
+            # the certificate, the specified password here is effectively ignored.
+            # We do not want to wait on STDIN, so a password-protected certificate without a
+            # password will cause this to raise an error. There are two checks here, to exclude
+            # an edge case where somebody did actually put '1234' as their password.
+            _trash = OpenSSL::PKey::RSA.new(result[:pem], '1234')
+            _trash = OpenSSL::PKey::RSA.new(result[:pem], '5678')
           end
         end
 

data/lib/octocatalog-diff/util/parallel.rb

@@ -1,18 +1,22 @@
 # frozen_string_literal: true
 
-# Helper to use the 'parallel' gem to perform tasks
+# A class to parallelize process executation.
+# This is a utility class to execute tasks in parallel, with our own forking implementation
+# that passes through logs and reliably handles errors. If parallel processing has been disabled,
+# this instead executes the tasks serially, but provides the same API as the parallel tasks.
 
-require 'parallel'
 require 'stringio'
 
 module OctocatalogDiff
   module Util
-    # This is a utility class to execute tasks in parallel, using the 'parallel' gem.
-    # If parallel processing has been disabled, this instead executes the tasks serially,
-    # but provides the same API as the parallel tasks.
     class Parallel
+      # This exception is called for a task that didn't complete.
+      class IncompleteTask < RuntimeError; end
+
+      # --------------------------------------
       # This class represents a parallel task. It requires a method reference, which will be executed with
       # any supplied arguments. It can optionally take a text description and a validator function.
+      # --------------------------------------
       class Task
         attr_reader :description
         attr_accessor :args
@@ -35,10 +39,12 @@ module OctocatalogDiff
         end
       end
 
+      # --------------------------------------
       # This class represents the result from a parallel task. The status is set to true (success), false (error),
       # or nil (task was killed before it could complete). The exception (for failure) and output object (for success)
       # are readable attributes. The validity of the results, determined by executing the 'validate' method of the Task,
       # is available to be set and fetched.
+      # --------------------------------------
       class Result
         attr_reader :output, :args
         attr_accessor :status, :exception
@@ -51,121 +57,170 @@ module OctocatalogDiff
         end
       end
 
+      # --------------------------------------
+      # Static methods in the class
+      # --------------------------------------
+
       # Entry point for parallel processing. By default this will perform parallel processing,
       # but it will also accept an option to do serial processing instead.
       # @param task_array [Array<Parallel::Task>] Tasks to run
       # @param logger [Logger] Optional logger object
       # @param parallelized [Boolean] True for parallel processing, false for serial processing
+      # @param raise_exception [Boolean] True to raise exception immediately if one occurs; false to return exception in results
       # @return [Array<Parallel::Result>] Parallel results (same order as tasks)
-      def self.run_tasks(task_array, logger = nil, parallelized = true)
+      def self.run_tasks(task_array, logger = nil, parallelized = true, raise_exception = false)
         # Create a throwaway logger object if one is not given
         logger ||= Logger.new(StringIO.new)
 
-        # Validate input - we need an array. If the array is empty then return an empty array right away.
+        # Validate input - we need an array of OctocatalogDiff::Util::Parallel::Task. If the array is empty then
+        # return an empty array right away.
         raise ArgumentError, "run_tasks() argument must be array, not #{task_array.class}" unless task_array.is_a?(Array)
         return [] if task_array.empty?
 
-        # Make sure each element in the array is a OctocatalogDiff::Util::Parallel::Task
-        task_array.each do |x|
-          next if x.is_a?(OctocatalogDiff::Util::Parallel::Task)
-          raise ArgumentError, "Element #{x.inspect} must be a OctocatalogDiff::Util::Parallel::Task, not a #{x.class}"
+        invalid_inputs = task_array.reject { |task| task.is_a?(OctocatalogDiff::Util::Parallel::Task) }
+        if invalid_inputs.any?
+          ele = invalid_inputs.first
+          raise ArgumentError, "Element #{ele.inspect} must be a OctocatalogDiff::Util::Parallel::Task, not a #{ele.class}"
         end
 
-        # Actually do the processing - choose here between parallel and serial
-        parallelized ? run_tasks_parallel(task_array, logger) : run_tasks_serial(task_array, logger)
+        # Initialize the result array. For now all entries in the array indicate that the task was killed.
+        # Actual statuses will replace this initial status. If the initial status wasn't replaced, then indeed,
+        # the task was killed.
+        result = task_array.map { |x| Result.new(exception: IncompleteTask.new('Killed'), args: x.args) }
+        logger.debug "Initialized parallel task result array: size=#{result.size}"
+
+        # Execute as per the requested method (serial or parallel) and handle results.
+        exception = parallelized ? run_tasks_parallel(result, task_array, logger) : run_tasks_serial(result, task_array, logger)
+        raise exception if exception && raise_exception
+        result
       end
 
-      # Use the parallel gem to run each task in the task array. Under the hood this is forking a process for
-      # each task, and serializing/deserializing the arguments and the outputs.
+      # Utility method! Not intended to be called from outside this class.
+      # ---
+      # Use a forking strategy to run tasks in parallel. Each task in the array is forked in a child
+      # process, and when that task completes it writes its result (OctocatalogDiff::Util::Parallel::Result)
+      # into a serialized data file. Once children are forked this method waits for their return, deserializing
+      # the output from each data file and updating the `result` array with actual results.
+      # @param result [Array<OctocatalogDiff::Util::Parallel::Result>] Parallel task results
       # @param task_array [Array<OctocatalogDiff::Util::Parallel::Task>] Tasks to perform
       # @param logger [Logger] Logger
-      # @return [Array<OctocatalogDiff::Util::Parallel::Result>] Parallel task results
-      def self.run_tasks_parallel(task_array, logger)
-        # Create an empty array of results. The status is nil and the exception is pre-populated. If the code
-        # runs successfully and doesn't get killed, all of these default values will be overwritten. If the code
-        # gets killed before the task finishes, this exception will remain.
-        result = task_array.map do |x|
-          Result.new(exception: ::Parallel::Kill.new('Killed'), args: x.args)
+      # @return [Exception] First exception encountered by a child process; returns nil if no exceptions encountered.
+      def self.run_tasks_parallel(result, task_array, logger)
+        pidmap = {}
+        ipc_tempdir = Dir.mktmpdir
+
+        # Child process forking
+        task_array.each_with_index do |task, index|
+          # simplecov doesn't see this because it's forked
+          # :nocov:
+          this_pid = fork do
+            task_result = execute_task(task, logger)
+            File.open(File.join(ipc_tempdir, "#{Process.pid}.dat"), 'w') { |f| f.write Marshal.dump(task_result) }
+            Kernel.exit! 0 # Kernel.exit! avoids at_exit from parents being triggered by children exiting
+          end
+          # :nocov:
+
+          pidmap[this_pid] = { index: index, start_time: Time.now }
+          logger.debug "Launched pid=#{this_pid} for index=#{index}"
+          logger.reopen if logger.respond_to?(:reopen)
         end
-        logger.debug "Initialized parallel task result array: size=#{result.size}"
 
-        # Do parallel processing
-        ::Parallel.each(task_array,
-                        finish: lambda do |item, i, parallel_result|
-                          # Set the result array element to the result
-                          result[i] = parallel_result
-
-                          # Kill all other parallel tasks if this task failed by throwing an exception
-                          raise ::Parallel::Kill unless parallel_result.exception.nil?
-
-                          # Run the validator to determine if the result is in fact valid. The validator
-                          # returns true or false. If true, set the 'valid' attribute in the result. If
-                          # false, kill all other parallel tasks.
-                          if item.validate(parallel_result.output, logger)
-                            logger.debug("Success #{item.description}")
-                          else
-                            logger.warn("Failed #{item.description}")
-                            result[i].status = false
-                            raise ::Parallel::Kill
-                          end
-                        end) do |ele|
-          # simplecov does not detect that this code runs because it's forked, but this is
-          # tested extensively in the parallel_spec.rb spec file.
-          # :nocov:
+        # Waiting for children and handling results
+        while pidmap.any?
+          this_pid, exit_obj = Process.wait2(0)
+          next unless this_pid && pidmap.key?(this_pid)
+          index = pidmap[this_pid][:index]
+          exitstatus = exit_obj.exitstatus
+          raise "PID=#{this_pid} exited abnormally: #{exit_obj.inspect}" if exitstatus.nil?
+          raise "PID=#{this_pid} exited with status #{exitstatus}" unless exitstatus.zero?
+
+          input = File.read(File.join(ipc_tempdir, "#{this_pid}.dat"))
+          result[index] = Marshal.load(input) # rubocop:disable Security/MarshalLoad
+          time_delta = Time.now - pidmap[this_pid][:start_time]
+          pidmap.delete(this_pid)
+
+          logger.debug "PID=#{this_pid} completed in #{time_delta} seconds, #{input.length} bytes"
+
+          next if result[index].status
+          return result[index].exception
+        end
+
+        logger.debug 'All child processes completed with no exceptions raised'
+
+      # Cleanup: Kill any child processes that are still running, and clean the temporary directory
+      # where data files were stored.
+      ensure
+        pidmap.each do |pid, _pid_data|
           begin
-            logger.debug("Begin #{ele.description}")
-            output = ele.execute(logger)
-            logger.debug("Success #{ele.description}")
-            Result.new(output: output, status: true, args: ele.args)
-          rescue => exc
-            logger.debug("Failed #{ele.description}: #{exc.class} #{exc.message}")
-            Result.new(exception: exc, status: false, args: ele.args)
+            Process.kill('TERM', pid)
+          rescue Errno::ESRCH # rubocop:disable Lint/HandleExceptions
+            # If the process doesn't exist, that's fine.
           end
-          # :nocov:
         end
 
-        # Return result
-        result
+        retries = 0
+        while File.directory?(ipc_tempdir) && retries < 10
+          retries += 1
+          begin
+            FileUtils.remove_entry_secure ipc_tempdir
+          rescue Errno::ENOTEMPTY, Errno::ENOENT # rubocop:disable Lint/HandleExceptions
+            # Errno::ENOTEMPTY will trigger a retry because the directory exists
+            # Errno::ENOENT will break the loop because the directory won't exist next time it's checked
+          end
+        end
       end
 
+      # Utility method! Not intended to be called from outside this class.
+      # ---
       # Perform the tasks in serial.
+      # @param result [Array<OctocatalogDiff::Util::Parallel::Result>] Parallel task results
       # @param task_array [Array<OctocatalogDiff::Util::Parallel::Task>] Tasks to perform
       # @param logger [Logger] Logger
-      # @return [Array<OctocatalogDiff::Util::Parallel::Result>] Parallel task results
-      def self.run_tasks_serial(task_array, logger)
-        # Create an empty array of results. The status is nil and the exception is pre-populated. If the code
-        # runs successfully, all of these default values will be overwritten. If a predecessor task fails, all
-        # later task will have the defined exception.
-        result = task_array.map do |x|
-          Result.new(exception: ::RuntimeError.new('Cancellation - A prior task failed'), args: x.args)
-        end
-
+      def self.run_tasks_serial(result, task_array, logger)
         # Perform the tasks 1 by 1 - each successful task will replace an element in the 'result' array,
         # whereas a failed task will replace the current element with an exception, and all later tasks
         # will not be replaced (thereby being populated with the cancellation error).
-        task_counter = 0
-        task_array.each do |ele|
-          begin
-            logger.debug("Begin #{ele.description}")
-            output = ele.execute(logger)
-            result[task_counter] = Result.new(output: output, status: true, args: ele.args)
-          rescue => exc
-            logger.debug("Failed #{ele.description}: #{exc.class} #{exc.message}")
-            result[task_counter] = Result.new(exception: exc, status: false, args: ele.args)
-          end
+        task_array.each_with_index do |ele, task_counter|
+          result[task_counter] = execute_task(ele, logger)
+          next if result[task_counter].status
+          return result[task_counter].exception
+        end
+        nil
+      end
 
-          if ele.validate(output, logger)
-            logger.debug("Success #{ele.description}")
+      # Utility method! Not intended to be called from outside this class.
+      # ---
+      # Process a single task. Called by run_tasks_parallel / run_tasks_serial.
+      # This method will report all exceptions in the OctocatalogDiff::Util::Parallel::Result object
+      # itself, and not raise them.
+      # @param task [OctocatalogDiff::Util::Parallel::Task] Task object
+      # @param logger [Logger] Logger
+      # @return [OctocatalogDiff::Util::Parallel::Result] Parallel task result
+      def self.execute_task(task, logger)
+        begin
+          logger.debug("Begin #{task.description}")
+          output = task.execute(logger)
+          result = Result.new(output: output, status: true, args: task.args)
+        rescue => exc
+          logger.debug("Failed #{task.description}: #{exc.class} #{exc.message}")
+          # Immediately return without running the validation, since this already failed.
+          return Result.new(exception: exc, status: false, args: task.args)
+        end
+
+        begin
+          if task.validate(output, logger)
+            logger.debug("Success #{task.description}")
           else
-            logger.warn("Failed #{ele.description}")
-            result[task_counter].status = false
+            # Preferably the validator method raised its own exception. However if it
+            # simply returned false, raise our own exception here.
+            raise "Failed #{task.description} validation (unspecified error)"
           end
-
-          break unless result[task_counter].status
-          task_counter += 1
+        rescue => exc
+          logger.warn("Failed #{task.description} validation: #{exc.class} #{exc.message}")
+          result.status = false
+          result.exception = exc
         end
 
-        # Return the result
         result
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: octocatalog-diff
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - GitHub, Inc.
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-09 00:00:00.000000000 Z
+date: 2017-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diffy
@@ -53,20 +53,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.0
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.11.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.11.1
 - !ruby/object:Gem::Dependency
   name: rugged
   requirement: !ruby/object:Gem::Requirement