octocatalog-diff 1.5.3 → 1.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d454b6d3a7fc5ec8ff2f55231e83fc2cad36d2a
-  data.tar.gz: 10dba5a6f4694bebd11fa01465fdbbdc1cb77774
+  metadata.gz: 0ba4dc520ab7e8ae27324beef80edd6c96f601de
+  data.tar.gz: d927873135a73e5f1b8c178e0cf4a54b9a29aa46
 SHA512:
-  metadata.gz: f4edfbcd8f3d5e4508e7610ba47f93159608f6572c3585fc506eba2c2037a541193fe76d018afee365ddf6c0b6a27e7df32ff0145ded794e3cab5d9ed7d67115
-  data.tar.gz: 0053a799e8342b2df02d722502c1dc1ead513148ce78e9e1616a715e33af63cb1990b29afe1abae81ef18659295efbc6b14190db498215c8b4438bf5397eb9a3
+  metadata.gz: e7a23ff7139158b5092765b190a801fbd670fe2931dde206fad3007fdf03995e3e6f62a881f9b3258d2b91880688664c7f43c947f5a2cfdb9b1c9b36516c94be
+  data.tar.gz: 4eae1cff736c29ced07644eac4cd4af812df56ac6c32afade41cf85ba0771d8136d79bf07783bec6a101cec3e5880ac647fbb83db55fe352b1181db6422288d0

data/.version

@@ -1 +1 @@
-1.5.3
+1.5.4

data/doc/CHANGELOG.md

@@ -8,6 +8,17 @@
 </tr>
 </thead><tbody>
 
+<tr valign=top>
+<td>1.5.4</td>
+<td>2018-12-11</td>
+<td>
+<li><a href="https://github.com/github/octocatalog-diff/pull/190">#190</a>: (Enhancement) Additional filtered out cases for compilation directory</li>
+<li><a href="https://github.com/github/octocatalog-diff/pull/195">#195</a>: (Enhancement) Parallel catalog-diff when multiple hostnames are passed</li>
+<li><a href="https://github.com/github/octocatalog-diff/pull/198">#198</a>: (Bug Fix) Portability fixes</li>
+<li><a href="https://github.com/github/octocatalog-diff/pull/200">#200</a>: (Bug Fix) Support name parameter when validating references</li>
+</td>
+</tr>
+
 <tr valign=top>
 <td>1.5.3</td>
 <td>2018-03-05</td>

data/doc/dev/api/v1/calls/catalog-diff.md

@@ -167,12 +167,16 @@ In this case, "owner", "notify", and "content" are nested under "parameters". In
 
   ```
   # Ignore all changes to the `owner` attribute of a file.
-  [ { type: Regexp.new('\AFile\z'), attr: Regexp.new("\Aparameters\fowner\z" } ]
+  [ { type: Regexp.new('\AFile\z'), attr: Regexp.new("\\Aparameters\fowner\\z" } ]
 
   # Ignore changes to `owner` or `group` for a file or an exec.
-  [ { type: Regexp.new('\A(File|Exec)\z'), attr: Regexp.new("\Aparameters\f(owner|group)\z" } ]
+  [ { type: Regexp.new('\A(File|Exec)\z'), attr: Regexp.new("\\Aparameters\f(owner|group)\\z" } ]
   ```
 
+When using regular expressions, `\f` (form feed character) is used to separate the structure (e.g. `parameters\fowner` refers to the `parameters` hash, `owner` key).
+
+:bulb: Note that `\A` in Ruby matches the beginning of the string and `\z` matches the end, but these are not actual characters. Therefore, if you are using `\A` or `\z` in double quotes (`"`), be sure to heed the examples above and write your expression like: `Regexp.new("\\Aparameters\fowner\\z")`.
+
 #### `:validate_references` (Array<String>, Optional)
 
 Invoke the [catalog validation](/doc/advanced-catalog-validation.md) feature to ensure resources targeted by `before`, `notify`, `require`, and/or `subscribe` exist in the catalog. If this parameter is not defined, no reference validation occurs.

data/doc/dev/api/v1/objects/diff.md

@@ -94,9 +94,9 @@ Returns the value of the resource from the new catalog.
     }
   }
 
-  # Demonstrates structure and old_value
+  # Demonstrates structure and new_value
   diff.structure #=> ['parameters', 'content']
-  diff.old_value #=> 'This is the NEW FILE!!!!!'
+  diff.new_value #=> 'This is the NEW FILE!!!!!'
   ```
 
 #### `#old_file` (String)
@@ -107,7 +107,7 @@ Note that this is a pass-through of information provided in the Puppet catalog,
 
 Note also that if the diff represents addition of a resource, this will return `nil`, because the resource does not exist in the old catalog.
 
-#### `#old_file` (String)
+#### `#old_line` (String)
 
 Returns the line number within the Puppet manifest giving rise to the resource as it exists in the old catalog. (See `#old_file` for the filename of the Puppet manifest.)
 

data/doc/dev/releasing.md

@@ -9,7 +9,7 @@ The project maintainers are responsible for bumping the version number, regenera
 To test the new version of `octocatalog-diff` in the GitHub Puppet repository, check out `github/puppet` and:
 
 - Start a new branch based off master
-- Run `script/update-octocatalog-diff -r <ocd_branch_name>`
+- Run `script/update-octocatalog-diff <ocd_branch_name>`
 - Confirm and commit the result
 - Make sure all CI jobs pass
 - Run the `puppet-catalog-diff` CI job and make sure it passes and shows expected results

data/doc/optionsref.md

@@ -9,7 +9,8 @@
 
 ```
 Usage: octocatalog-diff [command line options]
-    -n, --hostname HOSTNAME          Use PuppetDB facts from last run of hostname
+    -n HOSTNAME1[,HOSTNAME2[,...]],  Use PuppetDB facts from last run of a hostname or a comma separated list of multiple hostnames
+        --hostname
         --basedir DIRNAME            Use an alternate base directory (git checkout of puppet repository)
     -f, --from FROM_BRANCH           Branch you are coming from
     -t, --to TO_BRANCH               Branch you are going to
@@ -856,14 +857,17 @@ Puppet control repo template, the value of this should be 'hieradata', which is
 
   <tr>
     <td valign=top>
-      <pre><code>-n HOSTNAME
---hostname HOSTNAME</code></pre>
+      <pre><code>-n HOSTNAME1[,HOSTNAME2[,...]]
+--hostname HOSTNAME1[,HOSTNAME2[,...]]</code></pre>
     </td>
     <td valign=top>
-      Use PuppetDB facts from last run of hostname
+      Use PuppetDB facts from last run of a hostname or a comma separated list of multiple hostnames
     </td>
     <td valign=top>
-      Set hostname, which is used to look up facts in PuppetDB, and in the header of diff display. (<a href="../lib/octocatalog-diff/cli/options/hostname.rb">hostname.rb</a>)
+      Set hostname, which is used to look up facts in PuppetDB, and in the header of diff display.
+This option can recieve a single hostname, or a comma separated list of
+multiple hostnames, which are split into an Array. Multiple hostnames do not
+work with the `catalog-only` or `bootstrap-then-exit` options. (<a href="../lib/octocatalog-diff/cli/options/hostname.rb">hostname.rb</a>)
     </td>
   </tr>
 

data/lib/octocatalog-diff/catalog-diff/differ.rb

@@ -294,10 +294,13 @@ module OctocatalogDiff
         # Use diffy to get only the lines that have changed in a text object.
         # As we iterate through the diff, jump out if we have our answer: either
         # true if '=~>' finds ANY match, or false if '=&>' fails to find a match.
-        Diffy::Diff.new(old_val, new_val, context: 0).each do |line|
+        diffy_result = Diffy::Diff.new(old_val, new_val, context: 0)
+        newline_alerts = diffy_result.count { |line| line.strip == '\' }
+        diffy_result.each do |line|
           if regex.match(line.strip)
             return true if operator == '=~>'
           elsif operator == '=&>'
+            next if line.strip == '\' && newline_alerts == 2
             return false
           end
         end
@@ -394,6 +397,13 @@ module OctocatalogDiff
           return false unless rule[:title].casecmp(hsh[:title]).zero?
         end
 
+        # If rule[:attr] is a regular expression, handle that case here.
+        if rule[:attr].is_a?(Regexp)
+          return false unless hsh[:attr].is_a?(String)
+          return false unless rule[:attr].match(hsh[:attr])
+          return ignore_match_true(hsh, rule)
+        end
+
         # Special 'attributes': Ignore specific diff types (+ add, - remove, ~ and ! change)
         if rule[:attr] =~ /\A[\-\+~!]+\Z/
           return ignore_match_true(hsh, rule) if rule[:attr].include?(diff_type)

data/lib/octocatalog-diff/catalog-diff/filter/compilation_dir.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../filter'
+require_relative '../../util/util'
 
 module OctocatalogDiff
   module CatalogDiff
@@ -35,43 +36,46 @@ module OctocatalogDiff
 
           # Check for a change where the difference in a parameter exactly corresponds to the difference in the
           # compilation directory.
-          if diff.change? && (diff.old_value.is_a?(String) || diff.new_value.is_a?(String))
-            from_before = nil
-            from_after = nil
-            from_match = false
-            to_before = nil
-            to_after = nil
-            to_match = false
+          if diff.change?
+            o = remove_compilation_dir(diff.old_value, dir2)
+            n = remove_compilation_dir(diff.new_value, dir1)
 
-            if diff.old_value =~ /^(.*)#{dir2}(.*)$/m
-              from_before = Regexp.last_match(1) || ''
-              from_after = Regexp.last_match(2) || ''
-              from_match = true
-            end
-
-            if diff.new_value =~ /^(.*)#{dir1}(.*)$/m
-              to_before = Regexp.last_match(1) || ''
-              to_after = Regexp.last_match(2) || ''
-              to_match = true
-            end
-
-            if from_match && to_match && to_before == from_before && to_after == from_after
+            if o != diff.old_value || n != diff.new_value
               message = "Resource key #{diff.type}[#{diff.title}] #{diff.structure.join(' => ')}"
-              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
+              message += ' may depend on catalog compilation directory, but there may be differences.'
+              message += ' This is included in results for now, but please verify.'
               @logger.warn message
-              return true
             end
 
-            if from_match || to_match
+            if o == n
               message = "Resource key #{diff.type}[#{diff.title}] #{diff.structure.join(' => ')}"
-              message += ' may depend on catalog compilation directory, but there may be differences.'
-              message += ' This is included in results for now, but please verify.'
+              message += ' appears to depend on catalog compilation directory. Suppressed from results.'
               @logger.warn message
+              return true
             end
           end
 
           false
         end
+
+        def remove_compilation_dir(v, dir)
+          value = OctocatalogDiff::Util::Util.deep_dup(v)
+          traverse(value) do |e|
+            e.gsub!(dir, '') if e.respond_to?(:gsub!)
+          end
+          value
+        end
+
+        def traverse(a)
+          case a
+          when Array
+            a.map { |v| traverse(v, &Proc.new) }
+          when Hash
+            traverse(a.values, &Proc.new)
+          else
+            yield a
+          end
+        end
       end
     end
   end

data/lib/octocatalog-diff/catalog.rb

@@ -331,8 +331,9 @@ module OctocatalogDiff
         title = normalized_title(resource['title'], resource['type'])
         @resource_hash[resource['type']][title] = resource
 
-        if resource.key?('parameters') && resource['parameters'].key?('alias')
-          @resource_hash[resource['type']][resource['parameters']['alias']] = resource
+        if resource.key?('parameters')
+          @resource_hash[resource['type']][resource['parameters']['alias']] = resource if resource['parameters'].key?('alias')
+          @resource_hash[resource['type']][resource['parameters']['name']] = resource if resource['parameters'].key?('name')
         end
       end
     end

data/lib/octocatalog-diff/cli.rb

@@ -11,6 +11,7 @@ require_relative 'util/util'
 require_relative 'version'
 
 require 'logger'
+require 'parallel'
 require 'socket'
 
 module OctocatalogDiff
@@ -116,16 +117,46 @@ module OctocatalogDiff
       end
 
       # Compile catalogs and do catalog-diff
-      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options.merge(logger: logger))
+      node_set = options.delete(:node)
+      node_set = [node_set] unless node_set.is_a?(Array)
+
+      # run multiple node diffs in parallel
+      catalog_diffs = if node_set.size == 1
+        [run_octocatalog_diff(node_set.first, options, logger)]
+      else
+        ::Parallel.map(node_set, in_threads: 4) { |node| run_octocatalog_diff(node, options, logger) }
+      end
+
+      # Return the resulting diff object if requested (generally for testing)
+      # or otherwise return exit code
+      return catalog_diffs.first if opts[:INTEGRATION]
+
+      all_diffs = catalog_diffs.map(&:diffs)
+
+      all_diffs.each do |diff|
+        next unless diff.any?
+        return EXITCODE_SUCCESS_WITH_DIFFS
+      end
+
+      EXITCODE_SUCCESS_NO_DIFFS
+    end
+
+    # Run the octocatalog-diff process for a given node. Return the diffs for a contribution to
+    # the final exit status.
+    # node    - String with the node
+    # options - All of the currently defined options
+    # logger  - Logger object
+    def self.run_octocatalog_diff(node, options, logger)
+      options_copy = options.merge(node: node)
+      catalog_diff = OctocatalogDiff::API::V1.catalog_diff(options_copy.merge(logger: logger))
       diffs = catalog_diff.diffs
 
       # Display diffs
-      printer_obj = OctocatalogDiff::Cli::Printer.new(options, logger)
+      printer_obj = OctocatalogDiff::Cli::Printer.new(options_copy, logger)
       printer_obj.printer(diffs, catalog_diff.from.compilation_dir, catalog_diff.to.compilation_dir)
 
-      # Return the resulting diff object if requested (generally for testing) or otherwise return exit code
-      return catalog_diff if opts[:INTEGRATION]
-      diffs.any? ? EXITCODE_SUCCESS_WITH_DIFFS : EXITCODE_SUCCESS_NO_DIFFS
+      # Return catalog-diff object.
+      catalog_diff
     end
 
     # Parse command line options with 'optparse'. Returns a hash with the parsed arguments.

data/lib/octocatalog-diff/cli/options.rb

@@ -11,7 +11,7 @@ module OctocatalogDiff
     # This class contains the option parser. 'parse_options' is the external entry point.
     class Options
       # The usage banner.
-      BANNER = 'Usage: catalog-diff -n <hostname> [-f <from environment>] [-t <to environment>]'.freeze
+      BANNER = 'Usage: catalog-diff -n <hostname>[,<hostname>...] [-f <from environment>] [-t <to environment>]'.freeze
 
       # An error class specifically for passing information to the document build task.
       class DocBuildError < RuntimeError; end

data/lib/octocatalog-diff/cli/options/hostname.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 # Set hostname, which is used to look up facts in PuppetDB, and in the header of diff display.
+# This option can recieve a single hostname, or a comma separated list of
+# multiple hostnames, which are split into an Array. Multiple hostnames do not
+# work with the `catalog-only` or `bootstrap-then-exit` options.
 # @param parser [OptionParser object] The OptionParser argument
 # @param options [Hash] Options hash being constructed; this is modified in this method.
 
@@ -8,8 +11,16 @@ OctocatalogDiff::Cli::Options::Option.newoption(:hostname) do
   has_weight 1
 
   def parse(parser, options)
-    parser.on('--hostname HOSTNAME', '-n', 'Use PuppetDB facts from last run of hostname') do |hostname|
-      options[:node] = hostname
+    parser.on(
+      '--hostname HOSTNAME1[,HOSTNAME2[,...]]',
+      '-n',
+      'Use PuppetDB facts from last run of a hostname or a comma separated list of multiple hostnames'
+    ) do |hostname|
+      options[:node] = if hostname.include?(',')
+        hostname.split(',')
+      else
+        hostname
+      end
     end
   end
 end

data/scripts/env/env.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script echoes back the environment. This is used for spec testing
 # and possible debugging.

data/scripts/git-extract/git-extract.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script is called from lib/octocatalog-diff/catalog-util/git.rb and is used to
 # archive and extract a certain branch of a git repository into a target directory.

data/scripts/puppet/puppet.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to run Puppet. The default implementation here is simply to pass
 # through the command line arguments (which are likely to be numerous when

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: octocatalog-diff
 version: !ruby/object:Gem::Version
-  version: 1.5.3
+  version: 1.5.4
 platform: ruby
 authors:
 - GitHub, Inc.
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-05 00:00:00.000000000 Z
+date: 2018-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diffy
@@ -53,6 +53,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.0
 - !ruby/object:Gem::Dependency
   name: rugged
   requirement: !ruby/object:Gem::Requirement
@@ -87,14 +101,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 11.2.2
+        version: 12.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 11.2.2
+        version: 12.3.1
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
@@ -171,14 +185,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.4.0
+        version: 5.5.8
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.4.0
+        version: 5.5.8
 description: |
   Octocatalog-Diff assists with Puppet development and testing by enabling the user to
   compile 2 Puppet catalogs and compare them. It is possible to compare different