octocatalog-diff 0.5.6 → 0.6.0

data/lib/octocatalog-diff/catalog-util/enc.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'enc/noop'
 require_relative 'enc/pe'
 require_relative 'enc/script'

data/lib/octocatalog-diff/catalog-util/enc/noop.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module OctocatalogDiff
   module CatalogUtil
     class ENC

data/lib/octocatalog-diff/catalog-util/enc/pe.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'pe/v1'
 require_relative '../../util/httparty'
 require_relative '../facts'

data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'uri'
 require 'yaml'

data/lib/octocatalog-diff/catalog-util/enc/script.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 require 'open3'
 require 'shellwords'
@@ -79,7 +81,7 @@ module OctocatalogDiff
         def script_path(enc, tempdir)
           return enc if enc.start_with? '/'
           raise ArgumentError, 'OctocatalogDiff::CatalogUtil::ENC::Script#new requires :tempdir' unless tempdir.is_a?(String)
-          return File.join(tempdir, enc) if enc =~ %r{^environments/production/}
+          return File.join(tempdir, enc) if enc =~ %r{^environments/[^/]+/}
           File.join(tempdir, 'environments', 'production', enc)
         end
       end

data/lib/octocatalog-diff/catalog-util/facts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../facts'
 
 module OctocatalogDiff

data/lib/octocatalog-diff/catalog-util/fileresources.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest'
 
 module OctocatalogDiff
@@ -11,9 +13,9 @@ module OctocatalogDiff
       # Public method: Convert file resources to text. See the description of the class
       # just above for details.
       # @param obj [OctocatalogDiff::Catalog] Catalog object (will be modified)
-      def self.convert_file_resources(obj)
+      def self.convert_file_resources(obj, environment = 'production')
         return unless obj.valid? && obj.compilation_dir.is_a?(String) && !obj.compilation_dir.empty?
-        _convert_file_resources(obj.resources, obj.compilation_dir)
+        _convert_file_resources(obj.resources, obj.compilation_dir, environment)
         begin
           obj.catalog_json = ::JSON.generate(obj.catalog)
         rescue ::JSON::GeneratorError => exc
@@ -43,20 +45,23 @@ module OctocatalogDiff
       end
 
       # Internal method: Parse environment.conf to find the modulepath
-      # @param compilation_dir [String] Compilation directory
+      # @param dir [String] Directory in which to look for environment.conf
       # @return [Array] Module paths
-      def self.module_path(compilation_dir)
-        environment_conf = File.join(compilation_dir, 'environment.conf')
-        unless File.file?(environment_conf)
-          return [File.join(compilation_dir, 'modules')]
-        end
+      def self.module_path(dir)
+        environment_conf = File.join(dir, 'environment.conf')
+        return [File.join(dir, 'modules')] unless File.file?(environment_conf)
 
         # This doesn't support multi-line, continuations with backslash, etc.
         # Does it need to??
         if File.read(environment_conf) =~ /^modulepath\s*=\s*(.+)/
-          Regexp.last_match(1).split(/:/).map(&:strip).reject { |x| x =~ /^\$/ }.map { |x| File.join(compilation_dir, x) }
+          result = []
+          Regexp.last_match(1).split(/:/).map(&:strip).each do |path|
+            next if path.start_with?('$')
+            result << File.expand_path(path, dir)
+          end
+          result
         else
-          [File.join(compilation_dir, 'modules')]
+          [File.join(dir, 'modules')]
         end
       end
 
@@ -64,15 +69,15 @@ module OctocatalogDiff
       # required, or else this is a no-op. The passed-in array of resources is modified by this method.
       # @param resources [Array<Hash>] Array of catalog resources
       # @param compilation_dir [String] Compilation directory (so files can be looked up)
-      def self._convert_file_resources(resources, compilation_dir)
+      def self._convert_file_resources(resources, compilation_dir, environment = 'production')
         # Calculate compilation directory. There is not explicit error checking here because
         # there is on-demand, explicit error checking for each file within the modification loop.
         return unless compilation_dir.is_a?(String) && compilation_dir != ''
 
-        # Making sure that compilation_dir/environments/production/modules exists (and by inference,
-        # that compilation_dir/environments/production is pointing at the right place). Otherwise, try to find
+        # Making sure that compilation_dir/environments/<env>/modules exists (and by inference,
+        # that compilation_dir/environments/<env> is pointing at the right place). Otherwise, try to find
         # compilation_dir/modules. If neither of those exist, this code can't run.
-        env_dir = File.join(compilation_dir, 'environments', 'production')
+        env_dir = File.join(compilation_dir, 'environments', environment)
         modulepaths = module_path(env_dir) + module_path(compilation_dir)
         modulepaths.select! { |x| File.directory?(x) }
         return if modulepaths.empty?

data/lib/octocatalog-diff/catalog-util/git.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 require 'open3'
 require 'rugged'

data/lib/octocatalog-diff/catalog.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'stringio'
 
@@ -19,6 +21,7 @@ module OctocatalogDiff
     # Error classes that we can throw
     class PuppetVersionError < RuntimeError; end
     class CatalogError < RuntimeError; end
+    class ReferenceValidationError < RuntimeError; end
 
     # Constructor
     # @param :backend [Symbol] If set, this will force a backend
@@ -71,10 +74,18 @@ module OctocatalogDiff
       # Perform post-generation processing of the catalog
       return unless valid?
       unless @catalog_obj.respond_to?(:convert_file_resources) && @catalog_obj.convert_file_resources == false
-        OctocatalogDiff::CatalogUtil::FileResources.convert_file_resources(self) if @options.fetch(:compare_file_text, false)
+        if @options.fetch(:compare_file_text, false)
+          OctocatalogDiff::CatalogUtil::FileResources.convert_file_resources(self, environment)
+        end
       end
     end
 
+    # Compilation environment
+    # @return [String] Compilation environment (if set), else 'production' by default
+    def environment
+      @catalog_obj.respond_to?(:environment) ? @catalog_obj.environment : 'production'
+    end
+
     # For logging we may wish to know the backend being used
     # @return [String] Class of backend used
     def builder
@@ -155,7 +166,7 @@ module OctocatalogDiff
       # This is a bug condition
       # :nocov:
       raise "BUG: catalog has no data::resources or ::resources array. Please report this. #{@catalog.inspect}"
-      # :nocov
+      # :nocov:
     end
 
     # This retrieves the number of retries necessary to compile the catalog. If the underlying catalog
@@ -171,8 +182,66 @@ module OctocatalogDiff
       !@catalog.nil?
     end
 
+    # Determine if all of the (before, notify, require, subscribe) targets are actually in the catalog.
+    # Raise a ReferenceValidationError for any found to be missing.
+    # Uses @options[:validate_references] to influence which references are checked.
+    def validate_references
+      # Skip out early if no reference validation has been requested.
+      unless @options[:validate_references].is_a?(Array) && @options[:validate_references].any?
+        return
+      end
+
+      # Iterate over all the resources and check each one that has one of the attributes being checked.
+      # Keep track of all references that are missing for ultimate inclusion in the error message.
+      missing = []
+      resources.each do |x|
+        @options[:validate_references].each do |r|
+          next unless x.key?('parameters')
+          next unless x['parameters'].key?(r)
+          missing_resources = resources_missing_from_catalog(x['parameters'][r])
+          next unless missing_resources.any?
+          missing.concat missing_resources.map { |missing_target| { source: x, target_type: r, target_value: missing_target } }
+        end
+      end
+      return if missing.empty?
+
+      # At this point there is at least one broken/missing reference. Format an error message and
+      # raise. Error message will look like this:
+      # ---
+      # Catalog has broken references: exec[subscribe caller 1] -> subscribe[Exec[subscribe target]];
+      # exec[subscribe caller 2] -> subscribe[Exec[subscribe target]]; exec[subscribe caller 2] ->
+      # subscribe[Exec[subscribe target 2]]
+      # ---
+      formatted_references = missing.map do |obj|
+        # obj[:target_value] can be a string or an array. If it's an array, create a
+        # separate error message per element of that array. This allows the total number
+        # of errors to be correct.
+        src = "#{obj[:source]['type'].downcase}[#{obj[:source]['title']}]"
+        target_val = obj[:target_value].is_a?(Array) ? obj[:target_value] : [obj[:target_value]]
+        target_val.map { |tv| "#{src} -> #{obj[:target_type].downcase}[#{tv}]" }
+      end
+      formatted_references.flatten!
+      plural = formatted_references.size == 1 ? '' : 's'
+      errors = formatted_references.join('; ')
+      raise ReferenceValidationError, "Catalog has broken reference#{plural}: #{errors}"
+    end
+
     private
 
+    # Private method: Given a list of resources to check, return the references from
+    # that list that are missing from the catalog. (An empty array returned would indicate
+    # all references are present in the catalog.)
+    # @param resources_to_check [String / Array] Resources to check
+    # @return [Array] References that are missing from catalog
+    def resources_missing_from_catalog(resources_to_check)
+      [resources_to_check].flatten.select do |res|
+        unless res =~ /\A([\w:]+)\[(.+)\]\z/
+          raise ArgumentError, "Resource #{res} is not in the expected format"
+        end
+        resource(type: Regexp.last_match(1), title: Regexp.last_match(2)).nil?
+      end
+    end
+
     # Private method: Choose backend based on passed-in options
     # @param options [Hash] Options passed into constructor
     # @return [Object] OctocatalogDiff::Catalog::<whatever> object
@@ -203,6 +272,10 @@ module OctocatalogDiff
       resources.each do |resource|
         @resource_hash[resource['type']] ||= {}
         @resource_hash[resource['type']][resource['title']] = resource
+
+        if resource.key?('parameters') && resource['parameters'].key?('alias')
+          @resource_hash[resource['type']][resource['parameters']['alias']] = resource
+        end
       end
     end
   end

data/lib/octocatalog-diff/catalog/computed.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 require 'json'
 require 'open3'
@@ -76,6 +78,11 @@ module OctocatalogDiff
         @builddir.tempdir
       end
 
+      # Environment used to compile catalog
+      def environment
+        @opts[:preserve_environments] ? @opts.fetch(:environment, 'production') : 'production'
+      end
+
       private
 
       # Private method: Clean up a checkout directory, if it exists
@@ -179,11 +186,20 @@ module OctocatalogDiff
         }
       end
 
+      # Private method: Make sure that the Puppet environment directory exists.
+      def assert_that_puppet_environment_directory_exists
+        target_dir = File.join(@builddir.tempdir, 'environments', environment)
+        return if File.directory?(target_dir)
+        raise Errno::ENOENT, "Environment directory #{target_dir} does not exist"
+      end
+
       # Private method: Runs puppet on the command line to compile the catalog
       # Exit code is 0 if catalog generation was successful, non-zero otherwise.
       # @param logger [Logger] Logger object
       # @return [Hash] { stdout: <catalog as JSON>, stderr: <error messages>, exitcode: <hopefully 0> }
       def run_puppet(logger)
+        assert_that_puppet_environment_directory_exists
+
         # Run 'cmd' with environment 'env' from directory 'dir'
         # First line of a successful result needs to be stripped off. It will look like:
         # Notice: Compiled catalog for xxx in environment production in 27.88 seconds

data/lib/octocatalog-diff/catalog/json.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 
 module OctocatalogDiff

data/lib/octocatalog-diff/catalog/noop.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 
 module OctocatalogDiff

data/lib/octocatalog-diff/catalog/puppetdb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'stringio'
 

data/lib/octocatalog-diff/catalog/puppetmaster.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../catalog-util/facts'
 require_relative '../external/pson/pure'
 require_relative '../util/httparty'
@@ -68,13 +70,17 @@ module OctocatalogDiff
 
       # Returns a hash of parameters for each supported version of the Puppet Server Catalog API.
       # @return [Hash] Hash of parameters
+      #
+      # Note: The double escaping of the facts here is implemented to correspond to a long standing
+      # bug in the Puppet code. See https://github.com/puppetlabs/puppet/pull/1818 and
+      # https://docs.puppet.com/puppet/latest/http_api/http_catalog.html#parameters for explanation.
       def puppet_catalog_api
         {
           2 => {
             url: "https://#{@options[:puppet_master]}/#{@options[:branch]}/catalog/#{@node}",
             parameters: {
               'facts_format' => 'pson',
-              'facts' => @facts.fudge_timestamp.without('trusted').to_pson,
+              'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
               'transaction_uuid' => SecureRandom.uuid
             }
           },
@@ -83,7 +89,7 @@ module OctocatalogDiff
             parameters: {
               'environment' => @options[:branch],
               'facts_format' => 'pson',
-              'facts' => @facts.fudge_timestamp.without('trusted').to_pson,
+              'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
               'transaction_uuid' => SecureRandom.uuid
             }
           }

data/lib/octocatalog-diff/facts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'facts/json'
 require_relative 'facts/yaml'
 require_relative 'facts/puppetdb'

data/lib/octocatalog-diff/facts/json.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../facts'
 
 require 'json'

data/lib/octocatalog-diff/facts/puppetdb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../facts'
 require_relative '../puppetdb'
 require 'yaml'
@@ -6,6 +8,12 @@ module OctocatalogDiff
   class Facts
     # Deal with facts in PuppetDB
     class PuppetDB
+      # Supporting multiple versions of the PuppetDB API.
+      PUPPETDB_QUERY_FACTS_URL = {
+        '3' => '/v3/nodes/<NODE>/facts',
+        '4' => '/pdb/query/v4/nodes/<NODE>/facts'
+      }.freeze
+
       # Retrieve facts from PuppetDB for a specified node.
       # @param :puppetdb_url [String|Array] => URL to PuppetDB
       # @param :retry [Fixnum] => Retry after timeout (default 0 retries, can be more)
@@ -15,7 +23,8 @@ module OctocatalogDiff
         # Set up some variables from options
         raise ArgumentError, 'puppetdb_url is required' unless options[:puppetdb_url].is_a?(String)
         raise ArgumentError, 'node must be a non-empty string' unless node.is_a?(String) && node != ''
-        uri = "/pdb/query/v4/nodes/#{node}/facts"
+        puppetdb_api_version = options.fetch(:puppetdb_api_version, 4)
+        uri = PUPPETDB_QUERY_FACTS_URL.fetch(puppetdb_api_version.to_s).gsub('<NODE>', node)
         retries = options.fetch(:retry, 0).to_i
 
         # Construct puppetdb object and options

data/lib/octocatalog-diff/facts/yaml.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../facts'
 require 'yaml'
 

data/lib/octocatalog-diff/puppetdb.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'util/httparty'
 
 require 'uri'

data/lib/octocatalog-diff/util/colored.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Create colorizing methods in the 'String' class, but only if 'colors_enabled'
 # has been set.
 class String

data/lib/octocatalog-diff/util/httparty.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'httparty'
 require 'json'
 require_relative '../external/pson/pure'

data/lib/octocatalog-diff/util/parallel.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Helper to use the 'parallel' gem to perform tasks
 
 require 'parallel'
@@ -20,6 +22,7 @@ module OctocatalogDiff
           @args = opts.fetch(:args, {})
           @description = opts[:description] || @method.name
           @validator = opts[:validator]
+          @validator_args = opts[:validator_args] || {}
         end
 
         def execute(logger = Logger.new(StringIO.new))
@@ -28,7 +31,7 @@ module OctocatalogDiff
 
         def validate(result, logger = Logger.new(StringIO.new))
           return true if @validator.nil?
-          @validator.call(result, logger)
+          @validator.call(result, logger, @validator_args)
         end
       end
 

data/lib/octocatalog-diff/util/puppetversion.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Helper to determine the version of Puppet
 
 require 'fileutils'

data/lib/octocatalog-diff/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module OctocatalogDiff
   # Determine the version of octocatalog-diff
   class Version

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: octocatalog-diff
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.6.0
 platform: ruby
 authors:
 - GitHub, Inc.
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2017-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diffy
@@ -226,8 +226,10 @@ files:
 - doc/advanced-bootstrap.md
 - doc/advanced-cache-dir.md
 - doc/advanced-catalog-only.md
+- doc/advanced-catalog-validation.md
 - doc/advanced-ci.md
 - doc/advanced-dynamic-ignores.md
+- doc/advanced-environments.md
 - doc/advanced-future-parser.md
 - doc/advanced-hiera-path-stripping.md
 - doc/advanced-ignores.md
@@ -251,6 +253,7 @@ files:
 - doc/dev/how-to-add-options.md
 - doc/dev/integration-tests.md
 - doc/dev/releasing.md
+- doc/dev/run-from-branch.md
 - doc/installation.md
 - doc/limitations.md
 - doc/optionsref.md
@@ -274,13 +277,16 @@ files:
 - lib/octocatalog-diff/catalog-diff/cli/options/cached_master_dir.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/catalog_only.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/color.rb
+- lib/octocatalog-diff/catalog-diff/cli/options/command_line.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/compare_file_text.rb
+- lib/octocatalog-diff/catalog-diff/cli/options/create_symlinks.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/debug.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/debug_bootstrap.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/display_datatype_changes.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/display_detail_add.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/display_source_file_line.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/enc.rb
+- lib/octocatalog-diff/catalog-diff/cli/options/environment.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/existing_catalogs.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/fact_file.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/fact_override.rb
@@ -307,12 +313,14 @@ files:
 - lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token_file.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_url.rb
+- lib/octocatalog-diff/catalog-diff/cli/options/preserve_environments.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppet_binary.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppet_master.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_api_version.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_ca.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_cert.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_key.rb
+- lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_api_version.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_ca.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_cert.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_key.rb
@@ -325,6 +333,7 @@ files:
 - lib/octocatalog-diff/catalog-diff/cli/options/storeconfigs.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/suppress_absent_file_details.rb
 - lib/octocatalog-diff/catalog-diff/cli/options/to_from_branch.rb
+- lib/octocatalog-diff/catalog-diff/cli/options/validate_references.rb
 - lib/octocatalog-diff/catalog-diff/cli/printer.rb
 - lib/octocatalog-diff/catalog-diff/differ.rb
 - lib/octocatalog-diff/catalog-diff/display.rb