octocatalog-diff 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/.version +1 -0
  3. data/LICENSE +20 -0
  4. data/README.md +82 -0
  5. data/bin/octocatalog-diff +75 -0
  6. data/doc/advanced-bootstrap.md +33 -0
  7. data/doc/advanced-cache-dir.md +24 -0
  8. data/doc/advanced-catalog-only.md +37 -0
  9. data/doc/advanced-ci.md +13 -0
  10. data/doc/advanced-dynamic-ignores.md +123 -0
  11. data/doc/advanced-future-parser.md +11 -0
  12. data/doc/advanced-ignores.md +224 -0
  13. data/doc/advanced-output-formats.md +96 -0
  14. data/doc/advanced-output-hacks.md +45 -0
  15. data/doc/advanced-override-facts.md +67 -0
  16. data/doc/advanced-pe-enc.md +52 -0
  17. data/doc/advanced-puppet-master.md +50 -0
  18. data/doc/advanced-puppet-versions.md +9 -0
  19. data/doc/advanced-storeconfigs.md +72 -0
  20. data/doc/advanced-using-without-git.md +15 -0
  21. data/doc/advanced.md +43 -0
  22. data/doc/basic.md +70 -0
  23. data/doc/configuration-enc.md +69 -0
  24. data/doc/configuration-hiera.md +103 -0
  25. data/doc/configuration-puppetdb.md +49 -0
  26. data/doc/configuration.md +51 -0
  27. data/doc/dev/README.md +1 -0
  28. data/doc/dev/coverage.md +34 -0
  29. data/doc/dev/how-to-add-options.md +83 -0
  30. data/doc/dev/integration-tests.md +63 -0
  31. data/doc/dev/releasing.md +19 -0
  32. data/doc/installation.md +49 -0
  33. data/doc/limitations.md +34 -0
  34. data/doc/optionsref.md +947 -0
  35. data/doc/requirements.md +16 -0
  36. data/doc/roadmap.md +26 -0
  37. data/doc/similar.md +17 -0
  38. data/doc/troubleshooting.md +54 -0
  39. data/lib/octocatalog-diff.rb +12 -0
  40. data/lib/octocatalog-diff/bootstrap.rb +53 -0
  41. data/lib/octocatalog-diff/catalog-diff/cli.rb +205 -0
  42. data/lib/octocatalog-diff/catalog-diff/cli/catalogs.rb +240 -0
  43. data/lib/octocatalog-diff/catalog-diff/cli/diffs.rb +145 -0
  44. data/lib/octocatalog-diff/catalog-diff/cli/helpers/fact_override.rb +99 -0
  45. data/lib/octocatalog-diff/catalog-diff/cli/options.rb +173 -0
  46. data/lib/octocatalog-diff/catalog-diff/cli/options/basedir.rb +14 -0
  47. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_environment.rb +18 -0
  48. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_script.rb +14 -0
  49. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_then_exit.rb +12 -0
  50. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrapped_dirs.rb +18 -0
  51. data/lib/octocatalog-diff/catalog-diff/cli/options/cached_master_dir.rb +21 -0
  52. data/lib/octocatalog-diff/catalog-diff/cli/options/catalog_only.rb +14 -0
  53. data/lib/octocatalog-diff/catalog-diff/cli/options/color.rb +13 -0
  54. data/lib/octocatalog-diff/catalog-diff/cli/options/compare_file_text.rb +15 -0
  55. data/lib/octocatalog-diff/catalog-diff/cli/options/debug.rb +12 -0
  56. data/lib/octocatalog-diff/catalog-diff/cli/options/display_datatype_changes.rb +16 -0
  57. data/lib/octocatalog-diff/catalog-diff/cli/options/display_detail_add.rb +12 -0
  58. data/lib/octocatalog-diff/catalog-diff/cli/options/display_source_file_line.rb +12 -0
  59. data/lib/octocatalog-diff/catalog-diff/cli/options/enc.rb +31 -0
  60. data/lib/octocatalog-diff/catalog-diff/cli/options/existing_catalogs.rb +25 -0
  61. data/lib/octocatalog-diff/catalog-diff/cli/options/fact_file.rb +23 -0
  62. data/lib/octocatalog-diff/catalog-diff/cli/options/fact_override.rb +19 -0
  63. data/lib/octocatalog-diff/catalog-diff/cli/options/facts_terminus.rb +16 -0
  64. data/lib/octocatalog-diff/catalog-diff/cli/options/from_puppetdb.rb +13 -0
  65. data/lib/octocatalog-diff/catalog-diff/cli/options/header.rb +24 -0
  66. data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_config.rb +18 -0
  67. data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_path_strip.rb +12 -0
  68. data/lib/octocatalog-diff/catalog-diff/cli/options/hostname.rb +13 -0
  69. data/lib/octocatalog-diff/catalog-diff/cli/options/ignore.rb +24 -0
  70. data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_attr.rb +16 -0
  71. data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_tags.rb +23 -0
  72. data/lib/octocatalog-diff/catalog-diff/cli/options/include_tags.rb +12 -0
  73. data/lib/octocatalog-diff/catalog-diff/cli/options/master_cache_branch.rb +12 -0
  74. data/lib/octocatalog-diff/catalog-diff/cli/options/output_file.rb +15 -0
  75. data/lib/octocatalog-diff/catalog-diff/cli/options/output_format.rb +15 -0
  76. data/lib/octocatalog-diff/catalog-diff/cli/options/parallel.rb +12 -0
  77. data/lib/octocatalog-diff/catalog-diff/cli/options/parser.rb +48 -0
  78. data/lib/octocatalog-diff/catalog-diff/cli/options/pass_env_vars.rb +19 -0
  79. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_ca.rb +15 -0
  80. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_cert.rb +14 -0
  81. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_key.rb +14 -0
  82. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token.rb +15 -0
  83. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token_file.rb +17 -0
  84. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_url.rb +19 -0
  85. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_binary.rb +16 -0
  86. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master.rb +16 -0
  87. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_api_version.rb +20 -0
  88. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_ca.rb +19 -0
  89. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_cert.rb +19 -0
  90. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_key.rb +19 -0
  91. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_ca.rb +15 -0
  92. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_cert.rb +14 -0
  93. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_key.rb +14 -0
  94. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password.rb +14 -0
  95. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password_file.rb +13 -0
  96. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_url.rb +18 -0
  97. data/lib/octocatalog-diff/catalog-diff/cli/options/quiet.rb +12 -0
  98. data/lib/octocatalog-diff/catalog-diff/cli/options/retry_failed_catalog.rb +13 -0
  99. data/lib/octocatalog-diff/catalog-diff/cli/options/safe_to_delete_cached_master_dir.rb +15 -0
  100. data/lib/octocatalog-diff/catalog-diff/cli/options/storeconfigs.rb +12 -0
  101. data/lib/octocatalog-diff/catalog-diff/cli/options/suppress_absent_file_details.rb +14 -0
  102. data/lib/octocatalog-diff/catalog-diff/cli/options/to_from_branch.rb +16 -0
  103. data/lib/octocatalog-diff/catalog-diff/cli/printer.rb +52 -0
  104. data/lib/octocatalog-diff/catalog-diff/differ.rb +615 -0
  105. data/lib/octocatalog-diff/catalog-diff/display.rb +125 -0
  106. data/lib/octocatalog-diff/catalog-diff/display/json.rb +25 -0
  107. data/lib/octocatalog-diff/catalog-diff/display/text.rb +452 -0
  108. data/lib/octocatalog-diff/catalog-util/bootstrap.rb +145 -0
  109. data/lib/octocatalog-diff/catalog-util/builddir.rb +289 -0
  110. data/lib/octocatalog-diff/catalog-util/cached_master_directory.rb +169 -0
  111. data/lib/octocatalog-diff/catalog-util/command.rb +96 -0
  112. data/lib/octocatalog-diff/catalog-util/enc.rb +77 -0
  113. data/lib/octocatalog-diff/catalog-util/enc/noop.rb +22 -0
  114. data/lib/octocatalog-diff/catalog-util/enc/pe.rb +99 -0
  115. data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb +61 -0
  116. data/lib/octocatalog-diff/catalog-util/enc/script.rb +88 -0
  117. data/lib/octocatalog-diff/catalog-util/facts.rb +89 -0
  118. data/lib/octocatalog-diff/catalog-util/fileresources.rb +83 -0
  119. data/lib/octocatalog-diff/catalog-util/git.rb +65 -0
  120. data/lib/octocatalog-diff/catalog.rb +209 -0
  121. data/lib/octocatalog-diff/catalog/computed.rb +205 -0
  122. data/lib/octocatalog-diff/catalog/json.rb +30 -0
  123. data/lib/octocatalog-diff/catalog/noop.rb +19 -0
  124. data/lib/octocatalog-diff/catalog/puppetdb.rb +82 -0
  125. data/lib/octocatalog-diff/catalog/puppetmaster.rb +121 -0
  126. data/lib/octocatalog-diff/external/pson/LICENSE +17 -0
  127. data/lib/octocatalog-diff/external/pson/README.md +20 -0
  128. data/lib/octocatalog-diff/external/pson/common.rb +370 -0
  129. data/lib/octocatalog-diff/external/pson/pure.rb +15 -0
  130. data/lib/octocatalog-diff/external/pson/pure/generator.rb +395 -0
  131. data/lib/octocatalog-diff/external/pson/pure/parser.rb +307 -0
  132. data/lib/octocatalog-diff/external/pson/version.rb +8 -0
  133. data/lib/octocatalog-diff/facts.rb +125 -0
  134. data/lib/octocatalog-diff/facts/json.rb +20 -0
  135. data/lib/octocatalog-diff/facts/puppetdb.rb +59 -0
  136. data/lib/octocatalog-diff/facts/yaml.rb +29 -0
  137. data/lib/octocatalog-diff/puppetdb.rb +163 -0
  138. data/lib/octocatalog-diff/util/colored.rb +20 -0
  139. data/lib/octocatalog-diff/util/httparty.rb +158 -0
  140. data/lib/octocatalog-diff/util/parallel.rb +170 -0
  141. data/lib/octocatalog-diff/util/puppetversion.rb +24 -0
  142. data/lib/octocatalog-diff/version.rb +7 -0
  143. metadata +386 -0
@@ -0,0 +1,89 @@
1
+ require_relative '../facts'
2
+
3
+ module OctocatalogDiff
4
+ module CatalogUtil
5
+ # Helper class to construct a fact object based on options provided by
6
+ # cli/options. Supports a direct fact file, looking up a YAML file based on
7
+ # node name within Puppet fact directories, or retrieving from PuppetDB.
8
+ class Facts
9
+ # Constructor
10
+ # @param options [Hash] Options from cli/options
11
+ # @param logger [Logger] Logger object for debug messages (optional)
12
+ def initialize(options, logger = nil)
13
+ raise ArgumentError, "Argument to constructor must be Hash not #{options.class}" unless options.is_a?(Hash)
14
+ @options = options.dup
15
+ @logger = logger
16
+
17
+ # Environment variable recognition
18
+ @options[:puppetdb_url] ||= ENV['PUPPETDB_URL'] if ENV['PUPPETDB_URL']
19
+ @options[:puppet_fact_dir] ||= ENV['PUPPET_FACT_DIR'] if ENV['PUPPET_FACT_DIR']
20
+ end
21
+
22
+ # Compute facts if needed and then return them
23
+ # @return [Hash] Facts
24
+ def facts
25
+ @facts ||= compute_facts
26
+ end
27
+
28
+ private
29
+
30
+ # Retrieve facts from a YAML file in the puppet facts directory
31
+ # @param filename [String] Full path to file to read in
32
+ # @return [OctocatalogDiff::Facts] Facts object
33
+ def facts_from_file(filename)
34
+ @logger.debug("Retrieving facts from #{filename}") unless @logger.nil?
35
+ opts = {
36
+ node: @options[:node],
37
+ backend: :yaml,
38
+ fact_file_string: File.read(filename)
39
+ }
40
+ OctocatalogDiff::Facts.new(opts)
41
+ end
42
+
43
+ # Retrieve facts from PuppetDB. Either options[:puppetdb_url] or ENV['PUPPETDB_URL']
44
+ # needs to be set for this to work. Node name must also be set in options.
45
+ # @return [OctocatalogDiff::Facts] Facts object
46
+ def facts_from_puppetdb
47
+ @logger.debug('Retrieving facts from PuppetDB') unless @logger.nil?
48
+ OctocatalogDiff::Facts.new(@options.merge(backend: :puppetdb, retry: 2))
49
+ end
50
+
51
+ # Error message when the node is needed but not defined
52
+ # :nocov:
53
+ def error_node_not_provided
54
+ message = 'Unable to determine facts. You must either supply "--fact-file FILENAME"' \
55
+ ' or a node name "-n NODENAME" to look up a set of node facts in a fact' \
56
+ ' directory or in PuppetDB.'
57
+ raise ArgumentError, message
58
+ end
59
+ # :nocov:
60
+
61
+ # Does the actual computation/lookup of facts. Seeks to return a OctocatalogDiff::Facts
62
+ # object. Raises error if no fact sources are found.
63
+ # @return [OctocatalogDiff::Facts] Facts object
64
+ def compute_facts
65
+ if @options.key?(:facts) && @options[:facts].is_a?(OctocatalogDiff::Facts)
66
+ return @options[:facts]
67
+ end
68
+
69
+ if @options.key?(:fact_file)
70
+ raise Errno::ENOENT, 'Specified fact file does not exist' unless File.file?(@options[:fact_file])
71
+ return facts_from_file(@options[:fact_file])
72
+ end
73
+
74
+ error_node_not_provided if @options[:node].nil?
75
+
76
+ if @options[:puppet_fact_dir] && File.directory?(@options[:puppet_fact_dir])
77
+ filename = File.join(@options[:puppet_fact_dir], @options[:node] + '.yaml')
78
+ return facts_from_file(filename) if File.file?(filename)
79
+ end
80
+
81
+ return facts_from_puppetdb if @options[:puppetdb_url]
82
+
83
+ message = 'Unable to compute facts for node. Please use "--fact-file FILENAME" option' \
84
+ ' or set one of these environment variables: PUPPET_FACT_DIR or PUPPETDB_URL.'
85
+ raise ArgumentError, message
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,83 @@
1
+ require 'digest'
2
+
3
+ module OctocatalogDiff
4
+ module CatalogUtil
5
+ # Used to convert file resources such as:
6
+ # file { 'something': source => 'puppet:///modules/xxx/yyy'}
7
+ # to:
8
+ # file { 'something': content => $( cat modules/xxx/files/yyy )}
9
+ # This allows the displayed diff to show differences in static files.
10
+ class FileResources
11
+ # Public method: Convert file resources to text. See the description of the class
12
+ # just above for details.
13
+ # @param obj [OctocatalogDiff::Catalog] Catalog object (will be modified)
14
+ def self.convert_file_resources(obj)
15
+ return unless obj.valid? && obj.compilation_dir.is_a?(String) && !obj.compilation_dir.empty?
16
+ _convert_file_resources(obj.resources, obj.compilation_dir)
17
+ begin
18
+ obj.catalog_json = ::JSON.generate(obj.catalog)
19
+ rescue ::JSON::GeneratorError => exc
20
+ obj.error_message = "Failed to generate JSON: #{exc.message}"
21
+ end
22
+ end
23
+
24
+ # Internal method: Static method to convert file resources. The compilation directory is
25
+ # required, or else this is a no-op. The passed-in array of resources is modified by this method.
26
+ # @param resources [Array<Hash>] Array of catalog resources
27
+ # @param compilation_dir [String] Compilation directory (so files can be looked up)
28
+ def self._convert_file_resources(resources, compilation_dir)
29
+ # Calculate compilation directory. There is not explicit error checking here because
30
+ # there is on-demand, explicit error checking for each file within the modification loop.
31
+ return unless compilation_dir.is_a?(String) && compilation_dir != ''
32
+
33
+ # Making sure that compilation_dir/environments/production/modules exists (and by inference,
34
+ # that compilation_dir/environments/production is pointing at the right place). Otherwise, try to find
35
+ # compilation_dir/modules. If neither of those exist, this code can't run.
36
+ env_dir = File.join(compilation_dir, 'environments', 'production')
37
+ unless File.directory?(File.join(env_dir, 'modules'))
38
+ return unless File.directory?(File.join(compilation_dir, 'modules'))
39
+ env_dir = compilation_dir
40
+ end
41
+
42
+ # Modify the resources
43
+ resources.map! do |resource|
44
+ if resource_convertible?(resource)
45
+ # Parse the 'source' parameter into a file on disk
46
+ src = resource['parameters']['source']
47
+ raise "Bad parameter source #{src}" unless src =~ %r{^puppet:///modules/([^/]+)/(.+)}
48
+ path = File.join(env_dir, 'modules', Regexp.last_match(1), 'files', Regexp.last_match(2))
49
+
50
+ if File.file?(path)
51
+ # If the file is found, read its content. If the content is all ASCII, substitute it into
52
+ # the 'content' parameter for easier comparison. If not, instead populate the md5sum.
53
+ # Delete the 'source' attribute as well.
54
+ content = File.read(path)
55
+ is_ascii = content.force_encoding('UTF-8').ascii_only?
56
+ resource['parameters']['content'] = is_ascii ? content : '{md5}' + Digest::MD5.hexdigest(content)
57
+ resource['parameters'].delete('source')
58
+ elsif File.exist?(path)
59
+ # We are not handling recursive file installs from a directory or anything else.
60
+ # However, the fact that we found *something* at this location indicates that the catalog
61
+ # is probably correct. Hence, the very general .exist? check.
62
+ else
63
+ raise Errno::ENOENT, "Unable to find '#{src}' at #{path}!"
64
+ end
65
+ end
66
+ resource
67
+ end
68
+ end
69
+
70
+ # Internal method: Determine if a resource is convertible. It is convertible if it
71
+ # is a file resource with no declared 'content' and with a declared and parseable 'source'.
72
+ # @param resource [Hash] Resource to check
73
+ # @return [Boolean] True of resource is convertible, false if not
74
+ def self.resource_convertible?(resource)
75
+ return true if resource['type'] == 'File' && \
76
+ resource['parameters'].key?('source') && \
77
+ !resource['parameters'].key?('content') && \
78
+ resource['parameters']['source'] =~ %r{^puppet:///modules/([^/]+)/(.+)}
79
+ false
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,65 @@
1
+ require 'fileutils'
2
+ require 'open3'
3
+ require 'rugged'
4
+ require 'shellwords'
5
+ require 'tempfile'
6
+
7
+ module OctocatalogDiff
8
+ module CatalogUtil
9
+ # Class to perform a git checkout (via 'git archive') of a branch from the base git
10
+ # directory into another targeted directory.
11
+ class Git
12
+ # Trapped errors
13
+ class GitCheckoutError < RuntimeError
14
+ end
15
+
16
+ # Check out a branch via 'git archive' from one directory into another.
17
+ # @param options [Hash] Options hash:
18
+ # - :branch => Branch name to check out
19
+ # - :path => Where to check out to (must exist as a directory)
20
+ # - :basedir => Where to check out from (must exist as a directory)
21
+ # - :logger => Logger object
22
+ def self.check_out_git_archive(options = {})
23
+ branch = options.fetch(:branch)
24
+ path = options.fetch(:path)
25
+ dir = options.fetch(:basedir)
26
+ logger = options.fetch(:logger)
27
+
28
+ # Validate parameters
29
+ raise GitCheckoutError, "Source directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
30
+ raise GitCheckoutError, "Target directory #{path.inspect} does not exist" if dir.nil? || !File.directory?(path)
31
+
32
+ # To get the options working correctly (-o pipefail in particular) this needs to run under
33
+ # bash. It's just creating a script, rather than figuring out all the shell escapes...
34
+ begin
35
+ tmp_script = Tempfile.new(['git-checkout', '.sh'])
36
+ tmp_script.write "#!/bin/bash\n"
37
+ tmp_script.write "set -euf -o pipefail\n"
38
+ tmp_script.write "git archive --format=tar #{Shellwords.escape(branch)} | \\\n"
39
+ tmp_script.write " ( cd #{Shellwords.escape(path)} && tar -xf - )\n"
40
+ tmp_script.close
41
+ FileUtils.chmod 0o755, tmp_script.path
42
+
43
+ logger.debug("Begin git archive #{dir}:#{branch} -> #{path}")
44
+ output, status = Open3.capture2e(tmp_script.path, chdir: dir)
45
+ raise GitCheckoutError, "Git archive #{branch}->#{path} failed: #{output}" unless status.exitstatus.zero?
46
+ logger.debug("Success git archive #{dir}:#{branch}")
47
+ ensure
48
+ FileUtils.rm_f tmp_script.path if File.exist?(tmp_script.path)
49
+ end
50
+ end
51
+
52
+ # Determine the SHA of origin/master (or any other branch really) in the git repo
53
+ # @param options [Hash] Options hash:
54
+ # - :branch => Branch name to determine SHA of
55
+ # - :basedir => Where to check out from (must exist as a directory)
56
+ def self.branch_sha(options = {})
57
+ branch = options.fetch(:branch)
58
+ dir = options.fetch(:basedir)
59
+ raise GitCheckoutError, "Git directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
60
+ repo = Rugged::Repository.new(dir)
61
+ repo.branches[branch].target_id
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,209 @@
1
+ require 'json'
2
+ require 'stringio'
3
+
4
+ require_relative 'catalog/computed'
5
+ require_relative 'catalog/json'
6
+ require_relative 'catalog/noop'
7
+ require_relative 'catalog/puppetdb'
8
+ require_relative 'catalog/puppetmaster'
9
+ require_relative 'catalog-util/fileresources'
10
+
11
+ module OctocatalogDiff
12
+ # This class represents a catalog. Generation of the catalog is handled via one of the
13
+ # supported backends listed above as 'require_relative'. Usually, the 'computed' backend
14
+ # will build the catalog from the Puppet command.
15
+ class Catalog
16
+ # Readable
17
+ attr_reader :built, :catalog, :catalog_json
18
+
19
+ # Error classes that we can throw
20
+ class PuppetVersionError < RuntimeError; end
21
+ class CatalogError < RuntimeError; end
22
+
23
+ # Constructor
24
+ # @param :backend [Symbol] If set, this will force a backend
25
+ # @param :json [String] JSON catalog content (will avoid running Puppet to compile catalog)
26
+ # @param :puppetdb [Object] If set, pull the catalog from PuppetDB rather than building
27
+ # @param :node [String] Name of node whose catalog is being built
28
+ # @param :fact_file [String] OPTIONAL: Path to fact file (if not provided, look up in PuppetDB)
29
+ # @param :hiera_config [String] OPTIONAL: Path to hiera config file (munge temp. copy if not provided)
30
+ # @param :basedir [String] OPTIONAL: Base directory for catalog (default base directory of this checkout)
31
+ # @param :pass_env_vars [Array<String>] OPTIONAL: Additional environment vars to pass
32
+ # @param :convert_file_resources [Boolean] OPTIONAL: Convert file resource source to content
33
+ # @param :storeconfigs [Boolean] OPTIONAL: Pass the '-s' flag, for puppetdb (storeconfigs) integration
34
+ def initialize(options = {})
35
+ @options = options
36
+
37
+ # Call appropriate backend for catalog generation
38
+ @catalog_obj = backend(options)
39
+
40
+ # The catalog is not built yet, except if the backend has no build method
41
+ @built = false
42
+ build unless @catalog_obj.respond_to?(:build)
43
+
44
+ # The compilation directory can be overridden, e.g. when testing
45
+ @override_compilation_dir = nil
46
+ end
47
+
48
+ # Build catalog - this method needs to be called to build the catalog. It is separate due to
49
+ # the serialization of the logger object -- the parallel gem cannot serialize/deserialize a logger
50
+ # object so it cannot be part of any object that is passed around.
51
+ # @param logger [Logger] Logger object, initialized to a default throwaway value
52
+ def build(logger = Logger.new(StringIO.new))
53
+ # Only build once
54
+ return if @built
55
+ @built = true
56
+
57
+ # Call catalog's build method.
58
+ if @catalog_obj.respond_to?(:build)
59
+ logger.debug "Calling build for object #{@catalog_obj.class}"
60
+ @catalog_obj.build(logger)
61
+ end
62
+
63
+ # These methods must exist in all backends
64
+ @catalog = @catalog_obj.catalog
65
+ @catalog_json = @catalog_obj.catalog_json
66
+ @error_message = @catalog_obj.error_message
67
+
68
+ # The resource hash is computed the first time it's needed. For now initialize it as nil.
69
+ @resource_hash = nil
70
+
71
+ # Perform post-generation processing of the catalog
72
+ return unless valid?
73
+ unless @catalog_obj.respond_to?(:convert_file_resources) && @catalog_obj.convert_file_resources == false
74
+ OctocatalogDiff::CatalogUtil::FileResources.convert_file_resources(self) if @options.fetch(:compare_file_text, false)
75
+ end
76
+ end
77
+
78
+ # For logging we may wish to know the backend being used
79
+ # @return [String] Class of backend used
80
+ def builder
81
+ @catalog_obj.class.to_s
82
+ end
83
+
84
+ # Set the catalog JSON
85
+ # @param str [String] Catalog JSON
86
+ def catalog_json=(str)
87
+ @catalog_json = str
88
+ @resource_hash = nil
89
+ end
90
+
91
+ # This retrieves the compilation directory from the catalog, or otherwise the passed-in directory.
92
+ # @return [String] Compilation directory
93
+ def compilation_dir
94
+ return @override_compilation_dir if @override_compilation_dir
95
+ @catalog_obj.respond_to?(:compilation_dir) ? @catalog_obj.compilation_dir : @options[:basedir]
96
+ end
97
+
98
+ # The compilation directory can be overridden, e.g. during testing.
99
+ # @param dir [String] Compilation directory
100
+ def compilation_dir=(dir)
101
+ @override_compilation_dir = dir
102
+ end
103
+
104
+ # Determine whether the underlying catalog object supports :compare_file_text
105
+ # @return [Boolean] Whether underlying catalog object supports :compare_file_text
106
+ def convert_file_resources
107
+ return true unless @catalog_obj.respond_to?(:convert_file_resources)
108
+ @catalog_obj.convert_file_resources
109
+ end
110
+
111
+ # Retrieve the error message.
112
+ # @return [String] Error message (maximum 20,000 characters) - nil if no error.
113
+ def error_message
114
+ return nil if @error_message.nil? || !@error_message.is_a?(String)
115
+ @error_message[0, 20_000]
116
+ end
117
+
118
+ # Allow setting the error message. If the error message is set to a string, the catalog
119
+ # and catalog JSON are set to nil.
120
+ # @param error [String] Error message
121
+ def error_message=(error)
122
+ raise ArgumentError, 'Error message must be a string' unless error.is_a?(String)
123
+ @error_message = error
124
+ @catalog = nil
125
+ @catalog_json = nil
126
+ @resource_hash = nil
127
+ end
128
+
129
+ # This retrieves the version of Puppet used to compile a catalog. If the underlying catalog was not
130
+ # compiled by running Puppet (e.g., it was read in from JSON or puppetdb), then this returns the
131
+ # puppet version optionally passed in to the constructor. This can also be nil.
132
+ # @return [String] Puppet version
133
+ def puppet_version
134
+ @catalog_obj.respond_to?(:puppet_version) ? @catalog_obj.puppet_version : @options[:puppet_version]
135
+ end
136
+
137
+ # This allows retrieving a resource by type and title. This is intended for use when a O(1) lookup is required.
138
+ # @param :type [String] Type of resource
139
+ # @param :title [String] Title of resource
140
+ # @return [Hash] Resource item
141
+ def resource(opts = {})
142
+ raise ArgumentError, ':type and :title are required' unless opts[:type] && opts[:title]
143
+ build_resource_hash if @resource_hash.nil?
144
+ return nil unless @resource_hash[opts[:type]].is_a?(Hash)
145
+ @resource_hash[opts[:type]][opts[:title]]
146
+ end
147
+
148
+ # This is a compatibility layer for the resources, which are in a different place in Puppet 3.x and Puppet 4.x
149
+ # @return [Array] Resource array
150
+ def resources
151
+ raise CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
152
+ raise CatalogError, error_message unless valid?
153
+ return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
154
+ return @catalog['resources'] if @catalog['resources'].is_a?(Array)
155
+ # This is a bug condition
156
+ # :nocov:
157
+ raise "BUG: catalog has no data::resources or ::resources array. Please report this. #{@catalog.inspect}"
158
+ # :nocov
159
+ end
160
+
161
+ # This retrieves the number of retries necessary to compile the catalog. If the underlying catalog
162
+ # generation backend does not support retries, nil is returned.
163
+ # @return [Integer] Retry count
164
+ def retries
165
+ @retries = @catalog_obj.respond_to?(:retries) ? @catalog_obj.retries : nil
166
+ end
167
+
168
+ # Determine if the catalog build was successful.
169
+ # @return [Boolean] Whether the catalog is valid
170
+ def valid?
171
+ !@catalog.nil?
172
+ end
173
+
174
+ private
175
+
176
+ # Private method: Choose backend based on passed-in options
177
+ # @param options [Hash] Options passed into constructor
178
+ # @return [Object] OctocatalogDiff::Catalog::<whatever> object
179
+ def backend(options)
180
+ # Hard-coded backend
181
+ if options[:backend]
182
+ return OctocatalogDiff::Catalog::JSON.new(options) if options[:backend] == :json
183
+ return OctocatalogDiff::Catalog::PuppetDB.new(options) if options[:backend] == :puppetdb
184
+ return OctocatalogDiff::Catalog::PuppetMaster.new(options) if options[:backend] == :puppetmaster
185
+ return OctocatalogDiff::Catalog::Computed.new(options) if options[:backend] == :computed
186
+ return OctocatalogDiff::Catalog::Noop.new(options) if options[:backend] == :noop
187
+ raise ArgumentError, "Unknown backend :#{options[:backend]}"
188
+ end
189
+
190
+ # Determine backend based on arguments
191
+ return OctocatalogDiff::Catalog::JSON.new(options) if options[:json]
192
+ return OctocatalogDiff::Catalog::PuppetDB.new(options) if options[:puppetdb]
193
+ return OctocatalogDiff::Catalog::PuppetMaster.new(options) if options[:puppet_master]
194
+
195
+ # Default is to build catalog ourselves
196
+ OctocatalogDiff::Catalog::Computed.new(options)
197
+ end
198
+
199
+ # Private method: Build the resource hash to be used used for O(1) lookups by type and title.
200
+ # This method is called the first time the resource hash is accessed.
201
+ def build_resource_hash
202
+ @resource_hash = {}
203
+ resources.each do |resource|
204
+ @resource_hash[resource['type']] ||= {}
205
+ @resource_hash[resource['type']][resource['title']] = resource
206
+ end
207
+ end
208
+ end
209
+ end
@@ -0,0 +1,205 @@
1
+ require 'fileutils'
2
+ require 'json'
3
+ require 'open3'
4
+ require 'stringio'
5
+
6
+ require_relative '../catalog-util/bootstrap'
7
+ require_relative '../catalog-util/builddir'
8
+ require_relative '../catalog-util/command'
9
+ require_relative '../util/puppetversion'
10
+ require_relative '../catalog-util/facts'
11
+
12
+ module OctocatalogDiff
13
+ class Catalog
14
+ # Represents a Puppet catalog that is computed (via `puppet master --compile ...`)
15
+ # By instantiating this class, the catalog is computed.
16
+ class Computed
17
+ attr_reader :node, :error_message, :catalog, :catalog_json, :retries
18
+
19
+ # Constructor
20
+ # @param :node [String] REQUIRED: Node name
21
+ # @param :basedir [String] Directory in which to compile the catalog
22
+ # @param :pass_env_vars [Array<String>] Environment variables to pass when compiling catalog
23
+ # @param :retry_failed_catalog [Fixnum] Number of retries if a catalog compilation fails
24
+ # @param :tag [String] For display purposes, the catalog being compiled
25
+ # @param :puppet_binary [String] Full path to Puppet
26
+ # @param :puppet_version [String] Puppet version (optional; if not supplied, it is calculated)
27
+ # @param :puppet_command [String] Full command to run Puppet (optional; if not supplied, it is calculated)
28
+ def initialize(options)
29
+ raise ArgumentError, 'Usage: OctocatalogDiff::Catalog::Computed.initialize(options_hash)' unless options.is_a?(Hash)
30
+ raise ArgumentError, 'Node name must be passed to OctocatalogDiff::Catalog::Computed' unless options[:node].is_a?(String)
31
+
32
+ # Standard readable variables
33
+ @node = options[:node]
34
+ @error_message = nil
35
+ @catalog = nil
36
+ @catalog_json = nil
37
+
38
+ # Additional class variables
39
+ @pass_env_vars = options.fetch(:pass_env_vars, [])
40
+ @retry_failed_catalog = options.fetch(:retry_failed_catalog, 0)
41
+ @tag = options.fetch(:tag, 'catalog')
42
+ @puppet_binary = options[:puppet_binary]
43
+ @puppet_version = options[:puppet_version]
44
+ @puppet_command = options[:puppet_command]
45
+ @retries = nil
46
+ @builddir = nil
47
+
48
+ # Pass through the input for other access
49
+ @opts = options
50
+ raise ArgumentError, 'Branch is undefined' unless @opts[:branch]
51
+ end
52
+
53
+ # Actually build the catalog (populate @error_message, @catalog, @catalog_json)
54
+ def build(logger = Logger.new(StringIO.new))
55
+ facts_obj = OctocatalogDiff::CatalogUtil::Facts.new(@opts, logger)
56
+ logger.debug "Start retrieving facts for #{@node} from #{self.class}"
57
+ @opts[:facts] = facts_obj.facts
58
+ logger.debug "Success retrieving facts for #{@node} from #{self.class}"
59
+ build_catalog(logger)
60
+ end
61
+
62
+ # Get the Puppet version
63
+ # @return [String] Puppet version
64
+ def puppet_version
65
+ raise ArgumentError, '"puppet_binary" was not passed to OctocatalogDiff::Catalog::Computed' unless @puppet_binary
66
+ @puppet_version ||= OctocatalogDiff::Util::PuppetVersion.puppet_version(@puppet_binary)
67
+ end
68
+
69
+ # Compilation directory
70
+ # @return [String] Compilation directory
71
+ def compilation_dir
72
+ raise 'Catalog was not built' if @builddir.nil?
73
+ @builddir.tempdir
74
+ end
75
+
76
+ private
77
+
78
+ # Private method: Clean up a checkout directory, if it exists
79
+ def cleanup_checkout_dir(checkout_dir, logger)
80
+ return unless File.directory?(checkout_dir)
81
+ logger.debug("Cleaning up temporary directory #{checkout_dir}")
82
+ FileUtils.remove_entry_secure checkout_dir
83
+ end
84
+
85
+ # Private method: Bootstrap a directory
86
+ def bootstrap(logger)
87
+ return if @builddir
88
+
89
+ # Fill options for creating and populating the temporary directory
90
+ tmphash = @opts.dup
91
+
92
+ # Bootstrap directory if needed
93
+ if !@opts[:bootstrapped_dir].nil?
94
+ raise Errno::ENOENT, "Invalid dir #{@opts[:bootstrapped_dir]}" unless File.directory?(@opts[:bootstrapped_dir])
95
+ tmphash[:basedir] = @opts[:bootstrapped_dir]
96
+ elsif @opts[:branch] == '.'
97
+ tmphash[:basedir] = @opts[:basedir]
98
+ else
99
+ checkout_dir = Dir.mktmpdir
100
+ at_exit { cleanup_checkout_dir(checkout_dir, logger) }
101
+ tmphash[:basedir] = checkout_dir
102
+ OctocatalogDiff::CatalogUtil::Bootstrap.bootstrap_directory(@opts.merge(path: checkout_dir), logger)
103
+ end
104
+
105
+ # Create and populate the temporary directory
106
+ @builddir ||= OctocatalogDiff::CatalogUtil::BuildDir.new(tmphash, logger)
107
+ end
108
+
109
+ # Private method: Build catalog by running Puppet
110
+ # @param logger [Logger] Logger object
111
+ def build_catalog(logger = nil)
112
+ return nil unless @catalog.nil? && @error_message.nil?
113
+ bootstrap(logger)
114
+ result = run_puppet(logger)
115
+ @retries = result[:retries]
116
+ if (result[:exitcode]).zero?
117
+ begin
118
+ @catalog = ::JSON.parse(result[:stdout])
119
+ @catalog_json = result[:stdout]
120
+ @error_message = nil
121
+ rescue ::JSON::ParserError => exc
122
+ @catalog = nil
123
+ @catalog_json = nil
124
+ @error_message = "Catalog has invalid JSON: #{exc.message}"
125
+ end
126
+ else
127
+ @error_message = result[:stderr]
128
+ @catalog = nil
129
+ @catalog_json = nil
130
+ end
131
+ end
132
+
133
+ # Get the command to compile the catalog
134
+ # @return [String] Puppet command line
135
+ def puppet_command(options = @opts)
136
+ return @puppet_command if @puppet_command
137
+ raise ArgumentError, '"puppet_binary" was not passed to OctocatalogDiff::Catalog::Computed' unless @puppet_binary
138
+ command_opts = options.merge(
139
+ node: @node,
140
+ compilation_dir: @builddir.tempdir,
141
+ parser: options.fetch(:parser, :default),
142
+ puppet_binary: @puppet_binary,
143
+ fact_file: @builddir.fact_file,
144
+ dir: @builddir.tempdir,
145
+ enc: @builddir.enc
146
+ )
147
+ command = OctocatalogDiff::CatalogUtil::Command.new(command_opts)
148
+ @puppet_command = command.puppet_command
149
+ end
150
+
151
+ # Private method: Actually execute puppet
152
+ # @return [Hash] { stdout, stderr, exitcode }
153
+ def exec_puppet
154
+ # This is the environment provided to the puppet command.
155
+ env = {
156
+ 'HOME' => ENV['HOME'],
157
+ 'PATH' => ENV['PATH'],
158
+ 'PWD' => @builddir.tempdir
159
+ }
160
+ @pass_env_vars.each { |var| env[var] ||= ENV[var] }
161
+ out, err, status = Open3.capture3(env, puppet_command, unsetenv_others: true, chdir: @builddir.tempdir)
162
+ {
163
+ stdout: out,
164
+ stderr: err,
165
+ exitcode: status.exitstatus
166
+ }
167
+ end
168
+
169
+ # Private method: Runs puppet on the command line to compile the catalog
170
+ # Exit code is 0 if catalog generation was successful, non-zero otherwise.
171
+ # @param logger [Logger] Logger object
172
+ # @return [Hash] { stdout: <catalog as JSON>, stderr: <error messages>, exitcode: <hopefully 0> }
173
+ def run_puppet(logger)
174
+ # Run 'cmd' with environment 'env' from directory 'dir'
175
+ # First line of a successful result needs to be stripped off. It will look like:
176
+ # Notice: Compiled catalog for xxx in environment production in 27.88 seconds
177
+ retval = {}
178
+ 0.upto(@retry_failed_catalog) do |retry_num|
179
+ @retries = retry_num
180
+ time_begin = Time.now
181
+ logger.debug("(#{@tag}) Try #{1 + retry_num} executing Puppet #{puppet_version}: #{puppet_command}")
182
+ result = exec_puppet
183
+
184
+ # Success
185
+ if (result[:exitcode]).zero?
186
+ logger.debug("(#{@tag}) Catalog succeeded on try #{1 + retry_num} in #{Time.now - time_begin} seconds")
187
+ first_brace = result[:stdout].index('{') || 0
188
+ retval = {
189
+ stdout: result[:stdout][first_brace..-1],
190
+ stderr: nil,
191
+ exitcode: 0,
192
+ retries: retry_num
193
+ }
194
+ break
195
+ end
196
+
197
+ # Failure
198
+ logger.debug("(#{@tag}) Catalog failed on try #{1 + retry_num} in #{Time.now - time_begin} seconds")
199
+ retval = result.merge(retries: retry_num)
200
+ end
201
+ retval
202
+ end
203
+ end
204
+ end
205
+ end