octocatalog-diff 0.5.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/.version +1 -0
  3. data/LICENSE +20 -0
  4. data/README.md +82 -0
  5. data/bin/octocatalog-diff +75 -0
  6. data/doc/advanced-bootstrap.md +33 -0
  7. data/doc/advanced-cache-dir.md +24 -0
  8. data/doc/advanced-catalog-only.md +37 -0
  9. data/doc/advanced-ci.md +13 -0
  10. data/doc/advanced-dynamic-ignores.md +123 -0
  11. data/doc/advanced-future-parser.md +11 -0
  12. data/doc/advanced-ignores.md +224 -0
  13. data/doc/advanced-output-formats.md +96 -0
  14. data/doc/advanced-output-hacks.md +45 -0
  15. data/doc/advanced-override-facts.md +67 -0
  16. data/doc/advanced-pe-enc.md +52 -0
  17. data/doc/advanced-puppet-master.md +50 -0
  18. data/doc/advanced-puppet-versions.md +9 -0
  19. data/doc/advanced-storeconfigs.md +72 -0
  20. data/doc/advanced-using-without-git.md +15 -0
  21. data/doc/advanced.md +43 -0
  22. data/doc/basic.md +70 -0
  23. data/doc/configuration-enc.md +69 -0
  24. data/doc/configuration-hiera.md +103 -0
  25. data/doc/configuration-puppetdb.md +49 -0
  26. data/doc/configuration.md +51 -0
  27. data/doc/dev/README.md +1 -0
  28. data/doc/dev/coverage.md +34 -0
  29. data/doc/dev/how-to-add-options.md +83 -0
  30. data/doc/dev/integration-tests.md +63 -0
  31. data/doc/dev/releasing.md +19 -0
  32. data/doc/installation.md +49 -0
  33. data/doc/limitations.md +34 -0
  34. data/doc/optionsref.md +947 -0
  35. data/doc/requirements.md +16 -0
  36. data/doc/roadmap.md +26 -0
  37. data/doc/similar.md +17 -0
  38. data/doc/troubleshooting.md +54 -0
  39. data/lib/octocatalog-diff.rb +12 -0
  40. data/lib/octocatalog-diff/bootstrap.rb +53 -0
  41. data/lib/octocatalog-diff/catalog-diff/cli.rb +205 -0
  42. data/lib/octocatalog-diff/catalog-diff/cli/catalogs.rb +240 -0
  43. data/lib/octocatalog-diff/catalog-diff/cli/diffs.rb +145 -0
  44. data/lib/octocatalog-diff/catalog-diff/cli/helpers/fact_override.rb +99 -0
  45. data/lib/octocatalog-diff/catalog-diff/cli/options.rb +173 -0
  46. data/lib/octocatalog-diff/catalog-diff/cli/options/basedir.rb +14 -0
  47. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_environment.rb +18 -0
  48. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_script.rb +14 -0
  49. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_then_exit.rb +12 -0
  50. data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrapped_dirs.rb +18 -0
  51. data/lib/octocatalog-diff/catalog-diff/cli/options/cached_master_dir.rb +21 -0
  52. data/lib/octocatalog-diff/catalog-diff/cli/options/catalog_only.rb +14 -0
  53. data/lib/octocatalog-diff/catalog-diff/cli/options/color.rb +13 -0
  54. data/lib/octocatalog-diff/catalog-diff/cli/options/compare_file_text.rb +15 -0
  55. data/lib/octocatalog-diff/catalog-diff/cli/options/debug.rb +12 -0
  56. data/lib/octocatalog-diff/catalog-diff/cli/options/display_datatype_changes.rb +16 -0
  57. data/lib/octocatalog-diff/catalog-diff/cli/options/display_detail_add.rb +12 -0
  58. data/lib/octocatalog-diff/catalog-diff/cli/options/display_source_file_line.rb +12 -0
  59. data/lib/octocatalog-diff/catalog-diff/cli/options/enc.rb +31 -0
  60. data/lib/octocatalog-diff/catalog-diff/cli/options/existing_catalogs.rb +25 -0
  61. data/lib/octocatalog-diff/catalog-diff/cli/options/fact_file.rb +23 -0
  62. data/lib/octocatalog-diff/catalog-diff/cli/options/fact_override.rb +19 -0
  63. data/lib/octocatalog-diff/catalog-diff/cli/options/facts_terminus.rb +16 -0
  64. data/lib/octocatalog-diff/catalog-diff/cli/options/from_puppetdb.rb +13 -0
  65. data/lib/octocatalog-diff/catalog-diff/cli/options/header.rb +24 -0
  66. data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_config.rb +18 -0
  67. data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_path_strip.rb +12 -0
  68. data/lib/octocatalog-diff/catalog-diff/cli/options/hostname.rb +13 -0
  69. data/lib/octocatalog-diff/catalog-diff/cli/options/ignore.rb +24 -0
  70. data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_attr.rb +16 -0
  71. data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_tags.rb +23 -0
  72. data/lib/octocatalog-diff/catalog-diff/cli/options/include_tags.rb +12 -0
  73. data/lib/octocatalog-diff/catalog-diff/cli/options/master_cache_branch.rb +12 -0
  74. data/lib/octocatalog-diff/catalog-diff/cli/options/output_file.rb +15 -0
  75. data/lib/octocatalog-diff/catalog-diff/cli/options/output_format.rb +15 -0
  76. data/lib/octocatalog-diff/catalog-diff/cli/options/parallel.rb +12 -0
  77. data/lib/octocatalog-diff/catalog-diff/cli/options/parser.rb +48 -0
  78. data/lib/octocatalog-diff/catalog-diff/cli/options/pass_env_vars.rb +19 -0
  79. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_ca.rb +15 -0
  80. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_cert.rb +14 -0
  81. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_key.rb +14 -0
  82. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token.rb +15 -0
  83. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token_file.rb +17 -0
  84. data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_url.rb +19 -0
  85. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_binary.rb +16 -0
  86. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master.rb +16 -0
  87. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_api_version.rb +20 -0
  88. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_ca.rb +19 -0
  89. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_cert.rb +19 -0
  90. data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_key.rb +19 -0
  91. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_ca.rb +15 -0
  92. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_cert.rb +14 -0
  93. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_key.rb +14 -0
  94. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password.rb +14 -0
  95. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password_file.rb +13 -0
  96. data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_url.rb +18 -0
  97. data/lib/octocatalog-diff/catalog-diff/cli/options/quiet.rb +12 -0
  98. data/lib/octocatalog-diff/catalog-diff/cli/options/retry_failed_catalog.rb +13 -0
  99. data/lib/octocatalog-diff/catalog-diff/cli/options/safe_to_delete_cached_master_dir.rb +15 -0
  100. data/lib/octocatalog-diff/catalog-diff/cli/options/storeconfigs.rb +12 -0
  101. data/lib/octocatalog-diff/catalog-diff/cli/options/suppress_absent_file_details.rb +14 -0
  102. data/lib/octocatalog-diff/catalog-diff/cli/options/to_from_branch.rb +16 -0
  103. data/lib/octocatalog-diff/catalog-diff/cli/printer.rb +52 -0
  104. data/lib/octocatalog-diff/catalog-diff/differ.rb +615 -0
  105. data/lib/octocatalog-diff/catalog-diff/display.rb +125 -0
  106. data/lib/octocatalog-diff/catalog-diff/display/json.rb +25 -0
  107. data/lib/octocatalog-diff/catalog-diff/display/text.rb +452 -0
  108. data/lib/octocatalog-diff/catalog-util/bootstrap.rb +145 -0
  109. data/lib/octocatalog-diff/catalog-util/builddir.rb +289 -0
  110. data/lib/octocatalog-diff/catalog-util/cached_master_directory.rb +169 -0
  111. data/lib/octocatalog-diff/catalog-util/command.rb +96 -0
  112. data/lib/octocatalog-diff/catalog-util/enc.rb +77 -0
  113. data/lib/octocatalog-diff/catalog-util/enc/noop.rb +22 -0
  114. data/lib/octocatalog-diff/catalog-util/enc/pe.rb +99 -0
  115. data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb +61 -0
  116. data/lib/octocatalog-diff/catalog-util/enc/script.rb +88 -0
  117. data/lib/octocatalog-diff/catalog-util/facts.rb +89 -0
  118. data/lib/octocatalog-diff/catalog-util/fileresources.rb +83 -0
  119. data/lib/octocatalog-diff/catalog-util/git.rb +65 -0
  120. data/lib/octocatalog-diff/catalog.rb +209 -0
  121. data/lib/octocatalog-diff/catalog/computed.rb +205 -0
  122. data/lib/octocatalog-diff/catalog/json.rb +30 -0
  123. data/lib/octocatalog-diff/catalog/noop.rb +19 -0
  124. data/lib/octocatalog-diff/catalog/puppetdb.rb +82 -0
  125. data/lib/octocatalog-diff/catalog/puppetmaster.rb +121 -0
  126. data/lib/octocatalog-diff/external/pson/LICENSE +17 -0
  127. data/lib/octocatalog-diff/external/pson/README.md +20 -0
  128. data/lib/octocatalog-diff/external/pson/common.rb +370 -0
  129. data/lib/octocatalog-diff/external/pson/pure.rb +15 -0
  130. data/lib/octocatalog-diff/external/pson/pure/generator.rb +395 -0
  131. data/lib/octocatalog-diff/external/pson/pure/parser.rb +307 -0
  132. data/lib/octocatalog-diff/external/pson/version.rb +8 -0
  133. data/lib/octocatalog-diff/facts.rb +125 -0
  134. data/lib/octocatalog-diff/facts/json.rb +20 -0
  135. data/lib/octocatalog-diff/facts/puppetdb.rb +59 -0
  136. data/lib/octocatalog-diff/facts/yaml.rb +29 -0
  137. data/lib/octocatalog-diff/puppetdb.rb +163 -0
  138. data/lib/octocatalog-diff/util/colored.rb +20 -0
  139. data/lib/octocatalog-diff/util/httparty.rb +158 -0
  140. data/lib/octocatalog-diff/util/parallel.rb +170 -0
  141. data/lib/octocatalog-diff/util/puppetversion.rb +24 -0
  142. data/lib/octocatalog-diff/version.rb +7 -0
  143. metadata +386 -0
@@ -0,0 +1,89 @@
1
+ require_relative '../facts'
2
+
3
+ module OctocatalogDiff
4
+ module CatalogUtil
5
+ # Helper class to construct a fact object based on options provided by
6
+ # cli/options. Supports a direct fact file, looking up a YAML file based on
7
+ # node name within Puppet fact directories, or retrieving from PuppetDB.
8
+ class Facts
9
+ # Constructor
10
+ # @param options [Hash] Options from cli/options
11
+ # @param logger [Logger] Logger object for debug messages (optional)
12
+ def initialize(options, logger = nil)
13
+ raise ArgumentError, "Argument to constructor must be Hash not #{options.class}" unless options.is_a?(Hash)
14
+ @options = options.dup
15
+ @logger = logger
16
+
17
+ # Environment variable recognition
18
+ @options[:puppetdb_url] ||= ENV['PUPPETDB_URL'] if ENV['PUPPETDB_URL']
19
+ @options[:puppet_fact_dir] ||= ENV['PUPPET_FACT_DIR'] if ENV['PUPPET_FACT_DIR']
20
+ end
21
+
22
+ # Compute facts if needed and then return them
23
+ # @return [Hash] Facts
24
+ def facts
25
+ @facts ||= compute_facts
26
+ end
27
+
28
+ private
29
+
30
+ # Retrieve facts from a YAML file in the puppet facts directory
31
+ # @param filename [String] Full path to file to read in
32
+ # @return [OctocatalogDiff::Facts] Facts object
33
+ def facts_from_file(filename)
34
+ @logger.debug("Retrieving facts from #{filename}") unless @logger.nil?
35
+ opts = {
36
+ node: @options[:node],
37
+ backend: :yaml,
38
+ fact_file_string: File.read(filename)
39
+ }
40
+ OctocatalogDiff::Facts.new(opts)
41
+ end
42
+
43
+ # Retrieve facts from PuppetDB. Either options[:puppetdb_url] or ENV['PUPPETDB_URL']
44
+ # needs to be set for this to work. Node name must also be set in options.
45
+ # @return [OctocatalogDiff::Facts] Facts object
46
+ def facts_from_puppetdb
47
+ @logger.debug('Retrieving facts from PuppetDB') unless @logger.nil?
48
+ OctocatalogDiff::Facts.new(@options.merge(backend: :puppetdb, retry: 2))
49
+ end
50
+
51
+ # Error message when the node is needed but not defined
52
+ # :nocov:
53
+ def error_node_not_provided
54
+ message = 'Unable to determine facts. You must either supply "--fact-file FILENAME"' \
55
+ ' or a node name "-n NODENAME" to look up a set of node facts in a fact' \
56
+ ' directory or in PuppetDB.'
57
+ raise ArgumentError, message
58
+ end
59
+ # :nocov:
60
+
61
+ # Does the actual computation/lookup of facts. Seeks to return a OctocatalogDiff::Facts
62
+ # object. Raises error if no fact sources are found.
63
+ # @return [OctocatalogDiff::Facts] Facts object
64
+ def compute_facts
65
+ if @options.key?(:facts) && @options[:facts].is_a?(OctocatalogDiff::Facts)
66
+ return @options[:facts]
67
+ end
68
+
69
+ if @options.key?(:fact_file)
70
+ raise Errno::ENOENT, 'Specified fact file does not exist' unless File.file?(@options[:fact_file])
71
+ return facts_from_file(@options[:fact_file])
72
+ end
73
+
74
+ error_node_not_provided if @options[:node].nil?
75
+
76
+ if @options[:puppet_fact_dir] && File.directory?(@options[:puppet_fact_dir])
77
+ filename = File.join(@options[:puppet_fact_dir], @options[:node] + '.yaml')
78
+ return facts_from_file(filename) if File.file?(filename)
79
+ end
80
+
81
+ return facts_from_puppetdb if @options[:puppetdb_url]
82
+
83
+ message = 'Unable to compute facts for node. Please use "--fact-file FILENAME" option' \
84
+ ' or set one of these environment variables: PUPPET_FACT_DIR or PUPPETDB_URL.'
85
+ raise ArgumentError, message
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,83 @@
1
+ require 'digest'
2
+
3
+ module OctocatalogDiff
4
+ module CatalogUtil
5
+ # Used to convert file resources such as:
6
+ # file { 'something': source => 'puppet:///modules/xxx/yyy'}
7
+ # to:
8
+ # file { 'something': content => $( cat modules/xxx/files/yyy )}
9
+ # This allows the displayed diff to show differences in static files.
10
+ class FileResources
11
+ # Public method: Convert file resources to text. See the description of the class
12
+ # just above for details.
13
+ # @param obj [OctocatalogDiff::Catalog] Catalog object (will be modified)
14
+ def self.convert_file_resources(obj)
15
+ return unless obj.valid? && obj.compilation_dir.is_a?(String) && !obj.compilation_dir.empty?
16
+ _convert_file_resources(obj.resources, obj.compilation_dir)
17
+ begin
18
+ obj.catalog_json = ::JSON.generate(obj.catalog)
19
+ rescue ::JSON::GeneratorError => exc
20
+ obj.error_message = "Failed to generate JSON: #{exc.message}"
21
+ end
22
+ end
23
+
24
+ # Internal method: Static method to convert file resources. The compilation directory is
25
+ # required, or else this is a no-op. The passed-in array of resources is modified by this method.
26
+ # @param resources [Array<Hash>] Array of catalog resources
27
+ # @param compilation_dir [String] Compilation directory (so files can be looked up)
28
+ def self._convert_file_resources(resources, compilation_dir)
29
+ # Calculate compilation directory. There is not explicit error checking here because
30
+ # there is on-demand, explicit error checking for each file within the modification loop.
31
+ return unless compilation_dir.is_a?(String) && compilation_dir != ''
32
+
33
+ # Making sure that compilation_dir/environments/production/modules exists (and by inference,
34
+ # that compilation_dir/environments/production is pointing at the right place). Otherwise, try to find
35
+ # compilation_dir/modules. If neither of those exist, this code can't run.
36
+ env_dir = File.join(compilation_dir, 'environments', 'production')
37
+ unless File.directory?(File.join(env_dir, 'modules'))
38
+ return unless File.directory?(File.join(compilation_dir, 'modules'))
39
+ env_dir = compilation_dir
40
+ end
41
+
42
+ # Modify the resources
43
+ resources.map! do |resource|
44
+ if resource_convertible?(resource)
45
+ # Parse the 'source' parameter into a file on disk
46
+ src = resource['parameters']['source']
47
+ raise "Bad parameter source #{src}" unless src =~ %r{^puppet:///modules/([^/]+)/(.+)}
48
+ path = File.join(env_dir, 'modules', Regexp.last_match(1), 'files', Regexp.last_match(2))
49
+
50
+ if File.file?(path)
51
+ # If the file is found, read its content. If the content is all ASCII, substitute it into
52
+ # the 'content' parameter for easier comparison. If not, instead populate the md5sum.
53
+ # Delete the 'source' attribute as well.
54
+ content = File.read(path)
55
+ is_ascii = content.force_encoding('UTF-8').ascii_only?
56
+ resource['parameters']['content'] = is_ascii ? content : '{md5}' + Digest::MD5.hexdigest(content)
57
+ resource['parameters'].delete('source')
58
+ elsif File.exist?(path)
59
+ # We are not handling recursive file installs from a directory or anything else.
60
+ # However, the fact that we found *something* at this location indicates that the catalog
61
+ # is probably correct. Hence, the very general .exist? check.
62
+ else
63
+ raise Errno::ENOENT, "Unable to find '#{src}' at #{path}!"
64
+ end
65
+ end
66
+ resource
67
+ end
68
+ end
69
+
70
+ # Internal method: Determine if a resource is convertible. It is convertible if it
71
+ # is a file resource with no declared 'content' and with a declared and parseable 'source'.
72
+ # @param resource [Hash] Resource to check
73
+ # @return [Boolean] True of resource is convertible, false if not
74
+ def self.resource_convertible?(resource)
75
+ return true if resource['type'] == 'File' && \
76
+ resource['parameters'].key?('source') && \
77
+ !resource['parameters'].key?('content') && \
78
+ resource['parameters']['source'] =~ %r{^puppet:///modules/([^/]+)/(.+)}
79
+ false
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,65 @@
1
+ require 'fileutils'
2
+ require 'open3'
3
+ require 'rugged'
4
+ require 'shellwords'
5
+ require 'tempfile'
6
+
7
+ module OctocatalogDiff
8
+ module CatalogUtil
9
+ # Class to perform a git checkout (via 'git archive') of a branch from the base git
10
+ # directory into another targeted directory.
11
+ class Git
12
+ # Trapped errors
13
+ class GitCheckoutError < RuntimeError
14
+ end
15
+
16
+ # Check out a branch via 'git archive' from one directory into another.
17
+ # @param options [Hash] Options hash:
18
+ # - :branch => Branch name to check out
19
+ # - :path => Where to check out to (must exist as a directory)
20
+ # - :basedir => Where to check out from (must exist as a directory)
21
+ # - :logger => Logger object
22
+ def self.check_out_git_archive(options = {})
23
+ branch = options.fetch(:branch)
24
+ path = options.fetch(:path)
25
+ dir = options.fetch(:basedir)
26
+ logger = options.fetch(:logger)
27
+
28
+ # Validate parameters
29
+ raise GitCheckoutError, "Source directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
30
+ raise GitCheckoutError, "Target directory #{path.inspect} does not exist" if dir.nil? || !File.directory?(path)
31
+
32
+ # To get the options working correctly (-o pipefail in particular) this needs to run under
33
+ # bash. It's just creating a script, rather than figuring out all the shell escapes...
34
+ begin
35
+ tmp_script = Tempfile.new(['git-checkout', '.sh'])
36
+ tmp_script.write "#!/bin/bash\n"
37
+ tmp_script.write "set -euf -o pipefail\n"
38
+ tmp_script.write "git archive --format=tar #{Shellwords.escape(branch)} | \\\n"
39
+ tmp_script.write " ( cd #{Shellwords.escape(path)} && tar -xf - )\n"
40
+ tmp_script.close
41
+ FileUtils.chmod 0o755, tmp_script.path
42
+
43
+ logger.debug("Begin git archive #{dir}:#{branch} -> #{path}")
44
+ output, status = Open3.capture2e(tmp_script.path, chdir: dir)
45
+ raise GitCheckoutError, "Git archive #{branch}->#{path} failed: #{output}" unless status.exitstatus.zero?
46
+ logger.debug("Success git archive #{dir}:#{branch}")
47
+ ensure
48
+ FileUtils.rm_f tmp_script.path if File.exist?(tmp_script.path)
49
+ end
50
+ end
51
+
52
+ # Determine the SHA of origin/master (or any other branch really) in the git repo
53
+ # @param options [Hash] Options hash:
54
+ # - :branch => Branch name to determine SHA of
55
+ # - :basedir => Where to check out from (must exist as a directory)
56
+ def self.branch_sha(options = {})
57
+ branch = options.fetch(:branch)
58
+ dir = options.fetch(:basedir)
59
+ raise GitCheckoutError, "Git directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
60
+ repo = Rugged::Repository.new(dir)
61
+ repo.branches[branch].target_id
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,209 @@
1
+ require 'json'
2
+ require 'stringio'
3
+
4
+ require_relative 'catalog/computed'
5
+ require_relative 'catalog/json'
6
+ require_relative 'catalog/noop'
7
+ require_relative 'catalog/puppetdb'
8
+ require_relative 'catalog/puppetmaster'
9
+ require_relative 'catalog-util/fileresources'
10
+
11
+ module OctocatalogDiff
12
+ # This class represents a catalog. Generation of the catalog is handled via one of the
13
+ # supported backends listed above as 'require_relative'. Usually, the 'computed' backend
14
+ # will build the catalog from the Puppet command.
15
+ class Catalog
16
+ # Readable
17
+ attr_reader :built, :catalog, :catalog_json
18
+
19
+ # Error classes that we can throw
20
+ class PuppetVersionError < RuntimeError; end
21
+ class CatalogError < RuntimeError; end
22
+
23
+ # Constructor
24
+ # @param :backend [Symbol] If set, this will force a backend
25
+ # @param :json [String] JSON catalog content (will avoid running Puppet to compile catalog)
26
+ # @param :puppetdb [Object] If set, pull the catalog from PuppetDB rather than building
27
+ # @param :node [String] Name of node whose catalog is being built
28
+ # @param :fact_file [String] OPTIONAL: Path to fact file (if not provided, look up in PuppetDB)
29
+ # @param :hiera_config [String] OPTIONAL: Path to hiera config file (munge temp. copy if not provided)
30
+ # @param :basedir [String] OPTIONAL: Base directory for catalog (default base directory of this checkout)
31
+ # @param :pass_env_vars [Array<String>] OPTIONAL: Additional environment vars to pass
32
+ # @param :convert_file_resources [Boolean] OPTIONAL: Convert file resource source to content
33
+ # @param :storeconfigs [Boolean] OPTIONAL: Pass the '-s' flag, for puppetdb (storeconfigs) integration
34
+ def initialize(options = {})
35
+ @options = options
36
+
37
+ # Call appropriate backend for catalog generation
38
+ @catalog_obj = backend(options)
39
+
40
+ # The catalog is not built yet, except if the backend has no build method
41
+ @built = false
42
+ build unless @catalog_obj.respond_to?(:build)
43
+
44
+ # The compilation directory can be overridden, e.g. when testing
45
+ @override_compilation_dir = nil
46
+ end
47
+
48
+ # Build catalog - this method needs to be called to build the catalog. It is separate due to
49
+ # the serialization of the logger object -- the parallel gem cannot serialize/deserialize a logger
50
+ # object so it cannot be part of any object that is passed around.
51
+ # @param logger [Logger] Logger object, initialized to a default throwaway value
52
+ def build(logger = Logger.new(StringIO.new))
53
+ # Only build once
54
+ return if @built
55
+ @built = true
56
+
57
+ # Call catalog's build method.
58
+ if @catalog_obj.respond_to?(:build)
59
+ logger.debug "Calling build for object #{@catalog_obj.class}"
60
+ @catalog_obj.build(logger)
61
+ end
62
+
63
+ # These methods must exist in all backends
64
+ @catalog = @catalog_obj.catalog
65
+ @catalog_json = @catalog_obj.catalog_json
66
+ @error_message = @catalog_obj.error_message
67
+
68
+ # The resource hash is computed the first time it's needed. For now initialize it as nil.
69
+ @resource_hash = nil
70
+
71
+ # Perform post-generation processing of the catalog
72
+ return unless valid?
73
+ unless @catalog_obj.respond_to?(:convert_file_resources) && @catalog_obj.convert_file_resources == false
74
+ OctocatalogDiff::CatalogUtil::FileResources.convert_file_resources(self) if @options.fetch(:compare_file_text, false)
75
+ end
76
+ end
77
+
78
+ # For logging we may wish to know the backend being used
79
+ # @return [String] Class of backend used
80
+ def builder
81
+ @catalog_obj.class.to_s
82
+ end
83
+
84
+ # Set the catalog JSON
85
+ # @param str [String] Catalog JSON
86
+ def catalog_json=(str)
87
+ @catalog_json = str
88
+ @resource_hash = nil
89
+ end
90
+
91
+ # This retrieves the compilation directory from the catalog, or otherwise the passed-in directory.
92
+ # @return [String] Compilation directory
93
+ def compilation_dir
94
+ return @override_compilation_dir if @override_compilation_dir
95
+ @catalog_obj.respond_to?(:compilation_dir) ? @catalog_obj.compilation_dir : @options[:basedir]
96
+ end
97
+
98
+ # The compilation directory can be overridden, e.g. during testing.
99
+ # @param dir [String] Compilation directory
100
+ def compilation_dir=(dir)
101
+ @override_compilation_dir = dir
102
+ end
103
+
104
+ # Determine whether the underlying catalog object supports :compare_file_text
105
+ # @return [Boolean] Whether underlying catalog object supports :compare_file_text
106
+ def convert_file_resources
107
+ return true unless @catalog_obj.respond_to?(:convert_file_resources)
108
+ @catalog_obj.convert_file_resources
109
+ end
110
+
111
+ # Retrieve the error message.
112
+ # @return [String] Error message (maximum 20,000 characters) - nil if no error.
113
+ def error_message
114
+ return nil if @error_message.nil? || !@error_message.is_a?(String)
115
+ @error_message[0, 20_000]
116
+ end
117
+
118
+ # Allow setting the error message. If the error message is set to a string, the catalog
119
+ # and catalog JSON are set to nil.
120
+ # @param error [String] Error message
121
+ def error_message=(error)
122
+ raise ArgumentError, 'Error message must be a string' unless error.is_a?(String)
123
+ @error_message = error
124
+ @catalog = nil
125
+ @catalog_json = nil
126
+ @resource_hash = nil
127
+ end
128
+
129
+ # This retrieves the version of Puppet used to compile a catalog. If the underlying catalog was not
130
+ # compiled by running Puppet (e.g., it was read in from JSON or puppetdb), then this returns the
131
+ # puppet version optionally passed in to the constructor. This can also be nil.
132
+ # @return [String] Puppet version
133
+ def puppet_version
134
+ @catalog_obj.respond_to?(:puppet_version) ? @catalog_obj.puppet_version : @options[:puppet_version]
135
+ end
136
+
137
+ # This allows retrieving a resource by type and title. This is intended for use when a O(1) lookup is required.
138
+ # @param :type [String] Type of resource
139
+ # @param :title [String] Title of resource
140
+ # @return [Hash] Resource item
141
+ def resource(opts = {})
142
+ raise ArgumentError, ':type and :title are required' unless opts[:type] && opts[:title]
143
+ build_resource_hash if @resource_hash.nil?
144
+ return nil unless @resource_hash[opts[:type]].is_a?(Hash)
145
+ @resource_hash[opts[:type]][opts[:title]]
146
+ end
147
+
148
+ # This is a compatibility layer for the resources, which are in a different place in Puppet 3.x and Puppet 4.x
149
+ # @return [Array] Resource array
150
+ def resources
151
+ raise CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
152
+ raise CatalogError, error_message unless valid?
153
+ return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
154
+ return @catalog['resources'] if @catalog['resources'].is_a?(Array)
155
+ # This is a bug condition
156
+ # :nocov:
157
+ raise "BUG: catalog has no data::resources or ::resources array. Please report this. #{@catalog.inspect}"
158
+ # :nocov
159
+ end
160
+
161
+ # This retrieves the number of retries necessary to compile the catalog. If the underlying catalog
162
+ # generation backend does not support retries, nil is returned.
163
+ # @return [Integer] Retry count
164
+ def retries
165
+ @retries = @catalog_obj.respond_to?(:retries) ? @catalog_obj.retries : nil
166
+ end
167
+
168
+ # Determine if the catalog build was successful.
169
+ # @return [Boolean] Whether the catalog is valid
170
+ def valid?
171
+ !@catalog.nil?
172
+ end
173
+
174
+ private
175
+
176
+ # Private method: Choose backend based on passed-in options
177
+ # @param options [Hash] Options passed into constructor
178
+ # @return [Object] OctocatalogDiff::Catalog::<whatever> object
179
+ def backend(options)
180
+ # Hard-coded backend
181
+ if options[:backend]
182
+ return OctocatalogDiff::Catalog::JSON.new(options) if options[:backend] == :json
183
+ return OctocatalogDiff::Catalog::PuppetDB.new(options) if options[:backend] == :puppetdb
184
+ return OctocatalogDiff::Catalog::PuppetMaster.new(options) if options[:backend] == :puppetmaster
185
+ return OctocatalogDiff::Catalog::Computed.new(options) if options[:backend] == :computed
186
+ return OctocatalogDiff::Catalog::Noop.new(options) if options[:backend] == :noop
187
+ raise ArgumentError, "Unknown backend :#{options[:backend]}"
188
+ end
189
+
190
+ # Determine backend based on arguments
191
+ return OctocatalogDiff::Catalog::JSON.new(options) if options[:json]
192
+ return OctocatalogDiff::Catalog::PuppetDB.new(options) if options[:puppetdb]
193
+ return OctocatalogDiff::Catalog::PuppetMaster.new(options) if options[:puppet_master]
194
+
195
+ # Default is to build catalog ourselves
196
+ OctocatalogDiff::Catalog::Computed.new(options)
197
+ end
198
+
199
+ # Private method: Build the resource hash to be used used for O(1) lookups by type and title.
200
+ # This method is called the first time the resource hash is accessed.
201
+ def build_resource_hash
202
+ @resource_hash = {}
203
+ resources.each do |resource|
204
+ @resource_hash[resource['type']] ||= {}
205
+ @resource_hash[resource['type']][resource['title']] = resource
206
+ end
207
+ end
208
+ end
209
+ end
@@ -0,0 +1,205 @@
1
+ require 'fileutils'
2
+ require 'json'
3
+ require 'open3'
4
+ require 'stringio'
5
+
6
+ require_relative '../catalog-util/bootstrap'
7
+ require_relative '../catalog-util/builddir'
8
+ require_relative '../catalog-util/command'
9
+ require_relative '../util/puppetversion'
10
+ require_relative '../catalog-util/facts'
11
+
12
+ module OctocatalogDiff
13
+ class Catalog
14
+ # Represents a Puppet catalog that is computed (via `puppet master --compile ...`)
15
+ # By instantiating this class, the catalog is computed.
16
+ class Computed
17
+ attr_reader :node, :error_message, :catalog, :catalog_json, :retries
18
+
19
+ # Constructor
20
+ # @param :node [String] REQUIRED: Node name
21
+ # @param :basedir [String] Directory in which to compile the catalog
22
+ # @param :pass_env_vars [Array<String>] Environment variables to pass when compiling catalog
23
+ # @param :retry_failed_catalog [Fixnum] Number of retries if a catalog compilation fails
24
+ # @param :tag [String] For display purposes, the catalog being compiled
25
+ # @param :puppet_binary [String] Full path to Puppet
26
+ # @param :puppet_version [String] Puppet version (optional; if not supplied, it is calculated)
27
+ # @param :puppet_command [String] Full command to run Puppet (optional; if not supplied, it is calculated)
28
+ def initialize(options)
29
+ raise ArgumentError, 'Usage: OctocatalogDiff::Catalog::Computed.initialize(options_hash)' unless options.is_a?(Hash)
30
+ raise ArgumentError, 'Node name must be passed to OctocatalogDiff::Catalog::Computed' unless options[:node].is_a?(String)
31
+
32
+ # Standard readable variables
33
+ @node = options[:node]
34
+ @error_message = nil
35
+ @catalog = nil
36
+ @catalog_json = nil
37
+
38
+ # Additional class variables
39
+ @pass_env_vars = options.fetch(:pass_env_vars, [])
40
+ @retry_failed_catalog = options.fetch(:retry_failed_catalog, 0)
41
+ @tag = options.fetch(:tag, 'catalog')
42
+ @puppet_binary = options[:puppet_binary]
43
+ @puppet_version = options[:puppet_version]
44
+ @puppet_command = options[:puppet_command]
45
+ @retries = nil
46
+ @builddir = nil
47
+
48
+ # Pass through the input for other access
49
+ @opts = options
50
+ raise ArgumentError, 'Branch is undefined' unless @opts[:branch]
51
+ end
52
+
53
+ # Actually build the catalog (populate @error_message, @catalog, @catalog_json)
54
+ def build(logger = Logger.new(StringIO.new))
55
+ facts_obj = OctocatalogDiff::CatalogUtil::Facts.new(@opts, logger)
56
+ logger.debug "Start retrieving facts for #{@node} from #{self.class}"
57
+ @opts[:facts] = facts_obj.facts
58
+ logger.debug "Success retrieving facts for #{@node} from #{self.class}"
59
+ build_catalog(logger)
60
+ end
61
+
62
+ # Get the Puppet version
63
+ # @return [String] Puppet version
64
+ def puppet_version
65
+ raise ArgumentError, '"puppet_binary" was not passed to OctocatalogDiff::Catalog::Computed' unless @puppet_binary
66
+ @puppet_version ||= OctocatalogDiff::Util::PuppetVersion.puppet_version(@puppet_binary)
67
+ end
68
+
69
+ # Compilation directory
70
+ # @return [String] Compilation directory
71
+ def compilation_dir
72
+ raise 'Catalog was not built' if @builddir.nil?
73
+ @builddir.tempdir
74
+ end
75
+
76
+ private
77
+
78
+ # Private method: Clean up a checkout directory, if it exists
79
+ def cleanup_checkout_dir(checkout_dir, logger)
80
+ return unless File.directory?(checkout_dir)
81
+ logger.debug("Cleaning up temporary directory #{checkout_dir}")
82
+ FileUtils.remove_entry_secure checkout_dir
83
+ end
84
+
85
+ # Private method: Bootstrap a directory
86
+ def bootstrap(logger)
87
+ return if @builddir
88
+
89
+ # Fill options for creating and populating the temporary directory
90
+ tmphash = @opts.dup
91
+
92
+ # Bootstrap directory if needed
93
+ if !@opts[:bootstrapped_dir].nil?
94
+ raise Errno::ENOENT, "Invalid dir #{@opts[:bootstrapped_dir]}" unless File.directory?(@opts[:bootstrapped_dir])
95
+ tmphash[:basedir] = @opts[:bootstrapped_dir]
96
+ elsif @opts[:branch] == '.'
97
+ tmphash[:basedir] = @opts[:basedir]
98
+ else
99
+ checkout_dir = Dir.mktmpdir
100
+ at_exit { cleanup_checkout_dir(checkout_dir, logger) }
101
+ tmphash[:basedir] = checkout_dir
102
+ OctocatalogDiff::CatalogUtil::Bootstrap.bootstrap_directory(@opts.merge(path: checkout_dir), logger)
103
+ end
104
+
105
+ # Create and populate the temporary directory
106
+ @builddir ||= OctocatalogDiff::CatalogUtil::BuildDir.new(tmphash, logger)
107
+ end
108
+
109
+ # Private method: Build catalog by running Puppet
110
+ # @param logger [Logger] Logger object
111
+ def build_catalog(logger = nil)
112
+ return nil unless @catalog.nil? && @error_message.nil?
113
+ bootstrap(logger)
114
+ result = run_puppet(logger)
115
+ @retries = result[:retries]
116
+ if (result[:exitcode]).zero?
117
+ begin
118
+ @catalog = ::JSON.parse(result[:stdout])
119
+ @catalog_json = result[:stdout]
120
+ @error_message = nil
121
+ rescue ::JSON::ParserError => exc
122
+ @catalog = nil
123
+ @catalog_json = nil
124
+ @error_message = "Catalog has invalid JSON: #{exc.message}"
125
+ end
126
+ else
127
+ @error_message = result[:stderr]
128
+ @catalog = nil
129
+ @catalog_json = nil
130
+ end
131
+ end
132
+
133
+ # Get the command to compile the catalog
134
+ # @return [String] Puppet command line
135
+ def puppet_command(options = @opts)
136
+ return @puppet_command if @puppet_command
137
+ raise ArgumentError, '"puppet_binary" was not passed to OctocatalogDiff::Catalog::Computed' unless @puppet_binary
138
+ command_opts = options.merge(
139
+ node: @node,
140
+ compilation_dir: @builddir.tempdir,
141
+ parser: options.fetch(:parser, :default),
142
+ puppet_binary: @puppet_binary,
143
+ fact_file: @builddir.fact_file,
144
+ dir: @builddir.tempdir,
145
+ enc: @builddir.enc
146
+ )
147
+ command = OctocatalogDiff::CatalogUtil::Command.new(command_opts)
148
+ @puppet_command = command.puppet_command
149
+ end
150
+
151
+ # Private method: Actually execute puppet
152
+ # @return [Hash] { stdout, stderr, exitcode }
153
+ def exec_puppet
154
+ # This is the environment provided to the puppet command.
155
+ env = {
156
+ 'HOME' => ENV['HOME'],
157
+ 'PATH' => ENV['PATH'],
158
+ 'PWD' => @builddir.tempdir
159
+ }
160
+ @pass_env_vars.each { |var| env[var] ||= ENV[var] }
161
+ out, err, status = Open3.capture3(env, puppet_command, unsetenv_others: true, chdir: @builddir.tempdir)
162
+ {
163
+ stdout: out,
164
+ stderr: err,
165
+ exitcode: status.exitstatus
166
+ }
167
+ end
168
+
169
+ # Private method: Runs puppet on the command line to compile the catalog
170
+ # Exit code is 0 if catalog generation was successful, non-zero otherwise.
171
+ # @param logger [Logger] Logger object
172
+ # @return [Hash] { stdout: <catalog as JSON>, stderr: <error messages>, exitcode: <hopefully 0> }
173
+ def run_puppet(logger)
174
+ # Run 'cmd' with environment 'env' from directory 'dir'
175
+ # First line of a successful result needs to be stripped off. It will look like:
176
+ # Notice: Compiled catalog for xxx in environment production in 27.88 seconds
177
+ retval = {}
178
+ 0.upto(@retry_failed_catalog) do |retry_num|
179
+ @retries = retry_num
180
+ time_begin = Time.now
181
+ logger.debug("(#{@tag}) Try #{1 + retry_num} executing Puppet #{puppet_version}: #{puppet_command}")
182
+ result = exec_puppet
183
+
184
+ # Success
185
+ if (result[:exitcode]).zero?
186
+ logger.debug("(#{@tag}) Catalog succeeded on try #{1 + retry_num} in #{Time.now - time_begin} seconds")
187
+ first_brace = result[:stdout].index('{') || 0
188
+ retval = {
189
+ stdout: result[:stdout][first_brace..-1],
190
+ stderr: nil,
191
+ exitcode: 0,
192
+ retries: retry_num
193
+ }
194
+ break
195
+ end
196
+
197
+ # Failure
198
+ logger.debug("(#{@tag}) Catalog failed on try #{1 + retry_num} in #{Time.now - time_begin} seconds")
199
+ retval = result.merge(retries: retry_num)
200
+ end
201
+ retval
202
+ end
203
+ end
204
+ end
205
+ end