octocatalog-diff 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.version +1 -0
- data/LICENSE +20 -0
- data/README.md +82 -0
- data/bin/octocatalog-diff +75 -0
- data/doc/advanced-bootstrap.md +33 -0
- data/doc/advanced-cache-dir.md +24 -0
- data/doc/advanced-catalog-only.md +37 -0
- data/doc/advanced-ci.md +13 -0
- data/doc/advanced-dynamic-ignores.md +123 -0
- data/doc/advanced-future-parser.md +11 -0
- data/doc/advanced-ignores.md +224 -0
- data/doc/advanced-output-formats.md +96 -0
- data/doc/advanced-output-hacks.md +45 -0
- data/doc/advanced-override-facts.md +67 -0
- data/doc/advanced-pe-enc.md +52 -0
- data/doc/advanced-puppet-master.md +50 -0
- data/doc/advanced-puppet-versions.md +9 -0
- data/doc/advanced-storeconfigs.md +72 -0
- data/doc/advanced-using-without-git.md +15 -0
- data/doc/advanced.md +43 -0
- data/doc/basic.md +70 -0
- data/doc/configuration-enc.md +69 -0
- data/doc/configuration-hiera.md +103 -0
- data/doc/configuration-puppetdb.md +49 -0
- data/doc/configuration.md +51 -0
- data/doc/dev/README.md +1 -0
- data/doc/dev/coverage.md +34 -0
- data/doc/dev/how-to-add-options.md +83 -0
- data/doc/dev/integration-tests.md +63 -0
- data/doc/dev/releasing.md +19 -0
- data/doc/installation.md +49 -0
- data/doc/limitations.md +34 -0
- data/doc/optionsref.md +947 -0
- data/doc/requirements.md +16 -0
- data/doc/roadmap.md +26 -0
- data/doc/similar.md +17 -0
- data/doc/troubleshooting.md +54 -0
- data/lib/octocatalog-diff.rb +12 -0
- data/lib/octocatalog-diff/bootstrap.rb +53 -0
- data/lib/octocatalog-diff/catalog-diff/cli.rb +205 -0
- data/lib/octocatalog-diff/catalog-diff/cli/catalogs.rb +240 -0
- data/lib/octocatalog-diff/catalog-diff/cli/diffs.rb +145 -0
- data/lib/octocatalog-diff/catalog-diff/cli/helpers/fact_override.rb +99 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options.rb +173 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/basedir.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_environment.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_script.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_then_exit.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrapped_dirs.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/cached_master_dir.rb +21 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/catalog_only.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/color.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/compare_file_text.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/debug.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_datatype_changes.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_detail_add.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_source_file_line.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/enc.rb +31 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/existing_catalogs.rb +25 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/fact_file.rb +23 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/fact_override.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/facts_terminus.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/from_puppetdb.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/header.rb +24 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_config.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_path_strip.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hostname.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore.rb +24 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_attr.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_tags.rb +23 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/include_tags.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/master_cache_branch.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/output_file.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/output_format.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/parallel.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/parser.rb +48 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pass_env_vars.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_ca.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_cert.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_key.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token_file.rb +17 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_url.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_binary.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_api_version.rb +20 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_ca.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_cert.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_key.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_ca.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_cert.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_key.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password_file.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_url.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/quiet.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/retry_failed_catalog.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/safe_to_delete_cached_master_dir.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/storeconfigs.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/suppress_absent_file_details.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/to_from_branch.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/printer.rb +52 -0
- data/lib/octocatalog-diff/catalog-diff/differ.rb +615 -0
- data/lib/octocatalog-diff/catalog-diff/display.rb +125 -0
- data/lib/octocatalog-diff/catalog-diff/display/json.rb +25 -0
- data/lib/octocatalog-diff/catalog-diff/display/text.rb +452 -0
- data/lib/octocatalog-diff/catalog-util/bootstrap.rb +145 -0
- data/lib/octocatalog-diff/catalog-util/builddir.rb +289 -0
- data/lib/octocatalog-diff/catalog-util/cached_master_directory.rb +169 -0
- data/lib/octocatalog-diff/catalog-util/command.rb +96 -0
- data/lib/octocatalog-diff/catalog-util/enc.rb +77 -0
- data/lib/octocatalog-diff/catalog-util/enc/noop.rb +22 -0
- data/lib/octocatalog-diff/catalog-util/enc/pe.rb +99 -0
- data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb +61 -0
- data/lib/octocatalog-diff/catalog-util/enc/script.rb +88 -0
- data/lib/octocatalog-diff/catalog-util/facts.rb +89 -0
- data/lib/octocatalog-diff/catalog-util/fileresources.rb +83 -0
- data/lib/octocatalog-diff/catalog-util/git.rb +65 -0
- data/lib/octocatalog-diff/catalog.rb +209 -0
- data/lib/octocatalog-diff/catalog/computed.rb +205 -0
- data/lib/octocatalog-diff/catalog/json.rb +30 -0
- data/lib/octocatalog-diff/catalog/noop.rb +19 -0
- data/lib/octocatalog-diff/catalog/puppetdb.rb +82 -0
- data/lib/octocatalog-diff/catalog/puppetmaster.rb +121 -0
- data/lib/octocatalog-diff/external/pson/LICENSE +17 -0
- data/lib/octocatalog-diff/external/pson/README.md +20 -0
- data/lib/octocatalog-diff/external/pson/common.rb +370 -0
- data/lib/octocatalog-diff/external/pson/pure.rb +15 -0
- data/lib/octocatalog-diff/external/pson/pure/generator.rb +395 -0
- data/lib/octocatalog-diff/external/pson/pure/parser.rb +307 -0
- data/lib/octocatalog-diff/external/pson/version.rb +8 -0
- data/lib/octocatalog-diff/facts.rb +125 -0
- data/lib/octocatalog-diff/facts/json.rb +20 -0
- data/lib/octocatalog-diff/facts/puppetdb.rb +59 -0
- data/lib/octocatalog-diff/facts/yaml.rb +29 -0
- data/lib/octocatalog-diff/puppetdb.rb +163 -0
- data/lib/octocatalog-diff/util/colored.rb +20 -0
- data/lib/octocatalog-diff/util/httparty.rb +158 -0
- data/lib/octocatalog-diff/util/parallel.rb +170 -0
- data/lib/octocatalog-diff/util/puppetversion.rb +24 -0
- data/lib/octocatalog-diff/version.rb +7 -0
- metadata +386 -0
@@ -0,0 +1,89 @@
|
|
1
|
+
require_relative '../facts'
|
2
|
+
|
3
|
+
module OctocatalogDiff
|
4
|
+
module CatalogUtil
|
5
|
+
# Helper class to construct a fact object based on options provided by
|
6
|
+
# cli/options. Supports a direct fact file, looking up a YAML file based on
|
7
|
+
# node name within Puppet fact directories, or retrieving from PuppetDB.
|
8
|
+
class Facts
|
9
|
+
# Constructor
|
10
|
+
# @param options [Hash] Options from cli/options
|
11
|
+
# @param logger [Logger] Logger object for debug messages (optional)
|
12
|
+
def initialize(options, logger = nil)
|
13
|
+
raise ArgumentError, "Argument to constructor must be Hash not #{options.class}" unless options.is_a?(Hash)
|
14
|
+
@options = options.dup
|
15
|
+
@logger = logger
|
16
|
+
|
17
|
+
# Environment variable recognition
|
18
|
+
@options[:puppetdb_url] ||= ENV['PUPPETDB_URL'] if ENV['PUPPETDB_URL']
|
19
|
+
@options[:puppet_fact_dir] ||= ENV['PUPPET_FACT_DIR'] if ENV['PUPPET_FACT_DIR']
|
20
|
+
end
|
21
|
+
|
22
|
+
# Compute facts if needed and then return them
|
23
|
+
# @return [Hash] Facts
|
24
|
+
def facts
|
25
|
+
@facts ||= compute_facts
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
# Retrieve facts from a YAML file in the puppet facts directory
|
31
|
+
# @param filename [String] Full path to file to read in
|
32
|
+
# @return [OctocatalogDiff::Facts] Facts object
|
33
|
+
def facts_from_file(filename)
|
34
|
+
@logger.debug("Retrieving facts from #{filename}") unless @logger.nil?
|
35
|
+
opts = {
|
36
|
+
node: @options[:node],
|
37
|
+
backend: :yaml,
|
38
|
+
fact_file_string: File.read(filename)
|
39
|
+
}
|
40
|
+
OctocatalogDiff::Facts.new(opts)
|
41
|
+
end
|
42
|
+
|
43
|
+
# Retrieve facts from PuppetDB. Either options[:puppetdb_url] or ENV['PUPPETDB_URL']
|
44
|
+
# needs to be set for this to work. Node name must also be set in options.
|
45
|
+
# @return [OctocatalogDiff::Facts] Facts object
|
46
|
+
def facts_from_puppetdb
|
47
|
+
@logger.debug('Retrieving facts from PuppetDB') unless @logger.nil?
|
48
|
+
OctocatalogDiff::Facts.new(@options.merge(backend: :puppetdb, retry: 2))
|
49
|
+
end
|
50
|
+
|
51
|
+
# Error message when the node is needed but not defined
|
52
|
+
# :nocov:
|
53
|
+
def error_node_not_provided
|
54
|
+
message = 'Unable to determine facts. You must either supply "--fact-file FILENAME"' \
|
55
|
+
' or a node name "-n NODENAME" to look up a set of node facts in a fact' \
|
56
|
+
' directory or in PuppetDB.'
|
57
|
+
raise ArgumentError, message
|
58
|
+
end
|
59
|
+
# :nocov:
|
60
|
+
|
61
|
+
# Does the actual computation/lookup of facts. Seeks to return a OctocatalogDiff::Facts
|
62
|
+
# object. Raises error if no fact sources are found.
|
63
|
+
# @return [OctocatalogDiff::Facts] Facts object
|
64
|
+
def compute_facts
|
65
|
+
if @options.key?(:facts) && @options[:facts].is_a?(OctocatalogDiff::Facts)
|
66
|
+
return @options[:facts]
|
67
|
+
end
|
68
|
+
|
69
|
+
if @options.key?(:fact_file)
|
70
|
+
raise Errno::ENOENT, 'Specified fact file does not exist' unless File.file?(@options[:fact_file])
|
71
|
+
return facts_from_file(@options[:fact_file])
|
72
|
+
end
|
73
|
+
|
74
|
+
error_node_not_provided if @options[:node].nil?
|
75
|
+
|
76
|
+
if @options[:puppet_fact_dir] && File.directory?(@options[:puppet_fact_dir])
|
77
|
+
filename = File.join(@options[:puppet_fact_dir], @options[:node] + '.yaml')
|
78
|
+
return facts_from_file(filename) if File.file?(filename)
|
79
|
+
end
|
80
|
+
|
81
|
+
return facts_from_puppetdb if @options[:puppetdb_url]
|
82
|
+
|
83
|
+
message = 'Unable to compute facts for node. Please use "--fact-file FILENAME" option' \
|
84
|
+
' or set one of these environment variables: PUPPET_FACT_DIR or PUPPETDB_URL.'
|
85
|
+
raise ArgumentError, message
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'digest'
|
2
|
+
|
3
|
+
module OctocatalogDiff
|
4
|
+
module CatalogUtil
|
5
|
+
# Used to convert file resources such as:
|
6
|
+
# file { 'something': source => 'puppet:///modules/xxx/yyy'}
|
7
|
+
# to:
|
8
|
+
# file { 'something': content => $( cat modules/xxx/files/yyy )}
|
9
|
+
# This allows the displayed diff to show differences in static files.
|
10
|
+
class FileResources
|
11
|
+
# Public method: Convert file resources to text. See the description of the class
|
12
|
+
# just above for details.
|
13
|
+
# @param obj [OctocatalogDiff::Catalog] Catalog object (will be modified)
|
14
|
+
def self.convert_file_resources(obj)
|
15
|
+
return unless obj.valid? && obj.compilation_dir.is_a?(String) && !obj.compilation_dir.empty?
|
16
|
+
_convert_file_resources(obj.resources, obj.compilation_dir)
|
17
|
+
begin
|
18
|
+
obj.catalog_json = ::JSON.generate(obj.catalog)
|
19
|
+
rescue ::JSON::GeneratorError => exc
|
20
|
+
obj.error_message = "Failed to generate JSON: #{exc.message}"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# Internal method: Static method to convert file resources. The compilation directory is
|
25
|
+
# required, or else this is a no-op. The passed-in array of resources is modified by this method.
|
26
|
+
# @param resources [Array<Hash>] Array of catalog resources
|
27
|
+
# @param compilation_dir [String] Compilation directory (so files can be looked up)
|
28
|
+
def self._convert_file_resources(resources, compilation_dir)
|
29
|
+
# Calculate compilation directory. There is not explicit error checking here because
|
30
|
+
# there is on-demand, explicit error checking for each file within the modification loop.
|
31
|
+
return unless compilation_dir.is_a?(String) && compilation_dir != ''
|
32
|
+
|
33
|
+
# Making sure that compilation_dir/environments/production/modules exists (and by inference,
|
34
|
+
# that compilation_dir/environments/production is pointing at the right place). Otherwise, try to find
|
35
|
+
# compilation_dir/modules. If neither of those exist, this code can't run.
|
36
|
+
env_dir = File.join(compilation_dir, 'environments', 'production')
|
37
|
+
unless File.directory?(File.join(env_dir, 'modules'))
|
38
|
+
return unless File.directory?(File.join(compilation_dir, 'modules'))
|
39
|
+
env_dir = compilation_dir
|
40
|
+
end
|
41
|
+
|
42
|
+
# Modify the resources
|
43
|
+
resources.map! do |resource|
|
44
|
+
if resource_convertible?(resource)
|
45
|
+
# Parse the 'source' parameter into a file on disk
|
46
|
+
src = resource['parameters']['source']
|
47
|
+
raise "Bad parameter source #{src}" unless src =~ %r{^puppet:///modules/([^/]+)/(.+)}
|
48
|
+
path = File.join(env_dir, 'modules', Regexp.last_match(1), 'files', Regexp.last_match(2))
|
49
|
+
|
50
|
+
if File.file?(path)
|
51
|
+
# If the file is found, read its content. If the content is all ASCII, substitute it into
|
52
|
+
# the 'content' parameter for easier comparison. If not, instead populate the md5sum.
|
53
|
+
# Delete the 'source' attribute as well.
|
54
|
+
content = File.read(path)
|
55
|
+
is_ascii = content.force_encoding('UTF-8').ascii_only?
|
56
|
+
resource['parameters']['content'] = is_ascii ? content : '{md5}' + Digest::MD5.hexdigest(content)
|
57
|
+
resource['parameters'].delete('source')
|
58
|
+
elsif File.exist?(path)
|
59
|
+
# We are not handling recursive file installs from a directory or anything else.
|
60
|
+
# However, the fact that we found *something* at this location indicates that the catalog
|
61
|
+
# is probably correct. Hence, the very general .exist? check.
|
62
|
+
else
|
63
|
+
raise Errno::ENOENT, "Unable to find '#{src}' at #{path}!"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
resource
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Internal method: Determine if a resource is convertible. It is convertible if it
|
71
|
+
# is a file resource with no declared 'content' and with a declared and parseable 'source'.
|
72
|
+
# @param resource [Hash] Resource to check
|
73
|
+
# @return [Boolean] True of resource is convertible, false if not
|
74
|
+
def self.resource_convertible?(resource)
|
75
|
+
return true if resource['type'] == 'File' && \
|
76
|
+
resource['parameters'].key?('source') && \
|
77
|
+
!resource['parameters'].key?('content') && \
|
78
|
+
resource['parameters']['source'] =~ %r{^puppet:///modules/([^/]+)/(.+)}
|
79
|
+
false
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'open3'
|
3
|
+
require 'rugged'
|
4
|
+
require 'shellwords'
|
5
|
+
require 'tempfile'
|
6
|
+
|
7
|
+
module OctocatalogDiff
|
8
|
+
module CatalogUtil
|
9
|
+
# Class to perform a git checkout (via 'git archive') of a branch from the base git
|
10
|
+
# directory into another targeted directory.
|
11
|
+
class Git
|
12
|
+
# Trapped errors
|
13
|
+
class GitCheckoutError < RuntimeError
|
14
|
+
end
|
15
|
+
|
16
|
+
# Check out a branch via 'git archive' from one directory into another.
|
17
|
+
# @param options [Hash] Options hash:
|
18
|
+
# - :branch => Branch name to check out
|
19
|
+
# - :path => Where to check out to (must exist as a directory)
|
20
|
+
# - :basedir => Where to check out from (must exist as a directory)
|
21
|
+
# - :logger => Logger object
|
22
|
+
def self.check_out_git_archive(options = {})
|
23
|
+
branch = options.fetch(:branch)
|
24
|
+
path = options.fetch(:path)
|
25
|
+
dir = options.fetch(:basedir)
|
26
|
+
logger = options.fetch(:logger)
|
27
|
+
|
28
|
+
# Validate parameters
|
29
|
+
raise GitCheckoutError, "Source directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
|
30
|
+
raise GitCheckoutError, "Target directory #{path.inspect} does not exist" if dir.nil? || !File.directory?(path)
|
31
|
+
|
32
|
+
# To get the options working correctly (-o pipefail in particular) this needs to run under
|
33
|
+
# bash. It's just creating a script, rather than figuring out all the shell escapes...
|
34
|
+
begin
|
35
|
+
tmp_script = Tempfile.new(['git-checkout', '.sh'])
|
36
|
+
tmp_script.write "#!/bin/bash\n"
|
37
|
+
tmp_script.write "set -euf -o pipefail\n"
|
38
|
+
tmp_script.write "git archive --format=tar #{Shellwords.escape(branch)} | \\\n"
|
39
|
+
tmp_script.write " ( cd #{Shellwords.escape(path)} && tar -xf - )\n"
|
40
|
+
tmp_script.close
|
41
|
+
FileUtils.chmod 0o755, tmp_script.path
|
42
|
+
|
43
|
+
logger.debug("Begin git archive #{dir}:#{branch} -> #{path}")
|
44
|
+
output, status = Open3.capture2e(tmp_script.path, chdir: dir)
|
45
|
+
raise GitCheckoutError, "Git archive #{branch}->#{path} failed: #{output}" unless status.exitstatus.zero?
|
46
|
+
logger.debug("Success git archive #{dir}:#{branch}")
|
47
|
+
ensure
|
48
|
+
FileUtils.rm_f tmp_script.path if File.exist?(tmp_script.path)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Determine the SHA of origin/master (or any other branch really) in the git repo
|
53
|
+
# @param options [Hash] Options hash:
|
54
|
+
# - :branch => Branch name to determine SHA of
|
55
|
+
# - :basedir => Where to check out from (must exist as a directory)
|
56
|
+
def self.branch_sha(options = {})
|
57
|
+
branch = options.fetch(:branch)
|
58
|
+
dir = options.fetch(:basedir)
|
59
|
+
raise GitCheckoutError, "Git directory #{dir.inspect} does not exist" if dir.nil? || !File.directory?(dir)
|
60
|
+
repo = Rugged::Repository.new(dir)
|
61
|
+
repo.branches[branch].target_id
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,209 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'stringio'
|
3
|
+
|
4
|
+
require_relative 'catalog/computed'
|
5
|
+
require_relative 'catalog/json'
|
6
|
+
require_relative 'catalog/noop'
|
7
|
+
require_relative 'catalog/puppetdb'
|
8
|
+
require_relative 'catalog/puppetmaster'
|
9
|
+
require_relative 'catalog-util/fileresources'
|
10
|
+
|
11
|
+
module OctocatalogDiff
|
12
|
+
# This class represents a catalog. Generation of the catalog is handled via one of the
|
13
|
+
# supported backends listed above as 'require_relative'. Usually, the 'computed' backend
|
14
|
+
# will build the catalog from the Puppet command.
|
15
|
+
class Catalog
|
16
|
+
# Readable
|
17
|
+
attr_reader :built, :catalog, :catalog_json
|
18
|
+
|
19
|
+
# Error classes that we can throw
|
20
|
+
class PuppetVersionError < RuntimeError; end
|
21
|
+
class CatalogError < RuntimeError; end
|
22
|
+
|
23
|
+
# Constructor
|
24
|
+
# @param :backend [Symbol] If set, this will force a backend
|
25
|
+
# @param :json [String] JSON catalog content (will avoid running Puppet to compile catalog)
|
26
|
+
# @param :puppetdb [Object] If set, pull the catalog from PuppetDB rather than building
|
27
|
+
# @param :node [String] Name of node whose catalog is being built
|
28
|
+
# @param :fact_file [String] OPTIONAL: Path to fact file (if not provided, look up in PuppetDB)
|
29
|
+
# @param :hiera_config [String] OPTIONAL: Path to hiera config file (munge temp. copy if not provided)
|
30
|
+
# @param :basedir [String] OPTIONAL: Base directory for catalog (default base directory of this checkout)
|
31
|
+
# @param :pass_env_vars [Array<String>] OPTIONAL: Additional environment vars to pass
|
32
|
+
# @param :convert_file_resources [Boolean] OPTIONAL: Convert file resource source to content
|
33
|
+
# @param :storeconfigs [Boolean] OPTIONAL: Pass the '-s' flag, for puppetdb (storeconfigs) integration
|
34
|
+
def initialize(options = {})
|
35
|
+
@options = options
|
36
|
+
|
37
|
+
# Call appropriate backend for catalog generation
|
38
|
+
@catalog_obj = backend(options)
|
39
|
+
|
40
|
+
# The catalog is not built yet, except if the backend has no build method
|
41
|
+
@built = false
|
42
|
+
build unless @catalog_obj.respond_to?(:build)
|
43
|
+
|
44
|
+
# The compilation directory can be overridden, e.g. when testing
|
45
|
+
@override_compilation_dir = nil
|
46
|
+
end
|
47
|
+
|
48
|
+
# Build catalog - this method needs to be called to build the catalog. It is separate due to
|
49
|
+
# the serialization of the logger object -- the parallel gem cannot serialize/deserialize a logger
|
50
|
+
# object so it cannot be part of any object that is passed around.
|
51
|
+
# @param logger [Logger] Logger object, initialized to a default throwaway value
|
52
|
+
def build(logger = Logger.new(StringIO.new))
|
53
|
+
# Only build once
|
54
|
+
return if @built
|
55
|
+
@built = true
|
56
|
+
|
57
|
+
# Call catalog's build method.
|
58
|
+
if @catalog_obj.respond_to?(:build)
|
59
|
+
logger.debug "Calling build for object #{@catalog_obj.class}"
|
60
|
+
@catalog_obj.build(logger)
|
61
|
+
end
|
62
|
+
|
63
|
+
# These methods must exist in all backends
|
64
|
+
@catalog = @catalog_obj.catalog
|
65
|
+
@catalog_json = @catalog_obj.catalog_json
|
66
|
+
@error_message = @catalog_obj.error_message
|
67
|
+
|
68
|
+
# The resource hash is computed the first time it's needed. For now initialize it as nil.
|
69
|
+
@resource_hash = nil
|
70
|
+
|
71
|
+
# Perform post-generation processing of the catalog
|
72
|
+
return unless valid?
|
73
|
+
unless @catalog_obj.respond_to?(:convert_file_resources) && @catalog_obj.convert_file_resources == false
|
74
|
+
OctocatalogDiff::CatalogUtil::FileResources.convert_file_resources(self) if @options.fetch(:compare_file_text, false)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# For logging we may wish to know the backend being used
|
79
|
+
# @return [String] Class of backend used
|
80
|
+
def builder
|
81
|
+
@catalog_obj.class.to_s
|
82
|
+
end
|
83
|
+
|
84
|
+
# Set the catalog JSON
|
85
|
+
# @param str [String] Catalog JSON
|
86
|
+
def catalog_json=(str)
|
87
|
+
@catalog_json = str
|
88
|
+
@resource_hash = nil
|
89
|
+
end
|
90
|
+
|
91
|
+
# This retrieves the compilation directory from the catalog, or otherwise the passed-in directory.
|
92
|
+
# @return [String] Compilation directory
|
93
|
+
def compilation_dir
|
94
|
+
return @override_compilation_dir if @override_compilation_dir
|
95
|
+
@catalog_obj.respond_to?(:compilation_dir) ? @catalog_obj.compilation_dir : @options[:basedir]
|
96
|
+
end
|
97
|
+
|
98
|
+
# The compilation directory can be overridden, e.g. during testing.
|
99
|
+
# @param dir [String] Compilation directory
|
100
|
+
def compilation_dir=(dir)
|
101
|
+
@override_compilation_dir = dir
|
102
|
+
end
|
103
|
+
|
104
|
+
# Determine whether the underlying catalog object supports :compare_file_text
|
105
|
+
# @return [Boolean] Whether underlying catalog object supports :compare_file_text
|
106
|
+
def convert_file_resources
|
107
|
+
return true unless @catalog_obj.respond_to?(:convert_file_resources)
|
108
|
+
@catalog_obj.convert_file_resources
|
109
|
+
end
|
110
|
+
|
111
|
+
# Retrieve the error message.
|
112
|
+
# @return [String] Error message (maximum 20,000 characters) - nil if no error.
|
113
|
+
def error_message
|
114
|
+
return nil if @error_message.nil? || !@error_message.is_a?(String)
|
115
|
+
@error_message[0, 20_000]
|
116
|
+
end
|
117
|
+
|
118
|
+
# Allow setting the error message. If the error message is set to a string, the catalog
|
119
|
+
# and catalog JSON are set to nil.
|
120
|
+
# @param error [String] Error message
|
121
|
+
def error_message=(error)
|
122
|
+
raise ArgumentError, 'Error message must be a string' unless error.is_a?(String)
|
123
|
+
@error_message = error
|
124
|
+
@catalog = nil
|
125
|
+
@catalog_json = nil
|
126
|
+
@resource_hash = nil
|
127
|
+
end
|
128
|
+
|
129
|
+
# This retrieves the version of Puppet used to compile a catalog. If the underlying catalog was not
|
130
|
+
# compiled by running Puppet (e.g., it was read in from JSON or puppetdb), then this returns the
|
131
|
+
# puppet version optionally passed in to the constructor. This can also be nil.
|
132
|
+
# @return [String] Puppet version
|
133
|
+
def puppet_version
|
134
|
+
@catalog_obj.respond_to?(:puppet_version) ? @catalog_obj.puppet_version : @options[:puppet_version]
|
135
|
+
end
|
136
|
+
|
137
|
+
# This allows retrieving a resource by type and title. This is intended for use when a O(1) lookup is required.
|
138
|
+
# @param :type [String] Type of resource
|
139
|
+
# @param :title [String] Title of resource
|
140
|
+
# @return [Hash] Resource item
|
141
|
+
def resource(opts = {})
|
142
|
+
raise ArgumentError, ':type and :title are required' unless opts[:type] && opts[:title]
|
143
|
+
build_resource_hash if @resource_hash.nil?
|
144
|
+
return nil unless @resource_hash[opts[:type]].is_a?(Hash)
|
145
|
+
@resource_hash[opts[:type]][opts[:title]]
|
146
|
+
end
|
147
|
+
|
148
|
+
# This is a compatibility layer for the resources, which are in a different place in Puppet 3.x and Puppet 4.x
|
149
|
+
# @return [Array] Resource array
|
150
|
+
def resources
|
151
|
+
raise CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
|
152
|
+
raise CatalogError, error_message unless valid?
|
153
|
+
return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
|
154
|
+
return @catalog['resources'] if @catalog['resources'].is_a?(Array)
|
155
|
+
# This is a bug condition
|
156
|
+
# :nocov:
|
157
|
+
raise "BUG: catalog has no data::resources or ::resources array. Please report this. #{@catalog.inspect}"
|
158
|
+
# :nocov
|
159
|
+
end
|
160
|
+
|
161
|
+
# This retrieves the number of retries necessary to compile the catalog. If the underlying catalog
|
162
|
+
# generation backend does not support retries, nil is returned.
|
163
|
+
# @return [Integer] Retry count
|
164
|
+
def retries
|
165
|
+
@retries = @catalog_obj.respond_to?(:retries) ? @catalog_obj.retries : nil
|
166
|
+
end
|
167
|
+
|
168
|
+
# Determine if the catalog build was successful.
|
169
|
+
# @return [Boolean] Whether the catalog is valid
|
170
|
+
def valid?
|
171
|
+
!@catalog.nil?
|
172
|
+
end
|
173
|
+
|
174
|
+
private
|
175
|
+
|
176
|
+
# Private method: Choose backend based on passed-in options
|
177
|
+
# @param options [Hash] Options passed into constructor
|
178
|
+
# @return [Object] OctocatalogDiff::Catalog::<whatever> object
|
179
|
+
def backend(options)
|
180
|
+
# Hard-coded backend
|
181
|
+
if options[:backend]
|
182
|
+
return OctocatalogDiff::Catalog::JSON.new(options) if options[:backend] == :json
|
183
|
+
return OctocatalogDiff::Catalog::PuppetDB.new(options) if options[:backend] == :puppetdb
|
184
|
+
return OctocatalogDiff::Catalog::PuppetMaster.new(options) if options[:backend] == :puppetmaster
|
185
|
+
return OctocatalogDiff::Catalog::Computed.new(options) if options[:backend] == :computed
|
186
|
+
return OctocatalogDiff::Catalog::Noop.new(options) if options[:backend] == :noop
|
187
|
+
raise ArgumentError, "Unknown backend :#{options[:backend]}"
|
188
|
+
end
|
189
|
+
|
190
|
+
# Determine backend based on arguments
|
191
|
+
return OctocatalogDiff::Catalog::JSON.new(options) if options[:json]
|
192
|
+
return OctocatalogDiff::Catalog::PuppetDB.new(options) if options[:puppetdb]
|
193
|
+
return OctocatalogDiff::Catalog::PuppetMaster.new(options) if options[:puppet_master]
|
194
|
+
|
195
|
+
# Default is to build catalog ourselves
|
196
|
+
OctocatalogDiff::Catalog::Computed.new(options)
|
197
|
+
end
|
198
|
+
|
199
|
+
# Private method: Build the resource hash to be used used for O(1) lookups by type and title.
|
200
|
+
# This method is called the first time the resource hash is accessed.
|
201
|
+
def build_resource_hash
|
202
|
+
@resource_hash = {}
|
203
|
+
resources.each do |resource|
|
204
|
+
@resource_hash[resource['type']] ||= {}
|
205
|
+
@resource_hash[resource['type']][resource['title']] = resource
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
@@ -0,0 +1,205 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'json'
|
3
|
+
require 'open3'
|
4
|
+
require 'stringio'
|
5
|
+
|
6
|
+
require_relative '../catalog-util/bootstrap'
|
7
|
+
require_relative '../catalog-util/builddir'
|
8
|
+
require_relative '../catalog-util/command'
|
9
|
+
require_relative '../util/puppetversion'
|
10
|
+
require_relative '../catalog-util/facts'
|
11
|
+
|
12
|
+
module OctocatalogDiff
|
13
|
+
class Catalog
|
14
|
+
# Represents a Puppet catalog that is computed (via `puppet master --compile ...`)
|
15
|
+
# By instantiating this class, the catalog is computed.
|
16
|
+
class Computed
|
17
|
+
attr_reader :node, :error_message, :catalog, :catalog_json, :retries
|
18
|
+
|
19
|
+
# Constructor
|
20
|
+
# @param :node [String] REQUIRED: Node name
|
21
|
+
# @param :basedir [String] Directory in which to compile the catalog
|
22
|
+
# @param :pass_env_vars [Array<String>] Environment variables to pass when compiling catalog
|
23
|
+
# @param :retry_failed_catalog [Fixnum] Number of retries if a catalog compilation fails
|
24
|
+
# @param :tag [String] For display purposes, the catalog being compiled
|
25
|
+
# @param :puppet_binary [String] Full path to Puppet
|
26
|
+
# @param :puppet_version [String] Puppet version (optional; if not supplied, it is calculated)
|
27
|
+
# @param :puppet_command [String] Full command to run Puppet (optional; if not supplied, it is calculated)
|
28
|
+
def initialize(options)
|
29
|
+
raise ArgumentError, 'Usage: OctocatalogDiff::Catalog::Computed.initialize(options_hash)' unless options.is_a?(Hash)
|
30
|
+
raise ArgumentError, 'Node name must be passed to OctocatalogDiff::Catalog::Computed' unless options[:node].is_a?(String)
|
31
|
+
|
32
|
+
# Standard readable variables
|
33
|
+
@node = options[:node]
|
34
|
+
@error_message = nil
|
35
|
+
@catalog = nil
|
36
|
+
@catalog_json = nil
|
37
|
+
|
38
|
+
# Additional class variables
|
39
|
+
@pass_env_vars = options.fetch(:pass_env_vars, [])
|
40
|
+
@retry_failed_catalog = options.fetch(:retry_failed_catalog, 0)
|
41
|
+
@tag = options.fetch(:tag, 'catalog')
|
42
|
+
@puppet_binary = options[:puppet_binary]
|
43
|
+
@puppet_version = options[:puppet_version]
|
44
|
+
@puppet_command = options[:puppet_command]
|
45
|
+
@retries = nil
|
46
|
+
@builddir = nil
|
47
|
+
|
48
|
+
# Pass through the input for other access
|
49
|
+
@opts = options
|
50
|
+
raise ArgumentError, 'Branch is undefined' unless @opts[:branch]
|
51
|
+
end
|
52
|
+
|
53
|
+
# Actually build the catalog (populate @error_message, @catalog, @catalog_json)
|
54
|
+
def build(logger = Logger.new(StringIO.new))
|
55
|
+
facts_obj = OctocatalogDiff::CatalogUtil::Facts.new(@opts, logger)
|
56
|
+
logger.debug "Start retrieving facts for #{@node} from #{self.class}"
|
57
|
+
@opts[:facts] = facts_obj.facts
|
58
|
+
logger.debug "Success retrieving facts for #{@node} from #{self.class}"
|
59
|
+
build_catalog(logger)
|
60
|
+
end
|
61
|
+
|
62
|
+
# Get the Puppet version
|
63
|
+
# @return [String] Puppet version
|
64
|
+
def puppet_version
|
65
|
+
raise ArgumentError, '"puppet_binary" was not passed to OctocatalogDiff::Catalog::Computed' unless @puppet_binary
|
66
|
+
@puppet_version ||= OctocatalogDiff::Util::PuppetVersion.puppet_version(@puppet_binary)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Compilation directory
|
70
|
+
# @return [String] Compilation directory
|
71
|
+
def compilation_dir
|
72
|
+
raise 'Catalog was not built' if @builddir.nil?
|
73
|
+
@builddir.tempdir
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
# Private method: Clean up a checkout directory, if it exists
|
79
|
+
def cleanup_checkout_dir(checkout_dir, logger)
|
80
|
+
return unless File.directory?(checkout_dir)
|
81
|
+
logger.debug("Cleaning up temporary directory #{checkout_dir}")
|
82
|
+
FileUtils.remove_entry_secure checkout_dir
|
83
|
+
end
|
84
|
+
|
85
|
+
# Private method: Bootstrap a directory
|
86
|
+
def bootstrap(logger)
|
87
|
+
return if @builddir
|
88
|
+
|
89
|
+
# Fill options for creating and populating the temporary directory
|
90
|
+
tmphash = @opts.dup
|
91
|
+
|
92
|
+
# Bootstrap directory if needed
|
93
|
+
if !@opts[:bootstrapped_dir].nil?
|
94
|
+
raise Errno::ENOENT, "Invalid dir #{@opts[:bootstrapped_dir]}" unless File.directory?(@opts[:bootstrapped_dir])
|
95
|
+
tmphash[:basedir] = @opts[:bootstrapped_dir]
|
96
|
+
elsif @opts[:branch] == '.'
|
97
|
+
tmphash[:basedir] = @opts[:basedir]
|
98
|
+
else
|
99
|
+
checkout_dir = Dir.mktmpdir
|
100
|
+
at_exit { cleanup_checkout_dir(checkout_dir, logger) }
|
101
|
+
tmphash[:basedir] = checkout_dir
|
102
|
+
OctocatalogDiff::CatalogUtil::Bootstrap.bootstrap_directory(@opts.merge(path: checkout_dir), logger)
|
103
|
+
end
|
104
|
+
|
105
|
+
# Create and populate the temporary directory
|
106
|
+
@builddir ||= OctocatalogDiff::CatalogUtil::BuildDir.new(tmphash, logger)
|
107
|
+
end
|
108
|
+
|
109
|
+
# Private method: Build catalog by running Puppet
|
110
|
+
# @param logger [Logger] Logger object
|
111
|
+
def build_catalog(logger = nil)
|
112
|
+
return nil unless @catalog.nil? && @error_message.nil?
|
113
|
+
bootstrap(logger)
|
114
|
+
result = run_puppet(logger)
|
115
|
+
@retries = result[:retries]
|
116
|
+
if (result[:exitcode]).zero?
|
117
|
+
begin
|
118
|
+
@catalog = ::JSON.parse(result[:stdout])
|
119
|
+
@catalog_json = result[:stdout]
|
120
|
+
@error_message = nil
|
121
|
+
rescue ::JSON::ParserError => exc
|
122
|
+
@catalog = nil
|
123
|
+
@catalog_json = nil
|
124
|
+
@error_message = "Catalog has invalid JSON: #{exc.message}"
|
125
|
+
end
|
126
|
+
else
|
127
|
+
@error_message = result[:stderr]
|
128
|
+
@catalog = nil
|
129
|
+
@catalog_json = nil
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
# Get the command to compile the catalog
|
134
|
+
# @return [String] Puppet command line
|
135
|
+
def puppet_command(options = @opts)
|
136
|
+
return @puppet_command if @puppet_command
|
137
|
+
raise ArgumentError, '"puppet_binary" was not passed to OctocatalogDiff::Catalog::Computed' unless @puppet_binary
|
138
|
+
command_opts = options.merge(
|
139
|
+
node: @node,
|
140
|
+
compilation_dir: @builddir.tempdir,
|
141
|
+
parser: options.fetch(:parser, :default),
|
142
|
+
puppet_binary: @puppet_binary,
|
143
|
+
fact_file: @builddir.fact_file,
|
144
|
+
dir: @builddir.tempdir,
|
145
|
+
enc: @builddir.enc
|
146
|
+
)
|
147
|
+
command = OctocatalogDiff::CatalogUtil::Command.new(command_opts)
|
148
|
+
@puppet_command = command.puppet_command
|
149
|
+
end
|
150
|
+
|
151
|
+
# Private method: Actually execute puppet
|
152
|
+
# @return [Hash] { stdout, stderr, exitcode }
|
153
|
+
def exec_puppet
|
154
|
+
# This is the environment provided to the puppet command.
|
155
|
+
env = {
|
156
|
+
'HOME' => ENV['HOME'],
|
157
|
+
'PATH' => ENV['PATH'],
|
158
|
+
'PWD' => @builddir.tempdir
|
159
|
+
}
|
160
|
+
@pass_env_vars.each { |var| env[var] ||= ENV[var] }
|
161
|
+
out, err, status = Open3.capture3(env, puppet_command, unsetenv_others: true, chdir: @builddir.tempdir)
|
162
|
+
{
|
163
|
+
stdout: out,
|
164
|
+
stderr: err,
|
165
|
+
exitcode: status.exitstatus
|
166
|
+
}
|
167
|
+
end
|
168
|
+
|
169
|
+
# Private method: Runs puppet on the command line to compile the catalog
|
170
|
+
# Exit code is 0 if catalog generation was successful, non-zero otherwise.
|
171
|
+
# @param logger [Logger] Logger object
|
172
|
+
# @return [Hash] { stdout: <catalog as JSON>, stderr: <error messages>, exitcode: <hopefully 0> }
|
173
|
+
def run_puppet(logger)
|
174
|
+
# Run 'cmd' with environment 'env' from directory 'dir'
|
175
|
+
# First line of a successful result needs to be stripped off. It will look like:
|
176
|
+
# Notice: Compiled catalog for xxx in environment production in 27.88 seconds
|
177
|
+
retval = {}
|
178
|
+
0.upto(@retry_failed_catalog) do |retry_num|
|
179
|
+
@retries = retry_num
|
180
|
+
time_begin = Time.now
|
181
|
+
logger.debug("(#{@tag}) Try #{1 + retry_num} executing Puppet #{puppet_version}: #{puppet_command}")
|
182
|
+
result = exec_puppet
|
183
|
+
|
184
|
+
# Success
|
185
|
+
if (result[:exitcode]).zero?
|
186
|
+
logger.debug("(#{@tag}) Catalog succeeded on try #{1 + retry_num} in #{Time.now - time_begin} seconds")
|
187
|
+
first_brace = result[:stdout].index('{') || 0
|
188
|
+
retval = {
|
189
|
+
stdout: result[:stdout][first_brace..-1],
|
190
|
+
stderr: nil,
|
191
|
+
exitcode: 0,
|
192
|
+
retries: retry_num
|
193
|
+
}
|
194
|
+
break
|
195
|
+
end
|
196
|
+
|
197
|
+
# Failure
|
198
|
+
logger.debug("(#{@tag}) Catalog failed on try #{1 + retry_num} in #{Time.now - time_begin} seconds")
|
199
|
+
retval = result.merge(retries: retry_num)
|
200
|
+
end
|
201
|
+
retval
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|