octocatalog-diff 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.version +1 -0
- data/LICENSE +20 -0
- data/README.md +82 -0
- data/bin/octocatalog-diff +75 -0
- data/doc/advanced-bootstrap.md +33 -0
- data/doc/advanced-cache-dir.md +24 -0
- data/doc/advanced-catalog-only.md +37 -0
- data/doc/advanced-ci.md +13 -0
- data/doc/advanced-dynamic-ignores.md +123 -0
- data/doc/advanced-future-parser.md +11 -0
- data/doc/advanced-ignores.md +224 -0
- data/doc/advanced-output-formats.md +96 -0
- data/doc/advanced-output-hacks.md +45 -0
- data/doc/advanced-override-facts.md +67 -0
- data/doc/advanced-pe-enc.md +52 -0
- data/doc/advanced-puppet-master.md +50 -0
- data/doc/advanced-puppet-versions.md +9 -0
- data/doc/advanced-storeconfigs.md +72 -0
- data/doc/advanced-using-without-git.md +15 -0
- data/doc/advanced.md +43 -0
- data/doc/basic.md +70 -0
- data/doc/configuration-enc.md +69 -0
- data/doc/configuration-hiera.md +103 -0
- data/doc/configuration-puppetdb.md +49 -0
- data/doc/configuration.md +51 -0
- data/doc/dev/README.md +1 -0
- data/doc/dev/coverage.md +34 -0
- data/doc/dev/how-to-add-options.md +83 -0
- data/doc/dev/integration-tests.md +63 -0
- data/doc/dev/releasing.md +19 -0
- data/doc/installation.md +49 -0
- data/doc/limitations.md +34 -0
- data/doc/optionsref.md +947 -0
- data/doc/requirements.md +16 -0
- data/doc/roadmap.md +26 -0
- data/doc/similar.md +17 -0
- data/doc/troubleshooting.md +54 -0
- data/lib/octocatalog-diff.rb +12 -0
- data/lib/octocatalog-diff/bootstrap.rb +53 -0
- data/lib/octocatalog-diff/catalog-diff/cli.rb +205 -0
- data/lib/octocatalog-diff/catalog-diff/cli/catalogs.rb +240 -0
- data/lib/octocatalog-diff/catalog-diff/cli/diffs.rb +145 -0
- data/lib/octocatalog-diff/catalog-diff/cli/helpers/fact_override.rb +99 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options.rb +173 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/basedir.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_environment.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_script.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_then_exit.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrapped_dirs.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/cached_master_dir.rb +21 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/catalog_only.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/color.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/compare_file_text.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/debug.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_datatype_changes.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_detail_add.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_source_file_line.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/enc.rb +31 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/existing_catalogs.rb +25 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/fact_file.rb +23 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/fact_override.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/facts_terminus.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/from_puppetdb.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/header.rb +24 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_config.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_path_strip.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hostname.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore.rb +24 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_attr.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_tags.rb +23 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/include_tags.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/master_cache_branch.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/output_file.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/output_format.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/parallel.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/parser.rb +48 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pass_env_vars.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_ca.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_cert.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_key.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token_file.rb +17 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_url.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_binary.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_api_version.rb +20 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_ca.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_cert.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_key.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_ca.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_cert.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_key.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password_file.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_url.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/quiet.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/retry_failed_catalog.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/safe_to_delete_cached_master_dir.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/storeconfigs.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/suppress_absent_file_details.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/to_from_branch.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/printer.rb +52 -0
- data/lib/octocatalog-diff/catalog-diff/differ.rb +615 -0
- data/lib/octocatalog-diff/catalog-diff/display.rb +125 -0
- data/lib/octocatalog-diff/catalog-diff/display/json.rb +25 -0
- data/lib/octocatalog-diff/catalog-diff/display/text.rb +452 -0
- data/lib/octocatalog-diff/catalog-util/bootstrap.rb +145 -0
- data/lib/octocatalog-diff/catalog-util/builddir.rb +289 -0
- data/lib/octocatalog-diff/catalog-util/cached_master_directory.rb +169 -0
- data/lib/octocatalog-diff/catalog-util/command.rb +96 -0
- data/lib/octocatalog-diff/catalog-util/enc.rb +77 -0
- data/lib/octocatalog-diff/catalog-util/enc/noop.rb +22 -0
- data/lib/octocatalog-diff/catalog-util/enc/pe.rb +99 -0
- data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb +61 -0
- data/lib/octocatalog-diff/catalog-util/enc/script.rb +88 -0
- data/lib/octocatalog-diff/catalog-util/facts.rb +89 -0
- data/lib/octocatalog-diff/catalog-util/fileresources.rb +83 -0
- data/lib/octocatalog-diff/catalog-util/git.rb +65 -0
- data/lib/octocatalog-diff/catalog.rb +209 -0
- data/lib/octocatalog-diff/catalog/computed.rb +205 -0
- data/lib/octocatalog-diff/catalog/json.rb +30 -0
- data/lib/octocatalog-diff/catalog/noop.rb +19 -0
- data/lib/octocatalog-diff/catalog/puppetdb.rb +82 -0
- data/lib/octocatalog-diff/catalog/puppetmaster.rb +121 -0
- data/lib/octocatalog-diff/external/pson/LICENSE +17 -0
- data/lib/octocatalog-diff/external/pson/README.md +20 -0
- data/lib/octocatalog-diff/external/pson/common.rb +370 -0
- data/lib/octocatalog-diff/external/pson/pure.rb +15 -0
- data/lib/octocatalog-diff/external/pson/pure/generator.rb +395 -0
- data/lib/octocatalog-diff/external/pson/pure/parser.rb +307 -0
- data/lib/octocatalog-diff/external/pson/version.rb +8 -0
- data/lib/octocatalog-diff/facts.rb +125 -0
- data/lib/octocatalog-diff/facts/json.rb +20 -0
- data/lib/octocatalog-diff/facts/puppetdb.rb +59 -0
- data/lib/octocatalog-diff/facts/yaml.rb +29 -0
- data/lib/octocatalog-diff/puppetdb.rb +163 -0
- data/lib/octocatalog-diff/util/colored.rb +20 -0
- data/lib/octocatalog-diff/util/httparty.rb +158 -0
- data/lib/octocatalog-diff/util/parallel.rb +170 -0
- data/lib/octocatalog-diff/util/puppetversion.rb +24 -0
- data/lib/octocatalog-diff/version.rb +7 -0
- metadata +386 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
require_relative 'facts/json'
|
|
2
|
+
require_relative 'facts/yaml'
|
|
3
|
+
require_relative 'facts/puppetdb'
|
|
4
|
+
require_relative 'external/pson/pure'
|
|
5
|
+
|
|
6
|
+
module OctocatalogDiff
|
|
7
|
+
# Deal with facts in all forms, including:
|
|
8
|
+
# - In existing YAML files
|
|
9
|
+
# - In existing JSON files
|
|
10
|
+
# - Retrieved dynamically from PuppetDB
|
|
11
|
+
class Facts
|
|
12
|
+
# Constructor
|
|
13
|
+
# @param options [Hash] Initialization options, varies per backend
|
|
14
|
+
def initialize(options = {}, facts = nil)
|
|
15
|
+
@node = options.fetch(:node, '')
|
|
16
|
+
@timestamp = false
|
|
17
|
+
@options = options.dup
|
|
18
|
+
if facts
|
|
19
|
+
@facts = {}
|
|
20
|
+
facts.each { |k, v| @facts[k] = v.dup }
|
|
21
|
+
else
|
|
22
|
+
case options[:backend]
|
|
23
|
+
when :json
|
|
24
|
+
@orig_facts = OctocatalogDiff::Facts::JSON.fact_retriever(options, @node)
|
|
25
|
+
when :yaml
|
|
26
|
+
@orig_facts = OctocatalogDiff::Facts::Yaml.fact_retriever(options, @node)
|
|
27
|
+
when :puppetdb
|
|
28
|
+
@orig_facts = OctocatalogDiff::Facts::PuppetDB.fact_retriever(options, @node)
|
|
29
|
+
else
|
|
30
|
+
raise ArgumentError, 'Invalid fact source backend'
|
|
31
|
+
end
|
|
32
|
+
@facts = {}
|
|
33
|
+
@orig_facts.each { |k, v| @facts[k] = v.dup }
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def dup
|
|
38
|
+
self.class.new(@options, @orig_facts)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Facts - returned the 'cleansed' facts.
|
|
42
|
+
# Clean up facts by setting 'name' to the node if given, and deleting _timestamp and expiration
|
|
43
|
+
# which may cause Puppet catalog compilation to fail if the facts are old.
|
|
44
|
+
# @param node [String] Node name to override returned facts
|
|
45
|
+
# @return [Hash] Facts hash { 'name' => '...', 'values' => { ... } }
|
|
46
|
+
def facts(node = @node, timestamp = false)
|
|
47
|
+
raise "Expected @facts to be a hash but it is a #{@facts.class}" unless @facts.is_a?(Hash)
|
|
48
|
+
raise "Expected @facts['values'] to be a hash but it is a #{@facts['values'].class}" unless @facts['values'].is_a?(Hash)
|
|
49
|
+
f = @facts.dup
|
|
50
|
+
f['name'] = node unless node.nil? || node.empty?
|
|
51
|
+
f['values'].delete('_timestamp')
|
|
52
|
+
f.delete('expiration')
|
|
53
|
+
if timestamp
|
|
54
|
+
f['timestamp'] = Time.now.to_s
|
|
55
|
+
f['values']['timestamp'] = f['timestamp']
|
|
56
|
+
f['expiration'] = (Time.now + (24 * 60 * 60)).to_s
|
|
57
|
+
end
|
|
58
|
+
f
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Facts - Fudge the timestamp to right now and add include it in the facts when returned
|
|
62
|
+
# @return self
|
|
63
|
+
def fudge_timestamp
|
|
64
|
+
@timestamp = true
|
|
65
|
+
self
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Facts - remove one or more facts from the list.
|
|
69
|
+
# @param remove [String|Array<String>] Fact(s) to remove
|
|
70
|
+
# @return self
|
|
71
|
+
def without(remove)
|
|
72
|
+
r = remove.is_a?(Array) ? remove : [remove]
|
|
73
|
+
obj = dup
|
|
74
|
+
r.each { |fact| obj.remove_fact_from_list(fact) }
|
|
75
|
+
obj
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Facts - remove a fact from the list
|
|
79
|
+
# @param remove [String] Fact to remove
|
|
80
|
+
def remove_fact_from_list(remove)
|
|
81
|
+
@facts['values'].delete(remove)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Turn hash of facts into appropriate YAML for Puppet
|
|
85
|
+
# @param node [String] Node name to override returned facts
|
|
86
|
+
# @return [String] Puppet-compatible YAML facts
|
|
87
|
+
def facts_to_yaml(node = @node)
|
|
88
|
+
# Add the header that Puppet needs to treat this as facts. Save the results
|
|
89
|
+
# as a string in the option.
|
|
90
|
+
f = facts(node)
|
|
91
|
+
fact_file = f.to_yaml.split(/\n/)
|
|
92
|
+
fact_file[0] = '--- !ruby/object:Puppet::Node::Facts' if fact_file[0] =~ /^---/
|
|
93
|
+
fact_file.join("\n")
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Turn hash of facts into appropriate YAML for Puppet
|
|
97
|
+
# @param node [String] Node name to override returned facts
|
|
98
|
+
# @return [String] Puppet-compatible YAML facts
|
|
99
|
+
def to_pson
|
|
100
|
+
PSON.generate(facts)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Get the current value of a particular fact
|
|
104
|
+
# @param key [String] Fact key to override
|
|
105
|
+
# @return [?] Value for fact
|
|
106
|
+
def fact(key)
|
|
107
|
+
@facts['values'][key]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Override a particular fact
|
|
111
|
+
# @param key [String] Fact key to override
|
|
112
|
+
# @param value [?] Value for fact
|
|
113
|
+
def override(key, value)
|
|
114
|
+
if value.nil?
|
|
115
|
+
@facts['values'].delete(key)
|
|
116
|
+
else
|
|
117
|
+
@facts['values'][key] = value
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Separate classes to handle errors we throw explicitly
|
|
122
|
+
class FactSourceError < RuntimeError; end
|
|
123
|
+
class FactRetrievalError < RuntimeError; end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
require_relative '../facts'
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module OctocatalogDiff
|
|
6
|
+
class Facts
|
|
7
|
+
# Deal with facts in JSON files
|
|
8
|
+
class JSON
|
|
9
|
+
# @param options [Hash] Options hash specifically for this fact type.
|
|
10
|
+
# - :fact_file_string [String] => Fact data as a string
|
|
11
|
+
# @param node [String] Node name (overrides node name from fact data)
|
|
12
|
+
# @return [Hash] Facts
|
|
13
|
+
def self.fact_retriever(options = {}, node = '')
|
|
14
|
+
facts = ::JSON.parse(options.fetch(:fact_file_string))
|
|
15
|
+
node = facts.fetch('fqdn', 'unknown.node') if node.empty?
|
|
16
|
+
{ 'name' => node, 'values' => facts }
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
require_relative '../facts'
|
|
2
|
+
require_relative '../puppetdb'
|
|
3
|
+
require 'yaml'
|
|
4
|
+
|
|
5
|
+
module OctocatalogDiff
|
|
6
|
+
class Facts
|
|
7
|
+
# Deal with facts in PuppetDB
|
|
8
|
+
class PuppetDB
|
|
9
|
+
# Retrieve facts from PuppetDB for a specified node.
|
|
10
|
+
# @param :puppetdb_url [String|Array] => URL to PuppetDB
|
|
11
|
+
# @param :retry [Fixnum] => Retry after timeout (default 0 retries, can be more)
|
|
12
|
+
# @param node [String] Node name. (REQUIRED for PuppetDB fact source)
|
|
13
|
+
# @return [Hash] Facts
|
|
14
|
+
def self.fact_retriever(options = {}, node)
|
|
15
|
+
# Set up some variables from options
|
|
16
|
+
raise ArgumentError, 'puppetdb_url is required' unless options[:puppetdb_url].is_a?(String)
|
|
17
|
+
raise ArgumentError, 'node must be a non-empty string' unless node.is_a?(String) && node != ''
|
|
18
|
+
uri = "/pdb/query/v4/nodes/#{node}/facts"
|
|
19
|
+
retries = options.fetch(:retry, 0).to_i
|
|
20
|
+
|
|
21
|
+
# Construct puppetdb object and options
|
|
22
|
+
opts = options.merge(timeout: 5)
|
|
23
|
+
puppetdb = OctocatalogDiff::PuppetDB.new(opts)
|
|
24
|
+
|
|
25
|
+
# Use OctocatalogDiff::PuppetDB to pull facts
|
|
26
|
+
exception_class = nil
|
|
27
|
+
exception_message = nil
|
|
28
|
+
obj_to_return = nil
|
|
29
|
+
(retries + 1).times do
|
|
30
|
+
begin
|
|
31
|
+
result = puppetdb.get(uri)
|
|
32
|
+
facts = {}
|
|
33
|
+
result.map { |x| facts[x['name']] = x['value'] }
|
|
34
|
+
if facts.empty?
|
|
35
|
+
message = "Unable to retrieve facts for node #{node} from PuppetDB (empty or nil)!"
|
|
36
|
+
raise OctocatalogDiff::Facts::FactRetrievalError, message
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Create a structure compatible with YAML fact files.
|
|
40
|
+
obj_to_return = { 'name' => node, 'values' => {} }
|
|
41
|
+
facts.each { |k, v| obj_to_return['values'][k.sub(/^::/, '')] = v }
|
|
42
|
+
break # Not return, to avoid LocalJumpError in Ruby 2.2
|
|
43
|
+
rescue OctocatalogDiff::PuppetDB::ConnectionError => exc
|
|
44
|
+
exception_class = OctocatalogDiff::Facts::FactSourceError
|
|
45
|
+
exception_message = "Fact retrieval failed (#{exc.class}) (#{exc.message})"
|
|
46
|
+
rescue OctocatalogDiff::PuppetDB::NotFoundError => exc
|
|
47
|
+
exception_class = OctocatalogDiff::Facts::FactRetrievalError
|
|
48
|
+
exception_message = "Node #{node} not found in PuppetDB (#{exc.message})"
|
|
49
|
+
rescue OctocatalogDiff::PuppetDB::PuppetDBError => exc
|
|
50
|
+
exception_class = OctocatalogDiff::Facts::FactRetrievalError
|
|
51
|
+
exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})"
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
return obj_to_return unless obj_to_return.nil?
|
|
55
|
+
raise exception_class, exception_message
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
require_relative '../facts'
|
|
2
|
+
require 'yaml'
|
|
3
|
+
|
|
4
|
+
module OctocatalogDiff
|
|
5
|
+
class Facts
|
|
6
|
+
# Deal with facts in YAML files
|
|
7
|
+
class Yaml
|
|
8
|
+
# Manipulate a YAML file so it can be parsed and return the facts as a hash.
|
|
9
|
+
# If we leave it as Puppet::Node::Facts then it will require us to load puppet
|
|
10
|
+
# gems in order to parse it, and that's too heavy for simple fact retrieval.
|
|
11
|
+
# @param options [Hash] Options hash specifically for this fact type.
|
|
12
|
+
# - :fact_file_string [String] => Fact data as a string
|
|
13
|
+
# @param node [String] Node name (overrides node name from fact data)
|
|
14
|
+
# @return [Hash] Facts
|
|
15
|
+
def self.fact_retriever(options = {}, node = '')
|
|
16
|
+
fact_file_string = options.fetch(:fact_file_string)
|
|
17
|
+
|
|
18
|
+
# Touch up the first line before parsing.
|
|
19
|
+
fact_file_data = fact_file_string.split(/\n/)
|
|
20
|
+
fact_file_data[0] = '---' if fact_file_data[0] =~ /^---/
|
|
21
|
+
|
|
22
|
+
# Load and return the parsed fact file.
|
|
23
|
+
result = YAML.load(fact_file_data.join("\n"))
|
|
24
|
+
result['name'] = node unless node == ''
|
|
25
|
+
result
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
require_relative 'util/httparty'
|
|
2
|
+
|
|
3
|
+
require 'uri'
|
|
4
|
+
|
|
5
|
+
# Redefine constants to match PuppetDB defaults.
|
|
6
|
+
# This code avoids warnings about redefining constants.
|
|
7
|
+
URI::HTTP.send(:remove_const, :DEFAULT_PORT) if URI::HTTP.const_defined?(:DEFAULT_PORT)
|
|
8
|
+
URI::HTTP.const_set(:DEFAULT_PORT, 8080)
|
|
9
|
+
URI::HTTPS.send(:remove_const, :DEFAULT_PORT) if URI::HTTPS.const_defined?(:DEFAULT_PORT)
|
|
10
|
+
URI::HTTPS.const_set(:DEFAULT_PORT, 8081)
|
|
11
|
+
|
|
12
|
+
module OctocatalogDiff
|
|
13
|
+
# A standard way to connect to PuppetDB from the various scripts in this repository.
|
|
14
|
+
class PuppetDB
|
|
15
|
+
# This class raises errors for certain handled problems
|
|
16
|
+
class NotFoundError < RuntimeError; end
|
|
17
|
+
class PuppetDBError < RuntimeError; end
|
|
18
|
+
class ConnectionError < RuntimeError; end
|
|
19
|
+
|
|
20
|
+
# Allow connections to be read (used in tests for now)
|
|
21
|
+
attr_reader :connections
|
|
22
|
+
|
|
23
|
+
# Constructor - will construct connection parameters from a variety
|
|
24
|
+
# of sources, including arguments and environment variables. Supported
|
|
25
|
+
# environment variables:
|
|
26
|
+
# PUPPETDB_URL
|
|
27
|
+
# PUPPETDB_HOST [+ PUPPETDB_PORT] [+ PUPPETDB_SSL]
|
|
28
|
+
#
|
|
29
|
+
# Order of precedence:
|
|
30
|
+
# 1. :puppetdb_url argument (String or Array<String>)
|
|
31
|
+
# 2. :puppetdb_host argument [+ :puppetdb_port] [+ :puppetdb_ssl]
|
|
32
|
+
# 3. ENV['PUPPETDB_URL']
|
|
33
|
+
# 4. ENV['PUPPETDB_HOST'] [+ ENV['PUPPETDB_PORT']], [+ ENV['PUPPETDB_SSL']]
|
|
34
|
+
# When it finds one of these, it stops and does not process any others.
|
|
35
|
+
#
|
|
36
|
+
# When :puppetdb_url is an array, all given URLs are tried, in random order,
|
|
37
|
+
# until a connection succeeds. If a connection succeeds, any errors from previously
|
|
38
|
+
# failed connections are suppressed.
|
|
39
|
+
#
|
|
40
|
+
# Supported arguments:
|
|
41
|
+
# @param :puppetdb_url [String or Array<String>] PuppetDB URL(s) to try in random order
|
|
42
|
+
# @param :puppetdb_host [String] PuppetDB hostname, when constructing a URL
|
|
43
|
+
# @param :puppetdb_port [Fixnum] Port number, defaults to 8080 (non-SSL) or 8081 (SSL)
|
|
44
|
+
# @param :puppetdb_ssl [Boolean] defaults to true, because you should use SSL
|
|
45
|
+
# @param :puppetdb_ssl_ca [String] Path to file containing CA certificate
|
|
46
|
+
# @param :puppetdb_ssl_verify [Boolean] Override the CA verification setting guessed from parameters
|
|
47
|
+
# @param :puppetdb_ssl_client_pem [String] PEM-encoded client key and certificate
|
|
48
|
+
# @param :puppetdb_ssl_client_p12 [String] pkcs12-encoded client key and certificate
|
|
49
|
+
# @param :puppetdb_ssl_client_password [String] Path to file containing password for SSL client key (any format)
|
|
50
|
+
# @param :puppetdb_ssl_client_auth [Boolean] Override the client-auth that is guessed from parameters
|
|
51
|
+
# @param :timeout [Fixnum] Connection timeout for PuppetDB (default=10)
|
|
52
|
+
def initialize(options = {})
|
|
53
|
+
@connections =
|
|
54
|
+
if options.key?(:puppetdb_url)
|
|
55
|
+
urls = options[:puppetdb_url].is_a?(Array) ? options[:puppetdb_url] : [options[:puppetdb_url]]
|
|
56
|
+
urls.map { |url| parse_url(url) }
|
|
57
|
+
elsif options.key?(:puppetdb_host)
|
|
58
|
+
is_ssl = options.fetch(:puppetdb_ssl, true)
|
|
59
|
+
default_port = is_ssl ? URI::HTTPS::DEFAULT_PORT : URI::HTTP::DEFAULT_PORT
|
|
60
|
+
port = options.fetch(:puppetdb_port, default_port).to_i
|
|
61
|
+
[{ ssl: is_ssl, host: options[:puppetdb_host], port: port }]
|
|
62
|
+
elsif ENV['PUPPETDB_URL'] && !ENV['PUPPETDB_URL'].empty?
|
|
63
|
+
[parse_url(ENV['PUPPETDB_URL'])]
|
|
64
|
+
elsif ENV['PUPPETDB_HOST'] && !ENV['PUPPETDB_HOST'].empty?
|
|
65
|
+
# Because environment variables are strings...
|
|
66
|
+
# This will get the env var and see if it equals 'true'; the result
|
|
67
|
+
# of this == comparison is the true/false boolean we need.
|
|
68
|
+
is_ssl = ENV.fetch('PUPPETDB_SSL', 'true') == 'true'
|
|
69
|
+
default_port = is_ssl ? URI::HTTPS::DEFAULT_PORT : URI::HTTP::DEFAULT_PORT
|
|
70
|
+
port = ENV.fetch('PUPPETDB_PORT', default_port).to_i
|
|
71
|
+
[{ ssl: is_ssl, host: ENV['PUPPETDB_HOST'], port: port }]
|
|
72
|
+
else
|
|
73
|
+
[]
|
|
74
|
+
end
|
|
75
|
+
@timeout = options.fetch(:timeout, 10)
|
|
76
|
+
@options = options
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Wrapper around the httparty call in the private _get method.
|
|
80
|
+
# Returns the parsed result of getting the provided URL and returns
|
|
81
|
+
# a friendlier error message if there are network connection problems
|
|
82
|
+
# to PuppetDB.
|
|
83
|
+
# @param path [String] Path portion of the URL
|
|
84
|
+
# @return [Object] Parsed reply from PuppetDB as an object
|
|
85
|
+
def get(path)
|
|
86
|
+
_get(path)
|
|
87
|
+
rescue Net::OpenTimeout, Errno::ECONNREFUSED => exc
|
|
88
|
+
raise ConnectionError, "#{exc.class} connecting to PuppetDB (need VPN on?): #{exc.message}"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
# HTTP(S) Query - will attempt to retrieve URL from each connection
|
|
94
|
+
# @param path [String] Path portion of the URL
|
|
95
|
+
# @return [String] Parsed response
|
|
96
|
+
def _get(path)
|
|
97
|
+
# You need at least one connection or else this can't do anything
|
|
98
|
+
raise ArgumentError, 'No PuppetDB connections configured' if @connections.empty?
|
|
99
|
+
|
|
100
|
+
# Keep track of the latest exception seen
|
|
101
|
+
exc = nil
|
|
102
|
+
|
|
103
|
+
# Try each connection in random order. This will return the first successful
|
|
104
|
+
# response, and try the next connection if there's an error. Once it's out of
|
|
105
|
+
# connections to try it will raise the last exception encountered.
|
|
106
|
+
@connections.shuffle.each do |connection|
|
|
107
|
+
complete_url = [
|
|
108
|
+
connection[:ssl] ? 'https://' : 'http://',
|
|
109
|
+
connection[:host],
|
|
110
|
+
':',
|
|
111
|
+
connection[:port],
|
|
112
|
+
path
|
|
113
|
+
].join('')
|
|
114
|
+
|
|
115
|
+
begin
|
|
116
|
+
more_options = { headers: { 'Accept' => 'application/json' }, timeout: @timeout }
|
|
117
|
+
response = OctocatalogDiff::Util::HTTParty.get(complete_url, @options.merge(more_options), 'puppetdb')
|
|
118
|
+
|
|
119
|
+
# Handle all non-200's from PuppetDB
|
|
120
|
+
unless response[:code] == 200
|
|
121
|
+
raise NotFoundError, "404 - #{response[:error]}" if response[:code] == 404
|
|
122
|
+
raise PuppetDBError, "#{response[:code]} - #{response[:error]}"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# PuppetDB can return 'Not Found' as a string with a 200 response code
|
|
126
|
+
raise NotFoundError, '404 - Not Found' if response[:body] == 'Not Found'
|
|
127
|
+
|
|
128
|
+
# PuppetDB can also return an error message in a 200; we'll call this a 500
|
|
129
|
+
if response.key?(:error)
|
|
130
|
+
raise PuppetDBError, "500 - #{response[:error]}"
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# If we get here without raising an error, it will fall out of the begin/rescue
|
|
134
|
+
# with 'result' non-nil, and 'result' will then get returned.
|
|
135
|
+
raise "Unparseable response from puppetdb: '#{response.inspect}'" unless response[:parsed]
|
|
136
|
+
result = response[:parsed]
|
|
137
|
+
rescue => exc
|
|
138
|
+
# Set response to nil so the loop repeats itself if there are retries left.
|
|
139
|
+
# Also sets 'exc' to the most recent exception, in case all retries are
|
|
140
|
+
# exhausted and this exception has to be raised.
|
|
141
|
+
result = nil
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# If the previous query didn't error, return result
|
|
145
|
+
return result unless result.nil?
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# At this point no query has succeeded, so raise the last error encountered.
|
|
149
|
+
raise exc
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Parse a URL to determine hostname, port number, and whether or not SSL is used.
|
|
153
|
+
# @param url [String] URL to parse
|
|
154
|
+
# @return [Hash] { ssl: true/false, host: <String>, port: <Fixnum> }
|
|
155
|
+
def parse_url(url)
|
|
156
|
+
uri = URI(url)
|
|
157
|
+
raise ArgumentError, "URL #{url} has invalid scheme" unless uri.scheme =~ /^https?$/
|
|
158
|
+
{ ssl: uri.scheme == 'https', host: uri.host, port: uri.port }
|
|
159
|
+
rescue URI::InvalidURIError => exc
|
|
160
|
+
raise exc.class, "Invalid URL: #{url} (#{exc.message})"
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Create colorizing methods in the 'String' class, but only if 'colors_enabled'
|
|
2
|
+
# has been set.
|
|
3
|
+
class String
|
|
4
|
+
COLORS = {
|
|
5
|
+
'red' => 31,
|
|
6
|
+
'green' => 32,
|
|
7
|
+
'yellow' => 33,
|
|
8
|
+
'cyan' => 36
|
|
9
|
+
}.freeze
|
|
10
|
+
|
|
11
|
+
COLORS.each do |color, _value|
|
|
12
|
+
define_method(color) do
|
|
13
|
+
@@colors_enabled ? "\e[0;#{COLORS[color]};49m#{self}\e[0m" : self
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.colors_enabled=(value)
|
|
18
|
+
@@colors_enabled = value # rubocop:disable Style/ClassVars
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
require 'httparty'
|
|
2
|
+
require 'json'
|
|
3
|
+
require_relative '../external/pson/pure'
|
|
4
|
+
|
|
5
|
+
module OctocatalogDiff
|
|
6
|
+
module Util
|
|
7
|
+
# This is a wrapper around some common actions that octocatalog-diff does when preparing to talk
|
|
8
|
+
# to a web server using 'httparty'.
|
|
9
|
+
class HTTParty
|
|
10
|
+
# Wrap the 'get' method in httparty with SSL options
|
|
11
|
+
# @param url [String] URL to retrieve
|
|
12
|
+
# @param options [Hash] Options
|
|
13
|
+
# @param ssl_prefix [String] Strip "#{prefix}_" from the start of SSL options to generalize them
|
|
14
|
+
# @return [Hash] HTTParty response and codes
|
|
15
|
+
def self.get(url, options = {}, ssl_prefix = nil)
|
|
16
|
+
httparty_response_parse(::HTTParty.get(url, options.merge(wrap_ssl_options(options, ssl_prefix))))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Wrap the 'post' method in httparty with SSL options
|
|
20
|
+
# @param url [String] URL to retrieve
|
|
21
|
+
# @param options [Hash] Options
|
|
22
|
+
# @param post_body [String] Test to POST
|
|
23
|
+
# @param ssl_prefix [String] Strip "#{prefix}_" from the start of SSL options to generalize them
|
|
24
|
+
# @return [Hash] HTTParty response and codes
|
|
25
|
+
def self.post(url, options, post_body, ssl_prefix)
|
|
26
|
+
opts = options.merge(wrap_ssl_options(options, ssl_prefix))
|
|
27
|
+
httparty_response_parse(::HTTParty.post(url, opts.merge(body: post_body)))
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Common parser for HTTParty response
|
|
31
|
+
# @param response [HTTParty response object] HTTParty response object
|
|
32
|
+
# @return [Hash] HTTParty parsed response and codes
|
|
33
|
+
def self.httparty_response_parse(response)
|
|
34
|
+
# Handle HTTP errors
|
|
35
|
+
unless response.code == 200
|
|
36
|
+
begin
|
|
37
|
+
b = JSON.parse(response.body)
|
|
38
|
+
errormessage = b['error'] if b.is_a?(Hash) && b.key?('error')
|
|
39
|
+
rescue JSON::ParserError
|
|
40
|
+
errormessage = response.body
|
|
41
|
+
ensure
|
|
42
|
+
errormessage ||= response.body
|
|
43
|
+
end
|
|
44
|
+
return { code: response.code, body: response.body, error: errormessage }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Handle success
|
|
48
|
+
if response.headers.key?('content-type')
|
|
49
|
+
if response.headers['content-type'] =~ %r{/json}
|
|
50
|
+
begin
|
|
51
|
+
return { code: 200, body: response.body, parsed: JSON.parse(response.body) }
|
|
52
|
+
rescue JSON::ParserError => exc
|
|
53
|
+
return { code: 500, body: response.body, error: "JSON parse error: #{exc.message}" }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
if response.headers['content-type'] =~ %r{/pson}
|
|
57
|
+
begin
|
|
58
|
+
return { code: 200, body: response.body, parsed: PSON.parse(response.body) }
|
|
59
|
+
rescue PSON::ParserError => exc
|
|
60
|
+
return { code: 500, body: response.body, error: "PSON parse error: #{exc.message}" }
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
return { code: 500, body: response.body, error: "Don't know how to parse: #{response.headers['content-type']}" }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Return raw output
|
|
67
|
+
{ code: response.code, body: response.body }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Wrap context-specific options into generally named options for the other methods in this class
|
|
71
|
+
# @param options [Hash] Hash of all options
|
|
72
|
+
# @param prefix [String] Prefix to strip from SSL options
|
|
73
|
+
# @return [Hash] SSL options generally named
|
|
74
|
+
def self.wrap_ssl_options(options, prefix)
|
|
75
|
+
return {} unless prefix
|
|
76
|
+
result = {}
|
|
77
|
+
options.keys.each do |key|
|
|
78
|
+
next if key.to_s !~ /^#{prefix}_(ssl_.*)/
|
|
79
|
+
result[Regexp.last_match[1].to_sym] = options[key]
|
|
80
|
+
end
|
|
81
|
+
ssl_options(result)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# SSL options to add to the httparty options hash
|
|
85
|
+
# @param :ssl_ca [String] Optional: File with SSL CA certificate
|
|
86
|
+
# @param :ssl_client_key [String] Full text of SSL client private key
|
|
87
|
+
# @param :ssl_client_cert [String] Full text of SSL client public cert
|
|
88
|
+
# @param :ssl_client_pem [String] Full text of SSL client private key + client public cert
|
|
89
|
+
# @param :ssl_client_p12 [String] Full text of pkcs12-encoded keypair
|
|
90
|
+
# @param :ssl_client_password [String] Password to unlock private key
|
|
91
|
+
# @return [Hash] Hash of SSL options to pass to httparty
|
|
92
|
+
def self.ssl_options(options)
|
|
93
|
+
# Initialize the result
|
|
94
|
+
result = {}
|
|
95
|
+
|
|
96
|
+
# Verification of server against a known CA cert
|
|
97
|
+
if ssl_verify?(options)
|
|
98
|
+
result[:verify] = true
|
|
99
|
+
raise ArgumentError, ':ssl_ca must be passed' unless options[:ssl_ca].is_a?(String)
|
|
100
|
+
raise Errno::ENOENT, "'#{options[:ssl_ca]}' not a file" unless File.file?(options[:ssl_ca])
|
|
101
|
+
result[:ssl_ca_file] = options[:ssl_ca]
|
|
102
|
+
else
|
|
103
|
+
result[:verify] = false
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# SSL client certificate auth. This translates our options into httparty options.
|
|
107
|
+
if client_auth?(options)
|
|
108
|
+
if options[:ssl_client_key].is_a?(String) && options[:ssl_client_cert].is_a?(String)
|
|
109
|
+
result[:pem] = options[:ssl_client_key] + options[:ssl_client_cert]
|
|
110
|
+
elsif options[:ssl_client_pem].is_a?(String)
|
|
111
|
+
result[:pem] = options[:ssl_client_pem]
|
|
112
|
+
elsif options[:ssl_client_p12].is_a?(String)
|
|
113
|
+
result[:p12] = options[:ssl_client_p12]
|
|
114
|
+
raise ArgumentError, 'pkcs12 requires a password' unless options[:ssl_client_password]
|
|
115
|
+
result[:p12_password] = options[:ssl_client_password]
|
|
116
|
+
else
|
|
117
|
+
raise ArgumentError, 'SSL client auth enabled but no client keypair specified'
|
|
118
|
+
end
|
|
119
|
+
if result[:pem]
|
|
120
|
+
result[:pem_password] = options[:ssl_client_password] if options[:ssl_client_password]
|
|
121
|
+
# Make sure there's not a password required, or that if the password is given, it is correct.
|
|
122
|
+
# We do not want to wait on STDIN.
|
|
123
|
+
# This will raise OpenSSL::PKey::RSAError if the key needs a password.
|
|
124
|
+
OpenSSL::PKey::RSA.new(result[:pem], result[:pem_password] || '')
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Return result
|
|
129
|
+
result
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Determine, based on options, whether SSL client certificates need to be used.
|
|
133
|
+
# The order of precedence is:
|
|
134
|
+
# - If options[:ssl_client_auth] is not nil, return it
|
|
135
|
+
# - If (key and cert) or PEM or PKCS12 are set, return true
|
|
136
|
+
# - Else return false
|
|
137
|
+
# @return [Boolean] see description
|
|
138
|
+
def self.client_auth?(options)
|
|
139
|
+
return options[:ssl_client_auth] unless options[:ssl_client_auth].nil?
|
|
140
|
+
return true if options[:ssl_client_cert].is_a?(String) && options[:ssl_client_key].is_a?(String)
|
|
141
|
+
return true if options[:ssl_client_pem].is_a?(String)
|
|
142
|
+
return true if options[:ssl_client_p12].is_a?(String)
|
|
143
|
+
false
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Determine, based on options, whether SSL certificates should be verified.
|
|
147
|
+
# The order of precedence is:
|
|
148
|
+
# - If options[:ssl_verify] is not nil, return it
|
|
149
|
+
# - If options[:ssl_ca] is defined, return true
|
|
150
|
+
# - Else return false
|
|
151
|
+
# @return [Boolean] see description
|
|
152
|
+
def self.ssl_verify?(options)
|
|
153
|
+
return options[:ssl_verify] unless options[:ssl_verify].nil?
|
|
154
|
+
options[:ssl_ca].is_a?(String)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|