octocatalog-diff 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.version +1 -0
- data/LICENSE +20 -0
- data/README.md +82 -0
- data/bin/octocatalog-diff +75 -0
- data/doc/advanced-bootstrap.md +33 -0
- data/doc/advanced-cache-dir.md +24 -0
- data/doc/advanced-catalog-only.md +37 -0
- data/doc/advanced-ci.md +13 -0
- data/doc/advanced-dynamic-ignores.md +123 -0
- data/doc/advanced-future-parser.md +11 -0
- data/doc/advanced-ignores.md +224 -0
- data/doc/advanced-output-formats.md +96 -0
- data/doc/advanced-output-hacks.md +45 -0
- data/doc/advanced-override-facts.md +67 -0
- data/doc/advanced-pe-enc.md +52 -0
- data/doc/advanced-puppet-master.md +50 -0
- data/doc/advanced-puppet-versions.md +9 -0
- data/doc/advanced-storeconfigs.md +72 -0
- data/doc/advanced-using-without-git.md +15 -0
- data/doc/advanced.md +43 -0
- data/doc/basic.md +70 -0
- data/doc/configuration-enc.md +69 -0
- data/doc/configuration-hiera.md +103 -0
- data/doc/configuration-puppetdb.md +49 -0
- data/doc/configuration.md +51 -0
- data/doc/dev/README.md +1 -0
- data/doc/dev/coverage.md +34 -0
- data/doc/dev/how-to-add-options.md +83 -0
- data/doc/dev/integration-tests.md +63 -0
- data/doc/dev/releasing.md +19 -0
- data/doc/installation.md +49 -0
- data/doc/limitations.md +34 -0
- data/doc/optionsref.md +947 -0
- data/doc/requirements.md +16 -0
- data/doc/roadmap.md +26 -0
- data/doc/similar.md +17 -0
- data/doc/troubleshooting.md +54 -0
- data/lib/octocatalog-diff.rb +12 -0
- data/lib/octocatalog-diff/bootstrap.rb +53 -0
- data/lib/octocatalog-diff/catalog-diff/cli.rb +205 -0
- data/lib/octocatalog-diff/catalog-diff/cli/catalogs.rb +240 -0
- data/lib/octocatalog-diff/catalog-diff/cli/diffs.rb +145 -0
- data/lib/octocatalog-diff/catalog-diff/cli/helpers/fact_override.rb +99 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options.rb +173 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/basedir.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_environment.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_script.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrap_then_exit.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/bootstrapped_dirs.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/cached_master_dir.rb +21 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/catalog_only.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/color.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/compare_file_text.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/debug.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_datatype_changes.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_detail_add.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/display_source_file_line.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/enc.rb +31 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/existing_catalogs.rb +25 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/fact_file.rb +23 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/fact_override.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/facts_terminus.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/from_puppetdb.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/header.rb +24 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_config.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hiera_path_strip.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/hostname.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore.rb +24 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_attr.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/ignore_tags.rb +23 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/include_tags.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/master_cache_branch.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/output_file.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/output_format.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/parallel.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/parser.rb +48 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pass_env_vars.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_ca.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_cert.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_ssl_client_key.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_token_file.rb +17 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/pe_enc_url.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_binary.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_api_version.rb +20 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_ca.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_cert.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppet_master_ssl_client_key.rb +19 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_ca.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_cert.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_key.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_ssl_client_password_file.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/puppetdb_url.rb +18 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/quiet.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/retry_failed_catalog.rb +13 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/safe_to_delete_cached_master_dir.rb +15 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/storeconfigs.rb +12 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/suppress_absent_file_details.rb +14 -0
- data/lib/octocatalog-diff/catalog-diff/cli/options/to_from_branch.rb +16 -0
- data/lib/octocatalog-diff/catalog-diff/cli/printer.rb +52 -0
- data/lib/octocatalog-diff/catalog-diff/differ.rb +615 -0
- data/lib/octocatalog-diff/catalog-diff/display.rb +125 -0
- data/lib/octocatalog-diff/catalog-diff/display/json.rb +25 -0
- data/lib/octocatalog-diff/catalog-diff/display/text.rb +452 -0
- data/lib/octocatalog-diff/catalog-util/bootstrap.rb +145 -0
- data/lib/octocatalog-diff/catalog-util/builddir.rb +289 -0
- data/lib/octocatalog-diff/catalog-util/cached_master_directory.rb +169 -0
- data/lib/octocatalog-diff/catalog-util/command.rb +96 -0
- data/lib/octocatalog-diff/catalog-util/enc.rb +77 -0
- data/lib/octocatalog-diff/catalog-util/enc/noop.rb +22 -0
- data/lib/octocatalog-diff/catalog-util/enc/pe.rb +99 -0
- data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb +61 -0
- data/lib/octocatalog-diff/catalog-util/enc/script.rb +88 -0
- data/lib/octocatalog-diff/catalog-util/facts.rb +89 -0
- data/lib/octocatalog-diff/catalog-util/fileresources.rb +83 -0
- data/lib/octocatalog-diff/catalog-util/git.rb +65 -0
- data/lib/octocatalog-diff/catalog.rb +209 -0
- data/lib/octocatalog-diff/catalog/computed.rb +205 -0
- data/lib/octocatalog-diff/catalog/json.rb +30 -0
- data/lib/octocatalog-diff/catalog/noop.rb +19 -0
- data/lib/octocatalog-diff/catalog/puppetdb.rb +82 -0
- data/lib/octocatalog-diff/catalog/puppetmaster.rb +121 -0
- data/lib/octocatalog-diff/external/pson/LICENSE +17 -0
- data/lib/octocatalog-diff/external/pson/README.md +20 -0
- data/lib/octocatalog-diff/external/pson/common.rb +370 -0
- data/lib/octocatalog-diff/external/pson/pure.rb +15 -0
- data/lib/octocatalog-diff/external/pson/pure/generator.rb +395 -0
- data/lib/octocatalog-diff/external/pson/pure/parser.rb +307 -0
- data/lib/octocatalog-diff/external/pson/version.rb +8 -0
- data/lib/octocatalog-diff/facts.rb +125 -0
- data/lib/octocatalog-diff/facts/json.rb +20 -0
- data/lib/octocatalog-diff/facts/puppetdb.rb +59 -0
- data/lib/octocatalog-diff/facts/yaml.rb +29 -0
- data/lib/octocatalog-diff/puppetdb.rb +163 -0
- data/lib/octocatalog-diff/util/colored.rb +20 -0
- data/lib/octocatalog-diff/util/httparty.rb +158 -0
- data/lib/octocatalog-diff/util/parallel.rb +170 -0
- data/lib/octocatalog-diff/util/puppetversion.rb +24 -0
- data/lib/octocatalog-diff/version.rb +7 -0
- metadata +386 -0
@@ -0,0 +1,125 @@
|
|
1
|
+
require_relative 'facts/json'
|
2
|
+
require_relative 'facts/yaml'
|
3
|
+
require_relative 'facts/puppetdb'
|
4
|
+
require_relative 'external/pson/pure'
|
5
|
+
|
6
|
+
module OctocatalogDiff
|
7
|
+
# Deal with facts in all forms, including:
|
8
|
+
# - In existing YAML files
|
9
|
+
# - In existing JSON files
|
10
|
+
# - Retrieved dynamically from PuppetDB
|
11
|
+
class Facts
|
12
|
+
# Constructor
|
13
|
+
# @param options [Hash] Initialization options, varies per backend
|
14
|
+
def initialize(options = {}, facts = nil)
|
15
|
+
@node = options.fetch(:node, '')
|
16
|
+
@timestamp = false
|
17
|
+
@options = options.dup
|
18
|
+
if facts
|
19
|
+
@facts = {}
|
20
|
+
facts.each { |k, v| @facts[k] = v.dup }
|
21
|
+
else
|
22
|
+
case options[:backend]
|
23
|
+
when :json
|
24
|
+
@orig_facts = OctocatalogDiff::Facts::JSON.fact_retriever(options, @node)
|
25
|
+
when :yaml
|
26
|
+
@orig_facts = OctocatalogDiff::Facts::Yaml.fact_retriever(options, @node)
|
27
|
+
when :puppetdb
|
28
|
+
@orig_facts = OctocatalogDiff::Facts::PuppetDB.fact_retriever(options, @node)
|
29
|
+
else
|
30
|
+
raise ArgumentError, 'Invalid fact source backend'
|
31
|
+
end
|
32
|
+
@facts = {}
|
33
|
+
@orig_facts.each { |k, v| @facts[k] = v.dup }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def dup
|
38
|
+
self.class.new(@options, @orig_facts)
|
39
|
+
end
|
40
|
+
|
41
|
+
# Facts - returned the 'cleansed' facts.
|
42
|
+
# Clean up facts by setting 'name' to the node if given, and deleting _timestamp and expiration
|
43
|
+
# which may cause Puppet catalog compilation to fail if the facts are old.
|
44
|
+
# @param node [String] Node name to override returned facts
|
45
|
+
# @return [Hash] Facts hash { 'name' => '...', 'values' => { ... } }
|
46
|
+
def facts(node = @node, timestamp = false)
|
47
|
+
raise "Expected @facts to be a hash but it is a #{@facts.class}" unless @facts.is_a?(Hash)
|
48
|
+
raise "Expected @facts['values'] to be a hash but it is a #{@facts['values'].class}" unless @facts['values'].is_a?(Hash)
|
49
|
+
f = @facts.dup
|
50
|
+
f['name'] = node unless node.nil? || node.empty?
|
51
|
+
f['values'].delete('_timestamp')
|
52
|
+
f.delete('expiration')
|
53
|
+
if timestamp
|
54
|
+
f['timestamp'] = Time.now.to_s
|
55
|
+
f['values']['timestamp'] = f['timestamp']
|
56
|
+
f['expiration'] = (Time.now + (24 * 60 * 60)).to_s
|
57
|
+
end
|
58
|
+
f
|
59
|
+
end
|
60
|
+
|
61
|
+
# Facts - Fudge the timestamp to right now and add include it in the facts when returned
|
62
|
+
# @return self
|
63
|
+
def fudge_timestamp
|
64
|
+
@timestamp = true
|
65
|
+
self
|
66
|
+
end
|
67
|
+
|
68
|
+
# Facts - remove one or more facts from the list.
|
69
|
+
# @param remove [String|Array<String>] Fact(s) to remove
|
70
|
+
# @return self
|
71
|
+
def without(remove)
|
72
|
+
r = remove.is_a?(Array) ? remove : [remove]
|
73
|
+
obj = dup
|
74
|
+
r.each { |fact| obj.remove_fact_from_list(fact) }
|
75
|
+
obj
|
76
|
+
end
|
77
|
+
|
78
|
+
# Facts - remove a fact from the list
|
79
|
+
# @param remove [String] Fact to remove
|
80
|
+
def remove_fact_from_list(remove)
|
81
|
+
@facts['values'].delete(remove)
|
82
|
+
end
|
83
|
+
|
84
|
+
# Turn hash of facts into appropriate YAML for Puppet
|
85
|
+
# @param node [String] Node name to override returned facts
|
86
|
+
# @return [String] Puppet-compatible YAML facts
|
87
|
+
def facts_to_yaml(node = @node)
|
88
|
+
# Add the header that Puppet needs to treat this as facts. Save the results
|
89
|
+
# as a string in the option.
|
90
|
+
f = facts(node)
|
91
|
+
fact_file = f.to_yaml.split(/\n/)
|
92
|
+
fact_file[0] = '--- !ruby/object:Puppet::Node::Facts' if fact_file[0] =~ /^---/
|
93
|
+
fact_file.join("\n")
|
94
|
+
end
|
95
|
+
|
96
|
+
# Turn hash of facts into appropriate YAML for Puppet
|
97
|
+
# @param node [String] Node name to override returned facts
|
98
|
+
# @return [String] Puppet-compatible YAML facts
|
99
|
+
def to_pson
|
100
|
+
PSON.generate(facts)
|
101
|
+
end
|
102
|
+
|
103
|
+
# Get the current value of a particular fact
|
104
|
+
# @param key [String] Fact key to override
|
105
|
+
# @return [?] Value for fact
|
106
|
+
def fact(key)
|
107
|
+
@facts['values'][key]
|
108
|
+
end
|
109
|
+
|
110
|
+
# Override a particular fact
|
111
|
+
# @param key [String] Fact key to override
|
112
|
+
# @param value [?] Value for fact
|
113
|
+
def override(key, value)
|
114
|
+
if value.nil?
|
115
|
+
@facts['values'].delete(key)
|
116
|
+
else
|
117
|
+
@facts['values'][key] = value
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# Separate classes to handle errors we throw explicitly
|
122
|
+
class FactSourceError < RuntimeError; end
|
123
|
+
class FactRetrievalError < RuntimeError; end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative '../facts'
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module OctocatalogDiff
|
6
|
+
class Facts
|
7
|
+
# Deal with facts in JSON files
|
8
|
+
class JSON
|
9
|
+
# @param options [Hash] Options hash specifically for this fact type.
|
10
|
+
# - :fact_file_string [String] => Fact data as a string
|
11
|
+
# @param node [String] Node name (overrides node name from fact data)
|
12
|
+
# @return [Hash] Facts
|
13
|
+
def self.fact_retriever(options = {}, node = '')
|
14
|
+
facts = ::JSON.parse(options.fetch(:fact_file_string))
|
15
|
+
node = facts.fetch('fqdn', 'unknown.node') if node.empty?
|
16
|
+
{ 'name' => node, 'values' => facts }
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require_relative '../facts'
|
2
|
+
require_relative '../puppetdb'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module OctocatalogDiff
|
6
|
+
class Facts
|
7
|
+
# Deal with facts in PuppetDB
|
8
|
+
class PuppetDB
|
9
|
+
# Retrieve facts from PuppetDB for a specified node.
|
10
|
+
# @param :puppetdb_url [String|Array] => URL to PuppetDB
|
11
|
+
# @param :retry [Fixnum] => Retry after timeout (default 0 retries, can be more)
|
12
|
+
# @param node [String] Node name. (REQUIRED for PuppetDB fact source)
|
13
|
+
# @return [Hash] Facts
|
14
|
+
def self.fact_retriever(options = {}, node)
|
15
|
+
# Set up some variables from options
|
16
|
+
raise ArgumentError, 'puppetdb_url is required' unless options[:puppetdb_url].is_a?(String)
|
17
|
+
raise ArgumentError, 'node must be a non-empty string' unless node.is_a?(String) && node != ''
|
18
|
+
uri = "/pdb/query/v4/nodes/#{node}/facts"
|
19
|
+
retries = options.fetch(:retry, 0).to_i
|
20
|
+
|
21
|
+
# Construct puppetdb object and options
|
22
|
+
opts = options.merge(timeout: 5)
|
23
|
+
puppetdb = OctocatalogDiff::PuppetDB.new(opts)
|
24
|
+
|
25
|
+
# Use OctocatalogDiff::PuppetDB to pull facts
|
26
|
+
exception_class = nil
|
27
|
+
exception_message = nil
|
28
|
+
obj_to_return = nil
|
29
|
+
(retries + 1).times do
|
30
|
+
begin
|
31
|
+
result = puppetdb.get(uri)
|
32
|
+
facts = {}
|
33
|
+
result.map { |x| facts[x['name']] = x['value'] }
|
34
|
+
if facts.empty?
|
35
|
+
message = "Unable to retrieve facts for node #{node} from PuppetDB (empty or nil)!"
|
36
|
+
raise OctocatalogDiff::Facts::FactRetrievalError, message
|
37
|
+
end
|
38
|
+
|
39
|
+
# Create a structure compatible with YAML fact files.
|
40
|
+
obj_to_return = { 'name' => node, 'values' => {} }
|
41
|
+
facts.each { |k, v| obj_to_return['values'][k.sub(/^::/, '')] = v }
|
42
|
+
break # Not return, to avoid LocalJumpError in Ruby 2.2
|
43
|
+
rescue OctocatalogDiff::PuppetDB::ConnectionError => exc
|
44
|
+
exception_class = OctocatalogDiff::Facts::FactSourceError
|
45
|
+
exception_message = "Fact retrieval failed (#{exc.class}) (#{exc.message})"
|
46
|
+
rescue OctocatalogDiff::PuppetDB::NotFoundError => exc
|
47
|
+
exception_class = OctocatalogDiff::Facts::FactRetrievalError
|
48
|
+
exception_message = "Node #{node} not found in PuppetDB (#{exc.message})"
|
49
|
+
rescue OctocatalogDiff::PuppetDB::PuppetDBError => exc
|
50
|
+
exception_class = OctocatalogDiff::Facts::FactRetrievalError
|
51
|
+
exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
return obj_to_return unless obj_to_return.nil?
|
55
|
+
raise exception_class, exception_message
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative '../facts'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module OctocatalogDiff
|
5
|
+
class Facts
|
6
|
+
# Deal with facts in YAML files
|
7
|
+
class Yaml
|
8
|
+
# Manipulate a YAML file so it can be parsed and return the facts as a hash.
|
9
|
+
# If we leave it as Puppet::Node::Facts then it will require us to load puppet
|
10
|
+
# gems in order to parse it, and that's too heavy for simple fact retrieval.
|
11
|
+
# @param options [Hash] Options hash specifically for this fact type.
|
12
|
+
# - :fact_file_string [String] => Fact data as a string
|
13
|
+
# @param node [String] Node name (overrides node name from fact data)
|
14
|
+
# @return [Hash] Facts
|
15
|
+
def self.fact_retriever(options = {}, node = '')
|
16
|
+
fact_file_string = options.fetch(:fact_file_string)
|
17
|
+
|
18
|
+
# Touch up the first line before parsing.
|
19
|
+
fact_file_data = fact_file_string.split(/\n/)
|
20
|
+
fact_file_data[0] = '---' if fact_file_data[0] =~ /^---/
|
21
|
+
|
22
|
+
# Load and return the parsed fact file.
|
23
|
+
result = YAML.load(fact_file_data.join("\n"))
|
24
|
+
result['name'] = node unless node == ''
|
25
|
+
result
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,163 @@
|
|
1
|
+
require_relative 'util/httparty'
|
2
|
+
|
3
|
+
require 'uri'
|
4
|
+
|
5
|
+
# Redefine constants to match PuppetDB defaults.
|
6
|
+
# This code avoids warnings about redefining constants.
|
7
|
+
URI::HTTP.send(:remove_const, :DEFAULT_PORT) if URI::HTTP.const_defined?(:DEFAULT_PORT)
|
8
|
+
URI::HTTP.const_set(:DEFAULT_PORT, 8080)
|
9
|
+
URI::HTTPS.send(:remove_const, :DEFAULT_PORT) if URI::HTTPS.const_defined?(:DEFAULT_PORT)
|
10
|
+
URI::HTTPS.const_set(:DEFAULT_PORT, 8081)
|
11
|
+
|
12
|
+
module OctocatalogDiff
|
13
|
+
# A standard way to connect to PuppetDB from the various scripts in this repository.
|
14
|
+
class PuppetDB
|
15
|
+
# This class raises errors for certain handled problems
|
16
|
+
class NotFoundError < RuntimeError; end
|
17
|
+
class PuppetDBError < RuntimeError; end
|
18
|
+
class ConnectionError < RuntimeError; end
|
19
|
+
|
20
|
+
# Allow connections to be read (used in tests for now)
|
21
|
+
attr_reader :connections
|
22
|
+
|
23
|
+
# Constructor - will construct connection parameters from a variety
|
24
|
+
# of sources, including arguments and environment variables. Supported
|
25
|
+
# environment variables:
|
26
|
+
# PUPPETDB_URL
|
27
|
+
# PUPPETDB_HOST [+ PUPPETDB_PORT] [+ PUPPETDB_SSL]
|
28
|
+
#
|
29
|
+
# Order of precedence:
|
30
|
+
# 1. :puppetdb_url argument (String or Array<String>)
|
31
|
+
# 2. :puppetdb_host argument [+ :puppetdb_port] [+ :puppetdb_ssl]
|
32
|
+
# 3. ENV['PUPPETDB_URL']
|
33
|
+
# 4. ENV['PUPPETDB_HOST'] [+ ENV['PUPPETDB_PORT']], [+ ENV['PUPPETDB_SSL']]
|
34
|
+
# When it finds one of these, it stops and does not process any others.
|
35
|
+
#
|
36
|
+
# When :puppetdb_url is an array, all given URLs are tried, in random order,
|
37
|
+
# until a connection succeeds. If a connection succeeds, any errors from previously
|
38
|
+
# failed connections are suppressed.
|
39
|
+
#
|
40
|
+
# Supported arguments:
|
41
|
+
# @param :puppetdb_url [String or Array<String>] PuppetDB URL(s) to try in random order
|
42
|
+
# @param :puppetdb_host [String] PuppetDB hostname, when constructing a URL
|
43
|
+
# @param :puppetdb_port [Fixnum] Port number, defaults to 8080 (non-SSL) or 8081 (SSL)
|
44
|
+
# @param :puppetdb_ssl [Boolean] defaults to true, because you should use SSL
|
45
|
+
# @param :puppetdb_ssl_ca [String] Path to file containing CA certificate
|
46
|
+
# @param :puppetdb_ssl_verify [Boolean] Override the CA verification setting guessed from parameters
|
47
|
+
# @param :puppetdb_ssl_client_pem [String] PEM-encoded client key and certificate
|
48
|
+
# @param :puppetdb_ssl_client_p12 [String] pkcs12-encoded client key and certificate
|
49
|
+
# @param :puppetdb_ssl_client_password [String] Path to file containing password for SSL client key (any format)
|
50
|
+
# @param :puppetdb_ssl_client_auth [Boolean] Override the client-auth that is guessed from parameters
|
51
|
+
# @param :timeout [Fixnum] Connection timeout for PuppetDB (default=10)
|
52
|
+
def initialize(options = {})
|
53
|
+
@connections =
|
54
|
+
if options.key?(:puppetdb_url)
|
55
|
+
urls = options[:puppetdb_url].is_a?(Array) ? options[:puppetdb_url] : [options[:puppetdb_url]]
|
56
|
+
urls.map { |url| parse_url(url) }
|
57
|
+
elsif options.key?(:puppetdb_host)
|
58
|
+
is_ssl = options.fetch(:puppetdb_ssl, true)
|
59
|
+
default_port = is_ssl ? URI::HTTPS::DEFAULT_PORT : URI::HTTP::DEFAULT_PORT
|
60
|
+
port = options.fetch(:puppetdb_port, default_port).to_i
|
61
|
+
[{ ssl: is_ssl, host: options[:puppetdb_host], port: port }]
|
62
|
+
elsif ENV['PUPPETDB_URL'] && !ENV['PUPPETDB_URL'].empty?
|
63
|
+
[parse_url(ENV['PUPPETDB_URL'])]
|
64
|
+
elsif ENV['PUPPETDB_HOST'] && !ENV['PUPPETDB_HOST'].empty?
|
65
|
+
# Because environment variables are strings...
|
66
|
+
# This will get the env var and see if it equals 'true'; the result
|
67
|
+
# of this == comparison is the true/false boolean we need.
|
68
|
+
is_ssl = ENV.fetch('PUPPETDB_SSL', 'true') == 'true'
|
69
|
+
default_port = is_ssl ? URI::HTTPS::DEFAULT_PORT : URI::HTTP::DEFAULT_PORT
|
70
|
+
port = ENV.fetch('PUPPETDB_PORT', default_port).to_i
|
71
|
+
[{ ssl: is_ssl, host: ENV['PUPPETDB_HOST'], port: port }]
|
72
|
+
else
|
73
|
+
[]
|
74
|
+
end
|
75
|
+
@timeout = options.fetch(:timeout, 10)
|
76
|
+
@options = options
|
77
|
+
end
|
78
|
+
|
79
|
+
# Wrapper around the httparty call in the private _get method.
|
80
|
+
# Returns the parsed result of getting the provided URL and returns
|
81
|
+
# a friendlier error message if there are network connection problems
|
82
|
+
# to PuppetDB.
|
83
|
+
# @param path [String] Path portion of the URL
|
84
|
+
# @return [Object] Parsed reply from PuppetDB as an object
|
85
|
+
def get(path)
|
86
|
+
_get(path)
|
87
|
+
rescue Net::OpenTimeout, Errno::ECONNREFUSED => exc
|
88
|
+
raise ConnectionError, "#{exc.class} connecting to PuppetDB (need VPN on?): #{exc.message}"
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
# HTTP(S) Query - will attempt to retrieve URL from each connection
|
94
|
+
# @param path [String] Path portion of the URL
|
95
|
+
# @return [String] Parsed response
|
96
|
+
def _get(path)
|
97
|
+
# You need at least one connection or else this can't do anything
|
98
|
+
raise ArgumentError, 'No PuppetDB connections configured' if @connections.empty?
|
99
|
+
|
100
|
+
# Keep track of the latest exception seen
|
101
|
+
exc = nil
|
102
|
+
|
103
|
+
# Try each connection in random order. This will return the first successful
|
104
|
+
# response, and try the next connection if there's an error. Once it's out of
|
105
|
+
# connections to try it will raise the last exception encountered.
|
106
|
+
@connections.shuffle.each do |connection|
|
107
|
+
complete_url = [
|
108
|
+
connection[:ssl] ? 'https://' : 'http://',
|
109
|
+
connection[:host],
|
110
|
+
':',
|
111
|
+
connection[:port],
|
112
|
+
path
|
113
|
+
].join('')
|
114
|
+
|
115
|
+
begin
|
116
|
+
more_options = { headers: { 'Accept' => 'application/json' }, timeout: @timeout }
|
117
|
+
response = OctocatalogDiff::Util::HTTParty.get(complete_url, @options.merge(more_options), 'puppetdb')
|
118
|
+
|
119
|
+
# Handle all non-200's from PuppetDB
|
120
|
+
unless response[:code] == 200
|
121
|
+
raise NotFoundError, "404 - #{response[:error]}" if response[:code] == 404
|
122
|
+
raise PuppetDBError, "#{response[:code]} - #{response[:error]}"
|
123
|
+
end
|
124
|
+
|
125
|
+
# PuppetDB can return 'Not Found' as a string with a 200 response code
|
126
|
+
raise NotFoundError, '404 - Not Found' if response[:body] == 'Not Found'
|
127
|
+
|
128
|
+
# PuppetDB can also return an error message in a 200; we'll call this a 500
|
129
|
+
if response.key?(:error)
|
130
|
+
raise PuppetDBError, "500 - #{response[:error]}"
|
131
|
+
end
|
132
|
+
|
133
|
+
# If we get here without raising an error, it will fall out of the begin/rescue
|
134
|
+
# with 'result' non-nil, and 'result' will then get returned.
|
135
|
+
raise "Unparseable response from puppetdb: '#{response.inspect}'" unless response[:parsed]
|
136
|
+
result = response[:parsed]
|
137
|
+
rescue => exc
|
138
|
+
# Set response to nil so the loop repeats itself if there are retries left.
|
139
|
+
# Also sets 'exc' to the most recent exception, in case all retries are
|
140
|
+
# exhausted and this exception has to be raised.
|
141
|
+
result = nil
|
142
|
+
end
|
143
|
+
|
144
|
+
# If the previous query didn't error, return result
|
145
|
+
return result unless result.nil?
|
146
|
+
end
|
147
|
+
|
148
|
+
# At this point no query has succeeded, so raise the last error encountered.
|
149
|
+
raise exc
|
150
|
+
end
|
151
|
+
|
152
|
+
# Parse a URL to determine hostname, port number, and whether or not SSL is used.
|
153
|
+
# @param url [String] URL to parse
|
154
|
+
# @return [Hash] { ssl: true/false, host: <String>, port: <Fixnum> }
|
155
|
+
def parse_url(url)
|
156
|
+
uri = URI(url)
|
157
|
+
raise ArgumentError, "URL #{url} has invalid scheme" unless uri.scheme =~ /^https?$/
|
158
|
+
{ ssl: uri.scheme == 'https', host: uri.host, port: uri.port }
|
159
|
+
rescue URI::InvalidURIError => exc
|
160
|
+
raise exc.class, "Invalid URL: #{url} (#{exc.message})"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# Create colorizing methods in the 'String' class, but only if 'colors_enabled'
|
2
|
+
# has been set.
|
3
|
+
class String
|
4
|
+
COLORS = {
|
5
|
+
'red' => 31,
|
6
|
+
'green' => 32,
|
7
|
+
'yellow' => 33,
|
8
|
+
'cyan' => 36
|
9
|
+
}.freeze
|
10
|
+
|
11
|
+
COLORS.each do |color, _value|
|
12
|
+
define_method(color) do
|
13
|
+
@@colors_enabled ? "\e[0;#{COLORS[color]};49m#{self}\e[0m" : self
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.colors_enabled=(value)
|
18
|
+
@@colors_enabled = value # rubocop:disable Style/ClassVars
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'json'
|
3
|
+
require_relative '../external/pson/pure'
|
4
|
+
|
5
|
+
module OctocatalogDiff
|
6
|
+
module Util
|
7
|
+
# This is a wrapper around some common actions that octocatalog-diff does when preparing to talk
|
8
|
+
# to a web server using 'httparty'.
|
9
|
+
class HTTParty
|
10
|
+
# Wrap the 'get' method in httparty with SSL options
|
11
|
+
# @param url [String] URL to retrieve
|
12
|
+
# @param options [Hash] Options
|
13
|
+
# @param ssl_prefix [String] Strip "#{prefix}_" from the start of SSL options to generalize them
|
14
|
+
# @return [Hash] HTTParty response and codes
|
15
|
+
def self.get(url, options = {}, ssl_prefix = nil)
|
16
|
+
httparty_response_parse(::HTTParty.get(url, options.merge(wrap_ssl_options(options, ssl_prefix))))
|
17
|
+
end
|
18
|
+
|
19
|
+
# Wrap the 'post' method in httparty with SSL options
|
20
|
+
# @param url [String] URL to retrieve
|
21
|
+
# @param options [Hash] Options
|
22
|
+
# @param post_body [String] Test to POST
|
23
|
+
# @param ssl_prefix [String] Strip "#{prefix}_" from the start of SSL options to generalize them
|
24
|
+
# @return [Hash] HTTParty response and codes
|
25
|
+
def self.post(url, options, post_body, ssl_prefix)
|
26
|
+
opts = options.merge(wrap_ssl_options(options, ssl_prefix))
|
27
|
+
httparty_response_parse(::HTTParty.post(url, opts.merge(body: post_body)))
|
28
|
+
end
|
29
|
+
|
30
|
+
# Common parser for HTTParty response
|
31
|
+
# @param response [HTTParty response object] HTTParty response object
|
32
|
+
# @return [Hash] HTTParty parsed response and codes
|
33
|
+
def self.httparty_response_parse(response)
|
34
|
+
# Handle HTTP errors
|
35
|
+
unless response.code == 200
|
36
|
+
begin
|
37
|
+
b = JSON.parse(response.body)
|
38
|
+
errormessage = b['error'] if b.is_a?(Hash) && b.key?('error')
|
39
|
+
rescue JSON::ParserError
|
40
|
+
errormessage = response.body
|
41
|
+
ensure
|
42
|
+
errormessage ||= response.body
|
43
|
+
end
|
44
|
+
return { code: response.code, body: response.body, error: errormessage }
|
45
|
+
end
|
46
|
+
|
47
|
+
# Handle success
|
48
|
+
if response.headers.key?('content-type')
|
49
|
+
if response.headers['content-type'] =~ %r{/json}
|
50
|
+
begin
|
51
|
+
return { code: 200, body: response.body, parsed: JSON.parse(response.body) }
|
52
|
+
rescue JSON::ParserError => exc
|
53
|
+
return { code: 500, body: response.body, error: "JSON parse error: #{exc.message}" }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
if response.headers['content-type'] =~ %r{/pson}
|
57
|
+
begin
|
58
|
+
return { code: 200, body: response.body, parsed: PSON.parse(response.body) }
|
59
|
+
rescue PSON::ParserError => exc
|
60
|
+
return { code: 500, body: response.body, error: "PSON parse error: #{exc.message}" }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
return { code: 500, body: response.body, error: "Don't know how to parse: #{response.headers['content-type']}" }
|
64
|
+
end
|
65
|
+
|
66
|
+
# Return raw output
|
67
|
+
{ code: response.code, body: response.body }
|
68
|
+
end
|
69
|
+
|
70
|
+
# Wrap context-specific options into generally named options for the other methods in this class
|
71
|
+
# @param options [Hash] Hash of all options
|
72
|
+
# @param prefix [String] Prefix to strip from SSL options
|
73
|
+
# @return [Hash] SSL options generally named
|
74
|
+
def self.wrap_ssl_options(options, prefix)
|
75
|
+
return {} unless prefix
|
76
|
+
result = {}
|
77
|
+
options.keys.each do |key|
|
78
|
+
next if key.to_s !~ /^#{prefix}_(ssl_.*)/
|
79
|
+
result[Regexp.last_match[1].to_sym] = options[key]
|
80
|
+
end
|
81
|
+
ssl_options(result)
|
82
|
+
end
|
83
|
+
|
84
|
+
# SSL options to add to the httparty options hash
|
85
|
+
# @param :ssl_ca [String] Optional: File with SSL CA certificate
|
86
|
+
# @param :ssl_client_key [String] Full text of SSL client private key
|
87
|
+
# @param :ssl_client_cert [String] Full text of SSL client public cert
|
88
|
+
# @param :ssl_client_pem [String] Full text of SSL client private key + client public cert
|
89
|
+
# @param :ssl_client_p12 [String] Full text of pkcs12-encoded keypair
|
90
|
+
# @param :ssl_client_password [String] Password to unlock private key
|
91
|
+
# @return [Hash] Hash of SSL options to pass to httparty
|
92
|
+
def self.ssl_options(options)
|
93
|
+
# Initialize the result
|
94
|
+
result = {}
|
95
|
+
|
96
|
+
# Verification of server against a known CA cert
|
97
|
+
if ssl_verify?(options)
|
98
|
+
result[:verify] = true
|
99
|
+
raise ArgumentError, ':ssl_ca must be passed' unless options[:ssl_ca].is_a?(String)
|
100
|
+
raise Errno::ENOENT, "'#{options[:ssl_ca]}' not a file" unless File.file?(options[:ssl_ca])
|
101
|
+
result[:ssl_ca_file] = options[:ssl_ca]
|
102
|
+
else
|
103
|
+
result[:verify] = false
|
104
|
+
end
|
105
|
+
|
106
|
+
# SSL client certificate auth. This translates our options into httparty options.
|
107
|
+
if client_auth?(options)
|
108
|
+
if options[:ssl_client_key].is_a?(String) && options[:ssl_client_cert].is_a?(String)
|
109
|
+
result[:pem] = options[:ssl_client_key] + options[:ssl_client_cert]
|
110
|
+
elsif options[:ssl_client_pem].is_a?(String)
|
111
|
+
result[:pem] = options[:ssl_client_pem]
|
112
|
+
elsif options[:ssl_client_p12].is_a?(String)
|
113
|
+
result[:p12] = options[:ssl_client_p12]
|
114
|
+
raise ArgumentError, 'pkcs12 requires a password' unless options[:ssl_client_password]
|
115
|
+
result[:p12_password] = options[:ssl_client_password]
|
116
|
+
else
|
117
|
+
raise ArgumentError, 'SSL client auth enabled but no client keypair specified'
|
118
|
+
end
|
119
|
+
if result[:pem]
|
120
|
+
result[:pem_password] = options[:ssl_client_password] if options[:ssl_client_password]
|
121
|
+
# Make sure there's not a password required, or that if the password is given, it is correct.
|
122
|
+
# We do not want to wait on STDIN.
|
123
|
+
# This will raise OpenSSL::PKey::RSAError if the key needs a password.
|
124
|
+
OpenSSL::PKey::RSA.new(result[:pem], result[:pem_password] || '')
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# Return result
|
129
|
+
result
|
130
|
+
end
|
131
|
+
|
132
|
+
# Determine, based on options, whether SSL client certificates need to be used.
|
133
|
+
# The order of precedence is:
|
134
|
+
# - If options[:ssl_client_auth] is not nil, return it
|
135
|
+
# - If (key and cert) or PEM or PKCS12 are set, return true
|
136
|
+
# - Else return false
|
137
|
+
# @return [Boolean] see description
|
138
|
+
def self.client_auth?(options)
|
139
|
+
return options[:ssl_client_auth] unless options[:ssl_client_auth].nil?
|
140
|
+
return true if options[:ssl_client_cert].is_a?(String) && options[:ssl_client_key].is_a?(String)
|
141
|
+
return true if options[:ssl_client_pem].is_a?(String)
|
142
|
+
return true if options[:ssl_client_p12].is_a?(String)
|
143
|
+
false
|
144
|
+
end
|
145
|
+
|
146
|
+
# Determine, based on options, whether SSL certificates should be verified.
|
147
|
+
# The order of precedence is:
|
148
|
+
# - If options[:ssl_verify] is not nil, return it
|
149
|
+
# - If options[:ssl_ca] is defined, return true
|
150
|
+
# - Else return false
|
151
|
+
# @return [Boolean] see description
|
152
|
+
def self.ssl_verify?(options)
|
153
|
+
return options[:ssl_verify] unless options[:ssl_verify].nil?
|
154
|
+
options[:ssl_ca].is_a?(String)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|