ncio 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.rubocop.yml +7 -0
- data/.travis.yml +5 -0
- data/.yardopts +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +95 -0
- data/Rakefile +6 -0
- data/bin/console +10 -0
- data/bin/setup +8 -0
- data/exe/ncio +6 -0
- data/lib/ncio.rb +9 -0
- data/lib/ncio/api.rb +7 -0
- data/lib/ncio/api/v1.rb +109 -0
- data/lib/ncio/app.rb +114 -0
- data/lib/ncio/http_client.rb +149 -0
- data/lib/ncio/support.rb +116 -0
- data/lib/ncio/support/option_parsing.rb +200 -0
- data/lib/ncio/support/transform.rb +73 -0
- data/lib/ncio/trollop.rb +863 -0
- data/lib/ncio/version.rb +3 -0
- data/ncio.gemspec +33 -0
- metadata +181 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7f4228a74a79a10e47ffce674748ec755eae37ec
|
4
|
+
data.tar.gz: 5c80ffcca70d9ca50174b4f76cf72fed453e3794
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bc29b5864c421b08faf4609baa7560e2e5e8be82c6210522e82df39338ca7c5e2290ff53f4dd6ee9b34e3e79b4fd5bb1e21a7980f4bd3f347c0723d23727884d
|
7
|
+
data.tar.gz: b31ba443b49e416b5980e9a78996a0c9d8f53b7836c8d7b608f5e3d65b420929999acc34de959e7b92060e0255b05a274f45d389e6faf1ed9c973930ab1b4c80
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/.travis.yml
ADDED
data/.yardopts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--markup markdown
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Jeff McCune
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,95 @@
|
|
1
|
+
# ncio - Puppet Node Classifier backup / restore
|
2
|
+
|
3
|
+
This project implements a small command line utility to backup and restore node
|
4
|
+
classification data. The intended purpose is to backup node classification
|
5
|
+
groups on a Primary, monolithic PE master and restore the backup on a secondary
|
6
|
+
monolithic PE master. The purpose is to keep node classification groups in sync
|
7
|
+
and ready in the event the secondary master needs to take over service from the
|
8
|
+
primary.
|
9
|
+
|
10
|
+
|
11
|
+
## Transformation
|
12
|
+
|
13
|
+
To achieve the goal of replicating node classification groups from one PE
|
14
|
+
monolithic master to a secondary monolithic master, certain values need to be
|
15
|
+
transformed. For example, consider a primary named `master1.puppet.vm` and a
|
16
|
+
secondary named `master2.puppet.vm` Both are monolithic masters. When the
|
17
|
+
backup is taken on the primary, the hostname will be embedded in the data. This
|
18
|
+
is problematic because it will cause mis-configuration errors when imported into
|
19
|
+
the secondary which has a different name.
|
20
|
+
|
21
|
+
To illustrate, consider the PuppetDB classification group:
|
22
|
+
|
23
|
+
{
|
24
|
+
"name": "PE PuppetDB",
|
25
|
+
"rule": [
|
26
|
+
"or",
|
27
|
+
[
|
28
|
+
"=",
|
29
|
+
"name",
|
30
|
+
"master1.puppet.vm"
|
31
|
+
]
|
32
|
+
],
|
33
|
+
"classes": {
|
34
|
+
"puppet_enterprise::profile::puppetdb": {
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
38
|
+
|
39
|
+
Transformation from master1 to master2 is possible:
|
40
|
+
|
41
|
+
export PATH="/opt/pupeptlabs/puppet/bin:$PATH"
|
42
|
+
ncio --uri https://master1.puppet.vm:4433/classification-api/v1 backup \
|
43
|
+
| ncio transform --hostname master1.puppet.vm:master2.puppet.vm \
|
44
|
+
| ncio --uri https://master2.puppet.vm:4433/classification-api/v1 restore
|
45
|
+
|
46
|
+
This method of "replicating" node classification data has some caveats. It's
|
47
|
+
only been tested on PE Monolithic masters. The method assumes master1 and
|
48
|
+
master2 share the same Certificate Authority. By default, only the default
|
49
|
+
`puppet_enterprise` classification groups are transformed.
|
50
|
+
|
51
|
+
Additional groups and classes may be processed by chaining transfomation
|
52
|
+
processes and getting creative with the use of the `--class-matcher` option.
|
53
|
+
|
54
|
+
## Installation
|
55
|
+
|
56
|
+
Install this tool on the same node running the node classification service:
|
57
|
+
|
58
|
+
$ sudo /opt/puppetlabs/puppet/bin/gem install ncio
|
59
|
+
Successfully installed ncio-0.1.0
|
60
|
+
Parsing documentation for ncio-0.1.0
|
61
|
+
Installing ri documentation for ncio-0.1.0
|
62
|
+
Done installing documentation for ncio after 0 seconds
|
63
|
+
1 gem installed
|
64
|
+
|
65
|
+
## Usage
|
66
|
+
|
67
|
+
If the file `/etc/puppetlabs/puppet/ssl/certs/pe-internal-orchestrator.pem`
|
68
|
+
exists on the same node as the Node Classifier, then no configuration is
|
69
|
+
necessary. The default options will work to backup and restore node
|
70
|
+
classification data.
|
71
|
+
|
72
|
+
sudo -H -u pe-puppet /opt/puppetlabs/puppet/bin/ncio backup > /var/tmp/backup.json
|
73
|
+
I, [2016-06-28T19:25:55.507684 #2992] INFO -- : Backup completed successfully!
|
74
|
+
|
75
|
+
If this file does not exist, ncio will need to use a different client
|
76
|
+
certificate. It is recommended to use the same certificate used by the Puppet
|
77
|
+
Agent, which should be white-listed for node classification API access. The
|
78
|
+
white-list of certificates is located at
|
79
|
+
`/etc/puppetlabs/console-services/rbac-certificate-whitelist`
|
80
|
+
|
81
|
+
sudo -H -u pe-puppet /opt/puppetlabs/puppet/bin/ncio \
|
82
|
+
--cert /etc/puppetlabs/puppet/ssl/certs/${HOSTNAME}.pem \
|
83
|
+
--key /etc/puppetlabs/puppet/ssl/private_keys/${HOSTNAME}.pem \
|
84
|
+
backup > /var/tmp/backup.json
|
85
|
+
I, [2016-06-28T19:28:48.236257 #3148] INFO -- : Backup completed successfully!
|
86
|
+
|
87
|
+
## Contributing
|
88
|
+
|
89
|
+
Bug reports and pull requests are welcome on GitHub at
|
90
|
+
https://github.com/jeffmccune/ncio.
|
91
|
+
|
92
|
+
## License
|
93
|
+
|
94
|
+
The gem is available as open source under the terms of the [MIT
|
95
|
+
License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
data/bin/setup
ADDED
data/exe/ncio
ADDED
data/lib/ncio.rb
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
require 'ncio/version'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
##
|
5
|
+
# The top level module for Ncio (Node Classification Input/Output)
|
6
|
+
# This module contains instance methods used by all of the classes in this
|
7
|
+
# project. Mostly for option handling, logging, I.O, etc...
|
8
|
+
module Ncio
|
9
|
+
end
|
data/lib/ncio/api.rb
ADDED
data/lib/ncio/api/v1.rb
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'ncio/api'
|
2
|
+
require 'ncio/http_client'
|
3
|
+
require 'uri'
|
4
|
+
require 'socket'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
module Ncio
|
8
|
+
module Api
|
9
|
+
##
|
10
|
+
# Node Classifier API Version 1
|
11
|
+
#
|
12
|
+
# See [Groups
|
13
|
+
# endpoint](https://docs.puppet.com/pe/2016.1/nc_groups.html#get-v1groups)
|
14
|
+
class V1
|
15
|
+
attr_reader :host
|
16
|
+
attr_reader :port
|
17
|
+
attr_reader :opts
|
18
|
+
|
19
|
+
class ApiError < RuntimeError; end
|
20
|
+
|
21
|
+
DEFAULT_HEADERS = {
|
22
|
+
'Content-Type' => 'application/json',
|
23
|
+
'Transfer-Encoding' => 'chunked'
|
24
|
+
}.freeze
|
25
|
+
|
26
|
+
##
|
27
|
+
# Initialize and return a new API instance.
|
28
|
+
#
|
29
|
+
# @param [Hash<Symbol, String>] The global options hash created by the
|
30
|
+
# application instanece.
|
31
|
+
def initialize(opts)
|
32
|
+
@opts = opts
|
33
|
+
uri = URI(opts[:uri])
|
34
|
+
@host = uri.host
|
35
|
+
@port = uri.port
|
36
|
+
end
|
37
|
+
|
38
|
+
##
|
39
|
+
# Return a memoized HTTP connection
|
40
|
+
def connection
|
41
|
+
return @connection if @connection
|
42
|
+
conn_opts = {
|
43
|
+
host: host,
|
44
|
+
port: port,
|
45
|
+
cert: opts[:cert],
|
46
|
+
key: opts[:key],
|
47
|
+
cacert: opts[:cacert]
|
48
|
+
}
|
49
|
+
@connection = Ncio::HttpClient.new(conn_opts)
|
50
|
+
end
|
51
|
+
|
52
|
+
##
|
53
|
+
# Return all of the groups currently defined in the node classifier API.
|
54
|
+
#
|
55
|
+
# @param [Boolean] inherited If set to any value besides 0 or false, the
|
56
|
+
# node group will include the classes, class parameters, and variables
|
57
|
+
# that it inherits from its ancestors.
|
58
|
+
#
|
59
|
+
# @return [Array]
|
60
|
+
def groups(inherited = false)
|
61
|
+
uri = build_uri('groups', inherited: inherited.to_s)
|
62
|
+
req = Net::HTTP::Get.new(uri, DEFAULT_HEADERS)
|
63
|
+
resp = connection.request(req)
|
64
|
+
if resp.code == '200'
|
65
|
+
obj = JSON.parse(resp.body)
|
66
|
+
else
|
67
|
+
msg = "Expected 200 response, got #{resp.code} body: #{resp.body}"
|
68
|
+
raise ApiError, msg
|
69
|
+
end
|
70
|
+
obj
|
71
|
+
end
|
72
|
+
|
73
|
+
##
|
74
|
+
# Import all of the classification groups using the POST
|
75
|
+
# /v1/import-hierarchy endpoint. See: [NC Import
|
76
|
+
# Hierarchy](https://docs.puppet.com/pe/2016.1/nc_import-hierarchy.html)
|
77
|
+
#
|
78
|
+
# @return [Boolean] true if the upload was successful
|
79
|
+
def import_hierarchy(stream)
|
80
|
+
uri = build_uri('import-hierarchy')
|
81
|
+
req = Net::HTTP::Post.new(uri, DEFAULT_HEADERS)
|
82
|
+
req.body_stream = stream
|
83
|
+
resp = connection.request(req)
|
84
|
+
return true if resp.code == '204'
|
85
|
+
msg = "Expected 204 response, got #{resp.code} body: #{resp.body}"
|
86
|
+
raise ApiError, msg
|
87
|
+
end
|
88
|
+
|
89
|
+
##
|
90
|
+
# Return a URI instance with the given path and parameters. The
|
91
|
+
# connection base URI is used to construct the full URI to the service.
|
92
|
+
#
|
93
|
+
# @param [String] path The path relative to the classifier base service
|
94
|
+
# path and the API version, e.g. 'groups'.
|
95
|
+
#
|
96
|
+
# @param [Hash] params The query parameters to encode into the URI, e.g.
|
97
|
+
# `{inherited: 'false'}`.
|
98
|
+
#
|
99
|
+
# @return [URI] The API uri with query parameters and a fully constructed
|
100
|
+
# path.
|
101
|
+
def build_uri(path, params = {})
|
102
|
+
uri = connection.uri
|
103
|
+
uri.path = "/classifier-api/v1/#{path}"
|
104
|
+
uri.query = URI.encode_www_form(params)
|
105
|
+
uri
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
data/lib/ncio/app.rb
ADDED
@@ -0,0 +1,114 @@
|
|
1
|
+
# rubocop:disable Style/IndentationWidth
|
2
|
+
require 'ncio'
|
3
|
+
require 'ncio/support'
|
4
|
+
require 'ncio/support/option_parsing'
|
5
|
+
require 'ncio/support/transform'
|
6
|
+
require 'ncio/trollop'
|
7
|
+
require 'ncio/version'
|
8
|
+
require 'ncio/api/v1'
|
9
|
+
require 'uri'
|
10
|
+
require 'socket'
|
11
|
+
|
12
|
+
module Ncio
|
13
|
+
##
|
14
|
+
# The main Application class. Intended to be instantiated and called with
|
15
|
+
# using the `run` method by the `ncio` bin script.
|
16
|
+
class App
|
17
|
+
# rubocop:enable Style/IndentationWidth
|
18
|
+
# include support methods (option handling, logging, I/O helpers)
|
19
|
+
include Ncio::Support
|
20
|
+
include Ncio::Support::OptionParsing
|
21
|
+
include Ncio::Support::Transform
|
22
|
+
|
23
|
+
##
|
24
|
+
# @param [Array] argv The argument vector, passed to the option parser.
|
25
|
+
#
|
26
|
+
# @param [Hash] env The environment hash, passed to the option parser to
|
27
|
+
# supply defaults not specified on the command line argument vector.
|
28
|
+
#
|
29
|
+
# @return [Ncio::App] the application instance.
|
30
|
+
def initialize(argv = ARGV.dup, env = ENV.to_hash)
|
31
|
+
@argv = argv
|
32
|
+
@env = env
|
33
|
+
reset!
|
34
|
+
end
|
35
|
+
|
36
|
+
##
|
37
|
+
# Reset all state associated with this application instance.
|
38
|
+
def reset!
|
39
|
+
reset_options!
|
40
|
+
reset_logging!(opts)
|
41
|
+
@api = nil
|
42
|
+
end
|
43
|
+
|
44
|
+
##
|
45
|
+
# Run the application instance. This method is responsible for the
|
46
|
+
# application lifecycle. Command line arguments are parsed, unspecified
|
47
|
+
# options are read from the environment, and the specified subcommand is
|
48
|
+
# executed.
|
49
|
+
#
|
50
|
+
# @return [Fixnum] the exit code to pass to Kernel.exit in the calling
|
51
|
+
# script.
|
52
|
+
# rubocop:disable Metrics/MethodLength
|
53
|
+
def run
|
54
|
+
case opts[:subcommand]
|
55
|
+
when 'backup'
|
56
|
+
backup_groups if opts[:groups]
|
57
|
+
return 0
|
58
|
+
when 'restore'
|
59
|
+
restore_groups if opts[:groups]
|
60
|
+
return 0
|
61
|
+
when 'transform'
|
62
|
+
transform_groups
|
63
|
+
return 0
|
64
|
+
end
|
65
|
+
end
|
66
|
+
# rubocop:enable Metrics/MethodLength
|
67
|
+
|
68
|
+
##
|
69
|
+
# Backup all groups in a manner suitable for the node classification hierarchy
|
70
|
+
# import. See:
|
71
|
+
# [NC Groups](https://docs.puppet.com/pe/2016.1/nc_groups.html#get-v1groups)
|
72
|
+
def backup_groups
|
73
|
+
debug "GET #{uri}/groups"
|
74
|
+
str = JSON.pretty_generate(api.groups)
|
75
|
+
debug "Write #{str.bytesize} bytes to #{file} ..."
|
76
|
+
write_output(str, map_file_option(file))
|
77
|
+
info 'Backup completed successfully!'
|
78
|
+
end
|
79
|
+
|
80
|
+
##
|
81
|
+
# Restore all groups in a manner suitable for the node classification
|
82
|
+
# hierarchy import. See: [NC Import
|
83
|
+
# Hierarchy](https://docs.puppet.com/pe/2016.1/nc_import-hierarchy.html)
|
84
|
+
def restore_groups
|
85
|
+
api = self.api
|
86
|
+
debug "Open #{file} for streaming ..."
|
87
|
+
input_stream(map_file_option(file)) do |stream|
|
88
|
+
debug "POST #{uri}/import-hierarchy"
|
89
|
+
api.import_hierarchy(stream)
|
90
|
+
end
|
91
|
+
info 'Successfully restored node classification groups!'
|
92
|
+
end
|
93
|
+
|
94
|
+
##
|
95
|
+
# Transform a backup produced with backup_groups. The transformation is
|
96
|
+
# intended to allow restoration of the backup on PE Infrastructure cluster
|
97
|
+
# different from the one the backup was produced on.
|
98
|
+
#
|
99
|
+
# Currently only one PE cluster type is supported, the Monolithic master type.
|
100
|
+
# rubocop:disable Metrics/AbcSize
|
101
|
+
def transform_groups
|
102
|
+
# Read input
|
103
|
+
groups = JSON.parse(input_stream(map_file_option(opts[:input]), &:read))
|
104
|
+
groups.map! do |group|
|
105
|
+
group_matches?(group) ? transform_group(group) : group
|
106
|
+
end
|
107
|
+
str = JSON.pretty_generate(groups)
|
108
|
+
debug "Write #{str.bytesize} bytes to #{opts[:output]} ..."
|
109
|
+
write_output(str, map_file_option(opts[:output]))
|
110
|
+
info 'Transformation completed successful!'
|
111
|
+
end
|
112
|
+
# rubocop:enable Metrics/AbcSize
|
113
|
+
end
|
114
|
+
end
|