agentless-catalog-executor 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +7 -0
  2. data/.dependency_decisions.yml +118 -0
  3. data/.gitignore +15 -0
  4. data/.rspec +3 -0
  5. data/.rubocop.yml +97 -0
  6. data/.travis.yml +7 -0
  7. data/CHANGELOG.md +44 -0
  8. data/Dockerfile +42 -0
  9. data/Gemfile +22 -0
  10. data/LICENSE +201 -0
  11. data/README.md +23 -0
  12. data/Rakefile +52 -0
  13. data/acceptance/.gitignore +10 -0
  14. data/acceptance/Gemfile +28 -0
  15. data/acceptance/Rakefile +16 -0
  16. data/acceptance/config/gem/options.rb +6 -0
  17. data/acceptance/config/git/options.rb +7 -0
  18. data/acceptance/config/package/options.rb +6 -0
  19. data/acceptance/lib/acceptance/ace_command_helper.rb +49 -0
  20. data/acceptance/lib/acceptance/ace_setup_helper.rb +45 -0
  21. data/acceptance/setup/common/pre-suite/.gitkeep +0 -0
  22. data/acceptance/setup/gem/pre-suite/.gitkeep +0 -0
  23. data/acceptance/setup/git/pre-suite/.gitkeep +0 -0
  24. data/acceptance/setup/package/pre-suite/.gitkeep +0 -0
  25. data/acceptance/tests/.gitkeep +0 -0
  26. data/agentless-catalog-executor.gemspec +38 -0
  27. data/config/docker.conf +9 -0
  28. data/config/local.conf +9 -0
  29. data/config/transport_tasks_config.rb +49 -0
  30. data/developer-docs/api.md +139 -0
  31. data/developer-docs/docker.md +170 -0
  32. data/docker-compose.yml +12 -0
  33. data/lib/ace/config.rb +73 -0
  34. data/lib/ace/configurer.rb +17 -0
  35. data/lib/ace/error.rb +31 -0
  36. data/lib/ace/fork_util.rb +39 -0
  37. data/lib/ace/plugin_cache.rb +79 -0
  38. data/lib/ace/puppet_util.rb +57 -0
  39. data/lib/ace/schemas/ace-execute_catalog.json +48 -0
  40. data/lib/ace/schemas/ace-run_task.json +30 -0
  41. data/lib/ace/schemas/task.json +74 -0
  42. data/lib/ace/transport_app.rb +252 -0
  43. data/lib/ace/version.rb +5 -0
  44. data/lib/puppet/indirector/catalog/certless.rb +83 -0
  45. metadata +254 -0
@@ -0,0 +1,48 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-04/schema#",
3
+ "title": "ACE execute_catalog request",
4
+ "description": "POST /execute_catalog request schema for ACE",
5
+ "type": "object",
6
+ "properties": {
7
+ "target": {
8
+ "type": "object",
9
+ "description": "Contains the Transport schema to connect to the remote target",
10
+ "properties": {
11
+ "remote-transport": {
12
+ "type": "string",
13
+ "description": "The name of the transport being used"
14
+ }
15
+ },
16
+ "additionalProperties": true,
17
+ "required": ["remote-transport"]
18
+ },
19
+ "compiler": {
20
+ "type": "object",
21
+ "description": "Contains additional information to compile the catalog",
22
+ "properties": {
23
+ "certname": {
24
+ "type": "string",
25
+ "description": "The certname of the target"
26
+ },
27
+ "environment": {
28
+ "type": "string",
29
+ "description": "The name of the environment for which to compile the catalog."
30
+ },
31
+ "transaction_uuid": {
32
+ "type": "string",
33
+ "description": "The id for tracking the catalog compilation and report submission."
34
+ },
35
+ "job_id": {
36
+ "type": "string",
37
+ "description": "The id of the orchestrator job that triggered this run."
38
+ }
39
+ },
40
+ "additionalProperties": true,
41
+ "required": [
42
+ "certname",
43
+ "environment"
44
+ ]
45
+ }
46
+ },
47
+ "required": ["target", "compiler"]
48
+ }
@@ -0,0 +1,30 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-04/schema#",
3
+ "title": "ACE run_task request",
4
+ "description": "POST /run_task request schema for ACE",
5
+ "type": "object",
6
+ "properties": {
7
+ "target": {
8
+ "type": "object",
9
+ "description": "Contains the Transport schema to connect to the remote target",
10
+ "properties": {
11
+ "remote-transport": {
12
+ "type": "string",
13
+ "description": "The name of the transport being used"
14
+ },
15
+ "run-on": {
16
+ "type": "string",
17
+ "description": ""
18
+ }
19
+ },
20
+ "additionalProperties": true,
21
+ "required": ["remote-transport"]
22
+ },
23
+ "task": { "$ref": "file:task"},
24
+ "parameters": {
25
+ "type": "object",
26
+ "description": "JSON formatted parameters to be provided to task"
27
+ }
28
+ },
29
+ "required": ["target", "task"]
30
+ }
@@ -0,0 +1,74 @@
1
+ {
2
+ "id": "file:task",
3
+ "$schema": "http://json-schema.org/draft-04/schema#",
4
+ "title": "Task",
5
+ "description": "Task schema for bolt-server",
6
+ "type": "object",
7
+ "properties": {
8
+ "name": {
9
+ "type": "string",
10
+ "description": "Task name"
11
+ },
12
+ "metadata": {
13
+ "type": "object",
14
+ "description": "The metadata object is optional, and contains metadata about the task being run",
15
+ "properties": {
16
+ "description": {
17
+ "type": "string",
18
+ "description": "The task description from it's metadata"
19
+ },
20
+ "parameters": {
21
+ "type": "object",
22
+ "description": "Object whose keys are parameter names, and values are objects",
23
+ "properties": {
24
+ "description": {
25
+ "type": "string",
26
+ "description": "Parameter description"
27
+ },
28
+ "type": {
29
+ "type": "string",
30
+ "description": "The type the parameter should accept"
31
+ },
32
+ "sensitive": {
33
+ "description": "Whether the task runner should treat the parameter value as sensitive",
34
+ "type": "boolean"
35
+ }
36
+ }
37
+ },
38
+ "input_method": {
39
+ "type": "string",
40
+ "enum": ["stdin", "environment", "powershell"],
41
+ "description": "What input method should be used to pass params to the task"
42
+ }
43
+ }
44
+ },
45
+ "files": {
46
+ "type": "array",
47
+ "description": "Description of task files",
48
+ "items": {
49
+ "type": "object",
50
+ "properties": {
51
+ "uri": {
52
+ "type": "object",
53
+ "description": "Where is the file"
54
+ },
55
+ "sha256": {
56
+ "type": "string",
57
+ "description": "checksum of file"
58
+ },
59
+ "filename": {
60
+ "type": "string",
61
+ "description": "Name of file"
62
+ },
63
+ "size": {
64
+ "type": "number",
65
+ "description": "Size of file"
66
+ }
67
+ }
68
+ },
69
+ "required": ["filename", "uri", "sha256"]
70
+ }
71
+ },
72
+ "required": ["name", "files"],
73
+ "additionalProperties": false
74
+ }
@@ -0,0 +1,252 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'ace/error'
4
+ require 'ace/fork_util'
5
+ require 'ace/puppet_util'
6
+ require 'ace/configurer'
7
+ require 'ace/plugin_cache'
8
+ require 'bolt_server/file_cache'
9
+ require 'bolt/executor'
10
+ require 'bolt/inventory'
11
+ require 'bolt/target'
12
+ require 'bolt/task/puppet_server'
13
+ require 'json-schema'
14
+ require 'json'
15
+ require 'sinatra'
16
+ require 'puppet/util/network_device/base'
17
+
18
+ module ACE
19
+ class TransportApp < Sinatra::Base
20
+ def initialize(config = nil)
21
+ @config = config
22
+ @executor = Bolt::Executor.new(0, load_config: false)
23
+ tasks_cache_dir = File.join(@config['cache-dir'], 'tasks')
24
+ @file_cache = BoltServer::FileCache.new(@config.data.merge('cache-dir' => tasks_cache_dir)).setup
25
+ environments_cache_dir = File.join(@config['cache-dir'], 'environments')
26
+ @plugins = ACE::PluginCache.new(environments_cache_dir).setup
27
+
28
+ @schemas = {
29
+ "run_task" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ace-run_task.json'))),
30
+ "execute_catalog" => JSON.parse(File.read(File.join(__dir__, 'schemas', 'ace-execute_catalog.json')))
31
+ }
32
+ shared_schema = JSON::Schema.new(JSON.parse(File.read(File.join(__dir__, 'schemas', 'task.json'))),
33
+ Addressable::URI.parse("file:task"))
34
+ JSON::Validator.add_schema(shared_schema)
35
+
36
+ ACE::PuppetUtil.init_global_settings(config['ssl-ca-cert'],
37
+ config['ssl-ca-crls'],
38
+ config['ssl-key'],
39
+ config['ssl-cert'],
40
+ config['cache-dir'],
41
+ URI.parse(config['puppet-server-uri']))
42
+
43
+ super(nil)
44
+ end
45
+
46
+ # Initialises the puppet target.
47
+ # @param certname The certificate name of the target.
48
+ # @param transport The transport provider of the target.
49
+ # @param target Target connection hash or legacy connection URI
50
+ # @return [Puppet device instance] Returns Puppet device instance
51
+ # @raise [puppetlabs/ace/invalid_param] If nil parameter or no connection detail found
52
+ # @example Connect to device.
53
+ # init_puppet_target('test_device.domain.com', 'panos', JSON.parse("target":{
54
+ # "remote-transport":"panos",
55
+ # "host":"fw.example.net",
56
+ # "user":"foo",
57
+ # "password":"wibble"
58
+ # }) ) => panos.device
59
+ def self.init_puppet_target(certname, transport, target)
60
+ unless target
61
+ raise ACE::Error.new("There was an error parsing the Puppet target. 'target' not found",
62
+ 'puppetlabs/ace/invalid_param')
63
+ end
64
+ unless certname
65
+ raise ACE::Error.new("There was an error parsing the Puppet compiler details. 'certname' not found",
66
+ 'puppetlabs/ace/invalid_param')
67
+ end
68
+ unless transport
69
+ raise ACE::Error.new("There was an error parsing the Puppet target. 'transport' not found",
70
+ 'puppetlabs/ace/invalid_param')
71
+ end
72
+
73
+ if target['uri']
74
+ if target['uri'] =~ URI::DEFAULT_PARSER.make_regexp
75
+ # Correct URL
76
+ url = target['uri']
77
+ else
78
+ raise ACE::Error.new("There was an error parsing the URI of the Puppet target",
79
+ 'puppetlabs/ace/invalid_param')
80
+ end
81
+ else
82
+ url = Hash[target.map { |(k, v)| [k.to_sym, v] }]
83
+ url.delete(:"remote-transport")
84
+ end
85
+
86
+ device_struct = Struct.new(:provider, :url, :name, :options)
87
+ # Return device
88
+ Puppet::Util::NetworkDevice.init(device_struct.new(transport,
89
+ url,
90
+ certname,
91
+ {}))
92
+ end
93
+
94
+ def scrub_stack_trace(result)
95
+ if result.dig(:result, '_error', 'details', 'stack_trace')
96
+ result[:result]['_error']['details'].reject! { |k| k == 'stack_trace' }
97
+ end
98
+ if result.dig(:result, '_error', 'details', 'backtrace')
99
+ result[:result]['_error']['details'].reject! { |k| k == 'backtrace' }
100
+ end
101
+ result
102
+ end
103
+
104
+ def validate_schema(schema, body)
105
+ schema_error = JSON::Validator.fully_validate(schema, body)
106
+ if schema_error.any?
107
+ ACE::Error.new("There was an error validating the request body.",
108
+ 'puppetlabs/ace/schema-error',
109
+ schema_error)
110
+ end
111
+ end
112
+
113
+ # returns a hash of trusted facts that will be used
114
+ # to request a catalog for the target
115
+ def self.trusted_facts(certname)
116
+ # if the certname is a valid FQDN, it will split
117
+ # it in to the correct hostname.domain format
118
+ # otherwise hostname will be the certname and domain
119
+ # will be empty
120
+ hostname, domain = certname.split('.', 2)
121
+ trusted_facts = {
122
+ "authenticated": "remote",
123
+ "extensions": {},
124
+ "certname": certname,
125
+ "hostname": hostname
126
+ }
127
+ trusted_facts[:domain] = domain if domain
128
+ trusted_facts
129
+ end
130
+
131
+ get "/" do
132
+ 200
133
+ end
134
+
135
+ post "/check" do
136
+ [200, 'OK']
137
+ end
138
+
139
+ # :nocov:
140
+ if ENV['RACK_ENV'] == 'dev'
141
+ get '/admin/gc' do
142
+ GC.start
143
+ 200
144
+ end
145
+ end
146
+
147
+ get '/admin/gc_stat' do
148
+ [200, GC.stat.to_json]
149
+ end
150
+ # :nocov:
151
+
152
+ # run this with "curl -X POST http://0.0.0.0:44633/run_task -d '{}'"
153
+ post '/run_task' do
154
+ content_type :json
155
+
156
+ begin
157
+ body = JSON.parse(request.body.read)
158
+ rescue StandardError => e
159
+ request_error = {
160
+ _error: ACE::Error.new(e.message,
161
+ 'puppetlabs/ace/request_exception',
162
+ class: e.class, backtrace: e.backtrace)
163
+ }
164
+ return [400, request_error.to_json]
165
+ end
166
+
167
+ error = validate_schema(@schemas["run_task"], body)
168
+ return [400, error.to_json] unless error.nil?
169
+
170
+ opts = body['target'].merge('protocol' => 'remote')
171
+
172
+ # This is a workaround for Bolt due to the way it expects to receive the target info
173
+ # see: https://github.com/puppetlabs/bolt/pull/915#discussion_r268280535
174
+ # Systems calling into ACE will need to determine the nodename/certname and pass this as `name`
175
+ target = [Bolt::Target.new(body['target']['host'] || body['target']['name'], opts)]
176
+
177
+ inventory = Bolt::Inventory.new(nil)
178
+
179
+ target.first.inventory = inventory
180
+
181
+ task = Bolt::Task::PuppetServer.new(body['task'], @file_cache)
182
+
183
+ parameters = body['parameters'] || {}
184
+
185
+ result = ForkUtil.isolate do
186
+ # Since this will only be on one node we can just return the first result
187
+ results = @executor.run_task(target, task, parameters)
188
+ scrub_stack_trace(results.first.status_hash)
189
+ end
190
+ [200, result.to_json]
191
+ end
192
+
193
+ post '/execute_catalog' do
194
+ content_type :json
195
+
196
+ begin
197
+ body = JSON.parse(request.body.read)
198
+ rescue StandardError => e
199
+ request_error = {
200
+ _error: ACE::Error.new(e.message,
201
+ 'puppetlabs/ace/request_exception',
202
+ class: e.class, backtrace: e.backtrace)
203
+ }
204
+ return [400, request_error.to_json]
205
+ end
206
+
207
+ error = validate_schema(@schemas["execute_catalog"], body)
208
+ return [400, error.to_json] unless error.nil?
209
+
210
+ environment = body['compiler']['environment']
211
+ certname = body['compiler']['certname']
212
+
213
+ # TODO: (needs groomed) - proper error handling, errors within the block can be rescued
214
+ # and handled correctly, ACE::Errors we can handle similar to the task
215
+ # workflow, errors within the Configuer and not as pleasant - can have
216
+ # _some_ control over them, especially around status codes from
217
+ # the /v4/catalog endpoint
218
+ @plugins.with_synced_libdir(environment, certname) do
219
+ ACE::TransportApp.init_puppet_target(certname, body['target']['remote-transport'], body['target'])
220
+ configurer = ACE::Configurer.new(body['compiler']['transaction_uuid'], body['compiler']['job_id'])
221
+ configurer.run(transport_name: certname,
222
+ environment: environment,
223
+ network_device: true,
224
+ pluginsync: false,
225
+ trusted_facts: ACE::TransportApp.trusted_facts(certname))
226
+ end
227
+
228
+ # simulate expected error cases
229
+ if body['compiler']['certname'] == 'fail.example.net'
230
+ [200, { _error: {
231
+ msg: 'catalog compile failed',
232
+ kind: 'puppetlabs/ace/compile_failed',
233
+ details: 'upstream api errors go here'
234
+ } }.to_json]
235
+ elsif body['compiler']['certname'] == 'credentials.example.net'
236
+ [200, { _error: {
237
+ msg: 'target specification invalid',
238
+ kind: 'puppetlabs/ace/target_spec',
239
+ details: 'upstream api errors go here'
240
+ } }.to_json]
241
+ elsif body['compiler']['certname'] == 'reports.example.net'
242
+ [200, { _error: {
243
+ msg: 'report submission failed',
244
+ kind: 'puppetlabs/ace/reporting_failed',
245
+ details: 'upstream api errors go here'
246
+ } }.to_json]
247
+ else
248
+ [200, '{}']
249
+ end
250
+ end
251
+ end
252
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ACE
4
+ VERSION = "0.9.0"
5
+ end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'puppet/resource/catalog'
4
+ require 'puppet/indirector/rest'
5
+
6
+ module Puppet
7
+ class Resource
8
+ class Catalog
9
+ class Certless < Puppet::Indirector::REST
10
+ desc "Find certless catalogs over HTTP via REST."
11
+
12
+ # Override the REST indirector headers for the certless
13
+ # catalog indirector as the `puppet/pson` header throws of the
14
+ # request body and wraps it in a "value":{<request>} which
15
+ # causes a validation error on the v4/catalog endpoint
16
+ def headers
17
+ common_headers = {
18
+ "Content-Type" => 'text/json',
19
+ "Accept" => 'application/json',
20
+ Puppet::Network::HTTP::HEADER_PUPPET_VERSION => Puppet.version
21
+ }
22
+
23
+ add_accept_encoding(common_headers)
24
+ end
25
+
26
+ def find(request)
27
+ uri = "/puppet/v4/catalog"
28
+
29
+ body = {
30
+ "certname": request.key,
31
+ "persistence": {
32
+ "facts": true, "catalog": true
33
+ },
34
+ "environment": request.environment.name.to_s,
35
+ "facts": {
36
+ "values": request.options[:transport_facts]
37
+ },
38
+ "trusted_facts": {
39
+ "values": request.options[:trusted_facts]
40
+ },
41
+ "transaction_uuid": request.options[:transaction_uuid],
42
+ "job_id": request.options[:job_id],
43
+ "options": {
44
+ "prefer_requested_environment": true,
45
+ "capture_logs": false
46
+ }
47
+ }
48
+
49
+ response = do_request(request) do |req|
50
+ http_post(req, uri, body.to_json, headers)
51
+ end
52
+
53
+ if is_http_200?(response)
54
+ content_type, body = parse_response(response)
55
+ # the response from the `v4/catalog` endpoint is in the format of
56
+ # {"catalog": {}} whereas the configurer expects it to be a
57
+ # flatter structurer, so passing it the catalog contents from the body
58
+ # is suited as the API is unlikely to change for
59
+ # this release
60
+ result = deserialize_find(content_type, JSON.parse(body)['catalog'].to_json)
61
+ result.name = request.key if result.respond_to?(:name=)
62
+ result
63
+
64
+ elsif is_http_404?(response)
65
+ return nil unless request.options[:fail_on_404]
66
+
67
+ # 404 can get special treatment as the indirector API can not produce a meaningful
68
+ # reason to why something is not found - it may not be the thing the user is
69
+ # expecting to find that is missing, but something else (like the environment).
70
+ # While this way of handling the issue is not perfect, there is at least an error
71
+ # that makes a user aware of the reason for the failure.
72
+ #
73
+ _, body = parse_response(response)
74
+ msg = format(_("Find %<uri>s resulted in 404 with the message: %<body>s"),
75
+ uri: elide(uri, 100),
76
+ body: body)
77
+ raise Puppet::Error, msg
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end