choria-mcorpc-support 2.21.1 → 2.23.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. data/lib/mcollective.rb +1 -2
  3. data/lib/mcollective/agent/bolt_tasks.ddl +253 -0
  4. data/lib/mcollective/agent/bolt_tasks.json +365 -0
  5. data/lib/mcollective/agent/bolt_tasks.rb +178 -0
  6. data/lib/mcollective/agent/choria_util.ddl +152 -0
  7. data/lib/mcollective/agent/choria_util.json +244 -0
  8. data/lib/mcollective/agent/rpcutil.ddl +8 -4
  9. data/lib/mcollective/agent/rpcutil.json +333 -0
  10. data/lib/mcollective/agent/scout.ddl +169 -0
  11. data/lib/mcollective/agent/scout.json +224 -0
  12. data/lib/mcollective/agents.rb +7 -6
  13. data/lib/mcollective/aggregate.rb +4 -4
  14. data/lib/mcollective/aggregate/average.rb +2 -2
  15. data/lib/mcollective/aggregate/base.rb +2 -2
  16. data/lib/mcollective/aggregate/result.rb +3 -3
  17. data/lib/mcollective/aggregate/result/collection_result.rb +2 -2
  18. data/lib/mcollective/aggregate/result/numeric_result.rb +2 -2
  19. data/lib/mcollective/aggregate/sum.rb +2 -2
  20. data/lib/mcollective/aggregate/summary.rb +3 -4
  21. data/lib/mcollective/application.rb +57 -21
  22. data/lib/mcollective/application/choria.rb +189 -0
  23. data/lib/mcollective/application/completion.rb +6 -6
  24. data/lib/mcollective/application/facts.rb +11 -11
  25. data/lib/mcollective/application/federation.rb +237 -0
  26. data/lib/mcollective/application/find.rb +4 -4
  27. data/lib/mcollective/application/help.rb +3 -3
  28. data/lib/mcollective/application/inventory.rb +3 -341
  29. data/lib/mcollective/application/ping.rb +5 -51
  30. data/lib/mcollective/application/playbook.rb +207 -0
  31. data/lib/mcollective/application/plugin.rb +106 -106
  32. data/lib/mcollective/application/rpc.rb +3 -108
  33. data/lib/mcollective/application/tasks.rb +425 -0
  34. data/lib/mcollective/applications.rb +11 -10
  35. data/lib/mcollective/audit/choria.rb +33 -0
  36. data/lib/mcollective/cache.rb +2 -4
  37. data/lib/mcollective/client.rb +11 -10
  38. data/lib/mcollective/config.rb +21 -34
  39. data/lib/mcollective/connector/base.rb +2 -1
  40. data/lib/mcollective/connector/nats.ddl +9 -0
  41. data/lib/mcollective/connector/nats.rb +450 -0
  42. data/lib/mcollective/data.rb +8 -3
  43. data/lib/mcollective/data/agent_data.rb +1 -1
  44. data/lib/mcollective/data/base.rb +6 -5
  45. data/lib/mcollective/data/bolt_task_data.ddl +90 -0
  46. data/lib/mcollective/data/bolt_task_data.rb +32 -0
  47. data/lib/mcollective/data/collective_data.rb +1 -1
  48. data/lib/mcollective/data/fact_data.rb +6 -6
  49. data/lib/mcollective/data/fstat_data.rb +2 -4
  50. data/lib/mcollective/data/result.rb +7 -2
  51. data/lib/mcollective/ddl/agentddl.rb +5 -17
  52. data/lib/mcollective/ddl/base.rb +10 -13
  53. data/lib/mcollective/discovery.rb +24 -39
  54. data/lib/mcollective/discovery/choria.ddl +11 -0
  55. data/lib/mcollective/discovery/choria.rb +223 -0
  56. data/lib/mcollective/discovery/flatfile.rb +7 -8
  57. data/lib/mcollective/discovery/mc.rb +2 -2
  58. data/lib/mcollective/discovery/stdin.rb +17 -18
  59. data/lib/mcollective/exceptions.rb +13 -0
  60. data/lib/mcollective/facts/base.rb +9 -9
  61. data/lib/mcollective/facts/yaml_facts.rb +12 -12
  62. data/lib/mcollective/generators.rb +3 -3
  63. data/lib/mcollective/generators/agent_generator.rb +3 -4
  64. data/lib/mcollective/generators/base.rb +14 -15
  65. data/lib/mcollective/generators/data_generator.rb +5 -6
  66. data/lib/mcollective/log.rb +2 -2
  67. data/lib/mcollective/logger/base.rb +3 -2
  68. data/lib/mcollective/logger/console_logger.rb +10 -10
  69. data/lib/mcollective/logger/file_logger.rb +7 -7
  70. data/lib/mcollective/logger/syslog_logger.rb +11 -15
  71. data/lib/mcollective/message.rb +8 -39
  72. data/lib/mcollective/monkey_patches.rb +2 -4
  73. data/lib/mcollective/optionparser.rb +2 -1
  74. data/lib/mcollective/pluginmanager.rb +3 -5
  75. data/lib/mcollective/pluginpackager.rb +1 -3
  76. data/lib/mcollective/pluginpackager/agent_definition.rb +3 -8
  77. data/lib/mcollective/pluginpackager/forge_packager.rb +7 -9
  78. data/lib/mcollective/pluginpackager/standard_definition.rb +1 -2
  79. data/lib/mcollective/registration/base.rb +18 -16
  80. data/lib/mcollective/rpc.rb +2 -4
  81. data/lib/mcollective/rpc/actionrunner.rb +16 -18
  82. data/lib/mcollective/rpc/agent.rb +26 -43
  83. data/lib/mcollective/rpc/audit.rb +1 -0
  84. data/lib/mcollective/rpc/client.rb +67 -85
  85. data/lib/mcollective/rpc/helpers.rb +55 -62
  86. data/lib/mcollective/rpc/progress.rb +2 -2
  87. data/lib/mcollective/rpc/reply.rb +17 -19
  88. data/lib/mcollective/rpc/request.rb +7 -5
  89. data/lib/mcollective/rpc/result.rb +6 -8
  90. data/lib/mcollective/rpc/stats.rb +49 -58
  91. data/lib/mcollective/security/base.rb +13 -56
  92. data/lib/mcollective/security/choria.rb +765 -0
  93. data/lib/mcollective/shell.rb +9 -4
  94. data/lib/mcollective/signer/base.rb +28 -0
  95. data/lib/mcollective/signer/choria.rb +185 -0
  96. data/lib/mcollective/ssl.rb +8 -6
  97. data/lib/mcollective/util.rb +58 -55
  98. data/lib/mcollective/util/bolt_support.rb +176 -0
  99. data/lib/mcollective/util/bolt_support/plan_runner.rb +167 -0
  100. data/lib/mcollective/util/bolt_support/task_result.rb +94 -0
  101. data/lib/mcollective/util/bolt_support/task_results.rb +128 -0
  102. data/lib/mcollective/util/choria.rb +946 -0
  103. data/lib/mcollective/util/indifferent_hash.rb +12 -0
  104. data/lib/mcollective/util/natswrapper.rb +242 -0
  105. data/lib/mcollective/util/playbook.rb +435 -0
  106. data/lib/mcollective/util/playbook/data_stores.rb +201 -0
  107. data/lib/mcollective/util/playbook/data_stores/base.rb +99 -0
  108. data/lib/mcollective/util/playbook/data_stores/consul_data_store.rb +88 -0
  109. data/lib/mcollective/util/playbook/data_stores/environment_data_store.rb +33 -0
  110. data/lib/mcollective/util/playbook/data_stores/etcd_data_store.rb +42 -0
  111. data/lib/mcollective/util/playbook/data_stores/file_data_store.rb +106 -0
  112. data/lib/mcollective/util/playbook/data_stores/shell_data_store.rb +103 -0
  113. data/lib/mcollective/util/playbook/inputs.rb +265 -0
  114. data/lib/mcollective/util/playbook/nodes.rb +207 -0
  115. data/lib/mcollective/util/playbook/nodes/mcollective_nodes.rb +86 -0
  116. data/lib/mcollective/util/playbook/nodes/pql_nodes.rb +40 -0
  117. data/lib/mcollective/util/playbook/nodes/shell_nodes.rb +55 -0
  118. data/lib/mcollective/util/playbook/nodes/terraform_nodes.rb +65 -0
  119. data/lib/mcollective/util/playbook/nodes/yaml_nodes.rb +47 -0
  120. data/lib/mcollective/util/playbook/playbook_logger.rb +47 -0
  121. data/lib/mcollective/util/playbook/puppet_logger.rb +51 -0
  122. data/lib/mcollective/util/playbook/report.rb +152 -0
  123. data/lib/mcollective/util/playbook/task_result.rb +55 -0
  124. data/lib/mcollective/util/playbook/tasks.rb +196 -0
  125. data/lib/mcollective/util/playbook/tasks/base.rb +45 -0
  126. data/lib/mcollective/util/playbook/tasks/graphite_event_task.rb +64 -0
  127. data/lib/mcollective/util/playbook/tasks/mcollective_task.rb +356 -0
  128. data/lib/mcollective/util/playbook/tasks/shell_task.rb +93 -0
  129. data/lib/mcollective/util/playbook/tasks/slack_task.rb +105 -0
  130. data/lib/mcollective/util/playbook/tasks/webhook_task.rb +136 -0
  131. data/lib/mcollective/util/playbook/template_util.rb +98 -0
  132. data/lib/mcollective/util/playbook/uses.rb +169 -0
  133. data/lib/mcollective/util/tasks_support.rb +751 -0
  134. data/lib/mcollective/util/tasks_support/cli.rb +260 -0
  135. data/lib/mcollective/util/tasks_support/default_formatter.rb +138 -0
  136. data/lib/mcollective/util/tasks_support/json_formatter.rb +108 -0
  137. data/lib/mcollective/validator.rb +6 -1
  138. data/lib/mcollective/validator/bolt_task_name_validator.ddl +7 -0
  139. data/lib/mcollective/validator/bolt_task_name_validator.rb +11 -0
  140. data/lib/mcollective/validator/length_validator.rb +1 -3
  141. metadata +65 -6
  142. data/lib/mcollective/application/describe_filter.rb +0 -87
  143. data/lib/mcollective/matcher.rb +0 -220
  144. data/lib/mcollective/matcher/parser.rb +0 -128
  145. data/lib/mcollective/matcher/scanner.rb +0 -241
@@ -0,0 +1,946 @@
1
+ require "net/http"
2
+ require "resolv"
3
+
4
+ module MCollective
5
+ module Util
6
+ class Choria
7
+ class UserError < StandardError; end
8
+
9
+ class Abort < StandardError; end
10
+
11
+ unless defined?(Choria::VERSION) # rubocop:disable Style/IfUnlessModifier
12
+ VERSION = "0.19.0".freeze
13
+ end
14
+
15
+ attr_writer :ca
16
+
17
+ def initialize(check_ssl=true)
18
+ @config = Config.instance
19
+
20
+ check_ssl_setup if check_ssl
21
+ end
22
+
23
+ # Determines the configured path to the NATS credentials, empty when not set
24
+ #
25
+ # @return [String]
26
+ def credential_file
27
+ get_option("nats.credentials", "")
28
+ end
29
+
30
+ # Determines if a credential file is configured
31
+ #
32
+ # @return [Boolean]
33
+ def credential_file?
34
+ credential_file != ""
35
+ end
36
+
37
+ # Determines if we are connecting to NGS based on credentials and the nats.ngs setting
38
+ #
39
+ # @return [Boolean]
40
+ def ngs?
41
+ credential_file != "" && Util.str_to_bool(get_option("nats.ngs", "false"))
42
+ end
43
+
44
+ # Attempts to load the optional nkeys library
45
+ #
46
+ # @return [Boolean]
47
+ def nkeys?
48
+ require "nkeys"
49
+ true
50
+ rescue LoadError
51
+ false
52
+ end
53
+
54
+ # Creates a new TasksSupport instance with the configured cache dir
55
+ #
56
+ # @return [TasksSupport]
57
+ def tasks_support
58
+ require_relative "tasks_support"
59
+
60
+ Util::TasksSupport.new(self, tasks_cache_dir)
61
+ end
62
+
63
+ # Determines the Tasks Cache dir
64
+ #
65
+ # @return [String] path to the cache
66
+ def tasks_cache_dir
67
+ if Util.windows?
68
+ File.join(Util.windows_prefix, "tasks-cache")
69
+ elsif Process.uid == 0
70
+ "/opt/puppetlabs/mcollective/tasks-cache"
71
+ else
72
+ File.expand_path("~/.puppetlabs/mcollective/tasks-cache")
73
+ end
74
+ end
75
+
76
+ # Determines the Tasks Spool directory
77
+ #
78
+ # @return [String] path to the spool
79
+ def tasks_spool_dir
80
+ if Util.windows?
81
+ File.join(Util.windows_prefix, "tasks-spool")
82
+ elsif Process.uid == 0
83
+ "/opt/puppetlabs/mcollective/tasks-spool"
84
+ else
85
+ File.expand_path("~/.puppetlabs/mcollective/tasks-spool")
86
+ end
87
+ end
88
+
89
+ # Determines if there are any federations configured
90
+ #
91
+ # @return [Boolean]
92
+ def federated?
93
+ !federation_collectives.empty?
94
+ end
95
+
96
+ # List of active collectives that form the federation
97
+ #
98
+ # @return [Array<String>]
99
+ def federation_collectives
100
+ if override_networks = env_fetch("CHORIA_FED_COLLECTIVE", nil)
101
+ override_networks.split(",").map(&:strip).reject(&:empty?)
102
+ else
103
+ get_option("choria.federation.collectives", "").split(",").map(&:strip).reject(&:empty?)
104
+ end
105
+ end
106
+
107
+ # Retrieves a DNS resolver
108
+ #
109
+ # @note mainly used for testing
110
+ # @return [Resolv::DNS]
111
+ def resolver
112
+ Resolv::DNS.new
113
+ end
114
+
115
+ # Retrieves the domain from facter networking.domain if facter is found
116
+ #
117
+ # Potentially we could use the local facts in mcollective but that's a chicken
118
+ # and egg and sometimes its only set after initial connection if something like
119
+ # a cron job generates the yaml cache file
120
+ #
121
+ # @return [String,nil]
122
+ def facter_domain
123
+ if path = facter_cmd
124
+ `"#{path}" networking.domain`.chomp
125
+ end
126
+ end
127
+
128
+ # Determines the domain to do SRV lookups in
129
+ #
130
+ # This is settable using choria.srv_domain and defaults
131
+ # to the domain as reported by facter
132
+ #
133
+ # @return [String]
134
+ def srv_domain
135
+ get_option("choria.srv_domain", nil) || facter_domain
136
+ end
137
+
138
+ # Determines the SRV records to look up
139
+ #
140
+ # If an option choria.srv_domain is set that will be used else facter will be consulted,
141
+ # if neither of those provide a domain name a empty list is returned
142
+ #
143
+ # @param keys [Array<String>] list of keys to lookup
144
+ # @return [Array<String>] list of SRV records
145
+ def srv_records(keys)
146
+ domain = srv_domain
147
+
148
+ if domain.nil? || domain.empty?
149
+ Log.warn("Cannot look up SRV records, facter is not functional and choria.srv_domain was not supplied")
150
+ return []
151
+ end
152
+
153
+ keys.map do |key|
154
+ "%s.%s" % [key, domain]
155
+ end
156
+ end
157
+
158
+ # Determines if SRV records should be used
159
+ #
160
+ # Setting choria.use_srv_records to anything other than t, true, yes or 1 will disable
161
+ # SRV records
162
+ #
163
+ # @return [Boolean]
164
+ def should_use_srv?
165
+ ["t", "true", "yes", "1"].include?(get_option("choria.use_srv_records", "1").downcase)
166
+ end
167
+
168
+ # Query DNS for a series of records
169
+ #
170
+ # The given records will be passed through {#srv_records} to figure out the domain to query in.
171
+ #
172
+ # Querying of records can be bypassed by setting choria.use_srv_records to false
173
+ #
174
+ # @yield [Hash] each record for modification by the caller
175
+ # @param records [Array<String>] the records to query without their domain parts
176
+ # @return [Array<Hash>] with keys :port, :priority, :weight and :target
177
+ def query_srv_records(records)
178
+ unless should_use_srv?
179
+ Log.info("Skipping SRV record queries due to choria.query_srv_records setting")
180
+ return []
181
+ end
182
+
183
+ answers = Array(srv_records(records)).map do |record|
184
+ Log.debug("Attempting to resolve SRV record %s" % record)
185
+ answers = resolver.getresources(record, Resolv::DNS::Resource::IN::SRV)
186
+ Log.debug("Found %d SRV records for %s" % [answers.size, record])
187
+ answers
188
+ end.flatten
189
+
190
+ answers = answers.sort_by(&:priority).chunk(&:priority).sort
191
+ answers = sort_srv_answers(answers)
192
+
193
+ answers.map do |result|
194
+ Log.debug("Found %s:%s with priority %s and weight %s" % [result.target, result.port, result.priority, result.weight])
195
+
196
+ ans = {
197
+ :port => result.port,
198
+ :priority => result.priority,
199
+ :weight => result.weight,
200
+ :target => result.target
201
+ }
202
+
203
+ yield(ans) if block_given?
204
+
205
+ ans
206
+ end
207
+ end
208
+
209
+ # Sorts SRV records according to rfc2782
210
+ #
211
+ # @note this is probably still not correct :( so horrible
212
+ # @param answers [Array<Resolv::DNS::Resource::IN::SRV>]
213
+ # @return [Array<Resolv::DNS::Resource::IN::SRV>] sorted records
214
+ def sort_srv_answers(answers)
215
+ sorted_answers = []
216
+
217
+ # this is roughly based on the resolv-srv and supposedly mostly rfc2782 compliant
218
+ answers.each do |_, available|
219
+ total_weight = available.inject(0) {|a, e| a + e.weight + 1 }
220
+
221
+ until available.empty?
222
+ selector = Integer(rand * total_weight) + 1
223
+ selected_idx = available.find_index do |e|
224
+ selector -= e.weight + 1
225
+ selector <= 0
226
+ end
227
+ selected = available.delete_at(selected_idx)
228
+
229
+ total_weight -= selected.weight + 1
230
+
231
+ sorted_answers << selected
232
+ end
233
+ end
234
+
235
+ sorted_answers
236
+ end
237
+
238
+ # Create a Net::HTTP instance optionally set up with the Puppet certs
239
+ #
240
+ # If the client_private_key and client_public_cert both exist they will
241
+ # be used to validate the connection
242
+ #
243
+ # If the ca_path exist it will be used and full verification will be enabled
244
+ #
245
+ # @param server [Hash] as returned by {#try_srv}
246
+ # @param force_puppet_ssl [boolean] when true will call {#check_ssl_setup} and so force Puppet certs
247
+ # @return [Net::HTTP]
248
+ def https(server, force_puppet_ssl=false)
249
+ Log.debug("Creating new HTTPS connection to %s:%s" % [server[:target], server[:port]])
250
+
251
+ check_ssl_setup if force_puppet_ssl
252
+
253
+ http = Net::HTTP.new(server[:target], server[:port])
254
+
255
+ http.use_ssl = true
256
+
257
+ if has_client_private_key? && has_client_public_cert?
258
+ http.cert = OpenSSL::X509::Certificate.new(File.read(client_public_cert))
259
+ http.key = OpenSSL::PKey::RSA.new(File.read(client_private_key))
260
+ end
261
+
262
+ if has_ca?
263
+ http.ca_file = ca_path
264
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
265
+ else
266
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
267
+ end
268
+
269
+ http
270
+ end
271
+
272
+ # Creates a Net::HTTP::Get instance for a path that defaults to accepting JSON
273
+ #
274
+ # @param path [String]
275
+ # @return [Net::HTTP::Get]
276
+ def http_get(path, headers=nil)
277
+ headers ||= {}
278
+ headers = {
279
+ "Accept" => "application/json",
280
+ "User-Agent" => "Choria version %s http://choria.io" % VERSION
281
+ }.merge(headers)
282
+
283
+ Net::HTTP::Get.new(path, headers)
284
+ end
285
+
286
+ # Creates a Net::HTTP::Post instance for a path that defaults to accepting JSON
287
+ #
288
+ # @param path [String]
289
+ # @return [Net::HTTP::Post]
290
+ def http_post(path, headers=nil)
291
+ headers ||= {}
292
+ headers = {
293
+ "Accept" => "application/json",
294
+ "User-Agent" => "Choria version %s http://choria.io" % VERSION
295
+ }.merge(headers)
296
+
297
+ Net::HTTP::Post.new(path, headers)
298
+ end
299
+
300
+ # Does a proxied discovery request
301
+ #
302
+ # @param query [Hash] Discovery query as per pdbproxy standard
303
+ # @return [Array] JSON parsed result set
304
+ # @raise [StandardError] on any failures
305
+ def proxy_discovery_query(query)
306
+ transport = https(discovery_server, true)
307
+ request = http_get("/v1/discover")
308
+ request.body = query.to_json
309
+ request["Content-Type"] = "application/json"
310
+
311
+ resp, data = transport.request(request)
312
+
313
+ raise("Failed to make request to Discovery Proxy: %s: %s" % [resp.code, resp.body]) unless resp.code == "200"
314
+
315
+ result = JSON.parse(data || resp.body)
316
+
317
+ result["nodes"]
318
+ end
319
+
320
+ # Extract certnames from PQL results, deactivated nodes are ignored
321
+ #
322
+ # @param results [Array]
323
+ # @return [Array<String>] list of certnames
324
+ def pql_extract_certnames(results)
325
+ results.reject {|n| n["deactivated"]}.map {|n| n["certname"]}.compact
326
+ end
327
+
328
+ # Performs a PQL query against the configured PuppetDB
329
+ #
330
+ # @param query [String] PQL Query
331
+ # @param only_certnames [Boolean] extract certnames from the results
332
+ # @return [Array] JSON parsed result set
333
+ # @raise [StandardError] on any failures
334
+ def pql_query(query, only_certnames=false)
335
+ Log.debug("Performing PQL query: %s" % query)
336
+
337
+ path = "/pdb/query/v4?%s" % URI.encode_www_form("query" => query)
338
+
339
+ resp, data = https(puppetdb_server, true).request(http_get(path))
340
+
341
+ raise("Failed to make request to PuppetDB: %s: %s: %s" % [resp.code, resp.message, resp.body]) unless resp.code == "200"
342
+
343
+ result = JSON.parse(data || resp.body)
344
+
345
+ Log.debug("Found %d records for query %s" % [result.size, query])
346
+
347
+ only_certnames ? pql_extract_certnames(result) : result
348
+ end
349
+
350
+ # Checks if all the required SSL files exist
351
+ #
352
+ # @param log [Boolean] log warnings when true
353
+ # @return [Boolean]
354
+ def have_ssl_files?(log=true)
355
+ [client_public_cert, client_private_key, ca_path].map do |path|
356
+ Log.debug("Checking for SSL file %s" % path)
357
+
358
+ if File.exist?(path)
359
+ true
360
+ else
361
+ Log.warn("Cannot find SSL file %s" % path) if log
362
+ false
363
+ end
364
+ end.all?
365
+ end
366
+
367
+ # Validates a certificate against the CA
368
+ #
369
+ # @param pubcert [String] PEM encoded X509 public certificate
370
+ # @param name [String] name that should be present in the certificate
371
+ # @param log [Boolean] log warnings when true
372
+ # @return [String,false] when succesful, the certname else false
373
+ # @raise [StandardError] in case OpenSSL fails to open the various certificates
374
+ # @raise [OpenSSL::X509::CertificateError] if the CA is invalid
375
+ def valid_certificate?(pubcert, name, log=true)
376
+ return false unless name
377
+
378
+ raise("Cannot find or read the CA in %s, cannot verify public certificate" % ca_path) unless File.readable?(ca_path)
379
+
380
+ certs = parse_pubcert(pubcert, log)
381
+
382
+ return false if certs.empty?
383
+
384
+ incoming = certs.first
385
+
386
+ chain = certs[1..-1]
387
+
388
+ begin
389
+ ca = OpenSSL::X509::Store.new.add_file(ca_path)
390
+ rescue OpenSSL::X509::StoreError
391
+ Log.warn("Failed to load CA from %s: %s: %s" % [ca_path, $!.class, $!.to_s]) if log
392
+ raise
393
+ end
394
+
395
+ unless ca.verify(incoming, chain)
396
+ Log.warn("Failed to verify certificate %s against CA %s in %s" % [incoming.subject.to_s, incoming.issuer.to_s, ca_path]) if log
397
+
398
+ return false
399
+ end
400
+
401
+ Log.debug("Verified certificate %s against CA %s" % [incoming.subject.to_s, incoming.issuer.to_s]) if log
402
+
403
+ if !remote_signer_configured? && !OpenSSL::SSL.verify_certificate_identity(incoming, name)
404
+ raise("Could not parse certificate with subject %s as it has no CN part, or name %s invalid" % [incoming.subject.to_s, name])
405
+ end
406
+
407
+ name
408
+ end
409
+
410
+ # Determines if a remote signer is configured
411
+ #
412
+ # @return [Boolean]
413
+ def remote_signer_configured?
414
+ url = get_option("choria.security.request_signer.url", nil)
415
+
416
+ ![nil, ""].include?(url)
417
+ end
418
+
419
+ # Utility function to split a chained certificate String into an Array
420
+ #
421
+ # @param pemdata [String] PEM encoded certificate
422
+ # @return [Array<String,nil>]
423
+ def ssl_split_pem(pemdata)
424
+ # Chained certificates typically have the public certificate, along
425
+ # with every intermediate certificiate.
426
+ # OpenSSL will stop at the first certificate when using OpenSSL::X509::Certificate.new,
427
+ # so we need to separate them into a list
428
+ pemdata.scan(/-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----/m)
429
+ end
430
+
431
+ # Split a string containing chained certificates into an Array of OpenSSL::X509::Certificate.
432
+ #
433
+ # @param pemdata [String]
434
+ # @return [Array<OpenSSL::X509::Certificate,nil>]
435
+ def ssl_parse_chain(pemdata)
436
+ ssl_split_pem(pemdata).map do |cpem|
437
+ OpenSSL::X509::Certificate.new(cpem)
438
+ end
439
+ end
440
+
441
+ # Parses a public cert
442
+ #
443
+ # @param pubcert [String] PEM encoded public certificate
444
+ # @param log [Boolean] log warnings when true
445
+ # @return [Array<OpenSSL::X509::Certificate,nil>]
446
+ def parse_pubcert(pubcert, log=true)
447
+ ssl_parse_chain(pubcert)
448
+ rescue OpenSSL::X509::CertificateError
449
+ Log.warn("Received certificate is not a valid x509 certificate: %s: %s" % [$!.class, $!.to_s]) if log
450
+ nil
451
+ end
452
+
453
+ # The callerid for the current client
454
+ #
455
+ # @return [String]
456
+ # @raise [Exception] when remote JWT is invalid
457
+ def callerid
458
+ PluginManager["security_plugin"].callerid
459
+ end
460
+
461
+ # Checks all the required SSL files exist
462
+ #
463
+ # @param log [Boolean] log warnings when true
464
+ # @return [Boolean]
465
+ # @raise [StandardError] on failure
466
+ def check_ssl_setup(log=true)
467
+ return true if $choria_unsafe_disable_protocol_security # rubocop:disable Style/GlobalVars
468
+ return true if anon_tls?
469
+
470
+ raise(UserError, "The Choria client cannot be run as root") if Process.uid == 0 && PluginManager["security_plugin"].initiated_by == :client
471
+
472
+ raise(UserError, "Not all required SSL files exist") unless have_ssl_files?(log)
473
+
474
+ embedded_certname = nil
475
+
476
+ begin
477
+ embedded_certname = valid_certificate?(File.read(client_public_cert), certname)
478
+ rescue
479
+ raise(UserError, "The public certificate was not signed by the configured CA")
480
+ end
481
+
482
+ unless embedded_certname == certname
483
+ raise(UserError, "The certname %s found in %s does not match the configured certname of %s" % [embedded_certname, client_public_cert, certname])
484
+ end
485
+
486
+ true
487
+ end
488
+
489
+ # Resolves server lists based on config and SRV records
490
+ #
491
+ # Attempts to find server in the following order:
492
+ #
493
+ # * Configured hosts in `config_option`
494
+ # * SRV lookups of `srv_records`
495
+ # * Defaults
496
+ # * nil otherwise
497
+ #
498
+ # @param config_option [String] config to lookup
499
+ # @param srv_records [Array<String>] list of SRV records to query
500
+ # @param default_host [String] host to use when not found
501
+ # @param default_port [String] port to use when not found
502
+ # @return [Array, nil] groups of host and port pairs
503
+ def server_resolver(config_option, srv_records, default_host=nil, default_port=nil)
504
+ if servers = get_option(config_option, nil)
505
+ hosts = servers.split(",").map do |server|
506
+ server.split(":").map(&:strip)
507
+ end
508
+
509
+ return hosts
510
+ end
511
+
512
+ srv_answers = query_srv_records(srv_records)
513
+
514
+ unless srv_answers.empty?
515
+ hosts = srv_answers.map do |answer|
516
+ [answer[:target], answer[:port]]
517
+ end
518
+
519
+ return hosts
520
+ end
521
+
522
+ [[default_host, default_port]] if default_host && default_port
523
+ end
524
+
525
+ # Finds the middleware hosts in config or DNS
526
+ #
527
+ # Attempts to find servers in the following order:
528
+ #
529
+ # * connects.ngs.global if configured to be ngs and empty choria.middleware_hosts
530
+ # * Any federation servers if in a federation
531
+ # * Configured hosts in choria.middleware_hosts
532
+ # * SRV lookups in _mcollective-server._tcp and _x-puppet-mcollective._tcp
533
+ # * Supplied defaults
534
+ #
535
+ # Eventually it's intended that other middleware might be supported
536
+ # this would provide a single way to configure them all
537
+ #
538
+ # @param default_host [String] default hostname
539
+ # @param default_port [String] default port
540
+ # @return [Array<Array<String, String>>] groups of host and port
541
+ def middleware_servers(default_host="puppet", default_port="4222")
542
+ return [["connect.ngs.global", "4222"]] if ngs? && !has_option?("choria.middleware_hosts")
543
+
544
+ if federated? && federation = federation_middleware_servers
545
+ return federation
546
+ end
547
+
548
+ server_resolver("choria.middleware_hosts", ["_mcollective-server._tcp", "_x-puppet-mcollective._tcp"], default_host, default_port)
549
+ end
550
+
551
+ # Looks for federation middleware servers when federated
552
+ #
553
+ # Attempts to find servers in the following order:
554
+ #
555
+ # * Configured hosts in choria.federation_middleware_hosts
556
+ # * SRV lookups in _mcollective-federation_server._tcp and _x-puppet-mcollective_federation._tcp
557
+ #
558
+ # @note you'd still want to only get your middleware servers from {#middleware_servers}
559
+ # @return [Array,nil] groups of host and port, nil when not found
560
+ def federation_middleware_servers
561
+ server_resolver("choria.federation_middleware_hosts", ["_mcollective-federation_server._tcp", "_x-puppet-mcollective_federation._tcp"])
562
+ end
563
+
564
+ # Determines if servers should be randomized
565
+ #
566
+ # @return [Boolean]
567
+ def randomize_middleware_servers?
568
+ Util.str_to_bool(get_option("choria.randomize_middleware_hosts", "true"))
569
+ end
570
+
571
+ # Attempts to look up some SRV records falling back to defaults
572
+ #
573
+ # When given a array of multiple names it will try each name individually
574
+ # and check if it resolved to a answer, if it did it will use that one.
575
+ # Else it will move to the next. In this way you can prioritise one
576
+ # record over another like puppetdb over puppet and faill back to defaults.
577
+ #
578
+ # This is a pretty naive implementation that right now just returns
579
+ # the first result, the correct behaviour needs to be determined but
580
+ # for now this gets us going with easily iterable code.
581
+ #
582
+ # These names are mainly being used by {#https} so in theory it would
583
+ # be quite easy to support multiple results with fall back etc, but
584
+ # I am not really sure what would be the best behaviour here
585
+ #
586
+ # @param names [Array<String>, String] list of names to lookup without the domain
587
+ # @param default_target [String] default for the returned :target
588
+ # @param default_port [String] default for the returned :port
589
+ # @return [Hash] with :target and :port
590
+ def try_srv(names, default_target, default_port)
591
+ srv_answers = Array(names).map do |name|
592
+ answer = query_srv_records([name])
593
+
594
+ answer.empty? ? nil : answer
595
+ end.compact.flatten
596
+
597
+ if srv_answers.empty?
598
+ {:target => default_target, :port => default_port}
599
+ else
600
+ {:target => srv_answers[0][:target].to_s, :port => srv_answers[0][:port]}
601
+ end
602
+ end
603
+
604
+ # The Puppet server to connect to
605
+ #
606
+ # Will consult SRV records for _x-puppet._tcp.example.net first then
607
+ # configurable using choria.puppetserver_host and choria.puppetserver_port
608
+ # defaults to puppet:8140.
609
+ #
610
+ # @return [Hash] with :target and :port
611
+ def puppet_server
612
+ d_host = get_option("choria.puppetserver_host", "puppet")
613
+ d_port = get_option("choria.puppetserver_port", "8140")
614
+
615
+ try_srv(["_x-puppet._tcp"], d_host, d_port)
616
+ end
617
+
618
+ # The Puppet server to connect to
619
+ #
620
+ # Will consult _x-puppet-ca._tcp.example.net then _x-puppet._tcp.example.net
621
+ # then configurable using choria.puppetca_host, defaults to puppet:8140
622
+ #
623
+ # @return [Hash] with :target and :port
624
+ def puppetca_server
625
+ d_port = get_option("choria.puppetca_port", "8140")
626
+
627
+ if @ca
628
+ {:target => @ca, :port => d_port}
629
+ else
630
+ d_host = get_option("choria.puppetca_host", "puppet")
631
+ try_srv(["_x-puppet-ca._tcp", "_x-puppet._tcp"], d_host, d_port)
632
+ end
633
+ end
634
+
635
+ # The PuppetDB server to connect to
636
+ #
637
+ # Will consult _x-puppet-db._tcp.example.net then _x-puppet._tcp.example.net
638
+ # then configurable using choria.puppetdb_host and choria.puppetdb_port, defaults
639
+ # to puppet:8081
640
+ #
641
+ # @return [Hash] with :target and :port
642
+ def puppetdb_server
643
+ d_host = get_option("choria.puppetdb_host", "puppet")
644
+ d_port = get_option("choria.puppetdb_port", "8081")
645
+
646
+ answer = try_srv(["_x-puppet-db._tcp"], nil, nil)
647
+ return answer if answer[:target]
648
+
649
+ # In the case where we take _x-puppet._tcp SRV records we unfortunately have
650
+ # to force the port else it uses the one from Puppet which will 404
651
+ answer = try_srv(["_x-puppet._tcp"], d_host, d_port)
652
+ answer[:port] = d_port
653
+
654
+ answer
655
+ end
656
+
657
+ # Looks for discovery proxy servers
658
+ #
659
+ # Attempts to find servers in the following order:
660
+ #
661
+ # * If choria.discovery_proxy is set to false, returns nil
662
+ # * Configured hosts in choria.discovery_proxies
663
+ # * SRV lookups in _mcollective-discovery._tcp
664
+ #
665
+ # @return [Hash] with :target and :port
666
+ def discovery_server
667
+ return unless proxied_discovery?
668
+
669
+ d_host = get_option("choria.discovery_host", "puppet")
670
+ d_port = get_option("choria.discovery_port", "8085")
671
+
672
+ try_srv(["_mcollective-discovery._tcp"], d_host, d_port)
673
+ end
674
+
675
+ # Determines if this is using a discovery proxy
676
+ #
677
+ # @return [Boolean]
678
+ def proxied_discovery?
679
+ has_option?("choria.discovery_host") || has_option?("choria.discovery_port") || Util.str_to_bool(get_option("choria.discovery_proxy", "false"))
680
+ end
681
+
682
+ # The certname of the current context
683
+ #
684
+ # In the case of root that would be the configured `identity`
685
+ # for non root it would a string made up of the current username
686
+ # as determined by the USER environment variable or the configured
687
+ # `identity`
688
+ #
689
+ # At present windows clients are probably not supported automatically
690
+ # as they will default to the certificate based on identity. Same
691
+ # as root. Windows will have to rely on the environment override
692
+ # until we can figure out what the best behaviour is
693
+ #
694
+ # In all cases the certname can be overridden using the `MCOLLECTIVE_CERTNAME`
695
+ # environment variable
696
+ #
697
+ # @return [String]
698
+ def certname
699
+ if Process.uid == 0 || Util.windows?
700
+ certname = @config.identity
701
+ else
702
+ certname = "%s.mcollective" % [env_fetch("USER", @config.identity)]
703
+ end
704
+
705
+ env_fetch("MCOLLECTIVE_CERTNAME", certname)
706
+ end
707
+
708
+ # Initialises Puppet if needed and retrieve a config setting
709
+ #
710
+ # @param setting [Symbol] a Puppet setting name
711
+ # @return [String]
712
+ def puppet_setting(setting)
713
+ require "puppet"
714
+
715
+ unless Puppet.settings.app_defaults_initialized?
716
+ Puppet.settings.preferred_run_mode = :agent
717
+
718
+ Puppet.settings.initialize_global_settings([])
719
+ Puppet.settings.initialize_app_defaults(Puppet::Settings.app_defaults_for_run_mode(Puppet.run_mode))
720
+ Puppet.push_context(Puppet.base_context(Puppet.settings))
721
+ end
722
+
723
+ Puppet.settings[setting]
724
+ end
725
+
726
+ # Creates a SSL Context which includes the AIO SSL files
727
+ #
728
+ # @return [OpenSSL::SSL::SSLContext]
729
+ def ssl_context
730
+ context = OpenSSL::SSL::SSLContext.new
731
+ context.ca_file = ca_path
732
+ context.ssl_version = :TLSv1_2 # rubocop:disable Naming/VariableNumber
733
+
734
+ if anon_tls?
735
+ context.verify_mode = OpenSSL::SSL::VERIFY_NONE
736
+ return context
737
+ end
738
+
739
+ public_cert = File.read(client_public_cert)
740
+ private_key = File.read(client_private_key)
741
+
742
+ cert_chain = ssl_parse_chain(public_cert)
743
+
744
+ cert = cert_chain.first
745
+ key = OpenSSL::PKey::RSA.new(private_key)
746
+
747
+ extra_chain_cert = cert_chain[1..-1]
748
+
749
+ if OpenSSL::SSL::SSLContext.method_defined?(:add_certificate)
750
+ context.add_certificate(cert, key, extra_chain_cert)
751
+ else
752
+ context.cert = OpenSSL::X509::Certificate.new(File.read(client_public_cert))
753
+ context.key = OpenSSL::PKey::RSA.new(File.read(client_private_key))
754
+ context.extra_chain_cert = extra_chain_cert
755
+ end
756
+
757
+ context.verify_mode = OpenSSL::SSL::VERIFY_PEER
758
+
759
+ context
760
+ end
761
+
762
+ # The directory where SSL related files live
763
+ #
764
+ # This is configurable using choria.ssldir which should be a
765
+ # path expandable using File.expand_path
766
+ #
767
+ # On Windows or when running as root Puppet settings will be consulted
768
+ # but when running as a normal user it will default to the AIO path
769
+ # when not configured
770
+ #
771
+ # @return [String]
772
+ def ssl_dir
773
+ @_ssl_dir ||= if has_option?("choria.ssldir")
774
+ File.expand_path(get_option("choria.ssldir"))
775
+ elsif Util.windows? || Process.uid == 0
776
+ puppet_setting(:ssldir)
777
+ else
778
+ File.expand_path("~/.puppetlabs/etc/puppet/ssl")
779
+ end
780
+ end
781
+
782
+ # Determines the security provider
783
+ def security_provider
784
+ get_option("security.provider", "puppet")
785
+ end
786
+
787
+ # Determines if the file security provider is enabled
788
+ def file_security?
789
+ security_provider == "file"
790
+ end
791
+
792
+ # Determines if the puppet security provider is enabled
793
+ def puppet_security?
794
+ security_provider == "puppet"
795
+ end
796
+
797
+ # Expands full paths with special handling for empty string
798
+ #
799
+ # File.expand_path will expand `""` to cwd, this is not good for
800
+ # what we need in many cases so this returns `""` in that case
801
+ #
802
+ # @param path [String] the unexpanded path
803
+ # @return [String] `""` when empty string was given
804
+ def expand_path(path)
805
+ return "" if path == ""
806
+
807
+ File.expand_path(path)
808
+ end
809
+
810
+ # The path to a client public certificate
811
+ #
812
+ # @note paths determined by Puppet AIO packages
813
+ # @return [String]
814
+ def client_public_cert
815
+ return expand_path(get_option("security.file.certificate", "")) if file_security?
816
+
817
+ File.join(ssl_dir, "certs", "%s.pem" % certname)
818
+ end
819
+
820
+ # Determines if teh client_public_cert exist
821
+ #
822
+ # @return [Boolean]
823
+ def has_client_public_cert?
824
+ File.exist?(client_public_cert)
825
+ end
826
+
827
+ # The path to a client private key
828
+ #
829
+ # @note paths determined by Puppet AIO packages
830
+ # @return [String]
831
+ def client_private_key
832
+ return expand_path(get_option("security.file.key", "")) if file_security?
833
+
834
+ File.join(ssl_dir, "private_keys", "%s.pem" % certname)
835
+ end
836
+
837
+ # Determines if the client_private_key exist
838
+ #
839
+ # @return [Boolean]
840
+ def has_client_private_key?
841
+ File.exist?(client_private_key)
842
+ end
843
+
844
+ # The path to the CA
845
+ #
846
+ # @return [String]
847
+ def ca_path
848
+ return expand_path(get_option("security.file.ca", "")) if file_security?
849
+
850
+ File.join(ssl_dir, "certs", "ca.pem")
851
+ end
852
+
853
+ # Determines if Choria is configured for anonymous TLS mode
854
+ #
855
+ # @return [Boolean]
856
+ def anon_tls?
857
+ remote_signer_configured? && Util.str_to_bool(get_option("security.client_anon_tls", "false"))
858
+ end
859
+
860
+ # Determines if the CA exist
861
+ #
862
+ # @return [Boolean]
863
+ def has_ca?
864
+ File.exist?(ca_path)
865
+ end
866
+
867
+ # The path to a CSR for this user
868
+ #
869
+ # @return [String]
870
+ def csr_path
871
+ return "" if file_security?
872
+
873
+ File.join(ssl_dir, "certificate_requests", "%s.pem" % certname)
874
+ end
875
+
876
+ # Determines if the CSR exist
877
+ #
878
+ # @return [Boolean]
879
+ def has_csr?
880
+ File.exist?(csr_path)
881
+ end
882
+
883
+ # Searches the PATH for an executable command
884
+ #
885
+ # @param command [String] a command to search for
886
+ # @return [String,nil] the path to the command or nil
887
+ def which(command)
888
+ exts = Array(env_fetch("PATHEXT", "").split(";"))
889
+ exts << "" if exts.empty?
890
+
891
+ env_fetch("PATH", "").split(File::PATH_SEPARATOR).each do |path|
892
+ exts.each do |ext|
893
+ exe = File.join(path, "%s%s" % [command, ext])
894
+ return exe if File.executable?(exe) && !File.directory?(exe)
895
+ end
896
+ end
897
+
898
+ nil
899
+ end
900
+
901
+ # Searches the machine for a working facter
902
+ #
903
+ # It checks AIO path first and then attempts to find it in PATH and supports both
904
+ # unix and windows
905
+ #
906
+ # @return [String,nil]
907
+ def facter_cmd
908
+ return "/opt/puppetlabs/bin/facter" if File.executable?("/opt/puppetlabs/bin/facter")
909
+
910
+ which("facter")
911
+ end
912
+
913
+ # Gets a config option
914
+ #
915
+ # @param opt [String] config option to look up
916
+ # @param default [Object] default to return when not found
917
+ # @return [Object, Proc] the found data or default. When it's a proc the proc will be called only when needed
918
+ # @raise [StandardError] when no default is given and option is not found
919
+ def get_option(opt, default=:_unset)
920
+ return @config.pluginconf[opt] if has_option?(opt)
921
+
922
+ unless default == :_unset
923
+ if default.is_a?(Proc)
924
+ return default.call
925
+ else
926
+ return default
927
+ end
928
+ end
929
+
930
+ raise(UserError, "No plugin.%s configuration option given" % opt)
931
+ end
932
+
933
+ # Determines if a config option is set
934
+ #
935
+ # @param opt [String] config option to look up
936
+ # @return [Boolean]
937
+ def has_option?(opt)
938
+ @config.pluginconf.include?(opt)
939
+ end
940
+
941
+ def env_fetch(key, default=nil)
942
+ ENV.fetch(key, default)
943
+ end
944
+ end
945
+ end
946
+ end