choria-mcorpc-support 2.20.8 → 2.23.0.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/mcollective.rb +1 -1
- data/lib/mcollective/agent/bolt_tasks.ddl +235 -0
- data/lib/mcollective/agent/bolt_tasks.json +347 -0
- data/lib/mcollective/agent/bolt_tasks.rb +176 -0
- data/lib/mcollective/agent/choria_util.ddl +152 -0
- data/lib/mcollective/agent/choria_util.json +244 -0
- data/lib/mcollective/agent/rpcutil.ddl +7 -3
- data/lib/mcollective/agent/rpcutil.json +333 -0
- data/lib/mcollective/agent/scout.ddl +169 -0
- data/lib/mcollective/agent/scout.json +224 -0
- data/lib/mcollective/agents.rb +7 -6
- data/lib/mcollective/aggregate.rb +4 -4
- data/lib/mcollective/aggregate/average.rb +2 -2
- data/lib/mcollective/aggregate/base.rb +2 -2
- data/lib/mcollective/aggregate/result.rb +3 -3
- data/lib/mcollective/aggregate/result/collection_result.rb +2 -2
- data/lib/mcollective/aggregate/result/numeric_result.rb +2 -2
- data/lib/mcollective/aggregate/sum.rb +2 -2
- data/lib/mcollective/aggregate/summary.rb +3 -4
- data/lib/mcollective/application.rb +57 -21
- data/lib/mcollective/application/choria.rb +249 -0
- data/lib/mcollective/application/completion.rb +6 -6
- data/lib/mcollective/application/describe_filter.rb +20 -20
- data/lib/mcollective/application/facts.rb +19 -11
- data/lib/mcollective/application/federation.rb +239 -0
- data/lib/mcollective/application/find.rb +4 -4
- data/lib/mcollective/application/help.rb +3 -3
- data/lib/mcollective/application/inventory.rb +3 -341
- data/lib/mcollective/application/ping.rb +3 -77
- data/lib/mcollective/application/playbook.rb +207 -0
- data/lib/mcollective/application/plugin.rb +106 -106
- data/lib/mcollective/application/rpc.rb +3 -108
- data/lib/mcollective/application/tasks.rb +416 -0
- data/lib/mcollective/applications.rb +11 -10
- data/lib/mcollective/audit/choria.rb +33 -0
- data/lib/mcollective/cache.rb +2 -4
- data/lib/mcollective/client.rb +11 -10
- data/lib/mcollective/config.rb +21 -34
- data/lib/mcollective/connector/base.rb +2 -1
- data/lib/mcollective/connector/nats.ddl +9 -0
- data/lib/mcollective/connector/nats.rb +450 -0
- data/lib/mcollective/data.rb +8 -3
- data/lib/mcollective/data/agent_data.rb +1 -1
- data/lib/mcollective/data/base.rb +6 -5
- data/lib/mcollective/data/bolt_task_data.ddl +90 -0
- data/lib/mcollective/data/bolt_task_data.rb +32 -0
- data/lib/mcollective/data/collective_data.rb +1 -1
- data/lib/mcollective/data/fact_data.rb +6 -6
- data/lib/mcollective/data/fstat_data.rb +2 -4
- data/lib/mcollective/data/result.rb +7 -2
- data/lib/mcollective/ddl/agentddl.rb +5 -17
- data/lib/mcollective/ddl/base.rb +11 -14
- data/lib/mcollective/discovery.rb +12 -26
- data/lib/mcollective/discovery/choria.ddl +11 -0
- data/lib/mcollective/discovery/choria.rb +223 -0
- data/lib/mcollective/discovery/flatfile.rb +7 -8
- data/lib/mcollective/discovery/mc.rb +2 -2
- data/lib/mcollective/discovery/stdin.rb +17 -18
- data/lib/mcollective/exceptions.rb +13 -0
- data/lib/mcollective/facts/base.rb +9 -9
- data/lib/mcollective/facts/yaml_facts.rb +12 -12
- data/lib/mcollective/generators.rb +3 -3
- data/lib/mcollective/generators/agent_generator.rb +3 -4
- data/lib/mcollective/generators/base.rb +14 -15
- data/lib/mcollective/generators/data_generator.rb +5 -6
- data/lib/mcollective/log.rb +2 -2
- data/lib/mcollective/logger/base.rb +3 -2
- data/lib/mcollective/logger/console_logger.rb +10 -10
- data/lib/mcollective/logger/file_logger.rb +7 -7
- data/lib/mcollective/logger/syslog_logger.rb +11 -15
- data/lib/mcollective/matcher.rb +14 -14
- data/lib/mcollective/matcher/parser.rb +31 -41
- data/lib/mcollective/matcher/scanner.rb +69 -74
- data/lib/mcollective/message.rb +10 -17
- data/lib/mcollective/monkey_patches.rb +2 -4
- data/lib/mcollective/optionparser.rb +1 -0
- data/lib/mcollective/pluginmanager.rb +3 -5
- data/lib/mcollective/pluginpackager.rb +1 -3
- data/lib/mcollective/pluginpackager/agent_definition.rb +10 -11
- data/lib/mcollective/pluginpackager/forge_packager.rb +7 -9
- data/lib/mcollective/pluginpackager/standard_definition.rb +1 -2
- data/lib/mcollective/registration/base.rb +18 -16
- data/lib/mcollective/rpc.rb +2 -4
- data/lib/mcollective/rpc/actionrunner.rb +16 -18
- data/lib/mcollective/rpc/agent.rb +26 -43
- data/lib/mcollective/rpc/audit.rb +1 -0
- data/lib/mcollective/rpc/client.rb +67 -85
- data/lib/mcollective/rpc/helpers.rb +55 -62
- data/lib/mcollective/rpc/progress.rb +2 -2
- data/lib/mcollective/rpc/reply.rb +17 -19
- data/lib/mcollective/rpc/request.rb +7 -5
- data/lib/mcollective/rpc/result.rb +6 -8
- data/lib/mcollective/rpc/stats.rb +49 -58
- data/lib/mcollective/security/base.rb +29 -36
- data/lib/mcollective/security/choria.rb +765 -0
- data/lib/mcollective/shell.rb +9 -4
- data/lib/mcollective/signer/base.rb +28 -0
- data/lib/mcollective/signer/choria.rb +185 -0
- data/lib/mcollective/ssl.rb +8 -6
- data/lib/mcollective/util.rb +58 -55
- data/lib/mcollective/util/bolt_support.rb +176 -0
- data/lib/mcollective/util/bolt_support/plan_runner.rb +167 -0
- data/lib/mcollective/util/bolt_support/task_result.rb +94 -0
- data/lib/mcollective/util/bolt_support/task_results.rb +128 -0
- data/lib/mcollective/util/choria.rb +1103 -0
- data/lib/mcollective/util/indifferent_hash.rb +12 -0
- data/lib/mcollective/util/natswrapper.rb +242 -0
- data/lib/mcollective/util/playbook.rb +435 -0
- data/lib/mcollective/util/playbook/data_stores.rb +201 -0
- data/lib/mcollective/util/playbook/data_stores/base.rb +99 -0
- data/lib/mcollective/util/playbook/data_stores/consul_data_store.rb +88 -0
- data/lib/mcollective/util/playbook/data_stores/environment_data_store.rb +33 -0
- data/lib/mcollective/util/playbook/data_stores/etcd_data_store.rb +42 -0
- data/lib/mcollective/util/playbook/data_stores/file_data_store.rb +106 -0
- data/lib/mcollective/util/playbook/data_stores/shell_data_store.rb +103 -0
- data/lib/mcollective/util/playbook/inputs.rb +265 -0
- data/lib/mcollective/util/playbook/nodes.rb +207 -0
- data/lib/mcollective/util/playbook/nodes/mcollective_nodes.rb +86 -0
- data/lib/mcollective/util/playbook/nodes/pql_nodes.rb +40 -0
- data/lib/mcollective/util/playbook/nodes/shell_nodes.rb +55 -0
- data/lib/mcollective/util/playbook/nodes/terraform_nodes.rb +65 -0
- data/lib/mcollective/util/playbook/nodes/yaml_nodes.rb +47 -0
- data/lib/mcollective/util/playbook/playbook_logger.rb +47 -0
- data/lib/mcollective/util/playbook/puppet_logger.rb +51 -0
- data/lib/mcollective/util/playbook/report.rb +152 -0
- data/lib/mcollective/util/playbook/task_result.rb +55 -0
- data/lib/mcollective/util/playbook/tasks.rb +196 -0
- data/lib/mcollective/util/playbook/tasks/base.rb +45 -0
- data/lib/mcollective/util/playbook/tasks/graphite_event_task.rb +64 -0
- data/lib/mcollective/util/playbook/tasks/mcollective_task.rb +356 -0
- data/lib/mcollective/util/playbook/tasks/shell_task.rb +93 -0
- data/lib/mcollective/util/playbook/tasks/slack_task.rb +105 -0
- data/lib/mcollective/util/playbook/tasks/webhook_task.rb +136 -0
- data/lib/mcollective/util/playbook/template_util.rb +98 -0
- data/lib/mcollective/util/playbook/uses.rb +169 -0
- data/lib/mcollective/util/tasks_support.rb +733 -0
- data/lib/mcollective/util/tasks_support/cli.rb +260 -0
- data/lib/mcollective/util/tasks_support/default_formatter.rb +138 -0
- data/lib/mcollective/util/tasks_support/json_formatter.rb +108 -0
- data/lib/mcollective/validator.rb +8 -3
- data/lib/mcollective/validator/bolt_task_name_validator.ddl +7 -0
- data/lib/mcollective/validator/bolt_task_name_validator.rb +11 -0
- data/lib/mcollective/validator/length_validator.rb +1 -3
- data/lib/mcollective/validator/typecheck_validator.rb +4 -0
- metadata +67 -4
@@ -12,11 +12,11 @@ module MCollective
|
|
12
12
|
load_application(appname)
|
13
13
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
14
14
|
e.backtrace.first << Util.colorize(:red, " <----")
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
15
|
+
warn "Application '%s' failed to load:" % appname
|
16
|
+
$stderr.puts
|
17
|
+
warn Util.colorize(:red, " %s (%s)" % [$!, $!.class])
|
18
|
+
$stderr.puts
|
19
|
+
warn " %s" % [$!.backtrace.join("\n ")]
|
20
20
|
exit 1
|
21
21
|
end
|
22
22
|
|
@@ -40,7 +40,7 @@ module MCollective
|
|
40
40
|
rescue SystemExit
|
41
41
|
exit 1
|
42
42
|
rescue Exception # rubocop:disable Lint/RescueException
|
43
|
-
|
43
|
+
warn("Failed to generate application list: %s: %s" % [$!.class, $!])
|
44
44
|
exit 1
|
45
45
|
end
|
46
46
|
|
@@ -49,13 +49,14 @@ module MCollective
|
|
49
49
|
def self.filter_extra_options(opts)
|
50
50
|
words = Shellwords.shellwords(opts)
|
51
51
|
words.each_with_index do |word, idx|
|
52
|
-
|
52
|
+
case word
|
53
|
+
when "-c"
|
53
54
|
return "--config=#{words[idx + 1]}"
|
54
|
-
|
55
|
+
when "--config"
|
55
56
|
return "--config=#{words[idx + 1]}"
|
56
|
-
|
57
|
+
when /-c=/
|
57
58
|
return word
|
58
|
-
|
59
|
+
when /--config=/
|
59
60
|
return word
|
60
61
|
end
|
61
62
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module MCollective
|
2
|
+
module Audit
|
3
|
+
class Choria < RPC::Audit
|
4
|
+
def audit_request(request, connection)
|
5
|
+
logfile = Config.instance.pluginconf["rpcaudit.logfile"]
|
6
|
+
|
7
|
+
unless logfile
|
8
|
+
Log.warn("Auditing is not functional because rpcaudit.logfile is not set")
|
9
|
+
return
|
10
|
+
end
|
11
|
+
|
12
|
+
message = {
|
13
|
+
"timestamp" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6N%z"),
|
14
|
+
"request_id" => request.uniqid,
|
15
|
+
"request_time" => request.time,
|
16
|
+
"caller" => request.caller,
|
17
|
+
"sender" => request.sender,
|
18
|
+
"agent" => request.agent,
|
19
|
+
"action" => request.action,
|
20
|
+
"data" => request.data
|
21
|
+
}
|
22
|
+
|
23
|
+
begin
|
24
|
+
File.open(logfile, "a") do |f|
|
25
|
+
f.puts(message.to_json)
|
26
|
+
end
|
27
|
+
rescue
|
28
|
+
Log.warn("Auditing is not functional because writing to logfile '%s' failed %s: %s" % [logfile, $!.class, $!.to_s])
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/mcollective/cache.rb
CHANGED
@@ -122,14 +122,12 @@ module MCollective
|
|
122
122
|
end
|
123
123
|
end
|
124
124
|
|
125
|
-
def self.synchronize(cache_name)
|
125
|
+
def self.synchronize(cache_name, &block)
|
126
126
|
raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
|
127
127
|
|
128
128
|
raise("No block supplied to synchronize") unless block_given?
|
129
129
|
|
130
|
-
@cache_locks[cache_name].synchronize
|
131
|
-
yield
|
132
|
-
end
|
130
|
+
@cache_locks[cache_name].synchronize(&block)
|
133
131
|
end
|
134
132
|
|
135
133
|
def self.invalidate!(cache_name, key)
|
data/lib/mcollective/client.rb
CHANGED
@@ -7,10 +7,11 @@ module MCollective
|
|
7
7
|
@config = Config.instance
|
8
8
|
@options = nil
|
9
9
|
|
10
|
-
|
10
|
+
case options
|
11
|
+
when String
|
11
12
|
# String is the path to a config file
|
12
13
|
@config.loadconfig(options) unless @config.configured
|
13
|
-
|
14
|
+
when Hash
|
14
15
|
@config.loadconfig(options[:config]) unless @config.configured
|
15
16
|
@options = options
|
16
17
|
@connection_timeout = options[:connection_timeout]
|
@@ -43,6 +44,10 @@ module MCollective
|
|
43
44
|
|
44
45
|
@@request_sequence = 0 # rubocop:disable Style/ClassVars
|
45
46
|
|
47
|
+
def self.reset_request_sequence
|
48
|
+
@@request_sequence = 0 # rubocop:disable Style/ClassVars
|
49
|
+
end
|
50
|
+
|
46
51
|
def self.request_sequence
|
47
52
|
@@request_sequence
|
48
53
|
end
|
@@ -124,9 +129,7 @@ module MCollective
|
|
124
129
|
|
125
130
|
reply.decode!
|
126
131
|
|
127
|
-
unless reply.requestid == requestid
|
128
|
-
raise(MsgDoesNotMatchRequestID, "Message reqid #{reply.requestid} does not match our reqid #{requestid}")
|
129
|
-
end
|
132
|
+
raise(MsgDoesNotMatchRequestID, "Message reqid #{reply.requestid} does not match our reqid #{requestid}") unless reply.requestid == requestid
|
130
133
|
|
131
134
|
Log.debug("Received reply to #{reply.requestid} from #{reply.payload[:senderid]}")
|
132
135
|
rescue SecurityValidationFailed => e
|
@@ -171,7 +174,7 @@ module MCollective
|
|
171
174
|
request = createreq(body, agent, @options[:filter])
|
172
175
|
publish_timeout = @options[:publish_timeout] || @config.publish_timeout
|
173
176
|
stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
|
174
|
-
|
177
|
+
$stdout.sync = true
|
175
178
|
hosts_responded = 0
|
176
179
|
|
177
180
|
begin
|
@@ -180,7 +183,7 @@ module MCollective
|
|
180
183
|
else
|
181
184
|
hosts_responded = unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
|
182
185
|
end
|
183
|
-
rescue Interrupt # rubocop:disable Lint/
|
186
|
+
rescue Interrupt # rubocop:disable Lint/SuppressedException
|
184
187
|
ensure
|
185
188
|
unsubscribe(agent, :reply)
|
186
189
|
end
|
@@ -277,9 +280,7 @@ module MCollective
|
|
277
280
|
end
|
278
281
|
rescue Timeout::Error
|
279
282
|
if waitfor.is_a?(Array)
|
280
|
-
unless unfinished.empty?
|
281
|
-
Log.warn("Could not receive all responses. Did not receive responses from #{unfinished.keys.join(', ')}")
|
282
|
-
end
|
283
|
+
Log.warn("Could not receive all responses. Did not receive responses from #{unfinished.keys.join(', ')}") unless unfinished.empty?
|
283
284
|
elsif waitfor > hosts_responded
|
284
285
|
Log.warn("Could not receive all responses. Expected : #{waitfor}. Received : #{hosts_responded}")
|
285
286
|
end
|
data/lib/mcollective/config.rb
CHANGED
@@ -5,19 +5,13 @@ module MCollective
|
|
5
5
|
|
6
6
|
attr_accessor :mode
|
7
7
|
|
8
|
-
attr_reader :daemonize, :pluginconf, :configured
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
attr_reader :main_collective, :ssl_cipher, :registration_collective
|
16
|
-
attr_reader :direct_addressing, :direct_addressing_threshold, :ttl
|
17
|
-
attr_reader :default_discovery_method, :default_discovery_options
|
18
|
-
attr_reader :publish_timeout, :threaded, :soft_shutdown, :activate_agents
|
19
|
-
attr_reader :registration_splay, :discovery_timeout, :soft_shutdown_timeout
|
20
|
-
attr_reader :connection_timeout, :default_batch_size, :default_batch_sleep_time
|
8
|
+
attr_reader :daemonize, :pluginconf, :configured, :logfile, :keeplogs, :max_log_size, :loglevel, :logfacility,
|
9
|
+
:identity, :connector, :securityprovider, :factsource, :registration, :registerinterval, :classesfile,
|
10
|
+
:rpcauditprovider, :rpcaudit, :configdir, :rpcauthprovider, :rpcauthorization, :color, :configfile,
|
11
|
+
:rpclimitmethod, :logger_type, :fact_cache_time, :collectives, :main_collective, :ssl_cipher, :registration_collective,
|
12
|
+
:direct_addressing, :direct_addressing_threshold, :ttl, :default_discovery_method, :default_discovery_options,
|
13
|
+
:publish_timeout, :threaded, :soft_shutdown, :activate_agents, :registration_splay, :discovery_timeout, :soft_shutdown_timeout,
|
14
|
+
:connection_timeout, :default_batch_size, :default_batch_sleep_time
|
21
15
|
|
22
16
|
def initialize
|
23
17
|
@configured = false
|
@@ -34,19 +28,12 @@ module MCollective
|
|
34
28
|
|
35
29
|
next if line =~ /^#|^$/
|
36
30
|
next unless line =~ /(.+?)\s*=\s*(.+)/
|
31
|
+
|
37
32
|
key = $1.strip
|
38
33
|
val = $2
|
39
34
|
|
40
35
|
begin
|
41
36
|
case key
|
42
|
-
when "registration"
|
43
|
-
@registration = val.capitalize
|
44
|
-
when "registration_collective"
|
45
|
-
@registration_collective = val
|
46
|
-
when "registerinterval"
|
47
|
-
@registerinterval = Integer(val)
|
48
|
-
when "registration_splay"
|
49
|
-
@registration_splay = Util.str_to_bool(val)
|
50
37
|
when "collectives"
|
51
38
|
@collectives = val.split(",").map(&:strip)
|
52
39
|
when "main_collective"
|
@@ -128,10 +115,11 @@ module MCollective
|
|
128
115
|
@default_batch_size = Integer(val)
|
129
116
|
when "default_batch_sleep_time"
|
130
117
|
@default_batch_sleep_time = Float(val)
|
131
|
-
when "topicprefix", "topicsep", "queueprefix", "rpchelptemplate", "helptemplatedir"
|
132
|
-
Log.warn("Use of deprecated '#{key}' option. This option is ignored and should be removed from '#{configfile}'")
|
133
118
|
else
|
134
|
-
|
119
|
+
# server config might now be choria config which will divirge from mcollective
|
120
|
+
# in time, so we only raise this error when it looks like we aren't loading
|
121
|
+
# a server config else we try our best to load as much as we can
|
122
|
+
raise("Unknown config parameter '#{key}'") unless configfile =~ /server/
|
135
123
|
end
|
136
124
|
rescue ArgumentError
|
137
125
|
raise("Could not parse value for configuration option '%s' with value '%s'" % [key, val])
|
@@ -140,7 +128,7 @@ module MCollective
|
|
140
128
|
|
141
129
|
read_plugin_config_dir("#{@configdir}/plugin.d")
|
142
130
|
|
143
|
-
raise 'Identities can only match /\w\.\-/' unless @identity =~ /^[\w
|
131
|
+
raise 'Identities can only match /\w\.\-/' unless @identity =~ /^[\w.\-]+$/
|
144
132
|
|
145
133
|
@configured = true
|
146
134
|
|
@@ -152,14 +140,13 @@ module MCollective
|
|
152
140
|
$LOAD_PATH.unshift dir
|
153
141
|
end
|
154
142
|
|
155
|
-
if @logger_type == "syslog"
|
156
|
-
raise "The sylog logger is not usable on the Windows platform" if Util.windows?
|
157
|
-
end
|
143
|
+
raise "The sylog logger is not usable on the Windows platform" if @logger_type == "syslog" && Util.windows?
|
158
144
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
145
|
+
unless configfile =~ /server/
|
146
|
+
PluginManager.loadclass("Mcollective::Facts::#{@factsource}_facts")
|
147
|
+
PluginManager.loadclass("Mcollective::Connector::#{@connector}")
|
148
|
+
PluginManager.loadclass("Mcollective::Security::#{@securityprovider}")
|
149
|
+
end
|
163
150
|
|
164
151
|
Log.info("The Marionette Collective version #{MCollective::VERSION} started by #{$0} using config file #{configfile}")
|
165
152
|
else
|
@@ -168,7 +155,6 @@ module MCollective
|
|
168
155
|
end
|
169
156
|
|
170
157
|
def set_config_defaults(configfile) # rubocop:disable Naming/AccessorMethodName
|
171
|
-
@stomp = {}
|
172
158
|
@subscribe = []
|
173
159
|
@pluginconf = {}
|
174
160
|
@connector = "base"
|
@@ -221,7 +207,7 @@ module MCollective
|
|
221
207
|
return unless File.directory?(dir)
|
222
208
|
|
223
209
|
Dir.new(dir).each do |pluginconfigfile|
|
224
|
-
next unless pluginconfigfile =~ /^(
|
210
|
+
next unless pluginconfigfile =~ /^(\w+).cfg$/
|
225
211
|
|
226
212
|
plugin = $1
|
227
213
|
File.open("#{dir}/#{pluginconfigfile}", "r").each do |line|
|
@@ -229,6 +215,7 @@ module MCollective
|
|
229
215
|
line.gsub!(/\s*$/, "")
|
230
216
|
next if line =~ /^#|^$/
|
231
217
|
next unless line =~ /(.+?)\s*=\s*(.+)/
|
218
|
+
|
232
219
|
key = $1.strip
|
233
220
|
val = $2
|
234
221
|
@pluginconf["#{plugin}.#{key}"] = val
|
@@ -18,8 +18,9 @@ module MCollective
|
|
18
18
|
class Base
|
19
19
|
def self.inherited(klass)
|
20
20
|
plugin_name = klass.to_s.split("::").last.downcase
|
21
|
-
|
21
|
+
DDL.new(plugin_name, :connector)
|
22
22
|
PluginManager << {:type => "connector_plugin", :class => klass.to_s}
|
23
|
+
super
|
23
24
|
end
|
24
25
|
end
|
25
26
|
end
|
@@ -0,0 +1,9 @@
|
|
1
|
+
metadata :name => "nats",
|
2
|
+
:description => "Connector plugin for NATS.io middleware",
|
3
|
+
:author => "R.I.Pienaar <rip@devco.net>",
|
4
|
+
:license => "Apache-2.0",
|
5
|
+
:version => "0.19.0",
|
6
|
+
:url => "https://github.com/choria-io",
|
7
|
+
:timeout => 60
|
8
|
+
|
9
|
+
requires :mcollective => "2.8.4"
|
@@ -0,0 +1,450 @@
|
|
1
|
+
require "resolv"
|
2
|
+
require_relative "../util/choria"
|
3
|
+
require_relative "../util/natswrapper"
|
4
|
+
|
5
|
+
module MCollective
|
6
|
+
module Connector
|
7
|
+
class Nats < Base
|
8
|
+
attr_reader :connection
|
9
|
+
|
10
|
+
def initialize # rubocop:disable Lint/MissingSuper
|
11
|
+
@config = Config.instance
|
12
|
+
@subscriptions = []
|
13
|
+
@connection = Util::NatsWrapper.new
|
14
|
+
|
15
|
+
Log.info("Choria NATS.io connector using pure ruby nats/io/client %s with protocol version %s" % [NATS::IO::VERSION, NATS::IO::PROTOCOL])
|
16
|
+
end
|
17
|
+
|
18
|
+
# Determines if the NATS connection is active
|
19
|
+
#
|
20
|
+
# @return [Boolean]
|
21
|
+
def connected?
|
22
|
+
connection.connected?
|
23
|
+
end
|
24
|
+
|
25
|
+
# Current connected server
|
26
|
+
#
|
27
|
+
# @return [String,nil]
|
28
|
+
def connected_server
|
29
|
+
connection.connected_server
|
30
|
+
end
|
31
|
+
|
32
|
+
# Retrieves the NATS connection stats
|
33
|
+
#
|
34
|
+
# @return [Hash]
|
35
|
+
def stats
|
36
|
+
connection.stats
|
37
|
+
end
|
38
|
+
|
39
|
+
# Client library version
|
40
|
+
#
|
41
|
+
# @return [String]
|
42
|
+
def client_version
|
43
|
+
connection.client_version
|
44
|
+
end
|
45
|
+
|
46
|
+
# Client library flavour
|
47
|
+
#
|
48
|
+
# @return [String]
|
49
|
+
def client_flavour
|
50
|
+
connection.client_flavour
|
51
|
+
end
|
52
|
+
|
53
|
+
# Connection options from the NATS gem
|
54
|
+
#
|
55
|
+
# @return [Hash]
|
56
|
+
def active_options
|
57
|
+
connection.active_options
|
58
|
+
end
|
59
|
+
|
60
|
+
# Attempts to connect to the middleware, noop when already connected
|
61
|
+
#
|
62
|
+
# @return [void]
|
63
|
+
# @raise [StandardError] when SSL files are not readable
|
64
|
+
def connect
|
65
|
+
if connection && connection.started?
|
66
|
+
Log.debug("Already connection, not re-initializing connection")
|
67
|
+
return
|
68
|
+
end
|
69
|
+
|
70
|
+
parameters = {
|
71
|
+
:max_reconnect_attempts => -1,
|
72
|
+
:reconnect_time_wait => 1,
|
73
|
+
:dont_randomize_servers => !choria.randomize_middleware_servers?,
|
74
|
+
:name => @config.identity
|
75
|
+
}
|
76
|
+
|
77
|
+
parameters[:user_credentials] = choria.credential_file if choria.credential_file?
|
78
|
+
|
79
|
+
if $choria_unsafe_disable_nats_tls # rubocop:disable Style/GlobalVars
|
80
|
+
Log.warn("Disabling TLS in NATS connector, this is not a production supported setup")
|
81
|
+
elsif choria.ngs?
|
82
|
+
configure_ngs(parameters)
|
83
|
+
else
|
84
|
+
configure_tls(parameters)
|
85
|
+
end
|
86
|
+
|
87
|
+
servers = server_list
|
88
|
+
|
89
|
+
unless servers.empty?
|
90
|
+
Log.debug("Connecting to servers: %s" % servers.join(", "))
|
91
|
+
parameters[:servers] = servers
|
92
|
+
end
|
93
|
+
|
94
|
+
connection.start(parameters)
|
95
|
+
|
96
|
+
nil
|
97
|
+
end
|
98
|
+
|
99
|
+
def configure_tls(parameters)
|
100
|
+
parameters[:tls] = {:context => choria.ssl_context}
|
101
|
+
choria.check_ssl_setup
|
102
|
+
end
|
103
|
+
|
104
|
+
def configure_ngs(parameters)
|
105
|
+
Log.debug("Disabling specific TLS during connection to NGS")
|
106
|
+
|
107
|
+
raise("nkeys rubygem is required for connections with credentials") unless choria.nkeys?
|
108
|
+
|
109
|
+
tls = OpenSSL::SSL::SSLContext.new
|
110
|
+
tls.ssl_version = :TLSv1_2 # rubocop:disable Naming/VariableNumber
|
111
|
+
|
112
|
+
parameters[:tls] = {:context => tls}
|
113
|
+
end
|
114
|
+
|
115
|
+
# Disconnects from NATS
|
116
|
+
def disconnect
|
117
|
+
connection.stop
|
118
|
+
end
|
119
|
+
|
120
|
+
# Creates the middleware headers needed for a given message
|
121
|
+
#
|
122
|
+
# @param msg [Message]
|
123
|
+
# @return [Hash]
|
124
|
+
def headers_for(msg)
|
125
|
+
# mc_sender is only passed bacause M::Message incorrectly assumed this is some required
|
126
|
+
# part of messages when its just some internals of the stomp based connectors that bled out
|
127
|
+
headers = {
|
128
|
+
"mc_sender" => @config.identity
|
129
|
+
}
|
130
|
+
|
131
|
+
headers["seen-by"] = [] if msg.headers.include?("seen-by")
|
132
|
+
|
133
|
+
if [:request, :direct_request].include?(msg.type)
|
134
|
+
if msg.reply_to
|
135
|
+
headers["reply-to"] = msg.reply_to
|
136
|
+
else
|
137
|
+
# if its a request/direct_request style message and its not
|
138
|
+
# one we're replying to - ie. its a new message we're making
|
139
|
+
# we'll need to set a reply-to target that the daemon will
|
140
|
+
# subscribe to
|
141
|
+
headers["reply-to"] = make_target(msg.agent, :reply, msg.collective)
|
142
|
+
end
|
143
|
+
|
144
|
+
headers["seen-by"] << [@config.identity, connected_server.to_s] if msg.headers.include?("seen-by")
|
145
|
+
elsif msg.type == :reply
|
146
|
+
if msg.request.headers.include?("seen-by")
|
147
|
+
headers["seen-by"] = msg.request.headers["seen-by"]
|
148
|
+
headers["seen-by"].last << connected_server.to_s
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
headers
|
153
|
+
end
|
154
|
+
|
155
|
+
# Create a target structure for a message
|
156
|
+
#
|
157
|
+
# @example data
|
158
|
+
#
|
159
|
+
# {
|
160
|
+
# :name => "nats.name",
|
161
|
+
# :headers => { headers... }
|
162
|
+
# }
|
163
|
+
#
|
164
|
+
# @param msg [Message]
|
165
|
+
# @param identity [String,nil] override identity
|
166
|
+
# @return [Hash]
|
167
|
+
def target_for(msg, identity=nil)
|
168
|
+
target = nil
|
169
|
+
|
170
|
+
if msg.type == :reply
|
171
|
+
raise("Do not know how to reply, no reply-to header has been set on message %s" % msg.requestid) unless msg.request.headers["reply-to"]
|
172
|
+
|
173
|
+
target = {:name => msg.request.headers["reply-to"], :headers => {}}
|
174
|
+
|
175
|
+
elsif [:request, :direct_request].include?(msg.type)
|
176
|
+
target = {:name => make_target(msg.agent, msg.type, msg.collective, identity), :headers => {}}
|
177
|
+
|
178
|
+
else
|
179
|
+
raise("Don't now how to create a target for message type %s" % msg.type)
|
180
|
+
|
181
|
+
end
|
182
|
+
|
183
|
+
target[:headers].merge!(headers_for(msg))
|
184
|
+
|
185
|
+
target
|
186
|
+
end
|
187
|
+
|
188
|
+
# Retrieves the current process pid
|
189
|
+
#
|
190
|
+
# @note mainly used for testing
|
191
|
+
# @return [Fixnum]
|
192
|
+
def current_pid
|
193
|
+
$$
|
194
|
+
end
|
195
|
+
|
196
|
+
# Creates a target structure
|
197
|
+
#
|
198
|
+
# @param agent [String] agent name
|
199
|
+
# @param type [:directed, :broadcast, :reply, :request, :direct_request]
|
200
|
+
# @param collective [String] target collective name
|
201
|
+
# @param identity [String,nil] identity for the request, else node configured identity
|
202
|
+
# @return [String] target name
|
203
|
+
# @raise [StandardError] on invalid input
|
204
|
+
def make_target(agent, type, collective, identity=nil)
|
205
|
+
raise("Unknown target type %s" % type) unless [:directed, :broadcast, :reply, :request, :direct_request].include?(type)
|
206
|
+
|
207
|
+
raise("Unknown collective '%s' known collectives are '%s'" % [collective, @config.collectives.join(", ")]) unless @config.collectives.include?(collective)
|
208
|
+
|
209
|
+
identity ||= @config.identity
|
210
|
+
|
211
|
+
case type
|
212
|
+
when :reply
|
213
|
+
"%s.reply.%s.%d.%d" % [collective, Digest::MD5.hexdigest(choria.callerid), current_pid, Client.request_sequence]
|
214
|
+
|
215
|
+
when :broadcast, :request
|
216
|
+
"%s.broadcast.agent.%s" % [collective, agent]
|
217
|
+
|
218
|
+
when :direct_request, :directed
|
219
|
+
"%s.node.%s" % [collective, identity]
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
# Publishes a message to the middleware
|
224
|
+
#
|
225
|
+
# @param msg [Message]
|
226
|
+
def publish(msg)
|
227
|
+
msg.base64_encode!
|
228
|
+
|
229
|
+
if choria.federated?
|
230
|
+
msg.type == :direct_request ? publish_federated_directed(msg) : publish_federated_broadcast(msg)
|
231
|
+
else
|
232
|
+
msg.type == :direct_request ? publish_connected_directed(msg) : publish_connected_broadcast(msg)
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
# Publish a directed request via a Federation Broker
|
237
|
+
#
|
238
|
+
# @param msg [Message]
|
239
|
+
def publish_federated_directed(msg)
|
240
|
+
messages = []
|
241
|
+
target = target_for(msg, msg.discovered_hosts[0])
|
242
|
+
|
243
|
+
msg.discovered_hosts.in_groups_of(200) do |nodes|
|
244
|
+
node_targets = nodes.compact.map do |node|
|
245
|
+
target_for(msg, node)[:name]
|
246
|
+
end
|
247
|
+
|
248
|
+
data = {
|
249
|
+
"protocol" => "choria:transport:1",
|
250
|
+
"data" => msg.payload,
|
251
|
+
"headers" => {
|
252
|
+
"federation" => {
|
253
|
+
"target" => node_targets,
|
254
|
+
"req" => msg.requestid
|
255
|
+
}
|
256
|
+
}.merge(target[:headers])
|
257
|
+
}
|
258
|
+
|
259
|
+
messages << JSON.dump(data)
|
260
|
+
end
|
261
|
+
|
262
|
+
choria.federation_collectives.each do |network|
|
263
|
+
messages.each do |data|
|
264
|
+
network_target = "choria.federation.%s.federation" % network
|
265
|
+
|
266
|
+
Log.debug("Sending a federated direct message via NATS target '%s' for message type %s" % [network_target, msg.type])
|
267
|
+
|
268
|
+
connection.publish(network_target, data, target[:headers]["reply-to"])
|
269
|
+
end
|
270
|
+
end
|
271
|
+
end
|
272
|
+
|
273
|
+
# Publish a directed request to a connected collective
|
274
|
+
#
|
275
|
+
# @param msg [Message]
|
276
|
+
def publish_connected_directed(msg)
|
277
|
+
msg.discovered_hosts.each do |node|
|
278
|
+
target = target_for(msg, node)
|
279
|
+
data = {
|
280
|
+
"protocol" => "choria:transport:1",
|
281
|
+
"data" => msg.payload,
|
282
|
+
"headers" => target[:headers]
|
283
|
+
}
|
284
|
+
|
285
|
+
Log.debug("Sending a direct message to %s via NATS target '%s' for message type %s" % [node, target.inspect, msg.type])
|
286
|
+
|
287
|
+
connection.publish(target[:name], data.to_json, target[:headers]["reply-to"])
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
# Publish a broadcast message to via a Federation Broker
|
292
|
+
#
|
293
|
+
# @param msg [Message]
|
294
|
+
def publish_federated_broadcast(msg)
|
295
|
+
target = target_for(msg)
|
296
|
+
data = {
|
297
|
+
"protocol" => "choria:transport:1",
|
298
|
+
"data" => msg.payload,
|
299
|
+
"headers" => {
|
300
|
+
"federation" => {
|
301
|
+
"target" => [target[:name]],
|
302
|
+
"req" => msg.requestid
|
303
|
+
}
|
304
|
+
}.merge(target[:headers])
|
305
|
+
}
|
306
|
+
|
307
|
+
data = JSON.dump(data)
|
308
|
+
|
309
|
+
choria.federation_collectives.each do |network|
|
310
|
+
target[:name] = "choria.federation.%s.federation" % network
|
311
|
+
|
312
|
+
Log.debug("Sending a federated broadcast message to NATS target '%s' for message type %s" % [target.inspect, msg.type])
|
313
|
+
|
314
|
+
connection.publish(target[:name], data, target[:headers]["reply-to"])
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
# Publish a broadcast message to a connected collective
|
319
|
+
#
|
320
|
+
# @param msg [Message]
|
321
|
+
def publish_connected_broadcast(msg)
|
322
|
+
target = target_for(msg)
|
323
|
+
data = {
|
324
|
+
"protocol" => "choria:transport:1",
|
325
|
+
"data" => msg.payload,
|
326
|
+
"headers" => target[:headers]
|
327
|
+
}
|
328
|
+
|
329
|
+
# only happens when replying
|
330
|
+
received_message = msg.request
|
331
|
+
data["headers"]["federation"] = received_message.headers["federation"] if received_message && received_message.headers.include?("federation")
|
332
|
+
|
333
|
+
Log.debug("Sending a broadcast message to NATS target '%s' for message type %s" % [target.inspect, msg.type])
|
334
|
+
|
335
|
+
connection.publish(target[:name], JSON.dump(data), target[:headers]["reply-to"])
|
336
|
+
end
|
337
|
+
|
338
|
+
# Unsubscribe from the target for a agent
|
339
|
+
#
|
340
|
+
# @see make_target
|
341
|
+
# @param agent [String] agent name
|
342
|
+
# @param type [:reply, :broadcast, :request, :direct_request, :directed] type of message you want a subscription for
|
343
|
+
# @param collective [String] the collective to subscribe for
|
344
|
+
# @return [void]
|
345
|
+
def unsubscribe(agent, type, collective)
|
346
|
+
target = make_target(agent, type, collective)
|
347
|
+
Log.debug("Unsubscribing from %s" % target)
|
348
|
+
|
349
|
+
connection.unsubscribe(target)
|
350
|
+
end
|
351
|
+
|
352
|
+
# Subscribes to the topics/queues needed for a particular agent
|
353
|
+
#
|
354
|
+
# @see make_target
|
355
|
+
# @param agent [String] agent name
|
356
|
+
# @param type [:reply, :broadcast, :request, :direct_request, :directed] type of message you want a subscription for
|
357
|
+
# @param collective [String] the collective to subscribe for
|
358
|
+
# @return [void]
|
359
|
+
def subscribe(agent, type, collective)
|
360
|
+
target = make_target(agent, type, collective)
|
361
|
+
|
362
|
+
connection.subscribe(target)
|
363
|
+
end
|
364
|
+
|
365
|
+
# Receives a message from the middleware
|
366
|
+
#
|
367
|
+
# @note blocks until one is received
|
368
|
+
# @return [Message]
|
369
|
+
def receive
|
370
|
+
msg = nil
|
371
|
+
|
372
|
+
until msg
|
373
|
+
received = connection.receive
|
374
|
+
|
375
|
+
Thread.pass
|
376
|
+
|
377
|
+
begin
|
378
|
+
msg = JSON.parse(received)
|
379
|
+
rescue
|
380
|
+
Log.warn("Got non JSON data from the broker: %s" % [received])
|
381
|
+
msg = nil
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
msg["headers"]["seen-by"] << [connected_server.to_s, @config.identity] if msg["headers"].include?("seen-by")
|
386
|
+
|
387
|
+
Message.new(msg["data"], msg, :base64 => true, :headers => msg["headers"])
|
388
|
+
end
|
389
|
+
|
390
|
+
# Retrieves the list of server and port combos to attempt to connect to
|
391
|
+
#
|
392
|
+
# Configured servers are checked, then SRV records and finally a fall
|
393
|
+
# back to puppet:4222 is done
|
394
|
+
#
|
395
|
+
# @return [Array<String>] list of servers in form of a URI
|
396
|
+
def server_list
|
397
|
+
uris = choria.middleware_servers("puppet", "4222").map do |host, port|
|
398
|
+
URI("nats://%s:%s" % [host, port])
|
399
|
+
end
|
400
|
+
|
401
|
+
decorate_servers_with_users(uris).map(&:to_s)
|
402
|
+
end
|
403
|
+
|
404
|
+
# Add user and pass to a series of URIs
|
405
|
+
#
|
406
|
+
# @param servers [Array<URI>] list of URI's to decorate
|
407
|
+
# @return [Array<URI>]
|
408
|
+
def decorate_servers_with_users(servers)
|
409
|
+
user = get_option("nats.user", environment["MCOLLECTIVE_NATS_USERNAME"])
|
410
|
+
pass = get_option("nats.pass", environment["MCOLLECTIVE_NATS_PASSWORD"])
|
411
|
+
|
412
|
+
if choria.anon_tls?
|
413
|
+
user = PluginManager["security_plugin"].request_signer.token
|
414
|
+
pass = nil
|
415
|
+
end
|
416
|
+
|
417
|
+
if user || pass
|
418
|
+
servers.each do |uri|
|
419
|
+
uri.user = user
|
420
|
+
uri.password = pass
|
421
|
+
end
|
422
|
+
end
|
423
|
+
|
424
|
+
servers
|
425
|
+
end
|
426
|
+
|
427
|
+
# Retrieves the environment, mainly used for testing
|
428
|
+
def environment
|
429
|
+
ENV
|
430
|
+
end
|
431
|
+
|
432
|
+
# Gets a config option
|
433
|
+
#
|
434
|
+
# @param opt [String] config option to look up
|
435
|
+
# @param default [Object] default to return when not found
|
436
|
+
# @return [Object] the found data or default
|
437
|
+
# @raise [StandardError] when no default is given and option is not found
|
438
|
+
def get_option(opt, default=:_unset)
|
439
|
+
return @config.pluginconf[opt] if @config.pluginconf.include?(opt)
|
440
|
+
return default unless default == :_unset
|
441
|
+
|
442
|
+
raise("No plugin.%s configuration option given" % opt)
|
443
|
+
end
|
444
|
+
|
445
|
+
def choria
|
446
|
+
@_choria ||= Util::Choria.new(false)
|
447
|
+
end
|
448
|
+
end
|
449
|
+
end
|
450
|
+
end
|