choria-mcorpc-support 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/mco +64 -0
- data/lib/mcollective.rb +63 -0
- data/lib/mcollective/agent.rb +5 -0
- data/lib/mcollective/agents.rb +149 -0
- data/lib/mcollective/aggregate.rb +85 -0
- data/lib/mcollective/aggregate/average.ddl +33 -0
- data/lib/mcollective/aggregate/average.rb +29 -0
- data/lib/mcollective/aggregate/base.rb +40 -0
- data/lib/mcollective/aggregate/result.rb +9 -0
- data/lib/mcollective/aggregate/result/base.rb +25 -0
- data/lib/mcollective/aggregate/result/collection_result.rb +19 -0
- data/lib/mcollective/aggregate/result/numeric_result.rb +13 -0
- data/lib/mcollective/aggregate/sum.ddl +33 -0
- data/lib/mcollective/aggregate/sum.rb +18 -0
- data/lib/mcollective/aggregate/summary.ddl +33 -0
- data/lib/mcollective/aggregate/summary.rb +53 -0
- data/lib/mcollective/application.rb +365 -0
- data/lib/mcollective/application/completion.rb +104 -0
- data/lib/mcollective/application/describe_filter.rb +87 -0
- data/lib/mcollective/application/facts.rb +62 -0
- data/lib/mcollective/application/find.rb +23 -0
- data/lib/mcollective/application/help.rb +28 -0
- data/lib/mcollective/application/inventory.rb +344 -0
- data/lib/mcollective/application/ping.rb +82 -0
- data/lib/mcollective/application/plugin.rb +369 -0
- data/lib/mcollective/application/rpc.rb +111 -0
- data/lib/mcollective/applications.rb +134 -0
- data/lib/mcollective/cache.rb +145 -0
- data/lib/mcollective/client.rb +353 -0
- data/lib/mcollective/config.rb +245 -0
- data/lib/mcollective/connector.rb +18 -0
- data/lib/mcollective/connector/base.rb +26 -0
- data/lib/mcollective/data.rb +91 -0
- data/lib/mcollective/data/agent_data.ddl +22 -0
- data/lib/mcollective/data/agent_data.rb +17 -0
- data/lib/mcollective/data/base.rb +67 -0
- data/lib/mcollective/data/collective_data.ddl +20 -0
- data/lib/mcollective/data/collective_data.rb +9 -0
- data/lib/mcollective/data/fact_data.ddl +28 -0
- data/lib/mcollective/data/fact_data.rb +55 -0
- data/lib/mcollective/data/fstat_data.ddl +89 -0
- data/lib/mcollective/data/fstat_data.rb +56 -0
- data/lib/mcollective/data/result.rb +45 -0
- data/lib/mcollective/ddl.rb +113 -0
- data/lib/mcollective/ddl/agentddl.rb +253 -0
- data/lib/mcollective/ddl/base.rb +217 -0
- data/lib/mcollective/ddl/dataddl.rb +56 -0
- data/lib/mcollective/ddl/discoveryddl.rb +52 -0
- data/lib/mcollective/ddl/validatorddl.rb +6 -0
- data/lib/mcollective/discovery.rb +143 -0
- data/lib/mcollective/discovery/flatfile.ddl +11 -0
- data/lib/mcollective/discovery/flatfile.rb +48 -0
- data/lib/mcollective/discovery/mc.ddl +11 -0
- data/lib/mcollective/discovery/mc.rb +30 -0
- data/lib/mcollective/discovery/stdin.ddl +11 -0
- data/lib/mcollective/discovery/stdin.rb +68 -0
- data/lib/mcollective/exceptions.rb +28 -0
- data/lib/mcollective/facts.rb +39 -0
- data/lib/mcollective/facts/base.rb +100 -0
- data/lib/mcollective/facts/yaml_facts.rb +65 -0
- data/lib/mcollective/generators.rb +7 -0
- data/lib/mcollective/generators/agent_generator.rb +51 -0
- data/lib/mcollective/generators/base.rb +46 -0
- data/lib/mcollective/generators/data_generator.rb +51 -0
- data/lib/mcollective/generators/templates/action_snippet.erb +13 -0
- data/lib/mcollective/generators/templates/data_input_snippet.erb +7 -0
- data/lib/mcollective/generators/templates/ddl.erb +8 -0
- data/lib/mcollective/generators/templates/plugin.erb +7 -0
- data/lib/mcollective/log.rb +118 -0
- data/lib/mcollective/logger.rb +5 -0
- data/lib/mcollective/logger/base.rb +77 -0
- data/lib/mcollective/logger/console_logger.rb +61 -0
- data/lib/mcollective/logger/file_logger.rb +53 -0
- data/lib/mcollective/logger/syslog_logger.rb +53 -0
- data/lib/mcollective/matcher.rb +224 -0
- data/lib/mcollective/matcher/parser.rb +128 -0
- data/lib/mcollective/matcher/scanner.rb +241 -0
- data/lib/mcollective/message.rb +248 -0
- data/lib/mcollective/monkey_patches.rb +152 -0
- data/lib/mcollective/optionparser.rb +197 -0
- data/lib/mcollective/pluginmanager.rb +180 -0
- data/lib/mcollective/pluginpackager.rb +98 -0
- data/lib/mcollective/pluginpackager/agent_definition.rb +94 -0
- data/lib/mcollective/pluginpackager/debpackage_packager.rb +237 -0
- data/lib/mcollective/pluginpackager/modulepackage_packager.rb +127 -0
- data/lib/mcollective/pluginpackager/ospackage_packager.rb +59 -0
- data/lib/mcollective/pluginpackager/rpmpackage_packager.rb +180 -0
- data/lib/mcollective/pluginpackager/standard_definition.rb +69 -0
- data/lib/mcollective/pluginpackager/templates/debian/Makefile.erb +7 -0
- data/lib/mcollective/pluginpackager/templates/debian/changelog.erb +5 -0
- data/lib/mcollective/pluginpackager/templates/debian/compat.erb +1 -0
- data/lib/mcollective/pluginpackager/templates/debian/control.erb +15 -0
- data/lib/mcollective/pluginpackager/templates/debian/copyright.erb +8 -0
- data/lib/mcollective/pluginpackager/templates/debian/rules.erb +6 -0
- data/lib/mcollective/pluginpackager/templates/module/Modulefile.erb +5 -0
- data/lib/mcollective/pluginpackager/templates/module/README.md.erb +37 -0
- data/lib/mcollective/pluginpackager/templates/module/_manifest.pp.erb +9 -0
- data/lib/mcollective/pluginpackager/templates/redhat/rpm_spec.erb +63 -0
- data/lib/mcollective/registration/base.rb +91 -0
- data/lib/mcollective/rpc.rb +182 -0
- data/lib/mcollective/rpc/actionrunner.rb +158 -0
- data/lib/mcollective/rpc/agent.rb +374 -0
- data/lib/mcollective/rpc/audit.rb +38 -0
- data/lib/mcollective/rpc/client.rb +1066 -0
- data/lib/mcollective/rpc/helpers.rb +321 -0
- data/lib/mcollective/rpc/progress.rb +63 -0
- data/lib/mcollective/rpc/reply.rb +87 -0
- data/lib/mcollective/rpc/request.rb +86 -0
- data/lib/mcollective/rpc/result.rb +90 -0
- data/lib/mcollective/rpc/stats.rb +294 -0
- data/lib/mcollective/runnerstats.rb +90 -0
- data/lib/mcollective/security.rb +26 -0
- data/lib/mcollective/security/base.rb +244 -0
- data/lib/mcollective/shell.rb +126 -0
- data/lib/mcollective/ssl.rb +285 -0
- data/lib/mcollective/util.rb +579 -0
- data/lib/mcollective/validator.rb +85 -0
- data/lib/mcollective/validator/array_validator.ddl +7 -0
- data/lib/mcollective/validator/array_validator.rb +9 -0
- data/lib/mcollective/validator/ipv4address_validator.ddl +7 -0
- data/lib/mcollective/validator/ipv4address_validator.rb +16 -0
- data/lib/mcollective/validator/ipv6address_validator.ddl +7 -0
- data/lib/mcollective/validator/ipv6address_validator.rb +16 -0
- data/lib/mcollective/validator/length_validator.ddl +7 -0
- data/lib/mcollective/validator/length_validator.rb +11 -0
- data/lib/mcollective/validator/regex_validator.ddl +7 -0
- data/lib/mcollective/validator/regex_validator.rb +9 -0
- data/lib/mcollective/validator/shellsafe_validator.ddl +7 -0
- data/lib/mcollective/validator/shellsafe_validator.rb +13 -0
- data/lib/mcollective/validator/typecheck_validator.ddl +7 -0
- data/lib/mcollective/validator/typecheck_validator.rb +28 -0
- metadata +215 -0
@@ -0,0 +1,111 @@
|
|
1
|
+
class MCollective::Application::Rpc<MCollective::Application
|
2
|
+
description "Generic RPC agent client application"
|
3
|
+
|
4
|
+
usage "mco rpc [options] [filters] --agent <agent> --action <action> [--argument <key=val> --argument ...]"
|
5
|
+
usage "mco rpc [options] [filters] <agent> <action> [<key=val> <key=val> ...]"
|
6
|
+
|
7
|
+
option :show_results,
|
8
|
+
:description => "Do not process results, just send request",
|
9
|
+
:arguments => ["--no-results", "--nr"],
|
10
|
+
:default => true,
|
11
|
+
:type => :bool
|
12
|
+
|
13
|
+
option :agent,
|
14
|
+
:description => "Agent to call",
|
15
|
+
:arguments => ["-a", "--agent AGENT"]
|
16
|
+
|
17
|
+
option :action,
|
18
|
+
:description => "Action to call",
|
19
|
+
:arguments => ["--action ACTION"]
|
20
|
+
|
21
|
+
option :arguments,
|
22
|
+
:description => "Arguments to pass to agent",
|
23
|
+
:arguments => ["--arg", "--argument ARGUMENT"],
|
24
|
+
:type => :array,
|
25
|
+
:default => [],
|
26
|
+
:validate => Proc.new {|val| val.match(/^(.+?)=(.+)$/) ? true : "Could not parse --arg #{val} should be of the form key=val" }
|
27
|
+
|
28
|
+
def post_option_parser(configuration)
|
29
|
+
# handle the alternative format that optparse cant parse
|
30
|
+
unless (configuration.include?(:agent) && configuration.include?(:action))
|
31
|
+
if ARGV.length >= 2
|
32
|
+
configuration[:agent] = ARGV[0]
|
33
|
+
ARGV.delete_at(0)
|
34
|
+
|
35
|
+
configuration[:action] = ARGV[0]
|
36
|
+
ARGV.delete_at(0)
|
37
|
+
|
38
|
+
ARGV.each do |v|
|
39
|
+
if v =~ /^(.+?)=(.+)$/
|
40
|
+
configuration[:arguments] = [] unless configuration.include?(:arguments)
|
41
|
+
configuration[:arguments] << v
|
42
|
+
else
|
43
|
+
STDERR.puts("Could not parse --arg #{v}")
|
44
|
+
exit(1)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
else
|
48
|
+
STDERR.puts("No agent, action and arguments specified")
|
49
|
+
exit(1)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# convert arguments to symbols for keys to comply with simplerpc conventions
|
54
|
+
args = configuration[:arguments].clone
|
55
|
+
configuration[:arguments] = {}
|
56
|
+
|
57
|
+
args.each do |v|
|
58
|
+
if v =~ /^(.+?)=(.+)$/
|
59
|
+
configuration[:arguments][$1.to_sym] = $2
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def string_to_ddl_type(arguments, ddl)
|
65
|
+
return if ddl.empty?
|
66
|
+
|
67
|
+
arguments.keys.each do |key|
|
68
|
+
if ddl[:input].keys.include?(key)
|
69
|
+
case ddl[:input][key][:type]
|
70
|
+
when :boolean
|
71
|
+
arguments[key] = MCollective::DDL.string_to_boolean(arguments[key])
|
72
|
+
|
73
|
+
when :number, :integer, :float
|
74
|
+
arguments[key] = MCollective::DDL.string_to_number(arguments[key])
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def main
|
81
|
+
mc = rpcclient(configuration[:agent])
|
82
|
+
|
83
|
+
mc.agent_filter(configuration[:agent])
|
84
|
+
|
85
|
+
string_to_ddl_type(configuration[:arguments], mc.ddl.action_interface(configuration[:action])) if mc.ddl
|
86
|
+
|
87
|
+
mc.validate_request(configuration[:action], configuration[:arguments])
|
88
|
+
|
89
|
+
if mc.reply_to
|
90
|
+
configuration[:arguments][:process_results] = true
|
91
|
+
|
92
|
+
puts "Request sent with id: " + mc.send(configuration[:action], configuration[:arguments]) + " replies to #{mc.reply_to}"
|
93
|
+
elsif !configuration[:show_results]
|
94
|
+
configuration[:arguments][:process_results] = false
|
95
|
+
|
96
|
+
puts "Request sent with id: " + mc.send(configuration[:action], configuration[:arguments])
|
97
|
+
else
|
98
|
+
discover_args = {:verbose => true}
|
99
|
+
|
100
|
+
mc.detect_and_set_stdin_discovery
|
101
|
+
|
102
|
+
mc.discover discover_args
|
103
|
+
|
104
|
+
printrpc mc.send(configuration[:action], configuration[:arguments])
|
105
|
+
|
106
|
+
printrpcstats :summarize => true, :caption => "#{configuration[:agent]}##{configuration[:action]} call stats" if mc.discover.size > 0
|
107
|
+
|
108
|
+
halt mc.stats
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
module MCollective
|
2
|
+
class Applications
|
3
|
+
def self.[](appname)
|
4
|
+
load_application(appname)
|
5
|
+
PluginManager["#{appname}_application"]
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.run(appname)
|
9
|
+
load_config
|
10
|
+
|
11
|
+
begin
|
12
|
+
load_application(appname)
|
13
|
+
rescue Exception => e
|
14
|
+
e.backtrace.first << Util.colorize(:red, " <----")
|
15
|
+
STDERR.puts "Application '#{appname}' failed to load:"
|
16
|
+
STDERR.puts
|
17
|
+
STDERR.puts Util.colorize(:red, " #{e} (#{e.class})")
|
18
|
+
STDERR.puts
|
19
|
+
STDERR.puts " %s" % [e.backtrace.join("\n ")]
|
20
|
+
exit 1
|
21
|
+
end
|
22
|
+
|
23
|
+
PluginManager["#{appname}_application"].run
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.load_application(appname)
|
27
|
+
return if PluginManager.include?("#{appname}_application")
|
28
|
+
|
29
|
+
load_config
|
30
|
+
|
31
|
+
PluginManager.loadclass "MCollective::Application::#{appname.capitalize}"
|
32
|
+
PluginManager << {:type => "#{appname}_application", :class => "MCollective::Application::#{appname.capitalize}"}
|
33
|
+
end
|
34
|
+
|
35
|
+
# Returns an array of applications found in the lib dirs
|
36
|
+
def self.list
|
37
|
+
load_config
|
38
|
+
|
39
|
+
PluginManager.find("application")
|
40
|
+
rescue SystemExit
|
41
|
+
exit 1
|
42
|
+
rescue Exception => e
|
43
|
+
STDERR.puts "Failed to generate application list: #{e.class}: #{e}"
|
44
|
+
exit 1
|
45
|
+
end
|
46
|
+
|
47
|
+
# Filters a string of opts out using Shellwords
|
48
|
+
# keeping only things related to --config and -c
|
49
|
+
def self.filter_extra_options(opts)
|
50
|
+
res = ""
|
51
|
+
words = Shellwords.shellwords(opts)
|
52
|
+
words.each_with_index do |word,idx|
|
53
|
+
if word == "-c"
|
54
|
+
return "--config=#{words[idx + 1]}"
|
55
|
+
elsif word == "--config"
|
56
|
+
return "--config=#{words[idx + 1]}"
|
57
|
+
elsif word =~ /\-c=/
|
58
|
+
return word
|
59
|
+
elsif word =~ /\-\-config=/
|
60
|
+
return word
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
return ""
|
65
|
+
end
|
66
|
+
|
67
|
+
# We need to know the config file in order to know the libdir
|
68
|
+
# so that we can find applications.
|
69
|
+
#
|
70
|
+
# The problem is the CLI might be stuffed with options only the
|
71
|
+
# app in the libdir might understand so we have a chicken and
|
72
|
+
# egg situation.
|
73
|
+
#
|
74
|
+
# We're parsing and filtering MCOLLECTIVE_EXTRA_OPTS removing
|
75
|
+
# all but config related options and parsing the options looking
|
76
|
+
# just for the config file.
|
77
|
+
#
|
78
|
+
# We're handling failures gracefully and finally restoring the
|
79
|
+
# ARG and MCOLLECTIVE_EXTRA_OPTS to the state they were before
|
80
|
+
# we started parsing.
|
81
|
+
#
|
82
|
+
# This is mostly a hack, when we're redoing how config works
|
83
|
+
# this stuff should be made less sucky
|
84
|
+
def self.load_config
|
85
|
+
return if Config.instance.configured
|
86
|
+
|
87
|
+
original_argv = ARGV.clone
|
88
|
+
original_extra_opts = ENV["MCOLLECTIVE_EXTRA_OPTS"].clone rescue nil
|
89
|
+
configfile = nil
|
90
|
+
|
91
|
+
parser = OptionParser.new
|
92
|
+
parser.on("--config CONFIG", "-c", "Config file") do |f|
|
93
|
+
configfile = f
|
94
|
+
end
|
95
|
+
|
96
|
+
parser.program_name = $0
|
97
|
+
|
98
|
+
parser.on("--help")
|
99
|
+
|
100
|
+
# avoid option parsers own internal version handling that sux
|
101
|
+
parser.on("-v", "--verbose")
|
102
|
+
|
103
|
+
if original_extra_opts
|
104
|
+
begin
|
105
|
+
# optparse will parse the whole ENV in one go and refuse
|
106
|
+
# to play along with the retry trick I do below so in
|
107
|
+
# order to handle unknown options properly I parse out
|
108
|
+
# only -c and --config deleting everything else and
|
109
|
+
# then restore the environment variable later when I
|
110
|
+
# am done with it
|
111
|
+
ENV["MCOLLECTIVE_EXTRA_OPTS"] = filter_extra_options(ENV["MCOLLECTIVE_EXTRA_OPTS"].clone)
|
112
|
+
parser.environment("MCOLLECTIVE_EXTRA_OPTS")
|
113
|
+
rescue Exception => e
|
114
|
+
Log.error("Failed to parse MCOLLECTIVE_EXTRA_OPTS: #{e}")
|
115
|
+
end
|
116
|
+
|
117
|
+
ENV["MCOLLECTIVE_EXTRA_OPTS"] = original_extra_opts.clone
|
118
|
+
end
|
119
|
+
|
120
|
+
begin
|
121
|
+
parser.parse!
|
122
|
+
rescue OptionParser::InvalidOption => e
|
123
|
+
retry
|
124
|
+
end
|
125
|
+
|
126
|
+
ARGV.clear
|
127
|
+
original_argv.each {|a| ARGV << a}
|
128
|
+
|
129
|
+
configfile = Util.config_file_for_user unless configfile
|
130
|
+
|
131
|
+
Config.instance.loadconfig(configfile)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
@@ -0,0 +1,145 @@
|
|
1
|
+
module MCollective
|
2
|
+
# A class to manage a number of named caches. Each cache can have a unique
|
3
|
+
# timeout for keys in it and each cache has an independent Mutex protecting
|
4
|
+
# access to it.
|
5
|
+
#
|
6
|
+
# This cache is setup early in the process of loading the mcollective
|
7
|
+
# libraries, before any threads are created etc making it suitable as a
|
8
|
+
# cross thread cache or just a store for Mutexes you need to use across
|
9
|
+
# threads like in an Agent or something.
|
10
|
+
#
|
11
|
+
# # sets up a new cache, noop if it already exist
|
12
|
+
# Cache.setup(:ddl, 600)
|
13
|
+
# => true
|
14
|
+
#
|
15
|
+
# # writes an item to the :ddl cache, this item will
|
16
|
+
# # be valid on the cache for 600 seconds
|
17
|
+
# Cache.write(:ddl, :something, "value")
|
18
|
+
# => "value"
|
19
|
+
#
|
20
|
+
# # reads from the cache, read failures due to non existing
|
21
|
+
# # data or expired data will raise an exception
|
22
|
+
# Cache.read(:ddl, :something)
|
23
|
+
# => "value"
|
24
|
+
#
|
25
|
+
# # time left till expiry, raises if nothing is found
|
26
|
+
# Cache.ttl(:ddl, :something)
|
27
|
+
# => 500
|
28
|
+
#
|
29
|
+
# # forcibly evict something from the cache
|
30
|
+
# Cache.invalidate!(:ddl, :something)
|
31
|
+
# => "value"
|
32
|
+
#
|
33
|
+
# # deletes an entire named cache and its mutexes
|
34
|
+
# Cache.delete!(:ddl)
|
35
|
+
# => true
|
36
|
+
#
|
37
|
+
# # you can also just use this cache as a global mutex store
|
38
|
+
# Cache.setup(:mylock)
|
39
|
+
#
|
40
|
+
# Cache.synchronize(:mylock) do
|
41
|
+
# do_something()
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
module Cache
|
45
|
+
# protects access to @cache_locks and top level @cache
|
46
|
+
@locks_mutex = Mutex.new
|
47
|
+
|
48
|
+
# stores a mutex per named cache
|
49
|
+
@cache_locks = {}
|
50
|
+
|
51
|
+
# the named caches protected by items in @cache_locks
|
52
|
+
@cache = {}
|
53
|
+
|
54
|
+
def self.setup(cache_name, ttl=300)
|
55
|
+
@locks_mutex.synchronize do
|
56
|
+
break if @cache_locks.include?(cache_name)
|
57
|
+
|
58
|
+
@cache_locks[cache_name] = Mutex.new
|
59
|
+
|
60
|
+
@cache_locks[cache_name].synchronize do
|
61
|
+
@cache[cache_name] = {:max_age => Float(ttl)}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
true
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.has_cache?(cache_name)
|
69
|
+
@locks_mutex.synchronize do
|
70
|
+
@cache.include?(cache_name)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.delete!(cache_name)
|
75
|
+
@locks_mutex.synchronize do
|
76
|
+
raise("No cache called '%s'" % cache_name) unless @cache_locks.include?(cache_name)
|
77
|
+
|
78
|
+
@cache_locks.delete(cache_name)
|
79
|
+
@cache.delete(cache_name)
|
80
|
+
end
|
81
|
+
|
82
|
+
true
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.write(cache_name, key, value)
|
86
|
+
raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
|
87
|
+
|
88
|
+
@cache_locks[cache_name].synchronize do
|
89
|
+
@cache[cache_name][key] ||= {}
|
90
|
+
@cache[cache_name][key][:cache_create_time] = Time.now
|
91
|
+
@cache[cache_name][key][:value] = value
|
92
|
+
end
|
93
|
+
|
94
|
+
value
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.read(cache_name, key)
|
98
|
+
raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
|
99
|
+
|
100
|
+
unless ttl(cache_name, key) > 0
|
101
|
+
Log.debug("Cache expired on '%s' key '%s'" % [cache_name, key])
|
102
|
+
raise("Cache for item '%s' on cache '%s' has expired" % [key, cache_name])
|
103
|
+
end
|
104
|
+
|
105
|
+
Log.debug("Cache hit on '%s' key '%s'" % [cache_name, key])
|
106
|
+
|
107
|
+
@cache_locks[cache_name].synchronize do
|
108
|
+
@cache[cache_name][key][:value]
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def self.ttl(cache_name, key)
|
113
|
+
raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
|
114
|
+
|
115
|
+
@cache_locks[cache_name].synchronize do
|
116
|
+
unless @cache[cache_name].include?(key)
|
117
|
+
Log.debug("Cache miss on '%s' key '%s'" % [cache_name, key])
|
118
|
+
raise("No item called '%s' for cache '%s'" % [key, cache_name])
|
119
|
+
end
|
120
|
+
|
121
|
+
@cache[cache_name][:max_age] - (Time.now - @cache[cache_name][key][:cache_create_time])
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def self.synchronize(cache_name)
|
126
|
+
raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
|
127
|
+
|
128
|
+
raise ("No block supplied to synchronize") unless block_given?
|
129
|
+
|
130
|
+
@cache_locks[cache_name].synchronize do
|
131
|
+
yield
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def self.invalidate!(cache_name, key)
|
136
|
+
raise("No cache called '%s'" % cache_name) unless @cache.include?(cache_name)
|
137
|
+
|
138
|
+
@cache_locks[cache_name].synchronize do
|
139
|
+
return false unless @cache[cache_name].include?(key)
|
140
|
+
|
141
|
+
@cache[cache_name].delete(key)
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
@@ -0,0 +1,353 @@
|
|
1
|
+
module MCollective
|
2
|
+
# Helpers for writing clients that can talk to agents, do discovery and so forth
|
3
|
+
class Client
|
4
|
+
attr_accessor :options, :stats, :discoverer, :connection_timeout
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@config = Config.instance
|
8
|
+
@options = nil
|
9
|
+
|
10
|
+
if options.is_a?(String)
|
11
|
+
# String is the path to a config file
|
12
|
+
@config.loadconfig(options) unless @config.configured
|
13
|
+
elsif options.is_a?(Hash)
|
14
|
+
@config.loadconfig(options[:config]) unless @config.configured
|
15
|
+
@options = options
|
16
|
+
@connection_timeout = options[:connection_timeout]
|
17
|
+
else
|
18
|
+
raise "Invalid parameter passed to Client constructor. Valid types are Hash or String"
|
19
|
+
end
|
20
|
+
|
21
|
+
@connection_timeout ||= @config.connection_timeout
|
22
|
+
|
23
|
+
@connection = PluginManager["connector_plugin"]
|
24
|
+
@security = PluginManager["security_plugin"]
|
25
|
+
|
26
|
+
@security.initiated_by = :client
|
27
|
+
@subscriptions = {}
|
28
|
+
|
29
|
+
@discoverer = Discovery.new(self)
|
30
|
+
|
31
|
+
# Time box the connection if a timeout has been specified
|
32
|
+
# connection_timeout defaults to nil which means it will try forever if
|
33
|
+
# not specified
|
34
|
+
begin
|
35
|
+
Timeout::timeout(@connection_timeout, ClientTimeoutError) do
|
36
|
+
@connection.connect
|
37
|
+
end
|
38
|
+
rescue ClientTimeoutError => e
|
39
|
+
Log.error("Timeout occured while trying to connect to middleware")
|
40
|
+
raise e
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
@@request_sequence = 0
|
45
|
+
def self.request_sequence
|
46
|
+
@@request_sequence
|
47
|
+
end
|
48
|
+
|
49
|
+
# Returns the configured main collective if no
|
50
|
+
# specific collective is specified as options
|
51
|
+
def collective
|
52
|
+
if @options[:collective].nil?
|
53
|
+
@config.main_collective
|
54
|
+
else
|
55
|
+
@options[:collective]
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Disconnects cleanly from the middleware
|
60
|
+
def disconnect
|
61
|
+
Log.debug("Disconnecting from the middleware")
|
62
|
+
@connection.disconnect
|
63
|
+
end
|
64
|
+
|
65
|
+
# Sends a request and returns the generated request id, doesn't wait for
|
66
|
+
# responses and doesn't execute any passed in code blocks for responses
|
67
|
+
def sendreq(msg, agent, filter = {})
|
68
|
+
request = createreq(msg, agent, filter)
|
69
|
+
publish(request)
|
70
|
+
request.requestid
|
71
|
+
end
|
72
|
+
|
73
|
+
def createreq(msg, agent, filter ={})
|
74
|
+
if msg.is_a?(Message)
|
75
|
+
request = msg
|
76
|
+
agent = request.agent
|
77
|
+
else
|
78
|
+
ttl = @options[:ttl] || @config.ttl
|
79
|
+
request = Message.new(msg, nil, {:agent => agent, :type => :request, :collective => collective, :filter => filter, :ttl => ttl})
|
80
|
+
request.reply_to = @options[:reply_to] if @options[:reply_to]
|
81
|
+
end
|
82
|
+
|
83
|
+
@@request_sequence += 1
|
84
|
+
|
85
|
+
request.encode!
|
86
|
+
subscribe(agent, :reply) unless request.reply_to
|
87
|
+
request
|
88
|
+
end
|
89
|
+
|
90
|
+
def subscribe(agent, type)
|
91
|
+
unless @subscriptions.include?(agent)
|
92
|
+
subscription = Util.make_subscriptions(agent, type, collective)
|
93
|
+
Log.debug("Subscribing to #{type} target for agent #{agent}")
|
94
|
+
|
95
|
+
Util.subscribe(subscription)
|
96
|
+
@subscriptions[agent] = 1
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def unsubscribe(agent, type)
|
101
|
+
if @subscriptions.include?(agent)
|
102
|
+
subscription = Util.make_subscriptions(agent, type, collective)
|
103
|
+
Log.debug("Unsubscribing #{type} target for #{agent}")
|
104
|
+
|
105
|
+
Util.unsubscribe(subscription)
|
106
|
+
@subscriptions.delete(agent)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
# Blocking call that waits for ever for a message to arrive.
|
110
|
+
#
|
111
|
+
# If you give it a requestid this means you've previously send a request
|
112
|
+
# with that ID and now you just want replies that matches that id, in that
|
113
|
+
# case the current connection will just ignore all messages not directed at it
|
114
|
+
# and keep waiting for more till it finds a matching message.
|
115
|
+
def receive(requestid = nil)
|
116
|
+
reply = nil
|
117
|
+
|
118
|
+
begin
|
119
|
+
reply = @connection.receive
|
120
|
+
reply.type = :reply
|
121
|
+
reply.expected_msgid = requestid
|
122
|
+
|
123
|
+
reply.decode!
|
124
|
+
|
125
|
+
unless reply.requestid == requestid
|
126
|
+
raise(MsgDoesNotMatchRequestID, "Message reqid #{reply.requestid} does not match our reqid #{requestid}")
|
127
|
+
end
|
128
|
+
|
129
|
+
Log.debug("Received reply to #{reply.requestid} from #{reply.payload[:senderid]}")
|
130
|
+
rescue SecurityValidationFailed => e
|
131
|
+
Log.warn("Ignoring a message that did not pass security validations")
|
132
|
+
retry
|
133
|
+
rescue MsgDoesNotMatchRequestID => e
|
134
|
+
Log.debug("Ignoring a message for some other client : #{e.message}")
|
135
|
+
retry
|
136
|
+
end
|
137
|
+
|
138
|
+
reply
|
139
|
+
end
|
140
|
+
|
141
|
+
# Performs a discovery of nodes matching the filter passed
|
142
|
+
# returns an array of nodes
|
143
|
+
#
|
144
|
+
# An integer limit can be supplied this will have the effect
|
145
|
+
# of the discovery being cancelled soon as it reached the
|
146
|
+
# requested limit of hosts
|
147
|
+
def discover(filter, timeout, limit=0)
|
148
|
+
@discoverer.discover(filter.merge({'collective' => collective}), timeout, limit)
|
149
|
+
end
|
150
|
+
|
151
|
+
# Send a request, performs the passed block for each response
|
152
|
+
#
|
153
|
+
# times = req("status", "mcollectived", options, client) {|resp|
|
154
|
+
# pp resp
|
155
|
+
# }
|
156
|
+
#
|
157
|
+
# It returns a hash of times and timeouts for discovery and total run is taken from the options
|
158
|
+
# hash which in turn is generally built using MCollective::Optionparser
|
159
|
+
def req(body, agent=nil, options=false, waitfor=[], &block)
|
160
|
+
if body.is_a?(Message)
|
161
|
+
agent = body.agent
|
162
|
+
waitfor = body.discovered_hosts || []
|
163
|
+
@options = body.options
|
164
|
+
end
|
165
|
+
|
166
|
+
@options = options if options
|
167
|
+
threaded = @options[:threaded]
|
168
|
+
timeout = @discoverer.discovery_timeout(@options[:timeout], @options[:filter])
|
169
|
+
request = createreq(body, agent, @options[:filter])
|
170
|
+
publish_timeout = @options[:publish_timeout] || @config.publish_timeout
|
171
|
+
stat = {:starttime => Time.now.to_f, :discoverytime => 0, :blocktime => 0, :totaltime => 0}
|
172
|
+
STDOUT.sync = true
|
173
|
+
hosts_responded = 0
|
174
|
+
|
175
|
+
begin
|
176
|
+
if threaded
|
177
|
+
hosts_responded = threaded_req(request, publish_timeout, timeout, waitfor, &block)
|
178
|
+
else
|
179
|
+
hosts_responded = unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
|
180
|
+
end
|
181
|
+
rescue Interrupt => e
|
182
|
+
ensure
|
183
|
+
unsubscribe(agent, :reply)
|
184
|
+
end
|
185
|
+
|
186
|
+
return update_stat(stat, hosts_responded, request.requestid)
|
187
|
+
end
|
188
|
+
|
189
|
+
# Starts the client receiver and publisher unthreaded.
|
190
|
+
# This is the default client behaviour.
|
191
|
+
def unthreaded_req(request, publish_timeout, timeout, waitfor, &block)
|
192
|
+
start_publisher(request, publish_timeout)
|
193
|
+
start_receiver(request.requestid, waitfor, timeout, &block)
|
194
|
+
end
|
195
|
+
|
196
|
+
# Starts the client receiver and publisher in threads.
|
197
|
+
# This is activated when the 'threader_client' configuration
|
198
|
+
# option is set.
|
199
|
+
def threaded_req(request, publish_timeout, timeout, waitfor, &block)
|
200
|
+
Log.debug("Starting threaded client")
|
201
|
+
publisher = Thread.new do
|
202
|
+
start_publisher(request, publish_timeout)
|
203
|
+
end
|
204
|
+
|
205
|
+
# When the client is threaded we add the publishing timeout to
|
206
|
+
# the agent timeout so that the receiver doesn't time out before
|
207
|
+
# publishing has finished in cases where publish_timeout >= timeout.
|
208
|
+
total_timeout = publish_timeout + timeout
|
209
|
+
hosts_responded = 0
|
210
|
+
|
211
|
+
receiver = Thread.new do
|
212
|
+
hosts_responded = start_receiver(request.requestid, waitfor, total_timeout, &block)
|
213
|
+
end
|
214
|
+
|
215
|
+
receiver.join
|
216
|
+
hosts_responded
|
217
|
+
end
|
218
|
+
|
219
|
+
# Starts the request publishing routine
|
220
|
+
def start_publisher(request, publish_timeout)
|
221
|
+
Log.debug("Starting publishing with publish timeout of #{publish_timeout}")
|
222
|
+
begin
|
223
|
+
Timeout.timeout(publish_timeout) do
|
224
|
+
publish(request)
|
225
|
+
end
|
226
|
+
rescue Timeout::Error => e
|
227
|
+
Log.warn("Could not publish all messages. Publishing timed out.")
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
def publish(request)
|
232
|
+
Log.info("Sending request #{request.requestid} for agent '#{request.agent}' with ttl #{request.ttl} in collective '#{request.collective}'")
|
233
|
+
request.publish
|
234
|
+
end
|
235
|
+
|
236
|
+
# Starts the response receiver routine
|
237
|
+
# Expected to return the amount of received responses.
|
238
|
+
def start_receiver(requestid, waitfor, timeout, &block)
|
239
|
+
Log.debug("Starting response receiver with timeout of #{timeout}")
|
240
|
+
hosts_responded = 0
|
241
|
+
|
242
|
+
if (waitfor.is_a?(Array))
|
243
|
+
unfinished = Hash.new(0)
|
244
|
+
waitfor.each {|w| unfinished[w] += 1}
|
245
|
+
else
|
246
|
+
unfinished = []
|
247
|
+
end
|
248
|
+
|
249
|
+
begin
|
250
|
+
Timeout.timeout(timeout) do
|
251
|
+
loop do
|
252
|
+
resp = receive(requestid)
|
253
|
+
|
254
|
+
if block.arity == 2
|
255
|
+
yield resp.payload, resp
|
256
|
+
else
|
257
|
+
yield resp.payload
|
258
|
+
end
|
259
|
+
|
260
|
+
hosts_responded += 1
|
261
|
+
|
262
|
+
if (waitfor.is_a?(Array))
|
263
|
+
sender = resp.payload[:senderid]
|
264
|
+
if unfinished[sender] <= 1
|
265
|
+
unfinished.delete(sender)
|
266
|
+
else
|
267
|
+
unfinished[sender] -= 1
|
268
|
+
end
|
269
|
+
|
270
|
+
break if !waitfor.empty? && unfinished.empty?
|
271
|
+
else
|
272
|
+
break unless waitfor == 0 || hosts_responded < waitfor
|
273
|
+
end
|
274
|
+
end
|
275
|
+
end
|
276
|
+
rescue Timeout::Error => e
|
277
|
+
if waitfor.is_a?(Array)
|
278
|
+
if !unfinished.empty?
|
279
|
+
Log.warn("Could not receive all responses. Did not receive responses from #{unfinished.keys.join(', ')}")
|
280
|
+
end
|
281
|
+
elsif (waitfor > hosts_responded)
|
282
|
+
Log.warn("Could not receive all responses. Expected : #{waitfor}. Received : #{hosts_responded}")
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
hosts_responded
|
287
|
+
end
|
288
|
+
|
289
|
+
def update_stat(stat, hosts_responded, requestid)
|
290
|
+
stat[:totaltime] = Time.now.to_f - stat[:starttime]
|
291
|
+
stat[:blocktime] = stat[:totaltime] - stat[:discoverytime]
|
292
|
+
stat[:responses] = hosts_responded
|
293
|
+
stat[:noresponsefrom] = []
|
294
|
+
stat[:unexpectedresponsefrom] = []
|
295
|
+
stat[:requestid] = requestid
|
296
|
+
|
297
|
+
@stats = stat
|
298
|
+
end
|
299
|
+
|
300
|
+
def discovered_req(body, agent, options=false)
|
301
|
+
raise "Client#discovered_req has been removed, please port your agent and client to the SimpleRPC framework"
|
302
|
+
end
|
303
|
+
|
304
|
+
# Prints out the stats returns from req and discovered_req in a nice way
|
305
|
+
def display_stats(stats, options=false, caption="stomp call summary")
|
306
|
+
options = @options unless options
|
307
|
+
|
308
|
+
if options[:verbose]
|
309
|
+
puts("\n---- #{caption} ----")
|
310
|
+
|
311
|
+
if stats[:discovered]
|
312
|
+
puts(" Nodes: #{stats[:discovered]} / #{stats[:responses]}")
|
313
|
+
else
|
314
|
+
puts(" Nodes: #{stats[:responses]}")
|
315
|
+
end
|
316
|
+
|
317
|
+
printf(" Start Time: %s\n", Time.at(stats[:starttime]))
|
318
|
+
printf(" Discovery Time: %.2fms\n", stats[:discoverytime] * 1000)
|
319
|
+
printf(" Agent Time: %.2fms\n", stats[:blocktime] * 1000)
|
320
|
+
printf(" Total Time: %.2fms\n", stats[:totaltime] * 1000)
|
321
|
+
|
322
|
+
else
|
323
|
+
if stats[:discovered]
|
324
|
+
printf("\nFinished processing %d / %d hosts in %.2f ms\n\n", stats[:responses], stats[:discovered], stats[:blocktime] * 1000)
|
325
|
+
else
|
326
|
+
printf("\nFinished processing %d hosts in %.2f ms\n\n", stats[:responses], stats[:blocktime] * 1000)
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
if stats[:noresponsefrom].size > 0
|
331
|
+
puts("\nNo response from:\n")
|
332
|
+
|
333
|
+
stats[:noresponsefrom].each do |c|
|
334
|
+
puts if c % 4 == 1
|
335
|
+
printf("%30s", c)
|
336
|
+
end
|
337
|
+
|
338
|
+
puts
|
339
|
+
end
|
340
|
+
|
341
|
+
if stats[:unexpectedresponsefrom].size > 0
|
342
|
+
puts("\nUnexpected response from:\n")
|
343
|
+
|
344
|
+
stats[:unexpectedresponsefrom].each do |c|
|
345
|
+
puts if c % 4 == 1
|
346
|
+
printf("%30s", c)
|
347
|
+
end
|
348
|
+
|
349
|
+
puts
|
350
|
+
end
|
351
|
+
end
|
352
|
+
end
|
353
|
+
end
|