choria-mcorpc-support 2.23.2 → 2.24.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +5 -5
  2. data/lib/mcollective.rb +3 -3
  3. data/lib/mcollective/agent/choria_util.ddl +206 -107
  4. data/lib/mcollective/agent/choria_util.json +101 -1
  5. data/lib/mcollective/agent/rpcutil.json +2 -3
  6. data/lib/mcollective/agent/scout.json +107 -135
  7. data/lib/mcollective/application/facts.rb +2 -67
  8. data/lib/mcollective/application/find.rb +1 -1
  9. data/lib/mcollective/application/plugin.rb +8 -173
  10. data/lib/mcollective/client.rb +1 -1
  11. data/lib/mcollective/config.rb +145 -103
  12. data/lib/mcollective/ddl.rb +0 -1
  13. data/lib/mcollective/discovery.rb +11 -63
  14. data/lib/mcollective/discovery/broadcast.ddl +11 -0
  15. data/lib/mcollective/discovery/choria.ddl +6 -4
  16. data/lib/mcollective/discovery/delegate.ddl +13 -0
  17. data/lib/mcollective/discovery/delegate.rb +73 -0
  18. data/lib/mcollective/discovery/external.ddl +13 -0
  19. data/lib/mcollective/discovery/file.ddl +13 -0
  20. data/lib/mcollective/discovery/flatfile.ddl +7 -5
  21. data/lib/mcollective/discovery/inventory.ddl +13 -0
  22. data/lib/mcollective/discovery/mc.ddl +3 -3
  23. data/lib/mcollective/log.rb +1 -2
  24. data/lib/mcollective/optionparser.rb +1 -1
  25. data/lib/mcollective/pluginpackager/forge_packager.rb +1 -1
  26. data/lib/mcollective/rpc/client.rb +4 -2
  27. data/lib/mcollective/util.rb +2 -4
  28. data/lib/mcollective/util/choria.rb +14 -8
  29. data/lib/mcollective/util/tasks_support.rb +16 -2
  30. metadata +9 -29
  31. data/lib/mcollective/data.rb +0 -96
  32. data/lib/mcollective/data/agent_data.ddl +0 -22
  33. data/lib/mcollective/data/agent_data.rb +0 -17
  34. data/lib/mcollective/data/base.rb +0 -68
  35. data/lib/mcollective/data/bolt_task_data.ddl +0 -90
  36. data/lib/mcollective/data/bolt_task_data.rb +0 -32
  37. data/lib/mcollective/data/collective_data.ddl +0 -20
  38. data/lib/mcollective/data/collective_data.rb +0 -9
  39. data/lib/mcollective/data/fact_data.ddl +0 -28
  40. data/lib/mcollective/data/fact_data.rb +0 -55
  41. data/lib/mcollective/data/fstat_data.ddl +0 -89
  42. data/lib/mcollective/data/fstat_data.rb +0 -54
  43. data/lib/mcollective/data/result.rb +0 -50
  44. data/lib/mcollective/ddl/dataddl.rb +0 -56
  45. data/lib/mcollective/discovery/choria.rb +0 -223
  46. data/lib/mcollective/discovery/flatfile.rb +0 -47
  47. data/lib/mcollective/discovery/stdin.ddl +0 -11
  48. data/lib/mcollective/discovery/stdin.rb +0 -67
  49. data/lib/mcollective/generators.rb +0 -7
  50. data/lib/mcollective/generators/agent_generator.rb +0 -50
  51. data/lib/mcollective/generators/base.rb +0 -45
  52. data/lib/mcollective/generators/data_generator.rb +0 -50
  53. data/lib/mcollective/generators/templates/action_snippet.erb +0 -13
  54. data/lib/mcollective/generators/templates/data_input_snippet.erb +0 -7
  55. data/lib/mcollective/generators/templates/ddl.erb +0 -8
  56. data/lib/mcollective/generators/templates/plugin.erb +0 -7
@@ -150,7 +150,7 @@ module MCollective
150
150
  # of the discovery being cancelled soon as it reached the
151
151
  # requested limit of hosts
152
152
  def discover(filter, timeout, limit=0)
153
- @discoverer.discover(filter.merge("collective" => collective), timeout, limit)
153
+ @discoverer.discover(filter.merge("collective" => collective), timeout, limit, self)
154
154
  end
155
155
 
156
156
  # Send a request, performs the passed block for each response
@@ -17,123 +17,132 @@ module MCollective
17
17
  @configured = false
18
18
  end
19
19
 
20
- def loadconfig(configfile) # rubocop:disable Metrics/MethodLength
21
- set_config_defaults(configfile)
20
+ def parse_config_file(configfile, libdirs) # rubocop:disable Metrics/MethodLength
21
+ File.readlines(configfile).each do |line|
22
+ # strip blank spaces, tabs etc off the end of all lines
23
+ line.gsub!(/\s*$/, "")
22
24
 
23
- if File.exist?(configfile)
24
- libdirs = []
25
- File.readlines(configfile).each do |line|
26
- # strip blank spaces, tabs etc off the end of all lines
27
- line.gsub!(/\s*$/, "")
25
+ next if line =~ /^#|^$/
26
+ next unless line =~ /(.+?)\s*=\s*(.+)/
28
27
 
29
- next if line =~ /^#|^$/
30
- next unless line =~ /(.+?)\s*=\s*(.+)/
28
+ key = $1.strip
29
+ val = $2
31
30
 
32
- key = $1.strip
33
- val = $2
31
+ begin
32
+ case key
33
+ when "collectives"
34
+ @collectives = val.split(",").map(&:strip)
35
+ when "main_collective"
36
+ @main_collective = val
37
+ when "logfile"
38
+ @logfile = val
39
+ when "keeplogs"
40
+ @keeplogs = Integer(val)
41
+ when "max_log_size"
42
+ @max_log_size = Integer(val)
43
+ when "loglevel"
44
+ @loglevel = val
45
+ when "logfacility"
46
+ @logfacility = val
47
+ when "libdir"
48
+ paths = val.split(File::PATH_SEPARATOR)
49
+ paths.each do |path|
50
+ raise("libdir paths should be absolute paths but '%s' is relative" % path) unless Util.absolute_path?(path)
34
51
 
35
- begin
36
- case key
37
- when "collectives"
38
- @collectives = val.split(",").map(&:strip)
39
- when "main_collective"
40
- @main_collective = val
41
- when "logfile"
42
- @logfile = val
43
- when "keeplogs"
44
- @keeplogs = Integer(val)
45
- when "max_log_size"
46
- @max_log_size = Integer(val)
47
- when "loglevel"
48
- @loglevel = val
49
- when "logfacility"
50
- @logfacility = val
51
- when "libdir"
52
- paths = val.split(File::PATH_SEPARATOR)
53
- paths.each do |path|
54
- raise("libdir paths should be absolute paths but '%s' is relative" % path) unless Util.absolute_path?(path)
55
-
56
- libdirs << path
57
- end
58
- when "identity"
59
- @identity = val
60
- when "direct_addressing"
61
- @direct_addressing = Util.str_to_bool(val)
62
- when "direct_addressing_threshold"
63
- @direct_addressing_threshold = Integer(val)
64
- when "color"
65
- @color = Util.str_to_bool(val)
66
- when "daemonize"
67
- @daemonize = Util.str_to_bool(val)
68
- when "securityprovider"
69
- @securityprovider = val.capitalize
70
- when "factsource"
71
- @factsource = val.capitalize
72
- when "connector"
73
- @connector = val.capitalize
74
- when "classesfile"
75
- @classesfile = val
76
- when /^plugin.(.+)$/
77
- @pluginconf[$1] = val
78
- when "discovery_timeout"
79
- @discovery_timeout = Integer(val)
80
- when "publish_timeout"
81
- @publish_timeout = Integer(val)
82
- when "connection_timeout"
83
- @connection_timeout = Integer(val)
84
- when "rpcaudit"
85
- @rpcaudit = Util.str_to_bool(val)
86
- when "rpcauditprovider"
87
- @rpcauditprovider = val.capitalize
88
- when "rpcauthorization"
89
- @rpcauthorization = Util.str_to_bool(val)
90
- when "rpcauthprovider"
91
- @rpcauthprovider = val.capitalize
92
- when "rpclimitmethod"
93
- @rpclimitmethod = val.to_sym
94
- when "logger_type"
95
- @logger_type = val
96
- when "fact_cache_time"
97
- @fact_cache_time = Integer(val)
98
- when "ssl_cipher"
99
- @ssl_cipher = val
100
- when "threaded"
101
- @threaded = Util.str_to_bool(val)
102
- when "ttl"
103
- @ttl = Integer(val)
104
- when "default_discovery_options"
105
- @default_discovery_options << val
106
- when "default_discovery_method"
107
- @default_discovery_method = val
108
- when "soft_shutdown"
109
- @soft_shutdown = Util.str_to_bool(val)
110
- when "soft_shutdown_timeout"
111
- @soft_shutdown_timeout = Integer(val)
112
- when "activate_agents"
113
- @activate_agents = Util.str_to_bool(val)
114
- when "default_batch_size"
115
- @default_batch_size = Integer(val)
116
- when "default_batch_sleep_time"
117
- @default_batch_sleep_time = Float(val)
118
- else
119
- # server config might now be choria config which will divirge from mcollective
120
- # in time, so we only raise this error when it looks like we aren't loading
121
- # a server config else we try our best to load as much as we can
122
- raise("Unknown config parameter '#{key}'") unless configfile =~ /server/
52
+ libdirs << path
123
53
  end
124
- rescue ArgumentError
125
- raise("Could not parse value for configuration option '%s' with value '%s'" % [key, val])
54
+ when "identity"
55
+ @identity = val
56
+ when "direct_addressing"
57
+ @direct_addressing = Util.str_to_bool(val)
58
+ when "direct_addressing_threshold"
59
+ @direct_addressing_threshold = Integer(val)
60
+ when "color"
61
+ @color = Util.str_to_bool(val)
62
+ when "daemonize"
63
+ @daemonize = Util.str_to_bool(val)
64
+ when "securityprovider"
65
+ @securityprovider = val.capitalize
66
+ when "factsource"
67
+ @factsource = val.capitalize
68
+ when "connector"
69
+ @connector = val.capitalize
70
+ when "classesfile"
71
+ @classesfile = val
72
+ when /^plugin.(.+)$/
73
+ @pluginconf[$1] = val
74
+ when "discovery_timeout"
75
+ @discovery_timeout = Integer(val)
76
+ when "publish_timeout"
77
+ @publish_timeout = Integer(val)
78
+ when "connection_timeout"
79
+ @connection_timeout = Integer(val)
80
+ when "rpcaudit"
81
+ @rpcaudit = Util.str_to_bool(val)
82
+ when "rpcauditprovider"
83
+ @rpcauditprovider = val.capitalize
84
+ when "rpcauthorization"
85
+ @rpcauthorization = Util.str_to_bool(val)
86
+ when "rpcauthprovider"
87
+ @rpcauthprovider = val.capitalize
88
+ when "rpclimitmethod"
89
+ @rpclimitmethod = val.to_sym
90
+ when "logger_type"
91
+ @logger_type = val
92
+ when "fact_cache_time"
93
+ @fact_cache_time = Integer(val)
94
+ when "ssl_cipher"
95
+ @ssl_cipher = val
96
+ when "threaded"
97
+ @threaded = Util.str_to_bool(val)
98
+ when "ttl"
99
+ @ttl = Integer(val)
100
+ when "default_discovery_options"
101
+ @default_discovery_options << val
102
+ when "default_discovery_method"
103
+ @default_discovery_method = val
104
+ when "soft_shutdown"
105
+ @soft_shutdown = Util.str_to_bool(val)
106
+ when "soft_shutdown_timeout"
107
+ @soft_shutdown_timeout = Integer(val)
108
+ when "activate_agents"
109
+ @activate_agents = Util.str_to_bool(val)
110
+ when "default_batch_size"
111
+ @default_batch_size = Integer(val)
112
+ when "default_batch_sleep_time"
113
+ @default_batch_sleep_time = Float(val)
114
+ else
115
+ # server config might now be choria config which will divirge from mcollective
116
+ # in time, so we only raise this error when it looks like we aren't loading
117
+ # a server config else we try our best to load as much as we can
118
+ raise("Unknown config parameter '#{key}'") unless configfile =~ /server/
126
119
  end
120
+ rescue ArgumentError
121
+ raise("Could not parse value for configuration option '%s' with value '%s'" % [key, val])
127
122
  end
123
+ end
124
+ end
125
+
126
+ def loadconfig(configfile)
127
+ set_config_defaults(configfile)
128
+
129
+ if File.exist?(configfile)
130
+ libdirs = []
131
+
132
+ parse_config_file(configfile, libdirs)
128
133
 
129
134
  read_plugin_config_dir("#{@configdir}/plugin.d")
130
135
 
131
- raise 'Identities can only match /\w\.\-/' unless @identity =~ /^[\w.\-]+$/
136
+ parse_project_config(libdirs)
137
+
138
+ raise('Identities can only match /\w\.\-/') unless @identity =~ /^[\w.\-]+$/
139
+
140
+ check_deprecations
132
141
 
133
142
  @configured = true
134
143
 
135
144
  libdirs.each do |dir|
136
- Log.debug("Cannot find libdir: #{dir}") unless File.directory?(dir)
145
+ Log.debug("Cannot find libdir: %s" % dir) unless File.directory?(dir)
137
146
 
138
147
  # remove the old one if it exists, we're moving it to the front
139
148
  $LOAD_PATH.reject! { |elem| elem == dir }
@@ -154,6 +163,31 @@ module MCollective
154
163
  end
155
164
  end
156
165
 
166
+ def project_root
167
+ Dir.pwd
168
+ end
169
+
170
+ def parse_project_config(libdirs)
171
+ project_config_files(project_root).each do |pfile|
172
+ parse_config_file(pfile, libdirs)
173
+ end
174
+ end
175
+
176
+ def project_config_files(path)
177
+ paths = []
178
+
179
+ path = File.expand_path(path)
180
+ parent = File.expand_path("..", path)
181
+
182
+ paths << project_config_files(parent) if parent != path
183
+
184
+ config = File.join(path, "choria.conf")
185
+
186
+ paths << config if File.exist?(config)
187
+
188
+ paths.flatten
189
+ end
190
+
157
191
  def set_config_defaults(configfile) # rubocop:disable Naming/AccessorMethodName
158
192
  @subscribe = []
159
193
  @pluginconf = {}
@@ -222,5 +256,13 @@ module MCollective
222
256
  end
223
257
  end
224
258
  end
259
+
260
+ def check_deprecations
261
+ if @pluginconf["choria.use_srv_records"]
262
+ Log.warn("Configuration set 'choria.use_srv_records' which is deprecated in favor of 'choria.use_srv'.")
263
+ @pluginconf["choria.use_srv"] = @pluginconf["choria.use_srv_records"]
264
+ @pluginconf.delete("choria.use_srv_records")
265
+ end
266
+ end
225
267
  end
226
268
  end
@@ -50,7 +50,6 @@ module MCollective
50
50
  module DDL
51
51
  require "mcollective/ddl/base"
52
52
  require "mcollective/ddl/agentddl"
53
- require "mcollective/ddl/dataddl"
54
53
  require "mcollective/ddl/discoveryddl"
55
54
 
56
55
  # There used to be only one big nasty DDL class with a bunch of mashed
@@ -1,30 +1,26 @@
1
1
  module MCollective
2
2
  class Discovery
3
3
  def initialize(client)
4
- @known_methods = find_known_methods
5
- @default_method = Config.instance.default_discovery_method
6
4
  @client = client
7
5
  end
8
6
 
9
7
  def find_known_methods
10
- PluginManager.find("discovery")
8
+ ["broadcast", "mc", "inventory", "flatfile", "external", "choria", "file"]
11
9
  end
12
10
 
13
11
  def has_method?(method)
14
- @known_methods.include?(method)
12
+ find_known_methods.include?(method)
15
13
  end
16
14
 
17
15
  def force_direct_mode?
18
- discovery_method != "mc"
16
+ true
19
17
  end
20
18
 
21
19
  def discovery_method
22
- method = "mc"
23
-
24
20
  if @client.options[:discovery_method]
25
21
  method = @client.options[:discovery_method]
26
22
  else
27
- method = @default_method
23
+ method = Config.instance.default_discovery_method
28
24
  end
29
25
 
30
26
  raise "Unknown discovery method %s" % method unless has_method?(method)
@@ -35,11 +31,11 @@ module MCollective
35
31
  end
36
32
 
37
33
  def discovery_class
38
- method = discovery_method.capitalize
39
-
40
- PluginManager.loadclass("MCollective::Discovery::#{method}") unless self.class.const_defined?(method)
34
+ Delegate
35
+ end
41
36
 
42
- self.class.const_get(method)
37
+ def check_capabilities(filter)
38
+ true
43
39
  end
44
40
 
45
41
  def ddl
@@ -52,71 +48,23 @@ module MCollective
52
48
  @ddl
53
49
  end
54
50
 
55
- # Agent filters are always present no matter what, so we cant raise an error if the capabilities
56
- # suggest the discovery method cant do agents we just have to rely on the discovery plugin to not
57
- # do stupid things in the presense of a agent filter
58
- def check_capabilities(filter)
59
- capabilities = ddl.discovery_interface[:capabilities]
60
-
61
- raise "Cannot use class filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:classes) && !filter["cf_class"].empty?
62
-
63
- raise "Cannot use fact filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:facts) && !filter["fact"].empty?
64
-
65
- raise "Cannot use identity filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:identity) && !filter["identity"].empty?
66
-
67
- raise "Cannot use compound filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:compound) && !filter["compound"].empty?
68
- end
69
-
70
51
  # checks if compound filters are used and then forces the 'mc' discovery plugin
71
52
  def force_discovery_method_by_filter(filter)
72
- if discovery_method != "mc" && !filter["compound"].empty?
73
- Log.info "Switching to mc discovery method because compound filters are used"
74
- @client.options[:discovery_method] = "mc"
75
-
76
- return true
77
- end
78
-
79
53
  false
80
54
  end
81
55
 
82
- # if a compound filter is specified and it has any function
83
- # then we read the DDL for each of those plugins and sum up
84
- # the timeout declared in the DDL
85
56
  def timeout_for_compound_filter(compound_filter)
86
- return 0 if compound_filter.nil? || compound_filter.empty?
87
-
88
- # disabled while bringing in new compound filters
89
- # compound_filter.each do |filter|
90
- # filter.each do |statement|
91
- # next unless statement["fstatement"]
92
- #
93
- # pluginname = Data.pluginname(statement["fstatement"]["name"])
94
- # ddl = DDL.new(pluginname, :data)
95
- # timeout += ddl.meta[:timeout]
96
- # end
97
- # end
98
-
99
57
  0
100
58
  end
101
59
 
102
60
  def discovery_timeout(timeout, filter)
103
- timeout ||= ddl.meta[:timeout]
104
-
105
- if filter["compound"] && filter["compound"].empty?
106
- timeout
107
- else
108
- timeout + timeout_for_compound_filter(filter["compound"])
109
- end
61
+ timeout || ddl.meta[:timeout]
110
62
  end
111
63
 
112
- def discover(filter, timeout, limit)
64
+ def discover(filter, timeout, limit, client)
113
65
  raise "Limit has to be an integer" unless limit.is_a?(Integer)
114
66
 
115
- force_discovery_method_by_filter(filter)
116
-
117
- check_capabilities(filter)
118
-
119
- discovered = discovery_class.discover(filter, discovery_timeout(timeout, filter), limit, @client)
67
+ discovered = discovery_class.discover(filter, discovery_timeout(timeout, filter), limit, client)
120
68
 
121
69
  if limit > 0
122
70
  discovered[0, limit]
@@ -0,0 +1,11 @@
1
+ metadata :name => "broadcast",
2
+ :description => "Choria Broadcast based discovery",
3
+ :author => "R.I.Pienaar <rip@devco.net>",
4
+ :license => "Apache-2.0",
5
+ :version => "0.1",
6
+ :url => "https://choria.io",
7
+ :timeout => 2
8
+
9
+ discovery do
10
+ capabilities [:classes, :facts, :identity, :agents, :compound]
11
+ end
@@ -1,11 +1,13 @@
1
1
  metadata :name => "choria",
2
- :description => "PuppetDB based discovery for the Choria plugin suite",
2
+ :description => "PuppetDB based discovery method",
3
3
  :author => "R.I.Pienaar <rip@devco.net>",
4
4
  :license => "Apache-2.0",
5
- :version => "0.19.0",
6
- :url => "https://github.com/choria-io",
7
- :timeout => 0
5
+ :version => "0.1",
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
8
 
9
9
  discovery do
10
10
  capabilities [:classes, :facts, :identity, :agents]
11
11
  end
12
+
13
+