choria-mcorpc-support 2.23.3 → 2.24.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +5 -5
  2. data/lib/mcollective.rb +3 -3
  3. data/lib/mcollective/agent/choria_provision.ddl +214 -0
  4. data/lib/mcollective/agent/choria_provision.json +272 -0
  5. data/lib/mcollective/agent/choria_util.ddl +226 -107
  6. data/lib/mcollective/agent/choria_util.json +125 -25
  7. data/lib/mcollective/agent/rpcutil.ddl +242 -183
  8. data/lib/mcollective/agent/rpcutil.json +53 -57
  9. data/lib/mcollective/agent/scout.ddl +1 -1
  10. data/lib/mcollective/agent/scout.json +108 -136
  11. data/lib/mcollective/application/find.rb +1 -1
  12. data/lib/mcollective/application/plugin.rb +8 -173
  13. data/lib/mcollective/client.rb +1 -1
  14. data/lib/mcollective/config.rb +145 -103
  15. data/lib/mcollective/ddl.rb +0 -1
  16. data/lib/mcollective/discovery.rb +11 -63
  17. data/lib/mcollective/discovery/broadcast.ddl +11 -0
  18. data/lib/mcollective/discovery/choria.ddl +6 -4
  19. data/lib/mcollective/discovery/delegate.ddl +13 -0
  20. data/lib/mcollective/discovery/delegate.rb +72 -0
  21. data/lib/mcollective/discovery/external.ddl +13 -0
  22. data/lib/mcollective/discovery/file.ddl +13 -0
  23. data/lib/mcollective/discovery/flatfile.ddl +7 -5
  24. data/lib/mcollective/discovery/inventory.ddl +13 -0
  25. data/lib/mcollective/discovery/mc.ddl +3 -3
  26. data/lib/mcollective/log.rb +1 -2
  27. data/lib/mcollective/optionparser.rb +1 -1
  28. data/lib/mcollective/pluginpackager/forge_packager.rb +1 -1
  29. data/lib/mcollective/rpc/client.rb +4 -2
  30. data/lib/mcollective/util.rb +2 -4
  31. data/lib/mcollective/util/choria.rb +14 -8
  32. data/lib/mcollective/util/tasks_support.rb +15 -2
  33. metadata +11 -29
  34. data/lib/mcollective/data.rb +0 -96
  35. data/lib/mcollective/data/agent_data.ddl +0 -22
  36. data/lib/mcollective/data/agent_data.rb +0 -17
  37. data/lib/mcollective/data/base.rb +0 -68
  38. data/lib/mcollective/data/bolt_task_data.ddl +0 -90
  39. data/lib/mcollective/data/bolt_task_data.rb +0 -32
  40. data/lib/mcollective/data/collective_data.ddl +0 -20
  41. data/lib/mcollective/data/collective_data.rb +0 -9
  42. data/lib/mcollective/data/fact_data.ddl +0 -28
  43. data/lib/mcollective/data/fact_data.rb +0 -55
  44. data/lib/mcollective/data/fstat_data.ddl +0 -89
  45. data/lib/mcollective/data/fstat_data.rb +0 -54
  46. data/lib/mcollective/data/result.rb +0 -50
  47. data/lib/mcollective/ddl/dataddl.rb +0 -56
  48. data/lib/mcollective/discovery/choria.rb +0 -223
  49. data/lib/mcollective/discovery/flatfile.rb +0 -47
  50. data/lib/mcollective/discovery/stdin.ddl +0 -11
  51. data/lib/mcollective/discovery/stdin.rb +0 -67
  52. data/lib/mcollective/generators.rb +0 -7
  53. data/lib/mcollective/generators/agent_generator.rb +0 -50
  54. data/lib/mcollective/generators/base.rb +0 -45
  55. data/lib/mcollective/generators/data_generator.rb +0 -50
  56. data/lib/mcollective/generators/templates/action_snippet.erb +0 -13
  57. data/lib/mcollective/generators/templates/data_input_snippet.erb +0 -7
  58. data/lib/mcollective/generators/templates/ddl.erb +0 -8
  59. data/lib/mcollective/generators/templates/plugin.erb +0 -7
@@ -50,7 +50,6 @@ module MCollective
50
50
  module DDL
51
51
  require "mcollective/ddl/base"
52
52
  require "mcollective/ddl/agentddl"
53
- require "mcollective/ddl/dataddl"
54
53
  require "mcollective/ddl/discoveryddl"
55
54
 
56
55
  # There used to be only one big nasty DDL class with a bunch of mashed
@@ -1,30 +1,26 @@
1
1
  module MCollective
2
2
  class Discovery
3
3
  def initialize(client)
4
- @known_methods = find_known_methods
5
- @default_method = Config.instance.default_discovery_method
6
4
  @client = client
7
5
  end
8
6
 
9
7
  def find_known_methods
10
- PluginManager.find("discovery")
8
+ ["broadcast", "mc", "inventory", "flatfile", "external", "choria", "file"]
11
9
  end
12
10
 
13
11
  def has_method?(method)
14
- @known_methods.include?(method)
12
+ find_known_methods.include?(method)
15
13
  end
16
14
 
17
15
  def force_direct_mode?
18
- discovery_method != "mc"
16
+ true
19
17
  end
20
18
 
21
19
  def discovery_method
22
- method = "mc"
23
-
24
20
  if @client.options[:discovery_method]
25
21
  method = @client.options[:discovery_method]
26
22
  else
27
- method = @default_method
23
+ method = Config.instance.default_discovery_method
28
24
  end
29
25
 
30
26
  raise "Unknown discovery method %s" % method unless has_method?(method)
@@ -35,11 +31,11 @@ module MCollective
35
31
  end
36
32
 
37
33
  def discovery_class
38
- method = discovery_method.capitalize
39
-
40
- PluginManager.loadclass("MCollective::Discovery::#{method}") unless self.class.const_defined?(method)
34
+ Delegate
35
+ end
41
36
 
42
- self.class.const_get(method)
37
+ def check_capabilities(filter)
38
+ true
43
39
  end
44
40
 
45
41
  def ddl
@@ -52,71 +48,23 @@ module MCollective
52
48
  @ddl
53
49
  end
54
50
 
55
- # Agent filters are always present no matter what, so we cant raise an error if the capabilities
56
- # suggest the discovery method cant do agents we just have to rely on the discovery plugin to not
57
- # do stupid things in the presense of a agent filter
58
- def check_capabilities(filter)
59
- capabilities = ddl.discovery_interface[:capabilities]
60
-
61
- raise "Cannot use class filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:classes) && !filter["cf_class"].empty?
62
-
63
- raise "Cannot use fact filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:facts) && !filter["fact"].empty?
64
-
65
- raise "Cannot use identity filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:identity) && !filter["identity"].empty?
66
-
67
- raise "Cannot use compound filters while using the '%s' discovery method" % discovery_method if !capabilities.include?(:compound) && !filter["compound"].empty?
68
- end
69
-
70
51
  # checks if compound filters are used and then forces the 'mc' discovery plugin
71
52
  def force_discovery_method_by_filter(filter)
72
- if discovery_method != "mc" && !filter["compound"].empty?
73
- Log.info "Switching to mc discovery method because compound filters are used"
74
- @client.options[:discovery_method] = "mc"
75
-
76
- return true
77
- end
78
-
79
53
  false
80
54
  end
81
55
 
82
- # if a compound filter is specified and it has any function
83
- # then we read the DDL for each of those plugins and sum up
84
- # the timeout declared in the DDL
85
56
  def timeout_for_compound_filter(compound_filter)
86
- return 0 if compound_filter.nil? || compound_filter.empty?
87
-
88
- # disabled while bringing in new compound filters
89
- # compound_filter.each do |filter|
90
- # filter.each do |statement|
91
- # next unless statement["fstatement"]
92
- #
93
- # pluginname = Data.pluginname(statement["fstatement"]["name"])
94
- # ddl = DDL.new(pluginname, :data)
95
- # timeout += ddl.meta[:timeout]
96
- # end
97
- # end
98
-
99
57
  0
100
58
  end
101
59
 
102
60
  def discovery_timeout(timeout, filter)
103
- timeout ||= ddl.meta[:timeout]
104
-
105
- if filter["compound"] && filter["compound"].empty?
106
- timeout
107
- else
108
- timeout + timeout_for_compound_filter(filter["compound"])
109
- end
61
+ timeout || ddl.meta[:timeout]
110
62
  end
111
63
 
112
- def discover(filter, timeout, limit)
64
+ def discover(filter, timeout, limit, client)
113
65
  raise "Limit has to be an integer" unless limit.is_a?(Integer)
114
66
 
115
- force_discovery_method_by_filter(filter)
116
-
117
- check_capabilities(filter)
118
-
119
- discovered = discovery_class.discover(filter, discovery_timeout(timeout, filter), limit, @client)
67
+ discovered = discovery_class.discover(filter, discovery_timeout(timeout, filter), limit, client)
120
68
 
121
69
  if limit > 0
122
70
  discovered[0, limit]
@@ -0,0 +1,11 @@
1
+ metadata :name => "broadcast",
2
+ :description => "Choria Broadcast based discovery",
3
+ :author => "R.I.Pienaar <rip@devco.net>",
4
+ :license => "Apache-2.0",
5
+ :version => "0.1",
6
+ :url => "https://choria.io",
7
+ :timeout => 2
8
+
9
+ discovery do
10
+ capabilities [:classes, :facts, :identity, :agents, :compound]
11
+ end
@@ -1,11 +1,13 @@
1
1
  metadata :name => "choria",
2
- :description => "PuppetDB based discovery for the Choria plugin suite",
2
+ :description => "PuppetDB based discovery method",
3
3
  :author => "R.I.Pienaar <rip@devco.net>",
4
4
  :license => "Apache-2.0",
5
- :version => "0.19.0",
6
- :url => "https://github.com/choria-io",
7
- :timeout => 0
5
+ :version => "0.1",
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
8
 
9
9
  discovery do
10
10
  capabilities [:classes, :facts, :identity, :agents]
11
11
  end
12
+
13
+
@@ -0,0 +1,13 @@
1
+ metadata :name => "delegate",
2
+ :description => "Choria CLI based delegated discovery",
3
+ :author => "R.I.Pienaar <rip@devco.net>",
4
+ :license => "Apache-2.0",
5
+ :version => "0.1",
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
+
9
+ discovery do
10
+ capabilities [:classes, :facts, :identity, :agents, :compound]
11
+ end
12
+
13
+
@@ -0,0 +1,72 @@
1
+ module MCollective
2
+ class Discovery
3
+ class Delegate
4
+ def self.binary_name
5
+ "choria"
6
+ end
7
+
8
+ def self.discover(filter, timeout, limit, client)
9
+ raise("Cannot find the choria binary in your path") unless Util.command_in_path?("choria")
10
+
11
+ cmd = [binary_name, "discover", "-j", "--silent"]
12
+
13
+ cmd << "-T" << filter["collective"] if filter["collective"]
14
+
15
+ filter.fetch("identity", []).each do |i|
16
+ cmd << "-I" << i
17
+ end
18
+
19
+ filter.fetch("cf_class", []).each do |c|
20
+ cmd << "-C" << c
21
+ end
22
+
23
+ filter.fetch("fact", []).each do |f|
24
+ cmd << "-F" << "%s%s%s" % [f[:fact], f[:operator], f[:value]]
25
+ end
26
+
27
+ filter.fetch("agent", []).each do |a|
28
+ cmd << "-A" << a
29
+ end
30
+
31
+ filter.fetch("compound", []).each do |c|
32
+ next unless c.is_a?(Array)
33
+
34
+ cmd << "-S" << c.first["expr"]
35
+ end
36
+
37
+ client.options.fetch(:discovery_options, []).each do |opt|
38
+ cmd << "--do" << opt
39
+ end
40
+
41
+ cmd << "--dm" << (client.options.fetch(:discovery_method, "broadcast") rescue "broadcast")
42
+
43
+ run_discover(cmd, timeout)
44
+ end
45
+
46
+ def self.run_discover(cmd, timeout)
47
+ nodes = []
48
+
49
+ Log.debug("Executing choria for discovery using: %s" % cmd.join(" "))
50
+
51
+ Open3.popen3(ENV, *cmd) do |stdin, stdout, stderr, wait_thr|
52
+ stdin.close
53
+
54
+ begin
55
+ Timeout.timeout(timeout + 0.5) do
56
+ nodes.concat(JSON.parse(stdout.read))
57
+ status = wait_thr.value
58
+
59
+ raise("Choria discovery failed: %s" % stderr.read) unless status.exitstatus == 0
60
+ end
61
+ rescue Timeout::Error
62
+ Log.warn("Timeout waiting for Choria to perform discovery")
63
+ Process.kill("KILL", wait_thr[:pid])
64
+ raise("Choria failed to complete discovery within %d timeout" % timeout)
65
+ end
66
+ end
67
+
68
+ nodes
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,13 @@
1
+ metadata :name => "external",
2
+ :description => "External executable based discovery",
3
+ :author => "R.I.Pienaar <rip@devco.net>",
4
+ :license => "Apache-2.0",
5
+ :version => "0.1",
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
+
9
+ discovery do
10
+ capabilities [:classes, :facts, :identity, :agents, :compound]
11
+ end
12
+
13
+
@@ -0,0 +1,13 @@
1
+ metadata :name => "file",
2
+ :description => "Discovers from JSON, YAML and Text files",
3
+ :author => "R.I.Pienaar <rip@devco.net>",
4
+ :license => "Apache-2.0",
5
+ :version => "0.1",
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
+
9
+ discovery do
10
+ capabilities [:identity]
11
+ end
12
+
13
+
@@ -1,11 +1,13 @@
1
1
  metadata :name => "flatfile",
2
- :description => "Flatfile based discovery for node identities",
2
+ :description => "Discovers from JSON, YAML and Text files",
3
3
  :author => "R.I.Pienaar <rip@devco.net>",
4
- :license => "ASL 2.0",
4
+ :license => "Apache-2.0",
5
5
  :version => "0.1",
6
- :url => "https://docs.puppetlabs.com/mcollective/",
7
- :timeout => 0
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
8
 
9
9
  discovery do
10
- capabilities :identity
10
+ capabilities [:identity]
11
11
  end
12
+
13
+
@@ -0,0 +1,13 @@
1
+ metadata :name => "delegate",
2
+ :description => "Choria CLI based delegated discovery",
3
+ :author => "R.I.Pienaar <rip@devco.net>",
4
+ :license => "Apache 2.0",
5
+ :version => "0.1",
6
+ :url => "https://choria.io/",
7
+ :timeout => 2
8
+
9
+ discovery do
10
+ capabilities [:classes, :facts, :identity, :agents, :compound]
11
+ end
12
+
13
+
@@ -1,9 +1,9 @@
1
1
  metadata :name => "mc",
2
- :description => "MCollective Broadcast based discovery",
2
+ :description => "Choria Broadcast based discovery",
3
3
  :author => "R.I.Pienaar <rip@devco.net>",
4
- :license => "ASL 2.0",
4
+ :license => "Apache-2.0",
5
5
  :version => "0.1",
6
- :url => "https://docs.puppetlabs.com/mcollective/",
6
+ :url => "https://choria.io",
7
7
  :timeout => 2
8
8
 
9
9
  discovery do
@@ -96,9 +96,8 @@ module MCollective
96
96
  end
97
97
 
98
98
  @logger.start
99
- rescue Exception => e # rubocop:disable Lint/RescueException
99
+ rescue Exception # rubocop:disable Lint/RescueException
100
100
  @configured = false
101
- warn "Could not start logger: #{e.class} #{e}"
102
101
  end
103
102
 
104
103
  # figures out the filename that called us
@@ -168,7 +168,7 @@ module MCollective
168
168
  raise "Cannot read the discovery file #{v}" unless File.readable?(v)
169
169
 
170
170
  @options[:discovery_method] = "flatfile"
171
- @options[:discovery_options] << v
171
+ @options[:discovery_options] << "file=%s" % v
172
172
  end
173
173
 
174
174
  @parser.on("--publish_timeout TIMEOUT", Integer, "Timeout for publishing requests to remote agents.") do |pt|
@@ -191,7 +191,7 @@ module MCollective
191
191
  end
192
192
 
193
193
  def render_template(infile, outfile)
194
- erb = ERB.new(File.read(infile), 0, "%")
194
+ erb = ERB.new(File.read(infile), 0, "-")
195
195
  File.open(outfile, "w") do |f|
196
196
  f.puts erb.result(binding)
197
197
  end
@@ -533,6 +533,8 @@ module MCollective
533
533
 
534
534
  # All else fails we do it the hard way using a traditional broadcast
535
535
  unless @discovered_agents
536
+ raise("Invalid discovery method %s" % discovery_method) unless @client.discoverer.find_known_methods.include?(discovery_method)
537
+
536
538
  @stats.time_discovery :start
537
539
 
538
540
  @client.options = options
@@ -548,9 +550,9 @@ module MCollective
548
550
  actual_timeout = @client.discoverer.discovery_timeout(discovery_timeout, options[:filter])
549
551
 
550
552
  if actual_timeout > 0
551
- @stderr.print("Discovering hosts using the %s method for %d second(s) .... " % [@client.discoverer.discovery_method, actual_timeout])
553
+ @stderr.print("Discovering hosts using the %s method for %d second(s) .... " % [discovery_method, actual_timeout])
552
554
  else
553
- @stderr.print("Discovering hosts using the %s method .... " % [@client.discoverer.discovery_method])
555
+ @stderr.print("Discovering hosts using the %s method .... " % discovery_method)
554
556
  end
555
557
  end
556
558
 
@@ -147,13 +147,11 @@ module MCollective
147
147
 
148
148
  # Returns the PuppetLabs mcollective path for windows
149
149
  def self.windows_prefix
150
- require "win32/dir"
151
- File.join(Dir::COMMON_APPDATA, "PuppetLabs", "choria")
150
+ File.join(ENV["ALLUSERSPROFILE"], "PuppetLabs", "choria")
152
151
  end
153
152
 
154
153
  def self.choria_windows_prefix
155
- require "win32/dir"
156
- File.join(Dir::COMMON_APPDATA, "ChoriaIO", "choria")
154
+ File.join(ENV["ALLUSERSPROFILE"], "choria")
157
155
  end
158
156
 
159
157
  def self.config_paths_for_user
@@ -157,19 +157,19 @@ module MCollective
157
157
 
158
158
  # Determines if SRV records should be used
159
159
  #
160
- # Setting choria.use_srv_records to anything other than t, true, yes or 1 will disable
160
+ # Setting choria.use_srv to anything other than t, true, yes or 1 will disable
161
161
  # SRV records
162
162
  #
163
163
  # @return [Boolean]
164
164
  def should_use_srv?
165
- ["t", "true", "yes", "1"].include?(get_option("choria.use_srv_records", "1").downcase)
165
+ ["t", "true", "yes", "1"].include?(get_option("choria.use_srv", "1").downcase)
166
166
  end
167
167
 
168
168
  # Query DNS for a series of records
169
169
  #
170
170
  # The given records will be passed through {#srv_records} to figure out the domain to query in.
171
171
  #
172
- # Querying of records can be bypassed by setting choria.use_srv_records to false
172
+ # Querying of records can be bypassed by setting choria.use_srv to false
173
173
  #
174
174
  # @yield [Hash] each record for modification by the caller
175
175
  # @param records [Array<String>] the records to query without their domain parts
@@ -634,21 +634,27 @@ module MCollective
634
634
 
635
635
  # The PuppetDB server to connect to
636
636
  #
637
- # Will consult _x-puppet-db._tcp.example.net then _x-puppet._tcp.example.net
638
- # then configurable using choria.puppetdb_host and choria.puppetdb_port, defaults
639
- # to puppet:8081
637
+ # Use choria.puppetdb_host if set, otherwise query
638
+ # _x-puppet-db._tcp.example.net then _x-puppet._tcp.example.net if SRV
639
+ # lookup is enabled, and fallback to puppet:8081 if nothing else worked.
640
640
  #
641
641
  # @return [Hash] with :target and :port
642
642
  def puppetdb_server
643
- d_host = get_option("choria.puppetdb_host", "puppet")
644
643
  d_port = get_option("choria.puppetdb_port", "8081")
645
644
 
645
+ answer = {
646
+ :target => get_option("choria.puppetdb_host", nil),
647
+ :port => d_port
648
+ }
649
+
650
+ return answer if answer[:target]
651
+
646
652
  answer = try_srv(["_x-puppet-db._tcp"], nil, nil)
647
653
  return answer if answer[:target]
648
654
 
649
655
  # In the case where we take _x-puppet._tcp SRV records we unfortunately have
650
656
  # to force the port else it uses the one from Puppet which will 404
651
- answer = try_srv(["_x-puppet._tcp"], d_host, d_port)
657
+ answer = try_srv(["_x-puppet._tcp"], "puppet", d_port)
652
658
  answer[:port] = d_port
653
659
 
654
660
  answer