choria-mcorpc-support 2.23.0.pre → 2.24.0

data/lib/mcollective/discovery/delegate.ddl

@@ -0,0 +1,13 @@
+metadata    :name        => "delegate",
+            :description => "Choria CLI based delegated discovery",
+            :author      => "R.I.Pienaar <rip@devco.net>",
+            :license     => "Apache-2.0",
+            :version     => "0.1",
+            :url         => "https://choria.io/",
+            :timeout     => 2
+
+discovery do
+    capabilities [:classes, :facts, :identity, :agents, :compound]
+end
+
+

data/lib/mcollective/discovery/delegate.rb

@@ -0,0 +1,73 @@
+module MCollective
+  class Discovery
+    class Delegate
+      def self.binary_name
+        "choria"
+      end
+
+      def self.discover(filter, timeout, limit, client)
+        raise("Cannot find the choria binary in your path") unless Util.command_in_path?("choria")
+
+        cmd = [binary_name, "discover", "-j", "--silent"]
+
+        cmd << "-T" << filter["collective"] if filter["collective"]
+
+        filter.fetch("identity", []).each do |i|
+          cmd << "-I" << i
+        end
+
+        filter.fetch("cf_class", []).each do |c|
+          cmd << "-C" << c
+        end
+
+        filter.fetch("fact", []).each do |f|
+          cmd << "-F" << "%s%s%s" % [f[:fact], f[:operator], f[:value]]
+        end
+
+        filter.fetch("agent", []).each do |a|
+          cmd << "-A" << a
+        end
+
+        filter.fetch("compound", []).each do |c|
+          next unless c.is_a?(Array)
+
+          cmd << "-S" << c.first["expr"]
+        end
+
+        client.options.fetch(:discovery_options, []).each do |opt|
+          cmd << "--do" << opt
+        end
+
+        cmd << "--dm" << (client.options.fetch(:discovery_method, "broadcast") rescue "broadcast")
+
+        run_discover(cmd, timeout)
+      end
+
+      def self.run_discover(cmd, timeout)
+        nodes = []
+
+        Log.debug("Executing choria for discovery using: %s" % cmd.join(" "))
+
+        Open3.popen3(ENV, *cmd) do |stdin, stdout, stderr, wait_thr|
+          stdin.close
+
+          begin
+            Timeout.timeout(timeout + 0.5) do
+              status = wait_thr.value
+
+              raise("Choria discovery failed: %s" % stderr.read) unless status.exitstatus == 0
+            end
+          rescue Timeout::Error
+            Log.warn("Timeout waiting for Choria to perform discovery")
+            Process.kill("KILL", wait_thr[:pid])
+            raise("Choria failed to complete discovery within %d timeout" % timeout)
+          end
+
+          nodes.concat(JSON.parse(stdout.read))
+        end
+
+        nodes
+      end
+    end
+  end
+end

data/lib/mcollective/discovery/external.ddl

@@ -0,0 +1,13 @@
+metadata    :name        => "external",
+            :description => "External executable based discovery",
+            :author      => "R.I.Pienaar <rip@devco.net>",
+            :license     => "Apache-2.0",
+            :version     => "0.1",
+            :url         => "https://choria.io/",
+            :timeout     => 2
+
+discovery do
+    capabilities [:classes, :facts, :identity, :agents, :compound]
+end
+
+

data/lib/mcollective/discovery/file.ddl

@@ -0,0 +1,13 @@
+metadata    :name        => "file",
+            :description => "Discovers from JSON, YAML and Text files",
+            :author      => "R.I.Pienaar <rip@devco.net>",
+            :license     => "Apache-2.0",
+            :version     => "0.1",
+            :url         => "https://choria.io/",
+            :timeout     => 2
+
+discovery do
+    capabilities [:identity]
+end
+
+

data/lib/mcollective/discovery/flatfile.ddl

@@ -1,11 +1,13 @@
 metadata    :name        => "flatfile",
-            :description => "Flatfile based discovery for node identities",
+            :description => "Discovers from JSON, YAML and Text files",
             :author      => "R.I.Pienaar <rip@devco.net>",
-            :license     => "ASL 2.0",
+            :license     => "Apache-2.0",
             :version     => "0.1",
-            :url         => "https://docs.puppetlabs.com/mcollective/",
-            :timeout     => 0
+            :url         => "https://choria.io/",
+            :timeout     => 2
 
 discovery do
-    capabilities :identity
+    capabilities [:identity]
 end
+
+

data/lib/mcollective/discovery/inventory.ddl

@@ -0,0 +1,13 @@
+metadata    :name        => "delegate",
+            :description => "Choria CLI based delegated discovery",
+            :author      => "R.I.Pienaar <rip@devco.net>",
+            :license     => "Apache 2.0",
+            :version     => "0.1",
+            :url         => "https://choria.io/",
+            :timeout     => 2
+
+discovery do
+    capabilities [:classes, :facts, :identity, :agents, :compound]
+end
+
+

data/lib/mcollective/discovery/mc.ddl

@@ -1,9 +1,9 @@
 metadata    :name        => "mc",
-            :description => "MCollective Broadcast based discovery",
+            :description => "Choria Broadcast based discovery",
             :author      => "R.I.Pienaar <rip@devco.net>",
-            :license     => "ASL 2.0",
+            :license     => "Apache-2.0",
             :version     => "0.1",
-            :url         => "https://docs.puppetlabs.com/mcollective/",
+            :url         => "https://choria.io",
             :timeout     => 2
 
 discovery do

data/lib/mcollective/message.rb

@@ -148,8 +148,6 @@ module MCollective
         @requestid = request.payload[:requestid]
         @payload = PluginManager["security_plugin"].encodereply(agent, payload, requestid, request.payload[:callerid])
       when :request, :direct_request
-        validate_compound_filter(@filter["compound"]) unless @filter["compound"].empty?
-
         @requestid ||= create_reqid
         @payload = PluginManager["security_plugin"].encoderequest(Config.instance.identity, payload, requestid, filter, agent, collective, ttl)
       else
@@ -157,28 +155,6 @@ module MCollective
       end
     end
 
-    def validate_compound_filter(compound_filter)
-      compound_filter.each do |filter|
-        filter.each do |statement|
-          next unless statement["fstatement"]
-
-          functionname = statement["fstatement"]["name"]
-          pluginname = Data.pluginname(functionname)
-          value = statement["fstatement"]["value"]
-
-          ddl = DDL.new(pluginname, :data)
-
-          # parses numbers and booleans entered as strings into proper
-          # types of data so that DDL validation will pass
-          statement["fstatement"]["params"] = Data.ddl_transform_input(ddl, statement["fstatement"]["params"])
-
-          Data.ddl_validate(ddl, statement["fstatement"]["params"])
-
-          raise(DDLValidationError, "Data plugin '%s()' does not return a '%s' value" % [functionname, value]) unless value && Data.ddl_has_output?(ddl, value)
-        end
-      end
-    end
-
     def decode!
       raise "Cannot decode message type #{type}" unless [:request, :reply].include?(type)
 

data/lib/mcollective/optionparser.rb

@@ -86,7 +86,7 @@ module MCollective
       end
 
       @parser.on("-S", "--select FILTER", "Compound filter combining facts and classes") do |f|
-        @options[:filter]["compound"] << Matcher.create_compound_callstack(f)
+        @options[:filter]["compound"] << [{"expr" => f}]
       end
 
       @parser.on("-F", "--wf", "--with-fact fact=val", "Match hosts with a certain fact") do |f|
@@ -168,7 +168,7 @@ module MCollective
         raise "Cannot read the discovery file #{v}" unless File.readable?(v)
 
         @options[:discovery_method] = "flatfile"
-        @options[:discovery_options] << v
+        @options[:discovery_options] << "file=%s" % v
       end
 
       @parser.on("--publish_timeout TIMEOUT", Integer, "Timeout for publishing requests to remote agents.") do |pt|

data/lib/mcollective/pluginpackager/forge_packager.rb

@@ -191,7 +191,7 @@ module MCollective
       end
 
       def render_template(infile, outfile)
-        erb = ERB.new(File.read(infile), 0, "%")
+        erb = ERB.new(File.read(infile), 0, "-")
         File.open(outfile, "w") do |f|
           f.puts erb.result(binding)
         end

data/lib/mcollective/rpc/client.rb

@@ -442,7 +442,7 @@ module MCollective
 
       # Set a compound filter
       def compound_filter(filter)
-        @filter["compound"] = @filter["compound"] | [Matcher.create_compound_callstack(filter)]
+        @filter["compound"] = @filter["compound"] | [[{"expr" => filter}]]
         reset
       end
 
@@ -533,6 +533,8 @@ module MCollective
 
         # All else fails we do it the hard way using a traditional broadcast
         unless @discovered_agents
+          raise("Invalid discovery method %s" % discovery_method) unless @client.discoverer.find_known_methods.include?(discovery_method)
+
           @stats.time_discovery :start
 
           @client.options = options
@@ -548,9 +550,9 @@ module MCollective
             actual_timeout = @client.discoverer.discovery_timeout(discovery_timeout, options[:filter])
 
             if actual_timeout > 0
-              @stderr.print("Discovering hosts using the %s method for %d second(s) .... " % [@client.discoverer.discovery_method, actual_timeout])
+              @stderr.print("Discovering hosts using the %s method for %d second(s) .... " % [discovery_method, actual_timeout])
             else
-              @stderr.print("Discovering hosts using the %s method .... " % [@client.discoverer.discovery_method])
+              @stderr.print("Discovering hosts using the %s method .... " % discovery_method)
             end
           end
 
@@ -822,7 +824,7 @@ module MCollective
             @stdout.print twirl.twirl(respcount, discovered.size)
           end
 
-          if batch_size =~ /^(\d+)%$/
+          if batch_size.is_a?(String) && batch_size =~ /^(\d+)%$/
             # determine batch_size as a percentage of the discovered array's size
             batch_size = (discovered.size / 100.0 * Integer($1)).ceil
           else

data/lib/mcollective/security/base.rb

@@ -72,43 +72,7 @@ module MCollective
             end
 
           when "compound"
-            filter[key].each do |compound|
-              result = false
-              truth_values = []
-
-              begin
-                compound.each do |expression|
-                  case expression.keys.first
-                  when "statement"
-                    truth_values << Matcher.eval_compound_statement(expression).to_s
-                  when "fstatement"
-                    truth_values << Matcher.eval_compound_fstatement(expression.values.first)
-                  when "and"
-                    truth_values << "&&"
-                  when "or"
-                    truth_values << "||"
-                  when "("
-                    truth_values << "("
-                  when ")"
-                    truth_values << ")"
-                  when "not"
-                    truth_values << "!"
-                  end
-                end
-
-                result = eval(truth_values.join(" ")) # rubocop:disable Security/Eval
-              rescue DDLValidationError
-                result = false
-              end
-
-              if result
-                Log.debug("Passing based on class and fact composition")
-                passed += 1
-              else
-                Log.debug("Failing based on class and fact composition")
-                failed += 1
-              end
-            end
+            # removed while rebuilding compound filters, this whole method is probably unused now
 
           when "agent"
             filter[key].each do |f|

data/lib/mcollective/util.rb

@@ -156,19 +156,23 @@ module MCollective
       File.join(Dir::COMMON_APPDATA, "ChoriaIO", "choria")
     end
 
-    def self.mcollective_config_paths_for_user
+    def self.config_paths_for_user
       config_paths = []
 
-      begin
-        # File.expand_path will raise if HOME isn't set, catch it
-        user_path = File.expand_path("~/.mcollective")
-        config_paths << user_path
-      rescue Exception # rubocop:disable Lint/RescueException, Lint/SuppressedException
+      ["~/.choriarc", "~/.mcollective"].each do |f|
+        begin
+          # File.expand_path will raise if HOME isn't set, catch it
+          config_paths << File.expand_path(f)
+        rescue ArgumentError # rubocop:disable Lint/SuppressedException
+        end
       end
 
       if windows?
+        config_paths << File.join(choria_windows_prefix, "etc", "client.conf")
         config_paths << File.join(windows_prefix, "etc", "client.cfg")
       else
+        config_paths << "/etc/choria/client.conf"
+        config_paths << "/usr/local/etc/choria/client.conf"
         config_paths << "/etc/puppetlabs/mcollective/client.cfg"
         config_paths << "/etc/mcollective/client.cfg"
         config_paths << "/usr/local/etc/mcollective/client.cfg"
@@ -177,42 +181,30 @@ module MCollective
       config_paths
     end
 
-    def self.choria_config_paths_for_user
-      config_paths = []
-
-      begin
-        # File.expand_path will raise if HOME isn't set, catch it
-        user_path = File.expand_path("~/.choriarc")
-        config_paths << user_path
-      rescue Exception # rubocop:disable Lint/RescueException, Lint/SuppressedException
-      end
-
-      if windows?
-        config_paths << File.join(choria_windows_prefix, "etc", "client.conf")
-      else
-        config_paths << "/etc/choria/client.conf"
-        config_paths << "/usr/local/etc/choria/client.conf"
-      end
-
-      config_paths
-    end
-
     # Picks the default user config file, priorities are first Choria ones then old MCollective ones
     #
     # In roughly this order, first to exist is used:
     #
     # - ~/.choriarc
-    # - APPData/ChoriaIO/choria/etc/client.conf on windows
-    # - /etc/choria/client.conf then
-    # - /usr/local/etc/choria/client.conf on unix
     # - ~/.mcollective
-    # - APPData/PuppetLabs/mcollective/etc/client.cfg on windows
+    #
+    # On Unix:
+    #
+    # - /etc/choria/client.conf
+    # - /usr/local/etc/choria/client.conf
     # - /etc/puppetlabs/mcollective/client.cfg
     # - /etc/mcollective/client.cfg
     # - /usr/local/etc/mcollective/client.cfg
+    #
+    # On Windows:
+    #
+    # - APPData/ChoriaIO/choria/etc/client.conf on windows
+    # - APPData/PuppetLabs/mcollective/etc/client.cfg on windows
     def self.config_file_for_user
-      config_paths = choria_config_paths_for_user + mcollective_config_paths_for_user
+      config_paths = config_paths_for_user
+
       found = config_paths.find_index { |file| File.readable?(file) } || 0
+
       config_paths[found]
     end
 

data/lib/mcollective/util/choria.rb

@@ -86,14 +86,6 @@ module MCollective
         end
       end
 
-      # Which port to provide stats over HTTP on
-      #
-      # @return [Integer,nil]
-      # @raise [StandardError] when not numeric
-      def stats_port
-        Integer(get_option("choria.stats_port", "")) if has_option?("choria.stats_port")
-      end
-
       # Determines if there are any federations configured
       #
       # @return [Boolean]
@@ -888,15 +880,6 @@ module MCollective
         File.exist?(csr_path)
       end
 
-      # The formatted string representation of the CSR fingerprint
-      #
-      # @return [String]
-      def csr_fingerprint
-        require "puppet"
-        csr = OpenSSL::X509::Request.new(File.read(csr_path))
-        Puppet::SSL::Digest.new(nil, csr.to_der)
-      end
-
       # Searches the PATH for an executable command
       #
       # @param command [String] a command to search for
@@ -927,146 +910,6 @@ module MCollective
         which("facter")
       end
 
-      # Creates any missing SSL directories
-      #
-      # This prepares a Puppet like SSL tree in case Puppet
-      # has not been initialized yet
-      #
-      # @return [void]
-      def make_ssl_dirs
-        return if file_security?
-
-        FileUtils.mkdir_p(ssl_dir, :mode => 0o0771)
-
-        ["certificate_requests", "certs", "public_keys"].each do |dir|
-          FileUtils.mkdir_p(File.join(ssl_dir, dir), :mode => 0o0755)
-        end
-
-        ["private_keys", "private"].each do |dir|
-          FileUtils.mkdir_p(File.join(ssl_dir, dir), :mode => 0o0750)
-        end
-      end
-
-      # Creates a RSA key of a certain strenth
-      #
-      # @return [OpenSSL::PKey::RSA]
-      def create_rsa_key(bits)
-        OpenSSL::PKey::RSA.new(bits)
-      end
-
-      # Writes a new 4096 bit key in the puppet default locatioj
-      #
-      # @return [OpenSSL::PKey::RSA]
-      # @raise [StandardError] when the key already exist
-      def write_key
-        raise("Refusing to overwrite existing key in %s" % client_private_key) if has_client_private_key?
-
-        key = create_rsa_key(4096)
-        File.open(client_private_key, "w", 0o0640) {|f| f.write(key.to_pem)}
-
-        key
-      end
-
-      # Creates a basic CSR
-      #
-      # @return [OpenSSL::X509::Request] signed CSR
-      def create_csr(comonname, orgunit, key)
-        csr = OpenSSL::X509::Request.new
-        csr.version = 0
-        csr.public_key = key.public_key
-        csr.subject = OpenSSL::X509::Name.new(
-          [
-            ["CN", comonname, OpenSSL::ASN1::UTF8STRING],
-            ["OU", orgunit, OpenSSL::ASN1::UTF8STRING]
-          ]
-        )
-        csr.sign(key, OpenSSL::Digest.new("SHA1"))
-
-        csr
-      end
-
-      # Creates a new CSR signed by the given key
-      #
-      # @param key [OpenSSL::PKey::RSA]
-      # @return [String] PEM encoded CSR
-      def write_csr(key)
-        raise("Refusing to overwrite existing CSR in %s" % csr_path) if has_csr?
-
-        csr = create_csr(certname, "mcollective", key)
-
-        File.open(csr_path, "w", 0o0644) {|f| f.write(csr.to_pem)}
-
-        csr.to_pem
-      end
-
-      # Fetch and save the CA from Puppet
-      #
-      # @return [Boolean]
-      def fetch_ca
-        return true if has_ca?
-
-        server = puppetca_server
-
-        req = http_get("/puppet-ca/v1/certificate/ca?environment=production", "Accept" => "text/plain")
-        resp, _ = https(server).request(req)
-
-        if resp.code == "200"
-          File.open(ca_path, "w", 0o0644) {|f| f.write(resp.body)}
-        else
-          raise(UserError, "Failed to fetch CA from %s:%s: %s: %s" % [server[:target], server[:port], resp.code, resp.message])
-        end
-
-        has_ca?
-      end
-
-      # Requests a certificate from the Puppet CA
-      #
-      # This will attempt to create a new key, write a CSR and
-      # then sends it to the CA for signing
-      #
-      # @return [Boolean]
-      # @raise [UserError] when requesting the cert fails
-      def request_cert
-        key = write_key
-        csr = write_csr(key)
-
-        server = puppetca_server
-
-        req = Net::HTTP::Put.new("/puppet-ca/v1/certificate_request/%s?environment=production" % certname, "Content-Type" => "text/plain")
-        req.body = csr
-        resp, _ = https(server).request(req)
-
-        if resp.code == "200"
-          true
-        else
-          raise(UserError, "Failed to request certificate from %s:%s: %s: %s: %s" % [server[:target], server[:port], resp.code, resp.message, resp.body])
-        end
-      end
-
-      # Attempts to fetch a cert from the CA
-      #
-      # @return [Boolean]
-      def attempt_fetch_cert
-        return true if has_client_public_cert?
-
-        req = http_get("/puppet-ca/v1/certificate/%s?environment=production" % certname, "Accept" => "text/plain")
-        resp, _ = https(puppetca_server).request(req)
-
-        if resp.code == "200"
-          File.open(client_public_cert, "w", 0o0644) {|f| f.write(resp.body)}
-          true
-        else
-          false
-        end
-      end
-
-      # Determines if a CSR has been sent but not yet retrieved
-      #
-      # @return [Boolean]
-      def waiting_for_cert?
-        !has_client_public_cert? && has_client_private_key?
-      end
-
       # Gets a config option
       #
       # @param opt [String] config option to look up