choria-mcorpc-support 2.21.0 → 2.23.0

data/lib/mcollective/discovery/choria.ddl

@@ -0,0 +1,11 @@
+metadata    :name        => "choria",
+            :description => "PuppetDB based discovery for the Choria plugin suite",
+            :author      => "R.I.Pienaar <rip@devco.net>",
+            :license     => "Apache-2.0",
+            :version     => "0.19.0",
+            :url         => "https://github.com/choria-io",
+            :timeout     => 0
+
+discovery do
+    capabilities [:classes, :facts, :identity, :agents]
+end

data/lib/mcollective/discovery/choria.rb

@@ -0,0 +1,223 @@
+require "net/http"
+require_relative "../util/choria"
+
+module MCollective
+  class Discovery
+    class Choria
+      def self.discover(filter, timeout, limit=0, client=nil)
+        Choria.new(filter, timeout, limit, client).discover
+      end
+
+      attr_reader :timeout, :limit, :client, :config
+      attr_accessor :filter
+
+      def initialize(filter, timeout, limit, client)
+        @filter = filter
+        @timeout = timeout
+        @limit = limit
+        @client = client
+        @config = Config.instance
+      end
+
+      # Search for nodes
+      #
+      # @return [Array<String>] list of certnames found
+      def discover
+        queries = []
+
+        if choria.proxied_discovery?
+          Log.debug("Performing discovery against a PuppetDB Proxy")
+
+          choria.proxy_discovery_query(proxy_request)
+        else
+          Log.debug("Performing direct discovery against PuppetDB")
+          queries << discover_collective(filter["collective"]) if filter["collective"]
+          queries << discover_nodes(filter["identity"]) unless filter["identity"].empty?
+          queries << discover_classes(filter["cf_class"]) unless filter["cf_class"].empty?
+          queries << discover_facts(filter["fact"]) unless filter["fact"].empty?
+          queries << discover_agents(filter["agent"]) unless filter["agent"].empty?
+
+          choria.pql_query(node_search_string(queries.compact), true)
+        end
+      end
+
+      # Creates a request hash for the discovery proxy
+      #
+      # @return [Hash]
+      def proxy_request
+        request = {}
+
+        request["collective"] = filter["collective"] if filter["collective"]
+        request["identities"] = filter["identity"] unless filter["identity"].empty?
+        request["classes"] = filter["cf_class"] unless filter["cf_class"].empty?
+        request["facts"] = filter["fact"] unless filter["fact"].empty?
+        request["agents"] = filter["agent"] unless filter["agent"].empty?
+
+        request
+      end
+
+      # Discovers nodes in a specific collective
+      #
+      # @param filter [String] a collective name
+      # @return [String] a query string
+      def discover_collective(filter)
+        'certname in inventory[certname] { facts.mcollective.server.collectives.match("\d+") = "%s" }' % filter
+      end
+
+      # Searches for facts
+      #
+      # Nodes are searched using an `and` operator via the discover_classes method
+      #
+      # When the `rpcutil` or `scout` agent is required it will look for `Mcollective` class
+      # otherwise `Mcollective_avent_agentname` thus it will only find plugins
+      # installed using the `choria/mcollective` AIO plugin packager
+      #
+      # @param filter [Array<String>] agent names
+      # @return [Array<String>] list of nodes found
+      def discover_agents(filter)
+        pql = filter.map do |agent|
+          if ["rpcutil", "scout"].include?(agent)
+            "(%s or %s)" % [discover_classes(["mcollective::service"]), discover_classes(["choria::service"])]
+          elsif agent =~ /^\/(.+)\/$/
+            'resources {type = "File" and tag ~ "mcollective_agent_.*?%s.*?_server"}' % [string_regexi($1)]
+          else
+            'resources {type = "File" and tag = "mcollective_agent_%s_server"}' % [agent]
+          end
+        end
+
+        pql.join(" and ") unless pql.empty?
+      end
+
+      # Turns a string into a case insensitive regex string
+      #
+      # @param value [String]
+      # @return [String]
+      def string_regexi(value)
+        value =~ /^\/(.+)\/$/ ? derived_value = $1 : derived_value = value.dup
+
+        derived_value.each_char.map do |char|
+          if char =~ /[[:alpha:]]/
+            "[%s%s]" % [char.downcase, char.upcase]
+          else
+            char
+          end
+        end.join
+      end
+
+      # Capitalize a Puppet Resource
+      #
+      # foo::bar => Foo::Bar
+      #
+      # @param resource [String] a resource title
+      # @return [String]
+      def capitalize_resource(resource)
+        resource.split("::").map(&:capitalize).join("::")
+      end
+
+      # Searches for facts
+      #
+      # Nodes are searched using an `and` operator
+      #
+      # @param filter [Array<Hash>] hashes with :fact, :operator and :value
+      # @return [Array<String>] list of nodes found
+      def discover_facts(filter)
+        pql = filter.map do |q|
+          fact = q[:fact]
+          operator = q[:operator]
+          value = q[:value]
+
+          case operator
+          when "=~"
+            regex = string_regexi(value)
+
+            'inventory {facts.%s ~ "%s"}' % [fact, regex]
+          when "=="
+            if ["true", "false"].include?(value) || numeric?(value)
+              'inventory {facts.%s = %s or facts.%s = "%s"}' % [fact, value, fact, value]
+            else
+              'inventory {facts.%s = "%s"}' % [fact, value]
+            end
+          when "!="
+            if ["true", "false"].include?(value) || numeric?(value)
+              'inventory {!(facts.%s = %s or facts.%s = "%s")}' % [fact, value, fact, value]
+            else
+              'inventory {!(facts.%s = "%s")}' % [fact, value]
+            end
+          when ">=", ">", "<=", "<"
+            raise("Do not know how to do string fact comparisons using the '%s' operator with PuppetDB" % operator) unless numeric?(value)
+
+            "inventory {facts.%s %s %s}" % [fact, operator, value]
+          else
+            raise("Do not know how to do fact comparisons using the '%s' operator with PuppetDB" % operator)
+          end
+        end
+
+        pql.join(" and ") unless pql.empty?
+      end
+
+      # Searches for classes
+      #
+      # Nodes are searched using an `and` operator
+      #
+      # @return [Array<String>] list of nodes found
+      def discover_classes(filter)
+        pql = filter.map do |klass|
+          if klass =~ /^\/(.+)\/$/
+            'resources {type = "Class" and title ~ "%s"}' % [string_regexi($1)]
+          else
+            'resources {type = "Class" and title = "%s"}' % [capitalize_resource(klass)]
+          end
+        end
+
+        pql.join(" and ") unless pql.empty?
+      end
+
+      # Searches for nodes
+      #
+      # Nodes are searched using an `or` operator
+      #
+      # @return [Array<String>] list of nodes found
+      def discover_nodes(filter)
+        if filter.empty?
+          Log.debug("Empty node filter found, discovering all nodes")
+          nil
+        else
+          pql = filter.map do |ident|
+            case ident
+            when /^pql:\s*(.+)$/
+              "certname in %s" % $1
+            when /^\/(.+)\/$/
+              'certname ~ "%s"' % string_regexi($1)
+            else
+              'certname = "%s"' % ident
+            end
+          end
+
+          pql.join(" or ") unless pql.empty?
+        end
+      end
+
+      # Produce a nodes query with the supplied sub query included
+      #
+      # @param queries [Array<String>] PQL queries to be used as a sub query
+      # @return [String] nodes search string
+      def node_search_string(queries)
+        filter_queries = queries.map {|q| "(%s)" % q}.join(" and ")
+
+        "nodes[certname, deactivated] { %s }" % [filter_queries]
+      end
+
+      # Determines if a string is a number, either float or integer
+      #
+      # @param string [String]
+      # @return [boolean]
+      def numeric?(string)
+        true if Float(string) rescue false
+      end
+
+      def choria
+        @_choria ||= Util::Choria.new(false)
+      end
+    end
+  end
+end

data/lib/mcollective/discovery/flatfile.rb

@@ -5,10 +5,10 @@ module MCollective
   class Discovery
     class Flatfile
       def self.discover(filter, timeout, limit=0, client=nil)
-        unless client.options[:discovery_options].empty?
-          file = client.options[:discovery_options].first
-        else
+        if client.options[:discovery_options].empty?
           raise "The flatfile discovery method needs a path to a text file"
+        else
+          file = client.options[:discovery_options].first
         end
 
         raise "Cannot read the file %s specified as discovery source" % file unless File.readable?(file)
@@ -18,16 +18,15 @@ module MCollective
 
         File.readlines(file).each do |host|
           host = host.chomp.strip
-          if host.empty? || host.match(/^#/)
-            next
-          end
-          raise 'Identities can only match /^[\w\.\-]+$/' unless host.match(/^[\w\.\-]+$/)
+          next if host.empty? || host.match(/^#/)
+          raise 'Identities can only match /^[\w\.\-]+$/' unless host.match(/^[\w.\-]+$/)
+
           hosts << host
         end
 
         # this plugin only supports identity filters, do regex matches etc against
         # the list found in the flatfile
-        if !(filter["identity"].empty?)
+        if !filter["identity"].empty?
           filter["identity"].each do |identity|
             identity = Regexp.new(identity.gsub("\/", "")) if identity.match("^/")
 

data/lib/mcollective/discovery/mc.rb

@@ -16,8 +16,8 @@ module MCollective
               return hosts if limit > 0 && hosts.size == limit
             end
           end
-        rescue Timeout::Error => e
-        rescue Exception => e
+        rescue Timeout::Error # rubocop:disable Lint/SuppressedException
+        rescue Exception # rubocop:disable Lint/RescueException
           raise
         ensure
           client.unsubscribe("discovery", :reply)

data/lib/mcollective/discovery/stdin.rb

@@ -1,52 +1,54 @@
 # discovers against stdin instead of the traditional network discovery
 # the input must be a flat file with a node name per line which should match identities as configured,
 # or it should be a json string as output by the -j option of mco rpc
-require 'mcollective/rpc/helpers'
+require "mcollective/rpc/helpers"
 
 module MCollective
   class Discovery
     class Stdin
       def self.discover(filter, timeout, limit=0, client=nil)
-        unless client.options[:discovery_options].empty?
-          type = client.options[:discovery_options].first.downcase
+        if client.options[:discovery_options].empty?
+          type = "auto"
         else
-          type = 'auto'
+          type = client.options[:discovery_options].first.downcase
         end
 
         discovered = []
 
-        file = STDIN.read
+        file = $stdin.read
 
-        if file =~ /^\s*$/
-          raise("data piped on STDIN contained only whitespace - could not discover hosts from it.")
-        end
+        raise("data piped on STDIN contained only whitespace - could not discover hosts from it.") if file =~ /^\s*$/
 
-        if type == 'auto'
+        if type == "auto"
           if file =~ /^\s*\[/
-            type = 'json'
+            type = "json"
           else
-            type = 'text'
+            type = "text"
           end
         end
 
         Log.debug("Parsing STDIN input as type %s" % type)
 
-        if type == 'json'
+        case type
+        when "json"
           hosts = RPC::Helpers.extract_hosts_from_json(file)
-        elsif type == 'text'
+        when "text"
           hosts = file.split("\n")
         else
           raise("stdin discovery plugin only knows the types auto/text/json, not \"#{type}\"")
         end
 
         hosts.map do |host|
-          raise 'Identities can only match /\w\.\-/' unless host.match(/^[\w\.\-]+$/)
+          raise 'Identities can only match /\w\.\-/' unless host.match(/^[\w.\-]+$/)
+
           host
         end
 
         # this plugin only supports identity filters, do regex matches etc against
         # the list found in the flatfile
-        unless filter["identity"].empty?
+        if filter["identity"].empty?
+          discovered = hosts
+        else
           filter["identity"].each do |identity|
             identity = Regexp.new(identity.gsub("\/", "")) if identity.match("^/")
 
@@ -56,8 +58,6 @@ module MCollective
               discovered << identity
             end
           end
-        else
-          discovered = hosts
         end
 
         discovered
@@ -65,4 +65,3 @@ module MCollective
     end
   end
 end
-

data/lib/mcollective/exceptions.rb

@@ -1,12 +1,19 @@
 module MCollective
   # Exceptions for the RPC system
   class DDLValidationError < RuntimeError; end
+
   class ValidatorError < RuntimeError; end
+
   class ClientTimeoutError < RuntimeError; end
+
   class MsgDoesNotMatchRequestID < RuntimeError; end
+
   class MsgTTLExpired < RuntimeError; end
+
   class NotTargettedAtUs < RuntimeError; end
+
   class RPCError < StandardError; end
+
   class SecurityValidationFailed < RuntimeError; end
 
   class BackoffSuggestion < StandardError
@@ -14,15 +21,21 @@ module MCollective
 
     def initialize(backoff=nil)
       @backoff = backoff
+      super
     end
   end
 
   class MessageNotReceived < BackoffSuggestion; end
+
   class UnexpectedMessageType < BackoffSuggestion; end
 
   class InvalidRPCData < RPCError; end
+
   class MissingRPCData < RPCError; end
+
   class RPCAborted < RPCError; end
+
   class UnknownRPCAction < RPCError; end
+
   class UnknownRPCError < RPCError; end
 end

data/lib/mcollective/facts/base.rb

@@ -17,6 +17,7 @@ module MCollective
       # Registers new fact sources into the plugin manager
       def self.inherited(klass)
         PluginManager << {:type => "facts_plugin", :class => klass.to_s}
+        super
       end
 
       # Returns the value of a single fact
@@ -27,7 +28,7 @@ module MCollective
 
         @mutex.synchronize do
           begin
-            if (Time.now.to_i - @last_facts_load > cache_time.to_i ) || force_reload?
+            if (Time.now.to_i - @last_facts_load > cache_time.to_i) || force_reload?
               Log.debug("Resetting facter cache, now: #{Time.now.to_i} last-known-good: #{@last_facts_load}")
 
               tfacts = load_facts_from_source
@@ -42,7 +43,7 @@ module MCollective
             else
               Log.debug("Using cached facts now: #{Time.now.to_i} last-known-good: #{@last_facts_load}")
             end
-          rescue Exception => e
+          rescue Exception => e # rubocop:disable Lint/RescueException
             Log.error("Failed to load facts: #{e.class}: #{e}")
 
             # Avoid loops where failing fact loads cause huge CPU
@@ -55,17 +56,16 @@ module MCollective
           end
         end
 
-
         # If you do not supply a specific fact all facts will be returned
         if fact.nil?
-          return @facts
+          @facts
         else
           @facts.include?(fact) ? @facts[fact] : nil
         end
       end
 
       # Returns all facts
-      def get_facts
+      def get_facts # rubocop:disable Naming/AccessorMethodName
         get_fact(nil)
       end
 
@@ -84,15 +84,15 @@ module MCollective
       def normalize_facts(value)
         case value
         when Array
-          return value.map { |v| normalize_facts(v) }
+          value.map { |v| normalize_facts(v) }
         when Hash
           new_hash = {}
-          value.each do |k,v|
+          value.each do |k, v|
             new_hash[k.to_s] = normalize_facts(v)
           end
-          return new_hash
+          new_hash
         else
-          return value.to_s
+          value.to_s
         end
       end
     end