choria-mcorpc-support 2.22.1 → 2.23.0.pre

data/lib/mcollective/util/playbook/nodes.rb

@@ -0,0 +1,207 @@
+require_relative "nodes/mcollective_nodes"
+require_relative "nodes/pql_nodes"
+require_relative "nodes/yaml_nodes"
+require_relative "nodes/shell_nodes"
+require_relative "nodes/terraform_nodes"
+
+module MCollective
+  module Util
+    class Playbook
+      class Nodes
+        attr_reader :nodes
+
+        def initialize(playbook)
+          @playbook = playbook
+          @nodes = {}
+        end
+
+        # List of known node set names
+        #
+        # @return [Array<String>]
+        def keys
+          @nodes.keys
+        end
+
+        # Nodes belonging to a specific node set
+        #
+        # @param nodeset [String] node set name
+        # @return [Array<String>]
+        # @raise [StandardError] when node set is unknown
+        def [](nodeset)
+          if include?(nodeset)
+            @nodes[nodeset][:discovered]
+          else
+            raise("Unknown node set %s" % nodeset)
+          end
+        end
+
+        # Properties for a certain node set
+        #
+        # @param nodes [String] node set name
+        # @return [Hash]
+        # @raise [StandardError] when node set is unknown
+        def properties(nodes)
+          if include?(nodes)
+            @nodes[nodes][:properties]
+          else
+            raise("Unknown node set %s" % nodes)
+          end
+        end
+
+        # Determines if a node set is known
+        #
+        # @param nodes [String] node set name
+        # @return [Boolean]
+        def include?(nodes)
+          @nodes.include?(nodes)
+        end
+
+        def prepare
+          @nodes.each do |node_set, dets|
+            @playbook.in_context(node_set) do
+              Log.debug("Preparing nodeset %s" % node_set)
+
+              resolve_nodes(node_set)
+              check_empty(node_set)
+              limit_nodes(node_set)
+              validate_nodes(node_set)
+
+              Log.info("Discovered %d node(s) in node set %s" % [dets[:discovered].size, node_set])
+            end
+          end
+
+          @playbook.in_context("conn.test") { test_nodes }
+          @playbook.in_context("ddl.test") { check_uses }
+        end
+
+        # Resolve the node list using the resolver class
+        #
+        # @param node_set [String] node set name
+        def resolve_nodes(node_set)
+          node_props = @nodes[node_set]
+          node_props[:resolver].prepare
+          node_props[:discovered] = node_props[:resolver].discover.uniq
+        end
+
+        # Checks if the agents on the nodes matches the desired versions
+        #
+        # @raise [StandardError] on error
+        def check_uses
+          agent_nodes = {}
+
+          @nodes.map do |_, dets|
+            dets[:properties].fetch("uses", []).each do |agent|
+              agent_nodes[agent] ||= []
+              agent_nodes[agent].concat(dets[:discovered])
+            end
+          end
+
+          @playbook.validate_agents(agent_nodes) unless agent_nodes.empty?
+        end
+
+        # Determines if a nodeset needs connectivity test
+        #
+        # @param nodes [String] node set name
+        # @return [Boolean]
+        # @raise [StandardError] for unknown node sets
+        def should_test?(nodes)
+          !!properties(nodes)["test"]
+        end
+
+        def mcollective_task
+          Tasks::McollectiveTask.new(@playbook)
+        end
+
+        # Tests a RPC ping to the discovered nodes
+        #
+        # @todo is this really needed?
+        # @raise [StandardError] on error
+        def test_nodes
+          nodes_to_test = @nodes.map do |nodes, _|
+            self[nodes] if should_test?(nodes)
+          end.flatten.compact
+
+          return if nodes_to_test.empty?
+
+          Log.info("Checking connectivity for %d nodes" % nodes_to_test.size)
+
+          rpc = mcollective_task
+          rpc.from_hash(
+            "nodes" => nodes_to_test,
+            "action" => "rpcutil.ping",
+            "silent" => true
+          )
+          success, msg, _ = rpc.run
+
+          raise("Connectivity test failed for some nodes: %s" % [msg]) unless success
+        end
+
+        # Checks that discovered nodes matches stated expectations
+        #
+        # @param nodes [String] node set name
+        # @raise [StandardError] on error
+        def validate_nodes(nodes)
+          return if properties(nodes)["empty_ok"]
+
+          raise("Node set %s needs at least %d nodes, got %d" % [nodes, properties(nodes)["at_least"], self[nodes].size]) unless self[nodes].size >= properties(nodes)["at_least"]
+        end
+
+        # Handles an empty discovered list
+        #
+        # @param nodes [String] node set name
+        # @raise [StandardError] when empty
+        def check_empty(nodes)
+          raise(properties(nodes)["when_empty"] || "Did not discover any nodes for nodeset %s" % nodes) if self[nodes].empty? && !properties(nodes)["empty_ok"]
+        end
+
+        # Limits the discovered list for a node set based on the playbook limits
+        #
+        # @todo more intelegent limiting with weighted randoms like mco rpc client
+        # @param nodes [String] node set name
+        def limit_nodes(nodes)
+          return if self[nodes].empty?
+
+          if limit = properties(nodes)["limit"]
+            Log.debug("Limiting node set %s to %d nodes from %d" % [nodes, limit, @nodes[nodes][:discovered].size])
+            @nodes[nodes][:discovered] = @nodes[nodes][:discovered][0..(limit - 1)]
+          end
+        end
+
+        # Retrieves a new instance of the resolver for a certain type of discovery
+        #
+        # @param type [String] finds classes called *Nodes::TypeNodes* based on this type
+        def resolver_for(type)
+          klass_name = "%sNodes" % type.capitalize
+
+          Nodes.const_get(klass_name).new
+        rescue NameError
+          raise("Cannot find a handler for Node Set type %s" % type)
+        end
+
+        def from_hash(data)
+          data.each do |nodes, props|
+            resolver = resolver_for(props["type"])
+            resolver.from_hash(props)
+            resolver.validate_configuration!
+
+            node_props = {
+              "at_least" => 1,
+              "empty_ok" => false,
+              "when_empty" => "Did not discover any nodes for nodeset %s" % nodes
+            }.merge(props)
+
+            node_props["at_least"] = 0 if node_props["empty_ok"]
+
+            @nodes[nodes] = {
+              :resolver => resolver,
+              :discovered => [],
+              :properties => node_props
+            }
+          end
+
+          self
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/nodes/mcollective_nodes.rb

@@ -0,0 +1,86 @@
+module MCollective
+  module Util
+    class Playbook
+      class Nodes
+        class McollectiveNodes
+          def initialize
+            @discovery_method = "mc"
+            @agents = []
+            @facts = []
+            @classes = []
+            @identity = []
+            @compound = nil
+          end
+
+          def prepare; end
+
+          # @todo
+          def validate_configuration!; end
+
+          # Creates and cache an RPC::Client for the configured agent
+          #
+          # @param from_cache [Boolean] when false a new instance is always returned
+          # @return [RPC::Client]
+          def client(from_cache: true)
+            if from_cache
+              @_rpc_client ||= create_and_configure_client
+            else
+              create_and_configure_client
+            end
+          end
+
+          # Creates a new RPC::Client and configures it with the configured settings
+          #
+          # @todo discovery
+          # @return [RPC::Client]
+          def create_and_configure_client
+            client = RPC::Client.new(@agents[0], :configfile => Util.config_file_for_user, :options => Util.default_options)
+            client.progress = false
+            client.discovery_method = @discovery_method
+
+            @classes.each do |filter|
+              client.class_filter(filter)
+            end
+
+            @facts.each do |filter|
+              client.fact_filter(filter)
+            end
+
+            @agents.each do |filter|
+              client.agent_filter(filter)
+            end
+
+            @identity.each do |filter|
+              client.identity_filter(filter)
+            end
+
+            client.compound_filter(@compound) if @compound
+
+            client
+          end
+
+          # Initialize the nodes source from a hash
+          #
+          # @param data [Hash] input data matching nodes.json schema
+          # @return [McollectiveNodes]
+          def from_hash(data)
+            @discovery_method = data.fetch("discovery_method", "mc")
+            @agents = data.fetch("agents", ["rpcutil"])
+            @facts = data.fetch("facts", [])
+            @classes = data.fetch("classes", [])
+            @identity = data.fetch("identities", [])
+            @compound = data["compound"]
+
+            @_rpc_client = nil
+
+            self
+          end
+
+          def discover
+            client.discover
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/nodes/pql_nodes.rb

@@ -0,0 +1,40 @@
+require "mcollective/util/choria"
+
+module MCollective
+  module Util
+    class Playbook
+      class Nodes
+        class PqlNodes
+          def initialize
+            @query = nil
+          end
+
+          def prepare; end
+
+          def validate_configuration!
+            raise("No PQL query specified") unless @query
+          end
+
+          def choria
+            @_choria ||= Util::Choria.new(false)
+          end
+
+          # Initialize the nodes source from a hash
+          #
+          # @param data [Hash] input data matching nodes.json schema
+          # @return [PqlNodes]
+          def from_hash(data)
+            @query = data["query"]
+
+            self
+          end
+
+          # Performs the PQL query and extracts certnames
+          def discover
+            choria.pql_query(@query, true)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/nodes/shell_nodes.rb

@@ -0,0 +1,55 @@
+module MCollective
+  module Util
+    class Playbook
+      class Nodes
+        class ShellNodes
+          def initialize
+            @script = nil
+          end
+
+          def prepare; end
+
+          def validate_configuration!
+            raise("No node source script specified") unless @script
+            raise("Node source script is not executable") unless File.executable?(@script)
+            raise("Node source script produced no results") if data.empty?
+          end
+
+          def from_hash(data)
+            @script = data["script"]
+            @_data = nil
+
+            self
+          end
+
+          def valid_hostname?(host)
+            host =~ /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/
+          end
+
+          def data
+            return @_data if @_data
+
+            shell = Shell.new(@script, "timeout" => 10)
+            shell.runcommand
+
+            exitcode = shell.status.exitstatus
+
+            raise("Could not discover nodes via shell method, command exited with code %d" % [exitcode]) unless exitcode == 0
+
+            @_data = shell.stdout.lines.map do |line|
+              line.chomp!
+
+              raise("%s is not a valid hostname" % line) unless valid_hostname?(line)
+
+              line
+            end
+          end
+
+          def discover
+            data
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/nodes/terraform_nodes.rb

@@ -0,0 +1,65 @@
+module MCollective
+  module Util
+    class Playbook
+      class Nodes
+        class TerraformNodes
+          def prepare; end
+
+          def validate_configuration!
+            raise("The supplied terraform path %s is not executable" % @terraform) if @terraform && !File.executable?(@terraform)
+            raise("A terraform state file is needed") unless @state
+            raise("The terraform statefile %s is not readable" % @state) unless File.readable?(@state)
+            raise("An output name is needed") unless @output
+
+            Validator.validate(@terraform, :shellsafe)
+            Validator.validate(@state, :shellsafe)
+            Validator.validate(@output, :shellsafe)
+          end
+
+          def from_hash(data)
+            @state = data["statefile"]
+            @output = data["output"]
+            @terraform = data.fetch("terraform", choria.which("terraform"))
+
+            self
+          end
+
+          def valid_hostname?(host)
+            host =~ /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/
+          end
+
+          def tf_output
+            shell = Shell.new("%s output -state %s -json %s 2>&1" % [@terraform, @state, @output])
+            shell.runcommand
+
+            raise("Terraform exited with code %d: %s" % [shell.status.exitstatus, shell.stdout]) unless shell.status.exitstatus == 0
+
+            shell.stdout
+          end
+
+          def output_data
+            return @_data if @_data
+
+            data = JSON.parse(tf_output)
+
+            raise("Only terraform outputs of type list is supported") unless data["type"] == "list"
+
+            data["value"].each do |result|
+              raise("%s is not a valid hostname" % result) unless valid_hostname?(result)
+            end
+
+            @_data = data["value"]
+          end
+
+          def choria
+            @_choria ||= Util::Choria.new(false)
+          end
+
+          def discover
+            output_data
+          end
+        end
+      end
+    end
+  end
+end