choria-mcorpc-support 2.22.1 → 2.23.0.pre

data/lib/mcollective/util/playbook/tasks/shell_task.rb

@@ -0,0 +1,93 @@
+module MCollective
+  module Util
+    class Playbook
+      class Tasks
+        class ShellTask < Base
+          def validate_configuration!
+            raise("A command was not given") unless @command
+            raise("Nodes were given but is not an array") if @nodes && !@nodes.is_a?(Array)
+          end
+
+          def from_hash(data)
+            @cwd = data["cwd"] || Dir.pwd
+            @timeout = data["timeout"]
+            @nodes = data["nodes"]
+            @environment = data["environment"]
+
+            if @nodes.is_a?(Array)
+              @command = "%s --nodes %s" % [data["command"], @nodes.join(",")]
+            else
+              @command = data["command"]
+            end
+
+            self
+          end
+
+          def shell_options
+            options = {}
+            options["cwd"] = @cwd if @cwd
+            options["timeout"] = Integer(@timeout) if @timeout
+            options["environment"] = @environment if @environment
+            options["stdout"] = []
+            options["stderr"] = options["stdout"]
+            options
+          end
+
+          def to_execution_result(results)
+            result = {
+              "value" => results[2].join("\r\n").chomp,
+              "type" => "shell",
+              "fail_ok" => @fail_ok,
+              "error" => {
+                "msg" => results[1],
+                "kind" => "choria.playbook/taskerror",
+                "details" => {
+                  "command" => @command
+                }
+              }
+            }
+
+            result.delete("error") if results[0]
+
+            {"localhost" => result}
+          end
+
+          def run
+            if @nodes
+              Log.info("Starting command %s against %d nodes" % [@command, @nodes.size])
+            else
+              Log.info("Starting command %s" % [@command])
+            end
+
+            begin
+              options = shell_options
+
+              shell = Shell.new(@command, options)
+              shell.runcommand
+
+              options["stdout"].each do |output|
+                output.lines.each do |line|
+                  Log.info(line.chomp)
+                end
+              end
+
+              if shell.status.exitstatus == 0
+                Log.info("Successfully ran command %s" % [@command])
+                [true, "Command completed successfully", options["stdout"]]
+              else
+                Log.warn("Failed to run command %s with exit code %s" % [@command, shell.status.exitstatus])
+                [false, "Command failed with code %d" % [shell.status.exitstatus], options["stdout"]]
+              end
+            rescue
+              msg = "Could not run command %s: %s: %s" % [@command, $!.class, $!.to_s]
+              Log.debug(msg)
+              Log.debug($!.backtrace.join("\t\n"))
+
+              [false, msg, []]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/tasks/slack_task.rb

@@ -0,0 +1,105 @@
+require "cgi"
+require "json"
+
+module MCollective
+  module Util
+    class Playbook
+      class Tasks
+        class SlackTask < Base
+          def validate_configuration!
+            raise("A channel is required") unless @channel
+            raise("Message text is required") unless @text
+            raise("A bot token is required") unless @token
+          end
+
+          def from_hash(data)
+            @channel = data["channel"]
+            @text = data["text"]
+            @token = data["token"]
+            @color = data.fetch("color", "#ffa449")
+            @username = data.fetch("username", "Choria")
+            @icon = "https://choria.io/img/slack-48x48.png"
+
+            self
+          end
+
+          def choria
+            @_choria ||= Util::Choria.new(false)
+          end
+
+          def attachments
+            [
+              "fallback" => @text,
+              "color" => @color,
+              "text" => @text,
+              "pretext" => "Task: %s" % @description,
+              "mrkdwn_in" => ["text"],
+              "footer" => "Choria Playbooks",
+              "fields" => [
+                {
+                  "title" => "user",
+                  "value" => PluginManager["security_plugin"].callerid,
+                  "short" => true
+                },
+                {
+                  "title" => "playbook",
+                  "value" => @playbook.name,
+                  "short" => true
+                }
+              ]
+            ]
+          end
+
+          def to_execution_result(results)
+            result = {
+              "value" => results[1],
+              "type" => "slack",
+              "fail_ok" => @fail_ok
+            }
+
+            unless results[0]
+              result["error"] = {
+                "msg" => results[1],
+                "kind" => "choria.playbook/taskerror",
+                "details" => {
+                  "channel" => @channel
+                }
+              }
+              result["value"] = results[2].first
+            end
+
+            {"slack.com" => result}
+          end
+
+          def run
+            https = choria.https(:target => "slack.com", :port => 443)
+            path = "/api/chat.postMessage?token=%s&username=%s&channel=%s&icon_url=%s&attachments=%s" % [
+              CGI.escape(@token),
+              CGI.escape(@username),
+              CGI.escape(@channel),
+              CGI.escape(@icon),
+              CGI.escape(attachments.to_json)
+            ]
+
+            resp, data = https.request(choria.http_get(path))
+            data = JSON.parse(data || resp.body)
+
+            if resp.code == "200" && data["ok"]
+              Log.debug("Successfully sent message to slack channel %s" % [@channel])
+              [true, "Message submitted to slack channel %s" % [@channel], [data]]
+            else
+              Log.warn("Failed to send message to slack channel %s: %s" % [@channel, data["error"]])
+              [false, "Failed to send message to slack channel %s: %s" % [@channel, data["error"]], [data]]
+            end
+          rescue
+            msg = "Could not publish slack message to channel %s: %s: %s" % [@channel, $!.class, $!.to_s]
+            Log.debug(msg)
+            Log.debug($!.backtrace.join("\t\n"))
+
+            [false, msg, []]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/tasks/webhook_task.rb

@@ -0,0 +1,136 @@
+require "uri"
+require "cgi"
+require "json"
+
+module MCollective
+  module Util
+    class Playbook
+      class Tasks
+        class WebhookTask < Base
+          USER_AGENT = "Choria Playbooks http://choria.io".freeze
+
+          attr_reader :verify_ssl
+
+          def validate_configuration!
+            raise("A uri is required") unless @uri
+            raise("Only GET and POST is supported as methods") unless ["GET", "POST"].include?(@method)
+          end
+
+          def from_hash(data)
+            @headers = data.fetch("headers", {})
+            @data = data.fetch("data", {})
+            @uri = data["uri"]
+            @method = data.fetch("method", "POST").upcase
+            @request_id = SSL.uuid
+            @verify_ssl = Util.str_to_bool(data.fetch("verify_ssl", true))
+
+            self
+          end
+
+          def create_uri
+            uri = URI.parse(@uri)
+
+            if @method == "GET"
+              query = Array(uri.query)
+
+              @data.each do |k, v|
+                query << "%s=%s" % [CGI.escape(k), CGI.escape(v.to_s)]
+              end
+
+              uri.query = query.join("&") unless query.empty?
+            end
+
+            uri
+          end
+
+          def http_get_request(uri)
+            headers = {
+              "User-Agent" => USER_AGENT,
+              "X-Choria-Request-ID" => @request_id
+            }.merge(@headers)
+
+            Net::HTTP::Get.new(uri.request_uri, headers)
+          end
+
+          def http_post_request(uri)
+            headers = {
+              "Content-Type" => "application/json",
+              "User-Agent" => USER_AGENT,
+              "X-Choria-Request-ID" => @request_id
+            }.merge(@headers)
+
+            req = Net::HTTP::Post.new(uri.request_uri, headers)
+            req.body = @data.to_json
+            req
+          end
+
+          def http_request(uri)
+            return http_get_request(uri) if @method == "GET"
+            return http_post_request(uri) if @method == "POST"
+
+            raise("Unknown request method %s" % @method)
+          end
+
+          def choria
+            @_choria ||= Util::Choria.new(false)
+          end
+
+          def http(uri)
+            http = choria.https({:target => uri.host, :port => uri.port}, false)
+
+            if verify_ssl
+              http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+            else
+              http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            end
+
+            http.use_ssl = false if uri.scheme == "http"
+
+            http
+          end
+
+          def to_execution_result(results)
+            result = {
+              "value" => results[2].first,
+              "type" => "webhook",
+              "fail_ok" => @fail_ok,
+              "error" => {
+                "msg" => results[1],
+                "kind" => "choria.playbook/taskerror",
+                "details" => {
+                  "task" => "webhook",
+                  "uri" => @uri,
+                  "method" => @method,
+                  "request_id" => @request_id
+                }
+              }
+            }
+
+            result.delete("error") if results[0]
+
+            {create_uri.host => result}
+          end
+
+          def run
+            uri = create_uri
+            resp = http(uri).request(http_request(uri))
+
+            Log.debug("%s request to %s returned code %s with body: %s" % [@method, uri.to_s, resp.code, resp.body])
+
+            if ["200", "201"].include?(resp.code)
+              [true, "Successfully sent %s request to webhook %s with id %s" % [@method, @uri, @request_id], [resp.body]]
+            else
+              [false, "Failed to send %s request to webhook %s with id %s: %s: %s" % [@method, @uri, @request_id, resp.code, resp.body], [resp.body]]
+            end
+          rescue
+            msg = "Could not send %s to webhook %s: %s: %s" % [@method, @uri, $!.class, $!.to_s]
+            Log.debug(msg)
+            Log.debug($!.backtrace.join("\t\n"))
+
+            [false, msg, []]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/template_util.rb

@@ -0,0 +1,98 @@
+module MCollective
+  module Util
+    class Playbook
+      module TemplateUtil
+        # Recursively parse a data structure seeking strings that might contain templates
+        #
+        # Template strings look like `{{{scope.key}}}` where scope is one of `input`, `metadata`,
+        # `nodes` and the key is some item contained in those scopes like a named node list.
+        #
+        # You'll generally mix this into a class you wish to use it in, that class should have
+        # a `@playbook` variable set which is an instance of `Playbook`
+        #
+        # @param data [Object] data structure to traverse
+        # @return [Object] deep cloned copy of the structure with strings parsed
+        def t(data)
+          data = Marshal.load(Marshal.dump(data))
+
+          case data
+          when String
+            __template_process_string(data)
+          when Hash
+            data.each do |k, v|
+              data[k] = t(v)
+            end
+
+            data
+          when Array
+            data.map do |v|
+              t(v)
+            end
+          else
+            data
+          end
+        end
+
+        def __template_resolve(type, item)
+          Log.debug("Resolving template data for %s.%s" % [type, item])
+
+          case type
+          when "input", "inputs"
+            @playbook.input_value(item)
+          when "nodes"
+            @playbook.discovered_nodes(item)
+          when "metadata"
+            @playbook.metadata_item(item)
+          when "previous_task"
+            @playbook.previous_task(item)
+          when "date"
+            Time.now.strftime(item)
+          when "utc_date"
+            Time.now.utc.strftime(item)
+          when "elapsed_time"
+            @playbook.report.elapsed_time
+          when "uuid"
+            SSL.uuid
+          else
+            raise("Do not know how to process data of type %s" % type)
+          end
+        end
+
+        def __template_process_string(string)
+          raise("Playbook is not accessible") unless @playbook
+
+          front = '{{2,3}\s*'
+          back = '\s*}{2,3}'
+
+          data_regex = Regexp.new("%s%s%s" % [front, '(?<type>input(s*)|metadata|nodes)\.(?<item>[a-zA-Z0-9\_\-]+)', back])
+          date_regex = Regexp.new("%s%s%s" % [front, '(?<type>date|utc_date)\(\s*["\']*(?<format>.+?)["\']*\s*\)', back])
+          task_regex = Regexp.new("%s%s%s" % [front, '(?<type>previous_task)\.(?<item>(success|description|msg|message|data|runtime))', back])
+          singles_regex = Regexp.new("%s%s%s" % [front, "(?<type>uuid|elapsed_time)", back])
+
+          combined_regex = Regexp.union(data_regex, date_regex, task_regex, singles_regex)
+
+          if req = (string.match(/^#{data_regex}$/) || string.match(/^#{task_regex}$/))
+            __template_resolve(req["type"], req["item"])
+          elsif req = string.match(/^#{date_regex}$/)
+            __template_resolve(req["type"], req["format"])
+          elsif req = string.match(/^#{singles_regex}$/)
+            __template_resolve(req["type"], "")
+          else
+            string.gsub(/#{combined_regex}/) do |part|
+              value = __template_process_string(part)
+
+              case value
+              when Array
+                value.join(", ")
+              when Hash
+                value.to_json
+              else
+                value
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/uses.rb

@@ -0,0 +1,169 @@
+module MCollective
+  module Util
+    class Playbook
+      class Uses
+        def initialize(playbook)
+          @playbook = playbook
+          @uses = {}
+        end
+
+        def [](agent)
+          @uses[agent]
+        end
+
+        def keys
+          @uses.keys
+        end
+
+        # Retrieves the agent inventory for a set of nodes
+        #
+        # @param nodes [Array<String>] list of nodes to retrieve it for
+        # @return [Boolean, String, Array] success, message and inventory results
+        def agent_inventory(nodes)
+          rpc = Tasks::McollectiveTask.new(@playbook)
+
+          rpc.from_hash(
+            "nodes" => nodes,
+            "action" => "rpcutil.agent_inventory",
+            "silent" => true
+          )
+
+          success, msg, inventory = rpc.run
+
+          # rpcutil#agent_inventory is a hash in a hash not managed by the DDL
+          # this is not handled by the JSON encoding magic that does DDL based
+          # symbol and string conversion so we normalise the data always to symbol
+          # based structures
+          inventory.each do |node|
+            node["data"][:agents].each do |agent|
+              agent.keys.each do |key| # rubocop:disable Style/HashEachMethods
+                agent[key.intern] = agent.delete(key) if key.is_a?(String)
+              end
+            end
+          end
+
+          [success, msg, inventory]
+        end
+
+        # Validates agent versions on nodes
+        #
+        # @param agents [Hash] a hash of agent names and nodes that uses that agent
+        # @raise [StandardError] on failure
+        def validate_agents(agents)
+          nodes = agents.map {|_, agent_nodes| agent_nodes}.flatten.uniq
+
+          Log.info("Validating agent inventory on %d nodes" % nodes.size)
+
+          validation_fail = false
+
+          success, msg, inventory = agent_inventory(nodes)
+
+          raise("Could not determine agent inventory: %s" % msg) unless success
+
+          agents.each do |agent, agent_nodes|
+            unless @uses.include?(agent)
+              Log.warn("Agent %s is mentioned in node sets but not declared in the uses list" % agent)
+              validation_fail = true
+              next
+            end
+
+            agent_nodes.each do |node|
+              unless node_inventory = inventory.find {|i| i["sender"] == node}
+                Log.warn("Did not receive an inventory for node %s" % node)
+                validation_fail = true
+                next
+              end
+
+              unless metadata = node_inventory["data"][:agents].find {|i| i[:agent] == agent}
+                Log.warn("Node %s does not have the agent %s" % [node, agent])
+                validation_fail = true
+                next
+              end
+
+              if valid_version?(metadata[:version], @uses[agent])
+                Log.debug("Agent %s on %s version %s matches desired version %s" % [agent, node, metadata[:version], @uses[agent]])
+              else
+                Log.warn("Agent %s on %s version %s does not match desired version %s" % [agent, node, metadata[:version], @uses[agent]])
+                validation_fail = true
+              end
+            end
+          end
+
+          raise("Network agents did not match specified SemVer specifications in the playbook") if validation_fail
+
+          Log.info("Agent inventory on %d nodes validated" % nodes.size)
+        end
+
+        # Determines if a semver version is within a stated range
+        #
+        # @note mcollective never suggested semver, so versions like "1.1" becomes "1.1.0" for the compare
+        # @param have [String] SemVer of what you have
+        # @param want [String] SemVer range of what you need
+        # @return [Boolean]
+        # @raise [StandardError] on invalid version strings
+        def valid_version?(have, want)
+          have = "%s.0" % have if have.split(".").size == 2
+
+          require "semantic_puppet" unless defined?(SemanticPuppet)
+
+          semver_have = SemanticPuppet::Version.parse(have)
+          semver_want = SemanticPuppet::VersionRange.parse(want)
+          semver_want.include?(semver_have)
+        end
+
+        # Checks that all the declared agent DDLs exist
+        #
+        # @raise [StandardError] on invalid DDLs
+        def prepare
+          invalid = @uses.map do |agent, want|
+            begin
+              have = ddl_version(agent)
+
+              if valid_version?(have, want)
+                Log.debug("Agent %s DDL version %s matches desired %s" % [agent, have, want])
+                nil
+              else
+                Log.warn("Agent %s DDL version %s does not match desired %s" % [agent, have, want])
+                agent
+              end
+            rescue
+              Log.warn("Could not process DDL for agent %s: %s: %s" % [agent, $!.class, $!.to_s])
+              agent
+            end
+          end.compact
+
+          raise("DDLs for agent(s) %s did not match desired versions" % invalid.join(", ")) unless invalid.empty?
+        end
+
+        # Fetches the DDL version for an agent
+        #
+        # If the agent DDL has versions like 1.0 it will be
+        # turned into 1.0.0 as old mco stuff didnt do semver
+        #
+        # @param agent [String]
+        def ddl_version(agent)
+          ddl = agent_ddl(agent)
+          ddl.meta[:version]
+        end
+
+        # Returns the DDL for a specific agent
+        #
+        # @param agent [String]
+        # @return [DDL::AgentDDL]
+        # @raise [StandardError] should the DDL not exist
+        def agent_ddl(agent)
+          DDL::AgentDDL.new(agent)
+        end
+
+        def from_hash(data)
+          data.each do |agent, version|
+            Log.debug("Loading usage of %s version %s" % [agent, version])
+            @uses[agent] = version
+          end
+
+          self
+        end
+      end
+    end
+  end
+end