choria-mcorpc-support 2.22.1 → 2.23.0.pre

data/lib/mcollective/util/playbook/nodes/yaml_nodes.rb

@@ -0,0 +1,47 @@
+require "yaml"
+
+module MCollective
+  module Util
+    class Playbook
+      class Nodes
+        class YamlNodes
+          def initialize
+            @group = nil
+            @file = nil
+          end
+
+          def prepare; end
+
+          def validate_configuration!
+            raise("No node group YAML source file specified") unless @file
+            raise("Node group YAML source file %s is not readable" % @file) unless File.readable?(@file)
+            raise("No node group name specified") unless @group
+            raise("No data group %s defined in the data file %s" % [@group, @file]) unless data.include?(@group)
+            raise("Data group %s is not an array" % @group) unless data[@group].is_a?(Array)
+            raise("Data group %s is empty" % @group) if data[@group].empty?
+          end
+
+          # Initialize the nodes source from a hash
+          #
+          # @param data [Hash] input with `group` and `source` keys
+          # @return [PqlNodes]
+          def from_hash(data)
+            @group = data["group"]
+            @file = data["source"]
+
+            self
+          end
+
+          def data
+            @_data ||= YAML.safe_load(File.read(@file))
+          end
+
+          # Performs the PQL query and extracts certnames
+          def discover
+            data[@group]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/playbook_logger.rb

@@ -0,0 +1,47 @@
+require "mcollective/logger/console_logger"
+
+module MCollective
+  module Util
+    class Playbook
+      class Playbook_Logger < Logger::Console_logger
+        attr_writer :scope
+
+        def initialize(playbook)
+          @playbook = playbook
+          @report = playbook.report
+
+          super()
+        end
+
+        def start
+          set_level(@playbook.loglevel.intern)
+        end
+
+        def log(level, from, msg, normal_output=$stderr, last_resort_output=$stderr)
+          if @playbook.loglevel != "debug"
+            if should_show?
+              console_from = "%s#%-25s" % [@playbook.name, @playbook.context]
+            else
+              level = :debug
+            end
+          end
+
+          if @known_levels.index(level) >= @known_levels.index(@active_level)
+            r_time = Time.now
+            s_time = r_time.strftime("%H:%M:%S")
+
+            @report.append_log(r_time, level, from, msg)
+
+            normal_output.puts("%s %s: %s %s" % [colorize(level, level[0].capitalize), s_time, console_from, msg])
+          end
+        rescue
+          last_resort_output.puts("%s: %s" % [level, msg])
+        end
+
+        def should_show?
+          !caller(1..5).grep(/playbook/).empty?
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/puppet_logger.rb

@@ -0,0 +1,51 @@
+require "mcollective/logger/console_logger"
+
+module MCollective
+  module Util
+    class Playbook
+      class Puppet_Logger < Logger::Console_logger
+        attr_writer :scope
+
+        def initialize(playbook)
+          @playbook = playbook
+          @report = playbook.report
+
+          super()
+        end
+
+        def start
+          set_level(@playbook.loglevel.intern)
+        end
+
+        def log(level, from, msg, normal_output=$stderr, last_resort_output=$stderr)
+          return unless should_show?(level)
+
+          logmethod = case level
+                      when :info
+                        :notice
+                      when :warn
+                        :warning
+                      when :error
+                        :err
+                      when :fatal
+                        :crit
+                      else
+                        :debug
+                      end
+
+          if @scope
+            Puppet::Util::Log.log_func(@scope, logmethod, [msg])
+          else
+            Puppet.send(logmethod, msg)
+          end
+        end
+
+        def should_show?(level)
+          return true unless level == :info
+
+          !caller(2..5).grep(/playbook/).empty?
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/report.rb

@@ -0,0 +1,152 @@
+module MCollective
+  module Util
+    class Playbook
+      class Report
+        attr_reader :timestamp
+
+        def initialize(playbook)
+          @playbook = playbook
+          @logs = []
+          @timestamp = Time.now.utc
+          @version = 1
+          @nodes = {}
+          @tasks = []
+          @success = false
+          @final = false
+
+          @metrics = {
+            "start_time" => @timestamp,
+            "end_time" => @timestamp,
+            "run_time" => 0,
+            "task_count" => 0,
+            "task_types" => {}
+          }
+
+          @inputs = {
+            "static" => {},
+            "dynamic" => []
+          }
+        end
+
+        def start!
+          @metrics["start_time"] = Time.now.utc
+        end
+
+        def elapsed_time
+          Time.now.utc - @metrics["start_time"]
+        end
+
+        def finalize(success, fail_msg=nil)
+          return(to_report) if @final
+
+          @success = success
+          @fail_message = fail_msg
+
+          store_playbook_metadata
+          store_static_inputs
+          store_dynamic_inputs
+          store_node_groups
+          store_task_results
+          calculate_metrics
+
+          @final = true
+
+          to_report
+        end
+
+        def to_report
+          {
+            "report" => {
+              "version" => @version,
+              "timestamp" => @timestamp,
+              "success" => @success,
+              "playbook_error" => @fail_message
+            },
+
+            "playbook" => {
+              "name" => @playbook_name,
+              "version" => @playbook_version
+            },
+
+            "inputs" => @inputs,
+            "nodes" => @nodes,
+            "tasks" => @tasks,
+            "metrics" => @metrics,
+            "logs" => @logs
+          }
+        end
+
+        def calculate_metrics
+          @metrics["end_time"] = Time.now.utc
+          @metrics["run_time"] = @metrics["end_time"] - @metrics["start_time"]
+          @metrics["task_count"] = @tasks.size
+
+          @tasks.each do |task|
+            @metrics["task_types"][task["type"]] ||= {
+              "count" => 0,
+              "total_time" => 0,
+              "pass" => 0,
+              "fail" => 0
+            }
+
+            metrics = @metrics["task_types"][task["type"]]
+
+            metrics["count"] += 1
+            metrics["total_time"] += task["run_time"]
+            task["success"] ? metrics["pass"] += 1 : metrics["fail"] += 1
+          end
+
+          @metrics
+        end
+
+        def store_task_results
+          @playbook.task_results.each do |result|
+            @tasks << {
+              "type" => result.task_type,
+              "set" => result.set,
+              "description" => result.description,
+              "start_time" => result.start_time.utc,
+              "end_time" => result.end_time.utc,
+              "run_time" => result.run_time,
+              "ran" => result.ran,
+              "msg" => result.msg,
+              "success" => result.success
+            }
+          end
+
+          @tasks
+        end
+
+        def store_playbook_metadata
+          @playbook_name = @playbook.name
+          @playbook_version = @playbook.version
+        end
+
+        def store_node_groups
+          @playbook.nodes.each do |key|
+            @nodes[key] = @playbook.discovered_nodes(key)
+          end
+        end
+
+        def store_dynamic_inputs
+          @inputs["dynamic"] = @playbook.dynamic_inputs
+        end
+
+        def store_static_inputs
+          @playbook.static_inputs.each do |key|
+            @inputs["static"][key] = @playbook.input_value(key)
+          end
+        end
+
+        def append_log(time, level, from, msg)
+          @logs << {
+            "time" => time.utc,
+            "level" => level.to_s,
+            "from" => from.strip,
+            "msg" => msg
+          }
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/task_result.rb

@@ -0,0 +1,55 @@
+module MCollective
+  module Util
+    class Playbook
+      class TaskResult
+        attr_accessor :success, :msg, :data, :ran, :task, :set, :start_time, :end_time, :description
+
+        def initialize(task)
+          @start_time = Time.now
+          @end_time = @start_time
+          @ran = false
+          @task = task
+          @description = task[:description]
+        end
+
+        def task_type
+          @task[:type]
+        end
+
+        def run_time
+          @end_time - @start_time
+        end
+
+        def success?
+          !!success
+        end
+
+        def timed_run(set)
+          @start_time = Time.now
+          @set = set
+
+          begin
+            @success, @msg, @data = task[:runner].run
+
+            if !@success && task[:properties]["fail_ok"] && task[:properties]["tries"] == 1
+              Log.warn("Task failed but fail_ok is true, treating as success")
+              @success = true
+            end
+          rescue
+            @success = false
+            @data = [$!]
+            @msg = "Running task %s failed unexpectedly: %s: %s" % [task.to_s, $!.class, $!.to_s]
+
+            Log.warn(@msg)
+            Log.debug($!.backtrace.join("\n\t"))
+          end
+
+          @end_time = Time.now
+          @ran = true
+
+          self
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/tasks.rb

@@ -0,0 +1,196 @@
+require_relative "task_result"
+require_relative "tasks/base"
+require_relative "tasks/graphite_event_task"
+require_relative "tasks/mcollective_task"
+require_relative "tasks/shell_task"
+require_relative "tasks/slack_task"
+require_relative "tasks/webhook_task"
+
+module MCollective
+  module Util
+    class Playbook
+      class Tasks
+        include TemplateUtil
+
+        attr_reader :tasks, :results
+
+        def initialize(playbook)
+          @playbook = playbook
+
+          reset
+        end
+
+        def reset
+          @results = []
+          @tasks = {
+            "tasks" => [],
+            "pre_task" => [],
+            "post_task" => [],
+            "on_fail" => [],
+            "on_success" => [],
+            "pre_book" => [],
+            "post_book" => []
+          }
+        end
+
+        def prepare; end
+
+        # @todo support types of runner
+        def runner_for(type)
+          klass_name = "%sTask" % type.capitalize
+
+          Tasks.const_get(klass_name).new(@playbook)
+        rescue NameError
+          raise("Cannot find a handler for Task type %s" % type)
+        end
+
+        # Runs a specific task
+        #
+        # @param task [Hash] a task entry
+        # @param set [String] the set the task is running in
+        # @param hooks [Boolean] indicates if hooks should be run
+        # @return [Boolean] indicating task success
+        def run_task(task, set, hooks=true)
+          properties = task[:properties]
+          result = task[:result]
+          task_runner = task[:runner]
+
+          Log.info("About to run task: %s" % t(task[:runner].description))
+          result.description = t(task[:runner].description)
+
+          if hooks && !run_set("pre_task")
+            Log.warn("Failing task because a critical pre_task hook failed")
+            return false
+          end
+
+          if properties["pre_sleep"]
+            Log.info("Sleeping %d seconds before check" % [properties["pre_sleep"]])
+            sleep(Integer(properties["pre_sleep"]))
+          end
+
+          (1..properties["tries"]).each do |try|
+            task_runner.from_hash(t(properties))
+            task_runner.validate_configuration!
+
+            @results << result.timed_run(set)
+
+            if result.success?
+              Log.info(result.msg)
+            else
+              Log.error(result.msg)
+            end
+
+            if try != properties["tries"] && !result.success
+              Log.warn("Task failed on try %d/%d, sleeping %ds: %s" % [try, properties["tries"], properties["try_sleep"], result.msg])
+              sleep(Integer(properties["try_sleep"]))
+            end
+
+            break if result.success
+          end
+
+          if hooks && !run_set("post_task")
+            Log.warn("Failing task because a critical post_task hook failed")
+            return false
+          end
+
+          result.success
+        end
+
+        # Runs a specific task set
+        #
+        # @param set [String] one of the known task sets
+        # @return [Boolean] true if all tasks and all their hooks passed
+        def run_set(set)
+          set_tasks = @tasks[set]
+
+          return true if set_tasks.empty?
+
+          @playbook.in_context(set) do
+            Log.info("About to run task set %s with %d task(s)" % [set, set_tasks.size])
+
+            # would typically use map here but you cant break out of a map and keep the thus far built up array
+            # so it's either this or a inject loop
+            passed = set_tasks.take_while do |task|
+              @playbook.in_context("%s.%s" % [set, task[:type]]) { run_task(task, set, set == "tasks") }
+            end
+
+            set_success = passed.size == set_tasks.size
+
+            Log.info("Done running task set %s with %d task(s): success: %s" % [set, set_tasks.size, set_success])
+
+            set_success
+          end
+        end
+
+        def run
+          @playbook.in_context("running") do
+            unless run_set("pre_book")
+              Log.warn("Playbook pre_book hook failed to run, failing entire playbook")
+              return false
+            end
+
+            success = run_set("tasks")
+
+            Log.info("Finished running main tasks in playbook: success: %s" % success)
+
+            if success
+              success = run_set("on_success")
+            else
+              run_set("on_fail")
+            end
+
+            unless run_set("post_book")
+              Log.warn("Playbook post_book hook failed to run, failing entire playbookbook")
+              return false
+            end
+
+            success
+          end
+        end
+
+        def load_tasks(data, set)
+          data.each_with_index do |task, idx|
+            task.each do |type, props|
+              Log.debug("Loading task %d of type %s into %s" % [idx, type, set])
+
+              runner = runner_for(type)
+              runner.description = props.fetch("description", "%s task" % [type])
+
+              task_data = {
+                :description => runner.description,
+                :type => type,
+                :runner => runner,
+                :properties => {
+                  "tries" => 1,
+                  "try_sleep" => 10,
+                  "fail_ok" => false
+                }.merge(props)
+              }
+
+              runner.fail_ok = task_data[:properties]["fail_ok"]
+
+              task_data[:result] = TaskResult.new(task_data)
+
+              @tasks[set] << task_data
+            end
+          end
+
+          @tasks[set]
+        end
+
+        def from_hash(data)
+          case data
+          when Array
+            load_tasks(data, "tasks")
+          when Hash
+            data.each do |set, tasks|
+              load_tasks(tasks, set)
+            end
+          end
+
+          self
+        end
+      end
+    end
+  end
+end