smart_proxy_reports 0.0.7 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: daed1707b3085291e478f53a98ac15c72a0bf8389b2d4843784c4945f1dcace9
-  data.tar.gz: d54be2907dcdfadbac97820804e8afb6c9c66b1b507c4199d2b6b65c6daed2e1
+  metadata.gz: ada1a0027fc55bf9192b9f31e6547173c9e491184274f79dc29664dae8b54dc7
+  data.tar.gz: 15a2bd27cd02863d15b3a1352cfdf183e10e02226cb37d90777f735a14b5adce
 SHA512:
-  metadata.gz: 744f8177feeb96b3da827a570c66827c49a31ddc1ec33995af7fd48c7ee6c42b54bf5fa7810cae9e871fb27273c44f189c8a628c26d0c5bca34e2f905fdbffd1
-  data.tar.gz: 392c1d166303bec98edc9cf138529815415b1e56f4659ab60cb99ec2e47b19eefe7ec50165345460166ad33ca6a1312d3e8eb538f898e8ecfe6803c08d04d972
+  metadata.gz: b3f70e46aa78232547ac0ef5016ba8777be07e1ee3484fd60cde4e3bcb11dcf7bade9ff3fe40615cb944b827a81e104a0331f42e0638c36d7c1ec8b0769309d9
+  data.tar.gz: 488cdacf8e8685b46cd1163f889f22903b27e815baee13d3a139eedc921bdc5a3f4c81fb510d3fe4135c65e1d07745c2458adc7e53d405ab71712a2946dfd6b9

data/README.md

@@ -26,15 +26,17 @@ Few words about setting up a dev setup.
 Checkoud foreman-ansible-modules and build it via `make` command. Configure
 Ansible collection path to the build directory:
 
-    [defaults]
-    collection_path = /home/lzap/work/foreman-ansible-modules/build
-    callback_whitelist = foreman
-    [callback_foreman]
-    report_type = proxy
-    proxy_url = http://localhost:8000/reports
-    verify_certs = 0
-    client_cert = /home/lzap/DummyX509/client-one.crt
-    client_key = /home/lzap/DummyX509/client-one.key
+```ini
+[defaults]
+collection_path = /home/lzap/work/foreman-ansible-modules/build
+callback_whitelist = foreman
+[callback_foreman]
+report_type = proxy
+proxy_url = http://localhost:8000/reports
+verify_certs = 0
+client_cert = /home/lzap/DummyX509/client-one.crt
+client_key = /home/lzap/DummyX509/client-one.key
+```
 
 Configure Foreman Ansible callback with the correct Foreman URL:
 
@@ -46,7 +48,7 @@ Then call Ansible:
 
 For testing, there are several example data. Before importing them into Foreman, make sure to have `localhost` smart proxy and also a host named `report.example.com`. It is possible to capture example data via `incoming_save_dir` setting. Name generated files correctly and put them into the `contrib/fixtures` directory. There is a utility to use fixtures for development and testing purposes:
 
-```
+```console
 $ contrib/upload-fixture
 Usage:
   contrib/upload-fixture -h               Display this help message
@@ -90,7 +92,7 @@ systemctl enable --now puppetserver
 ```
 
 If you prefer to use HTTPS, set the different reporturl and configure the CA certificates according to the example below
-```
+```bash
 # use HTTPS, without Katello the port is 8443, with Katello it's 9090
 puppet config set reporturl https://$HOSTNAME:8443/reports/puppet
 # install the Smart Proxy CA certificate to the Puppet's localcacert store

data/lib/smart_proxy_reports/ansible_processor.rb

@@ -1,10 +1,10 @@
-require_relative "friendly_message"
-
 # frozen_string_literal: true
 
+require_relative "friendly_message"
+
 module Proxy::Reports
   class AnsibleProcessor < Processor
-    KEYS_TO_COPY = %w[status check_mode].freeze
+    KEYS_TO_COPY = %w[check_mode].freeze
 
     def initialize(data, json_body: true)
       super(data, json_body: json_body)
@@ -12,6 +12,9 @@ module Proxy::Reports
         @data = JSON.parse(data)
       end
       @body = {}
+      @failure = 0
+      @change = 0
+      @nochange = 0
       logger.debug "Processing report #{report_id}"
       debug_payload("Input", @data)
     end
@@ -20,36 +23,50 @@ module Proxy::Reports
       @data["uuid"] || generated_report_id
     end
 
+    def count_summary(result)
+      if result["result"]["changed"]
+        @change += 1
+      else
+        @nochange += 1
+      end
+      if result["failed"]
+        @failure += 1
+      end
+    end
+
     def process_results
       @data["results"]&.each do |result|
         raise("Report do not contain required 'results/result' element") unless result["result"]
         raise("Report do not contain required 'results/task' element") unless result["task"]
-        process_facts(result)
         process_level(result)
         friendly_message = FriendlyMessage.new(result)
         result["friendly_message"] = friendly_message.generate_message
         process_keywords(result)
+        count_summary(result)
       end
       @data["results"]
     rescue StandardError => e
-      logger.error "Unable to parse results", e
+      log_error("Unable to parse results", e)
       @data["results"]
     end
 
     def process
-      measure :process do
-        @body["format"] = "ansible"
-        @body["id"] = report_id
-        @body["host"] = hostname_from_config || @data["host"]
-        @body["proxy"] = Proxy::Reports::Plugin.settings.reported_proxy_hostname
-        @body["reported_at"] = @data["reported_at"]
+      @body["format"] = "ansible"
+      @body["id"] = report_id
+      @body["host"] = hostname_from_config || @data["host"]
+      @body["proxy"] = Proxy::Reports::Plugin.settings.reported_proxy_hostname
+      @body["reported_at"] = @data["reported_at"]
+      @body["reported_at_proxy"] = now_utc
+      measure :process_results do
         @body["results"] = process_results
-        @body["keywords"] = keywords
-        @body["telemetry"] = telemetry
-        @body["errors"] = errors if errors?
-        KEYS_TO_COPY.each do |key|
-          @body[key] = @data[key]
-        end
+      end
+      @body["summary"] = build_summary
+      process_root_keywords
+      @body["keywords"] = keywords
+      @body["telemetry"] = telemetry
+      @body["errors"] = errors if errors?
+      KEYS_TO_COPY.each do |key|
+        @body[key] = @data[key]
       end
     end
 
@@ -62,36 +79,121 @@ module Proxy::Reports
         format: "ansible",
         version: 1,
         host: @body["host"],
-        reported_at: @body["reported_at"],
-        statuses: process_statuses,
         proxy: @body["proxy"],
-        body: @body,
+        change: @body["summary"]["foreman"]["change"],
+        nochange: @body["summary"]["foreman"]["nochange"],
+        failure: @body["summary"]["foreman"]["failure"],
         keywords: @body["keywords"],
+        body: @body,
       )
     end
 
     def spool_report
+      facts_hash = measure :build_facts do
+        build_facts
+      end
+      if facts_hash
+        debug_payload("Facts output", facts_hash)
+        payload = measure :format_facts do
+          facts_hash.to_json
+        end
+        SpooledHttpClient.instance.spool(:ansible_facts, payload)
+      end
+
       report_hash = build_report
       debug_payload("Output", report_hash)
       payload = measure :format do
         report_hash.to_json
       end
-      SpooledHttpClient.instance.spool(report_id, payload)
+      SpooledHttpClient.instance.spool(:report, payload)
+    end
+
+    def find_facts_task
+      @data["results"]&.each do |result|
+        if result["result"] && result["result"]["ansible_facts"]
+          return result["result"]["ansible_facts"]
+        end
+      end
+      false
+    end
+
+    def build_facts
+      facts = find_facts_task
+      return nil unless facts
+      {
+        "name" => hostname_from_config || @data["host"],
+        "facts" => {
+          "ansible_facts" => facts,
+          "_type" => "ansible",
+          "_timestamp" => @data["reported_at"],
+        },
+      }
     end
 
     private
 
-    def process_facts(result)
-      # TODO: add fact processing and sending to the fact endpoint
-      result["result"]["ansible_facts"] = {}
+    # foreman-ansible-modules 3.0 does not contain summary field, convert it here
+    # https://github.com/theforeman/foreman-ansible-modules/pull/1325/files
+    def build_summary
+      if @data["summary"]
+        native = @data["summary"]
+      elsif (status = @data["status"])
+        native = {
+          "changed" => status["applied"] || 0,
+          "failures" => status["failed"] || 0,
+          "ignored" => 0,
+          "ok" => 0,
+          "rescued" => 0,
+          "skipped" => status["skipped"] || 0,
+          "unreachable" => 0,
+        }
+      else
+        native = {}
+      end
+      {
+        "foreman" => {
+          "change" => @change, "nochange" => @nochange, "failure" => @failure,
+        },
+        "native" => native,
+      }
+    rescue StandardError => e
+      log_error("Unable to build summary", e)
+      {
+        "foreman" => {
+          "change" => @change, "nochange" => @nochange, "failure" => @failure,
+        },
+        "native" => {},
+      }
+    end
+
+    def process_root_keywords
+      if (summary = @body["summary"])
+        if summary["changed"] && summary["changed"] > 0
+          add_keywords("AnsibleChanged")
+        elsif summary["failures"] && summary["failures"] > 0
+          add_keywords("AnsibleFailures")
+        elsif summary["unreachable"] && summary["unreachable"] > 0
+          add_keywords("AnsibleUnreachable")
+        elsif summary["rescued"] && summary["rescued"] > 0
+          add_keywords("AnsibleRescued")
+        elsif summary["ignored"] && summary["ignored"] > 0
+          add_keywords("AnsibleIgnored")
+        elsif summary["skipped"] && summary["skipped"] > 0
+          add_keywords("AnsibleSkipped")
+        end
+      end
+    rescue StandardError => e
+      log_error("Unable to parse root summary keywords", e)
     end
 
     def process_keywords(result)
       if result["failed"]
-        add_keywords("HasFailure", "AnsibleTaskFailed:#{result["task"]["action"]}")
+        add_keywords("AnsibleFailure", "AnsibleFailure:#{result["task"]["action"]}")
       elsif result["result"]["changed"]
-        add_keywords("HasChange")
+        add_keywords("AnsibleChanged")
       end
+    rescue StandardError => e
+      log_error("Unable to parse keywords", e)
     end
 
     def process_level(result)
@@ -102,18 +204,19 @@ module Proxy::Reports
       else
         result["level"] = "info"
       end
+    rescue StandardError => e
+      log_error("Unable to parse log level", e)
+      result["level"] = "info"
     end
+  end
 
-    def process_statuses
-      {
-        "applied" => @body["status"]["applied"],
-        "failed" => @body["status"]["failed"],
-        "pending" => @body["status"]["pending"] || 0, # It's only present in check mode
-        "other" => @body["status"]["skipped"],
-      }
-    rescue StandardError => e
-      logger.error "Unable to process statuses", e
-      { "applied" => 0, "failed" => 0, "pending" => 0, "other" => 0 }
+  def search_for_facts2222222222222222(result)
+    if result.respond_to?(:key?) && result.key?(:ansible_facts)
+      result[:ansible_facts]
+    elsif result.respond_to?(:each)
+      r = nil
+      result.find { |*a| r = search_for_facts(a.last) }
+      r
     end
   end
 end

data/lib/smart_proxy_reports/friendly_message.rb

@@ -13,8 +13,11 @@ class FriendlyMessage
 
     case @task_tree["action"]
     when "ansible.builtin.package", "package" then msg = package_message
+    when "ansible.builtin.known_hosts", "known_hosts" then msg = known_hosts_message
+    when "ansible.builtin.pip", "pip" then msg = pip_message
     when "ansible.builtin.template", "template" then msg = template_message
     when "ansible.builtin.service", "service" then msg = service_message
+    when "ansible.builtin.unarchive", "unarchive" then msg = unarchive_message
     when "ansible.builtin.group", "group" then msg = group_message
     when "ansible.builtin.user", "user" then msg = user_message
     when "ansible.builtin.cron", "cron" then msg = cron_message
@@ -40,6 +43,15 @@ class FriendlyMessage
     "Package(s) #{packages} are #{state}"
   end
 
+  def known_hosts_message
+    "#{@module_args_tree["name"]} is #{@module_args_tree["state"]} in #{@module_args_tree["path"]}"
+  end
+
+  def pip_message
+    packages = human_readable_array(@module_args_tree["name"]) || "contained in #{@module_args_tree["requirements"]}"
+    "Package(s) #{packages} are #{@module_args_tree["state"]}"
+  end
+
   def template_message
     "Render template #{@module_args_tree["_original_basename"]} to #{@result_tree["dest"]}"
   end
@@ -48,6 +60,10 @@ class FriendlyMessage
     "Service #{@result_tree["name"]} is #{@result_tree["state"]} and enabled: #{@result_tree["enabled"]}"
   end
 
+  def unarchive_message
+    "Archive #{@module_args_tree["src"]} unpacked into #{@module_args_tree["dest"]}"
+  end
+
   def group_message
     "User group #{@result_tree["name"]} is #{@result_tree["state"]} with gid: #{@result_tree["gid"]}"
   end

data/lib/smart_proxy_reports/processor.rb

@@ -31,19 +31,26 @@ module Proxy::Reports
       @hostname_from_config ||= Proxy::Reports::Plugin.settings.override_hostname
     end
 
-    def build_report_root(format:, version:, host:, reported_at:, statuses:, proxy:, body:, keywords:)
+    def now_utc
+      # make sure it contains TZ info: "2022-01-20 14:16:26 UTC"
+      Time.now.utc.to_s
+    end
+
+    def build_report_root(format:, version:, host:, proxy:, change:, nochange:, failure:, body:, keywords:)
       {
         "host_report" => {
           "format" => format,
           "version" => version,
           "host" => host,
-          "reported_at" => reported_at,
+          "reported_at" => now_utc,
           "proxy" => proxy,
           "body" => @json_body ? body.to_json : body,
           "keywords" => keywords,
-        }.merge(statuses),
+          "change" => change,
+          "nochange" => nochange,
+          "failure" => failure,
+        },
       }
-      # TODO add metric with total time
     end
 
     def debug_payload?
@@ -67,8 +74,14 @@ module Proxy::Reports
 
     attr_reader :errors
 
-    def log_error(message)
-      @errors << message.to_s
+    def log_error(message, exception = nil)
+      msg = if exception
+          "#{message}: #{exception}"
+        else
+          message
+        end
+      logger.error msg, exception
+      @errors << msg.to_s
     end
 
     def errors?

data/lib/smart_proxy_reports/puppet_processor.rb

@@ -34,37 +34,46 @@ module Proxy::Reports
       end
       logs
     rescue StandardError => e
-      logger.error "Unable to parse logs", e
+      log_error("Unable to parse logs", e)
       logs
     end
 
+    def process_root_keywords
+      status = @data["status"]
+      if status == "changed"
+        add_keywords("PuppetStatusChanged")
+      elsif status == "unchanged"
+        add_keywords("PuppetStatusUnchanged")
+      elsif status == "failed"
+        add_keywords("PuppetStatusFailed")
+      end
+      if @data["noop"] == "true"
+        add_keywords("PuppetNoop")
+      end
+      if @data["noop_pending"] == "true"
+        add_keywords("PuppetNoopPending")
+      end
+    rescue StandardError => e
+      log_error("Unable to parse root keywords", e)
+    end
+
     def process_resource_statuses
       statuses = []
       @data["resource_statuses"]&.each_pair do |key, value|
         statuses << key
         @evaluation_times << [key, value["evaluation_time"]]
-        # failures
-        add_keywords("PuppetResourceFailed:#{key}", "PuppetHasFailure") if value["failed"] || value["failed_to_restart"]
-        value["events"]&.each do |event|
-          add_keywords("PuppetResourceFailed:#{key}", "PuppetHasFailure") if event["status"] == "failed"
-          add_keywords("PuppetHasCorrectiveChange") if event["corrective_change"]
-        end
-        # changes
-        add_keywords("PuppetHasChange") if value["changed"]
-        add_keywords("PuppetHasChange") if value["change_count"] && value["change_count"] > 0
-        # changes
-        add_keywords("PuppetIsOutOfSync") if value["out_of_sync"]
-        add_keywords("PuppetIsOutOfSync") if value["out_of_sync_count"] && value["out_of_sync_count"] > 0
-        # skips
-        add_keywords("PuppetHasSkips") if value["skipped"]
-        # corrective changes
-        add_keywords("PuppetHasCorrectiveChange") if value["corrective_change"]
-        # others
+        add_keywords("PuppetFailed:#{key}", "PuppetFailed") if value["failed"]
+        add_keywords("PuppetFailedToRestart:#{key}", "PuppetFailedToRestart") if value["failed_to_restart"]
+        add_keywords("PuppetCorrectiveChange") if value["corrective_change"]
+        add_keywords("PuppetSkipped") if value["skipped"]
+        add_keywords("PuppetRestarted") if value["restarted"]
+        add_keywords("PuppetScheduled") if value["scheduled"]
+        add_keywords("PuppetOutOfSync") if value["out_of_sync"]
         add_keywords("PuppetEnvironment:#{@data["environment"]}") if @data["environment"]
       end
       statuses
     rescue StandardError => e
-      logger.error "Unable to parse resource_statuses", e
+      log_error("Unable to parse resource_statuses", e)
       statuses
     end
 
@@ -77,27 +86,36 @@ module Proxy::Reports
       end
       @evaluation_times
     rescue StandardError => e
-      logger.error "Unable to process evaluation_times", e
+      log_error("Unable to parse evaluation_times", e)
       []
     end
 
     def process
-      measure :process do
-        @body["format"] = "puppet"
-        @body["id"] = report_id
-        @body["host"] = hostname_from_config || @data["host"]
-        @body["proxy"] = Proxy::Reports::Plugin.settings.reported_proxy_hostname
-        @body["reported_at"] = @data["time"]
-        KEYS_TO_COPY.each do |key|
-          @body[key] = @data[key]
-        end
+      @body["format"] = "puppet"
+      @body["id"] = report_id
+      @body["host"] = hostname_from_config || @data["host"]
+      @body["proxy"] = Proxy::Reports::Plugin.settings.reported_proxy_hostname
+      @body["reported_at"] = @data["time"]
+      @body["reported_at_proxy"] = now_utc
+      KEYS_TO_COPY.each do |key|
+        @body[key] = @data[key]
+      end
+      process_root_keywords
+      measure :process_logs do
         @body["logs"] = process_logs
+      end
+      measure :process_resource_statuses do
         @body["resource_statuses"] = process_resource_statuses
-        @body["keywords"] = keywords
+      end
+      measure :process_summary do
+        @body["summary"] = process_summary
+      end
+      measure :process_evaluation_times do
         @body["evaluation_times"] = process_evaluation_times
-        @body["telemetry"] = telemetry
-        @body["errors"] = errors if errors?
       end
+      @body["telemetry"] = telemetry
+      @body["keywords"] = keywords
+      @body["errors"] = errors if errors?
     end
 
     def build_report
@@ -109,11 +127,12 @@ module Proxy::Reports
         format: "puppet",
         version: 1,
         host: @body["host"],
-        reported_at: @body["reported_at"],
-        statuses: process_statuses,
         proxy: @body["proxy"],
-        body: @body,
+        change: @body["summary"]["foreman"]["change"],
+        nochange: @body["summary"]["foreman"]["nochange"],
+        failure: @body["summary"]["foreman"]["failure"],
         keywords: @body["keywords"],
+        body: @body,
       )
     end
 
@@ -123,22 +142,32 @@ module Proxy::Reports
       payload = measure :format do
         report_hash.to_json
       end
-      SpooledHttpClient.instance.spool(report_id, payload)
+      SpooledHttpClient.instance.spool(:report, payload)
     end
 
     private
 
-    def process_statuses
-      stats = @body["metrics"]["resources"]["values"].collect { |s| [s[0], s[2]] }.to_h
+    def process_summary
+      events = @body["metrics"]["events"]["values"].collect { |s| [s[0], s[2]] }.to_h
+      resources = @body["metrics"]["resources"]["values"].collect { |s| [s[0], s[2]] }.to_h
       {
-        "applied" => stats["changed"] + stats["corrective_change"],
-        "failed" => stats["failed"] + stats["failed_to_restart"],
-        "pending" => stats["scheduled"],
-        "other" => stats["restarted"] + stats["skipped"] + stats["out_of_sync"],
+        "foreman" => {
+          "change" => events["success"],
+          "nochange" => resources["total"] - events["failure"] - events["success"],
+          "failure" => events["failure"],
+        },
+        "native" => resources,
       }
     rescue StandardError => e
-      logger.error "Unable to process statuses", e
-      { "applied" => 0, "failed" => 0, "pending" => 0, "other" => 0 }
+      log_error("Unable to parse summary counts", e)
+      {
+        "foreman" => {
+          "change" => 0,
+          "nochange" => 0,
+          "failure" => 0,
+        },
+        "native" => {},
+      }
     end
   end
 end

data/lib/smart_proxy_reports/reports_api.rb

@@ -47,6 +47,7 @@ module Proxy::Reports
       log_halt(415, "Missing body") if input.empty?
       json_body = to_bool(params[:json_body], true)
       processor = Processor.new_processor(format, input, json_body: json_body)
+      status 202
       processor.spool_report
     rescue => e
       log_halt 415, e, "Error during report processing: #{e.message}"

data/lib/smart_proxy_reports/spooled_http_client.rb

@@ -52,6 +52,14 @@ module Proxy::Reports
       @worker.join
     end
 
+    def get_endpoint(name)
+      if name&.end_with? 'ansible_facts'
+        "/api/v2/hosts/facts"
+      else
+        "/api/v2/host_reports"
+      end
+    end
+
     def process
       processed = 0
       client = ::Proxy::HttpRequest::ForemanRequest.new
@@ -59,11 +67,12 @@ module Proxy::Reports
       # send all files via a single persistent HTTP connection
       logger.debug "Opening HTTP connection to Foreman"
       client.http.start do |http|
-        Dir.glob(spool_path("todo", "*")) do |filename|
+        Dir.glob(spool_path("todo", "*")).sort.each do |filename|
           basename = File.basename(filename)
+          endpoint = get_endpoint(basename)
           logger.debug "Sending report #{basename}"
           begin
-            post = factory.create_post("/api/v2/host_reports", File.read(filename))
+            post = factory.create_post(endpoint, File.read(filename))
             response = http.request(post)
             logger.info "Report #{basename} sent with HTTP response #{response.code}"
             logger.debug { "Response body: #{response.body}" }
@@ -74,7 +83,7 @@ module Proxy::Reports
                 FileUtils.rm_f spool_path("todo", basename)
               end
             else
-              logger.debug { "Moving failed report to 'fail' spool directory" }
+              logger.debug { "Moving failed report to 'fail' spool directory" }
               spool_move("todo", "done", basename)
             end
             processed += 1
@@ -91,12 +100,20 @@ module Proxy::Reports
       @worker.wakeup if @worker
     end
 
-    def spool(filename, data)
-      filename = filename.gsub(/[^0-9a-z]/i, "")
+    def spool(prefix, data)
+      filename = unique_filename + "-" + prefix.to_s
       file = spool_path("temp", filename)
       File.open(file, "w") { |f| f.write(data) }
       spool_move("temp", "todo", filename)
       wakeup
     end
+
+    private
+
+    # Ensure that files are named so they can be sorted and processed in the same order
+    def unique_filename
+      Process.clock_gettime(Process::CLOCK_REALTIME, :second).to_s(36) +
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond).to_s(36)
+    end
   end
 end

data/lib/smart_proxy_reports/version.rb

@@ -1,5 +1,5 @@
 module Proxy
   module Reports
-    VERSION = "0.0.7"
+    VERSION = "1.0.0"
   end
 end

data/settings.d/reports.yml.example

@@ -16,7 +16,9 @@
 :keep_reports: false
 
 # Development settings (do not use)
+#
 # Override hostnames of incoming reports
 #:override_hostname: report.example.com
+#
 # Store input payloads in a directory
 #:incoming_save_dir: /var/lib/foreman-proxy/reports/incoming

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_reports
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 1.0.0
 platform: ruby
 authors:
 - Lukas Zapletal
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2022-02-16 00:00:00.000000000 Z
 dependencies: []
 description: Transform and upload Foreman host reports via REST API
 email: lukas-x@zapletalovi.com