foreman_rh_cloud 3.0.23 → 3.0.26

data/config/routes.rb

@@ -37,7 +37,7 @@ Rails.application.routes.draw do
     end
 
     scope '/r/insights' do
-      match '/*path', :constraints => lambda { |req| !req.path.include?('view/api') }, to: 'machine_telemetries#forward_request', via: [:get, :post, :delete,:put, :patch]
+      match '(/*path)(/)', :constraints => lambda { |req| !req.path.include?('view/api') }, to: 'machine_telemetries#forward_request', via: [:get, :post, :delete,:put, :patch]
     end
   end
 

data/db/migrate/20210720000001_remove_old_insights_statuses.foreman_rh_cloud.rb

@@ -0,0 +1,6 @@
+class RemoveOldInsightsStatuses < ActiveRecord::Migration[5.2]
+  def up
+    InsightsClientReportStatus.where(status: 2).update_all(status: InsightsClientReportStatus::NO_REPORT)
+    InsightsClientReportStatus.where(status: 3).update_all(status: InsightsClientReportStatus::REPORTING)
+  end
+end

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -27,6 +27,7 @@ module ForemanInventoryUpload
               'insights_client::obfuscate_ip_enabled',
               'insights_client::hostname',
               'insights_client::ips',
+              'insights_id',
             ]).pluck(:name, :id)
           ]
       end

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -49,6 +49,7 @@ module ForemanInventoryUpload
           @stream.simple_field('satellite_id', uuid_value!(host.subscription_facet&.uuid))
           @stream.simple_field('bios_uuid', bios_uuid(host))
           @stream.simple_field('vm_uuid', uuid_value(fact_value(host, 'virt::uuid')))
+          @stream.simple_field('insights_id', uuid_value(fact_value(host, 'insights_id')))
           report_ip_addresses(host, host_ips_cache)
           report_mac_addresses(host)
           @stream.object_field('system_profile') do

data/lib/foreman_inventory_upload/generators/tags.rb

@@ -19,7 +19,9 @@ module ForemanInventoryUpload
       def generate_parameters
         return [] unless Setting[:include_parameter_tags]
 
-        (@host.host_inherited_params_objects || []).map { |item| [item.name, item.value] }
+        (@host.host_inherited_params_objects || [])
+          .map { |item| [item.name, item.value] }
+          .select { |_name, value| value.present? || value.is_a?(FalseClass) }
       end
 
       private

data/lib/foreman_inventory_upload.rb

@@ -70,8 +70,16 @@ module ForemanInventoryUpload
     folder
   end
 
+  def self.inventory_base_url
+    ForemanRhCloud.base_url + "/api/inventory/v1/hosts"
+  end
+
   def self.inventory_export_url
     tags = URI.encode("satellite/satellite_instance_id=#{Foreman.instance_id}")
-    ForemanRhCloud.base_url + "/api/inventory/v1/hosts?tags=#{tags}"
+    inventory_base_url + "?tags=#{tags}"
+  end
+
+  def self.inventory_self_url
+    inventory_base_url + "?hostname_or_id=#{ForemanRhCloud.foreman_host.fqdn}"
   end
 end

data/lib/foreman_rh_cloud/engine.rb

@@ -6,6 +6,18 @@ module ForemanRhCloud
   class Engine < ::Rails::Engine
     engine_name 'foreman_rh_cloud'
 
+    def self.register_scheduled_task(task_class, cronline)
+      return if ForemanTasks::RecurringLogic.joins(:tasks)
+                .merge(ForemanTasks::Task.where(label: task_class.name))
+                .exists?
+
+      User.as_anonymous_admin do
+        recurring_logic = ForemanTasks::RecurringLogic.new_from_cronline(cronline)
+        recurring_logic.save!
+        recurring_logic.start(task_class)
+      end
+    end
+
     initializer 'foreman_rh_cloud.load_default_settings', :before => :load_config_initializers do
       require_dependency File.expand_path('../../app/models/setting/rh_cloud.rb', __dir__)
     end
@@ -81,8 +93,6 @@ module ForemanRhCloud
         register_custom_status InventorySync::InventoryStatus
         register_custom_status InsightsClientReportStatus
 
-        subscribe 'host_created.event.foreman', ForemanRhCloud::InsightsSubscriber
-
         describe_host do
           overview_buttons_provider :insights_host_overview_buttons
         end
@@ -138,15 +148,8 @@ module ForemanRhCloud
         # skip object creation when admin user is not present, for example in test DB
         if User.unscoped.find_by_login(User::ANONYMOUS_ADMIN).present?
           ::ForemanTasks.dynflow.config.on_init(false) do |world|
-            unless ForemanTasks::RecurringLogic.joins(:tasks).merge(
-              ForemanTasks::Task.where(label: 'InventorySync::Async::InventoryScheduledSync')
-            ).exists?
-              User.as_anonymous_admin do
-                recurring_logic = ForemanTasks::RecurringLogic.new_from_cronline("0 0 * * *")
-                recurring_logic.save!
-                recurring_logic.start(InventorySync::Async::InventoryScheduledSync)
-              end
-            end
+            ForemanRhCloud::Engine.register_scheduled_task(InventorySync::Async::InventoryScheduledSync, '0 0 * * *')
+            ForemanRhCloud::Engine.register_scheduled_task(InsightsCloud::Async::InsightsClientStatusAging, '0 0 * * *')
           end
         end
       end

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '3.0.23'.freeze
+  VERSION = '3.0.26'.freeze
 end

data/lib/insights_cloud/async/insights_client_status_aging.rb

@@ -0,0 +1,23 @@
+module InsightsCloud
+  module Async
+    class InsightsClientStatusAging < ::Actions::EntryAction
+      include ::Actions::RecurringAction
+
+      def run
+        host_ids = InsightsClientReportStatus.stale.reporting.pluck(:host_id)
+
+        # update all stale records
+        InsightsClientReportStatus.where(host_id: host_ids).update_all(status: InsightsClientReportStatus::NO_REPORT)
+
+        # refresh global status
+        Host.where(id: host_ids).preload(:host_statuses).find_in_batches do |hosts|
+          hosts.each { |host| host.refresh_global_status! }
+        end
+      end
+
+      def logger
+        action_logger
+      end
+    end
+  end
+end

data/lib/insights_cloud/async/insights_full_sync.rb

@@ -6,6 +6,11 @@ module InsightsCloud
       include ::ForemanRhCloud::CloudAuth
 
       def plan
+        unless cloud_auth_available?
+          logger.debug('Cloud authentication is not available, skipping insights sync')
+          return
+        end
+
         sequence do
           # This can be turned off when we enable automatic status syncs
           # This step will query cloud inventory to retrieve inventory uuids for each host

data/lib/insights_cloud/async/insights_resolutions_sync.rb

@@ -7,11 +7,21 @@ module InsightsCloud
 
       RULE_ID_REGEX = /[^:]*:(?<id>.*)/
 
+      def plan
+        unless cloud_auth_available?
+          logger.debug('Cloud authentication is not available, skipping resolutions sync')
+          return
+        end
+
+        plan_self
+      end
+
       def run
         InsightsResolution.transaction do
           InsightsResolution.delete_all
-          api_response = query_insights_resolutions(relevant_rules)
-          write_resolutions(api_response)
+          rule_ids = relevant_rules
+          api_response = query_insights_resolutions(rule_ids) unless rule_ids.empty?
+          write_resolutions(api_response) if api_response
         end
       end
 

data/lib/insights_cloud/async/insights_rules_sync.rb

@@ -6,8 +6,17 @@ module InsightsCloud
       include ::ForemanRhCloud::CloudAuth
 
       def plan
-        plan_self
-        plan_resolutions
+        unless cloud_auth_available?
+          logger.debug('Cloud authentication is not available, skipping rules sync')
+          return
+        end
+
+        # since the tasks are not connected, we need to force sequence execution here
+        # to make sure we don't run resolutions until we synced all our rules
+        sequence do
+          plan_self
+          plan_resolutions
+        end
       end
 
       def plan_resolutions

data/lib/inventory_sync/async/host_result.rb

@@ -1,6 +1,8 @@
 module InventorySync
   module Async
     class HostResult
+      attr_reader :uuid_by_fqdn
+
       def initialize(result)
         @total = result['total']
         @count = result['count']
@@ -8,6 +10,7 @@ module InventorySync
         @per_page = result['per_page']
         @sub_ids = result["results"].map { |host| host['subscription_manager_id'] }
         @uuid_by_sub_id = Hash[result["results"].map { |host| [host['subscription_manager_id'], host['id']] }]
+        @uuid_by_fqdn = Hash[result["results"].map { |host| [host['fqdn'].downcase, host['id']] }]
       end
 
       def status_hashes
@@ -19,6 +22,7 @@ module InventorySync
               host_id: host_id,
               status: InventorySync::InventoryStatus::SYNC,
               reported_at: DateTime.current,
+              type: InventorySync::InventoryStatus.name,
             }
           end
         end.compact

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -5,6 +5,11 @@ module InventorySync
       set_callback :step, :around, :update_statuses_batch
 
       def plan(organization)
+        unless cloud_auth_available?
+          logger.debug('Cloud authentication is not available, skipping inventory hosts sync')
+          return
+        end
+
         plan_self(organization_id: organization.id)
       end
 

data/lib/inventory_sync/async/inventory_hosts_sync.rb

@@ -4,6 +4,17 @@ module InventorySync
       set_callback :iteration, :around, :setup_facet_transaction
       set_callback :step, :around, :create_facets
 
+      def plan
+        unless cloud_auth_available?
+          logger.debug('Cloud authentication is not available, skipping inventory hosts sync')
+          return
+        end
+
+        # by default the tasks will be executed concurrently
+        plan_self
+        plan_self_host_sync
+      end
+
       def setup_facet_transaction
         InsightsFacet.transaction do
           yield
@@ -33,6 +44,10 @@ module InventorySync
           InsightsFacet.where(host_id: host_id).update_all(uuid: uuids_hash[host_id])
         end
       end
+
+      def plan_self_host_sync
+        plan_action InventorySync::Async::InventorySelfHostSync
+      end
     end
   end
 end

data/lib/inventory_sync/async/inventory_self_host_sync.rb

@@ -0,0 +1,39 @@
+module InventorySync
+  module Async
+    class InventorySelfHostSync < QueryInventoryJob
+      set_callback :step, :around, :create_facets
+
+      def plan
+        unless cloud_auth_available?
+          logger.debug('Cloud authentication is not available, skipping self host sync')
+          return
+        end
+
+        plan_self
+      end
+
+      def create_facets
+        # get the results from the event
+        results = yield
+
+        add_missing_insights_facet(results.uuid_by_fqdn) unless results.uuid_by_fqdn.empty?
+        results
+      end
+
+      private
+
+      def add_missing_insights_facet(uuids_hash)
+        facet = InsightsFacet.find_or_create_by(host_id: ForemanRhCloud.foreman_host.id) do |facet|
+          facet.uuid = uuids_hash.values.first
+        end
+
+        # fix empty uuid in case the facet already exists
+        facet.update(uuid: uuids_hash.values.first) unless facet.uuid
+      end
+
+      def request_url
+        ForemanInventoryUpload.inventory_self_url
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/query_inventory_job.rb

@@ -31,7 +31,7 @@ module InventorySync
       def query_inventory(page = 1)
         hosts_inventory_response = execute_cloud_request(
           method: :get,
-          url: ForemanInventoryUpload.inventory_export_url,
+          url: request_url,
           headers: {
             params: {
               per_page: 100,
@@ -46,6 +46,10 @@ module InventorySync
       def logger
         action_logger
       end
+
+      def request_url
+        ForemanInventoryUpload.inventory_export_url
+      end
     end
   end
 end

data/lib/tasks/insights.rake

@@ -4,4 +4,19 @@ namespace :rh_cloud_insights do
     ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync)
     puts "Synchronized Insights hosts hits data"
   end
+
+  desc "Remove insights client report statuses by searching on host criteria"
+  task clean_statuses: [:environment] do
+    hosts_search = ENV['SEARCH']
+
+    if hosts_search.empty?
+      puts 'Must specify SEARCH= criteria for hosts search'
+      next
+    end
+
+    cleaner = ForemanRhCloud::InsightsStatusCleaner.new
+    deleted_count = cleaner.clean(hosts_search)
+
+    puts "Deleted #{deleted_count} insights statuses"
+  end
 end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "3.0.23",
+  "version": "3.0.26",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {

data/test/controllers/insights_cloud/api/machine_telemetries_controller_test.rb

@@ -11,6 +11,39 @@ module InsightsCloud::Api
         host = FactoryBot.create(:host, :with_subscription, :organization => org)
         User.current = ::Katello::CpConsumerUser.new(:uuid => host.subscription_facet.uuid, :login => host.subscription_facet.uuid)
         InsightsCloud::Api::MachineTelemetriesController.any_instance.stubs(:upstream_owner).returns({ 'uuid' => 'abcdefg' })
+
+        @cert1 = "-----BEGIN CERTIFICATE-----\r\n" +
+        "MIIFdDCCA1ygAwIBAgIJAM5Uqykb3EAtMA0GCSqGSIb3DQEBCwUAME8xCzAJBgNV\r\n" +
+        "BAYTAklMMREwDwYDVQQIDAhUZWwgQXZpdjEUMBIGA1UECgwLVGhlIEZvcmVtYW4x\r\n" +
+        "FzAVBgNVBAMMDnRoZWZvcmVtYW4ub3JnMB4XDTE4MDMyNDEyMzYyOFoXDTI4MDMy\r\n" +
+        "MTEyMzYyOFowTzELMAkGA1UEBhMCSUwxETAPBgNVBAgMCFRlbCBBdml2MRQwEgYD\r\n" +
+        "VQQKDAtUaGUgRm9yZW1hbjEXMBUGA1UEAwwOdGhlZm9yZW1hbi5vcmcwggIiMA0G\r\n" +
+        "CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDF04/s4h+BgHPG1HDZ/sDlYq925pkc\r\n" +
+        "RTVAfnE2EXDAmZ6W4Q9ueDY65MHe3ZWO5Dg72kNSP2sK9kRI7Dk5CAFOgyw1rH8t\r\n" +
+        "Hd1+0xp/lv6e4SvSYghxIL68vFe0ftKkm1usqejBM5ZTgKr7JCI+XSIN36F65Kde\r\n" +
+        "c+vxwBnayuhP04r9/aaE/709SXML4eRVYW8I3qFy9FPtUOm+bY8U2PIv5fHayqbG\r\n" +
+        "cL/4t3+MCtMhHJsLzdBXya+1P5t+HcKjUNlmwoUF961YAktVuEFloGd0RMRlqF3/\r\n" +
+        "itU3QNlXgA5QBIciE5VPr/PiqgMC3zgd5avjF4OribZ+N9AATLiQMW78il5wSfcc\r\n" +
+        "kQjU9ChOLrzku455vQ8KE4bc0qvpCWGfUah6MvL9JB+TQkRl/8kxl0b9ZinIvJDH\r\n" +
+        "ynVMb4cB/TDEjrjOfzn9mWLH0ZJqjmc2bER/G12WQxOaYLxdVwRStD3Yh6PtiFWu\r\n" +
+        "sXOk19UOTVkeuvGFVtvzLfEwQ1lDEo7+VBQz8FG/HBu2Hpq3IwCFrHuicikwjQJk\r\n" +
+        "nfturgD0rBOKEc1qWNZRCvovYOLL6ihvv5Orujsx5ZCHOAtnVNxkvIlFt2RS45LF\r\n" +
+        "MtPJyhAc6SjitllfUEirxprsbmeSZqrIfzcGaEhgOSnyik1WMv6bYiqPfBg8Fzjh\r\n" +
+        "vOCbtiDNPmvgOwIDAQABo1MwUTAdBgNVHQ4EFgQUtkAgQopsTtG9zSG3MgW2IxHD\r\n" +
+        "MDwwHwYDVR0jBBgwFoAUtkAgQopsTtG9zSG3MgW2IxHDMDwwDwYDVR0TAQH/BAUw\r\n" +
+        "AwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAJq7iN+ZroRBweNhvUobxs75bLIV6tNn1\r\n" +
+        "MdNHDRA+hezwf+gxHZhFyaAHfTpst2/9leK5Qe5Zd6gZLr3E5/8ppQuRod72H39B\r\n" +
+        "vxMlG5zxDss0WMo3vZeKZbTY6QhXi/lY2IZ6OGV4feSvCsYxn27GTjjrRUSLFeHH\r\n" +
+        "JVemCwCDMavaE3+OIY4v2P4FcG+MjUvfOB9ahI24TWL7YgrsNVmJjCILq+EeUj0t\r\n" +
+        "Gde1SXVyLkqt7PoxHRJAE0BCEMJSnjxaVB329acJgeehBUxjj4CCPqtDxtbz9HEH\r\n" +
+        "mOKfNdaKpFor+DUeEKUWVGnr9U9xOaC+Ws+oX7MIEUCDM7p2ob4JwcjnFs1jZgHh\r\n" +
+        "Hwig+i7doTlc701PvKWO96fuNHK3B3/jTb1fVvSZ49O/RvY1VWODdUdxWmXGHNh3\r\n" +
+        "LoR8tSPEb46lC2DXGaIQumqQt8PnBG+vL1qkQa1SGTV7dJ8TTbxbv0S+sS+igkk9\r\n" +
+        "zsIEK8Ea3Ep935cXximz0faAAKHSA+It+xHLAyDtqy2KaAEBgGsBuuWlUfK6TaP3\r\n" +
+        "Gwdjct3y4yYUO45lUsUfHqX8vk/4ttW5zYeDiW+HArJz+9VUXNbEdury4kGuHgBj\r\n" +
+        "xHD4Bsul65+hHZ9QywKU26F1A6TLkYpQ2rk/Dx9LGICM4m4IlHjWJPFsQdtkyOor\r\n" +
+        "osxMtcaZZ1E=\r\n" +
+        "-----END CERTIFICATE-----"
       end
 
       test "should respond with response from cloud" do
@@ -30,6 +63,14 @@ module InsightsCloud::Api
         ::ForemanRhCloud::CloudRequestForwarder.any_instance.expects(:execute_cloud_request).with do |opts|
           opts[:headers][:content_type] == 'application/json'
         end.returns(res)
+        InsightsCloud::Api::MachineTelemetriesController.any_instance.expects(:candlepin_id_cert)
+          .returns(
+            {
+              cert: @cert1,
+              key: OpenSSL::PKey::RSA.new(1024).to_pem,
+            }
+          )
+        InsightsCloud::Api::MachineTelemetriesController.any_instance.expects(:cp_owner_id).returns('123')
 
         post :forward_request, as: :json, params: { "path" => "static/v1/test", "machine_telemetry" => {"foo" => "bar"} }
         assert_equal @body, @response.body

data/test/jobs/insights_client_status_aging_test.rb

@@ -0,0 +1,33 @@
+require 'test_plugin_helper'
+require 'foreman_tasks/test_helpers'
+
+class InsightsClientStatusAgingTest < ActiveSupport::TestCase
+  include ForemanTasks::TestHelpers::WithInThreadExecutor
+
+  setup do
+    User.current = User.find_by(login: 'secret_admin')
+
+    @host1 = FactoryBot.create(:host)
+    @host2 = FactoryBot.create(:host)
+    @host3 = FactoryBot.create(:host)
+    @host4 = FactoryBot.create(:host)
+
+    @hosts = [@host1, @host2, @host3, @host4]
+  end
+
+  test 'stale statuses should change' do
+    InsightsClientReportStatus.find_or_initialize_by(host_id: @host1.id).update(status: InsightsClientReportStatus::REPORTING, reported_at: Time.now - InsightsClientReportStatus::REPORT_INTERVAL + 1.day)
+    InsightsClientReportStatus.find_or_initialize_by(host_id: @host2.id).update(status: InsightsClientReportStatus::NO_REPORT, reported_at: Time.now - InsightsClientReportStatus::REPORT_INTERVAL + 1.day)
+    InsightsClientReportStatus.find_or_initialize_by(host_id: @host3.id).update(status: InsightsClientReportStatus::REPORTING, reported_at: Time.now - InsightsClientReportStatus::REPORT_INTERVAL - 1.day)
+    InsightsClientReportStatus.find_or_initialize_by(host_id: @host4.id).update(status: InsightsClientReportStatus::NO_REPORT, reported_at: Time.now - InsightsClientReportStatus::REPORT_INTERVAL - 1.day)
+
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsClientStatusAging)
+
+    @hosts.each(&:reload)
+
+    assert_equal InsightsClientReportStatus::REPORTING, @host1.get_status(InsightsClientReportStatus).status
+    assert_equal InsightsClientReportStatus::NO_REPORT, @host2.get_status(InsightsClientReportStatus).status
+    assert_equal InsightsClientReportStatus::NO_REPORT, @host3.get_status(InsightsClientReportStatus).status
+    assert_equal InsightsClientReportStatus::NO_REPORT, @host4.get_status(InsightsClientReportStatus).status
+  end
+end