foreman_rh_cloud 5.0.35 → 5.0.38

data/lib/insights_cloud/async/insights_rules_sync.rb

@@ -3,18 +3,13 @@ require 'rest-client'
 module InsightsCloud
   module Async
     class InsightsRulesSync < ::Actions::EntryAction
-      include ::ForemanRhCloud::CloudAuth
-
-      def plan
-        unless cloud_auth_available?
-          logger.debug('Cloud authentication is not available, skipping rules sync')
-          return
-        end
+      include ::ForemanRhCloud::CertAuth
 
+      def plan(organizations)
         # since the tasks are not connected, we need to force sequence execution here
         # to make sure we don't run resolutions until we synced all our rules
         sequence do
-          plan_self
+          plan_self(organization_ids: organizations.map(&:id))
           plan_resolutions
         end
       end
@@ -26,16 +21,20 @@ module InsightsCloud
       def run
         offset = 0
         InsightsRule.transaction do
-          InsightsRule.delete_all
-          loop do
-            api_response = query_insights_rules(offset)
-            results = RulesResult.new(api_response)
-            logger.debug("Downloaded #{offset + results.count} of #{results.total}")
-            write_rules_page(results.rules)
-            offset += results.count
-            output[:rules_count] = results.total
-            break if offset >= results.total
+          organizations.each do |organization|
+            loop do
+              api_response = query_insights_rules(offset, organization)
+              results = RulesResult.new(api_response)
+              logger.debug("Downloaded #{offset + results.count} of #{results.total}")
+              write_rules_page(results.rules)
+              offset += results.count
+              output[:rules_count] = results.total
+              break if offset >= results.total
+            end
           end
+
+          # Remove all rules that do not have hits associated with them
+          cleanup_rules
         end
       end
 
@@ -45,8 +44,9 @@ module InsightsCloud
 
       private
 
-      def query_insights_rules(offset)
+      def query_insights_rules(offset, organization)
         rules_response = execute_cloud_request(
+          organization: organization,
           method: :get,
           url: InsightsCloud.rules_url(offset: offset)
         )
@@ -57,7 +57,7 @@ module InsightsCloud
       def write_rules_page(rules)
         rules_attributes = rules.map { |rule| to_rule_hash(rule) }
 
-        InsightsRule.create(rules_attributes)
+        InsightsRule.upsert_all(rules_attributes, unique_by: :rule_id) unless rules_attributes.empty?
       end
 
       def to_rule_hash(rule_hash)
@@ -76,9 +76,17 @@ module InsightsCloud
         }
       end
 
+      def cleanup_rules
+        InsightsRule.left_outer_joins(:hits).where(insights_hits: { id: nil }).delete_all
+      end
+
       def rescue_strategy_for_self
         Dynflow::Action::Rescue::Fail
       end
+
+      def organizations
+        @organizations ||= Organization.where(id: input[:organization_ids])
+      end
     end
   end
 end

data/lib/insights_cloud/async/insights_scheduled_sync.rb

@@ -16,7 +16,7 @@ module InsightsCloud
       end
 
       def plan_full_sync
-        plan_action InsightsFullSync
+        plan_action(InsightsFullSync, Organization.unscoped.all)
       end
 
       def rescue_strategy_for_self

data/lib/insights_cloud.rb

@@ -3,26 +3,22 @@ module InsightsCloud
     ADVISOR = 'advisor'
     VULNERABILITY = 'vulnerability'
     PATCH = 'patch'
-
-    def self.system_url(subsystem, host_uuid)
-      ForemanRhCloud.base_url + "/insights/#{subsystem}/systems/#{host_uuid}"
-    end
   end
 
   def self.hits_export_url
-    ForemanRhCloud.base_url + '/api/insights/v1/export/hits/'
+    ForemanRhCloud.cert_base_url + '/api/insights/v1/export/hits/'
   end
 
   def self.rules_url(limit: ForemanRhCloud.query_limit, offset: 0)
-    ForemanRhCloud.base_url + "/api/insights/v1/rule/?impacting=true&rule_status=enabled&has_playbook=true&limit=#{limit}&offset=#{offset}"
+    ForemanRhCloud.cert_base_url + "/api/insights/v1/rule/?impacting=true&rule_status=enabled&has_playbook=true&limit=#{limit}&offset=#{offset}"
   end
 
   def self.resolutions_url
-    ForemanRhCloud.base_url + '/api/remediations/v1/resolutions'
+    ForemanRhCloud.cert_base_url + '/api/remediations/v1/resolutions'
   end
 
   def self.playbook_url
-    ForemanRhCloud.base_url + '/api/remediations/v1/playbook'
+    ForemanRhCloud.cert_base_url + '/api/remediations/v1/playbook'
   end
 
   def self.remediation_rule_id(rule_id)

data/lib/inventory_sync/async/host_result.rb

@@ -1,9 +1,10 @@
 module InventorySync
   module Async
     class HostResult
-      attr_reader :uuid_by_fqdn
+      attr_reader :uuid_by_fqdn, :organization
 
-      def initialize(result)
+      def initialize(result, organization)
+        @organization = organization
         @total = result['total']
         @count = result['count']
         @page = result['page']

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -5,12 +5,12 @@ module InventorySync
       set_callback :step, :around, :update_statuses_batch
 
       def plan(organization)
-        unless cloud_auth_available?
+        unless cert_auth_available?(organization)
           logger.debug('Cloud authentication is not available, skipping inventory hosts sync')
           return
         end
 
-        plan_self(organization_id: organization.id)
+        super(organization)
       end
 
       def setup_statuses
@@ -70,7 +70,7 @@ module InventorySync
 
       def affected_host_ids
         ForemanInventoryUpload::Generators::Queries.for_slice(
-          Host.unscoped.where(organization: input[:organization_id])
+          Host.unscoped.where(organization: organizations)
         ).pluck(:id)
       end
     end

data/lib/inventory_sync/async/inventory_hosts_sync.rb

@@ -4,14 +4,9 @@ module InventorySync
       set_callback :iteration, :around, :setup_facet_transaction
       set_callback :step, :around, :create_facets
 
-      def plan
-        unless cloud_auth_available?
-          logger.debug('Cloud authentication is not available, skipping inventory hosts sync')
-          return
-        end
-
+      def plan(organizations)
         # by default the tasks will be executed concurrently
-        plan_self
+        super(organizations)
         plan_self_host_sync
       end
 
@@ -24,7 +19,7 @@ module InventorySync
       def create_facets
         # get the results from the event
         results = yield
-        add_missing_insights_facets(results.host_uuids)
+        add_missing_insights_facets(results.organization, results.host_uuids)
         results
       end
 
@@ -34,8 +29,10 @@ module InventorySync
 
       private
 
-      def add_missing_insights_facets(uuids_hash)
-        all_facets = uuids_hash.map do |host_id, uuid|
+      def add_missing_insights_facets(organization, uuids_hash)
+        # Filter out hosts that belong to different organization (although they are visible by the query)
+        unrelated_hosts = Host.where.not(organization_id: organization.id).where(id: uuids_hash.keys).pluck(:id)
+        all_facets = uuids_hash.except(unrelated_hosts).map do |host_id, uuid|
           {
             host_id: host_id,
             uuid: uuid,
@@ -48,6 +45,10 @@ module InventorySync
       def plan_self_host_sync
         plan_action InventorySync::Async::InventorySelfHostSync
       end
+
+      def action_name
+        'inventory hosts sync'
+      end
     end
   end
 end

data/lib/inventory_sync/async/inventory_self_host_sync.rb

@@ -4,12 +4,7 @@ module InventorySync
       set_callback :step, :around, :create_facets
 
       def plan
-        unless cloud_auth_available?
-          logger.debug('Cloud authentication is not available, skipping self host sync')
-          return
-        end
-
-        plan_self
+        super(ForemanRhCloud.foreman_host.organization)
       end
 
       def create_facets
@@ -38,6 +33,10 @@ module InventorySync
       def request_url
         ForemanInventoryUpload.inventory_self_url
       end
+
+      def action_name
+        'self host sync'
+      end
     end
   end
 end

data/lib/inventory_sync/async/query_inventory_job.rb

@@ -4,32 +4,52 @@ module InventorySync
   module Async
     class QueryInventoryJob < ::Actions::EntryAction
       include ActiveSupport::Callbacks
-      include ::ForemanRhCloud::CloudAuth
+      include ::ForemanRhCloud::CertAuth
 
       define_callbacks :iteration, :step
 
+      def plan(organizations, **params)
+        actual_params = params.merge(
+          {
+            organization_ids: Array(organizations).map(&:id),
+          }
+        )
+
+        plan_self(actual_params)
+      end
+
       def run
         run_callbacks :iteration do
-          page = 1
-          loop do
-            api_response = query_inventory(page)
-            results = HostResult.new(api_response)
-            logger.debug("Downloaded cloud inventory data: #{results.percentage}%")
-
-            run_callbacks :step do
-              results
+          organizations.each do |organization|
+            if !cert_auth_available?(organization) || organization.manifest_expired?
+              logger.debug("Subscription manifest not available, skipping #{action_name} for organization #{organization.name}")
+              next
             end
 
-            page += 1
-            break if results.last?
+            logger.debug("Executing #{action_name} for organization #{organization.name}")
+
+            page = 1
+            loop do
+              api_response = query_inventory(organization, page)
+              results = HostResult.new(api_response, organization)
+              logger.debug("Downloaded cloud inventory data: #{results.percentage}%")
+
+              run_callbacks :step do
+                results
+              end
+
+              page += 1
+              break if results.last?
+            end
           end
         end
       end
 
       private
 
-      def query_inventory(page = 1)
+      def query_inventory(organization, page = 1)
         hosts_inventory_response = execute_cloud_request(
+          organization: organization,
           method: :get,
           url: request_url,
           headers: {
@@ -54,6 +74,14 @@ module InventorySync
       def rescue_strategy_for_self
         Dynflow::Action::Rescue::Fail
       end
+
+      def organizations
+        @organizations ||= Organization.where(id: input[:organization_ids])
+      end
+
+      def action_name
+        'inventory query'
+      end
     end
   end
 end

data/lib/tasks/insights.rake

@@ -1,7 +1,7 @@
 namespace :rh_cloud_insights do
   desc "Synchronize Insights hosts hits"
   task sync: [:environment, 'dynflow:client'] do
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync)
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync, Organization.unscoped.all)
     puts "Synchronized Insights hosts hits data"
   end
 

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "5.0.35",
+  "version": "5.0.38",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {
@@ -22,14 +22,14 @@
     "url": "http://projects.theforeman.org/projects/foreman_rh_cloud/issues"
   },
   "peerDependencies": {
-    "@theforeman/vendor": "~8.16.0"
+    "@theforeman/vendor": ">=8.16.0"
   },
   "devDependencies": {
     "@babel/core": "~7.7.0",
-    "@theforeman/builder": "~8.16.0",
-    "@theforeman/stories": "~8.16.0",
-    "@theforeman/test": "~8.16.0",
-    "@theforeman/eslint-plugin-foreman": "~8.16.0",
+    "@theforeman/builder": ">=8.16.0",
+    "@theforeman/stories": ">=8.16.0",
+    "@theforeman/test": ">=8.16.0",
+    "@theforeman/eslint-plugin-foreman": ">=8.16.0",
     "babel-eslint": "~10.0.0",
     "eslint": "~6.7.2",
     "eslint-plugin-spellcheck": "~0.0.17",

data/test/controllers/insights_cloud/api/cloud_request_controller_test.rb

@@ -19,7 +19,6 @@ module InsightsCloud::Api
     end
 
     test 'Starts playbook run for correct directive' do
-      Setting[:rh_cloud_token] = 'MOCK_TOKEN'
       host1 = FactoryBot.create(:host, :with_insights_hits)
       host1.insights.uuid = 'TEST_UUID1'
       host1.insights.save!

data/test/controllers/inventory_upload/cloud_status_controller_test.rb

@@ -10,10 +10,6 @@ class CloudStatusControllerTest < ActionController::TestCase
     User.stubs(:current).returns(user)
     user.stubs(:my_organizations).returns(organizations)
 
-    ForemanRhCloud::CloudPingService::TokenPing.any_instance.expects(:execute_cloud_request).returns(
-      RestClient::Response.new('TEST RESPONSE')
-    )
-
     setup_certs_expectation do
       ForemanRhCloud::CloudPingService::CertPing.any_instance.expects(:candlepin_id_cert).with { |actual| actual.id == organizations[0].id }
     end
@@ -36,8 +32,6 @@ class CloudStatusControllerTest < ActionController::TestCase
     assert_response :success
     actual = JSON.parse(response.body)
     assert_not_nil (actual_ping = actual['ping'])
-    assert actual_ping['token_auth']['success']
-    assert_nil actual_ping['token_auth']['error']
     assert actual_ping['cert_auth'][0]['success']
     assert_nil actual_ping['cert_auth'][0]['error']
     assert actual_ping['cert_auth'][1]['success']

data/test/jobs/cloud_connector_announce_task_test.rb

@@ -0,0 +1,125 @@
+require 'json'
+require 'test_plugin_helper'
+require 'foreman_tasks/test_helpers'
+require "#{ForemanTasks::Engine.root}/test/support/dummy_dynflow_action"
+
+class CloudConnectorAnnounceTaskTest < ActiveSupport::TestCase
+  include ForemanTasks::TestHelpers::WithInThreadExecutor
+
+  setup do
+    RemoteExecutionFeature.register(
+      :ansible_configure_cloud_connector,
+      N_('Configure Cloud Connector on given hosts'),
+      :description => N_('Configure Cloud Connector on given hosts'),
+      :proxy_selector_override => ::RemoteExecutionProxySelector::INTERNAL_PROXY
+    )
+
+    @job_invocation = generate_job_invocation(:ansible_configure_cloud_connector)
+
+    # reset connector feature ID cache
+    InsightsCloud::Async::CloudConnectorAnnounceTask.instance_variable_set(:@connector_feature_id, nil)
+  end
+
+  test 'It executes cloud presence announcer' do
+    ForemanRhCloud::CloudPresence.any_instance.expects(:announce_to_sources).times(Organization.unscoped.count)
+
+    ForemanTasks.sync_task(InsightsCloud::Async::CloudConnectorAnnounceTask, @job_invocation)
+  end
+
+  private
+
+  def generate_job_invocation(feature_name)
+    job_template = FactoryBot.build(
+      :job_template,
+      :template => 'BLEH'
+    )
+    feature = RemoteExecutionFeature.feature!(feature_name).id
+
+    job_invocation = FactoryBot.create(
+      :job_invocation,
+      remote_execution_feature_id: feature,
+      task_id: FactoryBot.create(:dynflow_task).id
+    )
+
+    job_template.template_inputs << playbook_url_input = FactoryBot.build(:template_input,
+      :name => 'playbook_url',
+      :input_type => 'user',
+      :required => true)
+    job_template.template_inputs << report_url_input = FactoryBot.build(:template_input,
+      :name => 'report_url',
+      :input_type => 'user',
+      :required => true)
+    job_template.template_inputs << correlation_id_input = FactoryBot.build(:template_input,
+      :name => 'correlation_id',
+      :input_type => 'user',
+      :required => true)
+    job_template.template_inputs << report_interval_input = FactoryBot.build(:template_input,
+      :name => 'report_interval',
+      :input_type => 'user',
+      :required => true)
+
+    template_invocation = FactoryBot.build(:template_invocation,
+      :template => job_template,
+      :job_invocation => job_invocation
+    )
+
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => playbook_url_input,
+      :value => 'http://example.com/TEST_PLAYBOOK'
+    )
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => report_url_input,
+      :value => 'http://example.com/TEST_REPORT'
+    )
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => correlation_id_input,
+      :value => 'TEST_CORRELATION'
+    )
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => report_interval_input,
+      :value => '1'
+    )
+
+    @host1 = FactoryBot.create(:host, :with_insights_hits, name: 'host1')
+    @host1.insights.uuid = 'TEST_UUID1'
+    @host1.insights.save!
+    @host2 = FactoryBot.create(:host, :with_insights_hits, name: 'host2')
+    @host2.insights.uuid = 'TEST_UUID2'
+    @host2.insights.save!
+
+    targeting = FactoryBot.create(:targeting, hosts: [@host1, @host2])
+    job_invocation.targeting = targeting
+    job_invocation.save!
+
+    job_invocation.template_invocations << FactoryBot.create(
+      :template_invocation,
+      run_host_job_task: FactoryBot.create(:dynflow_task),
+      host_id: @host1.id
+    )
+    job_invocation.template_invocations << FactoryBot.create(
+      :template_invocation,
+      run_host_job_task: FactoryBot.create(:dynflow_task),
+      host_id: @host2.id
+    )
+
+    fake_output = (1..5).map do |i|
+      { 'timestamp' => (Time.now - (5 - i)).to_f, 'output' => "#{i}\n" }
+    end
+    Support::DummyDynflowAction.any_instance.stubs(:live_output).returns(fake_output)
+    Support::DummyDynflowAction.any_instance.stubs(:exit_status).returns(0)
+
+    job_invocation
+  end
+
+  def read_jsonl(jsonl)
+    jsonl.lines.map { |l| JSON.parse(l) }
+  end
+end

data/test/jobs/insights_full_sync_test.rb

@@ -3,11 +3,15 @@ require 'foreman_tasks/test_helpers'
 
 class InsightsFullSyncTest < ActiveSupport::TestCase
   include ForemanTasks::TestHelpers::WithInThreadExecutor
+  include MockCerts
 
   setup do
     InsightsCloud::Async::InsightsFullSync.any_instance.stubs(:plan_rules_sync)
     InsightsCloud::Async::InsightsFullSync.any_instance.stubs(:plan_notifications)
-    Setting[:rh_cloud_token] = 'MOCK_TOKEN'
+
+    setup_certs_expectation do
+      InsightsCloud::Async::InsightsFullSync.any_instance.stubs(:candlepin_id_cert)
+    end
 
     uuid1 = 'accdf444-5628-451d-bf3e-cf909ad72756'
     @host1 = FactoryBot.create(:host, :managed, name: 'host1')
@@ -66,7 +70,7 @@ class InsightsFullSyncTest < ActiveSupport::TestCase
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:plan_hosts_sync)
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:plan_rules_sync)
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:plan_notifications)
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync)
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync, [@host1.organization, @host2.organization])
 
     @host1.reload
     @host2.reload
@@ -80,9 +84,9 @@ class InsightsFullSyncTest < ActiveSupport::TestCase
 
     InsightsCloud::Async::InsightsFullSync.any_instance.stubs(:plan_hosts_sync)
 
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync)
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync, [@host1.organization, @host2.organization])
     # Invoke again
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync)
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync, [@host1.organization, @host2.organization])
 
     @host1.reload
     @host2.reload
@@ -114,7 +118,7 @@ class InsightsFullSyncTest < ActiveSupport::TestCase
     InsightsCloud::Async::InsightsFullSync.any_instance.stubs(:plan_hosts_sync)
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(hits)
 
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync)
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsFullSync, [@host1.organization, @host2.organization])
 
     @host1.reload
     @host2.reload

data/test/jobs/insights_resolutions_sync_test.rb

@@ -3,6 +3,7 @@ require 'foreman_tasks/test_helpers'
 
 class InsightsResolutionsSyncTest < ActiveSupport::TestCase
   include ForemanTasks::TestHelpers::WithInThreadExecutor
+  include MockCerts
 
   setup do
     @resolutions = {
@@ -63,7 +64,12 @@ class InsightsResolutionsSyncTest < ActiveSupport::TestCase
     }
 
     @rule = FactoryBot.create(:insights_rule, rule_id: 'network_tcp_connection_hang|NETWORK_TCP_CONNECTION_HANG_WARN')
-    Setting[:rh_cloud_token] = 'MOCK_TOKEN'
+
+    setup_certs_expectation do
+      InsightsCloud::Async::InsightsResolutionsSync.any_instance.stubs(:candlepin_id_cert)
+    end
+
+    Organization.any_instance.stubs(:manifest_expired?).returns(false)
   end
 
   test 'Resolutions data is replaced with data from cloud' do

data/test/jobs/insights_rules_sync_test.rb

@@ -112,13 +112,14 @@ class InsightsRulesSyncTest < ActiveSupport::TestCase
     @hit = FactoryBot.create(:insights_hit, host_id: @host.id)
 
     InsightsCloud::Async::InsightsRulesSync.any_instance.stubs(:plan_resolutions)
-    Setting[:rh_cloud_token] = 'MOCK_TOKEN'
   end
 
   test 'Hits data is replaced with data from cloud' do
-    InsightsCloud::Async::InsightsRulesSync.any_instance.expects(:query_insights_rules).returns(@rules)
+    InsightsCloud::Async::InsightsRulesSync.any_instance.expects(:query_insights_rules).returns(@rules).times(Organization.count)
+    # do not cleanup unused rules for tests
+    InsightsCloud::Async::InsightsRulesSync.any_instance.stubs(:cleanup_rules)
 
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsRulesSync)
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsRulesSync, Organization.all)
     @hit.reload
 
     assert_equal 2, InsightsRule.all.count
@@ -197,7 +198,10 @@ class InsightsRulesSyncTest < ActiveSupport::TestCase
     InsightsCloud::Async::InsightsRulesSync.any_instance.
       stubs(:query_insights_rules).returns(@rules).then.returns(@last_rule)
 
-    ForemanTasks.sync_task(InsightsCloud::Async::InsightsRulesSync)
+    # do not cleanup unused rules for tests
+    InsightsCloud::Async::InsightsRulesSync.any_instance.stubs(:cleanup_rules)
+
+    ForemanTasks.sync_task(InsightsCloud::Async::InsightsRulesSync, Organization.all)
 
     assert_equal 3, InsightsRule.all.count
   end