foreman_rh_cloud 3.0.18.1 → 3.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72303b7bc3f1bc2deb3972acde6131cdbe4d1760381d1eb8cdaa5704e3189481
-  data.tar.gz: 725efab879cee6224007359547068820882319d078089760cac43a8c763df754
+  metadata.gz: 43110585639a77e33c9c4fb1fce44855b6fb4c9793db3fc624a01b567c09e400
+  data.tar.gz: '00285b699105835e629127cc96801082ece7ff41c9a93d82fcb7e7a6b2662c65'
 SHA512:
-  metadata.gz: 28d7e810476e8202a62aa7381e8d3ef86cec5f2f6a7bc9580d5a116a543275183201ad573e848f30a65253bee2f20e7b0385f0cc6f0ab90637cfb03398557188
-  data.tar.gz: 52560a0917aecaed1c9897c593769c19b7dde9348c1b0450cd32fb4998d5c63e4232d671a1b767e33e6dcec960b3f4647bc23d56a80f15c1f18930db8b772409
+  metadata.gz: aa89cca9fb72a26453a44cba2120c51ab3f47f67ec40ca8e539a1dfaeddae73687bd909e3f3880cd3eed94c7c5d4b509bb55e2450f5bf4e55a8db2f73ceaabe9
+  data.tar.gz: 33d4200395db6f2f9a0e2dbd8b27d8c55ff66d7da87efa9c9c39663468f0dd9c43c8949b0ce09b2158875d053375e39d03e4550b821732df4d49fb52f5872cb3

data/app/controllers/foreman_inventory_upload/tasks_controller.rb

@@ -13,12 +13,23 @@ module ForemanInventoryUpload
           message: N_('Nothing to sync, there are no hosts with subscription for this organization.'),
         }, status: :method_not_allowed
       else
-        host_statuses = InventorySync::Async::InventoryFullSync.perform_now(selected_org)
+        task = ForemanTasks.async_task(InventorySync::Async::InventoryFullSync, selected_org)
       end
+      return render json: { message: N_('there was an issue triggering the task') }, status: :internal_server_error unless task
 
       render json: {
-        syncHosts: host_statuses[:sync],
-        disconnectHosts: host_statuses[:disconnect],
+        task: task,
+      }, status: :ok
+    end
+
+    def show
+      task = ForemanTasks::Task.find_by(id: params[:id])
+      return render json: { message: N_('No task was found') }, status: :internal_server_error unless task
+
+      render json: {
+        endedAt: task.ended_at,
+        output: task.output,
+        result: task.result,
       }, status: :ok
     end
   end

data/app/controllers/foreman_inventory_upload/uploads_settings_controller.rb

@@ -8,6 +8,7 @@ module ForemanInventoryUpload
         excludePackagesEnabled: Setting[:exclude_installed_packages],
         CloudConnectorStatus: ForemanInventoryUpload::UploadsSettingsController.cloud_connector_status,
         cloudToken: !Setting[:rh_cloud_token].empty?,
+        lastSyncTask: last_successful_inventory_sync_task,
       }, status: :ok
     end
 
@@ -22,5 +23,12 @@ module ForemanInventoryUpload
       return nil unless job
       { id: job.id, task: ForemanTasks::Task.where(:id => job.task_id).first }
     end
+
+    def last_successful_inventory_sync_task
+      task = ForemanTasks::Task.where(label: 'InventorySync::Async::InventoryFullSync', result: 'success')
+                        .reorder('ended_at desc').first
+      return nil unless task
+      { output: task.output, endedAt: task.ended_at }
+    end
   end
 end

data/app/models/insights_resolution.rb

@@ -1,3 +1,3 @@
 class InsightsResolution < ApplicationRecord
-  belongs_to :rule, class_name: 'InsightsRule', foreign_key: 'rule_id', primary_key: 'rule_id', inverse_of: :resolutions
+  belongs_to :rule, class_name: 'InsightsRule', primary_key: 'rule_id', inverse_of: :resolutions
 end

data/config/routes.rb

@@ -10,7 +10,7 @@ Rails.application.routes.draw do
 
     post 'cloud_connector', to: 'uploads#enable_cloud_connector'
 
-    resources :tasks, only: [:create]
+    resources :tasks, only: [:create, :show]
   end
 
   namespace :insights_cloud do

data/lib/foreman_inventory_upload.rb

@@ -56,6 +56,11 @@ module ForemanInventoryUpload
     @upload_url ||= ENV['SATELLITE_INVENTORY_UPLOAD_URL'] || 'https://cert.cloud.redhat.com/api/ingress/v1/upload'
   end
 
+  def self.slice_size
+    # for testing set ENV to 'https://ci.cloud.redhat.com/api/ingress/v1/upload'
+    @slice_size ||= (ENV['SATELLITE_INVENTORY_SLICE_SIZE'] || '1000').to_i
+  end
+
   def self.ensure_folder(folder)
     FileUtils.mkdir_p(folder)
     folder

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -50,7 +50,7 @@ module ForemanInventoryUpload
       end
 
       def self.for_org(organization_id)
-        for_slice(Host.unscoped.where(organization_id: organization_id)).in_batches(of: 1_000)
+        for_slice(Host.unscoped.where(organization_id: organization_id)).in_batches(of: ForemanInventoryUpload.slice_size)
       end
 
       def self.organizations_for_user(portal_user)

data/lib/foreman_inventory_upload/scripts/uploader.sh.erb

@@ -37,9 +37,13 @@ mkdir -p $DONE_DIR
 for f in $FILES
 do
   curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST "$AUTH_KEY" "$AUTH_VAL"
-  if [ $? -eq 0 ]; then
+  status=$?
+  if [ $status -eq 0 ]; then
     mv $f $DONE_DIR
     echo "Done: $f"
   fi
 done
 echo "Uploaded files moved to done/ folder"
+
+# return the error code from the curl command
+exit $status

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '3.0.18.1'.freeze
+  VERSION = '3.0.19'.freeze
 end

data/lib/insights_cloud/async/insights_full_sync.rb

@@ -6,14 +6,22 @@ module InsightsCloud
       include ::ForemanRhCloud::CloudAuth
 
       def perform
+        # This can be turned off when we enable automatic status syncs
+        # This step will query cloud inventory to retrieve inventory uuids for each host
+        ForemanTasks.sync_task(InventorySync::Async::InventoryHostsSync)
+
+        perform_hits_sync
+
+        InsightsRulesSync.perform_later
+      end
+
+      def perform_hits_sync
         hits = query_insights_hits
 
-        @hits_host_names = Hash[hits.map { |hit| [hit['hostname'], hit['uuid']] }]
-        setup_host_names(@hits_host_names.keys)
+        uuids = hits.map { |hit| hit['uuid'] }
+        setup_host_ids(uuids)
 
         replace_hits_data(hits)
-
-        InsightsRulesSync.perform_later
       end
 
       def logger
@@ -50,39 +58,27 @@ module InsightsCloud
         JSON.parse(rules_response)
       end
 
-      def setup_host_names(host_names)
+      def setup_host_ids(uuids)
         @host_ids = Hash[
-          Host.unscoped.where(name: host_names).pluck(:name, :id)
+          InsightsFacet.where(uuid: uuids).pluck(:uuid, :host_id)
         ]
       end
 
-      def host_id(host_name)
-        @host_ids[host_name]
+      def host_id(uuid)
+        @host_ids[uuid]
       end
 
       def replace_hits_data(hits)
         InsightsHit.transaction do
           # Reset hit counters to 0, they will be recreated later
           InsightsFacet.unscoped.update_all(hits_count: 0)
-          # create new facets for hosts that are missing one
-          hosts_with_existing_facets = InsightsFacet.where(host_id: @host_ids.values).pluck(:host_id)
-          InsightsFacet.create(
-            @host_ids.map do |host_name, host_id|
-              unless hosts_with_existing_facets.include?(host_id)
-                {
-                  host_id: host_id,
-                  uuid: @hits_host_names[host_name],
-                }
-              end
-            end.compact
-          )
           InsightsHit.delete_all
           InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) }.compact)
         end
       end
 
       def to_model_hash(hit_hash)
-        hit_host_id = host_id(hit_hash['hostname'])
+        hit_host_id = host_id(hit_hash['uuid'])
 
         return unless hit_host_id
 

data/lib/inventory_sync/async/host_result.rb

@@ -6,12 +6,13 @@ module InventorySync
         @count = result['count']
         @page = result['page']
         @per_page = result['per_page']
-        @fqdns = result["results"].map { |host| host['fqdn'] }
+        @sub_ids = result["results"].map { |host| host['subscription_manager_id'] }
+        @uuid_by_sub_id = Hash[result["results"].map { |host| [host['subscription_manager_id'], host['id']] }]
       end
 
       def status_hashes
-        @fqdns.map do |fqdn|
-          host_id = host_id(fqdn)
+        @sub_ids.map do |sub_id|
+          host_id = host_id(sub_id)
           if host_id
             touched << host_id
             {
@@ -27,16 +28,20 @@ module InventorySync
         @touched ||= []
       end
 
-      def host_id(fqdn)
-        hosts[fqdn]
+      def host_id(sub_id)
+        hosts[sub_id]
       end
 
       def hosts
         @hosts ||= Hash[
-          Host.where(name: @fqdns).pluck(:name, :id)
+          Katello::Host::SubscriptionFacet.where(uuid: @sub_ids).pluck(:uuid, :host_id)
         ]
       end
 
+      def host_uuids
+        @host_uuids ||= Hash[@sub_ids.map { |sub_id| [host_id(sub_id), @uuid_by_sub_id[sub_id]] }].except(nil)
+      end
+
       def percentage
         ratio = @per_page * @page * 1.0 / @total * 100
         ratio > 100 ? 100 : ratio.truncate(2)

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -1,42 +1,36 @@
-require 'rest-client'
-
 module InventorySync
   module Async
-    class InventoryFullSync < ::ApplicationJob
-      include ::ForemanRhCloud::CloudAuth
+    class InventoryFullSync < InventoryHostsSync
+      set_callback :iteration, :around, :setup_statuses
+      set_callback :step, :around, :update_statuses_batch
+
+      def plan(organization)
+        plan_self(organization_id: organization.id)
+      end
 
-      def perform(organization)
-        @organization = organization
+      def setup_statuses
         @subscribed_hosts_ids = Set.new(
           ForemanInventoryUpload::Generators::Queries.for_slice(
-            Host.unscoped.where(organization: organization)
+            Host.unscoped.where(organization: input[:organization_id])
           ).pluck(:id)
         )
-        @host_statuses = {
-          sync: 0,
-          disconnect: 0,
-        }
 
         InventorySync::InventoryStatus.transaction do
           InventorySync::InventoryStatus.where(host_id: @subscribed_hosts_ids).delete_all
-          page = 1
-          loop do
-            api_response = query_inventory(page)
-            results = HostResult.new(api_response)
-            logger.debug("Downloading cloud inventory data: #{results.percentage}%")
-            update_hosts_status(results.status_hashes, results.touched)
-            @host_statuses[:sync] += results.touched.size
-            page += 1
-            break if results.last?
-          end
+          yield
           add_missing_hosts_statuses(@subscribed_hosts_ids)
-          @host_statuses[:disconnect] += @subscribed_hosts_ids.size
+          host_statuses[:disconnect] += @subscribed_hosts_ids.size
         end
 
-        logger.debug("Synced hosts amount: #{@host_statuses[:sync]}")
-        logger.debug("Disconnected hosts amount: #{@host_statuses[:disconnect]}")
+        logger.debug("Synced hosts amount: #{host_statuses[:sync]}")
+        logger.debug("Disconnected hosts amount: #{host_statuses[:disconnect]}")
+      end
+
+      def update_statuses_batch
+        results = yield
 
-        @host_statuses
+        update_hosts_status(results.status_hashes, results.touched)
+        host_statuses[:sync] += results.touched.size
       end
 
       private
@@ -58,22 +52,11 @@ module InventorySync
         )
       end
 
-      def query_inventory(page = 1)
-        hosts_inventory_response = RestClient::Request.execute(
-          method: :get,
-          url: ForemanInventoryUpload.inventory_export_url,
-          verify_ssl: ForemanRhCloud.verify_ssl_method,
-          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
-          headers: {
-            Authorization: "Bearer #{rh_credentials}",
-            params: {
-              per_page: 100,
-              page: page,
-            },
-          }
-        )
-
-        JSON.parse(hosts_inventory_response)
+      def host_statuses
+        output[:host_statuses] ||= {
+          sync: 0,
+          disconnect: 0,
+        }
       end
     end
   end

data/lib/inventory_sync/async/inventory_hosts_sync.rb

@@ -0,0 +1,34 @@
+module InventorySync
+  module Async
+    class InventoryHostsSync < QueryInventoryJob
+      set_callback :iteration, :around, :setup_facet_transaction
+      set_callback :step, :around, :create_facets
+
+      def setup_facet_transaction
+        InsightsFacet.transaction do
+          yield
+        end
+      end
+
+      def create_facets
+        # get the results from the event
+        results = yield
+        add_missing_insights_facets(results.host_uuids)
+        results
+      end
+
+      private
+
+      def add_missing_insights_facets(uuids_hash)
+        existing_facets = InsightsFacet.where(host_id: uuids_hash.keys).pluck(:host_id)
+        missing_facets = uuids_hash.except(*existing_facets).map do |host_id, uuid|
+          {
+            host_id: host_id,
+            uuid: uuid,
+          }
+        end
+        InsightsFacet.create(missing_facets)
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/query_inventory_job.rb

@@ -0,0 +1,54 @@
+require 'rest-client'
+
+module InventorySync
+  module Async
+    class QueryInventoryJob < ::Actions::EntryAction
+      include ActiveSupport::Callbacks
+      include ::ForemanRhCloud::CloudAuth
+
+      define_callbacks :iteration, :step
+
+      def run
+        run_callbacks :iteration do
+          page = 1
+          loop do
+            api_response = query_inventory(page)
+            results = HostResult.new(api_response)
+            logger.debug("Downloaded cloud inventory data: #{results.percentage}%")
+
+            run_callbacks :step do
+              results
+            end
+
+            page += 1
+            break if results.last?
+          end
+        end
+      end
+
+      private
+
+      def query_inventory(page = 1)
+        hosts_inventory_response = RestClient::Request.execute(
+          method: :get,
+          url: ForemanInventoryUpload.inventory_export_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
+          headers: {
+            Authorization: "Bearer #{rh_credentials}",
+            params: {
+              per_page: 100,
+              page: page,
+            },
+          }
+        )
+
+        JSON.parse(hosts_inventory_response)
+      end
+
+      def logger
+        action_logger
+      end
+    end
+  end
+end

data/lib/tasks/insights.rake

@@ -8,7 +8,7 @@ namespace :rh_cloud_insights do
     end
 
     organizations.each do |organization|
-      InventorySync::Async::InventoryFullSync.perform_now(organization)
+      ForemanTasks.async_task(InventorySync::Async::InventoryFullSync, organization)
       puts "Synchronized inventory for organization '#{organization.name}'"
     end
   end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "3.0.18.1",
+  "version": "3.0.19",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {

data/test/jobs/insights_full_sync_test.rb

@@ -2,15 +2,20 @@ require 'test_helper'
 
 class InsightsFullSyncTest < ActiveJob::TestCase
   setup do
+    uuid1 = 'accdf444-5628-451d-bf3e-cf909ad72756'
     @host1 = FactoryBot.create(:host, :managed, name: 'host1')
+    FactoryBot.create(:insights_facet, host_id: @host1.id, uuid: uuid1)
+
+    uuid2 = 'accdf444-5628-451d-bf3e-cf909ad72757'
     @host2 = FactoryBot.create(:host, :managed, name: 'host2')
+    FactoryBot.create(:insights_facet, host_id: @host2.id, uuid: uuid2)
 
     hits_json = <<-HITS_JSON
     [
         {
             "hostname": "#{@host1.name}",
             "rhel_version": "7.5",
-            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "uuid": "#{uuid1}",
             "last_seen": "2019-11-22T08:41:42.447244Z",
             "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
             "solution_url": "",
@@ -22,7 +27,7 @@ class InsightsFullSyncTest < ActiveJob::TestCase
         {
             "hostname": "#{@host1.name}",
             "rhel_version": "7.5",
-            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "uuid": "#{uuid1}",
             "last_seen": "2019-11-22T08:41:42.447244Z",
             "title": "CPU vulnerable to side-channel attacks using Microarchitectural Data Sampling (CVE-2018-12130, CVE-2018-12126, CVE-2018-12127, CVE-2019-11091)",
             "solution_url": "https://access.redhat.com/node/4134081",
@@ -34,7 +39,7 @@ class InsightsFullSyncTest < ActiveJob::TestCase
         {
             "hostname": "#{@host2.name}",
             "rhel_version": "7.5",
-            "uuid": "accdf444-5628-451d-bf3e-cf909ad72757",
+            "uuid": "#{uuid2}",
             "last_seen": "2019-11-22T08:41:42.447244Z",
             "title": "CPU vulnerable to side-channel attacks using L1 Terminal Fault (CVE-2018-3620)",
             "solution_url": "https://access.redhat.com/node/3560291",
@@ -51,6 +56,7 @@ class InsightsFullSyncTest < ActiveJob::TestCase
   test 'Hits data is replaced with data from cloud' do
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(@hits)
 
+    ForemanTasks.expects(:sync_task).with(InventorySync::Async::InventoryHostsSync)
     InsightsCloud::Async::InsightsFullSync.perform_now()
 
     @host1.reload
@@ -58,12 +64,11 @@ class InsightsFullSyncTest < ActiveJob::TestCase
 
     assert_equal 2, @host1.insights.hits.count
     assert_equal 1, @host2.insights.hits.count
-    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72756', @host1.insights.uuid
-    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72757', @host2.insights.uuid
   end
 
   test 'Hits counters are reset correctly' do
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(@hits).twice
+    ForemanTasks.stubs(:sync_task)
 
     InsightsCloud::Async::InsightsFullSync.perform_now()
     # Invoke again
@@ -75,8 +80,6 @@ class InsightsFullSyncTest < ActiveJob::TestCase
     # Check that the counters are correct
     assert_equal 2, @host1.insights.hits.count
     assert_equal 1, @host2.insights.hits.count
-    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72756', @host1.insights.uuid
-    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72757', @host2.insights.uuid
   end
 
   test 'Hits ignoring non-existent hosts' do
@@ -85,7 +88,7 @@ class InsightsFullSyncTest < ActiveJob::TestCase
         {
             "hostname": "#{@host1.name}_non_existent",
             "rhel_version": "7.5",
-            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad00000",
             "last_seen": "2019-11-22T08:41:42.447244Z",
             "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
             "solution_url": "",
@@ -98,6 +101,7 @@ class InsightsFullSyncTest < ActiveJob::TestCase
     HITS_JSON
     hits = JSON.parse(hits_json)
 
+    ForemanTasks.stubs(:sync_task)
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(hits)
 
     InsightsCloud::Async::InsightsFullSync.perform_now()
@@ -105,7 +109,7 @@ class InsightsFullSyncTest < ActiveJob::TestCase
     @host1.reload
     @host2.reload
 
-    assert_nil @host1.insights
-    assert_nil @host2.insights
+    assert_equal 0, @host1.insights.hits_count
+    assert_equal 0, @host2.insights.hits_count
   end
 end