foreman_rh_cloud 2.0.8 → 2.0.13

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '2.0.8'.freeze
+  VERSION = '2.0.13'.freeze
 end

data/lib/insights_cloud.rb

@@ -1,15 +1,15 @@
 module InsightsCloud
-  def self.base_url
-    # for testing set ENV to 'https://ci.cloud.redhat.com'
-    @base_url ||= ENV['SATELLITE_INSIGHTS_CLOUD_URL'] || 'https://cloud.redhat.com'
-  end
+  module WebUi
+    ADVISOR = 'advisor'
+    VULNERABILITY = 'vulnerability'
+    PATCH = 'patch'
 
-  def self.authentication_url
-    # https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token
-    @authentication_url ||= ENV['SATELLITE_INSIGHTS_CLOUD_SSO_URL'] || 'https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token'
+    def self.system_url(subsystem, host_uuid)
+      ForemanRhCloud.base_url + "/insights/#{subsystem}/systems/#{host_uuid}"
+    end
   end
 
   def self.hits_export_url
-    base_url + '/api/insights/v1/export/hits/'
+    ForemanRhCloud.base_url + '/api/insights/v1/export/hits/'
   end
 end

data/lib/insights_cloud/async/insights_full_sync.rb

@@ -3,13 +3,13 @@ require 'rest-client'
 module InsightsCloud
   module Async
     class InsightsFullSync < ::ApplicationJob
-      def perform(organization)
-        @organization = organization
+      include ::ForemanRhCloud::CloudAuth
 
+      def perform
         hits = query_insights_hits
 
-        host_names = hits.map { |hit| hit['hostname'] }.uniq
-        setup_host_names(host_names)
+        @hits_host_names = Hash[hits.map { |hit| [hit['hostname'], hit['uuid']] }]
+        setup_host_names(@hits_host_names.keys)
 
         replace_hits_data(hits)
       end
@@ -18,17 +18,14 @@ module InsightsCloud
         Foreman::Logging.logger('background')
       end
 
-      def rh_credentials
-        @rh_credentials ||= query_refresh_token
-      end
-
       private
 
       def query_insights_hits
         hits_response = RestClient::Request.execute(
           method: :get,
           url: InsightsCloud.hits_export_url,
-          verify_ssl: ENV['SATELLITE_INSIGHTS_CLOUD_URL'] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
           headers: {
             Authorization: "Bearer #{rh_credentials}",
           }
@@ -37,27 +34,9 @@ module InsightsCloud
         JSON.parse(hits_response)
       end
 
-      def query_refresh_token
-        token_response = RestClient::Request.execute(
-          method: :post,
-          url: InsightsCloud.authentication_url,
-          verify_ssl: ENV['SATELLITE_INSIGHTS_CLOUD_URL'] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER,
-          payload: {
-            grant_type: 'refresh_token',
-            client_id: 'rhsm-api',
-            refresh_token: Setting[:rh_cloud_token],
-          }
-        )
-
-        JSON.parse(token_response)['access_token']
-      rescue RestClient::ExceptionWithResponse => e
-        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
-        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
-      end
-
       def setup_host_names(host_names)
         @host_ids = Hash[
-          Host.where(name: host_names).pluck(:name, :id)
+          Host.unscoped.where(name: host_names).pluck(:name, :id)
         ]
       end
 
@@ -67,17 +46,30 @@ module InsightsCloud
 
       def replace_hits_data(hits)
         InsightsHit.transaction do
-          InsightsHit.delete_all
-          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) })
           # create new facets for hosts that are missing one
           hosts_with_existing_facets = InsightsFacet.where(host_id: @host_ids.values).pluck(:host_id)
-          InsightsFacet.create((@host_ids.values - hosts_with_existing_facets).map { |id| {host_id: id} })
+          InsightsFacet.create(
+            @host_ids.map do |host_name, host_id|
+              unless hosts_with_existing_facets.include?(host_id)
+                {
+                  host_id: host_id,
+                  uuid: @hits_host_names[host_name],
+                }
+              end
+            end.compact
+          )
+          InsightsHit.delete_all
+          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) }.compact)
         end
       end
 
       def to_model_hash(hit_hash)
+        hit_host_id = host_id(hit_hash['hostname'])
+
+        return unless hit_host_id
+
         {
-          host_id: host_id(hit_hash['hostname']),
+          host_id: hit_host_id,
           last_seen: DateTime.parse(hit_hash['last_seen']),
           publish_date: DateTime.parse(hit_hash['publish_date']),
           title: hit_hash['title'],

data/lib/insights_cloud/async/insights_scheduled_sync.rb

@@ -0,0 +1,23 @@
+module InsightsCloud
+  module Async
+    class InsightsScheduledSync < ::ApplicationJob
+      def perform
+        unless Setting[:allow_auto_insights_sync]
+          logger.debug(
+            'The scheduled process is disabled due to the "allow_auto_insights_sync"
+            setting being set to false.'
+          )
+          return
+        end
+
+        InsightsFullSync.perform_later()
+      ensure
+        self.class.set(:wait => 24.hours).perform_later
+      end
+
+      def self.singleton_job_name
+        name
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/host_result.rb

@@ -0,0 +1,50 @@
+module InventorySync
+  module Async
+    class HostResult
+      def initialize(result)
+        @total = result['total']
+        @count = result['count']
+        @page = result['page']
+        @per_page = result['per_page']
+        @fqdns = result["results"].map { |host| host['fqdn'] }
+      end
+
+      def status_hashes
+        @fqdns.map do |fqdn|
+          host_id = host_id(fqdn)
+          if host_id
+            touched << host_id
+            {
+              host_id: host_id,
+              status: InventorySync::InventoryStatus::SYNC,
+              reported_at: DateTime.current,
+            }
+          end
+        end.compact
+      end
+
+      def touched
+        @touched ||= []
+      end
+
+      def host_id(fqdn)
+        hosts[fqdn]
+      end
+
+      def hosts
+        @hosts ||= Hash[
+          Host.where(name: @fqdns).pluck(:name, :id)
+        ]
+      end
+
+      def percentage
+        ratio = @per_page * @page * 1.0 / @total * 100
+        ratio > 100 ? 100 : ratio.truncate(2)
+      end
+
+      def last?
+        @total <= @per_page * @page
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -0,0 +1,80 @@
+require 'rest-client'
+
+module InventorySync
+  module Async
+    class InventoryFullSync < ::ApplicationJob
+      include ::ForemanRhCloud::CloudAuth
+
+      def perform(organization)
+        @organization = organization
+        @subscribed_hosts_ids = Set.new(
+          ForemanInventoryUpload::Generators::Queries.for_slice(
+            Host.unscoped.where(organization: organization)
+          ).pluck(:id)
+        )
+        @host_statuses = {
+          sync: 0,
+          disconnect: 0,
+        }
+
+        InventorySync::InventoryStatus.transaction do
+          InventorySync::InventoryStatus.where(host_id: @subscribed_hosts_ids).delete_all
+          page = 1
+          loop do
+            api_response = query_inventory(page)
+            results = HostResult.new(api_response)
+            logger.debug("Downloading cloud inventory data: #{results.percentage}%")
+            update_hosts_status(results.status_hashes, results.touched)
+            @host_statuses[:sync] += results.touched.size
+            page += 1
+            break if results.last?
+          end
+          add_missing_hosts_statuses(@subscribed_hosts_ids)
+          @host_statuses[:disconnect] += @subscribed_hosts_ids.size
+        end
+
+        logger.debug("Synced hosts amount: #{@host_statuses[:sync]}")
+        logger.debug("Disconnected hosts amount: #{@host_statuses[:disconnect]}")
+
+        @host_statuses
+      end
+
+      private
+
+      def update_hosts_status(status_hashes, touched)
+        InventorySync::InventoryStatus.create(status_hashes)
+        @subscribed_hosts_ids.subtract(touched)
+      end
+
+      def add_missing_hosts_statuses(hosts_ids)
+        InventorySync::InventoryStatus.create(
+          hosts_ids.map do |host_id|
+            {
+              host_id: host_id,
+              status: InventorySync::InventoryStatus::DISCONNECT,
+              reported_at: DateTime.current,
+            }
+          end
+        )
+      end
+
+      def query_inventory(page = 1)
+        hosts_inventory_response = RestClient::Request.execute(
+          method: :get,
+          url: ForemanInventoryUpload.inventory_export_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
+          headers: {
+            Authorization: "Bearer #{rh_credentials}",
+            params: {
+              per_page: 100,
+              page: page,
+            },
+          }
+        )
+
+        JSON.parse(hosts_inventory_response)
+      end
+    end
+  end
+end

data/lib/tasks/insights.rake

@@ -0,0 +1,15 @@
+namespace :rh_cloud_insights do
+  desc "Synchronize Insights inventory"
+  task sync: :environment do
+    if ! ENV['organization_id'].nil?
+      organizations = [ Organization.where(:id => ENV['organization_id']).first ]
+    else
+      organizations = Organization.all
+    end
+
+    organizations.each do |organization|
+      InventorySync::Async::InventoryFullSync.perform_now(organization)
+      puts "Synchronized inventory for organization '#{organization.name}'"
+    end
+  end
+end

data/lib/tasks/rh_cloud_inventory.rake

@@ -0,0 +1,37 @@
+require 'tempfile'
+
+namespace :rh_cloud_inventory do
+  namespace :report do
+    desc 'Generate inventory report to be sent to Red Hat cloud'
+    task generate: :environment do
+      portal_user = ENV['portal_user']
+      organizations = [ENV['organization_id']]
+      base_folder = ENV['target'] || Dir.pwd
+
+      unless portal_user || organizations.empty?
+        puts "Must specify either portal_user or organization_id"
+      end
+
+      User.as_anonymous_admin do
+        if portal_user
+          puts "Generating report for all organizations associated with #{portal_user}"
+          base_folder = File.join(base_folder, portal_user)
+          organizations = ForemanInventoryUpload::Generators::Queries.organizations_for_user(portal_user).pluck(:id)
+        end
+
+        organizations.each do |organization|
+          target = File.join(base_folder, ForemanInventoryUpload.facts_archive_name(organization))
+          archived_report_generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
+          archived_report_generator.render(organization: organization)
+          puts "Successfully generated #{target} for organization id #{organization}"
+        end
+      end
+    end
+  end
+
+  desc "Synchronize Insights hosts hits"
+  task sync: :environment do
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+    puts "Synchronized Insights hosts hits data"
+  end
+end

data/package.json

@@ -1,10 +1,11 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "0.0.1",
+  "version": "2.0.13",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {
     "lint": "tfm-lint --plugin -d /webpack",
+    "lint:spelling": "eslint ./",
     "test": "tfm-test --plugin",
     "test:watch": "tfm-test --plugin --watchAll",
     "test:current": "tfm-test --plugin --watch",
@@ -32,6 +33,7 @@
     "@theforeman/eslint-plugin-foreman": "^4.0.2",
     "babel-eslint": "^10.0.0",
     "eslint": "^6.7.2",
+    "eslint-plugin-spellcheck": "^0.0.17",
     "prettier": "^1.13.5",
     "raf": "^3.4.0",
     "stylelint": "^9.3.0",

data/test/controllers/insights_sync/settings_controller_test.rb

@@ -0,0 +1,30 @@
+require 'test_plugin_helper'
+
+class SettingsControllerTest < ActionController::TestCase
+  tests InsightsCloud::SettingsController
+
+  test 'should return allow_auto_insights_sync setting' do
+    FactoryBot.create(:setting, :name => 'allow_auto_insights_sync', :settings_type => "boolean", :category => "Setting::RhCloud", :default => false, :value => false)
+
+    assert_equal false, Setting[:allow_auto_insights_sync]
+
+    get :show, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal false, actual['insightsSyncEnabled']
+  end
+
+  test 'should update allow_auto_insights_sync setting' do
+    FactoryBot.create(:setting, :name => 'allow_auto_insights_sync', :settings_type => "boolean", :category => "Setting::RhCloud", :default => false, :value => false)
+
+    assert_equal false, Setting[:allow_auto_insights_sync]
+
+    patch :update, params: { insightsSyncEnabled: true }, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal true, actual['insightsSyncEnabled']
+    assert_equal true, Setting[:allow_auto_insights_sync]
+  end
+end

data/test/factories/insights_factories.rb

@@ -0,0 +1,31 @@
+FactoryBot.define do
+  factory :insights_facet do
+    # sequence(:uuid) { |n| "uuid-#{n}" }
+
+    trait :with_hits do
+      hits do
+        Array.new(1) { association(:insights_hit, host_id: host_id) }
+      end
+    end
+  end
+
+  factory :insights_hit do
+    last_seen { DateTime.yesterday }
+    sequence(:title) { |n| "hit title #{n}" }
+    sequence(:solution_url) { |n| "http://example.com/solutions/#{n}" }
+    total_risk { 1 }
+    likelihood { 2 }
+    publish_date { DateTime.now }
+    sequence(:results_url) { |n| "http://example.com/results/#{n}" }
+  end
+end
+
+FactoryBot.modify do
+  factory :host do
+    trait :with_insights_hits do
+      after(:create) do |host, _evaluator|
+        host.insights = FactoryBot.create(:insights_facet, :with_hits, host_id: host.id)
+      end
+    end
+  end
+end

data/test/jobs/insights_full_sync_test.rb

@@ -51,12 +51,44 @@ class InsightsFullSyncTest < ActiveJob::TestCase
   test 'Hits data is replaced with data from cloud' do
     InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(@hits)
 
-    InsightsCloud::Async::InsightsFullSync.perform_now(@host1.organization)
+    InsightsCloud::Async::InsightsFullSync.perform_now()
 
     @host1.reload
     @host2.reload
 
     assert_equal 2, @host1.insights.hits.count
     assert_equal 1, @host2.insights.hits.count
+    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72756', @host1.insights.uuid
+    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72757', @host2.insights.uuid
+  end
+
+  test 'Hits ignoring non-existent hosts' do
+    hits_json = <<-HITS_JSON
+    [
+        {
+            "hostname": "#{@host1.name}_non_existent",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
+            "solution_url": "",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-04-16T10:03:16Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/stability/ansible_deprecated_repo%7CANSIBLE_DEPRECATED_REPO/accdf444-5628-451d-bf3e-cf909ad72756/"
+        }
+    ]
+    HITS_JSON
+    hits = JSON.parse(hits_json)
+
+    InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(hits)
+
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+
+    @host1.reload
+    @host2.reload
+
+    assert_nil @host1.insights
+    assert_nil @host2.insights
   end
 end