foreman_rh_cloud 2.0.6 → 2.0.11

data/lib/insights_cloud/async/insights_full_sync.rb

@@ -0,0 +1,103 @@
+require 'rest-client'
+
+module InsightsCloud
+  module Async
+    class InsightsFullSync < ::ApplicationJob
+      def perform
+        hits = query_insights_hits
+
+        @hits_host_names = Hash[hits.map { |hit| [hit['hostname'], hit['uuid']] }]
+        setup_host_names(@hits_host_names.keys)
+
+        replace_hits_data(hits)
+      end
+
+      def logger
+        Foreman::Logging.logger('background')
+      end
+
+      def rh_credentials
+        @rh_credentials ||= query_refresh_token
+      end
+
+      private
+
+      def query_insights_hits
+        hits_response = RestClient::Request.execute(
+          method: :get,
+          url: InsightsCloud.hits_export_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          headers: {
+            Authorization: "Bearer #{rh_credentials}",
+          }
+        )
+
+        JSON.parse(hits_response)
+      end
+
+      def query_refresh_token
+        token_response = RestClient::Request.execute(
+          method: :post,
+          url: ForemanRhCloud.authentication_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          payload: {
+            grant_type: 'refresh_token',
+            client_id: 'rhsm-api',
+            refresh_token: Setting[:rh_cloud_token],
+          }
+        )
+
+        JSON.parse(token_response)['access_token']
+      rescue RestClient::ExceptionWithResponse => e
+        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
+        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
+      end
+
+      def setup_host_names(host_names)
+        @host_ids = Hash[
+          Host.unscoped.where(name: host_names).pluck(:name, :id)
+        ]
+      end
+
+      def host_id(host_name)
+        @host_ids[host_name]
+      end
+
+      def replace_hits_data(hits)
+        InsightsHit.transaction do
+          InsightsHit.delete_all
+          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) }.compact)
+          # create new facets for hosts that are missing one
+          hosts_with_existing_facets = InsightsFacet.where(host_id: @host_ids.values).pluck(:host_id)
+          InsightsFacet.create(
+            @host_ids.map do |host_name, host_id|
+              unless hosts_with_existing_facets.include?(host_id)
+                {
+                  host_id: host_id,
+                  uuid: @hits_host_names[host_name],
+                }
+              end
+            end.compact
+          )
+        end
+      end
+
+      def to_model_hash(hit_hash)
+        hit_host_id = host_id(hit_hash['hostname'])
+
+        return unless hit_host_id
+
+        {
+          host_id: hit_host_id,
+          last_seen: DateTime.parse(hit_hash['last_seen']),
+          publish_date: DateTime.parse(hit_hash['publish_date']),
+          title: hit_hash['title'],
+          solution_url: hit_hash['solution_url'],
+          total_risk: hit_hash['total_risk'].to_i,
+          likelihood: hit_hash['likelihood'].to_i,
+          results_url: hit_hash['results_url'],
+        }
+      end
+    end
+  end
+end

data/lib/insights_cloud/async/insights_scheduled_sync.rb

@@ -0,0 +1,23 @@
+module InsightsCloud
+  module Async
+    class InsightsScheduledSync < ::ApplicationJob
+      def perform
+        unless Setting[:allow_auto_insights_sync]
+          logger.debug(
+            'The scheduled process is disabled due to the "allow_auto_insights_sync"
+            setting being set to false.'
+          )
+          return
+        end
+
+        InsightsFullSync.perform_later()
+      ensure
+        self.class.set(:wait => 24.hours).perform_later
+      end
+
+      def self.singleton_job_name
+        name
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/host_result.rb

@@ -0,0 +1,50 @@
+module InventorySync
+  module Async
+    class HostResult
+      def initialize(result)
+        @total = result['total']
+        @count = result['count']
+        @page = result['page']
+        @per_page = result['per_page']
+        @fqdns = result["results"].map { |host| host['fqdn'] }
+      end
+
+      def status_hashes
+        @fqdns.map do |fqdn|
+          host_id = host_id(fqdn)
+          if host_id
+            touched << host_id
+            {
+              host_id: host_id,
+              status: InventorySync::InventoryStatus::SYNC,
+              reported_at: DateTime.current,
+            }
+          end
+        end.compact
+      end
+
+      def touched
+        @touched ||= []
+      end
+
+      def host_id(fqdn)
+        hosts[fqdn]
+      end
+
+      def hosts
+        @hosts ||= Hash[
+          Host.where(name: @fqdns).pluck(:name, :id)
+        ]
+      end
+
+      def percentage
+        ratio = @per_page * @page * 1.0 / @total * 100
+        ratio > 100 ? 100 : ratio.truncate(2)
+      end
+
+      def last?
+        @total <= @per_page * @page
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -0,0 +1,99 @@
+require 'rest-client'
+
+module InventorySync
+  module Async
+    class InventoryFullSync < ::ApplicationJob
+      def perform(organization)
+        @organization = organization
+        @subscribed_hosts_ids = Set.new(
+          ForemanInventoryUpload::Generators::Queries.for_slice(
+            Host.unscoped.where(organization: organization)
+          ).pluck(:id)
+        )
+        @host_statuses = {
+          sync: 0,
+          disconnect: 0,
+        }
+
+        InventorySync::InventoryStatus.transaction do
+          InventorySync::InventoryStatus.where(host_id: @subscribed_hosts_ids).delete_all
+          page = 1
+          loop do
+            api_response = query_inventory(page)
+            results = HostResult.new(api_response)
+            logger.debug("Downloading cloud inventory data: #{results.percentage}%")
+            update_hosts_status(results.status_hashes, results.touched)
+            @host_statuses[:sync] += results.touched.size
+            page += 1
+            break if results.last?
+          end
+          add_missing_hosts_statuses(@subscribed_hosts_ids)
+          @host_statuses[:disconnect] += @subscribed_hosts_ids.size
+        end
+
+        logger.debug("Synced hosts amount: #{@host_statuses[:sync]}")
+        logger.debug("Disconnected hosts amount: #{@host_statuses[:disconnect]}")
+
+        @host_statuses
+      end
+
+      def rh_credentials
+        @rh_credentials ||= query_refresh_token
+      end
+
+      private
+
+      def update_hosts_status(status_hashes, touched)
+        InventorySync::InventoryStatus.create(status_hashes)
+        @subscribed_hosts_ids.subtract(touched)
+      end
+
+      def add_missing_hosts_statuses(hosts_ids)
+        InventorySync::InventoryStatus.create(
+          hosts_ids.map do |host_id|
+            {
+              host_id: host_id,
+              status: InventorySync::InventoryStatus::DISCONNECT,
+              reported_at: DateTime.current,
+            }
+          end
+        )
+      end
+
+      def query_inventory(page = 1)
+        hosts_inventory_response = RestClient::Request.execute(
+          method: :get,
+          url: ForemanInventoryUpload.inventory_export_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          headers: {
+            Authorization: "Bearer #{rh_credentials}",
+            params: {
+              per_page: 100,
+              page: page,
+            },
+          }
+        )
+
+        JSON.parse(hosts_inventory_response)
+      end
+
+      def query_refresh_token
+        token_response = RestClient::Request.execute(
+          method: :post,
+          url: ForemanRhCloud.authentication_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          payload: {
+            grant_type: 'refresh_token',
+            client_id: 'rhsm-api',
+            refresh_token: Setting[:rh_cloud_token],
+          }
+        )
+
+        JSON.parse(token_response)['access_token']
+      rescue RestClient::ExceptionWithResponse => e
+        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
+        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
+      end
+    end
+  end
+end

data/lib/tasks/insights.rake

@@ -0,0 +1,15 @@
+namespace :rh_cloud_insights do
+  desc "Synchronize Insights inventory"
+  task sync: :environment do
+    if ! ENV['organization_id'].nil?
+      organizations = [ Organization.where(:id => ENV['organization_id']).first ]
+    else
+      organizations = Organization.all
+    end
+
+    organizations.each do |organization|
+      InventorySync::Async::InventoryFullSync.perform_now(organization)
+      puts "Synchronized inventory for organization '#{organization.name}'"
+    end
+  end
+end

data/lib/tasks/{generator.rake → rh_cloud_inventory.rake}

@@ -1,6 +1,6 @@
 require 'tempfile'
 
-namespace :foreman_inventory_upload do
+namespace :rh_cloud_inventory do
   namespace :report do
     desc 'Generate inventory report to be sent to Red Hat cloud'
     task generate: :environment do
@@ -26,4 +26,10 @@ namespace :foreman_inventory_upload do
       end
     end
   end
+
+  desc "Synchronize Insights hosts hits"
+  task sync: :environment do
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+    puts "Synchronized Insights hosts hits data"
+  end
 end

data/package.json

@@ -1,10 +1,11 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "0.0.1",
+  "version": "2.0.11",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {
     "lint": "tfm-lint --plugin -d /webpack",
+    "lint:spelling": "eslint ./",
     "test": "tfm-test --plugin",
     "test:watch": "tfm-test --plugin --watchAll",
     "test:current": "tfm-test --plugin --watch",
@@ -32,6 +33,7 @@
     "@theforeman/eslint-plugin-foreman": "^4.0.2",
     "babel-eslint": "^10.0.0",
     "eslint": "^6.7.2",
+    "eslint-plugin-spellcheck": "^0.0.17",
     "prettier": "^1.13.5",
     "raf": "^3.4.0",
     "stylelint": "^9.3.0",

data/test/controllers/insights_sync/settings_controller_test.rb

@@ -0,0 +1,30 @@
+require 'test_plugin_helper'
+
+class SettingsControllerTest < ActionController::TestCase
+  tests InsightsCloud::SettingsController
+
+  test 'should return allow_auto_insights_sync setting' do
+    FactoryBot.create(:setting, :name => 'allow_auto_insights_sync', :settings_type => "boolean", :category => "Setting::RhCloud", :default => false, :value => false)
+
+    assert_equal false, Setting[:allow_auto_insights_sync]
+
+    get :show, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal false, actual['insightsSyncEnabled']
+  end
+
+  test 'should update allow_auto_insights_sync setting' do
+    FactoryBot.create(:setting, :name => 'allow_auto_insights_sync', :settings_type => "boolean", :category => "Setting::RhCloud", :default => false, :value => false)
+
+    assert_equal false, Setting[:allow_auto_insights_sync]
+
+    patch :update, params: { insightsSyncEnabled: true }, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal true, actual['insightsSyncEnabled']
+    assert_equal true, Setting[:allow_auto_insights_sync]
+  end
+end

data/test/factories/insights_factories.rb

@@ -0,0 +1,31 @@
+FactoryBot.define do
+  factory :insights_facet do
+    # sequence(:uuid) { |n| "uuid-#{n}" }
+
+    trait :with_hits do
+      hits do
+        Array.new(1) { association(:insights_hit, host_id: host_id) }
+      end
+    end
+  end
+
+  factory :insights_hit do
+    last_seen { DateTime.yesterday }
+    sequence(:title) { |n| "hit title #{n}" }
+    sequence(:solution_url) { |n| "http://example.com/solutions/#{n}" }
+    total_risk { 1 }
+    likelihood { 2 }
+    publish_date { DateTime.now }
+    sequence(:results_url) { |n| "http://example.com/results/#{n}" }
+  end
+end
+
+FactoryBot.modify do
+  factory :host do
+    trait :with_insights_hits do
+      after(:create) do |host, _evaluator|
+        host.insights = FactoryBot.create(:insights_facet, :with_hits, host_id: host.id)
+      end
+    end
+  end
+end

data/test/jobs/insights_full_sync_test.rb

@@ -0,0 +1,94 @@
+require 'test_helper'
+
+class InsightsFullSyncTest < ActiveJob::TestCase
+  setup do
+    @host1 = FactoryBot.create(:host, :managed, name: 'host1')
+    @host2 = FactoryBot.create(:host, :managed, name: 'host2')
+
+    hits_json = <<-HITS_JSON
+    [
+        {
+            "hostname": "#{@host1.name}",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
+            "solution_url": "",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-04-16T10:03:16Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/stability/ansible_deprecated_repo%7CANSIBLE_DEPRECATED_REPO/accdf444-5628-451d-bf3e-cf909ad72756/"
+        },
+        {
+            "hostname": "#{@host1.name}",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "CPU vulnerable to side-channel attacks using Microarchitectural Data Sampling (CVE-2018-12130, CVE-2018-12126, CVE-2018-12127, CVE-2019-11091)",
+            "solution_url": "https://access.redhat.com/node/4134081",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2019-05-14T17:00:00Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/security/CVE_2018_12130_cpu_kernel%7CCVE_2018_12130_CPU_KERNEL_NEED_UPDATE/accdf444-5628-451d-bf3e-cf909ad72756/"
+        },
+        {
+            "hostname": "#{@host2.name}",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72757",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "CPU vulnerable to side-channel attacks using L1 Terminal Fault (CVE-2018-3620)",
+            "solution_url": "https://access.redhat.com/node/3560291",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-08-14T17:00:00Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/security/CVE_2018_3620_cpu_kernel%7CCVE_2018_3620_CPU_KERNEL_NEED_UPDATE/accdf444-5628-451d-bf3e-cf909ad72756/"
+        }
+      ]
+    HITS_JSON
+    @hits = JSON.parse(hits_json)
+  end
+
+  test 'Hits data is replaced with data from cloud' do
+    InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(@hits)
+
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+
+    @host1.reload
+    @host2.reload
+
+    assert_equal 2, @host1.insights.hits.count
+    assert_equal 1, @host2.insights.hits.count
+    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72756', @host1.insights.uuid
+    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72757', @host2.insights.uuid
+  end
+
+  test 'Hits ignoring non-existent hosts' do
+    hits_json = <<-HITS_JSON
+    [
+        {
+            "hostname": "#{@host1.name}_non_existent",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
+            "solution_url": "",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-04-16T10:03:16Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/stability/ansible_deprecated_repo%7CANSIBLE_DEPRECATED_REPO/accdf444-5628-451d-bf3e-cf909ad72756/"
+        }
+    ]
+    HITS_JSON
+    hits = JSON.parse(hits_json)
+
+    InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(hits)
+
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+
+    @host1.reload
+    @host2.reload
+
+    assert_nil @host1.insights
+    assert_nil @host2.insights
+  end
+end