foreman_rh_cloud 1.0.7 → 1.0.12

data/lib/insights_cloud/async/insights_scheduled_sync.rb

@@ -0,0 +1,23 @@
+module InsightsCloud
+  module Async
+    class InsightsScheduledSync < ::ApplicationJob
+      def perform
+        unless Setting[:allow_auto_insights_sync]
+          logger.debug(
+            'The scheduled process is disabled due to the "allow_auto_insights_sync"
+            setting being set to false.'
+          )
+          return
+        end
+
+        InsightsFullSync.perform_later()
+      ensure
+        self.class.set(:wait => 24.hours).perform_later
+      end
+
+      def self.singleton_job_name
+        name
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/host_result.rb

@@ -0,0 +1,50 @@
+module InventorySync
+  module Async
+    class HostResult
+      def initialize(result)
+        @total = result['total']
+        @count = result['count']
+        @page = result['page']
+        @per_page = result['per_page']
+        @fqdns = result["results"].map { |host| host['fqdn'] }
+      end
+
+      def status_hashes
+        @fqdns.map do |fqdn|
+          host_id = host_id(fqdn)
+          if host_id
+            touched << host_id
+            {
+              host_id: host_id,
+              status: InventorySync::InventoryStatus::SYNC,
+              reported_at: DateTime.current,
+            }
+          end
+        end.compact
+      end
+
+      def touched
+        @touched ||= []
+      end
+
+      def host_id(fqdn)
+        hosts[fqdn]
+      end
+
+      def hosts
+        @hosts ||= Hash[
+          Host.where(name: @fqdns).pluck(:name, :id)
+        ]
+      end
+
+      def percentage
+        ratio = @per_page * @page * 1.0 / @total * 100
+        (ratio > 100) ? 100 : ratio.truncate(2)
+      end
+
+      def last?
+        @total <= @per_page * @page
+      end
+    end
+  end
+end

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -0,0 +1,101 @@
+require 'rest-client'
+
+module InventorySync
+  module Async
+    class InventoryFullSync < ::ApplicationJob
+      def perform(organization)
+        @organization = organization
+        @subscribed_hosts_ids = Set.new(
+          ForemanInventoryUpload::Generators::Queries.for_slice(
+            Host.unscoped.where(organization: organization)
+          ).pluck(:id)
+        )
+        @host_statuses = {
+          sync: 0,
+          disconnect: 0,
+        }
+
+        InventorySync::InventoryStatus.transaction do
+          InventorySync::InventoryStatus.where(host_id: @subscribed_hosts_ids).delete_all
+          page = 1
+          loop do
+            api_response = query_inventory(page)
+            results = HostResult.new(api_response)
+            logger.debug("Downloading cloud inventory data: #{results.percentage}%")
+            update_hosts_status(results.status_hashes, results.touched)
+            @host_statuses[:sync] += results.touched.size
+            page += 1
+            break if results.last?
+          end
+          add_missing_hosts_statuses(@subscribed_hosts_ids)
+          @host_statuses[:disconnect] += @subscribed_hosts_ids.size
+        end
+
+        logger.debug("Synced hosts amount: #{@host_statuses[:sync]}")
+        logger.debug("Disconnected hosts amount: #{@host_statuses[:disconnect]}")
+
+        @host_statuses
+      end
+
+      def rh_credentials
+        @rh_credentials ||= query_refresh_token
+      end
+
+      private
+
+      def update_hosts_status(status_hashes, touched)
+        InventorySync::InventoryStatus.create(status_hashes)
+        @subscribed_hosts_ids.subtract(touched)
+      end
+
+      def add_missing_hosts_statuses(hosts_ids)
+        InventorySync::InventoryStatus.create(
+          hosts_ids.map do |host_id|
+            {
+              host_id: host_id,
+              status: InventorySync::InventoryStatus::DISCONNECT,
+              reported_at: DateTime.current,
+            }
+          end
+        )
+      end
+
+      def query_inventory(page = 1)
+        hosts_inventory_response = RestClient::Request.execute(
+          method: :get,
+          url: ForemanInventoryUpload.inventory_export_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
+          headers: {
+            Authorization: "Bearer #{rh_credentials}",
+            params: {
+              per_page: 100,
+              page: page,
+            },
+          }
+        )
+
+        JSON.parse(hosts_inventory_response)
+      end
+
+      def query_refresh_token
+        token_response = RestClient::Request.execute(
+          method: :post,
+          url: ForemanRhCloud.authentication_url,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
+          payload: {
+            grant_type: 'refresh_token',
+            client_id: 'rhsm-api',
+            refresh_token: Setting[:rh_cloud_token],
+          }
+        )
+
+        JSON.parse(token_response)['access_token']
+      rescue RestClient::ExceptionWithResponse => e
+        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
+        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
+      end
+    end
+  end
+end

data/lib/tasks/insights.rake

@@ -0,0 +1,15 @@
+namespace :rh_cloud_insights do
+  desc "Synchronize Insights inventory"
+  task sync: :environment do
+    if ! ENV['organization_id'].nil?
+      organizations = [ Organization.where(:id => ENV['organization_id']).first ]
+    else
+      organizations = Organization.all
+    end
+
+    organizations.each do |organization|
+      InventorySync::Async::InventoryFullSync.perform_now(organization)
+      puts "Synchronized inventory for organization '#{organization.name}'"
+    end
+  end
+end

data/lib/tasks/rh_cloud_inventory.rake

@@ -0,0 +1,37 @@
+require 'tempfile'
+
+namespace :rh_cloud_inventory do
+  namespace :report do
+    desc 'Generate inventory report to be sent to Red Hat cloud'
+    task generate: :environment do
+      portal_user = ENV['portal_user']
+      organizations = [ENV['organization_id']]
+      base_folder = ENV['target'] || Dir.pwd
+
+      unless portal_user || organizations.empty?
+        puts "Must specify either portal_user or organization_id"
+      end
+
+      User.as_anonymous_admin do
+        if portal_user
+          puts "Generating report for all organizations associated with #{portal_user}"
+          base_folder = File.join(base_folder, portal_user)
+          organizations = ForemanInventoryUpload::Generators::Queries.organizations_for_user(portal_user).pluck(:id)
+        end
+
+        organizations.each do |organization|
+          target = File.join(base_folder, ForemanInventoryUpload.facts_archive_name(organization))
+          archived_report_generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
+          archived_report_generator.render(organization: organization)
+          puts "Successfully generated #{target} for organization id #{organization}"
+        end
+      end
+    end
+  end
+
+  desc "Synchronize Insights hosts hits"
+  task sync: :environment do
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+    puts "Synchronized Insights hosts hits data"
+  end
+end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "0.0.1",
+  "version": "1.0.12",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {
@@ -47,6 +47,8 @@
     "eslint": "^4.10.0",
     "eslint-import-resolver-babel-module": "^4.0.0",
     "eslint-plugin-patternfly-react": "0.2.0",
+    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-spellcheck": "^0.0.17",
     "identity-obj-proxy": "^3.0.0",
     "jed": "^1.1.1",
     "jest-cli": "^23.6.0",

data/test/controllers/insights_sync/settings_controller_test.rb

@@ -0,0 +1,30 @@
+require 'test_plugin_helper'
+
+class SettingsControllerTest < ActionController::TestCase
+  tests InsightsCloud::SettingsController
+
+  test 'should return allow_auto_insights_sync setting' do
+    FactoryBot.create(:setting, :name => 'allow_auto_insights_sync', :settings_type => "boolean", :category => "Setting::RhCloud", :default => false, :value => false)
+
+    assert_equal false, Setting[:allow_auto_insights_sync]
+
+    get :show, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal false, actual['insightsSyncEnabled']
+  end
+
+  test 'should update allow_auto_insights_sync setting' do
+    FactoryBot.create(:setting, :name => 'allow_auto_insights_sync', :settings_type => "boolean", :category => "Setting::RhCloud", :default => false, :value => false)
+
+    assert_equal false, Setting[:allow_auto_insights_sync]
+
+    patch :update, params: { insightsSyncEnabled: true }, session: set_session_user
+
+    assert_response :success
+    actual = JSON.parse(response.body)
+    assert_equal true, actual['insightsSyncEnabled']
+    assert_equal true, Setting[:allow_auto_insights_sync]
+  end
+end

data/test/factories/insights_factories.rb

@@ -0,0 +1,31 @@
+FactoryBot.define do
+  factory :insights_facet do
+    # sequence(:uuid) { |n| "uuid-#{n}" }
+
+    trait :with_hits do
+      hits do
+        Array.new(1) { association(:insights_hit, host_id: host_id) }
+      end
+    end
+  end
+
+  factory :insights_hit do
+    last_seen { DateTime.yesterday }
+    sequence(:title) { |n| "hit title #{n}" }
+    sequence(:solution_url) { |n| "http://example.com/solutions/#{n}" }
+    total_risk { 1 }
+    likelihood { 2 }
+    publish_date { DateTime.now }
+    sequence(:results_url) { |n| "http://example.com/results/#{n}" }
+  end
+end
+
+FactoryBot.modify do
+  factory :host do
+    trait :with_insights_hits do
+      after(:create) do |host, _evaluator|
+        host.insights = FactoryBot.create(:insights_facet, :with_hits, host_id: host.id)
+      end
+    end
+  end
+end

data/test/jobs/insights_full_sync_test.rb

@@ -0,0 +1,94 @@
+require 'test_helper'
+
+class InsightsFullSyncTest < ActiveJob::TestCase
+  setup do
+    @host1 = FactoryBot.create(:host, :managed, name: 'host1')
+    @host2 = FactoryBot.create(:host, :managed, name: 'host2')
+
+    hits_json = <<-HITS_JSON
+    [
+        {
+            "hostname": "#{@host1.name}",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
+            "solution_url": "",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-04-16T10:03:16Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/stability/ansible_deprecated_repo%7CANSIBLE_DEPRECATED_REPO/accdf444-5628-451d-bf3e-cf909ad72756/"
+        },
+        {
+            "hostname": "#{@host1.name}",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "CPU vulnerable to side-channel attacks using Microarchitectural Data Sampling (CVE-2018-12130, CVE-2018-12126, CVE-2018-12127, CVE-2019-11091)",
+            "solution_url": "https://access.redhat.com/node/4134081",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2019-05-14T17:00:00Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/security/CVE_2018_12130_cpu_kernel%7CCVE_2018_12130_CPU_KERNEL_NEED_UPDATE/accdf444-5628-451d-bf3e-cf909ad72756/"
+        },
+        {
+            "hostname": "#{@host2.name}",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72757",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "CPU vulnerable to side-channel attacks using L1 Terminal Fault (CVE-2018-3620)",
+            "solution_url": "https://access.redhat.com/node/3560291",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-08-14T17:00:00Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/security/CVE_2018_3620_cpu_kernel%7CCVE_2018_3620_CPU_KERNEL_NEED_UPDATE/accdf444-5628-451d-bf3e-cf909ad72756/"
+        }
+      ]
+    HITS_JSON
+    @hits = JSON.parse(hits_json)
+  end
+
+  test 'Hits data is replaced with data from cloud' do
+    InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(@hits)
+
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+
+    @host1.reload
+    @host2.reload
+
+    assert_equal 2, @host1.insights.hits.count
+    assert_equal 1, @host2.insights.hits.count
+    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72756', @host1.insights.uuid
+    assert_equal 'accdf444-5628-451d-bf3e-cf909ad72757', @host2.insights.uuid
+  end
+
+  test 'Hits ignoring non-existent hosts' do
+    hits_json = <<-HITS_JSON
+    [
+        {
+            "hostname": "#{@host1.name}_non_existent",
+            "rhel_version": "7.5",
+            "uuid": "accdf444-5628-451d-bf3e-cf909ad72756",
+            "last_seen": "2019-11-22T08:41:42.447244Z",
+            "title": "New Ansible Engine packages are inaccessible when dedicated Ansible repo is not enabled",
+            "solution_url": "",
+            "total_risk": 2,
+            "likelihood": 2,
+            "publish_date": "2018-04-16T10:03:16Z",
+            "results_url": "https://cloud.redhat.com/insights/overview/stability/ansible_deprecated_repo%7CANSIBLE_DEPRECATED_REPO/accdf444-5628-451d-bf3e-cf909ad72756/"
+        }
+    ]
+    HITS_JSON
+    hits = JSON.parse(hits_json)
+
+    InsightsCloud::Async::InsightsFullSync.any_instance.expects(:query_insights_hits).returns(hits)
+
+    InsightsCloud::Async::InsightsFullSync.perform_now()
+
+    @host1.reload
+    @host2.reload
+
+    assert_nil @host1.insights
+    assert_nil @host2.insights
+  end
+end

data/test/jobs/inventory_full_sync_test.rb

@@ -0,0 +1,91 @@
+require 'test_plugin_helper'
+
+class InventoryFullSyncTest < ActiveJob::TestCase
+  setup do
+    @host1 = FactoryBot.create(:host, :managed, name: 'host1')
+
+    User.current = User.find_by(login: 'secret_admin')
+
+    env = FactoryBot.create(:katello_k_t_environment)
+    cv = env.content_views << FactoryBot.create(:katello_content_view, organization: env.organization)
+
+    # this host would pass our plugin queries, so it could be uploaded to the cloud.
+    @host2 = FactoryBot.create(
+      :host,
+      :with_subscription,
+      :with_content,
+      content_view: cv.first,
+      lifecycle_environment: env,
+      organization: env.organization
+    )
+
+    @host2.subscription_facet.pools << FactoryBot.create(:katello_pool, account_number: '1234', cp_id: 1)
+
+    ForemanInventoryUpload::Generators::Queries.instance_variable_set(:@fact_names, nil)
+
+    inventory_json = <<-INVENTORY_JSON
+    {
+      "total": 3,
+      "count": 3,
+      "page": 1,
+      "per_page": 50,
+      "results": [{"fqdn": "#{@host1.fqdn}"}]
+    }
+    INVENTORY_JSON
+    @inventory = JSON.parse(inventory_json)
+  end
+
+  def interesting_facts
+    [
+      'dmi::system::uuid',
+      'virt::uuid',
+      'cpu::cpu(s)',
+      'cpu::cpu_socket(s)',
+      'cpu::core(s)_per_socket',
+      'memory::memtotal',
+      'dmi::bios::vendor',
+      'dmi::bios::version',
+      'dmi::bios::relase_date',
+      'uname::release',
+      'lscpu::flags',
+      'distribution::name',
+      'distribution::version',
+      'distribution::id',
+      'virt::is_guest',
+      'dmi::system::manufacturer',
+      'dmi::system::product_name',
+      'dmi::chassis::asset_tag',
+      'insights_client::obfuscate_hostname_enabled',
+      'insights_client::hostname',
+    ]
+  end
+
+  def fact_names
+    @fact_names ||= Hash[
+      interesting_facts.map do |fact|
+        [fact, FactoryBot.create(:fact_name, name: fact, type: 'Katello::RhsmFactName')]
+      end
+    ]
+  end
+
+  test 'Host status should be SYNC for inventory hosts' do
+    InventorySync::Async::InventoryFullSync.any_instance.expects(:query_inventory).returns(@inventory)
+
+    InventorySync::Async::InventoryFullSync.perform_now(@host1.organization)
+
+    @host1.reload
+
+    assert_equal InventorySync::InventoryStatus::SYNC, InventorySync::InventoryStatus.where(host_id: @host1.id).first.status
+  end
+
+  test 'Host status should be DISCONNECT for hosts that are not returned from cloud' do
+    InventorySync::Async::InventoryFullSync.any_instance.expects(:query_inventory).returns(@inventory)
+    FactoryBot.create(:fact_value, fact_name: fact_names['virt::uuid'], value: '1234', host: @host2)
+
+    InventorySync::Async::InventoryFullSync.perform_now(@host2.organization)
+
+    @host2.reload
+
+    assert_equal InventorySync::InventoryStatus::DISCONNECT, InventorySync::InventoryStatus.where(host_id: @host2.id).first.status
+  end
+end