foreman_rh_cloud 0.9.12 → 0.9.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4410aae42d90d765e8b16d784c50196a52b86b828f59272bd21baf163e7ab7c5
-  data.tar.gz: cef39240dfcf09f438b003bb9a9cf3e802ffd23a863fe7dd98485cfc0cb0db7f
+  metadata.gz: e16b8f617e39c1d76beec174071a7ed9f06f2d8bde88c3e0201f0aeb982fa0ce
+  data.tar.gz: 629b976255cd9abee97a0e77d5f1e9263cb356c6e80ae52c43b7bdf5c7acff01
 SHA512:
-  metadata.gz: a2d171f326d72a4d0eb1070b431a476f9a7eb2857ee108f726aeac752f78236a269a2d7387857ae83a1f3be1fb1d07ddbeeec3d8c7934d6c22f750a2bf290c4e
-  data.tar.gz: 1d1afc18d5614e1160ee48bdce951dfc2275126dcd7e9b5cd8a9771c61f7651920326e9d4c6faf7374b365abd21ebb882dd13b330fc3fc45d2bd75697c7eb37e
+  metadata.gz: d3511fa267a85232d787167ecae5225e1c6a63592d353a416a5acc085388efda9046aac95030ceb57b569f8208673613799b1b11c4f49a50d5eb98c8920f1eca
+  data.tar.gz: 33a6dac98b226bc4728af4e2fea437a4878ddecfe37daccb763c7d4a999741f7aa6be331b282da67eeb00c40f6f3d87e442ce3cbe46cd34ec374bc950bbdbda3

data/app/controllers/foreman_inventory_upload/accounts_controller.rb

@@ -23,7 +23,9 @@ module ForemanInventoryUpload
       render json: {
         autoUploadEnabled: Setting[:allow_auto_inventory_upload],
         hostObfuscationEnabled: Setting[:obfuscate_inventory_hostnames],
+        ipsObfuscationEnabled: Setting[:obfuscate_inventory_ips],
         cloudToken: Setting[:rh_cloud_token],
+        excludePackages: Setting[:exclude_installed_packages],
         accounts: accounts,
       }, status: :ok
     end

data/app/controllers/foreman_inventory_upload/uploads_controller.rb

@@ -46,6 +46,24 @@ module ForemanInventoryUpload
       ActiveModel::Type::Boolean.new.cast(params.require(:value))
     end
 
+    def installed_packages_inclusion
+      Setting[:exclude_installed_packages] = host_obfuscation_params
+      render_setting(:excludePackages, :exclude_installed_packages)
+    end
+
+    def installed_packages_inclusion_params
+      ActiveModel::Type::Boolean.new.cast(params.require(:value))
+    end
+
+    def ips_obfuscation
+      Setting[:obfuscate_inventory_ips] = ips_obfuscation_params
+      render_setting(:ipsObfuscationEnabled, :obfuscate_inventory_ips)
+    end
+
+    def ips_obfuscation_params
+      ActiveModel::Type::Boolean.new.cast(params.require(:value))
+    end
+
     private
 
     def render_setting(node_name, setting)

data/app/helpers/foreman_inventory_upload_host_helper.rb

@@ -5,11 +5,10 @@ module ForemanInventoryUploadHostHelper
 
   def hits_counts_cell(host)
     host_hits = hits_counts[host.id]
-    content = n_('One recommendation', '%{hits} recommendations', host_hits) % { hits: host_hits } if host_hits
-    tag.td class: ['hidden-xs', 'ellipsis'] do
+    tag.td class: ['hidden-xs', 'ellipsis', 'text-center'] do
       if host_hits
         link_to(
-          content,
+          host_hits,
           InsightsCloud::WebUi.system_url(InsightsCloud::WebUi::ADVISOR, host.insights.uuid),
           :rel => 'external noopener noreferrer',
           :target => '_blank'

data/app/models/concerns/rh_cloud_host.rb

@@ -6,7 +6,10 @@ module RhCloudHost
       :inventory_upload_facts,
       -> { where(fact_name_id: ForemanInventoryUpload::Generators::Queries.fact_names.values) },
       class_name: 'FactValue',
-      :foreign_key => :host_id
+      foreign_key: :host_id
     )
+
+    has_many :insights_hits, through: :insights, source: :hits
+    scoped_search :relation => :insights, :on => :hits_count, :only_explicit => true, :rename => :insights_recommendations_count
   end
 end

data/app/models/insights_hit.rb

@@ -2,5 +2,5 @@ class InsightsHit < ApplicationRecord
   has_one :host
   # since the facet is one-to-one association with a host, we can connect
   # through host_id column on both this model and facet.
-  has_one :insights_facet, foreign_key: 'host_id', primary_key: 'host_id'
+  belongs_to :insights_facet, foreign_key: 'host_id', primary_key: 'host_id', counter_cache: :hits_count
 end

data/app/models/setting/rh_cloud.rb

@@ -8,7 +8,9 @@ class Setting::RhCloud < Setting
         set('allow_auto_inventory_upload', N_('Allow automatic upload of the host inventory to the Red Hat cloud'), false),
         set('allow_auto_insights_sync', N_('Allow recommendations synchronization from Red Hat cloud'), false),
         set('obfuscate_inventory_hostnames', N_('Obfuscate host names sent to Red Hat cloud'), false),
+        set('obfuscate_inventory_ips', N_('Obfuscate ip addresses sent to Red Hat cloud'), false),
         set('rh_cloud_token', N_('Authentication token to Red Hat cloud services. Used to authenticate requests to cloud APIs'), 'DEFAULT', N_('Red Hat Cloud token'), nil, encrypted: true),
+        set('exclude_installed_packages', N_('Exclude installed packages from Red Hat cloud inventory report'), false),
       ].each { |s| create! s.update(:category => 'Setting::RhCloud')}
     end
 

data/app/overrides/hosts_list.rb

@@ -2,7 +2,7 @@ Deface::Override.new(
   virtual_path: 'hosts/_list',
   name: 'insights_hits_header',
   insert_before: 'thead tr th.hidden-xs:first-of-type',
-  text: '<th class="hidden-xs" width="10%"><%= _("Insights")%></th>'
+  text: '<th class="hidden-xs ellipsis" width="12%"><%= sort :insights_recommendations_count, :as => _("Recommendations")%></th>'
 )
 
 Deface::Override.new(

data/app/services/foreman_rh_cloud/cloud_auth.rb

@@ -0,0 +1,28 @@
+module ForemanRhCloud
+  module CloudAuth
+    extend ActiveSupport::Concern
+
+    def rh_credentials
+      @rh_credentials ||= query_refresh_token
+    end
+
+    def query_refresh_token
+      token_response = RestClient::Request.execute(
+        method: :post,
+        url: ForemanRhCloud.authentication_url,
+        verify_ssl: ForemanRhCloud.verify_ssl_method,
+        proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
+        payload: {
+          grant_type: 'refresh_token',
+          client_id: 'rhsm-api',
+          refresh_token: Setting[:rh_cloud_token],
+        }
+      )
+
+      JSON.parse(token_response)['access_token']
+    rescue RestClient::ExceptionWithResponse => e
+      Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
+      raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
+    end
+  end
+end

data/config/routes.rb

@@ -8,6 +8,8 @@ Rails.application.routes.draw do
     get 'auto_upload', to: 'uploads#show_auto_upload'
     post 'auto_upload', to: 'uploads#auto_upload'
     post 'host_obfuscation', to: 'uploads#host_obfuscation'
+    post 'installed_packages_inclusion', to: 'uploads#installed_packages_inclusion'
+    post 'ips_obfuscation', to: 'uploads#ips_obfuscation'
 
     resources :tasks, only: [:create]
   end

data/db/migrate/20201007115752_add_hits_count_to_insights_facets_table.foreman_rh_cloud.rb

@@ -0,0 +1,5 @@
+class AddHitsCountToInsightsFacetsTable < ActiveRecord::Migration[5.0]
+  def change
+    add_column :insights_facets, :hits_count, :integer
+  end
+end

data/db/migrate/20201007121540_setup_hits_count_cache.foreman_rh_cloud.rb

@@ -0,0 +1,10 @@
+class SetupHitsCountCache < ActiveRecord::Migration[5.0]
+  def up
+    ActiveRecord::Base.connection.execute <<-SQL.squish
+    UPDATE insights_facets
+    SET hits_count = (SELECT count(1)
+                              FROM insights_hits
+                              WHERE insights_hits.host_id = insights_facets.host_id)
+    SQL
+  end
+end

data/lib/foreman_inventory_upload/generators/fact_helpers.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 module ForemanInventoryUpload
   module Generators
     module FactHelpers
@@ -69,6 +71,39 @@ module ForemanInventoryUpload
       def obfuscate_fqdn(fqdn)
         "#{Digest::SHA1.hexdigest(fqdn)}.example.com"
       end
+
+      def obfuscate_ips?(host)
+        insights_client_setting = fact_value(host, 'insights_client::obfuscate_ip_enabled')
+        insights_client_setting = ActiveModel::Type::Boolean.new.cast(insights_client_setting)
+        return insights_client_setting unless insights_client_setting.nil?
+
+        Setting[:obfuscate_inventory_ips]
+      end
+
+      def host_ips(host)
+        return obfuscated_ips(host) if obfuscate_ips?(host)
+
+        # return a pass through proxy hash in case no obfuscation needed
+        Hash.new { |h, k| k }
+      end
+
+      def obfuscated_ips(host)
+        insights_client_ips = JSON.parse(fact_value(host, 'insights_client::ips') || '[]')
+
+        obfuscated_ips = Hash[
+          insights_client_ips.map { |ip_record| [ip_record['original'], ip_record['obfuscated']] }
+        ]
+
+        obfuscated_ips.default_proc = proc do |hash, key|
+          hash[key] = obfuscate_ip(key, hash)
+        end
+
+        obfuscated_ips
+      end
+
+      def obfuscate_ip(ip, ips_dict)
+        "10.230.230.#{ips_dict.count + 1}"
+      end
     end
   end
 end

data/lib/foreman_inventory_upload/generators/json_stream.rb

@@ -26,8 +26,13 @@ module ForemanInventoryUpload
         @out << string
       end
 
-      def simple_field(name, value, last = false)
-        @out << "\"#{name}\": #{stringify_value(value)}#{last ? '' : ','}" unless value.nil?
+      def simple_field(name, value, last = false, &block)
+        return if value.nil? || value.try(:empty?)
+        return if value.kind_of?(Array) && value.compact.empty?
+
+        block ||= ->(value) { value }
+
+        @out << "\"#{name}\": #{stringify_value(block.call(value))}#{last ? '' : ','}"
       end
 
       def array_field(name, last = false, &block)

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -24,7 +24,9 @@ module ForemanInventoryUpload
               'dmi::system::product_name',
               'dmi::chassis::asset_tag',
               'insights_client::obfuscate_hostname_enabled',
+              'insights_client::obfuscate_ip_enabled',
               'insights_client::hostname',
+              'insights_client::ips',
             ]).pluck(:name, :id)
           ]
       end

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -40,6 +40,7 @@ module ForemanInventoryUpload
       end
 
       def report_host(host)
+        host_ips_cache = host_ips(host)
         @stream.object do
           @stream.simple_field('fqdn', fqdn(host))
           @stream.simple_field('account', account_id(host.organization).to_s)
@@ -47,10 +48,10 @@ module ForemanInventoryUpload
           @stream.simple_field('satellite_id', host.subscription_facet&.uuid)
           @stream.simple_field('bios_uuid', fact_value(host, 'dmi::system::uuid'))
           @stream.simple_field('vm_uuid', fact_value(host, 'virt::uuid'))
-          report_ip_addresses(host)
+          report_ip_addresses(host, host_ips_cache)
           report_mac_addresses(host)
           @stream.object_field('system_profile') do
-            report_system_profile(host)
+            report_system_profile(host, host_ips_cache)
           end
           @stream.array_field('facts') do
             @stream.object do
@@ -96,15 +97,15 @@ module ForemanInventoryUpload
         end
       end
 
-      def report_system_profile(host)
-        @stream.simple_field('number_of_cpus', fact_value(host, 'cpu::cpu(s)').to_i)
-        @stream.simple_field('number_of_sockets', fact_value(host, 'cpu::cpu_socket(s)').to_i)
-        @stream.simple_field('cores_per_socket', fact_value(host, 'cpu::core(s)_per_socket').to_i)
-        @stream.simple_field('system_memory_bytes', kilobytes_to_bytes(fact_value(host, 'memory::memtotal').to_i))
+      def report_system_profile(host, host_ips_cache)
+        @stream.simple_field('number_of_cpus', fact_value(host, 'cpu::cpu(s)')) { |v| v.to_i }
+        @stream.simple_field('number_of_sockets', fact_value(host, 'cpu::cpu_socket(s)')) { |v| v.to_i }
+        @stream.simple_field('cores_per_socket', fact_value(host, 'cpu::core(s)_per_socket')) { |v| v.to_i }
+        @stream.simple_field('system_memory_bytes', fact_value(host, 'memory::memtotal')) { |v| kilobytes_to_bytes(v.to_i) }
         @stream.array_field('network_interfaces') do
           @stream.raw(host.interfaces.map do |nic|
             {
-              'ipv4_addresses': [nic.ip].compact,
+              'ipv4_addresses': [host_ips_cache[nic.ip]].compact,
               'ipv6_addresses': [nic.ip6].compact,
               'mtu': nic.try(:mtu),
               'mac_address': nic.mac,
@@ -123,18 +124,16 @@ module ForemanInventoryUpload
           end
         end
         @stream.simple_field(
-          'os_release',
-          os_release_value(
-            name: fact_value(host, 'distribution::name'),
-            version: fact_value(host, 'distribution::version'),
-            codename: fact_value(host, 'distribution::id')
-          )
-        )
+          'os_release', [
+            fact_value(host, 'distribution::name'),
+            fact_value(host, 'distribution::version'),
+            fact_value(host, 'distribution::id'),
+          ]
+        ) { |v| os_release_value(*v) }
         @stream.simple_field('os_kernel_version', fact_value(host, 'uname::release'))
         @stream.simple_field('arch', host.architecture&.name)
         @stream.simple_field('subscription_status', host.subscription_status_label)
         @stream.simple_field('katello_agent_running', host.content_facet&.katello_agent_installed?)
-        @stream.simple_field('satellite_managed', true)
         @stream.simple_field(
           'infrastructure_type',
           ActiveModel::Type::Boolean.new.cast(fact_value(host, 'virt::is_guest')) ? 'virtual' : 'physical'
@@ -150,13 +149,16 @@ module ForemanInventoryUpload
             end.join(', '))
           end
         end
-        @stream.array_field('installed_packages', :last) do
-          first = true
-          host.installed_packages.each do |package|
-            @stream.raw("#{first ? '' : ', '}#{@stream.stringify_value(package.nvra)}")
-            first = false
+        unless Setting[:exclude_installed_packages]
+          @stream.array_field('installed_packages') do
+            first = true
+            host.installed_packages.each do |package|
+              @stream.raw("#{first ? '' : ', '}#{@stream.stringify_value(package.nvra)}")
+              first = false
+            end
           end
         end
+        @stream.simple_field('satellite_managed', true, :last)
       end
 
       def report_satellite_facts(host)
@@ -175,8 +177,8 @@ module ForemanInventoryUpload
         @stream.simple_field('organization_id', host.organization_id, :last)
       end
 
-      def report_ip_addresses(host)
-        ip_addresses = host.interfaces.map { |nic| nic.ip }.compact
+      def report_ip_addresses(host, host_ips_cache)
+        ip_addresses = host.interfaces.map { |nic| host_ips_cache[nic.ip] }.compact
 
         @stream.string_array_value('ip_addresses', ip_addresses)
       end
@@ -187,7 +189,7 @@ module ForemanInventoryUpload
         @stream.string_array_value('mac_addresses', macs)
       end
 
-      def os_release_value(name:, version:, codename:)
+      def os_release_value(name, version, codename)
         "#{name} #{version} (#{codename})"
       end
     end

data/lib/foreman_rh_cloud.rb

@@ -27,7 +27,8 @@ module ForemanRhCloud
 
   def self.proxy_setting(logger: Foreman::Logging.logger('background'))
     ForemanRhCloud.cdn_proxy(logger: logger) ||
-    ForemanRhCloud.global_foreman_proxy
+    ForemanRhCloud.global_foreman_proxy ||
+    ''
   end
 
   def self.cdn_proxy(logger: Foreman::Logging.logger('app'))

data/lib/foreman_rh_cloud/engine.rb

@@ -13,6 +13,7 @@ module ForemanRhCloud
     config.autoload_paths += Dir["#{config.root}/app/helpers/concerns"]
     config.autoload_paths += Dir["#{config.root}/app/models/concerns"]
     config.autoload_paths += Dir["#{config.root}/app/overrides"]
+    config.autoload_paths += Dir["#{config.root}/app/services"]
     config.autoload_paths += Dir["#{config.root}/lib"]
 
     config.eager_load_paths += Dir["#{config.root}/lib"]
@@ -54,8 +55,8 @@ module ForemanRhCloud
 
         # Adding a sub menu after hosts menu
         divider :top_menu, caption: N_('RH Cloud'), parent: :configure_menu
-        menu :top_menu, :inventory_upload, :caption => N_('Inventory Upload'), :url_hash => { controller: :'foreman_rh_cloud/react', :action => :inventory_upload}, parent: :configure_menu
-        menu :top_menu, :insights_hits_import, :caption => N_('Insights'), :url_hash => { controller: :'foreman_rh_cloud/react', :action => :insights_cloud }, parent: :configure_menu
+        menu :top_menu, :inventory_upload, caption: N_('Inventory Upload'), url_hash: { controller: :'foreman_rh_cloud/react', action: :inventory_upload }, parent: :configure_menu
+        menu :top_menu, :insights_hits_import, caption: N_('Insights'), url_hash: { controller: :'foreman_rh_cloud/react', action: :insights_cloud }, parent: :configure_menu
 
         register_facet InsightsFacet, :insights do
           set_dependent_action :destroy

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '0.9.12'.freeze
+  VERSION = '0.9.13'.freeze
 end

data/lib/insights_cloud/async/insights_full_sync.rb

@@ -3,6 +3,8 @@ require 'rest-client'
 module InsightsCloud
   module Async
     class InsightsFullSync < ::ApplicationJob
+      include ::ForemanRhCloud::CloudAuth
+
       def perform
         hits = query_insights_hits
 
@@ -16,10 +18,6 @@ module InsightsCloud
         Foreman::Logging.logger('background')
       end
 
-      def rh_credentials
-        @rh_credentials ||= query_refresh_token
-      end
-
       private
 
       def query_insights_hits
@@ -36,25 +34,6 @@ module InsightsCloud
         JSON.parse(hits_response)
       end
 
-      def query_refresh_token
-        token_response = RestClient::Request.execute(
-          method: :post,
-          url: ForemanRhCloud.authentication_url,
-          verify_ssl: ForemanRhCloud.verify_ssl_method,
-          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
-          payload: {
-            grant_type: 'refresh_token',
-            client_id: 'rhsm-api',
-            refresh_token: Setting[:rh_cloud_token],
-          }
-        )
-
-        JSON.parse(token_response)['access_token']
-      rescue RestClient::ExceptionWithResponse => e
-        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
-        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
-      end
-
       def setup_host_names(host_names)
         @host_ids = Hash[
           Host.unscoped.where(name: host_names).pluck(:name, :id)
@@ -67,8 +46,6 @@ module InsightsCloud
 
       def replace_hits_data(hits)
         InsightsHit.transaction do
-          InsightsHit.delete_all
-          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) }.compact)
           # create new facets for hosts that are missing one
           hosts_with_existing_facets = InsightsFacet.where(host_id: @host_ids.values).pluck(:host_id)
           InsightsFacet.create(
@@ -81,6 +58,8 @@ module InsightsCloud
               end
             end.compact
           )
+          InsightsHit.delete_all
+          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) }.compact)
         end
       end
 

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -3,6 +3,8 @@ require 'rest-client'
 module InventorySync
   module Async
     class InventoryFullSync < ::ApplicationJob
+      include ::ForemanRhCloud::CloudAuth
+
       def perform(organization)
         @organization = organization
         @subscribed_hosts_ids = Set.new(
@@ -37,10 +39,6 @@ module InventorySync
         @host_statuses
       end
 
-      def rh_credentials
-        @rh_credentials ||= query_refresh_token
-      end
-
       private
 
       def update_hosts_status(status_hashes, touched)
@@ -77,25 +75,6 @@ module InventorySync
 
         JSON.parse(hosts_inventory_response)
       end
-
-      def query_refresh_token
-        token_response = RestClient::Request.execute(
-          method: :post,
-          url: ForemanRhCloud.authentication_url,
-          verify_ssl: ForemanRhCloud.verify_ssl_method,
-          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
-          payload: {
-            grant_type: 'refresh_token',
-            client_id: 'rhsm-api',
-            refresh_token: Setting[:rh_cloud_token],
-          }
-        )
-
-        JSON.parse(token_response)['access_token']
-      rescue RestClient::ExceptionWithResponse => e
-        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
-        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
-      end
     end
   end
 end