foreman_rh_cloud 0.9.8 → 0.9.13

data/lib/foreman_inventory_upload/generators/fact_helpers.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 module ForemanInventoryUpload
   module Generators
     module FactHelpers
@@ -67,7 +69,40 @@ module ForemanInventoryUpload
       end
 
       def obfuscate_fqdn(fqdn)
-        Digest::SHA1.hexdigest(fqdn)
+        "#{Digest::SHA1.hexdigest(fqdn)}.example.com"
+      end
+
+      def obfuscate_ips?(host)
+        insights_client_setting = fact_value(host, 'insights_client::obfuscate_ip_enabled')
+        insights_client_setting = ActiveModel::Type::Boolean.new.cast(insights_client_setting)
+        return insights_client_setting unless insights_client_setting.nil?
+
+        Setting[:obfuscate_inventory_ips]
+      end
+
+      def host_ips(host)
+        return obfuscated_ips(host) if obfuscate_ips?(host)
+
+        # return a pass through proxy hash in case no obfuscation needed
+        Hash.new { |h, k| k }
+      end
+
+      def obfuscated_ips(host)
+        insights_client_ips = JSON.parse(fact_value(host, 'insights_client::ips') || '[]')
+
+        obfuscated_ips = Hash[
+          insights_client_ips.map { |ip_record| [ip_record['original'], ip_record['obfuscated']] }
+        ]
+
+        obfuscated_ips.default_proc = proc do |hash, key|
+          hash[key] = obfuscate_ip(key, hash)
+        end
+
+        obfuscated_ips
+      end
+
+      def obfuscate_ip(ip, ips_dict)
+        "10.230.230.#{ips_dict.count + 1}"
       end
     end
   end

data/lib/foreman_inventory_upload/generators/json_stream.rb

@@ -26,8 +26,13 @@ module ForemanInventoryUpload
         @out << string
       end
 
-      def simple_field(name, value, last = false)
-        @out << "\"#{name}\": #{stringify_value(value)}#{last ? '' : ','}" unless value.nil?
+      def simple_field(name, value, last = false, &block)
+        return if value.nil? || value.try(:empty?)
+        return if value.kind_of?(Array) && value.compact.empty?
+
+        block ||= ->(value) { value }
+
+        @out << "\"#{name}\": #{stringify_value(block.call(value))}#{last ? '' : ','}"
       end
 
       def array_field(name, last = false, &block)
@@ -36,6 +41,16 @@ module ForemanInventoryUpload
         @out << ',' unless last
       end
 
+      def string_array_value(name, value, last = false)
+        return if value.empty?
+
+        string_value = value.map { |v| stringify_value(v) }
+
+        array_field(name, last) do
+          raw(string_value.join(', '))
+        end
+      end
+
       def object_field(name, last = false, &block)
         @out << "\"#{name}\": "
         object(&block)

data/lib/foreman_inventory_upload/generators/metadata.rb

@@ -21,6 +21,9 @@ module ForemanInventoryUpload
       private
 
       def render_report(metadata)
+        metadata ||= {}
+        metadata['foreman_rh_cloud_version'] = ForemanRhCloud::VERSION
+
         @stream.object do
           @stream.simple_field('report_id', Foreman.uuid)
           @stream.simple_field('host_inventory_api_version', '1.0')

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -24,29 +24,29 @@ module ForemanInventoryUpload
               'dmi::system::product_name',
               'dmi::chassis::asset_tag',
               'insights_client::obfuscate_hostname_enabled',
+              'insights_client::obfuscate_ip_enabled',
               'insights_client::hostname',
+              'insights_client::ips',
             ]).pluck(:name, :id)
           ]
       end
 
       def self.for_slice(base)
-        fact_values = FactValue.where(fact_name_id: fact_names.values)
         base
           .joins(:subscription_facet)
-          .eager_load(:fact_values)
           .preload(
             :interfaces,
             :installed_packages,
             :content_facet,
             :host_statuses,
+            :inventory_upload_facts,
             subscription_facet: [:pools, :installed_products, :hypervisor_host]
           )
-          .merge(fact_values)
       end
 
       def self.for_report(portal_user)
         org_ids = organizations_for_user(portal_user).pluck(:id)
-        for_slice(Host.unscoped.where(organization_id: org_ids)).in_batches(of: 1_000)
+        for_org(org_ids)
       end
 
       def self.for_org(organization_id)

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -3,6 +3,8 @@ module ForemanInventoryUpload
     class Slice
       include FactHelpers
 
+      SATELLITE_NAMESPACE = 'satellite'
+
       attr_accessor :slice_id
       attr_reader :hosts_count
 
@@ -38,6 +40,7 @@ module ForemanInventoryUpload
       end
 
       def report_host(host)
+        host_ips_cache = host_ips(host)
         @stream.object do
           @stream.simple_field('fqdn', fqdn(host))
           @stream.simple_field('account', account_id(host.organization).to_s)
@@ -45,22 +48,14 @@ module ForemanInventoryUpload
           @stream.simple_field('satellite_id', host.subscription_facet&.uuid)
           @stream.simple_field('bios_uuid', fact_value(host, 'dmi::system::uuid'))
           @stream.simple_field('vm_uuid', fact_value(host, 'virt::uuid'))
-          @stream.array_field('ip_addresses') do
-            @stream.raw(host.interfaces.map do |nic|
-              @stream.stringify_value(nic.ip) if nic.ip
-            end.compact.join(', '))
-          end
-          @stream.array_field('mac_addresses') do
-            @stream.raw(host.interfaces.map do |nic|
-              @stream.stringify_value(nic.mac) if nic.mac
-            end.compact.join(', '))
-          end
+          report_ip_addresses(host, host_ips_cache)
+          report_mac_addresses(host)
           @stream.object_field('system_profile') do
-            report_system_profile(host)
+            report_system_profile(host, host_ips_cache)
           end
           @stream.array_field('facts') do
             @stream.object do
-              @stream.simple_field('namespace', 'satellite')
+              @stream.simple_field('namespace', SATELLITE_NAMESPACE)
               @stream.object_field('facts', :last) do
                 report_satellite_facts(host)
               end
@@ -68,8 +63,15 @@ module ForemanInventoryUpload
           end
 
           @stream.array_field('tags', :last) do
-            report_tag('satellite', 'satellite_instance_id', Foreman.instance_id) if Foreman.respond_to?(:instance_id)
-            report_tag('satellite', 'organization_id', host.organization_id.to_s, :last)
+            report_satellite_tag('satellite_instance_id', Foreman.instance_id) if Foreman.respond_to?(:instance_id)
+            report_satellite_tag('lifecycle_environment', host.lifecycle_environment&.name)
+            report_satellite_tag('content_view', host.content_view&.name)
+            report_satellite_tag('activation_key', host.activation_keys) { |key| key.name }
+            report_satellite_tag('host_collection', host.host_collections) { |collection| collection.name }
+            report_satellite_tag('location', host.location.name)
+            report_satellite_tag('organization', host.organization.name)
+            report_satellite_tag('hostgroup', host.hostgroup&.name)
+            report_satellite_tag('organization_id', host.organization_id.to_s, :last)
           end
         end
       end
@@ -83,15 +85,27 @@ module ForemanInventoryUpload
         @stream.comma unless last
       end
 
-      def report_system_profile(host)
-        @stream.simple_field('number_of_cpus', fact_value(host, 'cpu::cpu(s)').to_i)
-        @stream.simple_field('number_of_sockets', fact_value(host, 'cpu::cpu_socket(s)').to_i)
-        @stream.simple_field('cores_per_socket', fact_value(host, 'cpu::core(s)_per_socket').to_i)
-        @stream.simple_field('system_memory_bytes', kilobytes_to_bytes(fact_value(host, 'memory::memtotal').to_i))
+      def report_satellite_tag(key, value, last = nil)
+        return if value.nil?
+
+        array_value = Array(value)
+        last_index = array_value.count - 1
+
+        array_value.each_with_index do |value, index|
+          value = yield(value) if block_given?
+          report_tag(SATELLITE_NAMESPACE, key, value, last && index == last_index)
+        end
+      end
+
+      def report_system_profile(host, host_ips_cache)
+        @stream.simple_field('number_of_cpus', fact_value(host, 'cpu::cpu(s)')) { |v| v.to_i }
+        @stream.simple_field('number_of_sockets', fact_value(host, 'cpu::cpu_socket(s)')) { |v| v.to_i }
+        @stream.simple_field('cores_per_socket', fact_value(host, 'cpu::core(s)_per_socket')) { |v| v.to_i }
+        @stream.simple_field('system_memory_bytes', fact_value(host, 'memory::memtotal')) { |v| kilobytes_to_bytes(v.to_i) }
         @stream.array_field('network_interfaces') do
           @stream.raw(host.interfaces.map do |nic|
             {
-              'ipv4_addresses': [nic.ip].compact,
+              'ipv4_addresses': [host_ips_cache[nic.ip]].compact,
               'ipv6_addresses': [nic.ip6].compact,
               'mtu': nic.try(:mtu),
               'mac_address': nic.mac,
@@ -110,18 +124,16 @@ module ForemanInventoryUpload
           end
         end
         @stream.simple_field(
-          'os_release',
-          os_release_value(
-            name: fact_value(host, 'distribution::name'),
-            version: fact_value(host, 'distribution::version'),
-            codename: fact_value(host, 'distribution::id')
-          )
-        )
+          'os_release', [
+            fact_value(host, 'distribution::name'),
+            fact_value(host, 'distribution::version'),
+            fact_value(host, 'distribution::id'),
+          ]
+        ) { |v| os_release_value(*v) }
         @stream.simple_field('os_kernel_version', fact_value(host, 'uname::release'))
         @stream.simple_field('arch', host.architecture&.name)
         @stream.simple_field('subscription_status', host.subscription_status_label)
         @stream.simple_field('katello_agent_running', host.content_facet&.katello_agent_installed?)
-        @stream.simple_field('satellite_managed', true)
         @stream.simple_field(
           'infrastructure_type',
           ActiveModel::Type::Boolean.new.cast(fact_value(host, 'virt::is_guest')) ? 'virtual' : 'physical'
@@ -137,13 +149,16 @@ module ForemanInventoryUpload
             end.join(', '))
           end
         end
-        @stream.array_field('installed_packages', :last) do
-          first = true
-          host.installed_packages.each do |package|
-            @stream.raw("#{first ? '' : ', '}#{@stream.stringify_value(package.nvra)}")
-            first = false
+        unless Setting[:exclude_installed_packages]
+          @stream.array_field('installed_packages') do
+            first = true
+            host.installed_packages.each do |package|
+              @stream.raw("#{first ? '' : ', '}#{@stream.stringify_value(package.nvra)}")
+              first = false
+            end
           end
         end
+        @stream.simple_field('satellite_managed', true, :last)
       end
 
       def report_satellite_facts(host)
@@ -154,6 +169,7 @@ module ForemanInventoryUpload
         end
         @stream.simple_field('system_purpose_usage', host.subscription_facet.purpose_usage)
         @stream.simple_field('system_purpose_role', host.subscription_facet.purpose_role)
+        @stream.simple_field('system_purpose_sla', host.subscription_facet.service_level)
         @stream.simple_field('distribution_version', fact_value(host, 'distribution::version'))
         @stream.simple_field('satellite_instance_id', Foreman.try(:instance_id))
         @stream.simple_field('is_simple_content_access', golden_ticket?(host.organization))
@@ -161,7 +177,19 @@ module ForemanInventoryUpload
         @stream.simple_field('organization_id', host.organization_id, :last)
       end
 
-      def os_release_value(name:, version:, codename:)
+      def report_ip_addresses(host, host_ips_cache)
+        ip_addresses = host.interfaces.map { |nic| host_ips_cache[nic.ip] }.compact
+
+        @stream.string_array_value('ip_addresses', ip_addresses)
+      end
+
+      def report_mac_addresses(host)
+        macs = host.interfaces.map { |nic| nic.mac }.compact
+
+        @stream.string_array_value('mac_addresses', macs)
+      end
+
+      def os_release_value(name, version, codename)
         "#{name} #{version} (#{codename})"
       end
     end

data/lib/foreman_rh_cloud.rb

@@ -1,4 +1,71 @@
 require 'foreman_rh_cloud/engine.rb'
+require 'cgi'
+require 'uri'
 
 module ForemanRhCloud
+  def self.base_url
+    # for testing set ENV to 'https://ci.cloud.redhat.com'
+    @base_url ||= ENV['SATELLITE_RH_CLOUD_URL'] || 'https://cloud.redhat.com'
+  end
+
+  def self.authentication_url
+    # https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token
+    @authentication_url ||= ENV['SATELLITE_RH_CLOUD_SSO_URL'] || 'https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token'
+  end
+
+  def self.verify_ssl_method
+    @verify_ssl_method ||= ENV['SATELLITE_RH_CLOUD_URL'] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+  end
+
+  def self.http_proxy_string(logger: Foreman::Logging.logger('background'))
+    ForemanRhCloud.proxy_setting(logger: logger)
+  end
+
+  def self.transformed_http_proxy_string(logger: Foreman::Logging.logger('background'))
+    ForemanRhCloud.transform_scheme(ForemanRhCloud.proxy_setting(logger: logger))
+  end
+
+  def self.proxy_setting(logger: Foreman::Logging.logger('background'))
+    ForemanRhCloud.cdn_proxy(logger: logger) ||
+    ForemanRhCloud.global_foreman_proxy ||
+    ''
+  end
+
+  def self.cdn_proxy(logger: Foreman::Logging.logger('app'))
+    proxy_config = SETTINGS[:katello][:cdn_proxy]
+    return nil unless proxy_config
+
+    uri = URI('')
+    uri.host = proxy_config[:host]
+    uri.port = proxy_config[:port]
+    uri.scheme = proxy_config[:scheme] || 'http'
+
+    if proxy_config[:user]
+      uri.user = CGI.escape(proxy_config[:user])
+      uri.password = CGI.escape(proxy_config[:password])
+    end
+    uri.to_s
+  rescue URI::Error => e
+    logger.warn("cdn_proxy parsing failed: #{e}")
+    nil
+  end
+
+  def self.global_foreman_proxy
+    Setting[:http_proxy]
+  end
+
+  # This method assumes uri_string contains uri-encoded username and p@$$word:
+  # http://user:p%40%24%24word@localhost:8888
+  def self.transform_scheme(uri_string)
+    transformed_uri = URI.parse(uri_string)
+
+    case transformed_uri.scheme
+    when "http"
+      transformed_uri.scheme = 'proxy'
+    when "https"
+      transformed_uri.scheme = 'proxys'
+    end
+
+    transformed_uri.to_s
+  end
 end

data/lib/foreman_rh_cloud/engine.rb

@@ -13,6 +13,7 @@ module ForemanRhCloud
     config.autoload_paths += Dir["#{config.root}/app/helpers/concerns"]
     config.autoload_paths += Dir["#{config.root}/app/models/concerns"]
     config.autoload_paths += Dir["#{config.root}/app/overrides"]
+    config.autoload_paths += Dir["#{config.root}/app/services"]
     config.autoload_paths += Dir["#{config.root}/lib"]
 
     config.eager_load_paths += Dir["#{config.root}/lib"]
@@ -54,20 +55,36 @@ module ForemanRhCloud
 
         # Adding a sub menu after hosts menu
         divider :top_menu, caption: N_('RH Cloud'), parent: :configure_menu
-        menu :top_menu, :inventory_upload, :caption => N_('Inventory Upload'), :url_hash => { controller: :'foreman_rh_cloud/react', :action => :inventory_upload}, parent: :configure_menu
-        menu :top_menu, :insights_hits_import, :caption => N_('Insights'), :url_hash => { controller: :'foreman_rh_cloud/react', :action => :insights_cloud }, parent: :configure_menu
+        menu :top_menu, :inventory_upload, caption: N_('Inventory Upload'), url_hash: { controller: :'foreman_rh_cloud/react', action: :inventory_upload }, parent: :configure_menu
+        menu :top_menu, :insights_hits_import, caption: N_('Insights'), url_hash: { controller: :'foreman_rh_cloud/react', action: :insights_cloud }, parent: :configure_menu
+
+        register_facet InsightsFacet, :insights do
+          set_dependent_action :destroy
+        end
 
-        register_facet InsightsFacet, :insights
         register_global_js_file 'subscriptions_extension'
+
+        register_custom_status(InventorySync::InventoryStatus)
+
+        extend_page 'hosts/show' do |context|
+          context.add_pagelet :main_tabs,
+            partial: 'hosts/insights_tab',
+            name: _('Insights'),
+            onlyif: proc { |host| host.insights }
+        end
       end
 
       ::Katello::UINotifications::Subscriptions::ManifestImportSuccess.include ForemanInventoryUpload::Notifications::ManifestImportSuccessNotificationOverride if defined?(Katello)
+
+      ::Host::Managed.include RhCloudHost
+      ::Host::Base.include RhCloudHost
     end
 
     initializer "foreman_rh_cloud.set_dynflow.config.on_init", :before => :finisher_hook do |_app|
       unless Rails.env.test?
         ForemanTasks.dynflow.config.on_init do |world|
           ForemanInventoryUpload::Async::GenerateAllReportsJob.spawn_if_missing(world)
+          InsightsCloud::Async::InsightsScheduledSync.spawn_if_missing(world)
         end
       end
     end

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '0.9.8'.freeze
+  VERSION = '0.9.13'.freeze
 end

data/lib/insights_cloud.rb

@@ -1,15 +1,15 @@
 module InsightsCloud
-  def self.base_url
-    # for testing set ENV to 'https://ci.cloud.redhat.com'
-    @base_url ||= ENV['SATELLITE_INSIGHTS_CLOUD_URL'] || 'https://cloud.redhat.com'
-  end
+  module WebUi
+    ADVISOR = 'advisor'
+    VULNERABILITY = 'vulnerability'
+    PATCH = 'patch'
 
-  def self.authentication_url
-    # https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token
-    @authentication_url ||= ENV['SATELLITE_INSIGHTS_CLOUD_SSO_URL'] || 'https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token'
+    def self.system_url(subsystem, host_uuid)
+      ForemanRhCloud.base_url + "/insights/#{subsystem}/systems/#{host_uuid}"
+    end
   end
 
   def self.hits_export_url
-    base_url + '/api/insights/v1/export/hits/'
+    ForemanRhCloud.base_url + '/api/insights/v1/export/hits/'
   end
 end

data/lib/insights_cloud/async/insights_full_sync.rb

@@ -3,13 +3,13 @@ require 'rest-client'
 module InsightsCloud
   module Async
     class InsightsFullSync < ::ApplicationJob
-      def perform(organization)
-        @organization = organization
+      include ::ForemanRhCloud::CloudAuth
 
+      def perform
         hits = query_insights_hits
 
-        host_names = hits.map { |hit| hit['hostname'] }.uniq
-        setup_host_names(host_names)
+        @hits_host_names = Hash[hits.map { |hit| [hit['hostname'], hit['uuid']] }]
+        setup_host_names(@hits_host_names.keys)
 
         replace_hits_data(hits)
       end
@@ -18,17 +18,14 @@ module InsightsCloud
         Foreman::Logging.logger('background')
       end
 
-      def rh_credentials
-        @rh_credentials ||= query_refresh_token
-      end
-
       private
 
       def query_insights_hits
         hits_response = RestClient::Request.execute(
           method: :get,
           url: InsightsCloud.hits_export_url,
-          verify_ssl: ENV['SATELLITE_INSIGHTS_CLOUD_URL'] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER,
+          verify_ssl: ForemanRhCloud.verify_ssl_method,
+          proxy: ForemanRhCloud.transformed_http_proxy_string(logger: logger),
           headers: {
             Authorization: "Bearer #{rh_credentials}",
           }
@@ -37,27 +34,9 @@ module InsightsCloud
         JSON.parse(hits_response)
       end
 
-      def query_refresh_token
-        token_response = RestClient::Request.execute(
-          method: :post,
-          url: InsightsCloud.authentication_url,
-          verify_ssl: ENV['SATELLITE_INSIGHTS_CLOUD_URL'] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER,
-          payload: {
-            grant_type: 'refresh_token',
-            client_id: 'rhsm-api',
-            refresh_token: Setting[:rh_cloud_token],
-          }
-        )
-
-        JSON.parse(token_response)['access_token']
-      rescue RestClient::ExceptionWithResponse => e
-        Foreman::Logging.exception('Unable to authenticate using rh_cloud_token setting', e)
-        raise ::Foreman::WrappedException.new(e, N_('Unable to authenticate using rh_cloud_token setting'))
-      end
-
       def setup_host_names(host_names)
         @host_ids = Hash[
-          Host.where(name: host_names).pluck(:name, :id)
+          Host.unscoped.where(name: host_names).pluck(:name, :id)
         ]
       end
 
@@ -67,17 +46,30 @@ module InsightsCloud
 
       def replace_hits_data(hits)
         InsightsHit.transaction do
-          InsightsHit.delete_all
-          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) })
           # create new facets for hosts that are missing one
           hosts_with_existing_facets = InsightsFacet.where(host_id: @host_ids.values).pluck(:host_id)
-          InsightsFacet.create((@host_ids.values - hosts_with_existing_facets).map { |id| {host_id: id} })
+          InsightsFacet.create(
+            @host_ids.map do |host_name, host_id|
+              unless hosts_with_existing_facets.include?(host_id)
+                {
+                  host_id: host_id,
+                  uuid: @hits_host_names[host_name],
+                }
+              end
+            end.compact
+          )
+          InsightsHit.delete_all
+          InsightsHit.create(hits.map { |hits_hash| to_model_hash(hits_hash) }.compact)
         end
       end
 
       def to_model_hash(hit_hash)
+        hit_host_id = host_id(hit_hash['hostname'])
+
+        return unless hit_host_id
+
         {
-          host_id: host_id(hit_hash['hostname']),
+          host_id: hit_host_id,
           last_seen: Time.parse(hit_hash['last_seen']),
           publish_date: Time.parse(hit_hash['publish_date']),
           title: hit_hash['title'],