foreman_rh_cloud 12.2.16 → 13.0.0

data/lib/foreman_inventory_upload/async/upload_report_job.rb

@@ -0,0 +1,96 @@
+require 'tempfile'
+
+module ForemanInventoryUpload
+  module Async
+    class UploadReportJob < ShellProcess
+      def self.output_label(label)
+        "upload_for_#{label}"
+      end
+
+      def plan(filename, organization_id, disconnected = false)
+        label = UploadReportJob.output_label(organization_id)
+        super(label, filename: filename, organization_id: organization_id, disconnected: disconnected)
+      end
+
+      def try_execute
+        if content_disconnected?
+          progress_output do |progress_output|
+            progress_output.write_line('Upload canceled because connection to Insights is not enabled or the --no-upload option was passed.')
+            progress_output.status = "Task aborted, exit 1"
+            done!
+          end
+          return
+        end
+
+        unless organization.owner_details&.fetch('upstreamConsumer')&.fetch('idCert')
+          logger.info("Skipping organization '#{organization}', no candlepin certificate defined.")
+          progress_output do |progress_output|
+            progress_output.write_line("Skipping organization #{organization}, no candlepin certificate defined.")
+            progress_output.status = "Task aborted, exit 1"
+            done!
+          end
+          return
+        end
+
+        Tempfile.create([organization.name, '.pem']) do |cer_file|
+          cer_file.write(certificate[:cert])
+          cer_file.write(certificate[:key])
+          cer_file.flush
+          @cer_path = cer_file.path
+          super
+        end
+      end
+
+      def command
+        ['/bin/bash', File.join(File.dirname(filename), ForemanInventoryUpload.upload_script_file)]
+      end
+
+      def env
+        env_vars = super.merge(
+          'FILES' => filename,
+          'CER_PATH' => @cer_path
+        )
+
+        http_proxy_string = ForemanRhCloud.http_proxy_string
+        if http_proxy_string
+          env_vars['http_proxy'] = http_proxy_string
+          env_vars['https_proxy'] = http_proxy_string
+        end
+        env_vars
+      end
+
+      def certificate
+        ForemanRhCloud.with_iop_smart_proxy? ? foreman_certificate : manifest_certificate
+      end
+
+      def manifest_certificate
+        @manifest_certificate ||= begin
+          candlepin_id_certificate = organization.owner_details['upstreamConsumer']['idCert']
+          {
+            cert: candlepin_id_certificate['cert'],
+            key: candlepin_id_certificate['key'],
+          }
+        end
+      end
+
+      def foreman_certificate
+        @foreman_certificate ||= {
+          cert: File.read(Setting[:ssl_certificate]),
+          key: File.read(Setting[:ssl_priv_key]),
+        }
+      end
+
+      def filename
+        input[:filename]
+      end
+
+      def organization
+        @organization ||= Organization.find(input[:organization_id])
+      end
+
+      def content_disconnected?
+        input[:disconnected] || !Setting[:subscription_connection_enabled]
+      end
+    end
+  end
+end

data/lib/foreman_inventory_upload/generators/fact_helpers.rb

@@ -59,7 +59,7 @@ module ForemanInventoryUpload
       def obfuscate_hostname?(host)
         # Returns true if hostname obfuscation should be applied for a given host, based on hierarchy:
         # 1. Global setting for hostname obfuscation.
-        return true if Setting[:obfuscate_inventory_hostnames] && !ForemanRhCloud.with_iop_smart_proxy?
+        return true if Setting[:obfuscate_inventory_hostnames]
 
         insights_client_setting = fact_value(host, 'insights_client::obfuscate_hostname_enabled')
         insights_client_setting = ActiveModel::Type::Boolean.new.cast(insights_client_setting)
@@ -99,7 +99,7 @@ module ForemanInventoryUpload
       def obfuscate_ips?(host)
         # Returns true if IP obfuscation should be applied for a given host, based on hierarchy:
         # 1. Global setting for IP obfuscation.
-        return true if Setting[:obfuscate_inventory_ips] && !ForemanRhCloud.with_iop_smart_proxy?
+        return true if Setting[:obfuscate_inventory_ips]
 
         insights_client_ipv4_setting = fact_value(host, 'insights_client::obfuscate_ipv4_enabled')
         insights_client_ipv6_setting = fact_value(host, 'insights_client::obfuscate_ipv6_enabled')

data/lib/foreman_inventory_upload/generators/slice.rb

@@ -90,7 +90,7 @@ module ForemanInventoryUpload
       def report_host(host)
         host_ips_cache = host_ips(host)
         @stream.object do
-          if Setting[:insights_minimal_data_collection] && !ForemanRhCloud.with_iop_smart_proxy?
+          if Setting[:insights_minimal_data_collection]
             insights_minimal_data_collection(host)
           else
             @stream.simple_field('fqdn', fqdn(host))
@@ -205,7 +205,7 @@ module ForemanInventoryUpload
             end.join(', '))
           end
         end
-        if ForemanRhCloud.with_iop_smart_proxy? || (!Setting[:insights_minimal_data_collection] && !Setting[:exclude_installed_packages])
+        if !Setting[:insights_minimal_data_collection] && !Setting[:exclude_installed_packages]
           @stream.array_field('installed_packages') do
             first = true
             host.installed_packages.each do |package|
@@ -256,7 +256,7 @@ module ForemanInventoryUpload
 
         @stream.array_field('yum_repos') do
           host.content_facet.bound_repositories.each_with_index do |repo, index|
-            report_yum_repo(host.content_source&.load_balancer_pulp_content_url || ::SmartProxy.pulp_primary.pulp_content_url, repo)
+            report_yum_repo(host.content_source.load_balancer_pulp_content_url, repo)
             @stream.comma unless index == host.content_facet.bound_repositories.count - 1
           end
         end

data/lib/foreman_inventory_upload/scripts/uploader.sh.erb

@@ -0,0 +1,49 @@
+#! /bin/bash
+
+DEST=<%= @upload_url %>
+RH_USERNAME=<%= @rh_username %>
+
+if [ -z "$FILES" ]
+then
+        FILES=./*.tar.gz
+fi
+
+if [ -n "$CER_PATH" ]
+then
+        AUTH_KEY="--cert"
+        AUTH_VAL="$CER_PATH"
+else
+        if [ -z "$RH_USERNAME" ]
+        then
+                IFS= read -rp "Enter username: " RH_USERNAME
+        fi
+
+        if [ -z "$RH_PASSWORD" ]
+        then
+                IFS= read -rsp "Enter password: " RH_PASSWORD
+        fi
+
+        AUTH_KEY="-u"
+        AUTH_VAL="\"$RH_USERNAME\":\"$RH_PASSWORD\""
+fi
+
+# /tmp/a b/x.pem
+# curl --cert /tmp/a\ b/x.pem
+
+SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE:-0}")")
+DONE_DIR="$SCRIPT_DIR/done/"
+mkdir -p $DONE_DIR
+
+for f in $FILES
+do
+  curl -k -vvv -# --fail -F "file=@$f;type=application/vnd.redhat.qpc.tar+tgz" $DEST "$AUTH_KEY" "$AUTH_VAL"
+  status=$?
+  if [ $status -eq 0 ]; then
+    mv $f $DONE_DIR
+    echo "Done: $f"
+  fi
+done
+echo "Uploaded files moved to done/ folder"
+
+# return the error code from the curl command
+exit $status

data/lib/foreman_inventory_upload.rb

@@ -48,6 +48,10 @@ module ForemanInventoryUpload
     @outputs_folder ||= ensure_folder(File.join(ForemanInventoryUpload.base_folder, 'outputs/'))
   end
 
+  def self.upload_script_file
+    'uploader.sh'
+  end
+
   def self.facts_archive_name(organization, filter = nil)
     "report_for_#{organization}#{filter.empty? ? nil : "[#{filter.to_s.parameterize}]"}.tar.xz"
   end
@@ -84,12 +88,8 @@ module ForemanInventoryUpload
     inventory_base_url + "?hostname_or_id=#{ForemanRhCloud.foreman_host.fqdn}"
   end
 
-  def self.host_by_id_url(host_uuid)
-    "#{inventory_base_url}/#{host_uuid}"
-  end
-
-  def self.hosts_by_ids_url(host_uuids)
-    host_ids_string = host_uuids.join(',')
+  def self.hosts_by_ids_url(host_ids)
+    host_ids_string = host_ids.join(',')
     "#{inventory_base_url}/#{host_ids_string}"
   end
 end

data/lib/foreman_rh_cloud/engine.rb

@@ -40,7 +40,6 @@ module ForemanRhCloud
         ::Host::Managed.include RhCloudHost
 
         ::Katello::Api::Rhsm::CandlepinDynflowProxyController.include InsightsCloud::PackageProfileUploadExtensions
-        ::Katello::RegistrationManager.singleton_class.prepend ::ForemanRhCloud::RegistrationManagerExtensions
       end
     end
 
@@ -84,27 +83,6 @@ module ForemanRhCloud
         :description => N_('Configure Cloud Connector on given hosts'),
         :proxy_selector_override => ::RemoteExecutionProxySelector::INTERNAL_PROXY
       )
-
-      ScopedSearch::AutoCompleteBuilder.class_eval do
-        # Insights rule IDs always contain a pipe character.
-        # example: hardening_ssh_config_perms|OPENSSH_HARDENING_CONFIG_PERMS
-        # We need to override this method of scoped_search so that autocomplete
-        # will correctly put the value in quotes (otherwise the "|" will be
-        # interpreted as an OR.)
-        # The only change from scoped_search code is adding the | to the regex in the final map.
-        def complete_value_from_db(field, special_values, val)
-          count = 20 - special_values.count
-          completer_scope(field)
-            .where(@options[:value_filter])
-            .where(value_conditions(field.quoted_field, val))
-            .select(field.quoted_field)
-            .limit(count)
-            .distinct
-            .map(&field.field)
-            .compact
-            .map { |v| v.to_s =~ /\s|\|/ ? "\"#{v.gsub('"', '\"')}\"" : v }
-        end
-      end
     end
 
     # Ideally this code belongs to an initializer. The problem is that Katello controllers are not initialized completely until after the end of the to_prepare blocks
@@ -116,20 +94,23 @@ module ForemanRhCloud
       if defined?(Katello) && !Foreman.in_setup_db_rake?
         Katello::Api::V2::OrganizationsController.include Foreman::Controller::SmartProxyAuth
         # patch the callbacks order for :download_debug_certificate, since local_find_taxonomy has to run after the user is already initialized
-        Katello::Api::V2::OrganizationsController.before_find_taxonomy_actions do
-          Katello::Api::V2::OrganizationsController.add_smart_proxy_filters(
-            [:index, :download_debug_certificate],
-            features: ForemanRhCloud.on_prem_smart_proxy_features
-          )
-        end
+        Katello::Api::V2::OrganizationsController.skip_before_action(:local_find_taxonomy, only: :download_debug_certificate)
+        Katello::Api::V2::OrganizationsController.add_smart_proxy_filters(
+          [:index, :download_debug_certificate],
+          features: ForemanRhCloud.on_prem_smart_proxy_features
+        )
+        Katello::Api::V2::OrganizationsController.before_action(:local_find_taxonomy, only: :download_debug_certificate)
+
         Katello::Api::V2::RepositoriesController.include Foreman::Controller::SmartProxyAuth
         # patch the callbacks order for :index, since find_product has to run after the user is already initialized
-        Katello::Api::V2::RepositoriesController.before_index_actions do
-          Katello::Api::V2::RepositoriesController.add_smart_proxy_filters(
-            :index,
-            features: ForemanRhCloud.on_prem_smart_proxy_features
-          )
-        end
+        Katello::Api::V2::RepositoriesController.skip_before_action(:find_product, only: :index)
+        Katello::Api::V2::RepositoriesController.skip_before_action(:find_optional_organization, only: :index)
+        Katello::Api::V2::RepositoriesController.add_smart_proxy_filters(
+          :index,
+          features: ForemanRhCloud.on_prem_smart_proxy_features
+        )
+        Katello::Api::V2::RepositoriesController.before_action(:find_product, only: :index)
+        Katello::Api::V2::RepositoriesController.before_action(:find_optional_organization, only: :index)
       end
     end
 

data/lib/foreman_rh_cloud/plugin.rb

@@ -13,15 +13,15 @@ module ForemanRhCloud
 
         settings do
           category(:rh_cloud, N_('Insights')) do
-            setting('allow_auto_inventory_upload', type: :boolean, description: N_('Enable automatic upload of your host inventory to the Red Hat cloud. Ignored when using local Insights.'), default: true, full_name: N_('Automatic inventory upload'))
-            setting('allow_auto_insights_sync', type: :boolean, description: N_('Enable automatic synchronization of Insights recommendations from the Red Hat cloud. Ignored when using local Insights.'), default: true, full_name: N_('Synchronize recommendations Automatically'))
-            setting('allow_auto_insights_mismatch_delete', type: :boolean, description: N_('Enable automatic deletion of mismatched host records from the Red Hat cloud. Ignored when using local Insights.'), default: false, full_name: N_('Automatic mismatch deletion'))
-            setting('obfuscate_inventory_hostnames', type: :boolean, description: N_('Obfuscate host names sent to the Red Hat cloud. (If insights_minimal_data_collection is set to true, this setting is ignored because host names are not included in the report.) Ignored when using local Insights.'), default: false, full_name: N_('Obfuscate host names'))
-            setting('obfuscate_inventory_ips', type: :boolean, description: N_('Obfuscate ipv4 addresses sent to the Red Hat cloud.  (If insights_minimal_data_collection is set to true, this setting is ignored because host IPv4 addresses are not included in the report.) Ignored when using local Insights.'), default: false, full_name: N_('Obfuscate host ipv4 addresses.'))
-            setting('exclude_installed_packages', type: :boolean, description: N_('Exclude installed packages from being uploaded to the Red Hat cloud. (If insights_minimal_data_collection is set to true, this setting is ignored and installed packages are always excluded.) Ignored when using local Insights.'), default: false, full_name: N_("Exclude installed packages"))
-            setting('include_parameter_tags', type: :boolean, description: N_('Should import include parameter tags from Foreman? Ignored when using local Insights.'), default: false, full_name: N_('Include parameters in insights-client reports'))
-            setting('rhc_instance_id', type: :string, description: N_('RHC daemon id. Ignored when using local Insights.'), default: nil, full_name: N_('ID of the RHC(Yggdrasil) daemon'))
-            setting('insights_minimal_data_collection', type: :boolean, default: false, full_name: N_('Minimal data collection'), description: N_('Only include the minimum required data in inventory reports for uploading to Red Hat cloud. When this is true, installed packages are excluded from the report regardless of the exclude_installed_packages setting, and host names and IPv4 addresses are excluded from the report regardless of obfuscation settings. Ignored when using local Insights.'))
+            setting('allow_auto_inventory_upload', type: :boolean, description: N_('Enable automatic upload of your host inventory to the Red Hat cloud'), default: true, full_name: N_('Automatic inventory upload'))
+            setting('allow_auto_insights_sync', type: :boolean, description: N_('Enable automatic synchronization of Insights recommendations from the Red Hat cloud'), default: true, full_name: N_('Synchronize recommendations Automatically'))
+            setting('allow_auto_insights_mismatch_delete', type: :boolean, description: N_('Enable automatic deletion of mismatched host records from the Red Hat cloud'), default: false, full_name: N_('Automatic mismatch deletion'))
+            setting('obfuscate_inventory_hostnames', type: :boolean, description: N_('Obfuscate host names sent to the Red Hat cloud. (If insights_minimal_data_collection is set to true, this setting is ignored because host names are not included in the report.)'), default: false, full_name: N_('Obfuscate host names'))
+            setting('obfuscate_inventory_ips', type: :boolean, description: N_('Obfuscate ipv4 addresses sent to the Red Hat cloud.  (If insights_minimal_data_collection is set to true, this setting is ignored because host IPv4 addresses are not included in the report.)'), default: false, full_name: N_('Obfuscate host ipv4 addresses.'))
+            setting('exclude_installed_packages', type: :boolean, description: N_('Exclude installed packages from being uploaded to the Red Hat cloud. (If insights_minimal_data_collection is set to true, this setting is ignored and installed packages are always excluded.)'), default: false, full_name: N_("Exclude installed packages"))
+            setting('include_parameter_tags', type: :boolean, description: N_('Should import include parameter tags from Foreman?'), default: false, full_name: N_('Include parameters in insights-client reports'))
+            setting('rhc_instance_id', type: :string, description: N_('RHC daemon id'), default: nil, full_name: N_('ID of the RHC(Yggdrasil) daemon'))
+            setting('insights_minimal_data_collection', type: :boolean, default: false, full_name: N_('Minimal data collection'), description: N_('Only include the minimum required data in inventory reports for uploading to Red Hat cloud. When this is true, installed packages are excluded from the report regardless of the exclude_installed_packages setting, and host names and IPv4 addresses are excluded from the report regardless of obfuscation settings.'))
           end
         end
 
@@ -117,17 +117,14 @@ module ForemanRhCloud
           end
         end
 
-        # Preload insights facet to avoid N+1 queries when rendering host list with facets
-        add_controller_action_scope('Api::V2::HostsController', :index) do |base_scope|
-          base_scope.preload(:insights)
-        end
-
         register_global_js_file 'global'
 
         register_custom_status InventorySync::InventoryStatus
         register_custom_status InsightsClientReportStatus
-        register_ping_extension { ForemanRhCloud::Ping.ping }
-        register_status_extension { ForemanRhCloud::Ping.status }
+        if ForemanRhCloud.with_iop_smart_proxy?
+          register_ping_extension { ForemanRhCloud::Ping.ping }
+          register_status_extension { ForemanRhCloud::Ping.status }
+        end
 
         describe_host do
           overview_buttons_provider :insights_host_overview_buttons
@@ -149,10 +146,6 @@ module ForemanRhCloud
           end
         end
 
-        ::Foreman::Plugin.app_metadata_registry.register(:foreman_rh_cloud, {
-          iop: -> { ForemanRhCloud.with_iop_smart_proxy? },
-        })
-
         extend_template_helpers ForemanRhCloud::TemplateRendererHelper
         allowed_template_helpers :remediations_playbook, :download_rh_playbook
       end

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '12.2.16'.freeze
+  VERSION = '13.0.0'.freeze
 end

data/lib/foreman_rh_cloud.rb

@@ -14,15 +14,15 @@ module ForemanRhCloud
 
   def self.base_url
     # for testing set ENV to 'https://ci.cloud.redhat.com'
-    env_or_on_premise_url('SATELLITE_RH_CLOUD_URL') || 'https://cloud.redhat.com'
+    @base_url ||= env_or_on_premise_url('SATELLITE_RH_CLOUD_URL') || 'https://cloud.redhat.com'
   end
 
   def self.cert_base_url
-    env_or_on_premise_url('SATELLITE_CERT_RH_CLOUD_URL') || 'https://cert.cloud.redhat.com'
+    @cert_base_url ||= env_or_on_premise_url('SATELLITE_CERT_RH_CLOUD_URL') || 'https://cert.cloud.redhat.com'
   end
 
   def self.legacy_insights_url
-    env_or_on_premise_url('SATELLITE_LEGACY_INSIGHTS_URL') || 'https://cert-api.access.redhat.com'
+    @legacy_insights_url ||= env_or_on_premise_url('SATELLITE_LEGACY_INSIGHTS_URL') || 'https://cert-api.access.redhat.com'
   end
 
   def self.verify_ssl_method

data/lib/insights_cloud/async/connector_playbook_execution_reporter_task.rb

@@ -126,10 +126,10 @@ module InsightsCloud
 
       def invocation_status
         hosts_state = Hash[job_invocation.targeting.hosts.map do |host|
-          next unless host.insights_uuid
+          next unless host.insights&.uuid
           [
-            host.insights_uuid,
-            task_status(job_invocation.sub_task_for_host(host), host.insights_uuid),
+            host.insights.uuid,
+            task_status(job_invocation.sub_task_for_host(host), host.insights.uuid),
           ]
         end.compact]
 

data/lib/inventory_sync/async/inventory_hosts_sync.rb

@@ -8,6 +8,8 @@ module InventorySync
       set_callback :step, :around, :create_missing_hosts
 
       def plan(organizations)
+        # Do not run for local advisor, since we use sub-man id to identify hosts.
+        return if ForemanRhCloud.with_iop_smart_proxy?
         # by default the tasks will be executed concurrently
         super(organizations)
         plan_self_host_sync

data/lib/tasks/rh_cloud_inventory.rake

@@ -9,20 +9,22 @@ namespace :rh_cloud_inventory do
       else
         organizations = [Organization.where(:id => ENV['organization_id']).first]
       end
+      disconnected = ForemanRhCloud.with_iop_smart_proxy?
       User.as_anonymous_admin do
         organizations.each do |organization|
           ForemanTasks.async_task(
             ForemanInventoryUpload::Async::GenerateReportJob,
             ForemanInventoryUpload.generated_reports_folder,
-            organization.id
+            organization.id,
+            disconnected
           )
           puts "Generated and uploaded inventory report for organization '#{organization.name}'"
         end
       end
     end
     desc 'Generate inventory report to be sent to Red Hat cloud'
-    task generate: [:environment, 'dynflow:client'] do
-      organization_ids = [ENV['organization_id']]
+    task generate: :environment do
+      organizations = [ENV['organization_id']]
       base_folder = ENV['target'] || Dir.pwd
       filter = ENV['hosts_filter']
 
@@ -32,22 +34,36 @@ namespace :rh_cloud_inventory do
         puts "Using #{base_folder} for the output"
       end
 
-      if organization_ids.empty?
+      if organizations.empty?
         puts "Must specify organization_id"
         return
       end
 
       User.as_anonymous_admin do
-        organization_ids.each do |organization_id|
-          ForemanTasks.sync_task(
-            ForemanInventoryUpload::Async::HostInventoryReportJob,
-            base_folder,
-            organization_id,
-            filter,
-            false # don't upload; the user ran report:generate and not report:generate_upload
-          )
+        organizations.each do |organization|
+          target = File.join(base_folder, ForemanInventoryUpload.facts_archive_name(organization, filter))
+          archived_report_generator = ForemanInventoryUpload::Generators::ArchivedReport.new(target, Logger.new(STDOUT))
+          archived_report_generator.render(organization: organization, filter: filter)
+          puts "Successfully generated #{target} for organization id #{organization}"
+
+          next unless ForemanRhCloud.with_iop_smart_proxy?
+
+          puts 'Creating missing insights facets'
+          hosts_without_facets = ForemanInventoryUpload::Generators::Queries.for_org(organization, hosts_query: 'null? insights_uuid')
+          hosts_without_facets.each do |batch|
+            facets = batch.pluck(:id, 'katello_subscription_facets.uuid').map do |host_id, uuid|
+              {
+                host_id: host_id,
+                uuid: uuid,
+              }
+            end
+            # We don't need to validate the facets here as we create the necessary fields.
+            # rubocop:disable Rails/SkipsModelValidations
+            InsightsFacet.upsert_all(facets, unique_by: :host_id) unless facets.empty?
+            # rubocop:enable Rails/SkipsModelValidations
+          end
+          puts 'Missing Insights facets created'
         end
-        puts "Check the Uploading tab for report uploading status." if Setting[:subscription_connection_enabled]
       end
     end
     desc 'Upload generated inventory report to Red Hat cloud'
@@ -55,7 +71,8 @@ namespace :rh_cloud_inventory do
       base_folder = ENV['target'] || ForemanInventoryUpload.generated_reports_folder
       organization_id = ENV['organization_id']
       report_file = ForemanInventoryUpload.facts_archive_name(organization_id)
-      ForemanTasks.sync_task(ForemanInventoryUpload::Async::QueueForUploadJob, base_folder, report_file, organization_id)
+      disconnected = ForemanRhCloud.with_iop_smart_proxy?
+      ForemanTasks.sync_task(ForemanInventoryUpload::Async::QueueForUploadJob, base_folder, report_file, organization_id, disconnected)
       puts "Uploaded #{report_file}"
     end
   end