katello 4.2.0.rc1 → 4.2.0.rc2

data/app/lib/actions/pulp3/repository/save_artifact.rb

@@ -4,15 +4,22 @@ module Actions
       class SaveArtifact < Pulp3::AbstractAsyncTask
         def plan(file, repository, smart_proxy, tasks, unit_type_id, options = {})
           options[:file_name] = file[:filename]
+          options[:sha256] = file[:sha256] || Digest::SHA256.hexdigest(File.read(file[:path]))
           plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :tasks => tasks, :unit_type_id => unit_type_id, :options => options)
         end
 
         def invoke_external_task
-          artifact_href = input[:tasks].last[:created_resources].first
+          artifact_href = input[:options][:artifact_href] || fetch_artifact_href
+          fail _("Content not uploaded to pulp") unless artifact_href
           content_type = input[:unit_type_id]
           content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
           output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name], artifact: artifact_href)]
         end
+
+        def fetch_artifact_href
+          sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": input[:options][:sha256])
+          sha_artifact_list&.results&.first&.pulp_href
+        end
       end
     end
   end

data/app/lib/actions/pulp3/repository/update_remote.rb

@@ -1,14 +1,17 @@
 module Actions
   module Pulp3
     module Repository
-      class UpdateRemote < Pulp3::Abstract
+      class UpdateRemote < Pulp3::AbstractAsyncTask
         def plan(repository, smart_proxy)
-          plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id)
+          if repository.root.url
+            repository.backend_service(smart_proxy).create_test_remote if smart_proxy.pulp_primary?
+            plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id)
+          end
         end
 
-        def run
+        def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:response] = repo.backend_service(smart_proxy).update_remote
+          repo.backend_service(smart_proxy).update_remote
         end
       end
     end

data/app/lib/actions/pulp3/repository/upload_file.rb

@@ -6,6 +6,7 @@ module Actions
           plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :file => file)
         end
 
+        # rubocop:disable Metrics/MethodLength
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
@@ -34,10 +35,21 @@ module Actions
 
             if response
               upload_href = response.pulp_href
-              output[:pulp_tasks] = [uploads_api.commit(upload_href, sha256: sha256)]
+              #Check for any duplicate artifacts created in parallel subtasks
+              duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": sha256)
+              duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
+              if duplicate_sha_artifact_href
+                uploads_api.delete(upload_href)
+                output[:artifact_href] = duplicate_sha_artifact_href
+                output[:pulp_tasks] = nil
+              else
+                output[:artifact_href] = nil
+                output[:pulp_tasks] = [uploads_api.commit(upload_href, sha256: sha256)]
+              end
             end
           end
         end
+        # rubocop:enable Metrics/MethodLength
 
         def external_task=(tasks)
           super
@@ -50,8 +62,18 @@ module Actions
         def check_error_details
           output[:pulp_tasks].each do |pulp_task|
             error_details = pulp_task.try(:[], "error")
-            if error_details && !error_details.nil?
-              fail _("An error occurred during the sync \n%{error_message}") % {:error_message => error_details}
+            if error_details && !error_details.nil? && !unique_error(error_details)
+              fail _("An error occurred during upload \n%{error_message}") % {:error_message => error_details}
+            end
+          end
+        end
+
+        def check_for_errors
+          combined_tasks.each do |task|
+            if unique_error task.error
+              warn _("Duplicate artifact detected")
+            else
+              super
             end
           end
         end
@@ -70,6 +92,10 @@ module Actions
           finish = finish > total ? total : finish
           "bytes #{start}-#{finish}/#{total}"
         end
+
+        def unique_error(message)
+          message&.include?("code='unique'")
+        end
       end
     end
   end

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -214,10 +214,15 @@ module Katello
         new_ids = new_available_module_streams.keys - old_associated_ids
         new_ids.each do |new_id|
           module_stream = new_available_module_streams[new_id]
+          status = module_stream["status"]
+          # Set status to "unknown" only if the active field is in use and set to false and the module is enabled
+          if enabled_module_stream_inactive?(module_stream)
+            status = "unknown"
+          end
           self.host_available_module_streams.create!(host_id: self.id,
                                                      available_module_stream_id: new_id,
                                                      installed_profiles: module_stream["installed_profiles"],
-                                                     status: module_stream["status"])
+                                                     status: status)
         end
 
         upgradable_streams.each do |hams|
@@ -227,6 +232,10 @@ module Katello
           if hams.attributes.slice(*shared_keys) != module_stream_data
             hams.update!(module_stream_data)
           end
+          # Set status to "unknown" only if the active field is in use and set to false and the module is enabled
+          if enabled_module_stream_inactive?(module_stream)
+            hams.update!(status: "unknown")
+          end
         end
       end
 
@@ -346,6 +355,10 @@ module Katello
         self.get_status(::Katello::TraceStatus).refresh!
         self.refresh_global_status!
       end
+
+      def enabled_module_stream_inactive?(module_stream)
+        !module_stream["active"].nil? && module_stream["active"] == false && module_stream["status"] == "enabled"
+      end
     end
   end
 end

data/app/models/katello/content_view.rb

@@ -270,7 +270,7 @@ module Katello
 
     def last_task
       last_task_id = history.order(:created_at)&.last&.task_id
-      last_task_id ? ForemanTasks::Task.find(last_task_id) : nil
+      last_task_id ? ForemanTasks::Task.find_by(id: last_task_id) : nil
     end
 
     def history

data/app/models/katello/content_view_repository.rb

@@ -5,8 +5,9 @@ module Katello
                                 Repository::OSTREE_TYPE,
                                 Repository::FILE_TYPE,
                                 Repository::DEB_TYPE,
-                                Repository::ANSIBLE_COLLECTION_TYPE
-                               ].freeze
+                                Repository::ANSIBLE_COLLECTION_TYPE,
+                                Katello::RepositoryTypeManager.generic_repository_types(enabled_only: false).keys.flatten
+                               ].flatten.freeze
 
     ALLOWED_IMPORT_REPOSITORY_TYPES = Repository::EXPORTABLE_TYPES
 

data/app/models/katello/content_view_version.rb

@@ -311,11 +311,13 @@ module Katello
 
     def update_content_counts!
       self.content_counts = {}
-      RepositoryTypeManager.indexable_content_types.map(&:model_class).each do |content_type|
-        if content_type::CONTENT_TYPE == DockerTag::CONTENT_TYPE
+      RepositoryTypeManager.indexable_content_types.each do |content_type|
+        if content_type&.model_class::CONTENT_TYPE == DockerTag::CONTENT_TYPE
           content_counts[DockerTag::CONTENT_TYPE] = docker_tags.count
+        elsif content_type&.model_class::CONTENT_TYPE == GenericContentUnit::CONTENT_TYPE
+          content_counts[content_type.content_type] = content_type&.model_class&.in_repositories(self.repositories.archived)&.where(:content_type => content_type.content_type)&.count
         else
-          content_counts[content_type::CONTENT_TYPE] = content_type.in_repositories(self.repositories.archived).count
+          content_counts[content_type&.model_class::CONTENT_TYPE] = content_type&.model_class&.in_repositories(self.repositories.archived)&.count
         end
       end
       save!

data/app/models/katello/erratum.rb

@@ -106,7 +106,7 @@ module Katello
       query_clauses = clauses.map do |clause|
         "(#{clause.to_sql})"
       end
-      statement = query_clauses.join(" OR ")
+      statement = query_clauses.join(" AND ")
 
       Katello::ErratumPackage.joins(:erratum => :repository_errata).
           where("#{RepositoryErratum.table_name}.repository_id" => repo.id).
@@ -117,7 +117,7 @@ module Katello
       query_clauses = clauses.map do |clause|
         "(#{clause.to_sql})"
       end
-      statement = query_clauses.join(" OR ")
+      statement = query_clauses.join(" AND ")
       ModuleStream.where(:id => ModuleStreamErratumPackage.joins(:erratum_package => {:erratum => :repository_errata}).
           where("#{RepositoryErratum.table_name}.repository_id" => repo.id).
           where(statement).select("#{ModuleStreamErratumPackage.table_name}.module_stream_id"))

data/app/models/katello/host/content_facet.rb

@@ -6,6 +6,7 @@ module Katello
       include Facets::Base
 
       HOST_TOOLS_PACKAGE_NAME = 'katello-host-tools'.freeze
+      HOST_TOOLS_TRACER_PACKAGE_NAME = 'katello-host-tools-tracer'.freeze
       SUBSCRIPTION_MANAGER_PACKAGE_NAME = 'subscription-manager'.freeze
 
       belongs_to :kickstart_repository, :class_name => "::Katello::Repository", :foreign_key => :kickstart_repository_id, :inverse_of => :kickstart_content_facets
@@ -196,11 +197,15 @@ module Katello
       end
 
       def tracer_installed?
-        self.host.installed_packages.where("#{Katello::InstalledPackage.table_name}.name" => 'katello-host-tools-tracer').any?
+        self.host.installed_packages.where("#{Katello::InstalledPackage.table_name}.name" => [ "python-#{HOST_TOOLS_TRACER_PACKAGE_NAME}",
+                                                                                               "python3-#{HOST_TOOLS_TRACER_PACKAGE_NAME}",
+                                                                                               HOST_TOOLS_TRACER_PACKAGE_NAME ]).any?
       end
 
       def host_tools_installed?
-        host.installed_packages.where("#{Katello::InstalledPackage.table_name}.name" => HOST_TOOLS_PACKAGE_NAME).any?
+        host.installed_packages.where("#{Katello::InstalledPackage.table_name}.name" => [ "python-#{HOST_TOOLS_PACKAGE_NAME}",
+                                                                                          "python3-#{HOST_TOOLS_PACKAGE_NAME}",
+                                                                                          HOST_TOOLS_PACKAGE_NAME ]).any?
       end
 
       def update_errata_status

data/app/models/katello/repository.rb

@@ -413,7 +413,7 @@ module Katello
       elsif ostree?
         "#{scheme}://#{pulp_uri.host.downcase}/pulp/content/web/#{relative_path}"
       elsif ansible_collection?
-        "#{scheme}://#{pulp_uri.host.downcase}/pulp_ansible/galaxy/#{relative_path}/api"
+        "#{scheme}://#{pulp_uri.host.downcase}/pulp_ansible/galaxy/#{relative_path}/api/"
       else
         "#{scheme}://#{pulp_uri.host.downcase}/pulp/content/#{relative_path}/"
       end

data/app/models/katello/root_repository.rb

@@ -187,10 +187,16 @@ module Katello
 
     def ensure_valid_collection_attributes
       errors.add(:base, _("URL needs to have a trailing /")) if !url.blank? && url[-1] != '/'
+      return unless ansible_collection_requirements
       begin
-        YAML.safe_load(ansible_collection_requirements) if ansible_collection_requirements
+        requirements = YAML.safe_load(ansible_collection_requirements)
+        if requirements.is_a?(Hash)
+          errors.add(:base,  _("Requirements yaml should have a 'collections' key")) unless requirements.key?('collections')
+        else
+          errors.add(:base,  _('Requirements yaml should be a key-value pair structure.'))
+        end
       rescue
-        errors.add(:base, _('Requirements yaml is invalid!'))
+        errors.add(:base, _('Requirements is not valid yaml.'))
       end
     end
 
@@ -283,9 +289,7 @@ module Katello
         return
       end
 
-      if self.ansible_collection_auth_url.blank?
-        errors.add(:base, N_("Auth token requires Auth URL be set."))
-      elsif !self.ansible_collection_auth_token
+      if !self.ansible_collection_auth_url.blank? && self.ansible_collection_auth_token.blank?
         errors.add(:base, N_("Auth URL requires Auth token be set."))
       end
     end
@@ -351,7 +355,7 @@ module Katello
                                  ssl_ca_cert_id ssl_client_cert_id ssl_client_key_id http_proxy_policy http_proxy_id download_concurrency)
       changeable_attributes += %w(name container_repository_name docker_tags_whitelist) if docker?
       changeable_attributes += %w(deb_releases deb_components deb_architectures gpg_key_id) if deb?
-      changeable_attributes += %w(ansible_collection_requirements) if ansible_collection?
+      changeable_attributes += %w(ansible_collection_requirements ansible_collection_auth_url ansible_collection_auth_token) if ansible_collection?
       changeable_attributes.any? { |key| previous_changes.key?(key) }
     end
 

data/app/models/setting/content.rb

@@ -157,7 +157,9 @@ class Setting::Content < Setting
       self.set('autosearch_delay', N_('If Autosearch is enabled, delay in milliseconds before executing searches while typing.'),
                500, N_('Autosearch delay')),
       self.set('bulk_load_size', N_('The number of items fetched from a single paged Pulp API call.'), 2000,
-               N_('Pulp bulk load size'))
+               N_('Pulp bulk load size')),
+      self.set('upload_profiles_without_dynflow', N_('Allow Katello to update host installed packages, enabled repos, and module inventory directly instead of wrapped in Dynflow tasks (try turning off if Puma processes are using too much memory)'), true,
+               N_('Upload profiles without Dynflow'))
     ]
   end
 

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -92,7 +92,8 @@ module Katello
         ::Katello::ModuleStream.
           joins("inner join katello_available_module_streams on
             katello_module_streams.name = katello_available_module_streams.name and
-            katello_module_streams.stream = katello_available_module_streams.stream").
+            katello_module_streams.stream = katello_available_module_streams.stream and
+            katello_module_streams.context = katello_available_module_streams.context").
           joins("inner join katello_host_available_module_streams on
             katello_available_module_streams.id = katello_host_available_module_streams.available_module_stream_id").
           where("katello_host_available_module_streams.host_id = :content_facet_id and

data/app/services/katello/host/package_profile_uploader.rb

@@ -0,0 +1,41 @@
+module Katello
+  module Host
+    class PackageProfileUploader
+      def initialize(profile_string:, host: nil)
+        @profile_string = profile_string
+        @host = host
+      end
+
+      def upload
+        profile = JSON.parse(@profile_string)
+        #free the huge string from the memory
+        @profile_string = 'TRIMMED'.freeze
+        import_package_profile(profile)
+      end
+
+      def import_package_profile(profile)
+        self.class.import_package_profile_for_host(@host&.id, profile)
+      end
+
+      def trigger_applicability_generation
+        ::Katello::Host::ContentFacet.trigger_applicability_generation(@host&.id)
+      end
+
+      def self.import_package_profile_for_host(host_id, profile)
+        host = ::Host.find_by(:id => host_id)
+        if host.nil?
+          Rails.logger.warn("Host with ID %s not found; continuing" % host_id)
+        elsif host.content_facet.nil? || host.content_facet.uuid.nil?
+          Rails.logger.warn("Host with ID %s has no content facet; continuing" % host_id)
+        else
+          begin
+            simple_packages = profile.map { |item| ::Katello::Pulp::SimplePackage.new(item) }
+            host.import_package_profile(simple_packages)
+          rescue ActiveRecord::InvalidForeignKey # this happens if the host gets deleted in between the "find_by" and "import_package_profile"
+            Rails.logger.warn("Host installed package list with ID %s was not able to be written to the DB (host likely is deleted); continuing" % host_id)
+          end
+        end
+      end
+    end # of class
+  end
+end

data/app/services/katello/host/profiles_uploader.rb

@@ -0,0 +1,74 @@
+module Katello
+  module Host
+    class ProfilesUploader
+      def initialize(profile_string:, host: nil)
+        @profile_string = profile_string
+        @host = host
+      end
+
+      def upload
+        profiles = JSON.parse(@profile_string)
+        #free the huge string from the memory
+        @profile_string = 'TRIMMED'.freeze
+        if @host.nil?
+          Rails.logger.warn("Host was not specified; continuing")
+        elsif @host.content_facet.nil? || @host.content_facet.uuid.nil?
+          Rails.logger.warn("Host with ID %s has no content facet; continuing" % @host.id)
+        elsif profiles.try(:has_key?, "deb_package_profile")
+          # remove this when deb_package_profile API is removed
+          payload = profiles.dig("deb_package_profile", "deb_packages") || []
+          import_deb_package_profile(payload)
+        else
+          module_streams = []
+          profiles.each do |profile|
+            payload = profile["profile"]
+            case profile["content_type"]
+            when "rpm"
+              PackageProfileUploader.import_package_profile_for_host(@host.id, payload)
+            when "deb"
+              import_deb_package_profile(payload)
+            when "enabled_repos"
+              @host.import_enabled_repositories(payload)
+            else
+              module_streams << payload
+            end
+          end
+
+          module_streams.each do |module_stream_payload|
+            import_module_streams(module_stream_payload)
+          end
+
+        end
+      end
+
+      def trigger_applicability_generation
+        if @host.nil?
+          Rails.logger.warn "Host was not specified; can't trigger applicability generation"
+          return
+        end
+        ::Katello::Host::ContentFacet.trigger_applicability_generation(@host.id)
+      end
+
+      private
+
+      def import_module_streams(payload)
+        enabled_payload = payload.map do |profile|
+          profile.slice("name", "stream", "version", "context", "arch").with_indifferent_access if profile["status"] == "enabled"
+        end
+        enabled_payload.compact!
+
+        @host.import_module_streams(payload)
+      end
+
+      def import_deb_package_profile(profile)
+        installed_deb_ids = profile.map do |item|
+          ::Katello::InstalledDeb.find_or_create_by(name: item['name'], architecture: item['architecture'], version: item['version']).id
+        end
+        @host.installed_deb_ids = installed_deb_ids
+        @host.save!
+      rescue ActiveRecord::InvalidForeignKey # this happens if the host gets deleted in between the "find_by" and "import_package_profile"
+        Rails.logger.warn("Host installed package list with ID %s was not able to be written to the DB (host likely is deleted); continuing" % @host.id)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/api/core.rb

@@ -100,6 +100,10 @@ module Katello
           PulpcoreClient::OrphansApi.new(core_api_client)
         end
 
+        def artifacts_api
+          PulpcoreClient::ArtifactsApi.new(core_api_client)
+        end
+
         def core_api_client
           client = PulpcoreClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpcoreClient::Configuration))
           api_client_class(client)

data/app/services/katello/pulp3/repository/ansible_collection.rb

@@ -5,15 +5,11 @@ module Katello
     class Repository
       class AnsibleCollection < ::Katello::Pulp3::Repository
         def remote_options
-          if root.url.blank?
-            super
-          else
-            common_remote_options.merge(url: root.url.chomp('/').concat('/'),
-                                        requirements_file: root.ansible_collection_requirements.blank? ? nil : root.ansible_collection_requirements,
-                                        auth_url: root.ansible_collection_auth_url,
-                                        token: root.ansible_collection_auth_token,
-                                        tls_validation: root.verify_ssl_on_sync)
-          end
+          common_remote_options.merge(url: root.url.chomp('/').concat('/'),
+                                      requirements_file: root.ansible_collection_requirements.blank? ? nil : root.ansible_collection_requirements,
+                                      auth_url: root.ansible_collection_auth_url,
+                                      token: root.ansible_collection_auth_token,
+                                      tls_validation: root.verify_ssl_on_sync)
         end
 
         def distribution_options(path)