katello 3.15.0.rc1.3 → 3.15.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a6bd47c5fa7163ee343c23d652b46abc14fdba414c1630b3b56335196aa6941
-  data.tar.gz: 18829650fef9ffdad3cea5d73a30a7bb31eca9305db9bbbda85d922b3ca93d90
+  metadata.gz: 198db45b1663ec5e4dd73cb999c9ef4860660fe9ca3428293d0c82ea45ff78a2
+  data.tar.gz: c541eaa6a075bf545ba668d2b80197375428a55047852ca7a4a8fddd2ac7f0cb
 SHA512:
-  metadata.gz: 7ab1ee973db81ec2f17436659b364cf1e7529aac79616bc7e8f6b7cf4fecad4b0821abafaeaf90077bcbaccaba6882731050719586d05239f6204ecadd506968
-  data.tar.gz: 3583ab23ec2bdb236ec056f79dd789d2d8455f2e071b93b200f4836845371f6c5aee7616077cb6d7f34b8ed9cd08ee5ec30fdabbaac97f6b6d27f4b7b96c4dd0
+  metadata.gz: 54851ee1cbf1de239a68264f377b4b96e58b310e656b67daed646927aa56bfbc6f6bdf307dc6f4a1240982675a98d2a05bf4d2991b9dae8dc5b829f0d897a4c2
+  data.tar.gz: e8466a953f3bedaa7f761f353fdddf8c858dbe9b09b72dcd338c5a2a132745f9b923e2ba1afdd6e4c2a4fd71fafc2bcb1bc88332866c0d6b08930b9c37246d69

data/app/controllers/katello/api/v2/api_controller.rb

@@ -35,6 +35,11 @@ module Katello
       @resource_class ||= "Katello::#{resource_name.classify}".constantize
     end
 
+    def deprecate_katello_agent
+      ::Foreman::Deprecation.api_deprecation_warning("Remote actions using katello-agent are deprecated and will be removed in Katello 4.0.  " \
+                                                         "You may consider switching to Remote Execution.")
+    end
+
     def full_result_response(collection)
       { :results => collection,
         :total => collection.count,

data/app/controllers/katello/api/v2/docker_tags_controller.rb

@@ -25,6 +25,23 @@ module Katello
       end
     end
 
+    api :GET, "/docker_tags/:id/repositories", N_("List of repositories for a docker meta tag")
+    def repositories
+      tag = DockerMetaTag.find(params[:id])
+
+      if tag.repositories.size > 1 #pulp3
+        repos = tag.repositories.non_archived
+      else
+        repos = []
+        tag.related_tags.each do |related|
+          repos << related.repositories.non_archived
+        end
+        repos.flatten!
+      end
+
+      respond_with_template_collection('index', 'repositories', collection: full_result_response(repos))
+    end
+
     private
 
     def find_repositories

data/app/controllers/katello/api/v2/host_packages_controller.rb

@@ -6,6 +6,7 @@ module Katello
     before_action :require_packages_only, :only => [:upgrade]
     before_action :find_editable_host_with_facet, :except => :index
     before_action :find_host, :only => :index
+    before_action :deprecate_katello_agent, :only => [:install, :remove, :upgrade, :upgrade_all]
 
     resource_description do
       api_version 'v2'

data/app/controllers/katello/api/v2/hosts_bulk_actions_controller.rb

@@ -10,6 +10,7 @@ module Katello
     before_action :find_deletable_hosts, :only => [:destroy_hosts]
     before_action :find_readable_hosts, :only => [:applicable_errata, :installable_errata, :available_incremental_updates]
     before_action :find_errata, :only => [:available_incremental_updates]
+    before_action :deprecate_katello_agent, :only => [:install_content, :update_content, :remove_content]
 
     before_action :validate_content_action, :only => [:install_content, :update_content, :remove_content]
 
@@ -118,7 +119,6 @@ module Katello
           :desc => N_("The type of content.  The following types are supported: 'package', 'package_group' and 'errata'."),
           :required => true
     param :content, Array, :desc => N_("List of content (e.g. package names, package group names or errata ids)")
-    param :install_all, :bool, :desc => N_("Only used for errata, but will install all installable errata.")
     def install_content
       content_action
     end

data/app/lib/actions/katello/capsule_content/sync_capsule.rb

@@ -31,6 +31,10 @@ module Actions
           end
         end
 
+        def resource_locks
+          :link
+        end
+
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end

data/app/lib/actions/katello/content_view/destroy.rb

@@ -17,6 +17,7 @@ module Actions
               end
             end
 
+            plan_action(Actions::Pulp3::ContentView::DeleteRepositoryReferences, content_view, SmartProxy.pulp_master)
             plan_self
           end
         end

data/app/lib/actions/katello/host/erratum/applicable_errata_install.rb

@@ -11,7 +11,7 @@ module Actions
             errata_ids = options.fetch(:errata_ids, nil)
 
             applicable_errata = host.content_facet.applicable_errata
-            applicable_errata = applicable_errata.with_identifiers(errata_ids) if errata_ids
+            applicable_errata = applicable_errata.with_identifiers(errata_ids) unless options[:update_all]
             plan_action(Actions::Katello::Host::Erratum::Install, host, applicable_errata.pluck(:errata_id))
             plan_self(:hostname => host.name)
           end

data/app/lib/actions/katello/host/upload_profiles.rb

@@ -33,16 +33,42 @@ module Actions
           Dynflow::Action::Rescue::Skip
         end
 
+        def self.upload_modules_to_pulp(available_streams, host)
+          query_name_streams = available_streams.map do |profile|
+            ::Katello::ModuleStream.where(profile.slice(:name, :stream))
+          end
+
+          query_name_streams = query_name_streams.inject(&:or)
+
+          bound_library_instances = host.content_facet.bound_repositories.map(&:library_instance_or_self)
+          query = ::Katello::ModuleStream.in_repositories(bound_library_instances).
+                                          select(:name, :stream, :version, :context, :arch).
+                                            merge(query_name_streams)
+
+          updated_profiles = query.map do |module_stream|
+            module_stream.slice(:name, :stream, :version, :context, :arch)
+          end
+
+          # We also need to pass module streams that are not found in the ModuleStream table
+          # but are present on the content host
+          unassociated_profiles = available_streams.select do |profile|
+            updated_profiles.none? { |p| p[:name] == profile[:name] && p[:stream] == profile[:stream] }
+          end
+
+          ::Katello::Pulp::Consumer.new(host.content_facet.uuid).
+              upload_module_stream_profile(updated_profiles + unassociated_profiles)
+        rescue RestClient::ResourceNotFound
+          Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % host.id)
+        end
+
         def import_module_streams(payload, host)
           enabled_payload = payload.map do |profile|
-            profile.slice("name", "stream", "version", "context", "arch") if profile["status"] == "enabled"
+            profile.slice("name", "stream", "version", "context", "arch").with_indifferent_access if profile["status"] == "enabled"
           end
           enabled_payload.compact!
 
-          ::Katello::Pulp::Consumer.new(host.content_facet.uuid).upload_module_stream_profile(enabled_payload)
+          UploadProfiles.upload_modules_to_pulp(enabled_payload, host)
           host.import_module_streams(payload)
-        rescue RestClient::ResourceNotFound
-          Rails.logger.warn("Host with ID %s was not known to Pulp, continuing" % input[:host_id])
         end
 
         def import_deb_package_profile(host, profile)
@@ -69,6 +95,7 @@ module Actions
             payload = profiles.dig("deb_package_profile", "deb_packages") || []
             import_deb_package_profile(host, payload)
           else
+            module_streams = []
             profiles.each do |profile|
               payload = profile["profile"]
               case profile["content_type"]
@@ -79,9 +106,13 @@ module Actions
               when "enabled_repos"
                 host.import_enabled_repositories(payload)
               else
-                import_module_streams(payload, host)
+                module_streams << payload
               end
             end
+
+            module_streams.each do |module_stream_payload|
+              import_module_streams(module_stream_payload, host)
+            end
           end
         end
       end

data/app/lib/actions/katello/orphan_cleanup/remove_orphans.rb

@@ -13,6 +13,19 @@ module Actions
                 Actions::Pulp3::Orchestration::OrphanCleanup::RemoveOrphans,
                 proxy)
             end
+            plan_self
+          end
+        end
+
+        def run
+          models = []
+          ::Katello::RepositoryTypeManager.repository_types.each_value do |repo_type|
+            indexable_types = repo_type.content_types_to_index
+            models += indexable_types&.map(&:model_class)
+            models.select! { |model| model.many_repository_associations }
+          end
+          models.each do |model|
+            model.joins("left join katello_#{model.repository_association} on #{model.table_name}.id = katello_#{model.repository_association}.#{model.unit_id_field}").where("katello_#{model.repository_association}.#{model.unit_id_field} IS NULL").destroy_all
           end
         end
       end

data/app/lib/actions/katello/pulp_selector.rb

@@ -3,6 +3,7 @@ module Actions
     module PulpSelector
       def plan_pulp_action(backend_actions, repository, smart_proxy, *args)
         fail "nil smart_proxy passed to PulpSelector" if smart_proxy.nil?
+        smart_proxy.fix_pulp3_capabilities(repository.content_type)
         planned = plan_optional_pulp_action(backend_actions, repository, smart_proxy, *args)
         fail "Could not locate an action for type #{smart_proxy.backend_service_type(repository)}" unless planned
         planned

data/app/lib/actions/pulp3/content_view/delete_repository_references.rb

@@ -0,0 +1,26 @@
+module Actions
+  module Pulp3
+    module ContentView
+      class DeleteRepositoryReferences < Pulp3::AbstractAsyncTask
+        def plan(content_view, smart_proxy)
+          if content_view.repository_references.any?
+            plan_self(:content_view_id => content_view.id, :smart_proxy_id => smart_proxy.id)
+          end
+        end
+
+        def invoke_external_task
+          tasks = []
+          content_view = ::Katello::ContentView.find(input[:content_view_id])
+          content_view.repository_references.each do |repository_reference|
+            repo = repository_reference.root_repository.library_instance
+            #force pulp3 in case we've done migrations, but haven't switched over yet
+            tasks << repo.backend_service(smart_proxy, true).delete(repository_reference.repository_href)
+          end
+          content_view.repository_references.destroy_all
+
+          output[:pulp_tasks] = tasks
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/repository/delete.rb

@@ -7,7 +7,14 @@ module Actions
             sequence do
               plan_action(Actions::Pulp3::Repository::DeleteRemote, repository.id, smart_proxy) if repository.remote_href
               plan_action(Actions::Pulp3::Repository::DeleteDistributions, repository.id, smart_proxy)
-              plan_action(Actions::Pulp3::Repository::Delete, repository.id, smart_proxy)
+
+              if repository.content_view.default?
+                #we're deleting the library instance, so just delete the whole pulp3 repo
+                plan_action(Actions::Pulp3::Repository::Delete, repository.id, smart_proxy)
+              elsif repository.environment.nil?
+                #we're deleting the archived instance, so delete the version
+                plan_action(Actions::Pulp3::Repository::DeleteVersion, repository, smart_proxy)
+              end
             end
           end
         end

data/app/lib/actions/pulp3/repository/delete_version.rb

@@ -0,0 +1,20 @@
+module Actions
+  module Pulp3
+    module Repository
+      class DeleteVersion < Pulp3::AbstractAsyncTask
+        def plan(repository, smart_proxy)
+          #A version from library might be reused in a composite, or could have been copied
+          # from a library repository, so only delete the version if its the last
+          if ::Katello::Repository.where(:version_href => repository.version_href).count == 1
+            plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id)
+          end
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          output[:response] = repo.backend_service(smart_proxy).delete_version
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -7,7 +7,8 @@ module Katello
         def allowed_helpers
           super + [:errata, :host_subscriptions, :host_applicable_errata_ids, :host_applicable_errata_filtered,
                    :host_latest_applicable_rpm_version, :load_pools, :load_errata_applications, :host_content_facet,
-                   :host_sla, :host_products, :sub_name, :sub_sku, :registered_through, :last_checkin]
+                   :host_sla, :host_products, :sub_name, :sub_sku, :registered_through, :last_checkin, :host_collections,
+                   :host_subscriptions_names, :host_subscriptions, :host_products_names, :host_collections_names]
         end
       end
 
@@ -23,6 +24,10 @@ module Katello
         host.subscriptions
       end
 
+      def host_subscriptions_names(host)
+        host.subscriptions.map(&:name)
+      end
+
       def host_content_facet(host)
         host.content_facet
       end
@@ -35,6 +40,23 @@ module Katello
         host_subscription_facet(host)&.installed_products
       end
 
+      def host_products_names(host)
+        products = host_products(host)
+        if products
+          products.map(&:name)
+        else
+          []
+        end
+      end
+
+      def host_collections(host)
+        host.host_collections
+      end
+
+      def host_collections_names(host)
+        host.host_collections.map(&:name)
+      end
+
       def sub_name(pool)
         return unless pool
         pool.subscription&.name

data/app/lib/katello/errors.rb

@@ -45,6 +45,12 @@ module Katello
       end
     end
 
+    class PulpcoreMissingCapabilities < StandardError
+      def message
+        _("A smart proxy seems to have been refreshed without pulpcore being running.  You may want to ")
+      end
+    end
+
     class ConnectionRefusedException < StandardError; end
 
     class MaxHostsReachedException < StandardError; end
@@ -115,6 +121,8 @@ module Katello
       end
     end
 
+    class Pulp3MigrationError < StandardError; end
+
     class PulpError < StandardError
       def self.from_task(task)
         if %w(error canceled).include?(task[:state])

data/app/lib/katello/util/package.rb

@@ -22,7 +22,6 @@ module Katello
       def self.parse_nvrea(name)
         name, suffix = extract_suffix(name)
         name, arch = extract_arch(name)
-        return unless arch
 
         if (nvre = parse_nvre(name))
           nvre.merge(:suffix => suffix, :arch => arch).delete_if { |_k, v| v.nil? }

data/app/models/katello/concerns/host_managed_extensions.rb

@@ -106,7 +106,7 @@ module Katello
 
       def import_package_profile(simple_packages)
         found = import_package_profile_in_bulk(simple_packages)
-        sync_package_associations(found.map(&:id))
+        sync_package_associations(found.map(&:id).uniq)
       end
 
       def import_package_profile_in_bulk(simple_packages)
@@ -127,9 +127,14 @@ module Katello
                                           :arch => simple_package.arch)
         end
         InstalledPackage.import(installed_packages, validate: false, on_duplicate_key_ignore: true)
+        #re-lookup all imported to pickup any duplicates/conflicts
+        imported = InstalledPackage.where(:nvrea => installed_packages.map(&:nvrea)).select(:id).to_a
 
-        found << installed_packages
-        found.flatten
+        if imported.count != installed_packages.count
+          Rails.logger.warn("Mismatch found in installed package insertion, expected #{installed_packages.count} but only could find #{imported.count}.  This is most likley a bug.")
+        end
+
+        (found + imported).flatten
       end
 
       def import_enabled_repositories(repos)

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -152,6 +152,19 @@ module Katello
         end
       end
 
+      def missing_pulp3_capabilities?
+        pulp3_enabled? && self.capabilities(PULP3_FEATURE).empty?
+      end
+
+      def fix_pulp3_capabilities(type)
+        if missing_pulp3_capabilities? && !pulp2_preferred_for_type?(type)
+          self.refresh
+          if self.capabilities(::SmartProxy::PULP3_FEATURE).empty?
+            fail Katello::Errors::PulpcoreMissingCapabilities
+          end
+        end
+      end
+
       def pulp3_repository_type_support?(repository_type, check_pulp2_preferred = true)
         repository_type_obj = repository_type.is_a?(String) ? Katello::RepositoryTypeManager.repository_types[repository_type] : repository_type
         fail "Cannot find repository type #{repository_type}, is it enabled?" unless repository_type_obj

data/app/models/katello/docker_meta_tag.rb

@@ -107,7 +107,7 @@ module Katello
       if grouped
         grouped_fields = "#{table_name}.name, katello_repositories.root_id"
         ids = distinct.select("ON (#{grouped_fields}) #{table_name}.id").joins(:repositories)
-        where(:id => ids).where("#{self.table_name}.schema1_id in (#{sql}) or #{self.table_name}.schema2_id in (#{sql})")
+        self.where(:id => ids)
       else
         self.where("#{self.table_name}.schema1_id in (#{sql}) or #{self.table_name}.schema2_id in (#{sql})")
       end
@@ -153,17 +153,14 @@ module Katello
         end
 
         unless params_to_query_for_delete.empty?
-          DockerMetaTag.where(:id => repository_association_class.
-                              where(:repository_id => repo.id).
-                              select(:docker_meta_tag_id)).
-                              where(params_to_query_for_delete.join(" OR ")).delete_all
+          RepositoryDockerMetaTag.where(:docker_meta_tag => DockerMetaTag.where(params_to_query_for_delete.join(" OR "))).delete_all
         end
 
         metatags = []
         (tag_table_values - meta_tag_table_values).each do |schema1, schema2, name|
           metatags << DockerMetaTag.where(:schema1_id => schema1,
                               :schema2_id => schema2,
-                              :name => name).create!
+                              :name => name).first_or_create
         end
         repo.docker_meta_tags += metatags
       end

data/app/models/katello/installed_package.rb

@@ -4,8 +4,8 @@ module Katello
       allow :nvra, :nvrea, :name
     end
 
-    has_many :hosts, :through => :host_installed_packages, :class_name => "::Host"
     has_many :host_installed_packages, :class_name => "Katello::HostInstalledPackage", :dependent => :destroy, :inverse_of => :installed_package
+    has_many :hosts, :through => :host_installed_packages, :class_name => "::Host"
 
     scoped_search :on => :name, :complete_value => true
     scoped_search :on => :nvrea

data/app/models/katello/repository.rb

@@ -171,8 +171,8 @@ module Katello
       joins(:root).where("#{RootRepository.table_name}.content_type" => content_type)
     end
 
-    def backend_service(smart_proxy)
-      if smart_proxy.pulp3_support?(self)
+    def backend_service(smart_proxy, force_pulp3 = false)
+      if force_pulp3 || smart_proxy.pulp3_support?(self)
         @service ||= Katello::Pulp3::Repository.instance_for_type(self, smart_proxy)
       else
         @service ||= Katello::Pulp::Repository.instance_for_type(self, smart_proxy)
@@ -301,17 +301,52 @@ module Katello
         group("#{errata}.id").count
     end
 
+    def self.errata_with_module_stream_counts(repo)
+      repository_errata = Katello::RepositoryErratum.table_name
+      errata = Katello::Erratum.table_name
+      erratum_package = Katello::ErratumPackage.table_name
+      repository_module_stream = Katello::RepositoryModuleStream.table_name
+      msep = ::Katello::ModuleStreamErratumPackage.table_name
+      ::Katello::Erratum.joins(
+        "INNER JOIN #{erratum_package} on #{erratum_package}.erratum_id = #{errata}.id",
+        "INNER JOIN #{msep} on #{msep}.erratum_package_id = #{erratum_package}.id",
+        "INNER JOIN #{repository_errata} on #{repository_errata}.erratum_id = #{errata}.id",
+        "INNER JOIN #{repository_module_stream} on #{repository_module_stream}.module_stream_id = #{msep}.module_stream_id").
+        where("#{repository_module_stream}.repository_id" => repo.id).
+        where("#{repository_errata}.repository_id" => repo.id).
+        group("#{errata}.id").count
+    end
+
+    def fetch_package_errata_to_keep
+      errata_counts = ::Katello::Repository.errata_with_package_counts(self)
+      if errata_counts.any?
+        errata_counts_in_library = ::Katello::Repository.errata_with_package_counts(library_instance)
+        errata_counts.keep_if { |id| errata_counts[id] == errata_counts_in_library[id] }
+        errata_counts.keys
+      else
+        []
+      end
+    end
+
+    def fetch_module_errata_to_filter
+      errata_counts = ::Katello::Repository.errata_with_module_stream_counts(self)
+      errata_counts_in_library = ::Katello::Repository.errata_with_module_stream_counts(library_instance)
+      if errata_counts_in_library.any?
+        errata_counts_in_library.keep_if { |id| errata_counts[id] != errata_counts_in_library[id] }
+        errata_counts_in_library.keys
+      else
+        []
+      end
+    end
+
     def partial_errata
       return [] if library_instance?
 
-      errata_with_package_counts = ::Katello::Repository.errata_with_package_counts(self)
       partial_errata = self.errata
-      if errata_with_package_counts.any?
-        errata_with_packages_in_library = ::Katello::Repository.errata_with_package_counts(library_instance)
-        errata_with_package_counts.keep_if { |id| errata_with_package_counts[id] == errata_with_packages_in_library[id] }
-        unless errata_with_package_counts.empty?
-          partial_errata = self.errata.where("#{Katello::Erratum.table_name}.id NOT IN (?)", errata_with_package_counts.keys)
-        end
+      errata_to_keep = fetch_package_errata_to_keep - fetch_module_errata_to_filter
+
+      if errata_to_keep.any?
+        partial_errata = self.errata.where("#{Katello::Erratum.table_name}.id NOT IN (?)", errata_to_keep)
       end
 
       partial_errata
@@ -387,8 +422,12 @@ module Katello
       path
     end
 
+    def library_instance_or_self
+      self.library_instance || self
+    end
+
     def generate_repo_path(content_path = nil)
-      _org, _content, content_path = (self.library_instance || self).relative_path.split("/", 3) if content_path.blank?
+      _org, _content, content_path = library_instance_or_self.relative_path.split("/", 3) if content_path.blank?
       content_path = content_path.sub(%r|^/|, '')
       if self.environment
         cve = ContentViewEnvironment.where(:environment_id => self.environment,
@@ -461,7 +500,7 @@ module Katello
         end
       end
       clone = Repository.new(:environment => to_env,
-                     :library_instance => self.library_instance || self,
+                     :library_instance => library_instance_or_self,
                      :root => self.root,
                      :content_view_version => to_version,
                      :saved_checksum_type => checksum_type)