katello 3.10.2 → 3.11.0.rc1

data/app/services/katello/pulp/repository.rb

@@ -21,6 +21,16 @@ module Katello
         Katello::RepositoryTypeManager.repository_types[repo.root.content_type].service_class.new(repo, smart_proxy)
       end
 
+      def unit_type_id(_uploads = [])
+        @repo.unit_type_id
+      end
+
+      def unit_keys(uploads)
+        uploads.map do |upload|
+          upload.except('id', 'name')
+        end
+      end
+
       def partial_repo_path
         fail NotImplementedError
       end
@@ -49,6 +59,22 @@ module Katello
         fail NotImplementedError
       end
 
+      def regenerate_applicability
+        fail NotImplementedError
+      end
+
+      def copy_contents(_destination_repo, _filters)
+        fail NotImplementedError
+      end
+
+      def content_service
+        Katello::Pulp::Content
+      end
+
+      def should_purge_empty_contents?
+        false
+      end
+
       def sync(overrides = {})
         sync_options = {}
         sync_options[:max_speed] = SETTINGS.dig(:katello, :pulp, :sync_KBlimit)
@@ -123,10 +149,9 @@ module Katello
       end
 
       def refresh
-        tasks = update_or_associate_importer
-        tasks += update_or_associate_distributors
-        tasks += remove_unnecessary_distributors
-        tasks
+        update_or_associate_importer
+        update_or_associate_distributors
+        remove_unnecessary_distributors
       end
 
       def update_or_associate_importer
@@ -134,7 +159,6 @@ module Katello
         importer = generate_importer
         found = existing_importers.find { |i| i['importer_type_id'] == importer.id }
 
-        tasks = []
         if found
           ssl_ca_cert = importer.config.delete('ssl_ca_cert')
           ssl_client_cert = importer.config.delete('ssl_client_cert')
@@ -142,40 +166,46 @@ module Katello
           importer.config['basic_auth_username'] = nil if importer.config['basic_auth_username'].blank?
           importer.config['basic_auth_password'] = nil if importer.config['basic_auth_password'].blank?
           # Update ssl options by themselves workaround for https://pulp.plan.io/issues/2727
-          tasks << smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], :ssl_client_cert => ssl_client_cert,
+          smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], :ssl_client_cert => ssl_client_cert,
                                                     :ssl_client_key => ssl_client_key, :ssl_ca_cert => ssl_ca_cert)
-          tasks << smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], importer.config)
+          smart_proxy.pulp_api.resources.repository.update_importer(repo.pulp_id, found['id'], importer.config)
         else
-          tasks << smart_proxy.pulp_api.resources.repository.associate_importer(repo.pulp_id, repo.importers.first['importer_type_id'], importer.config)
+          smart_proxy.pulp_api.resources.repository.associate_importer(repo.pulp_id, repo.importers.first['importer_type_id'], importer.config)
         end
-        tasks
       end
 
       def update_or_associate_distributors
-        tasks = []
         existing_distributors = backend_data["distributors"]
         generate_distributors.each do |distributor|
           found = existing_distributors.find { |i| i['distributor_type_id'] == distributor.type_id }
           if found
-            tasks << smart_proxy.pulp_api.resources.repository.update_distributor(repo.pulp_id, found['id'], distributor.config)
+            smart_proxy.pulp_api.resources.repository.
+                update_distributor(repo.pulp_id, found['id'], distributor.config)
           else
             smart_proxy.pulp_api.resources.repository.
                 associate_distributor(repo.pulp_id, distributor.type_id, distributor.config, :distributor_id => distributor.id,
                                       :auto_publish => distributor.auto_publish)
           end
         end
-        tasks
       end
 
       def remove_unnecessary_distributors
-        tasks = []
         existing_distributors = backend_data["distributors"]
         generated_distributors = generate_distributors
         existing_distributors.each do |distributor|
           found = generated_distributors.find { |dist| dist.type_id == distributor['distributor_type_id'] }
-          tasks << smart_proxy.pulp_api.resources.repository.delete_distributor(repo.pulp_id, distributor['id']) unless found
+          smart_proxy.pulp_api.resources.repository.delete_distributor(repo.pulp_id, distributor['id']) unless found
         end
-        tasks
+      end
+
+      def copy_units(destination_repo, units, options = {})
+        content_type = units.first.class::CONTENT_TYPE
+        unit_ids = units.pluck(:pulp_id)
+        override_config = {}
+        override_config[:recursive] = true if options[:recursive]
+        override_config[:resolve_dependencies] = true if options[:resolve_dependencies]
+
+        smart_proxy.pulp_api.extensions.send(content_type).copy(repo.pulp_id, destination_repo.pulp_id, ids: unit_ids, override_config: override_config)
       end
     end
   end

data/app/services/katello/pulp/repository/deb.rb

@@ -41,6 +41,14 @@ module Katello
         def importer_class
           Runcible::Models::DebImporter
         end
+
+        def copy_contents(destination_repo, _options = {})
+          [
+            @smart_proxy.pulp_api.extensions.deb.copy(@repo.pulp_id, destination_repo.pulp_id, fields: ::Katello::Pulp::Deb::PULP_SELECT_FIELDS),
+            @smart_proxy.pulp_api.extensions.deb_release.copy(@repo.pulp_id, destination_repo.pulp_id, {}),
+            @smart_proxy.pulp_api.extensions.deb_component.copy(@repo.pulp_id, destination_repo.pulp_id, {})
+          ]
+        end
       end
     end
   end

data/app/services/katello/pulp/repository/docker.rb

@@ -2,6 +2,16 @@ module Katello
   module Pulp
     class Repository
       class Docker < ::Katello::Pulp::Repository
+        def unit_type_id(uploads = [])
+          uploads.pluck('digest').any? ? 'docker_tag' : super
+        end
+
+        def unit_keys(uploads)
+          uploads.map do |upload|
+            upload.except('id')
+          end
+        end
+
         def generate_master_importer
           config = {
             feed: root.url,
@@ -37,6 +47,15 @@ module Katello
         def importer_class
           Runcible::Models::DockerImporter
         end
+
+        def copy_contents(destination_repo, options = {})
+          if options[:filters]
+            clause_gen = ::Katello::Util::DockerManifestClauseGenerator.new(@repo, options[:filters])
+            criteria = {filters: {:unit => clause_gen.copy_clause } }
+          end
+
+          @smart_proxy.pulp_api.extensions.docker_tag.copy(@repo.pulp_id, destination_repo.pulp_id, criteria || {})
+        end
       end
     end
   end

data/app/services/katello/pulp/repository/file.rb

@@ -2,6 +2,12 @@ module Katello
   module Pulp
     class Repository
       class File < ::Katello::Pulp::Repository
+        def unit_keys(uploads)
+          uploads.map do |upload|
+            upload.except('id')
+          end
+        end
+
         def generate_master_importer
           config = { feed: root.url }
           importer_class.new(config.merge(master_importer_connection_options))
@@ -25,6 +31,10 @@ module Katello
         def importer_class
           Runcible::Models::IsoImporter
         end
+
+        def copy_contents(destination_repo, _options = {})
+          @smart_proxy.pulp_api.extensions.file.copy(@repo.pulp_id, destination_repo.pulp_id, {})
+        end
       end
     end
   end

data/app/services/katello/pulp/repository/ostree.rb

@@ -34,6 +34,10 @@ module Katello
         def importer_class
           Runcible::Models::OstreeImporter
         end
+
+        def copy_contents(destination_repo, _options = {})
+          @smart_proxy.pulp_api.extensions.ostree_branch.copy(@repo.pulp_id, destination_repo.pulp_id, {})
+        end
       end
     end
   end

data/app/services/katello/pulp/repository/puppet.rb

@@ -37,6 +37,16 @@ module Katello
         def importer_class
           Runcible::Models::PuppetImporter
         end
+
+        def copy_contents(destination_repo, options = {})
+          if options[:puppet_modules]
+            module_uuids = options[:puppet_modules].pluck(:pulp_id)
+            clauses = {'filters': { 'association': { 'unit_id' => { "$in" => module_uuids } } } }
+          else
+            clauses = {}
+          end
+          @smart_proxy.pulp_api.extensions.puppet_module.copy(@repo.pulp_id, destination_repo.pulp_id, clauses)
+        end
       end
     end
   end

data/app/services/katello/pulp/repository/yum.rb

@@ -55,6 +55,79 @@ module Katello
             policy
           end
         end
+
+        def regenerate_applicability
+          smart_proxy.pulp_api.extensions.repository.regenerate_applicability_by_ids([repo.pulp_id], true)
+        end
+
+        def copy_contents(destination_repo, options = {})
+          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(options[:filters], options[:rpm_filenames])
+          tasks = [smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id, rpm_copy_clauses)]
+          if rpm_remove_clauses
+            tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
+                                                                         type_ids: [::Katello::Pulp::Rpm::CONTENT_TYPE],
+                                                                          filters: {unit: rpm_remove_clauses})
+          end
+
+          [:srpm, :errata, :package_group, :yum_repo_metadata_file, :distribution, :module, :module_default].each do |type|
+            tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
+          end
+          tasks
+        end
+
+        def purge_empty_contents
+          [purge_empty_errata, purge_empty_package_groups]
+        end
+
+        def should_purge_empty_contents?
+          true
+        end
+
+        private
+
+        def purge_empty_errata
+          task = nil
+          repo.empty_errata! do |errata_to_delete|
+            task = repo.unassociate_by_filter(::Katello::ContentViewErratumFilter::CONTENT_TYPE,
+                                                "id" => { "$in" => errata_to_delete.map(&:errata_id) })
+          end
+          task
+        end
+
+        def purge_empty_package_groups
+          rpm_names = repo.rpms.pluck(:name).uniq
+
+          # Remove all  package groups with no packages
+          package_groups_to_delete = repo.package_groups.select do |group|
+            (rpm_names & group.package_names).empty?
+          end
+
+          repo.repository_package_groups.where(:package_group_id => package_groups_to_delete.map(&:id)).delete_all
+
+          criteria = {:association => {"unit_id" => {"$in" => package_groups_to_delete.compact}}}
+          smart_proxy.pulp_api.extensions.repository.unassociate_units(repo.pulp_id, :filters => criteria)
+        end
+
+        def generate_copy_clauses(filters, rpm_filenames)
+          if rpm_filenames&.any?
+            copy_clauses = {filters: {unit: { 'filename' => { '$in' => rpm_filenames } }}}
+            remove_clauses = nil
+          elsif filters
+            clause_gen = ::Katello::Util::PackageClauseGenerator.new(repo, filters.yum)
+            clause_gen.generate
+
+            copy = clause_gen.copy_clause
+            copy_clauses = {filters: {unit: copy }} if copy
+
+            remove = clause_gen.remove_clause
+            remove_clauses = {filters: {unit: remove}} if remove
+          else
+            copy_clauses = {}
+            remove_clauses = nil
+          end
+          copy_clauses.merge!(fields: ::Katello::Pulp::Rpm::PULP_SELECT_FIELDS)
+          [copy_clauses, remove_clauses]
+        end
       end
     end
   end

data/app/services/katello/pulp/yum_metadata_file.rb

@@ -0,0 +1,7 @@
+module Katello
+  module Pulp
+    class YumMetadataFile < PulpContentUnit
+      CONTENT_TYPE = "yum_repo_metadata_file".freeze
+    end
+  end
+end

data/app/services/katello/repository_type.rb

@@ -13,6 +13,7 @@ module Katello
     end
 
     def_field :allow_creation_by_user, :service_class
+    attr_accessor :metadata_publish_matching_check
     attr_reader :id
 
     def initialize(id)
@@ -20,6 +21,10 @@ module Katello
       allow_creation_by_user(true)
     end
 
+    def prevent_unneeded_metadata_publish
+      self.metadata_publish_matching_check = true
+    end
+
     def <=>(other)
       self.id.to_s <=> other.id.to_s
     end

data/app/views/dashboard/_errata_widget.html.erb

@@ -23,7 +23,7 @@
         <tr>
           <td class="ellipsis" title="<%= erratum.errata_type %>"><%= erratum.errata_type %></td>
           <td class="ellipsis">
-            <a href="/errata/<%= erratum.uuid %>">
+            <a href="/errata/<%= erratum.pulp_id %>">
               <%= erratum.errata_id %> (<%= n_("%s Host", "%s Hosts", hosts.length) % hosts.length %>)
             </a>
           </td>

data/app/views/katello/api/v2/debs/base.json.rabl

@@ -7,4 +7,5 @@ attributes :architecture
 attributes :filename
 attributes :checksum
 attributes :description
-attributes :uuid
+attributes :pulp_id
+attributes :pulp_id => :uuid

data/app/views/katello/api/v2/docker_tags/_base.json.rabl

@@ -4,17 +4,17 @@ attributes :id, :name
 attributes :repository_id
 
 child :schema1_manifest => :manifest_schema1 do
-  attributes :uuid => :id
+  attributes :pulp_id => :id
   attributes :schema_version, :digest, :manifest_type
 end
 
 child :schema2_manifest => :manifest_schema2 do
-  attributes :uuid => :id
+  attributes :pulp_id => :id
   attributes :schema_version, :digest, :manifest_type
 end
 
 child :docker_manifest => :manifest do
-  attributes :uuid => :id
+  attributes :pulp_id => :id
   attributes :schema_version, :digest, :manifest_type
 end
 

data/app/views/katello/api/v2/errata/show.json.rabl

@@ -1,9 +1,10 @@
 object @resource
 
-attributes :id, :uuid, :title, :errata_id
+attributes :id, :pulp_id, :title, :errata_id
 attributes :issued, :updated, :version, :status, :release
 attributes :severity, :description, :solution, :summary, :reboot_suggested
 attributes :_href
+attributes :pulp_id => :uuid
 
 child :cves => :cves do
   attributes :cve_id, :href

data/app/views/katello/api/v2/file_units/base.json.rabl

@@ -3,5 +3,6 @@ object @resource
 attributes :id
 attributes :name
 attributes :path
-attributes :uuid
+attributes :pulp_id => :uuid
+attributes :pulp_id
 attributes :checksum

data/app/views/katello/api/v2/module_streams/base.json.rabl

@@ -1,4 +1,5 @@
 object @resource
 
-attributes :id, :name, :uuid, :version, :context, :stream,
+attributes :id, :name, :pulp_id, :version, :context, :stream,
            :arch, :description, :summary, :module_spec
+attributes :pulp_id => :uuid

data/app/views/katello/api/v2/ostree_branches/show.json.rabl

@@ -1,4 +1,5 @@
 object @resource
 
-attributes :uuid, :id
-attributes :name, :version, :commit
+attributes :pulp_id, :id
+attributes :name, :version, :commit, :version_date
+attributes :pulp_id => :uuid

data/app/views/katello/api/v2/package_groups/base.json.rabl

@@ -2,7 +2,8 @@ object @resource
 
 attributes :id
 attributes :name
-attributes :uuid
+attributes :pulp_id
+attributes :pulp_id => :uuid
 attributes :description
 
 node :repository do |package_group|

data/app/views/katello/api/v2/package_groups/show.json.rabl

@@ -1,4 +1,4 @@
 object @resource
 
 extends "katello/api/v2/package_groups/base"
-extends "katello/api/v2/package_groups/backend", :object => Katello::Pulp::PackageGroup.new(@resource.uuid)
+extends "katello/api/v2/package_groups/backend", :object => Katello::Pulp::PackageGroup.new(@resource.pulp_id)

data/app/views/katello/api/v2/packages/base.json.rabl

@@ -1,8 +1,9 @@
 object @resource
 
-attributes :id, :uuid, :name, :version, :release, :arch, :epoch, :summary
+attributes :id, :pulp_id, :name, :version, :release, :arch, :epoch, :summary
 attributes :filename, :sourcerpm, :checksum
 attributes :nvrea, :nvra
+attributes :pulp_id => :uuid
 
 node(:hosts_available_count) { |m| m.hosts_available(params[:organization_id]).count }
 node(:hosts_applicable_count) { |m| m.hosts_applicable(params[:organization_id]).count }

data/app/views/katello/api/v2/packages/show.json.rabl

@@ -1,4 +1,4 @@
 object @resource
 
 extends "katello/api/v2/packages/base"
-extends "katello/api/v2/packages/backend", :object => Katello::Pulp::Rpm.new(@resource.uuid)
+extends "katello/api/v2/packages/backend", :object => Katello::Pulp::Rpm.new(@resource.pulp_id)