katello 3.17.0 → 3.18.0

data/app/services/katello/pulp3/deb.rb

@@ -0,0 +1,38 @@
+module Katello
+  module Pulp3
+    class Deb < PulpContentUnit
+      include LazyAccessor
+      CONTENT_TYPE = "deb".freeze
+
+      def self.content_api
+        PulpDebClient::ContentPackagesApi.new(Katello::Pulp3::Api::Apt.new(SmartProxy.pulp_primary!).api_client)
+      end
+
+      def self.content_api_create(opts = {})
+        self.content_api.create(opts)
+      end
+
+      def self.create_content(options)
+        fail _("Artifact Id and relative path are needed to create content") unless options.dig(:file_name) && options.dig(:artifact)
+        PulpDebClient::DebContent.new(relative_path: options[:file_name], artifact: options[:artifact])
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::Apt.new(Katello::Repository.find(repo_id), SmartProxy.pulp_primary)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:pulp_href) }
+      end
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['checksum'] = backend_data['sha256']
+        custom_json['filename'] = backend_data['relative_path']
+        custom_json['name'] = backend_data['package']
+        custom_json['version'] = backend_data['version']
+        custom_json['description'] = backend_data['description']
+        custom_json['architecture'] = backend_data['architecture']
+        model.update!(custom_json)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerManifest < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_manifest".freeze
 
       def self.content_api
         PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/docker_tag.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerTag < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_tag".freeze
 
       def self.content_api
         PulpContainerClient::ContentTagsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/migration.rb

@@ -3,7 +3,7 @@ require 'pulp_2to3_migration_client'
 module Katello
   module Pulp3
     class Migration
-      attr_accessor :smart_proxy, :reimport_all
+      attr_accessor :smart_proxy, :reimport_all, :task_id
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -21,8 +21,10 @@ module Katello
         overridden.select { |type| SmartProxy.pulp_primary.pulp3_repository_type_support?(type.to_s, false) }.map { |t| t.to_s }
       end
 
-      def initialize(smart_proxy, repository_types = Migration.repository_types_for_migration, options = {})
-        self.reimport_all = options.fetch(:reimport, false)
+      def initialize(smart_proxy, options = {})
+        self.task_id = options.fetch(:task_id, nil)
+        self.reimport_all = options.fetch(:reimport_all, false)
+        repository_types = options.fetch(:repository_types, Migration.repository_types_for_migration)
 
         if (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).any?
           fail ::Katello::Errors::Pulp3MigrationError, _("Pulp 3 migration cannot run. Types %s have already been migrated.") %
@@ -62,6 +64,15 @@ module Katello
         [YumMetadataFile]
       end
 
+      def last_successful_migration_time
+        task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
+        if reimport_all || task.nil?
+          0
+        else
+          task.started_at.to_i
+        end
+      end
+
       def content_types_for_migration
         content_types = @repository_types.collect do |repository_type_label|
           Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index
@@ -70,7 +81,20 @@ module Katello
         content_types.flatten - Migration.ignorable_content_types
       end
 
+      def update_import_status(message, index = nil)
+        #reduce output updating, only update every 20 items
+        if (index.nil? || index % 20 == 0) && self.task_id
+          progress = Katello::ContentMigrationProgress.find_or_create_by(:task_id => self.task_id)
+          progress.update(:progress_message => message)
+          progress.save!
+
+          fail Katello::Errors::Pulp3MigrationError, "Cancelled by user." if progress.canceled?
+        end
+      end
+
       def import_pulp3_content
+        update_import_status("Starting katello import phase.")
+
         Katello::Logging.time("CONTENT_MIGRATION - Total Import Process") do
           @repository_types.each do |repository_type_label|
             Katello::Logging.time("CONTENT_MIGRATION - Importing Repository", data: {type: repository_type_label}) do
@@ -78,7 +102,7 @@ module Katello
             end
 
             Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
-              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type}) do
+              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type.label}) do
                 import_content_type(content_type)
               end
             end
@@ -114,7 +138,9 @@ module Katello
         if repository_type_label == 'yum'
           import_yum_repos(imported, katello_repos)
         else
-          katello_repos.each do |repo|
+          repo_count = katello_repos.count
+          katello_repos.each_with_index do |repo, index|
+            update_import_status("Importing migrated content units #{repository_type_label}: #{index + 1}/#{repo_count}", index)
             found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
             import_repo(repo, found) if found
           end
@@ -122,7 +148,9 @@ module Katello
       end
 
       def import_yum_repos(migrated_repo_items, repos)
-        repos.each do |yum_repo|
+        repo_count = repos.count
+        repos.each_with_index do |yum_repo, index|
+          update_import_status("Importing migrated yum repositories: #{index + 1}/#{repo_count}", index)
           to_find = nil
           if yum_repo.content_view.composite?
             if yum_repo.link?
@@ -165,10 +193,16 @@ module Katello
         process_distributions(pulp3_api, migrated_repo.pulp3_distribution_hrefs)
       end
 
+      def distribution_path_from_cache(href, pulp3_api)
+        @distribution_cache ||= {}
+        @distribution_cache[href] ||= pulp3_api.distributions_api.read(href).base_path
+        @distribution_cache[href]
+      end
+
       def process_distributions(pulp3_api, dist_hrefs_list)
         #distribution_path_hash(pulp3_api, dist_hrefs_list).each do |relative_path, href|
         dist_hrefs_list.each do |href|
-          relative_path = pulp3_api.distributions_api.read(href).base_path
+          relative_path = distribution_path_from_cache(href, pulp3_api)
 
           dist_ref = Katello::Pulp3::DistributionReference.find_or_initialize_by(:path => relative_path)
           if (distribution_repo = Katello::Repository.find_by(:relative_path => relative_path) || Katello::Repository.find_by(:container_repository_name => relative_path))
@@ -189,14 +223,16 @@ module Katello
       end
 
       def operate_on_errata
+        last_migration_time = last_successful_migration_time
         offset = 0
-        limit = 300
-        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+        limit = SETTINGS[:katello][:pulp][:bulk_load_size]
+        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
         total_count = response.count
         yield(response.results)
         until (offset + limit > total_count)
           offset += limit
-          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
+          update_import_status("Importing migrated content type erratum: #{offset + limit}/#{total_count}")
           yield(response.results)
         end
       end
@@ -236,7 +272,12 @@ module Katello
             unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
           end
 
+          total_count = unmigrated_units.count
+          current_count = 0
+
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
+            current_count += needing_hrefs.count
+            update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
             migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
             migrated_units.results.each do |migrated_unit|
               matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -11,6 +11,11 @@ module Katello
         fail NotImplementedError
       end
 
+      def self.content_api_create(opts = {})
+        relative_path = opts.delete(:relative_path)
+        self.content_api.create(relative_path, opts)
+      end
+
       def self.create_content
         fail NotImplementedError
       end

data/app/services/katello/pulp3/repository.rb

@@ -1,5 +1,4 @@
 require "pulpcore_client"
-
 module Katello
   module Pulp3
     class Repository
@@ -188,8 +187,9 @@ module Katello
         api.repositories_api.read(repository_reference.try(:repository_href))
       end
 
-      def delete(href = repository_reference.try(:repository_href))
-        repository_reference.try(:destroy)
+      def delete_repository(repo_reference = repository_reference)
+        href = repo_reference.try(:repository_href)
+        repo_reference.try(:destroy)
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
@@ -205,10 +205,16 @@ module Katello
       end
 
       def create_publication
-        publication_data = api.class.publication_class.new(repository_version: repo.version_href)
+        publication_data = api.class.publication_class.new(publication_options(repo.version_href))
         api.publications_api.create(publication_data)
       end
 
+      def publication_options(repository_version)
+        {
+          repository_version: repository_version
+        }
+      end
+
       def relative_path
         repo.relative_path.sub(/^\//, '')
       end

data/app/services/katello/pulp3/repository/ansible_collection.rb

@@ -19,6 +19,15 @@ module Katello
             name: "#{generate_backend_object_name}"
           }
         end
+
+        def partial_repo_path
+          "/pulp_ansible/galaxy/#{repo.relative_path}/api/v2/collections"
+        end
+
+        def mirror_remote_options
+          {
+          }
+        end
       end
     end
   end

data/app/services/katello/pulp3/repository/apt.rb

@@ -0,0 +1,63 @@
+require 'pulp_deb_client'
+
+module Katello
+  module Pulp3
+    class Repository
+      class Apt < ::Katello::Pulp3::Repository
+        SIGNING_SERVICE_NAME = 'katello_deb_sign'.freeze
+
+        def remote_options
+          deb_remote_options = {
+            distributions: root.deb_releases
+          }
+          deb_remote_options[:components] = root.deb_components.present? ? root.deb_components : nil
+          deb_remote_options[:architectures] = root.deb_architectures.present? ? root.deb_architectures : nil
+
+          if root.url.blank?
+            deb_remote_options[:url] = nil
+          end
+
+          deb_remote_options[:gpgkey] = root.gpg_key.present? ? root.gpg_key.content : nil
+
+          common_remote_options.merge(deb_remote_options)
+        end
+
+        def publication_options(repository_version)
+          ss = api.signing_services_api.list(name: SIGNING_SERVICE_NAME).results
+          popts = super(repository_version)
+          popts.merge!(
+            {
+              # structured is not necessary for subscription-manager
+              #structured: true, # publish real suites (e.g. 'stable')
+              simple: true # publish all into 'default'-suite
+            }
+          )
+          popts[:signing_service] = ss[0].pulp_href if ss && ss.length == 1
+          popts
+        end
+
+        def distribution_options(path)
+          {
+            base_path: path,
+            publication: repo.publication_href,
+            name: "#{generate_backend_object_name}"
+          }
+        end
+
+        def partial_repo_path
+          "/pulp/deb/#{repo.relative_path}/".sub('//', '/')
+        end
+
+        def copy_content_for_source
+          # TODO
+          fail NotImplementedError
+        end
+
+        def regenerate_applicability
+          # TODO
+          fail NotImplementedError
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/repository/docker.rb

@@ -8,6 +8,10 @@ module Katello
           repo.container_repository_name
         end
 
+        def partial_repo_path
+          ''
+        end
+
         def remote_options
           options = {url: root.url, upstream_name: root.docker_upstream_name}
           if root.docker_tags_whitelist&.any?
@@ -48,6 +52,11 @@ module Katello
                                       api.class.recursive_manage_class.new(content_units: options[:remove_content_units]))
         end
 
+        def tag_manifest(name, digest)
+          api.repositories_api.tag(repository_reference.repository_href,
+                                   api.class.tag_image_class.new(tag: name, digest: digest))
+        end
+
         def add_content(content_unit_href)
           content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
           api.repositories_api.add(repository_reference.repository_href, content_units: content_unit_href)

data/app/services/katello/pulp3/repository/yum.rb

@@ -159,8 +159,7 @@ module Katello
 
             data_dup.config[i][:content] = leftover_units.pop(copy_amount)
             unit_copy_counter += copy_amount
-            # Do copy call if limit is reached or if we're under the limit but on the last repo config.
-            if unit_copy_counter >= UNIT_LIMIT || (i == data_dup.config.size - 1 && leftover_units.empty?)
+            if unit_copy_counter != 0
               tasks << api.copy_api.copy_content(data_dup)
               unit_copy_counter = 0
             end
@@ -208,14 +207,20 @@ module Katello
           tasks
         end
 
-        def copy_all(source_repository)
-          data = PulpRpmClient::Copy.new
-          data.config = [{
-            source_repo_version: source_repository.version_href,
-            dest_repo: repository_reference.repository_href
-          }]
+        def copy_all(source_repository, mirror: false)
+          if mirror
+            data = PulpRpmClient::RepositoryAddRemoveContent.new(
+                      base_version: source_repository.version_href)
 
-          [api.copy_api.copy_content(data)]
+            [api.repositories_api.modify(repository_reference.repository_href, data)]
+          else
+            data = PulpRpmClient::Copy.new
+            data.config = [{
+              source_repo_version: source_repository.version_href,
+              dest_repo: repository_reference.repository_href
+            }]
+            [api.copy_api.copy_content(data)]
+          end
         end
 
         def remove_all_content_from_repo(repo_href)

data/app/services/katello/pulp3/repository_mirror.rb

@@ -91,10 +91,11 @@ module Katello
 
       def remote_options
         base_options = common_remote_options
-        base_options.merge(url: remote_feed_url)
+        base_options.merge!(url: remote_feed_url)
         if (type_specific_options = repo_service.try(:mirror_remote_options))
-          base_options.merge(type_specific_options)
+          base_options.merge!(type_specific_options)
         end
+        base_options
       end
 
       def create_remote
@@ -110,8 +111,11 @@ module Katello
         api.remotes_list(name: backend_object_name).first
       end
 
-      def sync
-        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: remote_href, mirror: true)
+      def sync(options = {})
+        sync_params = repo_service.sync_url_params(options)
+        sync_params[:remote] = remote_href
+        sync_params[:mirror] = true
+        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_params)
         [api.repositories_api.sync(repository_href, repository_sync_url_data)]
       end
 
@@ -152,6 +156,7 @@ module Katello
         dist_params[:publication] = options[:publication] if options[:publication]
         dist_params[:repository_version] = version_href if options[:use_repository_version]
         dist_options = distribution_options(path, dist_params)
+        dist_options.delete(:content_guard) if repo_service.repo.content_type == "docker"
         if (distro = repo_service.lookup_distributions(base_path: path).first) ||
           (distro = repo_service.lookup_distributions(name: "#{backend_object_name}").first)
           # update dist

data/app/services/katello/pulp3/task.rb

@@ -68,6 +68,10 @@ module Katello
         task_data[:finish_time] || FINISHED_STATES.include?(task_data[:state])
       end
 
+      def progress_reports
+        task_data['progress_reports']
+      end
+
       def poll
         task_data(true)
         self

data/app/services/katello/pulp3/task_group.rb

@@ -53,6 +53,10 @@ module Katello
         task_group_data['all_tasks_dispatched'] == true && IN_PROGRESS_STATES.all? { |state| task_group_data[state] == 0 }
       end
 
+      def group_progress_reports
+        task_group_data['group_progress_reports']
+      end
+
       def poll
         clear_task_group_data
         task_group_data

data/app/services/katello/repository_type.rb

@@ -70,7 +70,8 @@ module Katello
       {
         :name => self.id.to_s,
         :id => self.id,
-        :creatable => @allow_creation_by_user
+        :creatable => @allow_creation_by_user,
+        :pulp3_support => SmartProxy.pulp_primary.pulp3_repository_type_support?(self)
       }
     end