katello 3.12.3 → 3.13.0.rc1

data/app/services/katello/pulp/srpm.rb

@@ -4,7 +4,7 @@ module Katello
       include LazyAccessor
 
       PULP_SELECT_FIELDS = %w(name epoch version release arch checksumtype checksum).freeze
-      PULP_INDEXED_FIELDS = %w(name version release arch epoch summary checksum filename _id).freeze
+      PULP_INDEXED_FIELDS = %w(name version release arch epoch summary checksum filename).freeze
       CONTENT_TYPE = "srpm".freeze
 
       lazy_accessor :pulp_facts, :initializer => :backend_data

data/app/services/katello/pulp3/ansible_collection.rb

@@ -0,0 +1,31 @@
+module Katello
+  module Pulp3
+    class AnsibleCollection < PulpContentUnit
+      include LazyAccessor
+
+      def self.content_api
+        PulpAnsibleClient::ContentCollectionsApi.new(Katello::Pulp3::Repository::AnsibleCollection.api_client(SmartProxy.pulp_master!))
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::AnsibleCollection.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:_href) }
+      end
+
+      def self.pulp_data(_href)
+        #No content read method
+        fail NotImplementedError
+      end
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['checksum'] = backend_data['sha256']
+        custom_json['namespace'] = backend_data['namespace']
+        custom_json['version'] = backend_data['version']
+        custom_json['name'] = backend_data['name']
+        model.update_attributes!(custom_json)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_blob.rb

@@ -0,0 +1,17 @@
+module Katello
+  module Pulp3
+    class DockerBlob < PulpContentUnit
+      include LazyAccessor
+
+      def self.content_api
+        PulpDockerClient::ContentBlobsApi.new(Katello::Pulp3::Repository::Docker.api_client(SmartProxy.pulp_master!))
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:_href) }
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -0,0 +1,36 @@
+module Katello
+  module Pulp3
+    class DockerManifest < PulpContentUnit
+      include LazyAccessor
+
+      def self.content_api
+        PulpDockerClient::ContentManifestsApi.new(Katello::Pulp3::Repository::Docker.api_client(SmartProxy.pulp_master!))
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:_href) }
+      end
+
+      def self.content_unit_list(page_opts)
+        page_opts[:media_type] = "application/vnd.docker.distribution.manifest.v1+json"
+        data_v1 = self.content_api.list(page_opts)
+        page_opts[:media_type] = "application/vnd.docker.distribution.manifest.v2+json"
+        data_v2 = self.content_api.list(page_opts)
+
+        filtered = {}
+        filtered["count"] = data_v1.count + data_v2.count
+        filtered["results"] = data_v1.results + data_v2.results
+        filtered
+      end
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['schema_version'], = backend_data['schema_version']
+        custom_json['digest'], = backend_data['digest']
+        model.update_attributes!(custom_json)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_manifest_list.rb

@@ -0,0 +1,30 @@
+module Katello
+  module Pulp3
+    class DockerManifestList < PulpContentUnit
+      include LazyAccessor
+
+      def self.content_api
+        PulpDockerClient::ContentManifestsApi.new(Katello::Pulp3::Repository::Docker.api_client(SmartProxy.pulp_master!))
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:_href) }
+      end
+
+      def self.content_unit_list(page_opts)
+        page_opts[:media_type] = "application/vnd.docker.distribution.manifest.list.v2+json"
+        self.content_api.list(page_opts)
+      end
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['schema_version'], = backend_data['schema_version']
+        custom_json['digest'], = backend_data['digest']
+        custom_json['docker_manifests'] = ::Katello::DockerManifest.where(:pulp_id => backend_data[:listed_manifests])
+        model.update_attributes!(custom_json)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_tag.rb

@@ -0,0 +1,28 @@
+module Katello
+  module Pulp3
+    class DockerTag < PulpContentUnit
+      include LazyAccessor
+
+      def self.content_api
+        PulpDockerClient::ContentManifestTagsApi.new(Katello::Pulp3::Repository::Docker.api_client(SmartProxy.pulp_master!))
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:_href) }
+      end
+
+      def update_model(model)
+        custom_json = {}
+        taggable = ::Katello::DockerManifest.find_by(:pulp_id => backend_data['tagged_manifest'])
+        if taggable.nil?
+          taggable = ::Katello::DockerManifestList.find_by(:pulp_id => backend_data['tagged_manifest'])
+        end
+        custom_json['docker_taggable'] = taggable
+        custom_json['name'] = backend_data['name']
+        model.update_attributes!(custom_json)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/file_unit.rb

@@ -0,0 +1,25 @@
+module Katello
+  module Pulp3
+    class FileUnit < PulpContentUnit
+      include LazyAccessor
+
+      def self.content_api
+        PulpFileClient::ContentFilesApi.new(Katello::Pulp3::Repository::File.api_client(SmartProxy.pulp_master!))
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::File.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:_href) }
+      end
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['checksum'] = backend_data['sha256']
+        custom_json['path'] = backend_data['relative_path']
+        custom_json['name'] = File.basename(backend_data['relative_path'].try(:split, '/').try(:[], -1))
+        model.update_attributes!(custom_json)
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -0,0 +1,75 @@
+require 'set'
+require 'pulp_file_client'
+
+module Katello
+  module Pulp3
+    class PulpContentUnit
+      # Any class that extends this class should define:
+      # Class#update_model
+
+      def self.content_api
+        fail NotImplementedError
+      end
+
+      def update_model
+        fail NotImplementedError
+      end
+
+      attr_accessor :uuid
+      attr_writer :backend_data
+
+      def initialize(uuid)
+        self.uuid = uuid
+      end
+
+      def self.model_class
+        Katello::RepositoryTypeManager.model_class(self)
+      end
+
+      def self.unit_identifier
+        "_href"
+      end
+
+      def self.pulp_units_batch_for_repo(repository, options = {})
+        fetch_identifiers = options.fetch(:fetch_identifiers, false)
+        page_size = options.fetch(:page_size, SETTINGS[:katello][:pulp][:bulk_load_size])
+        repository_version_href = repository.version_href
+        page_opts = { "page" => 1, repository_version: repository_version_href, page_size: page_size}
+        page_opts[:fields] = '_href' if fetch_identifiers
+        response = {}
+        Enumerator.new do |yielder|
+          loop do
+            page_opts = page_opts.with_indifferent_access
+            break unless (response["next"] || page_opts["page"] == 1)
+            response = fetch_content_list page_opts
+            response = response.as_json.with_indifferent_access
+            yielder.yield response[:results]
+            page_opts[:page] += 1
+          end
+        end
+      end
+
+      def self.pulp_data(href)
+        content_unit = self.class.content_api.read(href)
+        content_unit.as_json
+      end
+
+      def self.content_unit_list(page_opts)
+        self.content_api.list page_opts
+      end
+
+      def backend_data
+        @backend_data ||= fetch_backend_data
+        @backend_data.try(:with_indifferent_access)
+      end
+
+      def fetch_backend_data
+        self.class.pulp_data(self.uuid)
+      end
+
+      def self.fetch_content_list(page_opts)
+        content_unit_list page_opts
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/repository.rb

@@ -1,44 +1,116 @@
+require "pulpcore_client"
+
 module Katello
   module Pulp3
     class Repository
+      include Katello::Util::HttpProxy
+
       attr_accessor :repo, :input
       attr_accessor :smart_proxy
       delegate :root, to: :repo
       delegate :pulp3_api, to: :smart_proxy
 
+      COPY_UNIT_PAGE_SIZE = 10_000
+
       def initialize(repo, smart_proxy)
         @repo = repo
         @smart_proxy = smart_proxy
       end
 
-      def create_remote
+      def api_exception_class
         fail NotImplementedError
       end
 
-      def remote_options
+      def remote_class
         fail NotImplementedError
       end
 
-      def update_remote
+      def publications_api
         fail NotImplementedError
       end
 
-      def sync
+      def publication_class
         fail NotImplementedError
       end
 
-      def create_publisher
+      def distribution_class
         fail NotImplementedError
       end
 
-      def create_publication
+      def distributions_api_class
+        fail NotImplementedError
+      end
+
+      def distribution_options
+        fail NotImplementedError
+      end
+
+      def remote_options
         fail NotImplementedError
       end
 
+      def self.api_client(_smart_proxy)
+        fail NotImplementedError
+      end
+
+      def api_client
+        self.class.api_client(smart_proxy)
+      end
+
+      def create_remote
+        remote_file_data = remote_class.new(remote_options)
+        response = remotes_api.create(remote_file_data)
+        repo.update_attributes!(:remote_href => response._href)
+      end
+
+      def update_remote
+        if remote_options[:url].blank?
+          if repo.remote_href
+            href = repo.remote_href
+            repo.update_attributes(remote_href: nil)
+            delete_remote href
+          end
+        else
+          if repo.remote_href?
+            remote_partial_update
+          else
+            create_remote
+          end
+        end
+      end
+
+      def remote_partial_update
+        remotes_api.partial_update(repo.remote_href, remote_options)
+      end
+
+      def delete_remote(href = repo.remote_href)
+        remotes_api.delete(href) if href
+      end
+
+      def list_remotes(args)
+        remotes_api.list(args).results
+      end
+
+      def core_api_client
+        PulpcoreClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpcoreClient::Configuration))
+      end
+
+      def repositories_api
+        PulpcoreClient::RepositoriesApi.new(core_api_client)
+      end
+
+      def repository_versions_api
+        PulpcoreClient::RepositoriesVersionsApi.new(core_api_client)
+      end
+
       def self.instance_for_type(repo, smart_proxy)
         Katello::RepositoryTypeManager.repository_types[repo.root.content_type].pulp3_service_class.new(repo, smart_proxy)
       end
 
+      def should_purge_empty_contents?
+        false
+      end
+
       def backend_object_name
         "#{root.label}-#{repo.id}#{rand(9999)}"
       end
@@ -53,71 +125,96 @@ module Katello
 
       def create
         unless repository_reference
-          response = pulp3_api.repositories_create(Zest::Repository.new(:name => backend_object_name))
-          RepositoryReference.create!(:root_repository_id => repo.root_id, :content_view_id => repo.content_view.id, :repository_href => response._href)
+          response = repositories_api.create(
+            name: backend_object_name)
+          RepositoryReference.create!(
+           root_repository_id: repo.root_id,
+           content_view_id: repo.content_view.id,
+           repository_href: response._href)
           response
         end
       end
 
+      def update
+        repositories_api.update(repository_reference.repository_href, name: backend_object_name)
+      end
+
       def list(args)
-        pulp3_api.repositories_list(args).results
+        repositories_api.list(args).results
       end
 
-      def delete(href = repository_reference.repository_href)
-        response = pulp3_api.repositories_delete(href)
-        repository_reference.destroy if repository_reference
-        response
+      def delete(href = repository_reference.try(:repository_href))
+        repository_reference.try(:destroy)
+        if href
+          response = repositories_api.delete(href)
+          response
+        end
       end
 
-      def update_distribution(path)
-        distribution_reference = distribution_reference(path)
-        pulp3_api.distributions_partial_update(distribution_reference.href, publication: repo.publication_href)
+      def sync
+        [remotes_api.sync(repo.remote_href, repository: repository_reference.repository_href)]
+      end
+
+      def create_publication
+        publication_data = publication_class.new(repository_version: repo.version_href)
+        publications_api.create(publication_data)
       end
 
       def refresh_distributions
-        paths.map do |prefix, path|
-          if distribution_reference(path)
-            update_distribution(path)
-          else
-            create_distribution(prefix, path)
-          end
+        path = repo.relative_path.sub(/^\//, '')
+        dist_ref = distribution_reference(path)
+        if dist_ref
+          update_distribution(path)
+        else
+          create_distribution(path)
         end
       end
 
-      def create_version
-        pulp3_api.repositories_versions_create(repository_reference.repository_href, {})
+      def create_distribution(path)
+        distribution_data = distribution_class.new(distribution_options(path))
+        distributions_api.create(distribution_data)
       end
 
-      def paths
-        list = {
-          https: "https/#{repo.relative_path}"
-        }
-        list['http'] = "http/#{repo.relative_path}" if repo.root.unprotected
-        list
+      def delete_distribution(href)
+        distributions_api.delete(href)
+      rescue api_exception_class => e
+        raise e if e.code != 404
+        nil
       end
 
-      private def delete_distribution(href)
-        pulp3_api.distributions_delete(href)
+      def lookup_distributions(args)
+        distributions_api.list(args).results
       end
 
-      def lookup_distributions(args)
-        pulp3_api.distributions_list(args).results
+      def update_distribution(path)
+        distribution_reference = distribution_reference(path)
+        if distribution_reference
+          options = distribution_options(path).except(:name, :base_path)
+          distributions_api.partial_update(distribution_reference.href, options)
+        end
       end
 
       def get_distribution(href)
-        pulp3_api.distributions_read(href)
-      rescue Zest::ApiError => e
+        distributions_api.read(href)
+      rescue api_exception_class => e
         raise e if e.code != 404
         nil
       end
 
-      def delete_distributions
-        paths.values.each do |path|
-          dists = lookup_distributions(base_path: path.sub(/^\//, ''))
-          delete_distribution(dists.first._href) if dists.first
-          distribution_reference = distribution_reference(path)
-          distribution_reference.destroy if distribution_reference
+      def create_version(options = {})
+        repository_versions_api.create(repository_reference.repository_href, options)
+      end
+
+      def copy_units_by_href(unit_hrefs)
+        tasks = []
+        unit_hrefs.each_slice(COPY_UNIT_PAGE_SIZE) do |slice|
+          tasks << create_version(:add_content_units => slice)
         end
+        tasks
+      end
+
+      def copy_version(from_repository)
+        create_version(:base_version => from_repository.version_href)
       end
 
       def save_distribution_references(hrefs)
@@ -129,17 +226,20 @@ module Katello
         end
       end
 
-      def create_distribution(prefix, path)
-        path = path.sub(/^\//, '') #remove leading / if present
-        pulp3_api.distributions_create(base_path: path, publication: repo.publication_href, name: "#{prefix}_#{backend_object_name}")
+      def delete_distributions
+        path = repo.relative_path.sub(/^\//, '')
+        dists = lookup_distributions(base_path: path)
+        delete_distribution(dists.first._href) if dists.first
+        dist_ref = distribution_reference(path)
+        dist_ref.destroy! if dist_ref
       end
 
       def common_remote_options
         remote_options = {
-          validate: true,
           ssl_validation: root.verify_ssl_on_sync,
           name: backend_object_name,
-          url: root.url
+          url: root.url,
+          proxy_url: root.http_proxy&.full_url
         }
         if root.upstream_username && root.upstream_password
           remote_options.merge(username: root.upstream_username,
@@ -165,6 +265,19 @@ module Katello
           {}
         end
       end
+
+      def lookup_version(href)
+        repository_versions_api.read(href)
+      rescue PulpcoreClient::ApiError => e
+        Rails.logger.error "Exception when calling RepositoriesApi->repositories_versions_read: #{e}"
+        nil
+      end
+
+      def remove_content(content_units)
+        data = PulpcoreClient::RepositoryVersionCreate.new(
+          remove_content_units: content_units.map(&:pulp_id))
+        repository_versions_api.create(repository_reference.repository_href, data)
+      end
     end
   end
 end