katello 3.13.4 → 3.14.0.rc1

data/app/services/katello/pulp/repository/file.rb

@@ -4,7 +4,7 @@ module Katello
       class File < ::Katello::Pulp::Repository
         def unit_keys(uploads)
           uploads.map do |upload|
-            upload.except('id')
+            upload.except('id', 'content_unit_id')
           end
         end
 

data/app/services/katello/pulp/repository/puppet.rb

@@ -18,8 +18,15 @@ module Katello
           importer_class.new(config.merge(mirror_importer_connection_options))
         end
 
+        def puppet_install_auto_publish?
+          (smart_proxy.try(:pulp_mirror?) && !@repo.library_instance?) || false
+        end
+
         def generate_distributors
-          puppet_install_dist = Runcible::Models::PuppetInstallDistributor.new(puppet_install_distributor_path, :subdir => 'modules', :id => repo.pulp_id, :auto_publish => smart_proxy.try(:pulp_mirror?) || false)
+          puppet_install_dist = Runcible::Models::PuppetInstallDistributor.new(puppet_install_distributor_path,
+                                                                               :subdir => 'modules',
+                                                                               :id => repo.pulp_id,
+                                                                               :auto_publish => puppet_install_auto_publish?)
           puppet_dist = Runcible::Models::PuppetDistributor.new(nil, (root.unprotected || false), true,
                                                                 :id => "#{repo.pulp_id}_puppet", :auto_publish => true)
           [puppet_dist, puppet_install_dist]

data/app/services/katello/pulp/repository/yum.rb

@@ -77,7 +77,7 @@ module Katello
 
         def copy_contents(destination_repo, options = {})
           override_config = Katello::Repository.build_override_config(options)
-          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(options[:filters], options[:rpm_filenames])
+          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(options[:filters]&.yum(false), options[:rpm_filenames])
           tasks = [smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id,
                    rpm_copy_clauses.merge(:override_config => override_config))]
 
@@ -87,11 +87,9 @@ module Katello
                                                                           filters: {unit: rpm_remove_clauses})
           end
 
-          # always copy modular rpms
-          modular_includes = {filters: {unit: { 'filename' => { '$in' => repo.rpms.modular.pluck(:filename) } }}}
-          tasks << smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id, modular_includes)
-
-          [:srpm, :errata, :package_group, :yum_repo_metadata_file, :distribution, :module, :module_default].each do |type|
+          tasks.concat(copy_module_contents(destination_repo, options[:filters], override_config))
+          [:srpm, :errata, :package_group, :package_environment,
+           :yum_repo_metadata_file, :distribution, :module_default].each do |type|
             tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
           end
           tasks
@@ -157,12 +155,46 @@ module Katello
           smart_proxy.pulp_api.extensions.repository.unassociate_units(repo.pulp_id, :filters => criteria)
         end
 
+        def copy_module_contents(destination_repo, filters, override_config)
+          if filters
+            filters = filters.module_stream.or(filters.errata)
+          end
+
+          copy_clauses, remove_clauses = generate_module_stream_copy_clauses(filters)
+          tasks = [smart_proxy.pulp_api.extensions.module.copy(repo.pulp_id, destination_repo.pulp_id,
+                   copy_clauses.merge(:override_config => override_config))]
+
+          if remove_clauses
+            tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
+                                                                         type_ids: [::Katello::Pulp::ModuleStream::CONTENT_TYPE],
+                                                                          filters: {unit: remove_clauses})
+          end
+          tasks
+        end
+
+        def generate_module_stream_copy_clauses(filters)
+          if filters&.any?
+            clause_gen = ::Katello::Util::ModuleStreamClauseGenerator.new(repo, filters)
+            clause_gen.generate
+
+            copy = clause_gen.copy_clause
+            copy_clauses = {filters: {unit: copy }} if copy
+
+            remove = clause_gen.remove_clause
+            remove_clauses = {filters: {unit: remove}} if remove
+          else
+            copy_clauses = {}
+            remove_clauses = nil
+          end
+          [copy_clauses, remove_clauses]
+        end
+
         def generate_copy_clauses(filters, rpm_filenames)
           if rpm_filenames&.any?
             copy_clauses = {filters: {unit: { 'filename' => { '$in' => rpm_filenames } }}}
             remove_clauses = nil
-          elsif filters
-            clause_gen = ::Katello::Util::PackageClauseGenerator.new(repo, filters.yum)
+          elsif filters&.any?
+            clause_gen = ::Katello::Util::PackageClauseGenerator.new(repo, filters.yum(false))
             clause_gen.generate
 
             copy = clause_gen.copy_clause
@@ -171,9 +203,10 @@ module Katello
             remove = clause_gen.remove_clause
             remove_clauses = {filters: {unit: remove}} if remove
           else
-            copy_clauses = {}
+            copy_clauses = {filters: {unit: ContentViewPackageFilter.generate_rpm_clauses(::Katello::Rpm.in_repositories(repo).non_modular.pluck(:filename))}}
             remove_clauses = nil
           end
+
           copy_clauses.merge!(fields: ::Katello::Pulp::Rpm::PULP_SELECT_FIELDS)
           [copy_clauses, remove_clauses]
         end

data/app/services/katello/pulp/smart_proxy_repository.rb

@@ -15,41 +15,6 @@ module Katello
         @smart_proxy.pulp_master?
       end
 
-      def repos_available_to_capsule(environments = nil, content_view = nil)
-        yum_repos = yum_repos_available_to_capsule(environments, content_view) || []
-        puppet_envs = puppet_environments_available_to_capsule(environments, content_view) || []
-        yum_repos + puppet_envs
-      end
-
-      def yum_repos_available_to_capsule(environments = nil, content_view = nil)
-        environments = @smart_proxy.lifecycle_environments if environments.nil?
-        yum_repos = Katello::Repository.in_environment(environments)
-        yum_repos = yum_repos.in_content_views([content_view]) if content_view
-        yum_repos.find_all { |repo| repo.node_syncable? }
-      end
-
-      def puppet_environments_available_to_capsule(environments = nil, content_view = nil)
-        environments = @smart_proxy.lifecycle_environments if environments.nil?
-        puppet_environments = Katello::ContentViewPuppetEnvironment.in_environment(environments)
-        puppet_environments = puppet_environments.in_content_view(content_view) if content_view
-        puppet_environments
-      end
-
-      def affected_repositories(environment, content_view, repository)
-        if repository
-          [repository]
-        else
-          repos_available_to_capsule(environment, content_view)
-        end
-      end
-
-      def repos_needing_updates(environment, content_view, repository)
-        repos = affected_repositories(environment, content_view, repository)
-        need_importer_update = needs_importer_updates(repos)
-        need_distributor_update = needs_distributor_updates(repos)
-        (need_distributor_update + need_importer_update).uniq
-      end
-
       def current_repositories(environment_id = nil, content_view_id = nil)
         yum_repos = current_yum_repos(environment_id, content_view_id) || []
         puppet_envs = current_puppet_environments(environment_id, content_view_id) || []
@@ -61,9 +26,10 @@ module Katello
         katello_repos = katello_repos.where(:environment_id => environment_id) if environment_id
         katello_repos = katello_repos.in_content_views([content_view_id]) if content_view_id
 
-        pulp_repos = self.smart_proxy.pulp_api.extensions.repository.search_by_repository_ids(katello_repos.pluck(:pulp_id))
+        pulp2_repos = self.smart_proxy.pulp_api.extensions.repository.search_by_repository_ids(katello_repos.pluck(:pulp_id))
+        pulp_repo_ids = pulp2_repos.map { |pulp_repo| pulp_repo['id'] }
 
-        katello_repos.where(:pulp_id => pulp_repos.map { |pulp_repo| pulp_repo['id'] })
+        katello_repos.where(:pulp_id => pulp_repo_ids)
       end
 
       def current_puppet_environments(environment_id = nil, content_view_id = nil)
@@ -92,73 +58,24 @@ module Katello
       end
 
       def orphaned_repos
-        @smart_proxy.pulp_repositories.map { |x| x["id"] } - current_repositories.map { |x| x.pulp_id }
+        @smart_proxy.pulp_repositories.map { |x| x["id"] } - repos_available_to_capsule.map { |x| x.pulp_id }
       end
 
-      def delete_orphaned_repos
-        orphaned_repos.map { |repo| self.smart_proxy.pulp_api.extensions.repository.delete(repo) }.compact
-      end
-
-      def needs_importer_updates(repos)
-        repos.select do |repo|
-          repo_details = repo.backend_service(self.smart_proxy).backend_data
-          next unless repo_details
-          capsule_importer = repo_details["importers"][0]
-          !importer_matches?(repo, capsule_importer)
-        end
-      end
-
-      def needs_distributor_updates(repos)
-        repos.select do |repo|
-          repo_details = repo.backend_service(smart_proxy).backend_data
-          next unless repo_details
-          !distributors_match?(repo, repo_details["distributors"])
-        end
+      def repos_available_to_capsule
+        yum_repos_available_to_capsule + puppet_environments_available_to_capsule
       end
 
-      def importer_matches?(repo, capsule_importer)
-        generated_importer = repo.backend_service(self.smart_proxy).generate_importer
-        capsule_importer.try(:[], 'importer_type_id') == generated_importer.id &&
-            generated_importer.config.compact == capsule_importer['config'].compact
-      end
-
-      def distributors_match?(repo, capsule_distributors)
-        generated_distributor_configs = repo.backend_service(self.smart_proxy).generate_distributors
-        generated_distributor_configs.all? do |gen_dist|
-          type = gen_dist.class.type_id
-          found_on_capsule = capsule_distributors.find { |dist| dist['distributor_type_id'] == type }
-          found_on_capsule && filtered_distribution_config_equal?(gen_dist.config, found_on_capsule['config'])
-        end
+      def yum_repos_available_to_capsule
+        yum_repos = Katello::Repository.in_environment(@smart_proxy.lifecycle_environments)
+        yum_repos.find_all { |repo| repo.node_syncable? }
       end
 
-      def get_repository_ids(environment, content_view, repository)
-        if environment
-          repository_ids = repos_available_to_capsule(environment, content_view).map(&:pulp_id)
-        elsif repository
-          repository_ids = [repository.pulp_id]
-          environment = repository.environment
-        else
-          repository_ids = repos_available_to_capsule.map(&:pulp_id)
-        end
-
-        if environment && !self.smart_proxy.lifecycle_environments.include?(environment)
-          fail _("Lifecycle environment '%{environment}' is not attached to this capsule.") % { :environment => environment.name }
-        end
-
-        repository_ids
+      def puppet_environments_available_to_capsule
+        Katello::ContentViewPuppetEnvironment.in_environment(@smart_proxy.lifecycle_environments)
       end
 
-      private
-
-      def filtered_distribution_config_equal?(generated_config, actual_config)
-        generated = generated_config.clone
-        actual = actual_config.clone
-        #We store 'default' checksum type as nil, but pulp will default to sha256, so if we haven't set it, ignore it
-        if generated.keys.include?('checksum_type') && generated['checksum_type'].nil?
-          generated.delete('checksum_type')
-          actual.delete('checksum_type')
-        end
-        generated.compact == actual.compact
+      def delete_orphaned_repos
+        orphaned_repos.map { |repo| self.smart_proxy.pulp_api.extensions.repository.delete(repo) }.compact
       end
     end
   end

data/app/services/katello/pulp3/ansible_collection.rb

@@ -4,16 +4,16 @@ module Katello
       include LazyAccessor
 
       def self.content_api
-        PulpAnsibleClient::ContentCollectionsApi.new(Katello::Pulp3::Repository::AnsibleCollection.api_client(SmartProxy.pulp_master!))
+        PulpAnsibleClient::ContentCollectionVersionsApi.new(Katello::Pulp3::Repository::AnsibleCollection.api_client(SmartProxy.pulp_master!))
       end
 
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::AnsibleCollection.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
-        repo_content_list.map { |content| content.try(:_href) }
+        repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def self.pulp_data(_href)
+      def self.pulp_data(_pulp_href)
         #No content read method
         fail NotImplementedError
       end
@@ -24,7 +24,11 @@ module Katello
         custom_json['namespace'] = backend_data['namespace']
         custom_json['version'] = backend_data['version']
         custom_json['name'] = backend_data['name']
+        custom_json['description'] = backend_data['description']
         model.update_attributes!(custom_json)
+
+        tags = backend_data['tags'].map { |tag| Katello::AnsibleTag.find_or_create_by(name: tag['name']) }
+        model.tags = tags
       end
     end
   end

data/app/services/katello/pulp3/content.rb

@@ -0,0 +1,60 @@
+require "pulpcore_client"
+module Katello
+  module Pulp3
+    class Content
+      extend Katello::Abstract::Pulp::Content
+      class << self
+        def create_upload(size = 0, checksum = nil, content_type = nil)
+          content_unit_href = nil
+          if checksum && content_type
+            content_backend_service = SmartProxy.pulp_master.content_service(content_type)
+            content_list = content_backend_service.content_api.list("digest": checksum)
+            content_unit_href = content_list.results.first.pulp_href unless content_list.results.empty?
+            return {"content_unit_href" => content_unit_href} if content_unit_href
+          end
+          upload_href = uploads_api.create(upload_class.new(size: size)).pulp_href
+          {"upload_id" => upload_href.split("/").last}
+        end
+
+        def delete_upload(upload_href)
+          #Commit deletes upload request for pulp3. Not needed other than to implement abstract method.
+        end
+
+        def upload_chunk(upload_href, offset, content, size)
+          upload_href = "/pulp/api/v3/uploads/" + upload_href + "/"
+          offset = offset.try(:to_i)
+          size = size.try(:to_i)
+          begin
+            filechunk = Tempfile.new('filechunk', :encoding => 'ascii-8bit')
+            filechunk.write(content)
+            filechunk.flush
+            actual_chunk_size = File.size(filechunk)
+            uploads_api.update(upload_href, content_range(offset, offset + actual_chunk_size - 1, size), filechunk)
+          ensure
+            filechunk.close
+            filechunk.unlink
+          end
+        end
+
+        private
+
+        def core_api_client
+          PulpcoreClient::ApiClient.new(SmartProxy.pulp_master.pulp3_configuration(PulpcoreClient::Configuration))
+        end
+
+        def uploads_api
+          PulpcoreClient::UploadsApi.new(core_api_client)
+        end
+
+        def upload_class
+          PulpcoreClient::Upload
+        end
+
+        def content_range(start, finish, total)
+          finish = finish > total ? total : finish
+          "bytes #{start}-#{finish}/#{total}"
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/docker_blob.rb

@@ -10,7 +10,7 @@ module Katello
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
-        repo_content_list.map { |content| content.try(:_href) }
+        repo_content_list.map { |content| content.try(:pulp_href) }
       end
     end
   end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -10,7 +10,7 @@ module Katello
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
-        repo_content_list.map { |content| content.try(:_href) }
+        repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
       def self.content_unit_list(page_opts)

data/app/services/katello/pulp3/docker_manifest_list.rb

@@ -10,7 +10,7 @@ module Katello
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
-        repo_content_list.map { |content| content.try(:_href) }
+        repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
       def self.content_unit_list(page_opts)

data/app/services/katello/pulp3/docker_tag.rb

@@ -4,13 +4,13 @@ module Katello
       include LazyAccessor
 
       def self.content_api
-        PulpDockerClient::ContentManifestTagsApi.new(Katello::Pulp3::Repository::Docker.api_client(SmartProxy.pulp_master!))
+        PulpDockerClient::ContentTagsApi.new(Katello::Pulp3::Repository::Docker.api_client(SmartProxy.pulp_master!))
       end
 
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::Docker.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
-        repo_content_list.map { |content| content.try(:_href) }
+        repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
       def update_model(model)

data/app/services/katello/pulp3/file_unit.rb

@@ -2,15 +2,21 @@ module Katello
   module Pulp3
     class FileUnit < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "file".freeze
 
       def self.content_api
         PulpFileClient::ContentFilesApi.new(Katello::Pulp3::Repository::File.api_client(SmartProxy.pulp_master!))
       end
 
+      def self.create_content(options)
+        fail _("Artifact Id and relative path are needed to create content") unless options.dig(:file_name) && options.dig(:artifact)
+        PulpFileClient::FileContent.new(relative_path: options[:file_name], artifact: options[:artifact])
+      end
+
       def self.ids_for_repository(repo_id)
         repo = Katello::Pulp3::Repository::File.new(Katello::Repository.find(repo_id), SmartProxy.pulp_master)
         repo_content_list = repo.content_list
-        repo_content_list.map { |content| content.try(:_href) }
+        repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
       def update_model(model)

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -11,6 +11,10 @@ module Katello
         fail NotImplementedError
       end
 
+      def self.create_content
+        fail NotImplementedError
+      end
+
       def update_model
         fail NotImplementedError
       end
@@ -27,24 +31,30 @@ module Katello
       end
 
       def self.unit_identifier
-        "_href"
+        "pulp_href"
+      end
+
+      def self.content_type
+        self::CONTENT_TYPE
       end
 
       def self.pulp_units_batch_for_repo(repository, options = {})
         fetch_identifiers = options.fetch(:fetch_identifiers, false)
         page_size = options.fetch(:page_size, SETTINGS[:katello][:pulp][:bulk_load_size])
         repository_version_href = repository.version_href
-        page_opts = { "page" => 1, repository_version: repository_version_href, page_size: page_size}
-        page_opts[:fields] = '_href' if fetch_identifiers
+        page_opts = { "offset" => 0, repository_version: repository_version_href, limit: page_size }
+        page_opts[:fields] = 'pulp_href' if fetch_identifiers
         response = {}
         Enumerator.new do |yielder|
           loop do
             page_opts = page_opts.with_indifferent_access
-            break unless (response["next"] || page_opts["page"] == 1)
+            break unless (
+              (response["count"] && page_opts["offset"] < response["count"]) ||
+              page_opts["offset"] == 0)
             response = fetch_content_list page_opts
             response = response.as_json.with_indifferent_access
             yielder.yield response[:results]
-            page_opts[:page] += 1
+            page_opts[:offset] += page_size
           end
         end
       end