katello 4.2.0.rc1 → 4.2.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8aa0f375342de9345f09a8a5b1cd5d7ab68af3282a4b8863cf9b63c33130fbcd
-  data.tar.gz: 871e47ae53fdd69760c042278b9feb68a1b0d81599db67cfae9ee375a270034e
+  metadata.gz: '0888b27382889ee155584abcf025ee49161afc59cd18d578a7fd517d5a50dae5'
+  data.tar.gz: 5e488c438f3a561d2b76eaf6d657842ec117efb95d887af8f7dfa040d265e5ae
 SHA512:
-  metadata.gz: 11757b1ba7c279ca6a6b70760996c8e81d0be8fc935ba0da3dd531beb69d00a4dadd9b113a7dd8f036ecd9d035c71a7f8ce2a5bebf4e53e8f0d9b80cccef4b92
-  data.tar.gz: 4c9337e95fda3f0e010f9fc11c3e838f15aea5f10fae1dd17312ea067c2bb20d5f5d2b05c36fee9ce274c3916db2c866d10e321c7b9a278eb2c8543983a7e3b4
+  metadata.gz: 7572571d4da028a2c7d8f1822632b1629a274ad6f0f21746f345f994ae60d04c4e5f8e22f3f2c848df6c809ba497d49487ccd491b4748ca31978926ee8ec9e9f
+  data.tar.gz: 6ef52bd64650d97097a29590bd8602045ec9e0c72978047be9e5f998bff112314f6be1028dc06f977682daea13ca1d270c4d0d13587e56dbabba8f5ea3e66c08

data/app/controllers/katello/api/rhsm/candlepin_dynflow_proxy_controller.rb

@@ -15,7 +15,16 @@ module Katello
     #param :id, String, :desc => N_("UUID of the consumer"), :required => true
     def upload_package_profile
       User.as_anonymous_admin do
-        async_task(::Actions::Katello::Host::UploadPackageProfile, @host, request.raw_post)
+        if Setting['upload_profiles_without_dynflow']
+          uploader = ::Katello::Host::PackageProfileUploader.new(
+            host: @host,
+            profile_string: request.raw_post
+          )
+          uploader.upload
+          uploader.trigger_applicability_generation
+        else
+          async_task(::Actions::Katello::Host::UploadPackageProfile, @host, request.raw_post)
+        end
       end
       render :json => Resources::Candlepin::Consumer.get(@host.subscription_facet.uuid)
     end
@@ -24,7 +33,16 @@ module Katello
     param :id, String, :desc => N_("UUID of the consumer"), :required => true
     def upload_profiles
       User.as_anonymous_admin do
-        async_task(::Actions::Katello::Host::UploadProfiles, @host, request.raw_post)
+        if Setting['upload_profiles_without_dynflow']
+          uploader = ::Katello::Host::ProfilesUploader.new(
+            host: @host,
+            profile_string: request.raw_post
+          )
+          uploader.upload
+          uploader.trigger_applicability_generation
+        else
+          async_task(::Actions::Katello::Host::UploadProfiles, @host, request.raw_post)
+        end
       end
       render :json => Resources::Candlepin::Consumer.get(@host.subscription_facet.uuid)
     end

data/app/controllers/katello/api/v2/content_uploads_controller.rb

@@ -12,7 +12,8 @@ module Katello
     param :checksum, String, :required => false, :desc => N_("Checksum of file to upload")
     param :content_type, RepositoryTypeManager.uploadable_content_types(false).map(&:label), :required => false, :desc => N_("content type ('deb', 'docker_manifest', 'file', 'ostree', 'rpm', 'srpm')")
     def create
-      content_type = params[:content_type] || ::Katello::RepositoryTypeManager.find(@repository.content_type).default_managed_content_type.label
+      fail Katello::Errors::InvalidRepositoryContent, _("Cannot upload Ansible collections.") if @repository.ansible_collection?
+      content_type = params[:content_type] || ::Katello::RepositoryTypeManager.find(@repository.content_type)&.default_managed_content_type&.label
       RepositoryTypeManager.check_content_matches_repo_type!(@repository, content_type)
 
       unit_type_id = SmartProxy.pulp_primary.content_service(content_type).content_type

data/app/controllers/katello/api/v2/repositories_controller.rb

@@ -360,6 +360,7 @@ module Katello
     param :content_type, RepositoryTypeManager.uploadable_content_types(false).map(&:label), :required => false, :desc => N_("content type ('deb', 'docker_manifest', 'file', 'ostree', 'rpm', 'srpm')")
     def upload_content
       fail Katello::Errors::InvalidRepositoryContent, _("Cannot upload Container Image content.") if @repository.docker?
+      fail Katello::Errors::InvalidRepositoryContent, _("Cannot upload Ansible collections.") if @repository.ansible_collection?
 
       filepaths = Array.wrap(params[:content]).compact.collect do |content|
         {path: content.path, filename: content.original_filename}

data/app/lib/actions/katello/content_view_version/create_repos.rb

@@ -10,7 +10,7 @@ module Actions
             source_repositories.each do |repositories|
               new_repository = repositories.first.build_clone(content_view: version.content_view,
                                                              version: version)
-              plan_action(Repository::Create, new_repository, true)
+              plan_action(Repository::Create, new_repository, clone: true)
               repository_mapping[repositories] = new_repository
             end
           end

data/app/lib/actions/katello/host/upload_package_profile.rb

@@ -30,28 +30,14 @@ module Actions
           Dynflow::Action::Rescue::Skip
         end
 
-        def self.upload(host_id, profile)
-          host = ::Host.find_by(:id => host_id)
-          if host.nil?
-            Rails.logger.warn("Host with ID %s not found, continuing" % host_id)
-          elsif host.content_facet.nil? || host.content_facet.uuid.nil?
-            Rails.logger.warn("Host with ID %s has no content facet, continuing" % host_id)
-          else
-            begin
-              simple_packages = profile.map { |item| ::Katello::Pulp::SimplePackage.new(item) }
-              host.import_package_profile(simple_packages)
-            rescue ActiveRecord::InvalidForeignKey # this happens if the host gets deleted in between the "find_by" and "import_package_profile"
-              Rails.logger.warn("Host installed package list with ID %s was not able to be written to the DB (host likely is deleted), continuing" % host_id)
-            end
-          end
-        end
-
         def run
-          profile = JSON.parse(input[:profile_string])
-          #free the huge string from the memory
-          input[:profile_string] = 'TRIMMED'.freeze
-          UploadPackageProfile.upload(input[:host_id], profile)
-          ::Katello::Host::ContentFacet.trigger_applicability_generation(input[:host_id])
+          host = ::Host.find_by(:id => input[:host_id])
+          uploader = ::Katello::Host::PackageProfileUploader.new(
+            host: host,
+            profile_string: input[:profile_string]
+          )
+          uploader.upload
+          uploader.trigger_applicability_generation
         end
       end
     end

data/app/lib/actions/katello/host/upload_profiles.rb

@@ -32,60 +32,14 @@ module Actions
           Dynflow::Action::Rescue::Skip
         end
 
-        def import_module_streams(payload, host)
-          enabled_payload = payload.map do |profile|
-            profile.slice("name", "stream", "version", "context", "arch").with_indifferent_access if profile["status"] == "enabled"
-          end
-          enabled_payload.compact!
-
-          host.import_module_streams(payload)
-        end
-
-        def import_deb_package_profile(host, profile)
-          installed_deb_ids = profile.map do |item|
-            ::Katello::InstalledDeb.find_or_create_by(name: item['name'], architecture: item['architecture'], version: item['version']).id
-          end
-          host.installed_deb_ids = installed_deb_ids
-          host.save!
-        rescue ActiveRecord::InvalidForeignKey # this happens if the host gets deleted in between the "find_by" and "import_package_profile"
-          Rails.logger.warn("Host installed package list with ID %s was not able to be written to the DB (host likely is deleted), continuing" % host.id)
-        end
-
         def run
-          profiles = JSON.parse(input[:profile_string])
-          #free the huge string from the memory
-          input[:profile_string] = 'TRIMMED'.freeze
           host = ::Host.find_by(:id => input[:host_id])
-          if host.nil?
-            Rails.logger.warn("Host with ID %s not found, continuing" % input[:host_id])
-          elsif host.content_facet.nil? || host.content_facet.uuid.nil?
-            Rails.logger.warn("Host with ID %s has no content facet, continuing" % input[:host_id])
-          elsif profiles.try(:has_key?, "deb_package_profile")
-            # remove this when deb_package_profile API is removed
-            payload = profiles.dig("deb_package_profile", "deb_packages") || []
-            import_deb_package_profile(host, payload)
-          else
-            module_streams = []
-            profiles.each do |profile|
-              payload = profile["profile"]
-              case profile["content_type"]
-              when "rpm"
-                UploadPackageProfile.upload(input[:host_id], payload)
-              when "deb"
-                import_deb_package_profile(host, payload)
-              when "enabled_repos"
-                host.import_enabled_repositories(payload)
-              else
-                module_streams << payload
-              end
-            end
-
-            module_streams.each do |module_stream_payload|
-              import_module_streams(module_stream_payload, host)
-            end
-
-            ::Katello::Host::ContentFacet.trigger_applicability_generation(host.id)
-          end
+          uploader = ::Katello::Host::ProfilesUploader.new(
+            host: host,
+            profile_string: input[:profile_string]
+          )
+          uploader.upload
+          uploader.trigger_applicability_generation
         end
       end
     end

data/app/lib/actions/katello/orphan_cleanup/remove_orphans.rb

@@ -27,6 +27,8 @@ module Actions
           models.each do |model|
             model.joins("left join katello_#{model.repository_association} on #{model.table_name}.id = katello_#{model.repository_association}.#{model.unit_id_field}").where("katello_#{model.repository_association}.#{model.unit_id_field} IS NULL").destroy_all
           end
+
+          ::Katello::RootRepository.orphaned.destroy_all
         end
       end
     end

data/app/lib/actions/katello/repository/clone_to_environment.rb

@@ -9,7 +9,7 @@ module Actions
 
           sequence do
             if clone.new_record?
-              plan_action(Repository::Create, clone, true)
+              plan_action(Repository::Create, clone, clone: true)
             end
 
             plan_action(::Actions::Katello::Repository::CloneContents, [repository], clone, :copy_contents => !clone.yum?)

data/app/lib/actions/katello/repository/create.rb

@@ -2,7 +2,9 @@ module Actions
   module Katello
     module Repository
       class Create < Actions::EntryAction
-        def plan(repository, clone = false)
+        def plan(repository, args = {})
+          clone = args[:clone] || false
+          force_repo_create = args[:force_repo_create] || false
           repository.save!
           root = repository.root
 
@@ -11,7 +13,7 @@ module Actions
           org = repository.organization
           sequence do
             create_action = plan_action(Pulp3::Orchestration::Repository::Create,
-                                        repository, SmartProxy.pulp_primary)
+                                        repository, SmartProxy.pulp_primary, force_repo_create)
 
             return if create_action.error
 
@@ -22,8 +24,10 @@ module Actions
               if repository.product.redhat?
                 plan_action(Actions::Candlepin::Environment::AddContentToEnvironment, :view_env_cp_id => view_env.cp_id, :content_id => repository.content_id)
               else
-                content_create = plan_action(Katello::Product::ContentCreate, root)
-                plan_action(Actions::Candlepin::Environment::AddContentToEnvironment, :view_env_cp_id => view_env.cp_id, :content_id => content_create.input[:content_id])
+                unless root.content
+                  content_create = plan_action(Katello::Product::ContentCreate, root)
+                  plan_action(Actions::Candlepin::Environment::AddContentToEnvironment, :view_env_cp_id => view_env.cp_id, :content_id => content_create.input[:content_id])
+                end
               end
             end
 

data/app/lib/actions/katello/repository/create_root.rb

@@ -10,7 +10,7 @@ module Actions
           repository.relative_path = repository.custom_repo_path
           repository.save!
           action_subject(repository)
-          plan_action(::Actions::Katello::Repository::Create, repository, false)
+          plan_action(::Actions::Katello::Repository::Create, repository)
         end
 
         def humanized_name

data/app/lib/actions/katello/repository/import_upload.rb

@@ -63,7 +63,7 @@ module Actions
         def results_to_json(results)
           json_results = []
           results.each do |result|
-            result[:pulp_tasks].each do |task|
+            result[:pulp_tasks]&.each do |task|
               details = task ? task.dig(:result, :details, :unit) : nil
               if details && details.dig('type_id') == 'docker_manifest'
                 manifest = ::Katello::DockerManifest.find_by(:pulp_id => details.dig(:metadata, :id))
@@ -79,6 +79,11 @@ module Actions
                 json_results << {:type => 'file'}
               end
             end
+            unless json_results.size
+              if result[:content_unit_href]
+                json_results << {:type => 'file'}
+              end
+            end
           end
           json_results
         end

data/app/lib/actions/katello/repository/upload_files.rb

@@ -21,14 +21,12 @@ module Actions
           generate_applicability = options.fetch(:generate_applicability, repository.yum?)
 
           sequence do
-            concurrence do
-              tmp_files.each do |file|
-                sequence do
-                  upload_action = plan_action(Pulp3::Orchestration::Repository::UploadContent,
-                                                   repository, SmartProxy.pulp_primary!, file, unit_type_id)
-
-                  upload_actions << upload_action.output
-                end
+            tmp_files.each do |file|
+              sequence do
+                upload_action = plan_action(Pulp3::Orchestration::Repository::UploadContent,
+                                                 repository, SmartProxy.pulp_primary!, file, unit_type_id)
+
+                upload_actions << upload_action.output
               end
             end
 

data/app/lib/actions/katello/repository_set/enable_repository.rb

@@ -15,7 +15,7 @@ module Actions
             fail ::Katello::Errors::ConflictException, _("The repository is already enabled")
           end
           repository = mapper.build_repository
-          plan_action(Repository::Create, repository, false)
+          plan_action(Repository::Create, repository, clone: false)
           action_subject(repository)
           plan_self
         end

data/app/lib/actions/pulp3/orchestration/repository/create.rb

@@ -3,9 +3,9 @@ module Actions
     module Orchestration
       module Repository
         class Create < Pulp3::Abstract
-          def plan(repository, smart_proxy)
+          def plan(repository, smart_proxy, force = false)
             sequence do
-              create_action = plan_action(Actions::Pulp3::Repository::Create, repository, smart_proxy)
+              create_action = plan_action(Actions::Pulp3::Repository::Create, repository, smart_proxy, force)
               plan_action(Actions::Pulp3::Repository::SaveVersion, repository, repository_details: create_action.output[:response])
 
               if repository.content_view.default? || !smart_proxy.pulp_primary?

data/app/lib/actions/pulp3/orchestration/repository/import_upload.rb

@@ -4,15 +4,36 @@ module Actions
     module Orchestration
       module Repository
         class ImportUpload < Pulp3::Abstract
+          # rubocop:disable Metrics/AbcSize
           def plan(repository, smart_proxy, args)
-            file = {:filename => args.dig(:unit_key, :name)}
+            file = {:filename => args.dig(:unit_key, :name), :sha256 => args.dig(:unit_key, :checksum) }
             content_unit_href = args.dig(:unit_key, :content_unit_id)
             docker_tag = (args.dig(:unit_type_id) == "docker_tag")
             sequence do
               if content_unit_href
-                plan_self(:commit_output => [], :content_unit_href => content_unit_href)
-                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
-                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+                content_backend_service = SmartProxy.pulp_primary.content_service(args.dig(:unit_type_id))
+                duplicate_sha_path_content_list = content_backend_service.content_api.list(
+                    "sha256": file[:sha256],
+                    "relative_path": file[:filename])
+                duplicate_content_href = duplicate_sha_path_content_list&.results&.first&.pulp_href
+                if duplicate_content_href
+                  plan_self(:commit_output => [], :content_unit_href => duplicate_content_href)
+                  action_output = plan_action(Pulp3::Repository::ImportUpload, duplicate_content_href, repository, smart_proxy).output
+                  plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+                else
+                  duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": file[:sha256])
+                  duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
+                  if duplicate_sha_artifact_href
+                    artifact_output = plan_action(Pulp3::Repository::SaveArtifact,
+                                                         file, repository, smart_proxy,
+                                                         nil, args.dig(:unit_type_id),
+                                                         artifact_href: duplicate_sha_artifact_href).output
+                    content_unit_href = artifact_output[:pulp_tasks]
+                    plan_self(:commit_output => nil, :artifact_output => artifact_output[:pulp_tasks])
+                    action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                    plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+                  end
+                end
               elsif docker_tag
                 tag_manifest_output = plan_action(Pulp3::Repository::UploadTag,
                             repository,

data/app/lib/actions/pulp3/orchestration/repository/upload_content.rb

@@ -7,14 +7,26 @@ module Actions
           def plan(repository, smart_proxy, file, unit_type_id)
             sequence do
               content_backend_service = SmartProxy.pulp_primary.content_service(unit_type_id)
-              content_list = content_backend_service.content_api.list("sha256": Digest::SHA256.hexdigest(File.read(file[:path])))
-              content_href = content_list&.results&.first&.pulp_href
+              duplicate_sha_path_content_list = content_backend_service.content_api.list(
+                  "sha256": Digest::SHA256.hexdigest(File.read(file[:path])),
+                  "relative_path": file[:filename])
+              duplicate_content_href = duplicate_sha_path_content_list&.results&.first&.pulp_href
 
-              unless content_href
-                upload_action_output = plan_action(Pulp3::Repository::UploadFile, repository, smart_proxy, file[:path]).output
-                artifact_action_output = plan_action(Pulp3::Repository::SaveArtifact, file, repository, smart_proxy, upload_action_output[:pulp_tasks], unit_type_id).output
+              unless duplicate_content_href
+                duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": Digest::SHA256.hexdigest(File.read(file[:path])))
+                duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
+              end
+
+              unless duplicate_content_href
+                if duplicate_sha_artifact_href
+                  artifact_action_output = plan_action(Pulp3::Repository::SaveArtifact, file, repository, smart_proxy, nil, unit_type_id, artifact_href: duplicate_sha_artifact_href).output
+                else
+                  upload_action_output = plan_action(Pulp3::Repository::UploadFile, repository, smart_proxy, file[:path]).output
+                  artifact_action_output = plan_action(Pulp3::Repository::SaveArtifact, file, repository, smart_proxy, upload_action_output[:pulp_tasks], unit_type_id).output
+                end
                 content_href = artifact_action_output[:pulp_tasks]
               end
+              content_href ||= duplicate_content_href
               action_output = plan_action(Pulp3::Repository::ImportUpload, content_href, repository, smart_proxy).output
               plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
               plan_self(:subaction_output => action_output)

data/app/lib/actions/pulp3/repository/commit_upload.rb

@@ -10,8 +10,31 @@ module Actions
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
           uploads_api = repo_backend_service.core_api.uploads_api
-          upload_commit = repo_backend_service.core_api.upload_commit_class.new(sha256: input[:sha256])
-          output[:pulp_tasks] = [uploads_api.commit(input[:upload_href], upload_commit)]
+          duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": input[:sha256])
+          duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
+          if duplicate_sha_artifact_href
+            uploads_api.delete(upload_href)
+            output[:artifact_href] = duplicate_sha_artifact_href
+            output[:pulp_tasks] = nil
+          else
+            output[:artifact_href] = nil
+            upload_commit = repo_backend_service.core_api.upload_commit_class.new(sha256: input[:sha256])
+            output[:pulp_tasks] = [uploads_api.commit(input[:upload_href], upload_commit)]
+          end
+        end
+
+        def check_for_errors
+          combined_tasks.each do |task|
+            if unique_error task.error
+              warn _("Duplicate artifact detected")
+            else
+              super
+            end
+          end
+        end
+
+        def unique_error(message)
+          message&.include?("code='unique'")
         end
       end
     end

data/app/lib/actions/pulp3/repository/create.rb

@@ -2,13 +2,14 @@ module Actions
   module Pulp3
     module Repository
       class Create < Pulp3::Abstract
-        def plan(repository, smart_proxy)
-          plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id)
+        def plan(repository, smart_proxy, force = false)
+          plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :force => force)
         end
 
         def run
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:response] = repo.backend_service(smart_proxy).with_mirror_adapter.create
+          force = input[:force] || false
+          output[:response] = repo.backend_service(smart_proxy).with_mirror_adapter.create(force)
         end
       end
     end

data/app/lib/actions/pulp3/repository/refresh_distribution.rb

@@ -6,13 +6,16 @@ module Actions
         middleware.use Actions::Middleware::ExecuteIfContentsChanged
 
         def plan(repository, smart_proxy, options = {})
-          smart_proxy = SmartProxy.find_by(id: smart_proxy) #support bulk actions
+          smart_proxy = SmartProxy.unscoped.find_by(id: smart_proxy) #support bulk actions
           sequence do
             if !repository.unprotected && !options[:assume_content_guard_exists]
               plan_action(::Actions::Pulp3::ContentGuard::Refresh, smart_proxy)
             end
 
-            refresh_options = {:repository_id => repository.id, :smart_proxy_id => smart_proxy.id}
+            refresh_options = {
+              :repository_id => repository.id,
+              :smart_proxy_id => smart_proxy.id
+            }
             refresh_options[:contents_changed] if options.key?(:contents_changed)
             action = plan_self(refresh_options)