katello 3.18.0.rc2.1 → 3.18.2.1

data/app/lib/actions/pulp3/orchestration/content_view_version/export_library.rb

@@ -0,0 +1,60 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class ExportLibrary < Actions::EntryAction
+          def plan(organization, destination_server: nil,
+                                 chunk_size: nil,
+                                 from_history: nil,
+                                 fail_on_missing_content: false)
+            action_subject(organization)
+            validate_repositories_immediate!(organization) if fail_on_missing_content
+
+            content_view = ::Katello::Pulp3::ContentViewVersion::Export.find_library_export_view(destination_server: destination_server,
+                                                                           organization: organization,
+                                                                           create_by_default: true)
+            repo_ids_in_library = organization.default_content_view_version.repositories.yum_type.immediate.pluck(:id)
+            content_view.update!(repository_ids: repo_ids_in_library)
+
+            sequence do
+              publish_action = plan_action(::Actions::Katello::ContentView::Publish, content_view, '')
+              export_action = plan_action(::Actions::Pulp3::Orchestration::ContentViewVersion::Export,
+                                          content_view_version: publish_action.version,
+                                          destination_server: destination_server,
+                                          chunk_size: chunk_size,
+                                          from_history: from_history,
+                                          validate_incremental: false,
+                                          fail_on_missing_content: fail_on_missing_content)
+              plan_self(export_action_output: export_action.output)
+            end
+          end
+
+          def run
+            output[:export_history_id] = input[:export_action_output][:export_history_id]
+          end
+
+          def humanized_name
+            _("Export Library")
+          end
+
+          def rescue_strategy
+            Dynflow::Action::Rescue::Skip
+          end
+
+          def validate_repositories_immediate!(organization)
+            non_immediate_repos = organization.default_content_view_version.repositories.yum_type.non_immediate
+            if non_immediate_repos.any?
+              fail _("NOTE: Unable to fully export '%{organization}' organization's library because"\
+                     " it contains repositories without the 'immediate' download policy."\
+                     " Update the download policy and sync affected repositories to include them in the export."\
+                     " \n %{repos}" %
+                     { organization: organization.name,
+                       repos: ::Katello::Pulp3::ContentViewVersion::Export
+                              .generate_product_repo_strings(repositories: non_immediate_repos)})
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/import.rb

@@ -36,10 +36,6 @@ module Actions
           def humanized_name
             _("Import")
           end
-
-          def rescue_strategy
-            Dynflow::Action::Rescue::Skip
-          end
         end
       end
     end

data/app/lib/actions/pulp3/orchestration/repository/import_upload.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Metrics/MethodLength
 module Actions
   module Pulp3
     module Orchestration
@@ -6,9 +7,19 @@ module Actions
           def plan(repository, smart_proxy, args)
             file = {:filename => args.dig(:unit_key, :name)}
             content_unit_href = args.dig(:unit_key, :content_unit_id)
+            docker_tag = (args.dig(:unit_type_id) == "docker_tag")
             sequence do
               if content_unit_href
                 plan_self(:commit_output => [], :content_unit_href => content_unit_href)
+                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+              elsif docker_tag
+                tag_manifest_output = plan_action(Pulp3::Repository::UploadTag,
+                            repository,
+                            smart_proxy,
+                            args).output
+                plan_self(:commit_output => tag_manifest_output[:pulp_tasks])
+                plan_action(Pulp3::Repository::SaveVersion, repository, {force_fetch_version: true, tasks: tag_manifest_output[:pulp_tasks]})
               else
                 commit_output = plan_action(Pulp3::Repository::CommitUpload,
                                             repository,
@@ -24,9 +35,9 @@ module Actions
                                               args.dig(:unit_type_id)).output
                 content_unit_href = artifact_output[:pulp_tasks]
                 plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output[:pulp_tasks])
+                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
               end
-              action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
-              plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
             end
           end
 
@@ -34,8 +45,10 @@ module Actions
             output[:pulp_tasks] = input[:commit_output]
             if input[:content_unit_href]
               output[:content_unit_href] = input[:content_unit_href]
-            else
+            elsif input[:artifact_output]
               output[:content_unit_href] = input[:artifact_output].last[:created_resources].first
+            else
+              output[:content_unit_href] = nil
             end
           end
         end

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -13,7 +13,7 @@ module Actions
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
           service = target.backend_service(smart_proxy)
           output[:pulp_tasks] = if input[:copy_all]
-                                  service.copy_all(source)
+                                  service.copy_all(source, mirror: input[:mirror] || false)
                                 else
                                   service.copy_content_for_source(source, input)
                                 end

data/app/lib/actions/pulp3/repository/delete.rb

@@ -8,7 +8,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:response] = repo.backend_service(smart_proxy).delete
+          output[:response] = repo.backend_service(smart_proxy).delete_repository
         end
       end
     end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -2,7 +2,7 @@ module Actions
   module Pulp3
     module Repository
       class SaveVersion < Pulp3::Abstract
-        def plan(repository, options)
+        def plan(repository, options = {})
           fail "Cannot accept tasks and repository_details into Save Version." if options[:tasks].present? && options[:repository_details].present?
           plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details], :force_fetch_version => options.fetch(:force_fetch_version, false))
         end

data/app/lib/actions/pulp3/repository/upload_tag.rb

@@ -0,0 +1,18 @@
+module Actions
+  module Pulp3
+    module Repository
+      class UploadTag < Pulp3::AbstractAsyncTask
+        def plan(repository, smart_proxy, args)
+          plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :args => args)
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          tag_name = input[:args].dig(:unit_key, :name)
+          manifest_digest = input[:args].dig(:unit_key, :digest)
+          output[:pulp_tasks] = [repo.backend_service(smart_proxy).tag_manifest(tag_name, manifest_digest)]
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/util/pulpcore_content_filters.rb

@@ -8,7 +8,7 @@ module Katello
       def filter_package_groups_by_pulp_href(package_groups, package_pulp_hrefs)
         rpms = Katello::Rpm.where(:pulp_id => package_pulp_hrefs)
         package_groups.reject do |package_group|
-          (package_group.package_names & rpms.pluck(:name)).empty?
+          (package_group.package_names - rpms.pluck(:name)).any?
         end
       end
 

data/app/models/katello/authorization/content_view_version.rb

@@ -10,9 +10,32 @@ module Katello
     end
 
     module ClassMethods
+      def with_content_view_scope(scope)
+        joins(:content_view).merge(Katello::ContentView.send(scope))
+      end
+
       def readable
-        view_ids = ::Katello::ContentView.readable.collect { |v| v.id }
-        joins(:content_view).where("#{Katello::ContentView.table_name}.id" => view_ids)
+        with_content_view_scope(:readable)
+      end
+
+      def exportable
+        with_content_view_scope(:exportable)
+      end
+
+      def editable
+        with_content_view_scope(:editable)
+      end
+
+      def publishable
+        with_content_view_scope(:publishable)
+      end
+
+      def deletable
+        with_content_view_scope(:deletable)
+      end
+
+      def promotable_or_removable
+        with_content_view_scope(:promotable_or_removable)
       end
     end
   end

data/app/models/katello/authorization/content_view_version_export_history.rb

@@ -4,7 +4,7 @@ module Katello
 
     module ClassMethods
       def readable
-        where(:content_view_version_id => Katello::ContentViewVersion.readable)
+        joins(:content_view_version).merge(Katello::ContentViewVersion.readable)
       end
     end
   end

data/app/models/katello/authorization/organization.rb

@@ -12,6 +12,14 @@ module Katello
       authorized?(:import_manifest)
     end
 
+    def can_import_library_content?
+      authorized?(:import_library_content)
+    end
+
+    def can_export_library_content?
+      authorized?(:export_library_content)
+    end
+
     def readable_promotion_paths
       permissible_promotion_paths(KTEnvironment.readable)
     end

data/app/models/katello/concerns/operatingsystem_extensions.rb

@@ -17,6 +17,8 @@ module Katello
           TemplateKind.all.each do |kind|
             if name == ::Operatingsystem::REDHAT_ATOMIC_HOST_OS && kind.name == "provision"
               provisioning_template_name = Setting["katello_default_atomic_provision"]
+            elsif kind.name == 'registration'
+              provisioning_template_name = 'Linux registration default'
             else
               provisioning_template_name = Setting["katello_default_#{kind.name}"]
             end

data/app/models/katello/concerns/pulp_database_unit.rb

@@ -91,6 +91,13 @@ module Katello
         service_class.pulp_units_batch_all(pulp_ids).each do |units|
           units.each do |unit|
             unit = unit.with_indifferent_access
+            if content_type == 'rpm' && repository
+              rpms_to_disassociate = ::Katello::Rpm.where(name: unit[:name], version: unit[:version], release: unit[:release],
+                                                          epoch: unit[:epoch], arch: unit[:arch]).select(:id)
+              if rpms_to_disassociate.any?
+                ::Katello::RepositoryRpm.where(rpm_id: rpms_to_disassociate, repository_id: repository.id).destroy_all
+              end
+            end
             model = Katello::Util::Support.active_record_retry do
               self.where(:pulp_id => unit[service_class.unit_identifier]).first_or_create
             end
@@ -211,6 +218,18 @@ module Katello
       def db_values(new_ids, pulp_id_href_map, repository)
         new_ids.map { |unit_id| [unit_id.to_i, pulp_id_href_map.dig(unit_id), repository.id.to_i, Time.now.utc.to_s(:db), Time.now.utc.to_s(:db)].compact }
       end
+
+      def unmigrated_content
+        self.where(migrated_pulp3_href: nil, ignore_missing_from_migration: false)
+      end
+
+      def missing_migrated_content #missing or corrupted content that could not be migrated
+        self.where(migrated_pulp3_href: nil, missing_from_migration: true, ignore_missing_from_migration: false)
+      end
+
+      def ignored_missing_migrated_content
+        self.where(migrated_pulp3_href: nil, missing_from_migration: true, ignore_missing_from_migration: true)
+      end
     end
   end
 end

data/app/models/katello/concerns/redhat_extensions.rb

@@ -44,8 +44,8 @@ module Katello
 
       def kickstart_repos(host)
         distros = distribution_repositories(host).where(distribution_bootable: true)
-        if distros && host.content_source
-          distros.map { |distro| distro.to_hash(host.content_source) }
+        if distros && host&.content_facet&.content_source
+          distros.map { |distro| distro.to_hash(host.content_facet.content_source) }
         else
           []
         end

data/app/models/katello/concerns/smart_proxy_extensions.rb

@@ -97,7 +97,7 @@ module Katello
         end
 
         def self.sync_needed?(environment)
-          unscoped.with_environment(environment).any?
+          Setting[:foreman_proxy_content_auto_sync] && unscoped.with_environment(environment).any?
         end
       end
 
@@ -107,9 +107,9 @@ module Katello
 
       def update_puppet_path
         if has_feature?(PULP_FEATURE)
-          path = ProxyAPI::Pulp.new(:url => self.url).capsule_puppet_path['puppet_content_dir']
+          path = ::ProxyAPI::Pulp.new(:url => self.url).capsule_puppet_path['puppet_content_dir']
         elsif has_feature?(PULP_NODE_FEATURE)
-          path = ProxyAPI::PulpNode.new(:url => self.url).capsule_puppet_path['puppet_content_dir']
+          path = ::ProxyAPI::PulpNode.new(:url => self.url).capsule_puppet_path['puppet_content_dir']
         end
         self.update_attribute(:puppet_path, path || '') if persisted?
         path
@@ -130,7 +130,7 @@ module Katello
           config.host = uri.host
           config.scheme = uri.scheme
           pulp3_ssl_configuration(config)
-          config.debugging = true
+          config.debugging = false
           config.logger = ::Foreman::Logging.logger('katello/pulp_rest')
           config.username = self.setting(PULP3_FEATURE, 'username')
           config.password = self.setting(PULP3_FEATURE, 'password')
@@ -179,7 +179,9 @@ module Katello
       end
 
       def fix_pulp3_capabilities(type)
-        if missing_pulp3_capabilities? && !pulp2_preferred_for_type?(type)
+        repository_type_obj = type.is_a?(String) ||  type.is_a?(Symbol) ? Katello::RepositoryTypeManager.repository_types[type] : type
+
+        if missing_pulp3_capabilities? && repository_type_obj.pulp3_plugin && !pulp2_preferred_for_type?(repository_type_obj.id)
           self.refresh
           if self.capabilities(::SmartProxy::PULP3_FEATURE).empty?
             fail Katello::Errors::PulpcoreMissingCapabilities

data/app/models/katello/content_migration_progress.rb

@@ -0,0 +1,4 @@
+module Katello
+  class ContentMigrationProgress < Katello::Model
+  end
+end

data/app/models/katello/content_view.rb

@@ -8,6 +8,7 @@ module Katello
     include ForemanTasks::Concerns::ActionSubject
 
     CONTENT_DIR = "content_views".freeze
+    IMPORT_LIBRARY = "Import-Library".freeze
 
     belongs_to :organization, :inverse_of => :content_views, :class_name => "::Organization"
 
@@ -102,6 +103,10 @@ module Katello
       name
     end
 
+    def library_import?
+      name == IMPORT_LIBRARY
+    end
+
     def content_host_count
       hosts.count
     end

data/app/models/katello/content_view_history.rb

@@ -32,7 +32,8 @@ module Katello
       publish: 1,
       promotion: 2,
       removal: 3,
-      export: 4
+      export: 4,
+      importing: 5
     }
 
     def content_view

data/app/models/katello/content_view_package_filter.rb

@@ -43,7 +43,7 @@ module Katello
     end
 
     def self.generate_rpm_clauses(package_filenames = [])
-      { 'filename' => { "$in" => package_filenames } } unless package_filenames.empty?
+      { 'filename' => { "$in" => package_filenames.sort } } unless package_filenames.empty?
     end
 
     def applicable_rpms

data/app/models/katello/content_view_version_export_history.rb

@@ -5,7 +5,7 @@ module Katello
     belongs_to :content_view_version, :class_name => "Katello::ContentViewVersion", :inverse_of => :export_histories
     validates_lengths_from_database
     validates :content_view_version_id, :presence => true
-    validates :destination_server, :presence => true, :uniqueness => { :scope => [:content_view_version_id, :destination_server, :path] }
+    validates :destination_server, :uniqueness => { :scope => [:content_view_version_id, :destination_server, :path] }
     validates :metadata, :presence => true
     serialize :metadata, Hash
 
@@ -20,5 +20,10 @@ module Katello
     scoped_search :on => :content_view_id, :relation => :content_view_version, :validator => ScopedSearch::Validators::INTEGER, :only_explicit => true
     scoped_search :on => :content_view_version_id, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER
     scoped_search :on => :id, :only_explicit => true, :validator => ScopedSearch::Validators::INTEGER
+
+    def self.latest(content_view, destination_server: nil)
+      where(content_view_version: content_view.versions,
+            destination_server: destination_server).order(:created_at).last
+    end
   end
 end

data/app/models/katello/file_unit.rb

@@ -13,6 +13,10 @@ module Katello
       order(:name)
     end
 
+    def filename
+      path
+    end
+
     def self.total_for_repositories(repos)
       self.in_repositories(repos).count
     end

data/app/models/katello/host/content_facet.rb

@@ -31,43 +31,21 @@ module Katello
       validates :host, :presence => true, :allow_blank => false
       validates_with Validators::ContentViewEnvironmentValidator
 
-      def bindable_types
-        [
-          {
-            type: Repository::DEB_TYPE,
-            matcher: '/pulp/deb/',
-            paths: []
-          },
-          {
-            type: Repository::YUM_TYPE,
-            matcher: '/pulp/repos/',
-            paths: []
-          }
-        ]
-      end
-
       def update_repositories_by_paths(paths)
-        bindable_paths = bindable_types
+        prefixes = %w(/pulp/deb/ /pulp/repos/ /pulp/content/)
         relative_paths = []
 
-        # paths == ["/pulp/repos/Default_Organization/Library/custom/Test_product/test2",
-        # "/pulp/repos/Default_Organization/Library/custom/Test_product/My_repo"]
-        paths.each do |absolute_path|
-          bindable_paths.each do |supported|
-            relative_path = absolute_path.gsub(supported[:matcher], '') # remove e.g. '/pulp/repos/' from beginning of string
-            relative_paths << relative_path unless relative_path == absolute_path
-            if absolute_path.starts_with?(supported[:matcher])
-              supported[:paths] << relative_path
-              break
-            end
+        # paths == ["/pulp/repos/Default_Organization/Library/custom/Test_product/test2"]
+        paths.each do |path|
+          if (prefix = prefixes.find { |pre| path.start_with?(pre) })
+            relative_paths << path.gsub(prefix, '')
+          else
+            Rails.logger.warn("System #{self.host.name} (#{self.host.id}) requested binding to repo with unknown prefix. #{path}")
           end
         end
 
-        repos = bindable_paths.flat_map do |supported|
-          repos = Repository.joins(:root).where(RootRepository.table_name => {content_type: supported[:type]}, relative_path: supported[:paths])
-          relative_paths -= repos.pluck(:relative_path) # remove relative paths that match our repos
-          repos
-        end
+        repos = Repository.where(relative_path: relative_paths)
+        relative_paths -= repos.pluck(:relative_path) # remove relative paths that match our repos
 
         # Any leftover relative paths do not match the repos we've just retrieved from the db,
         # so we should log warnings about them.