katello 4.3.0.rc4 → 4.4.0.rc2

data/app/services/katello/content_unit_indexer.rb

@@ -0,0 +1,166 @@
+module Katello
+  class ContentUnitIndexer
+    def initialize(content_type:, repository: nil, pulp_content_ids: nil, optimized: true)
+      @content_type = content_type
+      @model_class = content_type.model_class
+      @service_class = SmartProxy.pulp_primary!.content_service(content_type)
+      @repository = repository
+      @content_type = content_type
+      @pulp_content_ids = pulp_content_ids
+      @optimized = optimized
+    end
+
+    def remove_duplicates(unit)
+      #when we are uploading units, we need to remove any duplicates from our indexed data
+      if @content_type.label == 'rpm' && @repository && @pulp_content_ids
+        rpms_to_disassociate = ::Katello::Rpm.where(name: unit[:name], version: unit[:version], release: unit[:release],
+                                                    epoch: unit[:epoch], arch: unit[:arch]).select(:id)
+        if rpms_to_disassociate.any?
+          ::Katello::RepositoryRpm.where(rpm_id: rpms_to_disassociate, repository_id: @repository.id).destroy_all
+        end
+      end
+    end
+
+    def import_all
+      association_tracker = RepoAssociationTracker.new(@content_type, @service_class, @repository)
+
+      units_from_pulp.each do |units|
+        units.each do |unit|
+          association_tracker.push(unit)
+          remove_duplicates(unit)
+        end
+
+        unless fetch_only_ids
+          to_insert = units.map do |unit|
+            if @content_type.generic?
+              @service_class.generate_model_row(unit, @content_type)
+            else
+              @service_class.generate_model_row(unit)
+            end
+          end
+
+          next if to_insert.empty?
+          insert_timestamps(to_insert)
+          if @content_type.mutable
+            @model_class.upsert_all(to_insert, unique_by: :pulp_id)
+          else
+            @model_class.insert_all(to_insert, unique_by: :pulp_id)
+          end
+        end
+
+        import_associations(units) if @repository
+      end
+
+      if @model_class.many_repository_associations && @repository
+        sync_repository_associations(association_tracker)
+      end
+    end
+
+    def import_associations(units)
+      pulp_id_to_id = self.class.pulp_id_to_id_map(@content_type, units.map { |unit| unit[@service_class.unit_identifier] })
+      @service_class.insert_child_associations(units, pulp_id_to_id) if @service_class.respond_to?(:insert_child_associations)
+    end
+
+    def units_from_pulp(&block)
+      if @pulp_content_ids
+        @service_class.pulp_units_batch_all(@pulp_content_ids, &block)
+      elsif @repository
+        @service_class.pulp_units_batch_for_repo(@repository, fetch_identifiers: fetch_only_ids, content_type: @content_type, &block)
+      end
+    end
+
+    def self.pulp_id_to_id_map(content_type, pulp_ids)
+      map = {}
+      content_type.model_class.with_pulp_id(pulp_ids).select(:id, :pulp_id).each do |model|
+        map[model.pulp_id] = model.id
+      end
+      map
+    end
+
+    class RepoAssociationTracker
+      def initialize(content_type, service_class, repository)
+        @values = {}
+        @content_type = content_type
+        @repository = repository
+        @service_class = service_class
+      end
+
+      def unit_ids
+        db_values.map { |row| row[@content_type.model_class.unit_id_field] }
+      end
+
+      #pulp_href is only provided if we're storing a different 'pulp_id' on the repo association
+      def push(unit)
+        if @service_class.backend_unit_identifier
+          pulp_href = unit.dig(@service_class.backend_unit_identifier)
+        else
+          pulp_href = nil
+        end
+        unit_id = unit[@service_class.unit_identifier]
+        @values[unit_id] = pulp_href
+      end
+
+      def db_values
+        return @final_values if @final_values
+        @final_value = []
+
+        @final_values = ::Katello::ContentUnitIndexer.pulp_id_to_id_map(@content_type, @values.keys).map do |pulp_id, katello_id|
+          #:repository_id => X, :erratum_id => y
+          row = {:repository_id => @repository.id, @content_type.model_class.unit_id_field => katello_id}
+          row[pulp_href_association_name] = @values[pulp_id] if pulp_href_association_name
+          row
+        end
+        ContentUnitIndexer.insert_timestamps(@content_type.model_class, @final_values)
+        @final_values
+      end
+
+      def pulp_href_association_name
+        'erratum_pulp3_href' if @content_type.label == 'erratum'
+      end
+    end
+
+    def insert_timestamps(units)
+      self.class.insert_timestamps(@model_class, units)
+    end
+
+    def self.insert_timestamps(model_class, units)
+      dates = model_class.columns.map(&:name).include?("created_at")
+      return units unless dates
+      units.each do |row|
+        row[:created_at] = DateTime.now
+        row[:updated_at] = DateTime.now
+      end
+      units
+    end
+
+    def fetch_only_ids
+      @optimized && @repository &&
+        !@repository.content_view.default? &&
+        !@repository.repository_type.unique_content_per_repo &&
+        @service_class.supports_id_fetch?
+    end
+
+    def sync_repository_associations(assocication_tracker, additive: false)
+      unless additive
+        ActiveRecord::Base.connection.uncached do
+          @model_class.repository_association_class.where(repository_id: @repository.id).where.
+            not(@model_class.unit_id_field => assocication_tracker.unit_ids).delete_all
+        end
+      end
+      return if assocication_tracker.db_values.empty?
+      @model_class.repository_association_class.upsert_all(assocication_tracker.db_values, :unique_by => association_class_uniqiness_attributes)
+    end
+
+    def association_class_uniqiness_attributes
+      columns = [@model_class.unit_id_field, 'repository_id']
+      found = ActiveRecord::Base.connection.indexes(@model_class.repository_association_class.table_name).find do |index|
+        index.columns.sort == columns.sort
+      end
+      if found
+        found.columns
+      else
+        fail "Unable to find unique index for #{columns} on table #{self.repository_association_class.table_name}"
+      end
+    end
+  end
+end

data/app/services/katello/organization_creator.rb

@@ -1,5 +1,11 @@
 module Katello
   class OrganizationCreator
+    DEFAULT_CONTENT_VIEW_NAME = 'Default Organization View'.freeze
+    DEFAULT_CONTENT_VIEW_LABEL = 'Default_Organization_View'.freeze
+
+    DEFAULT_LIFECYCLE_ENV_NAME = 'Library'.freeze
+    DEFAULT_LIFECYCLE_ENV_LABEL = 'Library'.freeze
+
     attr_reader :library_view, :library_environment, :library_cvv, :content_view_environment, :anonymous_provider, :redhat_provider
 
     def self.seed_all_organizations!
@@ -16,7 +22,7 @@ module Katello
       User.as_anonymous_admin do
         Organization.not_created_in_katello.each do |org|
           creator = self.new(org)
-          creator.create!
+          creator.create!(raise_validation_errors: false)
         end
       end
     end
@@ -28,7 +34,9 @@ module Katello
     def seed!
       ActiveRecord::Base.transaction do
         @organization.setup_label_from_name
-        @organization.save!
+
+        # existing validation errors are not resolvable here, so don't validatate
+        @organization.save(validate: false)
 
         create_library_environment
         create_library_view
@@ -40,14 +48,19 @@ module Katello
       end
     end
 
-    def create!
+    def create!(raise_validation_errors: true)
       ActiveRecord::Base.transaction do
         seed!
 
         create_backend_objects!
 
         @organization.created_in_katello = true
-        @organization.save!
+
+        begin
+          @organization.save!
+        rescue => e
+          raise e if raise_validation_errors
+        end
       end
     end
 
@@ -80,8 +93,8 @@ module Katello
 
     def create_library_environment
       @library_environment = Katello::KTEnvironment.where(
-        :name => "Library",
-        :label => "Library",
+        :name => DEFAULT_LIFECYCLE_ENV_NAME,
+        :label => DEFAULT_LIFECYCLE_ENV_LABEL,
         :library => true,
         :organization => @organization
       ).first_or_create!
@@ -90,7 +103,8 @@ module Katello
     def create_library_view
       @library_view = Katello::ContentView.where(
         default: true,
-        name: 'Default Organization View',
+        name: DEFAULT_CONTENT_VIEW_NAME,
+        label: DEFAULT_CONTENT_VIEW_LABEL,
         organization: @organization
       ).first_or_create!
     end
@@ -121,7 +135,8 @@ module Katello
     def create_cdn_configuration
       @cdn_configuration = Katello::CdnConfiguration.where(
         organization: @organization,
-        url: ::Katello::Resources::CDN::CdnResource.redhat_cdn_url
+        url: ::Katello::Resources::CDN::CdnResource.redhat_cdn_url,
+        type: ::Katello::CdnConfiguration::CDN_TYPE
       ).first_or_create!
     end
   end

data/app/services/katello/pulp/repository/docker.rb

@@ -18,7 +18,7 @@ module Katello
           config = {
             feed: root.url,
             upstream_name: root.docker_upstream_name,
-            tags: root.docker_tags_whitelist,
+            tags: root.include_tags,
             enable_v1: false
           }
           importer_class.new(config.merge(primary_importer_connection_options))

data/app/services/katello/pulp/repository/yum.rb

@@ -105,37 +105,6 @@ module Katello
           config
         end
 
-        def copy_contents(destination_repo, filters: nil,
-                                            solve_dependencies: false,
-                                            rpm_filenames: [])
-          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(filters&.yum(false), rpm_filenames)
-          tasks = []
-          if rpm_copy_clauses
-            override_config = build_override_config(destination_repo,
-                                                    filters: filters,
-                                                    solve_dependencies: solve_dependencies)
-
-            tasks << smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id,
-                      rpm_copy_clauses.merge(:override_config => override_config))
-          end
-
-          if rpm_remove_clauses
-            tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
-                                                                         type_ids: [::Katello::Pulp::Rpm::CONTENT_TYPE],
-                                                                          filters: {unit: rpm_remove_clauses})
-          end
-
-          tasks.concat(copy_module_contents(destination_repo,
-                                              filters: filters,
-                                              solve_dependencies: solve_dependencies))
-
-          [:srpm, :errata, :package_group, :package_environment,
-           :yum_repo_metadata_file, :distribution, :module_default, :drpm].each do |type|
-            tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
-          end
-          tasks
-        end
-
         def import_distribution_data
           distribution = smart_proxy.pulp_api.extensions.repository.distributions(repo.pulp_id).first
           if distribution
@@ -230,29 +199,6 @@ module Katello
           end
           [copy_clauses, remove_clauses]
         end
-
-        def generate_copy_clauses(filters, rpm_filenames)
-          if rpm_filenames&.any?
-            copy_clauses = {filters: {unit: { 'filename' => { '$in' => rpm_filenames } }}}
-            remove_clauses = nil
-          elsif filters&.any?
-            clause_gen = ::Katello::Util::PackageClauseGenerator.new(repo, filters.yum(false))
-            clause_gen.generate
-
-            copy = clause_gen.copy_clause
-            copy_clauses = {filters: {unit: copy }} if copy
-
-            remove = clause_gen.remove_clause
-            remove_clauses = {filters: {unit: remove}} if remove
-          else
-            non_modular_rpms = ::Katello::Rpm.in_repositories(repo).non_modular.pluck(:filename)
-            copy_clauses = non_modular_rpms.blank? ? nil : {filters: {unit: ContentViewPackageFilter.generate_rpm_clauses(non_modular_rpms)}}
-            remove_clauses = nil
-          end
-
-          copy_clauses&.merge!(fields: ::Katello::Pulp::Rpm::PULP_SELECT_FIELDS)
-          [copy_clauses, remove_clauses]
-        end
       end
     end
   end

data/app/services/katello/pulp/repository.rb

@@ -278,12 +278,6 @@ module Katello
         Katello::RepositoryTypeManager.find(repo.content_type)
       end
 
-      def index_content_units
-        content_unit_types.each do |content_type|
-          content_type.pulp2_service_class
-        end
-      end
-
       def proxy_options
         if repo.root.http_proxy_policy == RootRepository::NO_DEFAULT_HTTP_PROXY
           return { proxy_host: '' }

data/app/services/katello/pulp3/ansible_collection.rb

@@ -18,17 +18,33 @@ module Katello
         fail NotImplementedError
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['checksum'] = backend_data['sha256']
-        custom_json['namespace'] = backend_data['namespace']
-        custom_json['version'] = backend_data['version']
-        custom_json['name'] = backend_data['name']
-        custom_json['description'] = backend_data['description']
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          pulp_id: unit['pulp_href'],
+          checksum: unit['sha256'],
+          namespace: unit['namespace'],
+          version: unit['version'],
+          name: unit['name'],
+          description: unit['description']
+        }
+      end
+
+      def self.insert_child_associations(units, pulp_id_to_id)
+        tag_names = units.map { |unit| unit['tags'].map { |tag| tag[:name] } }.flatten
+        tag_rows = tag_names.map { |name| {name: name } }
+        Katello::AnsibleTag.insert_all(tag_rows, unique_by: [:name]) if tag_rows.any?
+
+        collection_tag_rows = []
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit['pulp_href']]
+          #delete old tags
+          unit_tags = unit['tags'].map { |tag| tag[:name] }
+          Katello::AnsibleCollectionTag.where(:ansible_collection_id => katello_id).where.not(:ansible_tag_id => Katello::AnsibleTag.where(:name => unit_tags)).delete_all
+          collection_tag_rows += Katello::AnsibleTag.where(:name => unit_tags).pluck(:id).map { |tag_id| {ansible_collection_id: katello_id, ansible_tag_id: tag_id} }
+        end
 
-        tags = backend_data['tags'].map { |tag| Katello::AnsibleTag.find_or_create_by(name: tag['name']) }
-        model.tags = tags
+        collection_tag_rows.flatten!
+        Katello::AnsibleCollectionTag.insert_all(collection_tag_rows, unique_by: [:ansible_collection_id, :ansible_tag_id])
       end
     end
   end

data/app/services/katello/pulp3/api/apt.rb

@@ -4,6 +4,13 @@ module Katello
   module Pulp3
     module Api
       class Apt < Core
+        def publication_verbatim_class
+          PulpDebClient::DebVerbatimPublication
+        end
+
+        def publications_verbatim_api
+          PulpDebClient::PublicationsVerbatimApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/api/yum.rb

@@ -36,6 +36,10 @@ module Katello
           PulpRpmClient::ContentPackageenvironmentsApi.new(api_client)
         end
 
+        def content_modulemd_defaults_api
+          PulpRpmClient::ContentModulemdDefaultsApi.new(api_client)
+        end
+
         def content_repo_metadata_files_api
           PulpRpmClient::ContentRepoMetadataFilesApi.new(api_client)
         end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -121,11 +121,20 @@ module Katello
                         }
           end
 
+          fail _("Content View label not provided.") if metadata[:label].blank?
+
           cv = ::Katello::ContentView.find_by(label: metadata[:label],
-                                              organization: organization,
-                                              import_only: true)
+                                              organization: organization)
           if cv.blank?
             ::Katello::ContentView.create!(metadata.merge(organization: organization, import_only: true))
+          elsif !cv.import_only?
+            msg = _("Unable to import in to Content View specified in the metadata - '%{name}'. "\
+                     "The 'import_only' attribute for the content view is set to false. "\
+                     "To mark this Content View as importable, have your system administrator"\
+                     " run the following command on the server. "\
+                        % { name: cv.name })
+            command = "foreman-rake katello:set_content_view_import_only ID=#{cv.id}"
+            fail msg + "\n" + command
           else
             cv.update!(description: cv_metadata[:description]) if cv.description != metadata[:description]
             cv

data/app/services/katello/pulp3/content_view_version/metadata_generator.rb

@@ -43,7 +43,7 @@ module Katello
 
         def generate_repository_metadata(repo)
           repo.slice(:name, :label, :description, :arch, :content_type, :unprotected,
-                     :checksum_type, :os_versions, :major, :minor).
+                     :checksum_type, :os_versions, :major, :minor, :download_policy, :mirroring_policy).
             merge(product: generate_product_metadata(repo.product),
                   gpg_key: generate_gpg_metadata(repo.gpg_key),
                   content: generate_content_metadata(repo.content),

data/app/services/katello/pulp3/deb.rb

@@ -23,15 +23,16 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['checksum'] = backend_data['sha256']
-        custom_json['filename'] = backend_data['relative_path']
-        custom_json['name'] = backend_data['package']
-        custom_json['version'] = backend_data['version']
-        custom_json['description'] = backend_data['description']
-        custom_json['architecture'] = backend_data['architecture']
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          pulp_id: unit[unit_identifier],
+          checksum: unit[:sha256],
+          filename: unit[:relative_path],
+          name: unit[:package],
+          version: unit[:version],
+          description: unit[:description]&.truncate(255),
+          architecture: unit[:architecture]
+        }
       end
     end
   end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -20,11 +20,12 @@ module Katello
         self.content_api.list(page_opts)
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['schema_version'], = backend_data['schema_version']
-        custom_json['digest'], = backend_data['digest']
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          schema_version: unit['schema_version'],
+          digest: unit['digest'],
+          pulp_id: unit[unit_identifier]
+        }
       end
     end
   end

data/app/services/katello/pulp3/docker_manifest_list.rb

@@ -18,12 +18,29 @@ module Katello
         self.content_api.list(page_opts)
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['schema_version'], = backend_data['schema_version']
-        custom_json['digest'], = backend_data['digest']
-        custom_json['docker_manifests'] = ::Katello::DockerManifest.where(:pulp_id => backend_data[:listed_manifests])
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          schema_version: unit['schema_version'],
+          digest: unit['digest'],
+          pulp_id: unit[unit_identifier]
+        }
+      end
+
+      def self.insert_child_associations(units, pulp_id_to_id)
+        manifest_list_manifests = []
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit[unit_identifier]]
+          manifest_ids = ::Katello::DockerManifest.where(:pulp_id => unit[:listed_manifests]).pluck(:id)
+          manifest_list_manifests += manifest_ids.map do |manifest_id|
+            {
+              docker_manifest_list_id: katello_id,
+              docker_manifest_id: manifest_id
+            }
+          end
+        end
+
+        manifest_list_manifests.flatten!
+        Katello::DockerManifestListManifest.insert_all(manifest_list_manifests, unique_by: [:docker_manifest_list_id, :docker_manifest_id]) if manifest_list_manifests.any?
       end
     end
   end

data/app/services/katello/pulp3/docker_tag.rb

@@ -14,15 +14,24 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def update_model(model)
-        custom_json = {}
-        taggable = ::Katello::DockerManifest.find_by(:pulp_id => backend_data['tagged_manifest'])
+      def self.generate_model_row(unit)
+        row = {
+          pulp_id: unit[unit_identifier],
+          name: unit['name']
+        }
+
+        taggable = ::Katello::DockerManifest.find_by(:pulp_id => unit['tagged_manifest'])
+        taggable_type = ::Katello::DockerManifest.name
         if taggable.nil?
-          taggable = ::Katello::DockerManifestList.find_by(:pulp_id => backend_data['tagged_manifest'])
+          taggable = ::Katello::DockerManifestList.find_by(:pulp_id => unit['tagged_manifest'])
+          taggable_type = ::Katello::DockerManifestList.name
+        end
+
+        if taggable
+          row[:docker_taggable_id] = taggable.id
+          row[:docker_taggable_type] = taggable_type
         end
-        custom_json['docker_taggable'] = taggable
-        custom_json['name'] = backend_data['name']
-        model.update!(custom_json)
+        row
       end
     end
   end