katello 4.3.0 → 4.4.0.rc1

data/app/models/katello/root_repository.rb

@@ -4,6 +4,8 @@ module Katello
     audited :except => [:content_id]
     serialize :ignorable_content
     serialize :docker_tags_whitelist
+    serialize :include_tags
+    serialize :exclude_tags
     serialize :os_versions
 
     include Ext::LabelFromName
@@ -21,7 +23,7 @@ module Katello
     SUBSCRIBABLE_TYPES = [Repository::YUM_TYPE, Repository::OSTREE_TYPE, Repository::DEB_TYPE].freeze
 
     CONTENT_ATTRIBUTE_RESTRICTIONS = {
-      :download_policy => [Repository::YUM_TYPE, Repository::DEB_TYPE]
+      :download_policy => [Repository::YUM_TYPE, Repository::DEB_TYPE, Repository::DOCKER_TYPE]
     }.freeze
 
     NO_DEFAULT_HTTP_PROXY = 'none'.freeze
@@ -54,7 +56,7 @@ module Katello
     has_many :repositories, :class_name => "Katello::Repository", :foreign_key => :root_id,
                           :inverse_of => :root, :dependent => :destroy
 
-    has_many :repository_references, :class_name => 'Katello::Pulp3::RepositoryReference', :foreign_key => :root_repository_id,
+    has_many :repository_references, :class_name => 'Katello::Pulp3::RepositoryReference',
              :dependent => :destroy, :inverse_of => :root_repository
 
     validates_lengths_from_database :except => [:label]
@@ -72,7 +74,8 @@ module Katello
     validate :ensure_valid_collection_attributes, :if => :ansible_collection?
     validate :ensure_valid_auth_url_token, :if => :ansible_collection?
     validate :ensure_valid_ignorable_content
-    validate :ensure_valid_docker_tags_whitelist
+    validate :ensure_valid_include_tags
+    validate :ensure_valid_exclude_tags
     validate :ensure_valid_os_versions
     validate :ensure_content_attribute_restrictions
     validate :ensure_valid_upstream_authorization
@@ -223,10 +226,17 @@ module Katello
       end
     end
 
-    def ensure_valid_docker_tags_whitelist
-      return if docker_tags_whitelist.blank?
-      unless docker_tags_whitelist.is_a?(Array)
-        errors.add(:docker_tags_whitelist, N_("Invalid value specified for Container Image repositories."))
+    def ensure_valid_include_tags
+      return if include_tags.blank?
+      unless include_tags.is_a?(Array)
+        errors.add(:include_tags, N_("Invalid value specified for Container Image repositories."))
+      end
+    end
+
+    def ensure_valid_exclude_tags
+      return if exclude_tags.blank?
+      unless exclude_tags.is_a?(Array)
+        errors.add(:exclude_tags, N_("Invalid value specified for Container Image repositories."))
       end
     end
 
@@ -315,6 +325,11 @@ module Katello
       Katello::Content.find_by(:cp_content_id => self.content_id, :organization_id => self.product.organization_id)
     end
 
+    # For API support during deprecation period.
+    def docker_tags_whitelist
+      include_tags
+    end
+
     def docker?
       self.content_type == Repository::DOCKER_TYPE
     end
@@ -359,7 +374,7 @@ module Katello
       changeable_attributes = %w(url unprotected checksum_type docker_upstream_name download_policy mirroring_policy verify_ssl_on_sync
                                  upstream_username upstream_password ignorable_content
                                  ssl_ca_cert_id ssl_client_cert_id ssl_client_key_id http_proxy_policy http_proxy_id download_concurrency)
-      changeable_attributes += %w(name container_repository_name docker_tags_whitelist) if docker?
+      changeable_attributes += %w(name container_repository_name include_tags exclude_tags) if docker?
       changeable_attributes += %w(deb_releases deb_components deb_architectures gpg_key_id) if deb?
       changeable_attributes += %w(ansible_collection_requirements ansible_collection_auth_url ansible_collection_auth_token) if ansible_collection?
       changeable_attributes.any? { |key| previous_changes.key?(key) }
@@ -396,11 +411,7 @@ module Katello
 
     def format_arches
       if content_type == ::Katello::Repository::DEB_TYPE
-        # FIXME: This should be set to self.deb_architectures but it needs to have the
-        # subscription-manager PR https://github.com/candlepin/subscription-manager/pull/2213
-        # merged. Otherwise subscription-manager returns _NO_ debian repository as described in
-        # https://community.theforeman.org/t/katello-3-16-1-1-el7-subscription-manager-doesnt-create-rhsm-repos-for-ubuntu/20928/38
-        nil
+        self.deb_architectures
       else
         self.arch == "noarch" ? nil : self.arch
       end

data/app/models/katello/rpm.rb

@@ -8,6 +8,7 @@ module Katello
     has_many :content_facets, :through => :content_facet_applicable_rpms, :class_name => "Katello::Host::ContentFacet"
     has_many :module_stream_rpms, class_name: "Katello::ModuleStreamRpm", inverse_of: :rpm, dependent: :destroy
     has_many :module_streams, :through => :module_stream_rpms
+    scoped_search :on => :id, :complete_value => true
     scoped_search :on => :name, :complete_value => true
     scoped_search :on => :version, :complete_value => true, :ext_method => :scoped_search_version
     scoped_search :on => :release, :complete_value => true, :ext_method => :scoped_search_release
@@ -19,8 +20,6 @@ module Katello
     scoped_search :on => :modular, :complete_value => true, :only_explicit => true
     scoped_search :on => :checksum
 
-    before_save lambda { |rpm| rpm.summary = rpm.summary.truncate(255) unless rpm.summary.blank? }
-
     scope :modular, -> { where(modular: true) }
     scope :non_modular, -> { where(modular: false) }
 
@@ -215,6 +214,13 @@ module Katello
         where("#{Katello::Host::ContentFacet.table_name}.host_id" => hosts).distinct
     end
 
+    # Return RPMs that are not installed on a host, but could be installed
+    # the word 'installable' has a different meaning here than elsewhere
+    def self.yum_installable_for_host(host)
+      repos = host.content_facet.bound_repositories.pluck(:id)
+      Katello::Rpm.in_repositories(repos).where.not(name: host.installed_packages.pluck(:name)).order(:name)
+    end
+
     def self.latest(relation)
       # There are several different ways to implement this:
 

data/app/models/setting/content.rb

@@ -75,8 +75,8 @@ class Setting::Content < Setting
                20, N_('Accept action timeout')),
       self.set('content_action_finish_timeout', N_("Time in seconds to wait for a Host to finish a remote action"),
                3600, N_('Finish action timeout')),
-      self.set('content_disconnected', N_("A server operating in disconnected mode does not communicate with the Red Hat CDN."),
-               false, N_('Disconnected mode')),
+      self.set('subscription_connection_enabled', N_("Can communicate with the Red Hat Portal for subscriptions."),
+               true, N_('Subscription connection enabled')),
       self.set('errata_status_installable', N_("Calculate errata host status based only on errata in a Host's Content View and Lifecycle Environment"),
                false, N_('Installable errata from Content View')),
       self.set('restrict_composite_view', N_("If this is enabled, a composite content view may not be published or "\
@@ -88,6 +88,7 @@ class Setting::Content < Setting
                100, N_('Batch size to sync repositories in.')),
       self.set('foreman_proxy_content_auto_sync', N_("Whether or not to auto sync the Smart Proxies after a Content View promotion."),
                true, N_('Sync Smart Proxies after Content View promotion')),
+      self.set('download_rate_limit', N_("Maximum download rate when syncing a repository (requests per second). Use 0 for no limit."), 0, N_("Download rate limit")),
       self.set('default_download_policy', N_("Default download policy for custom repositories (either 'immediate' or 'on_demand')"), "immediate",
                N_('Default Custom Repository download policy'), nil, :collection => download_policies),
       self.set('default_redhat_download_policy', N_("Default download policy for enabled Red Hat repositories (either 'immediate' or 'on_demand')"), "on_demand",
@@ -148,7 +149,9 @@ class Setting::Content < Setting
                N_('Pulp bulk load size')),
       self.set('upload_profiles_without_dynflow', N_('Allow Katello to update host installed packages, enabled repos, and module inventory directly instead of wrapped in Dynflow tasks (try turning off if Puma processes are using too much memory)'), true,
                N_('Upload profiles without Dynflow')),
-      self.set('orphan_protection_time', N_('Time in minutes to consider orphan content as orphaned.'), 1440, N_('Orphaned Content Protection Time'))
+      self.set('orphan_protection_time', N_('Time in minutes to consider orphan content as orphaned.'), 1440, N_('Orphaned Content Protection Time')),
+      self.set('remote_execution_prefer_registered_through_proxy', N_('Prefer using a proxy to which a host is registered when using remote execution'), false,
+               N_('Prefer registered through proxy for remote execution'))
     ]
   end
 

data/app/services/cert/rhsm_client.rb

@@ -10,7 +10,7 @@ module Cert
     end
 
     def uuid
-      drop_cn_prefix_from_subject(@cert.subject.to_s)
+      @uuid ||= @cert.subject.to_a.detect { |name, _, _| name == 'CN' }&.second
     end
 
     private
@@ -26,10 +26,6 @@ module Cert
       end
     end
 
-    def drop_cn_prefix_from_subject(subject_string)
-      subject_string.sub(/\/CN=/i, '')
-    end
-
     def strip_cert(cert)
       cert = cert.to_s.gsub("-----BEGIN CERTIFICATE-----", "").gsub("-----END CERTIFICATE-----", "")
       cert.delete!(' ')

data/app/services/katello/content_unit_indexer.rb

@@ -0,0 +1,166 @@
+module Katello
+  class ContentUnitIndexer
+    def initialize(content_type:, repository: nil, pulp_content_ids: nil, optimized: true)
+      @content_type = content_type
+      @model_class = content_type.model_class
+      @service_class = SmartProxy.pulp_primary!.content_service(content_type)
+      @repository = repository
+      @content_type = content_type
+      @pulp_content_ids = pulp_content_ids
+      @optimized = optimized
+    end
+
+    def remove_duplicates(unit)
+      #when we are uploading units, we need to remove any duplicates from our indexed data
+      if @content_type.label == 'rpm' && @repository && @pulp_content_ids
+        rpms_to_disassociate = ::Katello::Rpm.where(name: unit[:name], version: unit[:version], release: unit[:release],
+                                                    epoch: unit[:epoch], arch: unit[:arch]).select(:id)
+        if rpms_to_disassociate.any?
+          ::Katello::RepositoryRpm.where(rpm_id: rpms_to_disassociate, repository_id: @repository.id).destroy_all
+        end
+      end
+    end
+
+    def import_all
+      association_tracker = RepoAssociationTracker.new(@content_type, @service_class, @repository)
+
+      units_from_pulp.each do |units|
+        units.each do |unit|
+          association_tracker.push(unit)
+          remove_duplicates(unit)
+        end
+
+        unless fetch_only_ids
+          to_insert = units.map do |unit|
+            if @content_type.generic?
+              @service_class.generate_model_row(unit, @content_type)
+            else
+              @service_class.generate_model_row(unit)
+            end
+          end
+
+          next if to_insert.empty?
+          insert_timestamps(to_insert)
+          if @content_type.mutable
+            @model_class.upsert_all(to_insert, unique_by: :pulp_id)
+          else
+            @model_class.insert_all(to_insert, unique_by: :pulp_id)
+          end
+        end
+
+        import_associations(units) if @repository
+      end
+
+      if @model_class.many_repository_associations && @repository
+        sync_repository_associations(association_tracker)
+      end
+    end
+
+    def import_associations(units)
+      pulp_id_to_id = self.class.pulp_id_to_id_map(@content_type, units.map { |unit| unit[@service_class.unit_identifier] })
+      @service_class.insert_child_associations(units, pulp_id_to_id) if @service_class.respond_to?(:insert_child_associations)
+    end
+
+    def units_from_pulp(&block)
+      if @pulp_content_ids
+        @service_class.pulp_units_batch_all(@pulp_content_ids, &block)
+      elsif @repository
+        @service_class.pulp_units_batch_for_repo(@repository, fetch_identifiers: fetch_only_ids, content_type: @content_type, &block)
+      end
+    end
+
+    def self.pulp_id_to_id_map(content_type, pulp_ids)
+      map = {}
+      content_type.model_class.with_pulp_id(pulp_ids).select(:id, :pulp_id).each do |model|
+        map[model.pulp_id] = model.id
+      end
+      map
+    end
+
+    class RepoAssociationTracker
+      def initialize(content_type, service_class, repository)
+        @values = {}
+        @content_type = content_type
+        @repository = repository
+        @service_class = service_class
+      end
+
+      def unit_ids
+        db_values.map { |row| row[@content_type.model_class.unit_id_field] }
+      end
+
+      #pulp_href is only provided if we're storing a different 'pulp_id' on the repo association
+      def push(unit)
+        if @service_class.backend_unit_identifier
+          pulp_href = unit.dig(@service_class.backend_unit_identifier)
+        else
+          pulp_href = nil
+        end
+        unit_id = unit[@service_class.unit_identifier]
+        @values[unit_id] = pulp_href
+      end
+
+      def db_values
+        return @final_values if @final_values
+        @final_value = []
+
+        @final_values = ::Katello::ContentUnitIndexer.pulp_id_to_id_map(@content_type, @values.keys).map do |pulp_id, katello_id|
+          #:repository_id => X, :erratum_id => y
+          row = {:repository_id => @repository.id, @content_type.model_class.unit_id_field => katello_id}
+          row[pulp_href_association_name] = @values[pulp_id] if pulp_href_association_name
+          row
+        end
+        ContentUnitIndexer.insert_timestamps(@content_type.model_class, @final_values)
+        @final_values
+      end
+
+      def pulp_href_association_name
+        'erratum_pulp3_href' if @content_type.label == 'erratum'
+      end
+    end
+
+    def insert_timestamps(units)
+      self.class.insert_timestamps(@model_class, units)
+    end
+
+    def self.insert_timestamps(model_class, units)
+      dates = model_class.columns.map(&:name).include?("created_at")
+      return units unless dates
+      units.each do |row|
+        row[:created_at] = DateTime.now
+        row[:updated_at] = DateTime.now
+      end
+      units
+    end
+
+    def fetch_only_ids
+      @optimized && @repository &&
+        !@repository.content_view.default? &&
+        !@repository.repository_type.unique_content_per_repo &&
+        @service_class.supports_id_fetch?
+    end
+
+    def sync_repository_associations(assocication_tracker, additive: false)
+      unless additive
+        ActiveRecord::Base.connection.uncached do
+          @model_class.repository_association_class.where(repository_id: @repository.id).where.
+            not(@model_class.unit_id_field => assocication_tracker.unit_ids).delete_all
+        end
+      end
+      return if assocication_tracker.db_values.empty?
+      @model_class.repository_association_class.upsert_all(assocication_tracker.db_values, :unique_by => association_class_uniqiness_attributes)
+    end
+
+    def association_class_uniqiness_attributes
+      columns = [@model_class.unit_id_field, 'repository_id']
+      found = ActiveRecord::Base.connection.indexes(@model_class.repository_association_class.table_name).find do |index|
+        index.columns.sort == columns.sort
+      end
+      if found
+        found.columns
+      else
+        fail "Unable to find unique index for #{columns} on table #{self.repository_association_class.table_name}"
+      end
+    end
+  end
+end

data/app/services/katello/organization_creator.rb

@@ -1,5 +1,11 @@
 module Katello
   class OrganizationCreator
+    DEFAULT_CONTENT_VIEW_NAME = 'Default Organization View'.freeze
+    DEFAULT_CONTENT_VIEW_LABEL = 'Default_Organization_View'.freeze
+
+    DEFAULT_LIFECYCLE_ENV_NAME = 'Library'.freeze
+    DEFAULT_LIFECYCLE_ENV_LABEL = 'Library'.freeze
+
     attr_reader :library_view, :library_environment, :library_cvv, :content_view_environment, :anonymous_provider, :redhat_provider
 
     def self.seed_all_organizations!
@@ -87,8 +93,8 @@ module Katello
 
     def create_library_environment
       @library_environment = Katello::KTEnvironment.where(
-        :name => "Library",
-        :label => "Library",
+        :name => DEFAULT_LIFECYCLE_ENV_NAME,
+        :label => DEFAULT_LIFECYCLE_ENV_LABEL,
         :library => true,
         :organization => @organization
       ).first_or_create!
@@ -97,7 +103,8 @@ module Katello
     def create_library_view
       @library_view = Katello::ContentView.where(
         default: true,
-        name: 'Default Organization View',
+        name: DEFAULT_CONTENT_VIEW_NAME,
+        label: DEFAULT_CONTENT_VIEW_LABEL,
         organization: @organization
       ).first_or_create!
     end
@@ -128,7 +135,8 @@ module Katello
     def create_cdn_configuration
       @cdn_configuration = Katello::CdnConfiguration.where(
         organization: @organization,
-        url: ::Katello::Resources::CDN::CdnResource.redhat_cdn_url
+        url: ::Katello::Resources::CDN::CdnResource.redhat_cdn_url,
+        type: ::Katello::CdnConfiguration::CDN_TYPE
       ).first_or_create!
     end
   end

data/app/services/katello/pulp/repository/docker.rb

@@ -18,7 +18,7 @@ module Katello
           config = {
             feed: root.url,
             upstream_name: root.docker_upstream_name,
-            tags: root.docker_tags_whitelist,
+            tags: root.include_tags,
             enable_v1: false
           }
           importer_class.new(config.merge(primary_importer_connection_options))

data/app/services/katello/pulp/repository/yum.rb

@@ -105,37 +105,6 @@ module Katello
           config
         end
 
-        def copy_contents(destination_repo, filters: nil,
-                                            solve_dependencies: false,
-                                            rpm_filenames: [])
-          rpm_copy_clauses, rpm_remove_clauses = generate_copy_clauses(filters&.yum(false), rpm_filenames)
-          tasks = []
-          if rpm_copy_clauses
-            override_config = build_override_config(destination_repo,
-                                                    filters: filters,
-                                                    solve_dependencies: solve_dependencies)
-
-            tasks << smart_proxy.pulp_api.extensions.rpm.copy(repo.pulp_id, destination_repo.pulp_id,
-                      rpm_copy_clauses.merge(:override_config => override_config))
-          end
-
-          if rpm_remove_clauses
-            tasks << smart_proxy.pulp_api.extensions.repository.unassociate_units(destination_repo.pulp_id,
-                                                                         type_ids: [::Katello::Pulp::Rpm::CONTENT_TYPE],
-                                                                          filters: {unit: rpm_remove_clauses})
-          end
-
-          tasks.concat(copy_module_contents(destination_repo,
-                                              filters: filters,
-                                              solve_dependencies: solve_dependencies))
-
-          [:srpm, :errata, :package_group, :package_environment,
-           :yum_repo_metadata_file, :distribution, :module_default, :drpm].each do |type|
-            tasks << smart_proxy.pulp_api.extensions.send(type).copy(repo.pulp_id, destination_repo.pulp_id)
-          end
-          tasks
-        end
-
         def import_distribution_data
           distribution = smart_proxy.pulp_api.extensions.repository.distributions(repo.pulp_id).first
           if distribution
@@ -230,29 +199,6 @@ module Katello
           end
           [copy_clauses, remove_clauses]
         end
-
-        def generate_copy_clauses(filters, rpm_filenames)
-          if rpm_filenames&.any?
-            copy_clauses = {filters: {unit: { 'filename' => { '$in' => rpm_filenames } }}}
-            remove_clauses = nil
-          elsif filters&.any?
-            clause_gen = ::Katello::Util::PackageClauseGenerator.new(repo, filters.yum(false))
-            clause_gen.generate
-
-            copy = clause_gen.copy_clause
-            copy_clauses = {filters: {unit: copy }} if copy
-
-            remove = clause_gen.remove_clause
-            remove_clauses = {filters: {unit: remove}} if remove
-          else
-            non_modular_rpms = ::Katello::Rpm.in_repositories(repo).non_modular.pluck(:filename)
-            copy_clauses = non_modular_rpms.blank? ? nil : {filters: {unit: ContentViewPackageFilter.generate_rpm_clauses(non_modular_rpms)}}
-            remove_clauses = nil
-          end
-
-          copy_clauses&.merge!(fields: ::Katello::Pulp::Rpm::PULP_SELECT_FIELDS)
-          [copy_clauses, remove_clauses]
-        end
       end
     end
   end

data/app/services/katello/pulp/repository.rb

@@ -278,12 +278,6 @@ module Katello
         Katello::RepositoryTypeManager.find(repo.content_type)
       end
 
-      def index_content_units
-        content_unit_types.each do |content_type|
-          content_type.pulp2_service_class
-        end
-      end
-
       def proxy_options
         if repo.root.http_proxy_policy == RootRepository::NO_DEFAULT_HTTP_PROXY
           return { proxy_host: '' }

data/app/services/katello/pulp3/ansible_collection.rb

@@ -18,17 +18,33 @@ module Katello
         fail NotImplementedError
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['checksum'] = backend_data['sha256']
-        custom_json['namespace'] = backend_data['namespace']
-        custom_json['version'] = backend_data['version']
-        custom_json['name'] = backend_data['name']
-        custom_json['description'] = backend_data['description']
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          pulp_id: unit['pulp_href'],
+          checksum: unit['sha256'],
+          namespace: unit['namespace'],
+          version: unit['version'],
+          name: unit['name'],
+          description: unit['description']
+        }
+      end
+
+      def self.insert_child_associations(units, pulp_id_to_id)
+        tag_names = units.map { |unit| unit['tags'].map { |tag| tag[:name] } }.flatten
+        tag_rows = tag_names.map { |name| {name: name } }
+        Katello::AnsibleTag.insert_all(tag_rows, unique_by: [:name]) if tag_rows.any?
+
+        collection_tag_rows = []
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit['pulp_href']]
+          #delete old tags
+          unit_tags = unit['tags'].map { |tag| tag[:name] }
+          Katello::AnsibleCollectionTag.where(:ansible_collection_id => katello_id).where.not(:ansible_tag_id => Katello::AnsibleTag.where(:name => unit_tags)).delete_all
+          collection_tag_rows += Katello::AnsibleTag.where(:name => unit_tags).pluck(:id).map { |tag_id| {ansible_collection_id: katello_id, ansible_tag_id: tag_id} }
+        end
 
-        tags = backend_data['tags'].map { |tag| Katello::AnsibleTag.find_or_create_by(name: tag['name']) }
-        model.tags = tags
+        collection_tag_rows.flatten!
+        Katello::AnsibleCollectionTag.insert_all(collection_tag_rows, unique_by: [:ansible_collection_id, :ansible_tag_id])
       end
     end
   end

data/app/services/katello/pulp3/api/apt.rb

@@ -4,6 +4,13 @@ module Katello
   module Pulp3
     module Api
       class Apt < Core
+        def publication_verbatim_class
+          PulpDebClient::DebVerbatimPublication
+        end
+
+        def publications_verbatim_api
+          PulpDebClient::PublicationsVerbatimApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -121,11 +121,20 @@ module Katello
                         }
           end
 
+          fail _("Content View label not provided.") if metadata[:label].blank?
+
           cv = ::Katello::ContentView.find_by(label: metadata[:label],
-                                              organization: organization,
-                                              import_only: true)
+                                              organization: organization)
           if cv.blank?
             ::Katello::ContentView.create!(metadata.merge(organization: organization, import_only: true))
+          elsif !cv.import_only?
+            msg = _("Unable to import in to Content View specified in the metadata - '%{name}'. "\
+                     "The 'import_only' attribute for the content view is set to false. "\
+                     "To mark this Content View as importable, have your system administrator"\
+                     " run the following command on the server. "\
+                        % { name: cv.name })
+            command = "foreman-rake katello:set_content_view_import_only ID=#{cv.id}"
+            fail msg + "\n" + command
           else
             cv.update!(description: cv_metadata[:description]) if cv.description != metadata[:description]
             cv

data/app/services/katello/pulp3/deb.rb

@@ -23,15 +23,16 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['checksum'] = backend_data['sha256']
-        custom_json['filename'] = backend_data['relative_path']
-        custom_json['name'] = backend_data['package']
-        custom_json['version'] = backend_data['version']
-        custom_json['description'] = backend_data['description']
-        custom_json['architecture'] = backend_data['architecture']
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          pulp_id: unit[unit_identifier],
+          checksum: unit[:sha256],
+          filename: unit[:relative_path],
+          name: unit[:package],
+          version: unit[:version],
+          description: unit[:description]&.truncate(255),
+          architecture: unit[:architecture]
+        }
       end
     end
   end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -20,11 +20,12 @@ module Katello
         self.content_api.list(page_opts)
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['schema_version'], = backend_data['schema_version']
-        custom_json['digest'], = backend_data['digest']
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          schema_version: unit['schema_version'],
+          digest: unit['digest'],
+          pulp_id: unit[unit_identifier]
+        }
       end
     end
   end

data/app/services/katello/pulp3/docker_manifest_list.rb

@@ -18,12 +18,29 @@ module Katello
         self.content_api.list(page_opts)
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['schema_version'], = backend_data['schema_version']
-        custom_json['digest'], = backend_data['digest']
-        custom_json['docker_manifests'] = ::Katello::DockerManifest.where(:pulp_id => backend_data[:listed_manifests])
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        {
+          schema_version: unit['schema_version'],
+          digest: unit['digest'],
+          pulp_id: unit[unit_identifier]
+        }
+      end
+
+      def self.insert_child_associations(units, pulp_id_to_id)
+        manifest_list_manifests = []
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit[unit_identifier]]
+          manifest_ids = ::Katello::DockerManifest.where(:pulp_id => unit[:listed_manifests]).pluck(:id)
+          manifest_list_manifests += manifest_ids.map do |manifest_id|
+            {
+              docker_manifest_list_id: katello_id,
+              docker_manifest_id: manifest_id
+            }
+          end
+        end
+
+        manifest_list_manifests.flatten!
+        Katello::DockerManifestListManifest.insert_all(manifest_list_manifests, unique_by: [:docker_manifest_list_id, :docker_manifest_id]) if manifest_list_manifests.any?
       end
     end
   end