katello 4.3.0.rc4 → 4.4.0.rc2

data/app/services/katello/pulp3/erratum.rb

@@ -25,13 +25,13 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      # rubocop:disable Metrics/AbcSize
-      def update_model(model)
-        updated = false
+      def self.generate_model_row(unit)
         keys = %w(title id severity issued_date type description reboot_suggested solution updated_date summary)
-        custom_json = backend_data.slice(*keys)
+        custom_json = unit.slice(*keys)
+        custom_json['pulp_id'] = custom_json['id']
         custom_json["issued"] = custom_json.delete("issued_date")
         custom_json["updated"] = custom_json.delete("updated_date")
+        custom_json['title'] = custom_json['title']&.truncate(255)
 
         # handle SUSE epoch dates
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])
@@ -41,86 +41,81 @@ module Katello
         custom_json['errata_type'] = custom_json.delete('type')
         custom_json['issued'] = custom_json['issued'].to_datetime.strftime('%Y-%m-%d').to_datetime
         custom_json['updated'] = custom_json['updated'].blank? ? custom_json['issued'] : custom_json['updated'].to_datetime.strftime('%Y-%m-%d').to_datetime
+        custom_json
+      end
 
-        if model.updated.blank? ||
-          model.attributes.excluding(model.attributes.keys - custom_json.keys) != custom_json
-          model.update!(custom_json)
-          updated = true
-
-          unless backend_data['references'].blank?
-            update_bugzillas(model, backend_data['references'])
-            update_cves(model, backend_data['references'])
-          end
+      def self.insert_child_associations(units, pulp_id_to_id)
+        bugzillas = []
+        cves = []
+        packages = []
+        modules = []
+
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit['id']]
+          bugzillas += build_bugzillas(katello_id, unit['references'])
+          cves += build_cves(katello_id, unit['references'])
+          packages += build_packages(katello_id, unit['pkglist'])
+          modules += build_modules(katello_id, unit['pkglist'])
         end
-        update_packages(model, backend_data['pkglist']) unless backend_data['pkglist'].blank?
-        update_modules(model, backend_data['pkglist']) unless backend_data['pkglist'].blank?
 
-        return model.id if updated
+        Katello::ErratumBugzilla.insert_all(bugzillas, unique_by: [:erratum_id, :bug_id, :href]) if bugzillas.any?
+        Katello::ErratumCve.insert_all(cves, unique_by: [:erratum_id, :cve_id, :href]) if cves.any?
+        Katello::ErratumPackage.insert_all(packages, unique_by: [:erratum_id, :nvrea, :name, :filename]) if packages.any?
+        ModuleStreamErratumPackage.insert_all(modules, unique_by: [:module_stream_id, :erratum_package_id]) if modules.any?
+        nil
       end
-      # rubocop:enable Metrics/AbcSize
 
-      def update_bugzillas(model, ref_list)
-        ref_list.select { |r| r[:type] == "bugzilla" }.each do |bugzilla|
-          Katello::Util::Support.active_record_retry do
-            model.bugzillas.where(bug_id: bugzilla[:id]).first_or_create!(bug_id: bugzilla[:id], href: bugzilla[:href], erratum_id: model.id)
-          end
+      def self.build_bugzillas(katello_id, ref_list)
+        ref_list.select { |r| r[:type] == "bugzilla" }.map do |bugzilla|
+          {
+            bug_id: bugzilla[:id],
+            href: bugzilla[:href],
+            erratum_id: katello_id
+          }
         end
       end
 
-      def update_cves(model, ref_list)
-        ref_list.select { |r| r[:type] == "cve" }.each do |cve|
-          Katello::Util::Support.active_record_retry do
-            model.cves.where(cve_id: cve[:id]).first_or_create!(cve_id: cve[:id], href: cve[:href], erratum_id: model.id)
-          end
+      def self.build_cves(katello_id, ref_list)
+        ref_list.select { |r| r[:type] == "cve" }.map do |cve|
+          {
+            cve_id: cve[:id],
+            href: cve[:href],
+            erratum_id: katello_id
+          }
         end
       end
 
-      def update_packages(model, package_list)
-        package_list.each do |json|
+      def self.build_packages(katello_id, pkg_list)
+        list = pkg_list.map do |json|
           package_hashes = json[:packages]
-          package_attributes = package_hashes.map do |hash|
+          package_hashes.map do |hash|
             nvrea = Util::Package.build_nvra(hash)
-            {'name' => hash[:name], 'nvrea' => nvrea, 'filename' => hash[:filename]}
-          end
-          existing_nvreas = model.packages.pluck(:nvrea)
-          package_attributes.delete_if { |pkg| existing_nvreas.include?(pkg['nvrea']) }
-          package_attributes.uniq.each do |package|
-            Katello::Util::Support.active_record_retry do
-              model.packages.where(filename: package["filename"]).first_or_create!(package)
-            end
+            {'name' => hash[:name], 'nvrea' => nvrea, 'filename' => hash[:filename], :erratum_id => katello_id}
           end
         end
+        list.flatten
       end
 
-      def update_modules(model, module_stream_list)
+      def self.build_modules(katello_id, module_stream_list)
         module_stream_attributes = []
         module_stream_list.each do |package_item|
           if package_item[:module]
             module_stream = ::Katello::ModuleStream.where(package_item[:module]).first
             next if module_stream.blank?
             nvreas = package_item[:packages].map { |hash| Util::Package.build_nvra(hash) }
-            module_stream_id_column = "#{ModuleStreamErratumPackage.table_name}.module_stream_id"
-            existing = ErratumPackage.joins(:module_streams).
-                                      where(module_stream_id_column => module_stream.id,
-                                            :nvrea => nvreas).pluck(:nvrea)
-
-            (nvreas - existing).each do |nvrea|
-              package = model.packages.find_by(:nvrea => nvrea)
-              module_stream_attributes << { :module_stream_id => module_stream.id,
-                                            :erratum_package_id => package.id }
-            end
-          end
-        end
-        module_stream_attributes.uniq.each do |module_stream|
-          Katello::Util::Support.active_record_retry do
-            if model.module_streams.empty? || !model.module_streams.pluck(:module_stream_id).include?(module_stream[:module_stream_id])
-              ModuleStreamErratumPackage.create!(module_stream)
+            package_ids = Katello::ErratumPackage.where(:erratum_id => katello_id, :nvrea => nvreas).pluck(:id)
+
+            module_stream_attributes += package_ids.map do |pkg_id|
+              { :module_stream_id => module_stream.id,
+                :erratum_package_id => pkg_id }
             end
+
           end
         end
+        module_stream_attributes.uniq
       end
 
-      def convert_date_if_epoch(date)
+      def self.convert_date_if_epoch(date)
         date.to_i.to_s == date ? epoch_to_date(date) : date
       end
 

data/app/services/katello/pulp3/file_unit.rb

@@ -19,12 +19,15 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def update_model(model)
-        custom_json = {}
-        custom_json['checksum'] = backend_data['sha256']
-        custom_json['path'] = backend_data['relative_path']
-        custom_json['name'] = File.basename(backend_data['relative_path'].try(:split, '/').try(:[], -1))
-        model.update!(custom_json)
+      def self.generate_model_row(unit)
+        filename = File.basename(unit['relative_path'].try(:split, '/').try(:[], -1))
+
+        {
+          pulp_id: unit[unit_identifier],
+          name: filename,
+          path: unit['relative_path'],
+          checksum: unit['sha256']
+        }
       end
     end
   end

data/app/services/katello/pulp3/generic_content_unit.rb

@@ -13,20 +13,19 @@ module Katello
       end
 
       def self.content_api(repository_type, content_type)
-        repository_type.content_types.find { |type| type.content_type == content_type }.pulp3_api.new(repository_type.pulp3_api_class.new(SmartProxy.pulp_primary!, repository_type).api_client)
+        label = content_type.is_a?(String) ? content_type : content_type.label
+        repository_type.content_types.find { |type| type.content_type == label }.pulp3_api.new(repository_type.pulp3_api_class.new(SmartProxy.pulp_primary!, repository_type).api_client)
       end
 
-      def update_model(model, content_type)
-        content_type = ::Katello::RepositoryTypeManager.find_content_type(content_type)
-
-        custom_json = {}
-        custom_json['pulp_id'] = backend_data['pulp_href']
-        custom_json['name'] = content_type&.model_name&.call(backend_data)
-        custom_json['version'] = content_type&.model_version&.call(backend_data)
-        custom_json['filename'] = content_type&.model_filename&.call(backend_data)
-        custom_json['additional_metadata'] = content_type&.model_additional_metadata&.call(backend_data)
-        custom_json['content_type'] = content_type&.label
-        model.update!(custom_json)
+      def self.generate_model_row(unit, content_type)
+        {
+          pulp_id: unit['pulp_href'],
+          name: content_type&.model_name&.call(unit),
+          version: content_type&.model_version&.call(unit),
+          filename: content_type&.model_filename&.call(unit),
+          additional_metadata: content_type&.model_additional_metadata&.call(unit),
+          content_type: content_type&.label
+        }
       end
     end
   end

data/app/services/katello/pulp3/module_stream.rb

@@ -13,44 +13,90 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def create_stream_rpms(model, packages)
-        packages_found = Katello::Rpm.where(:pulp_id => packages)
-        existing_rpms = model.rpms
-        new_packages = packages_found - existing_rpms
-        packages_to_delete = existing_rpms - packages_found
-        model.rpms.delete(packages_to_delete)
-        model.rpms << new_packages
-      end
-
-      def create_stream_artifacts(model, artifacts_json)
-        artifacts_json.each do |name|
-          Katello::Util::Support.active_record_retry do
-            model.artifacts.where(name: name).first_or_create!
-          end
+      def self.build_stream_rpms(katello_id, package_hrefs)
+        package_ids = Katello::Rpm.where(:pulp_id => package_hrefs).pluck(:id)
+        rpms = package_ids.map do |package_id|
+          {
+            module_stream_id: katello_id,
+            rpm_id: package_id
+          }
         end
+        add_timestamps(rpms)
       end
 
-      def create_profiles(model, profiles_json)
-        profiles_json.each do |profile, rpms|
-          Katello::Util::Support.active_record_retry do
-            profile = model.profiles.where(name: profile).first_or_create!
-          end
-          rpms.each do |rpm|
-            Katello::Util::Support.active_record_retry do
-              profile.rpms.where(name: rpm).first_or_create!
-            end
+      def self.build_artifacts(katello_id, artifacts_json)
+        return [] if artifacts_json.nil?
+        artifacts = artifacts_json.map do |name|
+          {name: name,
+           module_stream_id: katello_id}
+        end
+        add_timestamps(artifacts)
+      end
+
+      def self.build_profiles(katello_id, profiles_json)
+        return [] if profiles_json.nil?
+        profiles = profiles_json.map do |profile, _rpms|
+          {
+            module_stream_id: katello_id,
+            name: profile
+          }
+        end
+        add_timestamps(profiles)
+      end
+
+      def self.build_profile_rpms(katello_id, profiles_json)
+        return [] if profiles_json.nil?
+        profile_rpms = profiles_json.map do |profile, rpms|
+          profile_id = Katello::ModuleProfile.find_by(module_stream_id: katello_id, name: profile).id
+          rpms.map do |rpm|
+            {
+              module_profile_id: profile_id,
+              name: rpm
+            }
           end
         end
+        add_timestamps(profile_rpms.flatten)
       end
 
-      def update_model(model)
-        shared_attributes = backend_data.keys & model.class.column_names
-        shared_json = backend_data.select { |key, _v| shared_attributes.include?(key) }
-        model.update!(shared_json)
+      def self.generate_model_row(unit)
+        shared_attributes = unit.keys & Katello::ModuleStream.column_names
+        to_return = unit.select { |key, _v| shared_attributes.include?(key) }
+        to_return['pulp_id'] = unit['pulp_href']
+        to_return[:created_at] = DateTime.now
+        to_return[:updated_at] = DateTime.now
+        to_return
+      end
+
+      def self.add_timestamps(rows)
+        rows.each do |row|
+          row[:created_at] = DateTime.now
+          row[:updated_at] = DateTime.now
+        end
+        rows
+      end
 
-        create_stream_artifacts(model, backend_data['artifacts']) if backend_data.key?('artifacts')
-        create_profiles(model, backend_data['profiles']) if backend_data.key?('profiles')
-        create_stream_rpms(model, backend_data['packages']) if backend_data.key?('packages')
+      def self.insert_child_associations(units, pulp_id_to_id)
+        artifacts = []
+        profiles = []
+        stream_rpms = []
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit[unit_identifier]]
+          artifacts += build_artifacts(katello_id, unit['artifacts'])
+          profiles += build_profiles(katello_id, unit['profiles'])
+          stream_rpms += build_stream_rpms(katello_id, unit['packages'])
+        end
+
+        Katello::ModuleStreamArtifact.insert_all(artifacts, unique_by: [:module_stream_id, :name]) if artifacts.any?
+        Katello::ModuleProfile.insert_all(profiles, unique_by: [:module_stream_id, :name]) if profiles.any?
+        Katello::ModuleStreamRpm.insert_all(stream_rpms, unique_by: [:module_stream_id, :rpm_id]) if stream_rpms.any?
+
+        #have to import profile_rpms after profiles
+        profile_rpms = []
+        units.each do |unit|
+          katello_id = pulp_id_to_id[unit[unit_identifier]]
+          profile_rpms += build_profile_rpms(katello_id, unit['profiles'])
+        end
+        Katello::ModuleProfileRpm.insert_all(profile_rpms, unique_by: [:module_profile_id, :name]) if profile_rpms.any?
       end
     end
   end

data/app/services/katello/pulp3/package_group.rb

@@ -54,12 +54,12 @@ module Katello
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def update_model(model)
+      def self.generate_model_row(unit)
         custom_json = {}
-        custom_json['pulp_id'] = backend_data['pulp_href']
-        custom_json['name'] = backend_data['name']
-        custom_json['description'] = backend_data['description']
-        model.update!(custom_json)
+        custom_json['pulp_id'] = unit['pulp_href']
+        custom_json['name'] = unit['name']
+        custom_json['description'] = unit['description']
+        custom_json
       end
     end
   end

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -34,10 +34,6 @@ module Katello
         true
       end
 
-      def update_model
-        fail NotImplementedError
-      end
-
       attr_accessor :uuid
       attr_writer :backend_data
 
@@ -57,6 +53,20 @@ module Katello
         self::CONTENT_TYPE
       end
 
+      def self.pulp_units_for_ids(content_unit_hrefs)
+        Enumerator.new do |yielder|
+          yielder.yield content_unit_hrefs.collect { |href| pulp_data(href).with_indifferent_access }
+        end
+      end
+
+      def self.add_timestamps(rows)
+        rows.each do |row|
+          row[:created_at] = DateTime.now
+          row[:updated_at] = DateTime.now
+        end
+        rows
+      end
+
       def self.pulp_units_batch_all(content_unit_hrefs)
         Enumerator.new do |yielder|
           yielder.yield content_unit_hrefs.collect { |href| pulp_data(href) }
@@ -116,30 +126,28 @@ module Katello
         content_unit_list page_opts
       end
 
-      # rubocop:disable Lint/UselessAssignment
       def self.find_duplicate_unit(repository, unit_type_id, file, checksum)
-        filter_label = 'sha256'
+        filter_label = :sha256
         if unit_type_id == 'ostree_ref'
-          filter_label = 'checksum'
+          filter_label = :checksum
         end
         content_backend_service = SmartProxy.pulp_primary.content_service(unit_type_id)
         duplicates_allowed = ::Katello::RepositoryTypeManager.find_content_type(unit_type_id).try(:duplicates_allowed)
         if repository.generic? && duplicates_allowed
           filename_key = ::Katello::RepositoryTypeManager.find_content_type(unit_type_id).filename_key
           duplicate_sha_path_content_list = content_backend_service.content_api(repository.repository_type, unit_type_id).list(
-            filter_label: checksum,
+            filter_label => checksum,
             filename_key => file[:filename])
         elsif repository.generic?
           duplicate_sha_path_content_list = content_backend_service.content_api(repository.repository_type, unit_type_id).list(
-            filter_label: checksum)
+            filter_label => checksum)
         else
           duplicate_sha_path_content_list = content_backend_service.content_api.list(
-            filter_label: checksum,
+            filter_label => checksum,
             "relative_path": file[:filename])
         end
         duplicate_sha_path_content_list
       end
-      # rubocop:enable Lint/UselessAssignment
     end
   end
 end

data/app/services/katello/pulp3/repository/apt.rb

@@ -24,9 +24,11 @@ module Katello
         end
 
         def mirror_remote_options
-          {
-            distributions: repo.deb_releases + "#{' default' unless repo.deb_releases.include? 'default'}"
-          }
+          super.merge(
+            {
+              distributions: repo.deb_releases + "#{' default' unless repo.deb_releases.include? 'default'}"
+            }
+          )
         end
 
         def publication_options(repository_version)

data/app/services/katello/pulp3/repository/docker.rb

@@ -13,12 +13,17 @@ module Katello
         end
 
         def remote_options
-          options = {url: root.url, upstream_name: root.docker_upstream_name}
-          if root.docker_tags_whitelist&.any?
-            options[:include_tags] = root.docker_tags_whitelist
+          options = {url: root.url, upstream_name: root.docker_upstream_name, policy: root.download_policy}
+          if root.include_tags&.any?
+            options[:include_tags] = root.include_tags
           else
             options[:include_tags] = nil
           end
+          if root.exclude_tags&.any?
+            options[:exclude_tags] = root.exclude_tags
+          else
+            options[:exclude_tags] = nil
+          end
           common_remote_options.merge(options)
         end
 
@@ -31,10 +36,12 @@ module Katello
         end
 
         def mirror_remote_options
-          {
-            url: "https://#{SmartProxy.pulp_primary.pulp3_host!.downcase}",
-            upstream_name: repo.container_repository_name
-          }
+          super.merge(
+            {
+              url: "https://#{SmartProxy.pulp_primary.pulp3_host!.downcase}",
+              upstream_name: repo.container_repository_name
+            }
+          )
         end
 
         def distribution_options(path)

data/app/services/katello/pulp3/repository/generic.rb

@@ -24,7 +24,7 @@ module Katello
         def remote_options
           generic_remote_options = JSON.parse(root.generic_remote_options)
           if generic_remote_options.any?
-            common_remote_options.merge(generic_remote_options.select { |_, v| !v.nil? }).symbolize_keys
+            common_remote_options.merge(generic_remote_options).symbolize_keys
           else
             common_remote_options
           end

data/app/services/katello/pulp3/repository/yum.rb

@@ -49,16 +49,6 @@ module Katello
           }
         end
 
-        def mirror_remote_options
-          policy = smart_proxy.download_policy
-
-          if smart_proxy.download_policy == SmartProxy::DOWNLOAD_INHERIT
-            policy = repo.root.download_policy
-          end
-
-          { policy: policy }
-        end
-
         def import_distribution_data
           distribution = ::Katello::Pulp3::Distribution.fetch_content_list(repository_version: repo.version_href)
           if distribution.results.present?
@@ -78,6 +68,8 @@ module Katello
 
         def sync_url_params(sync_options)
           params = super
+          params.delete(:mirror)
+          params[:sync_policy] = repo.mirroring_policy
           params[:optimize] = sync_options[:optimize] if sync_options.key?(:optimize)
           params
         end
@@ -93,6 +85,7 @@ module Katello
           "/pulp/content/#{repo.relative_path}/".sub('//', '/')
         end
 
+        # rubocop:disable Metrics/MethodLength
         def multi_copy_units(repo_id_map, dependency_solving)
           tasks = []
 
@@ -105,14 +98,22 @@ module Katello
               dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_primary).repository_reference.repository_href
               content_unit_hrefs = dest_repo_id_map[:content_unit_hrefs]
               # Not needed during incremental update due to dest_base_version
+              # -> Unless incrementally updating a CV repo that is a soft copy of its library instance.
+              # -> I.e. no filters and not an incremental version.
               unless dest_repo_id_map[:base_version]
-                source_repo_for_package_envs = ::Katello::Repository.find(source_repo_ids.first)
-                unless source_repo_for_package_envs.library_instance?
-                  source_repo_for_package_envs = source_repo_for_package_envs.library_instance
+                source_repo_for_content = ::Katello::Repository.find(source_repo_ids.first)
+                unless source_repo_for_content.library_instance?
+                  source_repo_for_content = source_repo_for_content.library_instance
+                end
+                modulemd_defaults_hrefs = modulemd_defaults({ :repository_version => source_repo_for_content.version_href }).map(&:pulp_href).sort
+                package_env_hrefs = packageenvironments({ :repository_version => source_repo_for_content.version_href }).map(&:pulp_href).sort
+                # Don't perform extra content actions if the repo is a soft copy of its library instance.
+                # Taken care of by the IncrementalUpdate action.
+                unless dest_repo.soft_copy_of_library?
+                  tasks << remove_all_content_from_repo(dest_repo_href)
+                  tasks << add_content_for_repo(dest_repo_href, package_env_hrefs) unless package_env_hrefs.empty?
+                  tasks << add_content_for_repo(dest_repo_href, modulemd_defaults_hrefs) unless modulemd_defaults_hrefs.empty?
                 end
-                package_env_hrefs = packageenvironments({ :repository_version => source_repo_for_package_envs.version_href }).map(&:pulp_href).sort
-                tasks << remove_all_content_from_repo(dest_repo_href)
-                tasks << add_content_for_repo(dest_repo_href, package_env_hrefs) unless package_env_hrefs.empty?
               end
               source_repo_ids.each do |source_repo_id|
                 source_repo_version = ::Katello::Repository.find(source_repo_id).version_href
@@ -127,6 +128,7 @@ module Katello
           end
           tasks.flatten
         end
+        # rubocop:enable Metrics/MethodLength
 
         def copy_api_data_dup(data)
           data_dup = PulpRpmClient::Copy.new
@@ -201,6 +203,7 @@ module Katello
 
           if content_unit_hrefs.sort!.any?
             content_unit_hrefs += packageenvironments({ :repository_version => source_repository.version_href }).map(&:pulp_href).sort
+            content_unit_hrefs += modulemd_defaults({ :repository_version => source_repository.version_href }).map(&:pulp_href).sort
             first_slice = remove_all
             content_unit_hrefs.each_slice(UNIT_LIMIT) do |slice|
               tasks << add_content(slice, first_slice)
@@ -228,6 +231,10 @@ module Katello
           Katello::Pulp3::Api::Core.fetch_from_list { |page_opts| api.content_package_environments_api.list(page_opts.merge(options)) }
         end
 
+        def modulemd_defaults(options = {})
+          Katello::Pulp3::Api::Core.fetch_from_list { |page_opts| api.content_modulemd_defaults_api.list(page_opts.merge(options)) }
+        end
+
         def metadatafiles(options = {})
           api.content_repo_metadata_files_api.list(options)
         end