katello 3.16.0.rc5 → 3.16.1.2

data/app/models/katello/content_view.rb

@@ -289,9 +289,21 @@ module Katello
         where("#{Katello::ContentViewVersion.table_name}.content_view_id" => self.id)
     end
 
-    def repositories_to_publish
+    def repositories_to_publish(override_components = nil)
       if composite?
-        ids = components.flat_map { |version| version.repositories.archived }.map(&:id)
+        components_to_publish = []
+        components.each do |component|
+          override_component = override_components&.detect do |override_cvv|
+            override_cvv.content_view == component.content_view
+          end
+
+          if override_component
+            components_to_publish << override_component
+          else
+            components_to_publish << component
+          end
+        end
+        ids = components_to_publish.flat_map { |version| version.repositories.archived }.map(&:id)
         Repository.where(:id => ids)
       else
         repositories
@@ -302,11 +314,11 @@ module Katello
       composite? ? repositories_to_publish.pluck(&:id) : repository_ids
     end
 
-    def repositories_to_publish_by_library_instance
+    def repositories_to_publish_by_library_instance(override_components = nil)
       # retrieve the list of repositories in a hash, where the key
       # is the library instance id, and the value is an array
       # of the repositories for that instance.
-      repositories_to_publish.inject({}) do |result, repo|
+      repositories_to_publish(override_components).inject({}) do |result, repo|
         result[repo.library_instance] ||= []
         result[repo.library_instance] << repo
         result
@@ -326,8 +338,8 @@ module Katello
       component_composites.where(latest: true).joins(:composite_content_view).where(self.class.table_name => {auto_publish: true})
     end
 
-    def publish_repositories
-      repositories = composite? ? repositories_to_publish_by_library_instance.values : repositories_to_publish
+    def publish_repositories(override_components = nil)
+      repositories = composite? ? repositories_to_publish_by_library_instance(override_components).values : repositories_to_publish
       repositories.each do |repos|
         if repos.is_a? Array
           yield repos

data/app/models/katello/content_view_erratum_filter.rb

@@ -30,6 +30,7 @@ module Katello
       if filter_by_id?
         errata_ids = erratum_rules.map(&:errata_id)
         errata_pulp_ids = errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
+        errata_pulp_ids += errata_module_stream_pulp_ids_from_errata_ids(errata_ids)
       else
         clauses = []
         clauses << errata_from
@@ -37,7 +38,9 @@ module Katello
         clauses << types_clause
         package_filenames = Erratum.list_filenames_by_clauses(repo, clauses.compact)
         errata_pulp_ids = errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
+        errata_pulp_ids += errata_module_stream_pulp_ids_from_clauses(repo, clauses.compact)
       end
+
       errata_pulp_ids
     end
 
@@ -48,10 +51,20 @@ module Katello
       repo.rpms.where("filename ILIKE ANY ( array[?] )", query_params)
     end
 
+    def errata_module_stream_pulp_ids_from_clauses(repo, clauses)
+      module_streams = Erratum.list_modular_streams_by_clauses(repo, clauses)
+      module_streams.pluck(:pulp_id)
+    end
+
     def errata_package_pulp_ids_from_package_filenames(repo, package_filenames)
       rpms_by_filename(repo, package_filenames).pluck(:pulp_id)
     end
 
+    def errata_module_stream_pulp_ids_from_errata_ids(errata_ids)
+      module_streams = Katello::Erratum.where(:errata_id => errata_ids).map(&:module_streams).compact.flatten
+      module_streams.pluck(:pulp_id)
+    end
+
     def errata_package_pulp_ids_from_errata_ids(repo, errata_ids)
       package_filenames = Katello::ErratumPackage.joins(:erratum).where("#{Erratum.table_name}.errata_id" => errata_ids).pluck(:filename)
       rpms_by_filename(repo, package_filenames).pluck(:pulp_id)

data/app/models/katello/content_view_filter.rb

@@ -182,6 +182,10 @@ module Katello
       fail "setting original_packages not supported for #{self.class.name}"
     end
 
+    def original_module_streams=(_include_original)
+      fail "setting original_module_streams not supported for #{self.class.name}"
+    end
+
     protected
 
     def validate_repos

data/app/models/katello/content_view_module_stream_filter.rb

@@ -6,9 +6,36 @@ module Katello
 
     validates_lengths_from_database
 
-    def generate_clauses(_repo)
-      return if module_stream_rules.blank?
-      module_stream_rules.map(&:module_stream_id)
+    def generate_clauses(repo)
+      rules = module_stream_rules || []
+      ids = rules.map(&:module_stream_id)
+      if self.original_module_streams
+        ids.concat(repo.module_streams_without_errata.map(&:id))
+      end
+      ids
+    end
+
+    def original_module_streams=(value)
+      self[:original_module_streams] = value
+    end
+
+    def content_unit_pulp_ids(repo, dependents = false)
+      content_unit_ids = []
+      module_ids = []
+
+      self.module_stream_rules.each do |rule|
+        module_ids << rule.module_stream_id
+      end
+      if self.original_module_streams
+        module_ids.concat(repo.module_streams_without_errata.map(&:id))
+      end
+      modules_streams = ModuleStream.where(id: module_ids).includes(:rpms)
+      content_unit_ids += modules_streams.pluck(:pulp_id).flatten.uniq
+      if dependents && !modules_streams.empty?
+        rpms = modules_streams.map(&:rpms).flatten
+        content_unit_ids += rpms.pluck(:pulp_id).flatten.uniq
+      end
+      content_unit_ids.uniq
     end
   end
 end

data/app/models/katello/content_view_package_filter.rb

@@ -37,7 +37,7 @@ module Katello
       self.package_rules.each do |rule|
         package_filenames.concat(query_rpms(repo, rule))
       end
-
+      package_filenames.concat(repo.packages_without_errata.map(&:filename)) if self.original_packages
       rpms = Rpm.in_repositories(repo)
       rpms.where(filename: package_filenames).where(:modular => false).pluck(:pulp_id).flatten.uniq
     end

data/app/models/katello/host/content_facet.rb

@@ -126,6 +126,7 @@ module Katello
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::Erratum, bound_repos).calculate_and_import
         ::Katello::Applicability::ApplicableContentHelper.new(self, ::Katello::ModuleStream, bound_repos).calculate_and_import
         update_applicability_counts
+        self.update_errata_status
       end
 
       def import_applicability(partial = false)

data/app/models/katello/module_stream.rb

@@ -65,7 +65,7 @@ module Katello
     end
 
     def module_spec_hash
-      {:name => name, :stream => stream, :version => version, :context => context, :arch => arch, :id => id}.compact
+      {:name => name, :stream => stream, :version => version, :context => context, :arch => arch, :pulp_id => pulp_id, :id => id}.compact
     end
 
     def self.parse_module_spec(module_spec)

data/app/models/katello/ping.rb

@@ -31,9 +31,7 @@ module Katello
 
         # set overall status result code
         result = {:services => result}
-        result[:services].each_value do |v|
-          result[:status] = v[:status] == OK_RETURN_CODE ? OK_RETURN_CODE : FAIL_RETURN_CODE
-        end
+        result[:status] = result[:services].each_value.any? { |v| v[:status] == FAIL_RETURN_CODE } ? FAIL_RETURN_CODE : OK_RETURN_CODE
         result
       end
 

data/app/models/katello/repository.rb

@@ -133,6 +133,7 @@ module Katello
     scope :in_published_environments, -> { in_content_views(Katello::ContentView.non_default).where.not(:environment_id => nil) }
     scope :order_by_root, ->(attr) { joins(:root).order("#{Katello::RootRepository.table_name}.#{attr}") }
     scope :with_content, ->(content) { joins(Katello::RepositoryTypeManager.find_content_type(content).model_class.repository_association_class.name.demodulize.underscore.pluralize.to_sym).distinct }
+    scope :by_rpm_count, -> { left_joins(:repository_rpms).group(:id).order("count(katello_repository_rpms.id) ASC") } # smallest count first
 
     scoped_search :on => :name, :relation => :root, :complete_value => true
     scoped_search :rename => :product, :on => :name, :relation => :product, :complete_value => true
@@ -469,6 +470,17 @@ module Katello
       end
     end
 
+    def module_streams_without_errata
+      module_stream_errata = Katello::ModuleStreamErratumPackage.joins(:erratum_package => {:erratum => :repository_errata})
+                              .where("#{RepositoryErratum.table_name}.repository_id" => self.id)
+                              .pluck("#{Katello::ModuleStreamErratumPackage.table_name}.module_stream_id")
+      if module_stream_errata.any?
+        self.module_streams.where("#{ModuleStream.table_name}.id NOT in (?)", module_stream_errata)
+      else
+        self.module_streams
+      end
+    end
+
     def self.with_errata(errata)
       joins(:repository_errata).where("#{Katello::RepositoryErratum.table_name}.erratum_id" => errata)
     end
@@ -581,6 +593,10 @@ module Katello
       environment
     end
 
+    def self.smart_proxy_syncable
+      where.not(:environment_id => nil)
+    end
+
     def exist_for_environment?(environment, content_view, attribute = nil)
       if environment.present? && content_view.in_environment?(environment)
         repos = content_view.version(environment).repos(environment)

data/app/models/setting/content.rb

@@ -92,7 +92,7 @@ class Setting::Content < Setting
       self.set('check_services_before_actions', N_("Whether or not to check the status of backend services such as pulp and candlepin prior to performing some actions."),
                true, N_('Check services before actions')),
       self.set('foreman_proxy_content_batch_size', N_("How many repositories should be synced concurrently on the capsule.  A smaller number may lead to longer sync times.  A larger number will increase dynflow load."),
-               25, N_('Batch size to sync repositories in.')),
+               100, N_('Batch size to sync repositories in.')),
       self.set('foreman_proxy_content_auto_sync', N_("Whether or not to auto sync the Smart Proxies after a Content View promotion."),
                true, N_('Sync Smart Proxies after Content View promotion')),
       self.set('default_download_policy', N_("Default download policy for custom repositories (either 'immediate', 'on_demand', or 'background')"), "immediate",

data/app/presenters/katello/sync_status_presenter.rb

@@ -19,6 +19,8 @@ module Katello
     def sync_progress
       return {:state => nil} unless @repo
       return empty_task(@repo) unless @task
+      display_output = @task.humanized[:output]
+      display_output = display_output.split("\n")[0] if (display_output && @repo.version_href)
       {
         :id => @repo.id,
         :product_id => @repo.product.id,
@@ -29,8 +31,8 @@ module Katello
         :start_time => format_date(@task.started_at),
         :finish_time => format_date(@task.ended_at),
         :duration => format_duration(@task.ended_at, @task.started_at),
-        :display_size => @task.humanized[:output],
-        :size => @task.humanized[:output],
+        :display_size => display_output,
+        :size => display_output,
         :is_running => @task.pending && @task.state != 'paused',
         :error_details => @task.errors
       }

data/app/services/cert/certs.rb

@@ -9,11 +9,19 @@ module Cert
     end
 
     def self.ssl_client_cert
-      @ssl_client_cert ||= OpenSSL::X509::Certificate.new(File.open(Setting['pulp_client_cert'], 'r').read)
+      @ssl_client_cert ||= OpenSSL::X509::Certificate.new(File.open(ssl_client_cert_filename, 'r').read)
+    end
+
+    def self.ssl_client_cert_filename
+      Setting['pulp_client_cert']
     end
 
     def self.ssl_client_key
-      @ssl_client_key ||= OpenSSL::PKey::RSA.new(File.open(Setting['pulp_client_key'], 'r').read)
+      @ssl_client_key ||= OpenSSL::PKey::RSA.new(File.open(ssl_client_key_filename, 'r').read)
+    end
+
+    def self.ssl_client_key_filename
+      Setting['pulp_client_key']
     end
 
     def self.verify_ueber_cert(organization)

data/app/services/katello/pulp/repository/yum.rb

@@ -246,7 +246,8 @@ module Katello
             remove = clause_gen.remove_clause
             remove_clauses = {filters: {unit: remove}} if remove
           else
-            copy_clauses = {}
+            non_modular_rpms = ::Katello::Rpm.in_repositories(repo).non_modular.pluck(:filename)
+            copy_clauses = non_modular_rpms.blank? ? nil : {filters: {unit: ContentViewPackageFilter.generate_rpm_clauses(non_modular_rpms)}}
             remove_clauses = nil
           end
 

data/app/services/katello/pulp3/api/core.rb

@@ -74,6 +74,10 @@ module Katello
           PulpcoreClient::TasksApi.new(core_api_client)
         end
 
+        def task_groups_api
+          PulpcoreClient::TaskGroupsApi.new(core_api_client)
+        end
+
         def upload_class
           PulpcoreClient::Upload
         end

data/app/services/katello/pulp3/erratum.rb

@@ -26,8 +26,10 @@ module Katello
       end
 
       def update_model(model)
-        keys = %w(title id severity issued type description reboot_suggested solution updated summary)
+        keys = %w(title id severity issued_date type description reboot_suggested solution updated_date summary)
         custom_json = backend_data.slice(*keys)
+        custom_json["issued"] = custom_json.delete("issued_date")
+        custom_json["updated"] = custom_json.delete("updated_date")
 
         # handle SUSE epoch dates
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])

data/app/services/katello/pulp3/migration.rb

@@ -35,11 +35,15 @@ module Katello
       end
 
       def pulp2_content_api
-        Pulp2to3MigrationClient::Pulp2contentApi.new(api_client)
+        Pulp2to3MigrationClient::Pulp2ContentApi.new(api_client)
+      rescue NameError
+        Pulp2to3MigrationClient::Pulp2contentApi.new(api_client) #backwards compatible
       end
 
       def pulp2_repositories_api
-        Pulp2to3MigrationClient::Pulp2repositoriesApi.new(api_client)
+        Pulp2to3MigrationClient::Pulp2RepositoriesApi.new(api_client)
+      rescue NameError
+        Pulp2to3MigrationClient::Pulp2repositoriesApi.new(api_client) #backwards compatible
       end
 
       def create_and_run_migrations
@@ -64,8 +68,9 @@ module Katello
       end
 
       def create_migrations
-        migration_plans = @repository_types.map { |label| Katello::Pulp3::MigrationPlan.new(label) }
-        migration_plans.map { |plan| migration_plan_api.create(plan: plan.generate.as_json).pulp_href }
+        plan = Katello::Pulp3::MigrationPlan.new(@repository_types)
+        Rails.logger.info(plan)
+        [migration_plan_api.create(plan: plan.generate.as_json).pulp_href]
       end
 
       def start_migration(plan_href)
@@ -112,7 +117,7 @@ module Katello
           relative_path = pulp3_api.distributions_api.read(href).base_path
 
           dist_ref = Katello::Pulp3::DistributionReference.find_or_initialize_by(:path => relative_path)
-          if (distribution_repo = Katello::Repository.find_by(:relative_path => relative_path))
+          if (distribution_repo = Katello::Repository.find_by(:relative_path => relative_path) || Katello::Repository.find_by(:container_repository_name => relative_path))
             dist_ref.href = href
             dist_ref.repository_id = distribution_repo.id
             dist_ref.save!

data/app/services/katello/pulp3/migration_plan.rb

@@ -1,8 +1,8 @@
 module Katello
   module Pulp3
     class MigrationPlan
-      def initialize(repository_type_label)
-        @repository_type = repository_type_label
+      def initialize(repository_type_labels)
+        @repository_types = repository_type_labels
       end
 
       def master_proxy
@@ -17,12 +17,12 @@ module Katello
       end
 
       def generate_plugins
-        [
+        @repository_types.map do |repository_type|
           {
-            type: pulp2_repository_type(@repository_type),
-            repositories: repository_migrations(@repository_type)
+            type: pulp2_repository_type(repository_type),
+            repositories: repository_migrations(repository_type)
           }
-        ]
+        end
       end
 
       def pulp2_repository_type(repository_type)

data/app/services/katello/pulp3/repository.rb

@@ -193,14 +193,17 @@ module Katello
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
-      def sync
-        sync_url_params = {remote: repo.remote_href, mirror: repo.root.mirror_on_sync}
-        skip_type_param = skip_types
-        sync_url_params[:skip_types] = skip_type_param if skip_type_param
-        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_url_params)
+      def sync(options = {})
+        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_url_params(options))
         [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
       end
 
+      def sync_url_params(_sync_options)
+        params = {remote: repo.remote_href, mirror: repo.root.mirror_on_sync}
+        params[:skip_types] = skip_types if skip_types
+        params
+      end
+
       def create_publication
         publication_data = api.class.publication_class.new(repository_version: repo.version_href)
         api.publications_api.create(publication_data)
@@ -352,6 +355,11 @@ module Katello
         api.repositories_api.modify(repository_reference.repository_href, add_content_units: content_unit_href)
       end
 
+      def add_content_for_repo(repository_href, content_unit_href)
+        content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
+        api.repositories_api.modify(repository_href, add_content_units: content_unit_href)
+      end
+
       def unit_keys(uploads)
         uploads.map do |upload|
           upload.except('id')

data/app/services/katello/pulp3/repository/yum.rb

@@ -3,10 +3,13 @@ require 'pulp_rpm_client'
 module Katello
   module Pulp3
     class Repository
+      # rubocop:disable Metrics/ClassLength
       class Yum < ::Katello::Pulp3::Repository
         include Katello::Util::Errata
         include Katello::Util::PulpcoreContentFilters
 
+        UNIT_LIMIT = 10_000
+
         def remote_options
           if root.url.blank?
             common_remote_options.merge(url: nil, policy: root.download_policy)
@@ -59,6 +62,12 @@ module Katello
           end
         end
 
+        def sync_url_params(sync_options)
+          params = super
+          params[:optimize] = sync_options[:optimize] if sync_options.key?(:optimize)
+          params
+        end
+
         def self.distribution_bootable?(distribution)
           file_paths = distribution.results.first.images.map(&:path)
           file_paths.any? do |path|
@@ -70,48 +79,141 @@ module Katello
           "/pulp/repos/#{repo.relative_path}/".sub('//', '/')
         end
 
-        def copy_units(source_repository, content_unit_hrefs, dependency_solving, dest_base_version = 0,
-                       additional_repo_map = {})
+        def multi_copy_units(repo_id_map, dependency_solving)
           tasks = []
 
-          content_unit_hrefs.sort!
-          if content_unit_hrefs.any?
+          if repo_id_map.values.pluck(:content_unit_hrefs).flatten.any?
+            data = PulpRpmClient::Copy.new
+            data.dependency_solving = dependency_solving
+            data.config = []
+            repo_id_map.each do |source_repo_ids, dest_repo_id_map|
+              dest_repo = ::Katello::Repository.find(dest_repo_id_map[:dest_repo])
+              dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_master).repository_reference.repository_href
+              content_unit_hrefs = dest_repo_id_map[:content_unit_hrefs]
+              # Not needed during incremental update due to dest_base_version
+              unless dest_repo_id_map[:base_version]
+                source_repo_for_package_envs = ::Katello::Repository.find(source_repo_ids.first)
+                unless source_repo_for_package_envs.library_instance?
+                  source_repo_for_package_envs = source_repo_for_package_envs.library_instance
+                end
+                package_env_hrefs = packageenvironments({ :repository_version => source_repo_for_package_envs.version_href }).map(&:pulp_href).sort
+                tasks << remove_all_content_from_repo(dest_repo_href)
+                tasks << add_content_for_repo(dest_repo_href, package_env_hrefs) unless package_env_hrefs.empty?
+              end
+              source_repo_ids.each do |source_repo_id|
+                source_repo_version = ::Katello::Repository.find(source_repo_id).version_href
+                config = { source_repo_version: source_repo_version, dest_repo: dest_repo_href, content: content_unit_hrefs }
+                config[:dest_base_version] = dest_repo_id_map[:base_version] if dest_repo_id_map[:base_version]
+                data.config << config
+              end
+            end
+            tasks << copy_content_chunked(data)
+          else
+            tasks << remove_all_content_from_mapping(repo_id_map)
+          end
+          tasks.flatten
+        end
+
+        def copy_api_data_dup(data)
+          data_dup = PulpRpmClient::Copy.new
+          data_dup.dependency_solving = data.dependency_solving
+          data_dup.config = []
+          data.config.each do |repo_config|
+            config_hash = {
+              source_repo_version: repo_config[:source_repo_version],
+              dest_repo: repo_config[:dest_repo],
+              content: []
+            }
+            config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version]
+            data_dup.config << config_hash
+          end
+          data_dup
+        end
+
+        def copy_content_chunked(data)
+          tasks = []
+          # Don't chunk if there aren't enough content units
+          if data.config.sum { |repo_config| repo_config[:content].size } <= UNIT_LIMIT
+            return api.copy_api.copy_content(data)
+          end
+
+          unit_copy_counter = 0
+          i = 0
+          leftover_units = data.config.first[:content].deep_dup
+
+          # Copy data and clear its content fields
+          data_dup = copy_api_data_dup(data)
+
+          while i < data_dup.config.size
+            # Copy all units within repo or only some?
+            if leftover_units.length < UNIT_LIMIT - unit_copy_counter
+              copy_amount = leftover_units.length
+            else
+              copy_amount = UNIT_LIMIT - unit_copy_counter
+            end
+
+            data_dup.config[i][:content] = leftover_units.pop(copy_amount)
+            unit_copy_counter += copy_amount
+            # Do copy call if limit is reached or if we're under the limit but on the last repo config.
+            if unit_copy_counter >= UNIT_LIMIT || (i == data_dup.config.size - 1 && leftover_units.empty?)
+              tasks << api.copy_api.copy_content(data_dup)
+              unit_copy_counter = 0
+            end
+
+            if leftover_units.empty?
+              # Nothing more to copy -- clear current config's content
+              data_dup.config[i][:content] = []
+              i += 1
+              # Fetch unit list for next data config
+              leftover_units = data.config[i][:content].deep_dup unless i == data_dup.config.size
+            end
+          end
+
+          tasks
+        end
+
+        def remove_all_content_from_mapping(repo_id_map)
+          tasks = []
+          repo_id_map.each do |_source_repo_ids, dest_repo_id_map|
+            dest_repo = ::Katello::Repository.find(dest_repo_id_map[:dest_repo])
+            dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_master).repository_reference.repository_href
+            tasks << remove_all_content_from_repo(dest_repo_href)
+          end
+          tasks
+        end
+
+        def copy_units(source_repository, content_unit_hrefs, dependency_solving)
+          tasks = []
+
+          if content_unit_hrefs.sort!.any?
             data = PulpRpmClient::Copy.new
             data.config = [{
               source_repo_version: source_repository.version_href,
               dest_repo: repository_reference.repository_href,
-              dest_base_version: dest_base_version,
+              dest_base_version: 0,
               content: content_unit_hrefs
             }]
             data.dependency_solving = dependency_solving
-            if dependency_solving
-              # repo_map example: {
-              #   <source_repo_id>: {
-              #     dest_repo: <dest_repo_id>,
-              #     base_version: <base_version>
-              #   }
-              # }
-              additional_repo_map.each do |source_repo, dest_repo_map|
-                source_repo_version = ::Katello::Repository.find(source_repo).version_href
-
-                dest_repo = ::Katello::Repository.find(dest_repo_map[:dest_repo])
-                dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_master).repository_reference.repository_href
-                data.config << {
-                  source_repo_version: source_repo_version,
-                  dest_repo: dest_repo_href,
-                  dest_base_version: dest_repo_map[:base_version],
-                  content: content_unit_hrefs
-                }
-              end
-            end
+            package_env_hrefs = packageenvironments({ :repository_version => source_repository.version_href }).map(&:pulp_href).sort
             tasks << api.copy_api.copy_content(data)
+            tasks << add_content(package_env_hrefs) unless package_env_hrefs.empty?
           else
             tasks << remove_all_content
           end
-
           tasks
         end
 
+        def remove_all_content_from_repo(repo_href)
+          data = PulpRpmClient::RepositoryAddRemoveContent.new(
+            remove_content_units: ['*'])
+          api.repositories_api.modify(repo_href, data)
+        end
+
+        def repair(repository_version_href)
+          data = PulpRpmClient::RepositoryVersion.new
+          api.repository_versions_api.repair(repository_version_href, data)
+        end
+
         def remove_all_content
           data = PulpRpmClient::RepositoryAddRemoveContent.new(
             remove_content_units: ['*'])
@@ -119,7 +221,7 @@ module Katello
         end
 
         def packageenvironments(options = {})
-          api.content_package_environments_api.list(options)
+          Katello::Pulp3::Api::Core.fetch_from_list { |page_opts| api.content_package_environments_api.list(page_opts.merge(options)) }
         end
 
         def metadatafiles(options = {})
@@ -130,6 +232,94 @@ module Katello
           api.content_distribution_trees_api.list(options)
         end
 
+        def add_filter_content(source_repo_ids, filters, filter_list_map)
+          filters.each do |filter|
+            if filter.inclusion
+              source_repo_ids.each do |repo_id|
+                filter_list_map[:whitelist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id))
+              end
+            else
+              source_repo_ids.each do |repo_id|
+                filter_list_map[:blacklist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id))
+              end
+            end
+          end
+          filter_list_map
+        end
+
+        def add_un_modular_rpms(source_repo_ids, filters, filter_list_map)
+          if (filter_list_map[:whitelist_ids].empty? && filters.select { |filter| filter.inclusion }.empty?)
+            filter_list_map[:whitelist_ids] += source_repo_ids.collect do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              source_repo.rpms.where(:modular => false).pluck(:pulp_id).sort
+            end
+          end
+          filter_list_map
+        end
+
+        def add_modular_content(source_repo_ids, filters, modular_filters, filter_list_map)
+          inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion }
+          exclusion_modular_filters = modular_filters - inclusion_modular_filters
+          if inclusion_modular_filters.empty? &&
+              !(filters.any? { |filter| filter.class == ContentViewErratumFilter && filter.inclusion })
+            source_repo_ids.each do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              filter_list_map[:whitelist_ids] += source_repo.rpms.where(:modular => true).pluck(:pulp_id).sort
+              filter_list_map[:whitelist_ids] += source_repo.module_streams.pluck(:pulp_id).sort
+            end
+          end
+
+          unless inclusion_modular_filters.empty?
+            filter_list_map[:whitelist_ids] += source_repo_ids.collect do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              modular_packages(source_repo, inclusion_modular_filters)
+            end
+          end
+
+          unless exclusion_modular_filters.empty?
+            filter_list_map[:blacklist_ids] += source_repo_ids.collect do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              modular_packages(source_repo, exclusion_modular_filters)
+            end
+          end
+
+          filter_list_map
+        end
+
+        def copy_content_from_mapping(repo_id_map, options = {})
+          repo_id_map.each do |source_repo_ids, dest_repo_map|
+            filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
+              filter_class.where(:id => dest_repo_map[:filter_ids])
+            end
+            modular_filters = ContentViewModuleStreamFilter.where(:id => dest_repo_map[:filter_ids])
+            filters.flatten!.compact!
+
+            filter_list_map = { whitelist_ids: [], blacklist_ids: [] }
+            filter_list_map = add_filter_content(source_repo_ids, filters, filter_list_map)
+            filter_list_map = add_un_modular_rpms(source_repo_ids, filters, filter_list_map)
+            filter_list_map = add_modular_content(source_repo_ids, filters, modular_filters, filter_list_map)
+
+            whitelist_ids = filter_list_map[:whitelist_ids].flatten&.uniq
+            blacklist_ids = filter_list_map[:blacklist_ids].flatten&.uniq
+            content_unit_hrefs = whitelist_ids - blacklist_ids
+
+            if content_unit_hrefs.any?
+              source_repo_ids.each do |source_repo_id|
+                content_unit_hrefs += additional_content_hrefs(::Katello::Repository.find(source_repo_id), content_unit_hrefs)
+              end
+            end
+            source_repo_ids.each do |source_repo_id|
+              content_unit_hrefs += ::Katello::Repository.find(source_repo_id).srpms.pluck(:pulp_id)
+            end
+
+            dest_repo_map[:content_unit_hrefs] = content_unit_hrefs.uniq.sort
+          end
+
+          dependency_solving = options[:solve_dependencies] || false
+
+          multi_copy_units(repo_id_map, dependency_solving)
+        end
+
         def copy_content_for_source(source_repository, options = {})
           filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
             filter_class.where(:id => options[:filter_ids])
@@ -147,12 +337,18 @@ module Katello
             end
           end
 
-          if whitelist_ids.empty? && filters.select { |filter| filter.inclusion }.empty?
-            whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort
-          end
+          whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort if (whitelist_ids.empty? && filters.select { |filter| filter.inclusion }.empty?)
 
+          modular_filters = ContentViewModuleStreamFilter.where(:id => options[:filter_ids])
+          inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion }
+          exclusion_modular_filters = modular_filters - inclusion_modular_filters
+          if inclusion_modular_filters.empty? && !(filters.any? { |filter| filter.class == ContentViewErratumFilter && filter.inclusion })
+            whitelist_ids += source_repository.rpms.where(:modular => true).pluck(:pulp_id).sort
+            whitelist_ids += source_repository.module_streams.pluck(:pulp_id).sort
+          end
+          whitelist_ids += modular_packages(source_repository, inclusion_modular_filters) unless inclusion_modular_filters.empty?
+          blacklist_ids += modular_packages(source_repository, exclusion_modular_filters) unless exclusion_modular_filters.empty?
           content_unit_hrefs = whitelist_ids - blacklist_ids
-
           if content_unit_hrefs.any?
             content_unit_hrefs += additional_content_hrefs(source_repository, content_unit_hrefs)
           end
@@ -162,6 +358,14 @@ module Katello
           copy_units(source_repository, content_unit_hrefs.uniq, dependency_solving)
         end
 
+        def modular_packages(source_repository, filters)
+          list_ids = []
+          filters.each do |filter|
+            list_ids += filter.content_unit_pulp_ids(source_repository, true)
+          end
+          list_ids
+        end
+
         def additional_content_hrefs(source_repository, content_unit_hrefs)
           repo_service = source_repository.backend_service(SmartProxy.pulp_master)
           options = { :repository_version => source_repository.version_href }
@@ -172,10 +376,6 @@ module Katello
           package_groups_to_include = filter_package_groups_by_pulp_href(source_repository.package_groups, content_unit_hrefs)
           content_unit_hrefs += package_groups_to_include.pluck(:pulp_id)
 
-          package_environment_hrefs_to_include = filter_package_environments_by_pulp_hrefs(
-            repo_service.packageenvironments(options).results, package_groups_to_include.pluck(:pulp_id))
-          content_unit_hrefs += package_environment_hrefs_to_include
-
           metadata_file_hrefs_to_include = filter_metadatafiles_by_pulp_hrefs(
             repo_service.metadatafiles(options).results, content_unit_hrefs)
           content_unit_hrefs += metadata_file_hrefs_to_include