katello 3.16.0.rc3.1 → 3.16.0

data/app/services/katello/pulp3/erratum.rb

@@ -26,8 +26,10 @@ module Katello
       end
 
       def update_model(model)
-        keys = %w(title id severity issued type description reboot_suggested solution updated summary)
+        keys = %w(title id severity issued_date type description reboot_suggested solution updated_date summary)
         custom_json = backend_data.slice(*keys)
+        custom_json["issued"] = custom_json.delete("issued_date")
+        custom_json["updated"] = custom_json.delete("updated_date")
 
         # handle SUSE epoch dates
         custom_json["issued"] = convert_date_if_epoch(custom_json["issued"])

data/app/services/katello/pulp3/migration.rb

@@ -35,11 +35,15 @@ module Katello
       end
 
       def pulp2_content_api
-        Pulp2to3MigrationClient::Pulp2contentApi.new(api_client)
+        Pulp2to3MigrationClient::Pulp2ContentApi.new(api_client)
+      rescue NameError
+        Pulp2to3MigrationClient::Pulp2contentApi.new(api_client) #backwards compatible
       end
 
       def pulp2_repositories_api
-        Pulp2to3MigrationClient::Pulp2repositoriesApi.new(api_client)
+        Pulp2to3MigrationClient::Pulp2RepositoriesApi.new(api_client)
+      rescue NameError
+        Pulp2to3MigrationClient::Pulp2repositoriesApi.new(api_client) #backwards compatible
       end
 
       def create_and_run_migrations
@@ -64,8 +68,9 @@ module Katello
       end
 
       def create_migrations
-        migration_plans = @repository_types.map { |label| Katello::Pulp3::MigrationPlan.new(label) }
-        migration_plans.map { |plan| migration_plan_api.create(plan: plan.generate.as_json).pulp_href }
+        plan = Katello::Pulp3::MigrationPlan.new(@repository_types)
+        Rails.logger.info(plan)
+        [migration_plan_api.create(plan: plan.generate.as_json).pulp_href]
       end
 
       def start_migration(plan_href)

data/app/services/katello/pulp3/migration_plan.rb

@@ -1,8 +1,8 @@
 module Katello
   module Pulp3
     class MigrationPlan
-      def initialize(repository_type_label)
-        @repository_type = repository_type_label
+      def initialize(repository_type_labels)
+        @repository_types = repository_type_labels
       end
 
       def master_proxy
@@ -17,12 +17,12 @@ module Katello
       end
 
       def generate_plugins
-        [
+        @repository_types.map do |repository_type|
           {
-            type: pulp2_repository_type(@repository_type),
-            repositories: repository_migrations(@repository_type)
+            type: pulp2_repository_type(repository_type),
+            repositories: repository_migrations(repository_type)
           }
-        ]
+        end
       end
 
       def pulp2_repository_type(repository_type)

data/app/services/katello/pulp3/repository.rb

@@ -48,6 +48,10 @@ module Katello
         nil
       end
 
+      def skip_types
+        nil
+      end
+
       def content_service
         Katello::Pulp3::Content
       end
@@ -189,8 +193,15 @@ module Katello
         ignore_404_exception { api.repositories_api.delete(href) } if href
       end
 
+      def sync_params
+        {remote: repo.remote_href, mirror: repo.root.mirror_on_sync}
+      end
+
       def sync
-        repository_sync_url_data = api.class.repository_sync_url_class.new(remote: repo.remote_href, mirror: repo.root.mirror_on_sync)
+        sync_url_params = sync_params
+        skip_type_param = skip_types
+        sync_url_params[:skip_types] = skip_type_param if skip_type_param
+        repository_sync_url_data = api.class.repository_sync_url_class.new(sync_url_params)
         [api.repositories_api.sync(repository_reference.repository_href, repository_sync_url_data)]
       end
 
@@ -345,6 +356,11 @@ module Katello
         api.repositories_api.modify(repository_reference.repository_href, add_content_units: content_unit_href)
       end
 
+      def add_content_for_repo(repository_href, content_unit_href)
+        content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
+        api.repositories_api.modify(repository_href, add_content_units: content_unit_href)
+      end
+
       def unit_keys(uploads)
         uploads.map do |upload|
           upload.except('id')

data/app/services/katello/pulp3/repository/yum.rb

@@ -15,6 +15,14 @@ module Katello
           end
         end
 
+        def skip_types
+          skip_types = []
+          if root.ignorable_content.try(:include?, "srpm")
+            skip_types << "srpm"
+          end
+          skip_types
+        end
+
         def distribution_options(path)
           {
             base_path: path,
@@ -23,6 +31,10 @@ module Katello
           }
         end
 
+        def sync_params
+          {remote: repo.remote_href, mirror: repo.root.mirror_on_sync, optimize: false}
+        end
+
         def mirror_remote_options
           policy = smart_proxy.download_policy
 
@@ -62,30 +74,87 @@ module Katello
           "/pulp/repos/#{repo.relative_path}/".sub('//', '/')
         end
 
-        def copy_units(source_repository_version_href, content_unit_hrefs, dependency_solving)
+        def multi_copy_units(repo_id_map, dependency_solving)
           tasks = []
 
-          if content_unit_hrefs.any?
+          if repo_id_map.values.pluck(:content_unit_hrefs).flatten.any?
+            data = PulpRpmClient::Copy.new
+            data.dependency_solving = dependency_solving
+            data.config = []
+            repo_id_map.each do |source_repo_ids, dest_repo_id_map|
+              dest_repo = ::Katello::Repository.find(dest_repo_id_map[:dest_repo])
+              dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_master).repository_reference.repository_href
+              content_unit_hrefs = dest_repo_id_map[:content_unit_hrefs]
+              # Not needed during incremental update due to dest_base_version
+              unless dest_repo_id_map[:base_version]
+                source_repo_for_package_envs = ::Katello::Repository.find(source_repo_ids.first)
+                unless source_repo_for_package_envs.library_instance?
+                  source_repo_for_package_envs = source_repo_for_package_envs.library_instance
+                end
+                package_env_hrefs = packageenvironments({ :repository_version => source_repo_for_package_envs.version_href }).map(&:pulp_href).sort
+                tasks << remove_all_content_from_repo(dest_repo_href)
+                tasks << add_content_for_repo(dest_repo_href, package_env_hrefs) unless package_env_hrefs.empty?
+              end
+              source_repo_ids.each do |source_repo_id|
+                source_repo_version = ::Katello::Repository.find(source_repo_id).version_href
+                config = { source_repo_version: source_repo_version, dest_repo: dest_repo_href, content: content_unit_hrefs }
+                config[:dest_base_version] = dest_repo_id_map[:base_version] if dest_repo_id_map[:base_version]
+                data.config << config
+              end
+            end
+            # FIXME: data's content being [] causes all content to be copied back
+            tasks << api.copy_api.copy_content(data)
+          else
+            tasks << remove_all_content_from_mapping(repo_id_map)
+          end
+          tasks.flatten
+        end
+
+        def remove_all_content_from_mapping(repo_id_map)
+          tasks = []
+          repo_id_map.each do |_source_repo_ids, dest_repo_id_map|
+            dest_repo = ::Katello::Repository.find(dest_repo_id_map[:dest_repo])
+            dest_repo_href = ::Katello::Pulp3::Repository::Yum.new(dest_repo, SmartProxy.pulp_master).repository_reference.repository_href
+            tasks << remove_all_content_from_repo(dest_repo_href)
+          end
+          tasks
+        end
+
+        def copy_units(source_repository, content_unit_hrefs, dependency_solving)
+          tasks = []
+
+          if content_unit_hrefs.sort!.any?
             data = PulpRpmClient::Copy.new
-            data.config = [
-              source_repo_version: source_repository_version_href,
+            data.config = [{
+              source_repo_version: source_repository.version_href,
               dest_repo: repository_reference.repository_href,
               dest_base_version: 0,
-              content: content_unit_hrefs.sort
-            ]
+              content: content_unit_hrefs
+            }]
             data.dependency_solving = dependency_solving
+            package_env_hrefs = packageenvironments({ :repository_version => source_repository.version_href }).map(&:pulp_href).sort
             tasks << api.copy_api.copy_content(data)
+            tasks << add_content(package_env_hrefs) unless package_env_hrefs.empty?
           else
-            data = PulpRpmClient::RepositoryAddRemoveContent.new(
-              remove_content_units: ['*'])
-            tasks << api.repositories_api.modify(repository_reference.repository_href, data)
+            tasks << remove_all_content
           end
-
           tasks
         end
 
+        def remove_all_content_from_repo(repo_href)
+          data = PulpRpmClient::RepositoryAddRemoveContent.new(
+            remove_content_units: ['*'])
+          api.repositories_api.modify(repo_href, data)
+        end
+
+        def remove_all_content
+          data = PulpRpmClient::RepositoryAddRemoveContent.new(
+            remove_content_units: ['*'])
+          api.repositories_api.modify(repository_reference.repository_href, data)
+        end
+
         def packageenvironments(options = {})
-          api.content_package_environments_api.list(options)
+          Katello::Pulp3::Api::Core.fetch_from_list { |page_opts| api.content_package_environments_api.list(page_opts.merge(options)) }
         end
 
         def metadatafiles(options = {})
@@ -96,6 +165,94 @@ module Katello
           api.content_distribution_trees_api.list(options)
         end
 
+        def add_filter_content(source_repo_ids, filters, filter_list_map)
+          filters.each do |filter|
+            if filter.inclusion
+              source_repo_ids.each do |repo_id|
+                filter_list_map[:whitelist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id))
+              end
+            else
+              source_repo_ids.each do |repo_id|
+                filter_list_map[:blacklist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id))
+              end
+            end
+          end
+          filter_list_map
+        end
+
+        def add_un_modular_rpms(source_repo_ids, filters, filter_list_map)
+          if (filter_list_map[:whitelist_ids].empty? && filters.select { |filter| filter.inclusion }.empty?)
+            filter_list_map[:whitelist_ids] += source_repo_ids.collect do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              source_repo.rpms.where(:modular => false).pluck(:pulp_id).sort
+            end
+          end
+          filter_list_map
+        end
+
+        def add_modular_content(source_repo_ids, filters, modular_filters, filter_list_map)
+          inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion }
+          exclusion_modular_filters = modular_filters - inclusion_modular_filters
+          if inclusion_modular_filters.empty? &&
+              !(filters.any? { |filter| filter.class == ContentViewErratumFilter && filter.inclusion })
+            source_repo_ids.each do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              filter_list_map[:whitelist_ids] += source_repo.rpms.where(:modular => true).pluck(:pulp_id).sort
+              filter_list_map[:whitelist_ids] += source_repo.module_streams.pluck(:pulp_id).sort
+            end
+          end
+
+          unless inclusion_modular_filters.empty?
+            filter_list_map[:whitelist_ids] += source_repo_ids.collect do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              modular_packages(source_repo, inclusion_modular_filters)
+            end
+          end
+
+          unless exclusion_modular_filters.empty?
+            filter_list_map[:blacklist_ids] += source_repo_ids.collect do |source_repo_id|
+              source_repo = ::Katello::Repository.find(source_repo_id)
+              modular_packages(source_repo, exclusion_modular_filters)
+            end
+          end
+
+          filter_list_map
+        end
+
+        def copy_content_from_mapping(repo_id_map, options = {})
+          repo_id_map.each do |source_repo_ids, dest_repo_map|
+            filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
+              filter_class.where(:id => dest_repo_map[:filter_ids])
+            end
+            modular_filters = ContentViewModuleStreamFilter.where(:id => dest_repo_map[:filter_ids])
+            filters.flatten!.compact!
+
+            filter_list_map = { whitelist_ids: [], blacklist_ids: [] }
+            filter_list_map = add_filter_content(source_repo_ids, filters, filter_list_map)
+            filter_list_map = add_un_modular_rpms(source_repo_ids, filters, filter_list_map)
+            filter_list_map = add_modular_content(source_repo_ids, filters, modular_filters, filter_list_map)
+
+            whitelist_ids = filter_list_map[:whitelist_ids].flatten&.uniq
+            blacklist_ids = filter_list_map[:blacklist_ids].flatten&.uniq
+            content_unit_hrefs = whitelist_ids - blacklist_ids
+
+            if content_unit_hrefs.any?
+              source_repo_ids.each do |source_repo_id|
+                content_unit_hrefs += additional_content_hrefs(::Katello::Repository.find(source_repo_id), content_unit_hrefs)
+              end
+            end
+            source_repo_ids.each do |source_repo_id|
+              content_unit_hrefs += ::Katello::Repository.find(source_repo_id).srpms.pluck(:pulp_id)
+            end
+
+            dest_repo_map[:content_unit_hrefs] = content_unit_hrefs.uniq.sort
+          end
+
+          dependency_solving = options[:solve_dependencies] || false
+
+          multi_copy_units(repo_id_map, dependency_solving)
+        end
+
         def copy_content_for_source(source_repository, options = {})
           filters = [ContentViewErratumFilter, ContentViewPackageGroupFilter, ContentViewPackageFilter].collect do |filter_class|
             filter_class.where(:id => options[:filter_ids])
@@ -113,19 +270,33 @@ module Katello
             end
           end
 
-          if whitelist_ids.empty? && filters.select { |filter| filter.inclusion }.empty?
-            whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort
-          end
+          whitelist_ids = source_repository.rpms.where(:modular => false).pluck(:pulp_id).sort if (whitelist_ids.empty? && filters.select { |filter| filter.inclusion }.empty?)
 
+          modular_filters = ContentViewModuleStreamFilter.where(:id => options[:filter_ids])
+          inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion }
+          exclusion_modular_filters = modular_filters - inclusion_modular_filters
+          if inclusion_modular_filters.empty? && !(filters.any? { |filter| filter.class == ContentViewErratumFilter && filter.inclusion })
+            whitelist_ids += source_repository.rpms.where(:modular => true).pluck(:pulp_id).sort
+            whitelist_ids += source_repository.module_streams.pluck(:pulp_id).sort
+          end
+          whitelist_ids += modular_packages(source_repository, inclusion_modular_filters) unless inclusion_modular_filters.empty?
+          blacklist_ids += modular_packages(source_repository, exclusion_modular_filters) unless exclusion_modular_filters.empty?
           content_unit_hrefs = whitelist_ids - blacklist_ids
-
           if content_unit_hrefs.any?
             content_unit_hrefs += additional_content_hrefs(source_repository, content_unit_hrefs)
           end
           content_unit_hrefs += source_repository.srpms.pluck(:pulp_id)
 
           dependency_solving = options[:solve_dependencies] || false
-          copy_units(source_repository.version_href, content_unit_hrefs.uniq, dependency_solving)
+          copy_units(source_repository, content_unit_hrefs.uniq, dependency_solving)
+        end
+
+        def modular_packages(source_repository, filters)
+          list_ids = []
+          filters.each do |filter|
+            list_ids += filter.content_unit_pulp_ids(source_repository, true)
+          end
+          list_ids
         end
 
         def additional_content_hrefs(source_repository, content_unit_hrefs)
@@ -138,10 +309,6 @@ module Katello
           package_groups_to_include = filter_package_groups_by_pulp_href(source_repository.package_groups, content_unit_hrefs)
           content_unit_hrefs += package_groups_to_include.pluck(:pulp_id)
 
-          package_environment_hrefs_to_include = filter_package_environments_by_pulp_hrefs(
-            repo_service.packageenvironments(options).results, package_groups_to_include.pluck(:pulp_id))
-          content_unit_hrefs += package_environment_hrefs_to_include
-
           metadata_file_hrefs_to_include = filter_metadatafiles_by_pulp_hrefs(
             repo_service.metadatafiles(options).results, content_unit_hrefs)
           content_unit_hrefs += metadata_file_hrefs_to_include
@@ -150,11 +317,6 @@ module Katello
             repo_service.distributiontrees(options).results, content_unit_hrefs)
           content_unit_hrefs + distribution_tree_hrefs_to_include
         end
-
-        def regenerate_applicability
-          # TODO
-          fail NotImplementedError
-        end
       end
     end
   end

data/app/services/katello/pulp3/task.rb

@@ -0,0 +1,100 @@
+module Katello
+  module Pulp3
+    class Task
+      # A call report Looks like:  {"task":"/pulp/api/v3/tasks/5/"}
+      # {
+      #    "pulp_href":"/pulp/api/v3/tasks/4/",
+      #    "pulp_created":"2019-02-21T19:50:40.476767Z",
+      #    "job_id":"d0359658-d926-47a2-b430-1b2092b3bd86",
+      #    "state":"completed",
+      #    "name":"pulp_file.app.tasks.publishing.publish",
+      #    "started_at":"2019-02-21T19:50:40.556002Z",
+      #    "finished_at":"2019-02-21T19:50:40.618397Z",
+      #    "non_fatal_errors":[
+      #
+      #    ],
+      #    "error":null,
+      #    "worker":"/pulp/api/v3/workers/1/",
+      #    "parent":null,
+      #    "spawned_tasks":[
+      #
+      #    ],
+      #    "progress_reports":[
+      #
+      #    ],
+      #    "created_resources":[
+      #       "/pulp/api/v3/publications/1/"
+      #    ]
+      # }
+
+      WAITING = 'waiting'.freeze
+      SKIPPED = 'skipped'.freeze
+      RUNNING = 'running'.freeze
+      COMPLETED = 'completed'.freeze
+      FAILED = 'failed'.freeze
+      CANCELED = 'canceled'.freeze
+
+      FINISHED_STATES = [COMPLETED, FAILED, CANCELED, SKIPPED].freeze
+
+      #needed for serialization in dynflow
+
+      delegate :[], :key?, :dig, :to_hash, :to => :task_data
+
+      def initialize(smart_proxy, data)
+        @smart_proxy = smart_proxy
+        if (href = data['task'])
+          @href = href
+        else
+          @pulp_data = data.with_indifferent_access
+          @href = @pulp_data['pulp_href']
+          Rails.logger.error("Got empty pulp_href on #{@pulp_data}") if @href.nil?
+        end
+      end
+
+      def task_data(force_refresh = false)
+        @pulp_data = nil if force_refresh
+        @pulp_data ||= tasks_api.read(@href).as_json.with_indifferent_access
+      end
+
+      def tasks_api
+        ::Katello::Pulp3::Api::Core.new(@smart_proxy).tasks_api
+      end
+
+      def task_group_href
+        task_data[:task_group]
+      end
+
+      def done?
+        task_data[:finish_time] || FINISHED_STATES.include?(task_data[:state])
+      end
+
+      def poll
+        task_data(true)
+        self
+      end
+
+      def started?
+        task_data[:start_time]
+      end
+
+      def error
+        if task_data[:state] == CANCELED
+          self.new(_("Task canceled"))
+        elsif task_data[:state] == FAILED
+          if task_data[:error][:description].blank?
+            _("Pulp task error")
+          else
+            task_data[:error][:description]
+          end
+        end
+      end
+
+      def cancel
+        data = PulpcoreClient::Task.new(state: 'canceled')
+        tasks_api.tasks_cancel(pulp_task['pulp_href'], data)
+        #the main task may have completed, so cancel spawned tasks too
+        task_data['spawned_tasks']&.each { |spawned| tasks_api.tasks_cancel(spawned['pulp_href'], data) }
+      end
+    end
+  end
+end