katello 3.18.0.rc2.1 → 4.0.0.rc2

data/app/services/katello/event_monitor/poller_thread.rb

@@ -1,8 +1,8 @@
 module Katello
   module EventMonitor
+    # TODO: Move this class to app/lib/katello/event_daemon/services with other service definitions
     class PollerThread
       SLEEP_INTERVAL = 3
-      STATUS_CACHE_KEY = 'katello_events_status'.freeze
 
       cattr_accessor :instance
 
@@ -15,7 +15,6 @@ module Katello
           self.instance.close
           self.instance = nil
         end
-        reset_status
       end
 
       def self.run
@@ -24,14 +23,8 @@ module Katello
         instance.poll_for_events
       end
 
-      def self.status(refresh: true)
-        Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-          instance&.status
-        end
-      end
-
-      def self.reset_status
-        Rails.cache.delete(STATUS_CACHE_KEY)
+      def self.status
+        instance&.status
       end
 
       def initialize(logger = nil)
@@ -85,7 +78,6 @@ module Katello
       end
 
       def poll_for_events
-        @thread&.kill
         @thread = Thread.new do
           @logger.info("Polling Katello Event Queue")
           loop do

data/app/services/katello/pulp/smart_proxy_repository.rb

@@ -42,21 +42,6 @@ module Katello
         puppet_repos.where(:pulp_id => pulp_repos.map { |pulp_repo| pulp_repo['id'] })
       end
 
-      def current_repositories_data(environment = nil, content_view = nil)
-        @pulp_repositories ||= smart_proxy.pulp_repositories
-
-        repos = Katello::Repository
-        repos = repos.in_environment(environment) if environment
-        repos = repos.in_content_views([content_view]) if content_view
-        puppet_envs = Katello::ContentViewPuppetEnvironment
-        puppet_envs = puppet_envs.in_environment(environment) if environment
-        puppet_envs = puppet_envs.in_content_view(content_view) if content_view
-
-        repo_ids = repos.pluck(:pulp_id) + puppet_envs.pluck(:pulp_id)
-
-        @pulp_repositories.select { |r| repo_ids.include?(r['id']) }
-      end
-
       def orphaned_repos
         @smart_proxy.pulp_repositories.map { |x| x["id"] } - repos_available_to_capsule.map { |x| x.pulp_id }
       end

data/app/services/katello/pulp3/api/docker.rb

@@ -32,6 +32,10 @@ module Katello
           PulpContainerClient::RecursiveManage
         end
 
+        def self.tag_image_class
+          PulpContainerClient::TagImage
+        end
+
         def api_client
           PulpContainerClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpContainerClient::Configuration))
         end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -4,13 +4,37 @@ module Katello
       class Export
         include ImportExportCommon
 
-        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil, from_content_view_version: nil)
+        def initialize(smart_proxy:,
+                        content_view_version: nil,
+                        destination_server: nil,
+                        from_content_view_version: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
           @from_content_view_version = from_content_view_version
         end
 
+        def repository_hrefs
+          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
+        end
+
+        def version_hrefs
+          repositories.pluck(:version_href).compact
+        end
+
+        def repositories(fetch_all: false)
+          repos = if @content_view_version.default?
+                    @content_view_version.repositories.yum_type
+                  else
+                    @content_view_version.archived_repos.yum_type
+                  end
+          if fetch_all
+            repos
+          else
+            repos.immediate
+          end
+        end
+
         def generate_exporter_path
           export_path = "#{@content_view_version.content_view}/#{@content_view_version.version}/#{@destination_server}/#{date_dir}".gsub(/\s/, '_')
           "#{@content_view_version.organization.label}/#{export_path}"
@@ -66,12 +90,29 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
+        def validate!(fail_on_missing_content: true, validate_incremental: true)
+          validate_repositories_immediate! if fail_on_missing_content
+          validate_incremental_export! if validate_incremental && !@from_content_view_version.blank?
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).non_immediate
+          if non_immediate_repos.any?
+            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: @content_view_version.content_view.name,
+                     current: @content_view_version.version,
+                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
         def validate_incremental_export!
-          return if @from_content_view_version.blank?
           from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
 
-          from_exporter_repos = generate_repo_mapping(from_exporter.repositories)
-          to_exporter_repos = generate_repo_mapping(repositories)
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
 
           invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
             from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
@@ -99,7 +140,8 @@ module Katello
           ret = { organization: @content_view_version.organization.name,
                   repository_mapping: {},
                   content_view: @content_view_version.content_view.name,
-                  content_view_version: @content_view_version.slice(:major, :minor)
+                  content_view_version: @content_view_version.slice(:major, :minor),
+                  incremental: @from_content_view_version.present?
           }
 
           unless @from_content_view_version.blank?
@@ -120,6 +162,22 @@ module Katello
           end
           ret
         end
+
+        def self.find_library_export_view(create_by_default: false,
+                                          destination_server:,
+                                          organization:)
+          name = "Export-Library"
+          name += "-#{destination_server}" unless destination_server.blank?
+          select_method = create_by_default ? :first_or_create : :first
+          ::Katello::ContentView.where(name: name, organization: organization).send(select_method)
+        end
+
+        def self.generate_product_repo_strings(repositories:)
+          repositories.map do |repo|
+            _("Product: '%{product}', Repository: '%{repository}'" % { product: repo.product.name,
+                                                                       repository: repo.name})
+          end
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -46,6 +46,46 @@ module Katello
         def self.check!(content_view:, metadata:, path:)
           ImportValidator.new(content_view: content_view, metadata: metadata, path: path).check!
         end
+
+        def self.reset_content_view_repositories_from_metadata!(content_view:, metadata:)
+          # Given metadata from the dump and a content view
+          # this method
+          # 1) Fetches ids of the library repos whose product name, repo name amd redhat?
+          # =>  match values provided in the metadata's repository mapping
+          # 2) Removes all the repositories associated to this content view
+          # 3) Adds the repositories matched from the dump
+          # The main intent of this method is to assume that the user intends for the
+          # content view to exaclty look like what is specified in metadata
+
+          repos_in_library = Katello::Repository.
+                    in_default_view.
+                    yum_type.
+                    joins(:product => :provider, :content_view_version => :content_view).
+                    joins(:root).
+                    where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                    pluck("#{::Katello::Repository.table_name}.id",
+                          "#{::Katello::RootRepository.table_name}.name",
+                          "#{::Katello::Product.table_name}.name",
+                          "#{::Katello::Provider.table_name}.provider_type"
+                          )
+          repos_in_library_map = {}
+          # repos_in_library_map is going to look like {['repo1', 'product1', false] => 100, ['repo1', 'product1', true] => 200 }
+          repos_in_library.each do |id, repo, product, provider_type|
+            repos_in_library_map[[repo, product, provider_type == Katello::Provider::REDHAT]] = id
+          end
+
+          repo_ids = metadata[:repository_mapping].values.map do |repo|
+            repos_in_library_map[[repo[:repository], repo[:product], repo[:redhat]]]
+          end
+          content_view.update!(repository_ids: repo_ids)
+        end
+
+        def self.find_or_create_library_import_view(organization)
+          name = ::Katello::ContentView::IMPORT_LIBRARY
+          ::Katello::ContentView.where(name: name,
+                                       organization: organization,
+                                       import_only: true).first_or_create
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -19,22 +19,6 @@ module Katello
           repo_api.read(version_href_to_repository_href(version_href))
         end
 
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          repositories.pluck(:version_href).compact
-        end
-
-        def repositories
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type
-          else
-            @content_view_version.archived_repos.yum_type
-          end
-        end
-
         def version_href_to_repository_href(version_href)
           version_href.split("/")[0..-3].join("/") + "/"
         end

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -2,7 +2,6 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportValidator
-        BASEDIR = '/var/lib/pulp'.freeze
         attr_accessor :metadata, :path, :content_view
         def initialize(content_view:, path:, metadata:)
           self.content_view = content_view
@@ -11,10 +10,11 @@ module Katello
         end
 
         def check!
-          check_permissions!
-          ensure_importing_cvv_does_not_exist!
-          ensure_from_cvv_exists!
-          ensure_repositories_metadata_and_content_view_match!
+          unless content_view.default?
+            ensure_importing_cvv_does_not_exist!
+            ensure_from_cvv_exists!
+          end
+          ensure_repositories_metadata_are_in_the_library!
         end
 
         def ensure_importing_cvv_does_not_exist!
@@ -43,55 +43,32 @@ module Katello
           end
         end
 
-        def ensure_repositories_metadata_and_content_view_match!
-          product_repos_in_content_view = content_view.repositories.yum_type.map { |repo| [repo.product.name, repo.name, repo.redhat?] }
+        def ensure_repositories_metadata_are_in_the_library!
+          repos_in_library = Katello::Repository.
+                              in_default_view.
+                              yum_type.
+                              joins(:product => :provider, :content_view_version => :content_view).
+                              joins(:root).
+                              where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                              pluck("#{::Katello::Product.table_name}.name",
+                                    "#{::Katello::RootRepository.table_name}.name",
+                                    "#{::Katello::Provider.table_name}.provider_type"
+                                    )
+
+          # repos_in_library look like [["prod1", "repo1", "Anonymous"], ["prod2", "repo2", "Red Hat"]]
+          product_repos_in_library = repos_in_library.map { |product, repo, provider| [product, repo, provider == ::Katello::Provider::REDHAT] }
           product_repos_in_metadata = metadata[:repository_mapping].values.map { |repo| [repo[:product], repo[:repository], repo[:redhat]] }
-
-          product_repos_in_content_view.sort!
-          product_repos_in_metadata.sort!
-          # product_repos_in_content_view & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
-
-          if product_repos_in_content_view != product_repos_in_metadata
-            repos_in_content_view = generate_product_repo_i18n_string(product_repos_in_content_view)
-            repos_in_import = generate_product_repo_i18n_string(product_repos_in_metadata)
-
-            fail _("Repositories in the importing content view do not match the repositories provided in the import metadata.\n "\
-                    "Repositories in Content View '%{content_view}': %{repos_in_content_view}\n "\
-                    "Repositories in the Import Metadata: %{repos_in_import}" % { content_view: content_view.name,
-                                                                                  repos_in_content_view: repos_in_content_view.join(""),
-                                                                                  repos_in_import: repos_in_import.join("")}
+          # product_repos_in_library & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos_not_in_library = product_repos_in_metadata - product_repos_in_library
+          unless product_repos_not_in_library.blank?
+            repos_in_import = generate_product_repo_i18n_string(product_repos_not_in_library)
+            fail _("The following repositories provided in the import metadata are either not available in the Library or are of incorrect Respository Type. "\
+                    "Please add or enable the repositories before importing\n "\
+                    "%{repos}" % { content_view: content_view.name, repos: repos_in_import.join("")}
                   )
           end
         end
 
-        def check_permissions!
-          fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
-          fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
-          fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
-
-          Dir.glob("#{path}/*").each do |file|
-            fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
-          end
-          toc_path = "#{path}/#{metadata[:toc]}"
-          fail _("The TOC file specified in the metadata does not exist. %s " % toc_path) unless File.exist?(toc_path)
-        end
-
-        def pulp_user_accessible?(path)
-          pulp_info = fetch_pulp_user_info
-          return false if pulp_info.blank?
-
-          stat = File.stat(path)
-          stat.gid.to_s == pulp_info.gid ||
-            stat.uid.to_s == pulp_info.uid ||
-            stat.mode.to_s(8)[-1].to_i >= 4
-        end
-
-        def fetch_pulp_user_info
-          pulp_user = nil
-          Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
-          pulp_user
-        end
-
         def generate_product_repo_i18n_string(product_repos)
           # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
           product_repos.map do |product, repo, redhat|

data/app/services/katello/pulp3/docker_manifest.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerManifest < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_manifest".freeze
 
       def self.content_api
         PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/docker_tag.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class DockerTag < PulpContentUnit
       include LazyAccessor
+      CONTENT_TYPE = "docker_tag".freeze
 
       def self.content_api
         PulpContainerClient::ContentTagsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

data/app/services/katello/pulp3/migration.rb

@@ -3,7 +3,7 @@ require 'pulp_2to3_migration_client'
 module Katello
   module Pulp3
     class Migration
-      attr_accessor :smart_proxy, :reimport_all
+      attr_accessor :smart_proxy, :reimport_all, :task_id
       GET_QUERY_ID_LENGTH = 90
 
       MUTABLE_CONTENT_TYPES = [
@@ -21,8 +21,10 @@ module Katello
         overridden.select { |type| SmartProxy.pulp_primary.pulp3_repository_type_support?(type.to_s, false) }.map { |t| t.to_s }
       end
 
-      def initialize(smart_proxy, repository_types = Migration.repository_types_for_migration, options = {})
-        self.reimport_all = options.fetch(:reimport, false)
+      def initialize(smart_proxy, options = {})
+        self.task_id = options.fetch(:task_id, nil)
+        self.reimport_all = options.fetch(:reimport_all, false)
+        repository_types = options.fetch(:repository_types, Migration.repository_types_for_migration)
 
         if (repository_types - smart_proxy.supported_pulp_types[:pulp3][:overriden_to_pulp2]).any?
           fail ::Katello::Errors::Pulp3MigrationError, _("Pulp 3 migration cannot run. Types %s have already been migrated.") %
@@ -62,6 +64,15 @@ module Katello
         [YumMetadataFile]
       end
 
+      def last_successful_migration_time
+        task = ForemanTasks::Task.where(:label => Actions::Pulp3::ContentMigration.to_s, :result => 'success').order("started_at desc").first
+        if reimport_all || task.nil?
+          0
+        else
+          task.started_at.to_i
+        end
+      end
+
       def content_types_for_migration
         content_types = @repository_types.collect do |repository_type_label|
           Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index
@@ -70,7 +81,20 @@ module Katello
         content_types.flatten - Migration.ignorable_content_types
       end
 
+      def update_import_status(message, index = nil)
+        #reduce output updating, only update every 20 items
+        if (index.nil? || index % 20 == 0) && self.task_id
+          progress = Katello::ContentMigrationProgress.find_or_create_by(:task_id => self.task_id)
+          progress.update(:progress_message => message)
+          progress.save!
+
+          fail Katello::Errors::Pulp3MigrationError, "Cancelled by user." if progress.canceled?
+        end
+      end
+
       def import_pulp3_content
+        update_import_status("Starting katello import phase.")
+
         Katello::Logging.time("CONTENT_MIGRATION - Total Import Process") do
           @repository_types.each do |repository_type_label|
             Katello::Logging.time("CONTENT_MIGRATION - Importing Repository", data: {type: repository_type_label}) do
@@ -78,7 +102,7 @@ module Katello
             end
 
             Katello::RepositoryTypeManager.repository_types[repository_type_label].content_types_to_index.each do |content_type|
-              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type}) do
+              Katello::Logging.time("CONTENT_MIGRATION - Importing Content", data: {type: content_type.label}) do
                 import_content_type(content_type)
               end
             end
@@ -114,7 +138,9 @@ module Katello
         if repository_type_label == 'yum'
           import_yum_repos(imported, katello_repos)
         else
-          katello_repos.each do |repo|
+          repo_count = katello_repos.count
+          katello_repos.each_with_index do |repo, index|
+            update_import_status("Importing migrated content units #{repository_type_label}: #{index + 1}/#{repo_count}", index)
             found = imported.find { |migrated_repo| migrated_repo.pulp2_repo_id == repo.pulp_id }
             import_repo(repo, found) if found
           end
@@ -122,7 +148,9 @@ module Katello
       end
 
       def import_yum_repos(migrated_repo_items, repos)
-        repos.each do |yum_repo|
+        repo_count = repos.count
+        repos.each_with_index do |yum_repo, index|
+          update_import_status("Importing migrated yum repositories: #{index + 1}/#{repo_count}", index)
           to_find = nil
           if yum_repo.content_view.composite?
             if yum_repo.link?
@@ -195,14 +223,16 @@ module Katello
       end
 
       def operate_on_errata
+        last_migration_time = last_successful_migration_time
         offset = 0
-        limit = 300
-        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+        limit = SETTINGS[:katello][:pulp][:bulk_load_size]
+        response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
         total_count = response.count
         yield(response.results)
         until (offset + limit > total_count)
           offset += limit
-          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit)
+          response = pulp2_content_api.list(pulp2_content_type_id: 'erratum', offset: offset, limit: limit, pulp2_last_updated__gt: last_migration_time)
+          update_import_status("Importing migrated content type erratum: #{offset + limit}/#{total_count}")
           yield(response.results)
         end
       end
@@ -242,7 +272,12 @@ module Katello
             unmigrated_units = unmigrated_units.where(:migrated_pulp3_href => nil)
           end
 
+          total_count = unmigrated_units.count
+          current_count = 0
+
           unmigrated_units.select(:id, :pulp_id).find_in_batches(batch_size: GET_QUERY_ID_LENGTH) do |needing_hrefs|
+            current_count += needing_hrefs.count
+            update_import_status("Importing migrated content type #{content_type.label}: #{current_count}/#{total_count}")
             migrated_units = pulp2_content_api.list(pulp2_id__in: needing_hrefs.map { |unit| unit.pulp_id }.join(','))
             migrated_units.results.each do |migrated_unit|
               matching_record = needing_hrefs.find { |db_unit| db_unit.pulp_id == migrated_unit.pulp2_id }