katello 4.4.1 → 4.5.0

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -2,20 +2,21 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportValidator
-        attr_accessor :metadata, :path, :content_view, :smart_proxy
-
-        delegate :organization, :to => :content_view
-
-        def initialize(content_view:, path:, metadata:, smart_proxy:)
-          self.content_view = content_view
-          self.path = path
-          self.metadata = metadata
-          self.smart_proxy = smart_proxy
+        def initialize(import:)
+          @content_view = import.content_view
+          @path = import.path
+          @smart_proxy = import.smart_proxy
+          @organization = import.organization
+          @metadata_map = import.metadata_map
+          @interested_repos = import.intersecting_repos_library_and_metadata
+          @redhat_library_products = redhat_library_products
         end
 
         def check!
+          fail _("Content view not provided in the metadata") if @metadata_map.content_view.blank?
+
           ensure_pulp_importable!
-          unless content_view.default?
+          unless @content_view.default?
             ensure_importing_cvv_does_not_exist!
             ensure_from_cvv_exists!
           end
@@ -26,113 +27,107 @@ module Katello
 
         def ensure_pulp_importable!
           api = ::Katello::Pulp3::Api::Core.new(@smart_proxy).importer_check_api
-          response = api.pulp_import_check_post(toc: "#{@path}/#{@metadata[:toc]}")
+          response = api.pulp_import_check_post(toc: "#{@path}/#{@metadata_map.toc}")
           unless response.toc.is_valid
             fail response.toc.messages.join("\n")
           end
         end
 
         def ensure_importing_cvv_does_not_exist!
-          major = metadata[:content_view_version][:major]
-          minor = metadata[:content_view_version][:minor]
+          major = @metadata_map.content_view_version.major
+          minor = @metadata_map.content_view_version.minor
 
-          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: content_view).exists?
+          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: @content_view).exists?
             fail _("Content View Version specified in the metadata - '%{name}' already exists. "\
-                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: "#{content_view.name} #{major}.#{minor}" })
+                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: "#{@content_view.name} #{major}.#{minor}" })
           end
         end
 
         def ensure_from_cvv_exists!
-          major = metadata[:content_view_version][:major]
-          minor = metadata[:content_view_version][:minor]
+          major = @metadata_map.content_view_version.major
+          minor = @metadata_map.content_view_version.minor
 
-          if metadata[:from_content_view_version].present?
-            from_major = metadata[:from_content_view_version][:major]
-            from_minor = metadata[:from_content_view_version][:minor]
+          if @metadata_map.from_content_view_version
+            from_major = @metadata_map.from_content_view_version.major
+            from_minor = @metadata_map.from_content_view_version.minor
 
-            unless ::Katello::ContentViewVersion.where(major: from_major, minor: from_minor, content_view: content_view).exists?
+            unless ::Katello::ContentViewVersion.where(major: from_major, minor: from_minor, content_view: @content_view).exists?
               fail _("Prior Content View Version specified in the metadata - '%{name}' does not exist. "\
-                      "Please import the metadata for '%{name}' before importing '%{current}' " % { name: "#{content_view.name} #{from_major}.#{from_minor}",
-                                                                                                    current: "#{content_view.name} #{major}.#{minor}"})
+                      "Please import the metadata for '%{name}' before importing '%{current}' " % { name: "#{@content_view.name} #{from_major}.#{from_minor}",
+                                                                                                    current: "#{@content_view.name} #{major}.#{minor}"})
             end
           end
         end
 
         def ensure_manifest_imported!
-          rh_repos = ::Katello::Pulp3::ContentViewVersion::Import.metadata_map(metadata, redhat_only: true)
-          if rh_repos.any? && !content_view.organization.manifest_imported?
+          any_rh_repos = @metadata_map.repositories.any?(&:redhat)
+          if any_rh_repos && !@organization.manifest_imported?
             fail _("No manifest found. Import a manifest with the appropriate subscriptions "\
                    "before importing content.")
           end
         end
 
-        def repos_in_library
-          ::Katello::Pulp3::ContentViewVersion::Import.
-                              repositories_in_library(content_view.organization)
-        end
+        def ensure_metadata_matches_repos_in_library!
+          bad_repos = @interested_repos.select do |katello_repo|
+            metadata_repo = metadata_repo_for_katello_repo(katello_repo)
 
-        def metadata_map
-          @metadata_map ||= ::Katello::Pulp3::ContentViewVersion::Import.metadata_map(metadata)
-        end
+            next unless metadata_repo
 
-        def ensure_metadata_matches_repos_in_library!
-          metadata_map = ::Katello::Pulp3::ContentViewVersion::Import.metadata_map(metadata)
-          interested_repos = ::Katello::Pulp3::ContentViewVersion::Import.
-                                      intersecting_repos_library_and_metadata(organization: organization,
-                                                                              metadata: metadata)
-          bad_repos = interested_repos.select do |repo|
-            repo_in_metadata = metadata_map[[repo.product.label, repo.label]]
-            repo.redhat? != repo_in_metadata[:redhat] ||
-              repo.slice(:name, :label, :content_type) != repo_in_metadata.slice(:name, :label, :content_type)
+            !(katello_repo.content_type == metadata_repo.content_type &&
+              katello_repo.redhat? == metadata_repo.redhat)
           end
 
           if bad_repos.any?
-            repos_to_report = bad_repos.map { |repo| [repo.product.label, repo.label] }
             fail _("The following repositories provided in the import metadata have an incorrect content type or provider type. "\
                     "Make sure the export and import repositories are of the same type before importing\n "\
-                    "%{repos}" % { content_view: content_view.name,
-                                   repos: generate_product_repo_i18n_string(repos_to_report).join("")}
+                    "%{repos}" % { content_view: @content_view.name,
+                                   repos: generate_product_repo_i18n_string(bad_repos).join("")}
                   )
           end
         end
 
         def ensure_redhat_products_metadata_are_in_the_library!
-          products_in_library = ::Katello::Product.in_org(organization).redhat.pluck(:label)
-
-          # Map products to repositories in the metadata
-          # {product_label: [repositories with this product]}
-          products_in_metadata_map = {}
+          missing = @metadata_map.repositories.select do |repo|
+            repo.redhat && katello_product_for_metadata_repo(repo).nil?
+          end
 
-          metadata[:repositories].values.each do |repo|
-            next unless repo[:redhat]
-            products_in_metadata_map[repo[:product][:label]] ||= []
-            products_in_metadata_map[repo[:product][:label]] << repo
+          if missing.any?
+            repos_in_import = generate_product_repo_i18n_string(missing)
+            fail _("The organization's manifest does not contain the subscriptions required to enable the following repositories.\n "\
+                    "%{repos}" % { content_view: @content_view.name, repos: repos_in_import.join("")}
+                  )
           end
+        end
+
+        private
 
-          products_not_in_library = products_in_metadata_map.keys - products_in_library
-
-          unless products_not_in_library.blank?
-            # make a list of [product_name, repo_name] pairs to report to the user
-            product_repos = []
-            products_not_in_library.each do |prod|
-              products_in_metadata_map[prod].each do |repo|
-                product_name = metadata[:products][prod][:name]
-                repo_name = repo[:name]
-                product_repos << [product_name, repo_name]
-              end
+        def katello_product_for_metadata_repo(metadata_repo)
+          @redhat_library_products.find do |product|
+            if metadata_repo.redhat && metadata_repo.product.cp_id
+              product.cp_id == metadata_repo.product.cp_id
+            else
+              product.label == metadata_repo.product.label
             end
+          end
+        end
 
-            repos_in_import = generate_product_repo_i18n_string(product_repos)
-            fail _("The organization's manifest does not contain the subscriptions required to enable the following repositories.\n "\
-                    "%{repos}" % { content_view: content_view.name, repos: repos_in_import.join("")}
-                  )
+        def metadata_repo_for_katello_repo(repo)
+          @metadata_map.repositories.find do |metadata_repo|
+            if repo.redhat? && metadata_repo.product.cp_id
+              repo.label == metadata_repo.label && repo.product.cp_id == metadata_repo.product.cp_id
+            else
+              repo.label == metadata_repo.label && repo.product.label == metadata_repo.product.label
+            end
           end
         end
 
-        def generate_product_repo_i18n_string(product_repos)
-          # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
-          product_repos.map do |product, repo|
-            _("\n* Product = '%{product}', Repository = '%{repository}'" % { product: product, repository: repo })
+        def redhat_library_products
+          ::Katello::Product.in_org(@organization).redhat
+        end
+
+        def generate_product_repo_i18n_string(metadata_repos)
+          metadata_repos.map do |repo|
+            _("\n* Product = '%{product}', Repository = '%{repository}'" % { product: repo.product.name, repository: repo.name })
           end
         end
       end

data/app/services/katello/pulp3/content_view_version/importable_products.rb

@@ -2,19 +2,14 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportableProducts
-        attr_accessor :creatable, :updatable, :organization, :metadata
+        attr_reader :creatable, :updatable
 
-        def initialize(organization:, metadata:)
-          self.organization = organization
-          self.metadata = metadata
-          self.creatable = []
-          self.updatable = []
-        end
-
-        def products_in_library
-          # Get the product labels in library
-          return @products_in_library unless @products_in_library.blank?
-          @products_in_library = Set.new(::Katello::Product.in_org(organization).custom.pluck(:label))
+        def initialize(organization:, metadata_products:)
+          @organization = organization
+          @metadata_products = metadata_products
+          @creatable = []
+          @updatable = []
+          @products_in_library = ::Katello::Product.in_org(@organization).custom
         end
 
         def generate!
@@ -23,24 +18,45 @@ module Katello
           #            They are ready to be created
           # updatable: products that are both in the metadata and library.
           #            These may contain updates to the product and hence ready to be updated.
-          metadata[:products].each do |product_label, params|
-            next if params[:redhat]
-            if params[:gpg_key].blank?
-              params[:gpg_key_id] = nil
-            else
-              params[:gpg_key_id] = organization.gpg_keys.find_by(name: params[:gpg_key][:name]).id
-            end
-            params = params.except(:gpg_key, :redhat)
-            if products_in_library.include? product_label
+          @metadata_products.each do |product|
+            next if product.redhat
+
+            library_product = @products_in_library.find { |p| p.label == product.label }
+            if library_product
               # add to the update list if product is already available
-              product = ::Katello::Product.in_org(organization).find_by(label: product_label)
-              updatable << { product: product, options: params.except(:name, :label) }
+              updatable << { product: library_product, options: update_params(product) }
             else
               # add to the create list if its  a new product
-              creatable << { product: ::Katello::Product.new(params) }
+              creatable << { product: ::Katello::Product.new(create_params(product)) }
             end
           end
         end
+
+        private
+
+        def create_params(metadata_product)
+          {
+            gpg_key_id: gpg_key_id(metadata_product),
+            name: metadata_product.name,
+            label: metadata_product.label,
+            description: metadata_product.description
+          }
+        end
+
+        def update_params(metadata_product)
+          params = {
+            gpg_key_id: gpg_key_id(metadata_product),
+            description: metadata_product.description
+          }
+
+          params
+        end
+
+        def gpg_key_id(metadata_product)
+          if metadata_product.gpg_key
+            @organization.gpg_keys.where(name: metadata_product.gpg_key.name).pluck(:id).first
+          end
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/importable_repositories.rb

@@ -2,63 +2,127 @@ module Katello
   module Pulp3
     module ContentViewVersion
       class ImportableRepositories
-        attr_reader :creatable, :updatable, :organization, :metadata, :redhat
+        attr_reader :creatable, :updatable
 
-        def initialize(organization:, metadata:, redhat: false)
+        def initialize(organization:, metadata_repositories:)
           @organization = organization
-          @metadata = metadata
+          @metadata_repositories = metadata_repositories
           @creatable = []
           @updatable = []
-          @redhat = redhat
-        end
-
-        def product_content_by_label(content_label)
-          ::Katello::Content.find_by_label(content_label)
-        end
-
-        def repositories_in_library
-          return @repositories_in_library unless @repositories_in_library.blank?
-          repo_type = redhat ? :redhat : :custom
-          # fetch a list of [product, repo] pairs for every non-redhat library repo
-          product_repos_in_library = Import.repositories_in_library(organization).send(repo_type).
-                                      pluck("#{Katello::Product.table_name}.label",
-                                            "#{Katello::RootRepository.table_name}.label")
-          @repositories_in_library = Set.new(product_repos_in_library.compact)
         end
 
         def generate!
-          # This set's up a 2 different list of importable root repositories
+          # For Red Hat repositories or Custom Repositories in the metadata exclusively
+          # Set up a 2 different list of importable root repositories
           # creatable: repos that are part of the metadata but not in the library.
           #         They are ready to be created
           # updatable: repo that are both in the metadata and library.
           #         These may contain updates to the repo and hence ready to be updated.
-          metadata_map = Import.metadata_map(metadata, custom_only: !redhat, redhat_only: redhat)
-          metadata_map.keys.each do |product_label, repo_label|
-            product = Katello::Product.in_org(organization).find_by(label: product_label)
-            fail _("Unable to find product '%s' in organization '%s'" % [product_label, organization.name]) if product.blank?
-            params = metadata_map[[product_label, repo_label]]
-            if params[:gpg_key].blank?
-              params[:gpg_key_id] = nil
-            else
-              params[:gpg_key_id] = organization.gpg_keys.find_by(name: params[:gpg_key][:name]).id
+          @metadata_repositories.each do |repo|
+            product = product_for_metadata_repo(repo)
+            fail _("Unable to find product '%s' in organization '%s'" % [repo.product.label, @organization.name]) if product.blank?
+
+            root = product.root_repositories.find do |r|
+              if repo.content&.id && repo.redhat
+                r.content.cp_content_id == repo.content.id &&
+                  r.arch == repo.arch &&
+                  r.major == repo.major &&
+                  r.minor == repo.minor
+              else
+                r.label == repo.label
+              end
             end
-            content = params[:content]
-            params = params.except(:redhat, :product, :gpg_key, :content)
-            if repositories_in_library.include? [product_label, repo_label]
-              repo = ::Katello::RootRepository.find_by(product: product, label: repo_label)
-              updatable << { repository: repo, options: params.except(:label, :name, :content_type) }
-            elsif redhat
-              product_content = product_content_by_label(content[:label])
+
+            if root
+              updatable << { repository: root, options: update_repo_params(repo) }
+            elsif repo.redhat
+              product_content = fetch_product_content(repo.content, product)
               substitutions = {
-                basearch: params[:arch],
-                releasever: params[:minor]
+                basearch: repo.arch,
+                releasever: repo.minor
               }
               creatable << { product: product, content: product_content, substitutions: substitutions }
             else
-              creatable << { repository: product.add_repo(params) }
+              creatable << { repository: product.add_repo(create_repo_params(repo)) }
             end
           end
         end
+
+        private
+
+        def product_for_metadata_repo(repo)
+          if repo.redhat && repo.product.cp_id
+            @organization.products.includes(:root_repositories).find_by(cp_id: repo.product.cp_id)
+          else
+            @organization.products.includes(:root_repositories).find_by(label: repo.product.label)
+          end
+        end
+
+        def fetch_product_content(content_metadata, product)
+          query = ::Katello::Content.joins(:product_contents).where("#{Katello::ProductContent.table_name}.product_id": product.id)
+          table_name = Katello::Content.table_name
+          if content_metadata&.id
+            query.find_by("#{table_name}.cp_content_id": content_metadata.id)
+          else
+            query.find_by("#{table_name}.label": content_metadata.label)
+          end
+        end
+
+        def gpg_key_id(metadata_repo)
+          if metadata_repo.gpg_key
+            @organization.gpg_keys.find_by(name: metadata_repo.gpg_key.name).id
+          end
+        end
+
+        def create_repo_params(metadata_repo)
+          keys = [
+            :name,
+            :label,
+            :description,
+            :arch,
+            :unprotected,
+            :content_type,
+            :checksum_type,
+            :os_versions,
+            :major,
+            :minor,
+            :download_policy,
+            :mirroring_policy
+          ]
+
+          params = {}
+          params[:gpg_key_id] = gpg_key_id(metadata_repo)
+
+          keys.each do |key|
+            params[key] = metadata_repo.send(key)
+          end
+
+          params
+        end
+
+        def update_repo_params(metadata_repo)
+          keys = [
+            :description,
+            :arch,
+            :unprotected,
+            :checksum_type,
+            :os_versions,
+            :major,
+            :minor,
+            :download_policy,
+            :mirroring_policy
+          ]
+
+          params = {}
+          params[:gpg_key_id] = gpg_key_id(metadata_repo)
+
+          keys.each do |key|
+            value = metadata_repo.send(key)
+            params[key] = value if value
+          end
+
+          params
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/metadata_generator.rb

@@ -17,7 +17,7 @@ module Katello
         def generate!
           ret = { organization: organization.name,
                   repositories: {},
-                  content_view: content_view.slice(:name, :label, :description),
+                  content_view: content_view.slice(:name, :label, :description, :generated_for),
                   content_view_version: content_view_version.slice(:major, :minor, :description),
                   incremental: from_content_view_version.present?
           }
@@ -51,7 +51,7 @@ module Katello
         end
 
         def generate_product_metadata(product)
-          product.slice(:name, :label, :description).
+          product.slice(:name, :label, :description, :cp_id).
             merge(gpg_key: generate_gpg_metadata(product.gpg_key),
                   redhat: product.redhat?)
         end

data/app/services/katello/pulp3/content_view_version/metadata_map.rb

@@ -0,0 +1,117 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class MetadataMap
+        class MetadataContentView < OpenStruct; end
+
+        class MetadataContentViewVersion < OpenStruct; end
+
+        class MetadataProduct < OpenStruct; end
+
+        class MetadataRepository < OpenStruct; end
+
+        class MetadataGpgKey < OpenStruct; end
+
+        class MetadataRepositoryContent < OpenStruct; end
+
+        attr_reader :toc,
+                    :products,
+                    :repositories,
+                    :gpg_keys,
+                    :content_view,
+                    :content_view_version,
+                    :from_content_view_version
+
+        def initialize(metadata:)
+          @toc = metadata[:toc]
+          @gpg_keys = parse_gpg_keys(metadata[:gpg_keys]) if metadata[:gpg_keys]
+          @products = parse_products(metadata[:products]) if metadata[:products]
+          @repositories = parse_repositories(metadata[:repositories]) if metadata[:repositories]
+          @content_view = parse_content_view(metadata[:content_view]) if metadata[:content_view]
+          @content_view_version = parse_content_view_version(metadata[:content_view_version]) if metadata[:content_view_version]
+          @from_content_view_version = parse_content_view_version(metadata[:from_content_view_version]) if metadata[:from_content_view_version]
+        end
+
+        private
+
+        def parse_gpg_keys(gpg_keys)
+          gpg_keys.values.map do |g|
+            MetadataGpgKey.new(
+              name: g[:name],
+              content_type: g[:content_type],
+              content: g[:content]
+            )
+          end
+        end
+
+        def parse_content_view(content_view)
+          MetadataContentView.new(
+            name: content_view['name'],
+            label: content_view['label'],
+            description: content_view['description'],
+            generated_for: content_view['generated_for']
+          )
+        end
+
+        def parse_products(products)
+          products.values.map do |p|
+            MetadataProduct.new(
+              name: p[:name],
+              label: p[:label],
+              cp_id: p[:cp_id],
+              description: p[:description],
+              gpg_key: gpg_key_for_product(p),
+              redhat: ::Foreman::Cast.to_bool(p[:redhat])
+            )
+          end
+        end
+
+        def parse_content_view_version(version)
+          MetadataContentViewVersion.new(
+            major: version[:major],
+            minor: version[:minor],
+            description: version[:description]
+          )
+        end
+
+        def parse_repositories(repositories)
+          repositories.map do |pulp_name, repo|
+            MetadataRepository.new(
+              pulp_name: pulp_name,
+              name: repo[:name],
+              label: repo[:label],
+              description: repo[:description],
+              arch: repo[:arch],
+              content_type: repo[:content_type],
+              unprotected: repo[:unprotected],
+              checksum_type: repo[:checksum_type],
+              os_versions: repo[:os_versions],
+              major: repo[:major],
+              minor: repo[:minor],
+              download_policy: repo[:download_policy],
+              mirroring_policy: repo[:mirroring_policy],
+              redhat: repo[:redhat],
+              product: product_for_repo(repo),
+              gpg_key: gpg_key_for_repo(repo),
+              content: repo[:content] ? MetadataRepositoryContent.new(id: repo[:content][:id], label: repo[:content][:label]) : nil
+            )
+          rescue => e
+            raise _("Invalid repository in the metadata %{repo} error=%{error}") % { repo: repo, error: e.message }
+          end
+        end
+
+        def gpg_key_for_repo(repo)
+          @gpg_keys.find { |g| g.name == repo[:gpg_key][:name] } if repo[:gpg_key].present?
+        end
+
+        def gpg_key_for_product(product)
+          @gpg_keys.find { |g| g.name == product[:gpg_key][:name] } if product[:gpg_key].present?
+        end
+
+        def product_for_repo(repo)
+          @products.find { |p| p.label == repo[:product][:label] }
+        end
+      end
+    end
+  end
+end

data/app/services/katello/pulp3/pulp_content_unit.rb

@@ -128,9 +128,14 @@ module Katello
 
       def self.find_duplicate_unit(repository, unit_type_id, file, checksum)
         filter_label = :sha256
+        path_label = :relative_path
         if unit_type_id == 'ostree_ref'
           filter_label = :checksum
         end
+        if unit_type_id == 'rpm'
+          filter_label = :pkg_id
+          path_label = :location_href
+        end
         content_backend_service = SmartProxy.pulp_primary.content_service(unit_type_id)
         duplicates_allowed = ::Katello::RepositoryTypeManager.find_content_type(unit_type_id).try(:duplicates_allowed)
         if repository.generic? && duplicates_allowed
@@ -144,7 +149,7 @@ module Katello
         else
           duplicate_sha_path_content_list = content_backend_service.content_api.list(
             filter_label => checksum,
-            "relative_path": file[:filename])
+            path_label => file[:filename])
         end
         duplicate_sha_path_content_list
       end