katello 4.14.1 → 4.15.0.rc2

data/app/services/katello/product_content_finder.rb

@@ -15,22 +15,22 @@ module Katello
 
     def product_content
       if match_environment
-        if consumable.is_a?(::Katello::ActivationKey) # TODO: remove this when AKs are refactored for multi CV
-          environment = consumable.lifecycle_environment
-          view = consumable.content_view
-          return ProductContent.none unless environment && view
-          versions = ContentViewVersion.in_environment(environment).where(:content_view_id => view).first
-        else # consumable is a SubscriptionFacet
-          versions = consumable.content_view_environments.select(:content_view_version_id).map(&:content_view_version_id)
-        end
+        versions = consumable.content_view_environments.select(:content_view_version_id).map(&:content_view_version_id)
       end
 
       considered_products = match_subscription ? consumable.products : consumable.organization.products.enabled.uniq
 
       roots = Katello::RootRepository.where(:product_id => considered_products).subscribable
-      roots = roots.in_content_view_version(versions) if versions.present?
+      roots = roots.in_content_view_version(versions).distinct if versions.present?
+      content_ids = roots.where.not(:content_id => nil).pluck(:content_id)
+      structured_apt_roots = roots.where(:content_id => nil)
+      if structured_apt_roots.any?
+        deb_repos_query = Katello::Repository.where(root: structured_apt_roots)
+        deb_repos = match_environment ? deb_repos_query.where(content_view_version: versions, environment: consumable.environment) : deb_repos_query.where(:library_instance_id => nil)
+        content_ids += deb_repos.pluck(:content_id)
+      end
 
-      consumable.organization.enabled_product_content_for(roots)
+      consumable.organization.enabled_product_content_for(content_ids)
     end
 
     def custom_content_labels

data/app/services/katello/product_content_importer.rb

@@ -152,7 +152,7 @@ module Katello
         vendor: content[:vendor],
         gpg_url: content[:gpgUrl],
         content_url: content[:contentUrl],
-        organization_id: product.organization_id
+        organization_id: product.organization_id,
       }
     end
 

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -54,7 +54,7 @@ module Katello
           proxy_url: smart_proxy.http_proxy&.url,
           proxy_username: smart_proxy.http_proxy&.username,
           proxy_password: smart_proxy.http_proxy&.password,
-          total_timeout: Setting[:sync_connect_timeout]
+          total_timeout: Setting[:sync_connect_timeout],
         }
         if acs.content_type == ::Katello::Repository::FILE_TYPE && acs.subpaths.empty? && !remote_options[:url].end_with?('/PULP_MANIFEST')
           remote_options[:url] = acs.base_url + '/PULP_MANIFEST'
@@ -73,7 +73,7 @@ module Katello
           {
             client_cert: acs.ssl_client_cert&.content,
             client_key: acs.ssl_client_key&.content,
-            ca_cert: acs.ssl_ca_cert&.content
+            ca_cert: acs.ssl_ca_cert&.content,
           }
         end
       end

data/app/services/katello/pulp3/ansible_collection.rb

@@ -26,7 +26,7 @@ module Katello
           namespace: unit['namespace'],
           version: unit['version'],
           name: unit['name'],
-          description: unit['description']
+          description: unit['description'],
         }
       end
 

data/app/services/katello/pulp3/api/apt.rb

@@ -23,6 +23,10 @@ module Katello
         def copy_api
           PulpDebClient::DebCopyApi.new(api_client)
         end
+
+        def content_release_components_api
+          PulpDebClient::ContentReleaseComponentsApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/content.rb

@@ -4,6 +4,7 @@ module Katello
     class Content
       extend Katello::Abstract::Pulp::Content
       class << self
+        # rubocop:disable Metrics/ParameterLists
         def create_upload(size = 0, checksum = nil, content_type = nil, repository = nil)
           content_unit_href = nil
           content_type = ::Katello::RepositoryTypeManager.find_content_type(content_type)

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -130,11 +130,7 @@ module Katello
           MetadataGenerator.new(export_service: self).generate!
         end
 
-        def self.find_generated_export_view(create_by_default: false,
-                                            destination_server:,
-                                            organization:,
-                                            name:,
-                                            generated_for:)
+        def self.find_generated_export_view(destination_server:, organization:, name:, generated_for:, create_by_default: false)
           name += "-#{destination_server}" unless destination_server.blank?
           select_method = create_by_default ? :first_or_create : :first
           ::Katello::ContentView.where(name: name,
@@ -146,9 +142,7 @@ module Katello
           is_a?(SyncableFormatExport) ? SYNCABLE : IMPORTABLE
         end
 
-        def self.find_library_export_view(create_by_default: false,
-                                          destination_server:,
-                                          organization:,
+        def self.find_library_export_view(destination_server:, organization:, create_by_default: false,
                                           format: IMPORTABLE)
           if format == IMPORTABLE
             generated_for = :library_export
@@ -165,8 +159,7 @@ module Katello
                                      generated_for: generated_for)
         end
 
-        def self.find_repository_export_view(create_by_default: false,
-                                              repository:,
+        def self.find_repository_export_view(repository:, create_by_default: false,
                                               format: IMPORTABLE)
           if format == IMPORTABLE
             generated_for = :repository_export

data/app/services/katello/pulp3/content_view_version/export_validator.rb

@@ -30,7 +30,7 @@ module Katello
                    " is required to be able to export: '%{exportable_types}'." %
                    { content_view: content_view_version.content_view.name,
                      current: content_view_version.version,
-                     exportable_types: ::Katello::Repository.exportable_types(format: format).join("', '")
+                     exportable_types: ::Katello::Repository.exportable_types(format: format).join("', '"),
                    })
           end
         end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -135,7 +135,7 @@ module Katello
               name: metadata_map.content_view.name,
               label: metadata_map.content_view.label,
               description: metadata_map.content_view.description,
-              generated_for: generated_for
+              generated_for: generated_for,
             }
           end
 
@@ -150,7 +150,7 @@ module Katello
             name: import_cv_name_from_export(name: metadata_map.content_view.name, generated_for: generated_for),
             label: import_cv_name_from_export(name: metadata_map.content_view.label, generated_for: generated_for),
             description: "Content View used for importing into library",
-            generated_for: generated_for
+            generated_for: generated_for,
           }
         end
 

data/app/services/katello/pulp3/content_view_version/import_gpg_keys.rb

@@ -22,7 +22,7 @@ module Katello
             params = {
               name: gpg.name,
               content_type: ::Katello::ContentCredential::GPG_KEY_TYPE,
-              content: gpg.content
+              content: gpg.content,
             }
 
             create_or_update_gpg!(params)

data/app/services/katello/pulp3/content_view_version/importable_products.rb

@@ -39,7 +39,7 @@ module Katello
             gpg_key_id: gpg_key_id(metadata_product),
             name: find_unique_name(metadata_product),
             label: metadata_product.label,
-            description: metadata_product.description
+            description: metadata_product.description,
           }
         end
 
@@ -56,7 +56,7 @@ module Katello
         def update_params(metadata_product)
           {
             gpg_key_id: gpg_key_id(metadata_product),
-            description: metadata_product.description
+            description: metadata_product.description,
           }
         end
 

data/app/services/katello/pulp3/content_view_version/importable_repositories.rb

@@ -46,14 +46,14 @@ module Katello
               product_content = fetch_product_content(repo.content, product)
               substitutions = {
                 basearch: repo.arch,
-                releasever: repo.minor
+                releasever: repo.minor,
               }
               creatable << {
                 product: product,
                 content: product_content,
                 substitutions: substitutions,
                 override_url: fetch_feed_url(repo),
-                override_arch: repo.arch
+                override_arch: repo.arch,
               }
             else
               creatable << { repository: product.add_repo(create_repo_params(repo, product)) }
@@ -109,7 +109,7 @@ module Katello
             :major,
             :minor,
             :download_policy,
-            :mirroring_policy
+            :mirroring_policy,
           ]
 
           params = { name: find_unique_name(metadata_repo, product) }
@@ -142,7 +142,7 @@ module Katello
             :major,
             :minor,
             :download_policy,
-            :mirroring_policy
+            :mirroring_policy,
           ]
           params = {}
           url = fetch_feed_url(metadata_repo)

data/app/services/katello/pulp3/content_view_version/metadata_generator.rb

@@ -21,7 +21,7 @@ module Katello
                   content_view: content_view.slice(:name, :label, :description, :generated_for),
                   content_view_version: content_view_version.slice(:major, :minor, :description),
                   incremental: from_content_view_version.present?,
-                  format: export_service.format
+                  format: export_service.format,
           }
           unless from_content_view_version.blank?
             ret[:from_content_view_version] = from_content_view_version.slice(:major, :minor)
@@ -72,7 +72,7 @@ module Katello
                                                                    content_path: content.content_url)
           { id: content.cp_content_id,
             label: content.label,
-            url: content_data[:path]
+            url: content_data[:path],
           }
         end
 

data/app/services/katello/pulp3/deb.rb

@@ -5,6 +5,8 @@ module Katello
       CONTENT_TYPE = "deb".freeze
       PULPCORE_CONTENT_TYPE = "deb.package".freeze
 
+      lazy_accessor :initializer => :backend_data
+
       def self.content_api
         PulpDebClient::ContentPackagesApi.new(Katello::Pulp3::Api::Apt.new(SmartProxy.pulp_primary!).api_client)
       end
@@ -34,9 +36,41 @@ module Katello
           name: unit[:package],
           version: unit[:version],
           description: unit[:description]&.truncate(255),
-          architecture: unit[:architecture]
+          architecture: unit[:architecture],
+          section: unit[:section],
+          maintainer: unit[:maintainer],
+          homepage: unit[:homepage],
+          installed_size: unit[:installed_size],
         }
       end
+
+      def depends
+        Util::Deb.parse_dependencies(backend_data['depends'])
+      end
+
+      def pre_depends
+        Util::Deb.parse_dependencies(backend_data['pre_depends'])
+      end
+
+      def recommends
+        Util::Deb.parse_dependencies(backend_data['recommends'])
+      end
+
+      def suggests
+        Util::Deb.parse_dependencies(backend_data['suggests'])
+      end
+
+      def enhances
+        Util::Deb.parse_dependencies(backend_data['enhances'])
+      end
+
+      def breaks
+        Util::Deb.parse_dependencies(backend_data['breaks'])
+      end
+
+      def conflicts
+        Util::Deb.parse_dependencies(backend_data['conflicts'])
+      end
     end
   end
 end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -30,7 +30,7 @@ module Katello
           annotations: unit['annotations'],
           labels: unit['labels'],
           is_bootable: unit['is_bootable'],
-          is_flatpak: unit['is_flatpak']
+          is_flatpak: unit['is_flatpak'],
         }
       end
     end

data/app/services/katello/pulp3/docker_manifest_list.rb

@@ -28,7 +28,11 @@ module Katello
         {
           schema_version: unit['schema_version'],
           digest: unit['digest'],
-          pulp_id: unit[unit_identifier]
+          pulp_id: unit[unit_identifier],
+          annotations: unit['annotations'],
+          labels: unit['labels'],
+          is_bootable: unit['is_bootable'],
+          is_flatpak: unit['is_flatpak'],
         }
       end
 
@@ -40,7 +44,7 @@ module Katello
           manifest_list_manifests += manifest_ids.map do |manifest_id|
             {
               docker_manifest_list_id: katello_id,
-              docker_manifest_id: manifest_id
+              docker_manifest_id: manifest_id,
             }
           end
         end

data/app/services/katello/pulp3/docker_tag.rb

@@ -18,7 +18,7 @@ module Katello
       def self.generate_model_row(unit)
         row = {
           pulp_id: unit[unit_identifier],
-          name: unit['name']
+          name: unit['name'],
         }
 
         taggable = ::Katello::DockerManifest.find_by(:pulp_id => unit['tagged_manifest'])

data/app/services/katello/pulp3/erratum.rb

@@ -75,7 +75,7 @@ module Katello
           {
             bug_id: bugzilla[:id],
             href: bugzilla[:href],
-            erratum_id: katello_id
+            erratum_id: katello_id,
           }
         end
       end
@@ -85,7 +85,7 @@ module Katello
           {
             cve_id: cve[:id],
             href: cve[:href],
-            erratum_id: katello_id
+            erratum_id: katello_id,
           }
         end
       end

data/app/services/katello/pulp3/file_unit.rb

@@ -27,7 +27,7 @@ module Katello
           pulp_id: unit[unit_identifier],
           name: filename,
           path: unit['relative_path'],
-          checksum: unit['sha256']
+          checksum: unit['sha256'],
         }
       end
     end

data/app/services/katello/pulp3/generic_content_unit.rb

@@ -24,7 +24,7 @@ module Katello
           version: content_type&.model_version&.call(unit),
           filename: content_type&.model_filename&.call(unit),
           additional_metadata: content_type&.model_additional_metadata&.call(unit),
-          content_type: content_type&.label
+          content_type: content_type&.label,
         }
       end
     end

data/app/services/katello/pulp3/module_stream.rb

@@ -19,7 +19,7 @@ module Katello
         rpms = package_ids.map do |package_id|
           {
             module_stream_id: katello_id,
-            rpm_id: package_id
+            rpm_id: package_id,
           }
         end
         add_timestamps(rpms)
@@ -38,7 +38,7 @@ module Katello
         profiles = profiles_json.map do |profile, _rpms|
           {
             module_stream_id: katello_id,
-            name: profile
+            name: profile,
           }
         end
         add_timestamps(profiles)

data/app/services/katello/pulp3/repository/ansible_collection.rb

@@ -20,7 +20,7 @@ module Katello
           {
             base_path: path,
             repository_version: repo.version_href,
-            name: "#{generate_backend_object_name}"
+            name: "#{generate_backend_object_name}",
           }
         end
 

data/app/services/katello/pulp3/repository/apt.rb

@@ -7,10 +7,18 @@ module Katello
         UNIT_LIMIT = 10_000
         SIGNING_SERVICE_NAME = 'katello_deb_sign'.freeze
 
+        def initialize_empty
+          # For every empty APT library instance repository we must add at least a release component to
+          # ensure we have a publishable repo with consumable metadata. Otherwise smart proxy syncs will
+          # fail, and consuming hosts will choke on empty repos.
+          opts = {:repository => repository_reference.repository_href, :component => "empty", :distribution => "katello"}
+          api.content_release_components_api.create(opts)
+        end
+
         def remote_options
           deb_remote_options = {
             policy: root.download_policy,
-            distributions: root.deb_releases
+            distributions: root.deb_releases,
           }
           deb_remote_options[:components] = root.deb_components.present? ? root.deb_components : nil
           deb_remote_options[:architectures] = root.deb_architectures.present? ? root.deb_architectures : nil
@@ -31,22 +39,20 @@ module Katello
         end
 
         def mirror_remote_options
-          super.merge(
-            {
-              distributions: repo.deb_releases + "#{' default' unless repo.deb_releases.include? 'default'}"
-            }
-          )
+          distributions = if repo.deb_using_structured_apt?
+                            repo.deb_pulp_distributions.join(' ')
+                          else
+                            'default'
+                          end
+
+          super.merge({distributions: distributions})
         end
 
-        def publication_options(repository_version)
+        def publication_options(repository)
           ss = api.signing_services_api.list(name: SIGNING_SERVICE_NAME).results
-          popts = super(repository_version)
-          popts.merge!(
-            {
-              structured: true, # publish real suites (e.g. 'stable')
-              simple: true # publish all into 'default'-suite
-            }
-          )
+          popts = super(repository)
+          popts.merge!({ structured: true })
+          popts.merge!({ simple: true }) unless repository.deb_using_structured_apt?
           popts[:signing_service] = ss[0].pulp_href if ss && ss.length == 1
           popts
         end
@@ -55,7 +61,7 @@ module Katello
           {
             base_path: path,
             publication: repo.publication_href,
-            name: "#{generate_backend_object_name}"
+            name: "#{generate_backend_object_name}",
           }
         end
 
@@ -104,7 +110,7 @@ module Katello
             config_hash = {
               source_repo_version: repo_config[:source_repo_version],
               dest_repo: repo_config[:dest_repo],
-              content: []
+              content: [],
             }
             config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version]
             data_dup.config << config_hash

data/app/services/katello/pulp3/repository/docker.rb

@@ -39,7 +39,7 @@ module Katello
           super.merge(
             {
               url: "https://#{SmartProxy.pulp_primary.pulp3_host!.downcase}",
-              upstream_name: repo.container_repository_name
+              upstream_name: repo.container_repository_name,
             }
           )
         end
@@ -48,7 +48,7 @@ module Katello
           {
             base_path: path,
             repository_version: repo.version_href,
-            name: "#{generate_backend_object_name}"
+            name: "#{generate_backend_object_name}",
           }
         end
 

data/app/services/katello/pulp3/repository/file.rb

@@ -12,7 +12,7 @@ module Katello
           {
             base_path: path,
             publication: repo.publication_href,
-            name: "#{generate_backend_object_name}"
+            name: "#{generate_backend_object_name}",
           }
         end
 

data/app/services/katello/pulp3/repository/generic.rb

@@ -9,7 +9,7 @@ module Katello
         def distribution_options(path)
           options = {
             base_path: path,
-            name: "#{generate_backend_object_name}"
+            name: "#{generate_backend_object_name}",
           }
 
           if ::Katello::RepositoryTypeManager.find(repo.content_type).pulp3_skip_publication

data/app/services/katello/pulp3/repository/yum.rb

@@ -20,8 +20,8 @@ module Katello
           options.merge!(policy: root.download_policy)
         end
 
-        def publication_options(repository_version)
-          options = super(repository_version)
+        def publication_options(repository)
+          options = super(repository)
           # To work around https://projects.theforeman.org/issues/37715
           checksum_type = if root.checksum_type.nil?
                             'sha256'
@@ -50,7 +50,7 @@ module Katello
           {
             base_path: path,
             publication: repo.publication_href,
-            name: "#{generate_backend_object_name}"
+            name: "#{generate_backend_object_name}",
           }
         end
 
@@ -141,7 +141,7 @@ module Katello
             config_hash = {
               source_repo_version: repo_config[:source_repo_version],
               dest_repo: repo_config[:dest_repo],
-              content: []
+              content: [],
             }
             config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version]
             data_dup.config << config_hash
@@ -253,7 +253,7 @@ module Katello
               source_repo_ids.each do |repo_id|
                 filter_list_map[:whitelist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id))
               end
-            elsif filter.class == ContentViewErratumFilter
+            elsif filter.instance_of?(ContentViewErratumFilter)
               source_repo_ids.each do |repo_id|
                 filter_list_map[:blacklist_ids] += filter.content_unit_pulp_ids(::Katello::Repository.find(repo_id), additional_included_errata)
               end
@@ -280,7 +280,7 @@ module Katello
           inclusion_modular_filters = modular_filters.select { |filter| filter.inclusion }
           exclusion_modular_filters = modular_filters - inclusion_modular_filters
           if inclusion_modular_filters.empty? &&
-              !(filters.any? { |filter| filter.class == ContentViewErratumFilter && filter.inclusion })
+              !(filters.any? { |filter| filter.instance_of?(ContentViewErratumFilter) && filter.inclusion })
             source_repo_ids.each do |source_repo_id|
               source_repo = ::Katello::Repository.find(source_repo_id)
               filter_list_map[:whitelist_ids] += source_repo.rpms.where(:modular => true).pluck(:pulp_id).sort
@@ -401,7 +401,7 @@ module Katello
           (errata_filters + package_filters).each do |filter|
             if filter.inclusion
               whitelist_ids += filter.content_unit_pulp_ids(source_repository)
-            elsif filter.class == ContentViewErratumFilter
+            elsif filter.instance_of?(ContentViewErratumFilter)
               blacklist_ids += filter.content_unit_pulp_ids(source_repository, additional_included_errata)
             else
               blacklist_ids += filter.content_unit_pulp_ids(source_repository)

data/app/services/katello/pulp3/repository.rb

@@ -181,6 +181,12 @@ module Katello
         end
       end
 
+      def initialize_empty
+        # Can be used to initialize a newly created library instance repository with some initial content.
+        # Currently only used for deb type repositories!
+        fail NotImplementedError
+      end
+
       def update
         api.repositories_api.update(repository_reference.try(:repository_href), create_options)
       end
@@ -211,7 +217,7 @@ module Katello
       end
 
       def create_publication
-        publication_data = api.publication_class.new(publication_options(repo.version_href))
+        publication_data = api.publication_class.new(publication_options(repo))
         api.publications_api.create(publication_data)
       end
 
@@ -219,9 +225,9 @@ module Katello
         ignore_404_exception { api.publications_api.delete(repo.publication_href) } if repo.publication_href
       end
 
-      def publication_options(repository_version)
+      def publication_options(repository)
         {
-          repository_version: repository_version
+          repository_version: repository.version_href,
         }
       end
 
@@ -319,7 +325,7 @@ module Katello
           data = api.class.copy_class.new
           data.config = [{
             source_repo_version: source_repository.version_href,
-            dest_repo: repository_reference.repository_href
+            dest_repo: repository_reference.repository_href,
           }]
           tasks << api.copy_api.copy_content(data)
           tasks
@@ -400,7 +406,7 @@ module Katello
           connect_timeout: Setting[:sync_connect_timeout_v2],
           sock_connect_timeout: Setting[:sync_sock_connect_timeout],
           sock_read_timeout: Setting[:sync_sock_read_timeout],
-          rate_limit: Setting[:download_rate_limit]
+          rate_limit: Setting[:download_rate_limit],
         }
         remote_options[:url] = root.url unless root.url.blank?
         remote_options[:download_concurrency] = root.download_concurrency unless root.download_concurrency.blank?
@@ -446,23 +452,23 @@ module Katello
           options = {
             client_cert: root.product.certificate,
             client_key: root.product.key,
-            ca_cert: Katello::Repository.feed_ca_cert(root.url)
+            ca_cert: Katello::Repository.feed_ca_cert(root.url),
           }
         elsif root.redhat? && root.cdn_configuration.custom_cdn?
           options = {
-            ca_cert: root.cdn_configuration.ssl_ca
+            ca_cert: root.cdn_configuration.ssl_ca,
           }
         elsif root.redhat? && root.cdn_configuration.network_sync?
           options = {
             client_cert: root.cdn_configuration.ssl_cert,
             client_key: root.cdn_configuration.ssl_key,
-            ca_cert: root.cdn_configuration.ssl_ca
+            ca_cert: root.cdn_configuration.ssl_ca,
           }
         elsif root.custom?
           options = {
             client_cert: root.ssl_client_cert&.content,
             client_key: root.ssl_client_key&.content,
-            ca_cert: root.ssl_ca_cert&.content
+            ca_cert: root.ssl_ca_cert&.content,
           }
         end
         append_proxy_cacert(options) if options.key?(:cacert)