katello 3.17.0 → 3.18.0

data/app/services/katello/pulp3/api/core.rb

@@ -86,6 +86,14 @@ module Katello
           PulpcoreClient::UploadsApi.new(core_api_client)
         end
 
+        def upload_commit_class
+          PulpcoreClient::UploadCommit
+        end
+
+        def signing_services_api
+          PulpcoreClient::SigningServicesApi.new(core_api_client)
+        end
+
         def tasks_api
           PulpcoreClient::TasksApi.new(core_api_client)
         end

data/app/services/katello/pulp3/api/docker.rb

@@ -32,6 +32,10 @@ module Katello
           PulpContainerClient::RecursiveManage
         end
 
+        def self.tag_image_class
+          PulpContainerClient::TagImage
+        end
+
         def api_client
           PulpContainerClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpContainerClient::Configuration))
         end

data/app/services/katello/pulp3/content_view_version/export.rb

@@ -3,12 +3,36 @@ module Katello
     module ContentViewVersion
       class Export
         include ImportExportCommon
-        METADATA_FILE = "metadata.json".freeze
 
-        def initialize(smart_proxy:, content_view_version: nil, destination_server: nil)
+        def initialize(smart_proxy:,
+                        content_view_version: nil,
+                        destination_server: nil,
+                        from_content_view_version: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @destination_server = destination_server
+          @from_content_view_version = from_content_view_version
+        end
+
+        def repository_hrefs
+          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
+        end
+
+        def version_hrefs
+          repositories.pluck(:version_href).compact
+        end
+
+        def repositories(fetch_all: false)
+          repos = if @content_view_version.default?
+                    @content_view_version.repositories.yum_type
+                  else
+                    @content_view_version.archived_repos.yum_type
+                  end
+          if fetch_all
+            repos
+          else
+            repos.immediate
+          end
         end
 
         def generate_exporter_path
@@ -26,8 +50,33 @@ module Katello
                                   repositories: repository_hrefs)
         end
 
-        def create_export(exporter_href)
-          [api.export_api.create(exporter_href, { versions: version_hrefs })]
+        def create_export(exporter_href, chunk_size: nil)
+          options = { versions: version_hrefs }
+          options[:chunk_size] = "#{chunk_size}MB" if chunk_size
+          if @from_content_view_version
+            from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
+            start_versions = from_exporter.version_hrefs
+
+            # current_cvv - cvv_from , i.e. repos in current cvv that are not in from
+            # implying something got added to the current cvv
+            # make sure you set the start_versions as 0
+            added_repo_hrefs = repository_hrefs - from_exporter.repository_hrefs
+            added_repo_hrefs.each do |added_repo_href|
+              start_versions << zero_version_href(added_repo_href)
+            end
+
+            # cvv_from - current_cvv , i.e. repos in from cvv that are not in current
+            # implying something got removed the current cvv
+            # make sure the start_versions doesn't contain those
+            deleted_repo_hrefs = from_exporter.repository_hrefs - repository_hrefs
+            start_versions.select! do |href|
+              !deleted_repo_hrefs.include?(version_href_to_repository_href(href))
+            end
+
+            options[:start_versions] = start_versions
+            options[:full] = 'false'
+          end
+          [api.export_api.create(exporter_href, options)]
         end
 
         def fetch_export(exporter_href)
@@ -41,15 +90,67 @@ module Katello
           api.exporter_api.delete(exporter_href)
         end
 
+        def validate!(fail_on_missing_content: true, validate_incremental: true)
+          validate_repositories_immediate! if fail_on_missing_content
+          validate_incremental_export! if validate_incremental && !@from_content_view_version.blank?
+        end
+
+        def validate_repositories_immediate!
+          non_immediate_repos = repositories(fetch_all: true).non_immediate
+          if non_immediate_repos.any?
+            fail _("NOTE: Unable to fully export Content View Version '%{content_view} %{current}'"\
+                   " it contains repositories without the 'immediate' download policy."\
+                   " Update the download policy and sync affected repositories. Once synced republish the content view"\
+                   " and export the generated version. \n %{repos}" %
+                   { content_view: @content_view_version.content_view.name,
+                     current: @content_view_version.version,
+                     repos: self.class.generate_product_repo_strings(repositories: non_immediate_repos)})
+          end
+        end
+
+        def validate_incremental_export!
+          from_exporter = Export.new(smart_proxy: @smart_proxy, content_view_version: @from_content_view_version)
+
+          from_exporter_repos = generate_repo_mapping(from_exporter.repositories(fetch_all: true))
+          to_exporter_repos = generate_repo_mapping(repositories(fetch_all: true))
+
+          invalid_repos_exist = (from_exporter_repos.keys & to_exporter_repos.keys).any? do |repo_id|
+            from_exporter_repos[repo_id] != to_exporter_repos[repo_id]
+          end
+
+          if invalid_repos_exist
+            fail _("The exported Content View Version '%{content_view} %{current}' cannot be incrementally updated from version '%{from}'."\
+                   " Please do a full export." % { content_view: @content_view_version.content_view.name,
+                                                   current: @content_view_version.version,
+                                                   from: @from_content_view_version.version})
+          end
+        end
+
+        def generate_repo_mapping(repositories)
+          # return a repo mapping with key being the  library_instance_id and value being the repostiory_href
+          # used by validate_incremental_export
+          repo_map = {}
+          repositories.each do |repo|
+            repo_map[repo.library_instance_id] = version_href_to_repository_href(repo.version_href)
+          end
+          repo_map
+        end
+
         def generate_metadata
           ret = { organization: @content_view_version.organization.name,
                   repository_mapping: {},
                   content_view: @content_view_version.content_view.name,
-                  content_view_version: {
-                    major: @content_view_version.major,
-                    minor: @content_view_version.minor
-                  }
+                  content_view_version: @content_view_version.slice(:major, :minor),
+                  incremental: @from_content_view_version.present?
           }
+
+          unless @from_content_view_version.blank?
+            ret[:from_content_view_version] = {
+              major: @from_content_view_version.major,
+              minor: @from_content_view_version.minor
+            }
+          end
+
           repositories.each do |repo|
             next if repo.version_href.blank?
             pulp3_repo = fetch_repository_info(repo.version_href).name
@@ -61,6 +162,22 @@ module Katello
           end
           ret
         end
+
+        def self.find_library_export_view(create_by_default: false,
+                                          destination_server:,
+                                          organization:)
+          name = "Export-Library"
+          name += "-#{destination_server}" unless destination_server.blank?
+          select_method = create_by_default ? :first_or_create : :first
+          ::Katello::ContentView.where(name: name, organization: organization).send(select_method)
+        end
+
+        def self.generate_product_repo_strings(repositories:)
+          repositories.map do |repo|
+            _("Product: '%{product}', Repository: '%{repository}'" % { product: repo.product.name,
+                                                                       repository: repo.name})
+          end
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import.rb

@@ -3,17 +3,17 @@ module Katello
     module ContentViewVersion
       class Import
         include ImportExportCommon
-        BASEDIR = '/var/lib/pulp'.freeze
 
-        def initialize(smart_proxy:, content_view_version: nil, path: nil)
+        def initialize(smart_proxy:, content_view_version: nil, path: nil, metadata: nil)
           @smart_proxy = smart_proxy
           @content_view_version = content_view_version
           @path = path
+          @metadata = metadata
         end
 
         def repository_mapping
           mapping = {}
-          metadata[:repository_mapping].each do |key, value|
+          @metadata[:repository_mapping].each do |key, value|
             repo = @content_view_version.importable_repositories.joins(:root, :product).
                         where("#{::Katello::Product.table_name}" => {:name => value[:product]},
                                                   "#{::Katello::RootRepository.table_name}" => {:name => value[:repository]}).first
@@ -30,7 +30,7 @@ module Katello
         end
 
         def create_import(importer_href)
-          [api.import_api.create(importer_href, toc: "#{@path}/#{metadata[:toc]}")]
+          [api.import_api.create(importer_href, toc: "#{@path}/#{@metadata[:toc]}")]
         end
 
         def fetch_import(importer_href)
@@ -43,43 +43,48 @@ module Katello
           api.importer_api.delete(importer_href)
         end
 
-        def metadata
-          @metadata ||= self.class.metadata(@path)
+        def self.check!(content_view:, metadata:, path:)
+          ImportValidator.new(content_view: content_view, metadata: metadata, path: path).check!
         end
 
-        class << self
-          def metadata(path)
-            JSON.parse(File.read("#{path}/#{Export::METADATA_FILE}")).with_indifferent_access
-          end
+        def self.reset_content_view_repositories_from_metadata!(content_view:, metadata:)
+          # Given metadata from the dump and a content view
+          # this method
+          # 1) Fetches ids of the library repos whose product name, repo name amd redhat?
+          # =>  match values provided in the metadata's repository mapping
+          # 2) Removes all the repositories associated to this content view
+          # 3) Adds the repositories matched from the dump
+          # The main intent of this method is to assume that the user intends for the
+          # content view to exaclty look like what is specified in metadata
 
-          def check_permissions!(path)
-            fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
-            fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
-            metadata_file = "#{path}/#{::Katello::Pulp3::ContentViewVersion::Export::METADATA_FILE}"
-            fail _("Could not find metadata.json at '%s'." % metadata_file) unless File.exist?(metadata_file)
-            fail _("Unable to read the metadata.json at '%s'." % metadata_file) unless File.readable?(metadata_file)
-            fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
-            Dir.glob("#{path}/*").each do |file|
-              next if file == metadata_file
-              fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
-            end
+          repos_in_library = Katello::Repository.
+                    in_default_view.
+                    yum_type.
+                    joins(:product => :provider, :content_view_version => :content_view).
+                    joins(:root).
+                    where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                    pluck("#{::Katello::Repository.table_name}.id",
+                          "#{::Katello::RootRepository.table_name}.name",
+                          "#{::Katello::Product.table_name}.name",
+                          "#{::Katello::Provider.table_name}.provider_type"
+                          )
+          repos_in_library_map = {}
+          # repos_in_library_map is going to look like {['repo1', 'product1', false] => 100, ['repo1', 'product1', true] => 200 }
+          repos_in_library.each do |id, repo, product, provider_type|
+            repos_in_library_map[[repo, product, provider_type == Katello::Provider::REDHAT]] = id
           end
 
-          def pulp_user_accessible?(path)
-            pulp_info = fetch_pulp_user_info
-            return false if pulp_info.blank?
-
-            stat = File.stat(path)
-            stat.gid.to_s == pulp_info.gid ||
-              stat.uid.to_s == pulp_info.uid ||
-              stat.mode.to_s(8)[-1].to_i >= 4
+          repo_ids = metadata[:repository_mapping].values.map do |repo|
+            repos_in_library_map[[repo[:repository], repo[:product], repo[:redhat]]]
           end
+          content_view.update!(repository_ids: repo_ids)
+        end
 
-          def fetch_pulp_user_info
-            pulp_user = nil
-            Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
-            pulp_user
-          end
+        def self.find_or_create_library_import_view(organization)
+          name = ::Katello::ContentView::IMPORT_LIBRARY
+          ::Katello::ContentView.where(name: name,
+                                       organization: organization,
+                                       import_only: true).first_or_create
         end
       end
     end

data/app/services/katello/pulp3/content_view_version/import_export_common.rb

@@ -19,25 +19,15 @@ module Katello
           repo_api.read(version_href_to_repository_href(version_href))
         end
 
-        def repository_hrefs
-          version_hrefs.map { |href| version_href_to_repository_href(href) }.uniq
-        end
-
-        def version_hrefs
-          repositories.pluck(:version_href).compact
-        end
-
-        def repositories
-          if @content_view_version.default?
-            @content_view_version.repositories.yum_type
-          else
-            @content_view_version.archived_repos.yum_type
-          end
-        end
-
         def version_href_to_repository_href(version_href)
           version_href.split("/")[0..-3].join("/") + "/"
         end
+
+        def zero_version_href(repository_href)
+          #  /pulp/api/v3/repositories/rpm/rpm/e59c4334-81d2-4d6b-a1a1-b61fa55ed664/versions/0/
+          repository_href += "/" unless repository_href.ends_with?('/')
+          "#{repository_href}versions/0/"
+        end
       end
     end
   end

data/app/services/katello/pulp3/content_view_version/import_validator.rb

@@ -0,0 +1,114 @@
+module Katello
+  module Pulp3
+    module ContentViewVersion
+      class ImportValidator
+        BASEDIR = '/var/lib/pulp'.freeze
+        attr_accessor :metadata, :path, :content_view
+        def initialize(content_view:, path:, metadata:)
+          self.content_view = content_view
+          self.path = path
+          self.metadata = metadata
+        end
+
+        def check!
+          check_permissions!
+          unless content_view.default?
+            ensure_importing_cvv_does_not_exist!
+            ensure_from_cvv_exists!
+          end
+          ensure_repositories_metadata_are_in_the_library!
+        end
+
+        def ensure_importing_cvv_does_not_exist!
+          major = metadata[:content_view_version][:major]
+          minor = metadata[:content_view_version][:minor]
+
+          if ::Katello::ContentViewVersion.where(major: major, minor: minor, content_view: content_view).exists?
+            fail _("Content View Version specified in the metadata - '%{name}' already exists. "\
+                    "If you wish to replace the existing version, delete %{name} and try again. " % { name: "#{content_view.name} #{major}.#{minor}" })
+          end
+        end
+
+        def ensure_from_cvv_exists!
+          major = metadata[:content_view_version][:major]
+          minor = metadata[:content_view_version][:minor]
+
+          if metadata[:from_content_view_version].present?
+            from_major = metadata[:from_content_view_version][:major]
+            from_minor = metadata[:from_content_view_version][:minor]
+
+            unless ::Katello::ContentViewVersion.where(major: from_major, minor: from_minor, content_view: content_view).exists?
+              fail _("Prior Content View Version specified in the metadata - '%{name}' does not exist. "\
+                      "Please import the metadata for '%{name}' before importing '%{current}' " % { name: "#{content_view.name} #{from_major}.#{from_minor}",
+                                                                                                    current: "#{content_view.name} #{major}.#{minor}"})
+            end
+          end
+        end
+
+        def ensure_repositories_metadata_are_in_the_library!
+          repos_in_library = Katello::Repository.
+                              in_default_view.
+                              yum_type.
+                              joins(:product => :provider, :content_view_version => :content_view).
+                              joins(:root).
+                              where("#{::Katello::ContentView.table_name}.organization_id" => content_view.organization_id).
+                              pluck("#{::Katello::Product.table_name}.name",
+                                    "#{::Katello::RootRepository.table_name}.name",
+                                    "#{::Katello::Provider.table_name}.provider_type"
+                                    )
+
+          # repos_in_library look like [["prod1", "repo1", "Anonymous"], ["prod2", "repo2", "Red Hat"]]
+          product_repos_in_library = repos_in_library.map { |product, repo, provider| [product, repo, provider == ::Katello::Provider::REDHAT] }
+          product_repos_in_metadata = metadata[:repository_mapping].values.map { |repo| [repo[:product], repo[:repository], repo[:redhat]] }
+          # product_repos_in_library & product_repos_in_metadata look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos_not_in_library = product_repos_in_metadata - product_repos_in_library
+          unless product_repos_not_in_library.blank?
+            repos_in_import = generate_product_repo_i18n_string(product_repos_not_in_library)
+            fail _("The following repositories provided in the import metadata are either not available in the Library or are of incorrect Respository Type. "\
+                    "Please add or enable the repositories before importing\n "\
+                    "%{repos}" % { content_view: content_view.name, repos: repos_in_import.join("")}
+                  )
+          end
+        end
+
+        def check_permissions!
+          fail _("Invalid path specified.") if path.blank? || !File.directory?(path)
+          fail _("The import path must be in a subdirectory under '%s'." % BASEDIR) unless path.starts_with?(BASEDIR)
+          fail _("Pulp user or group unable to read content in '%s'." % path) unless pulp_user_accessible?(path)
+
+          Dir.glob("#{path}/*").each do |file|
+            fail _("Pulp user or group unable to read '%s'." % file) unless pulp_user_accessible?(file)
+          end
+          toc_path = "#{path}/#{metadata[:toc]}"
+          fail _("The TOC file specified in the metadata does not exist. %s " % toc_path) unless File.exist?(toc_path)
+        end
+
+        def pulp_user_accessible?(path)
+          pulp_info = fetch_pulp_user_info
+          return false if pulp_info.blank?
+
+          stat = File.stat(path)
+          stat.gid.to_s == pulp_info.gid ||
+            stat.uid.to_s == pulp_info.uid ||
+            stat.mode.to_s(8)[-1].to_i >= 4
+        end
+
+        def fetch_pulp_user_info
+          pulp_user = nil
+          Etc.passwd { |u| pulp_user = u if u.name == 'pulp' }
+          pulp_user
+        end
+
+        def generate_product_repo_i18n_string(product_repos)
+          # product_repos look like [["prod1", "repo1", false], ["prod2", "repo2", false]]
+          product_repos.map do |product, repo, redhat|
+            repo_type = redhat ? _("Red Hat") : _("Custom")
+            _("\n* Product = '%{product}', Repository = '%{repository}', Repository Type = '%{repo_type}'" % { product: product,
+                                                                                                               repository: repo,
+                                                                                                               repo_type: repo_type})
+          end
+        end
+      end
+    end
+  end
+end