katello 4.12.1 → 4.13.0

data/app/models/katello/host/content_facet.rb

@@ -44,7 +44,7 @@ module Katello
 
       validates_with ::AssociationExistsValidator, attributes: [:content_source]
       validates_with Katello::Validators::GeneratedContentViewValidator
-      validates_associated :content_view_environment_content_facets
+      validates_associated :content_view_environment_content_facets, :message => _("invalid: The content source must sync the lifecycle environment assigned to the host. See the logs for more information.")
       validates :host, :presence => true, :allow_blank => false
 
       attr_accessor :cves_changed

data/app/models/katello/host/subscription_facet.rb

@@ -67,7 +67,7 @@ module Katello
           self.purpose_addon_ids = consumer_params['addOns'].map { |addon_name| ::Katello::PurposeAddon.find_or_create_by(name: addon_name).id }
         end
 
-        unless consumer_params['releaseVer'].blank?
+        unless consumer_params['releaseVer'].nil?
           release = consumer_params['releaseVer']
           release = release['releaseVer'] if release.is_a?(Hash)
           self.release_version = release

data/app/models/katello/ping.rb

@@ -2,7 +2,7 @@ module Katello
   class Ping
     OK_RETURN_CODE = 'ok'.freeze
     FAIL_RETURN_CODE = 'FAIL'.freeze
-    PACKAGES = %w(katello candlepin pulp foreman hammer).freeze
+    PACKAGES = %w(katello candlepin pulp foreman hammer dynflow).freeze
 
     class << self
       def services(capsule_id = nil)

data/app/models/katello/repository.rb

@@ -118,7 +118,7 @@ module Katello
     end}
 
     before_validation :set_pulp_id
-    before_validation :set_container_repository_name, :if => :docker?
+    before_validation :set_container_repository_name, :unless => :skip_container_name?
 
     scope :has_url, -> { joins(:root).where.not("#{RootRepository.table_name}.url" => nil) }
     scope :on_demand, -> { joins(:root).where("#{RootRepository.table_name}.download_policy" => ::Katello::RootRepository::DOWNLOAD_ON_DEMAND) }
@@ -268,6 +268,10 @@ module Katello
       self.content_view_version.content_view
     end
 
+    def skip_container_name?
+      self.library_instance? && self.root.docker? && self.root.is_container_push
+    end
+
     def library_instance?
       self.content_view.default?
     end
@@ -651,6 +655,33 @@ module Katello
                                 for_resource(self).order(:started_at).last
     end
 
+    def blocking_task
+      blocking_task_labels = [
+        ::Actions::Katello::Repository::Sync.name,
+        ::Actions::Katello::Repository::UploadFiles.name,
+        ::Actions::Katello::Repository::RemoveContent.name,
+        ::Actions::Katello::Repository::MetadataGenerate.name
+      ]
+      ForemanTasks::Task::DynflowTask.where(:label => blocking_task_labels)
+                                     .where.not(state: 'stopped')
+                                     .for_resource(self)
+                                     .order(:started_at)
+                                     .last
+    end
+
+    def check_ready_to_act!
+      blocking_tasks = content_views&.map { |cv| cv.blocking_task }&.compact
+
+      if blocking_tasks&.any?
+        errored_tasks = blocking_tasks
+                          .uniq
+                          .map { |task| "- #{Setting['foreman_url']}/foreman_tasks/tasks/#{task&.id}" }
+                          .join("\n")
+        fail _("This repository has pending tasks in associated content views. Please wait for the tasks: " + errored_tasks +
+               " to complete before proceeding.")
+      end
+    end
+
     # returns other instances of this repo with the same library
     # equivalent of repo
     def environmental_instances(view)

data/app/models/katello/root_repository.rb

@@ -17,7 +17,7 @@ module Katello
     DOWNLOAD_POLICIES = [DOWNLOAD_IMMEDIATE, DOWNLOAD_ON_DEMAND].freeze
 
     IGNORABLE_CONTENT_UNIT_TYPES = %w(srpm treeinfo).freeze
-    CHECKSUM_TYPES = %w(sha1 sha256).freeze
+    CHECKSUM_TYPES = %w(sha256 sha384 sha512).freeze
 
     SUBSCRIBABLE_TYPES = [Repository::YUM_TYPE, Repository::OSTREE_TYPE, Repository::DEB_TYPE].freeze
     SKIPABLE_METADATA_TYPES = [Repository::YUM_TYPE, Repository::DEB_TYPE].freeze
@@ -112,6 +112,8 @@ module Katello
         only_integer: true
       }
 
+    validates :container_push_name_format, inclusion: { in: ['label', 'id'].freeze, allow_nil: true}
+
     scope :subscribable, -> { where(content_type: RootRepository::SUBSCRIBABLE_TYPES) }
     scope :skipable_metadata_check, -> { where(content_type: RootRepository::SKIPABLE_METADATA_TYPES) }
     scope :has_url, -> { where.not(:url => nil) }
@@ -195,7 +197,7 @@ module Katello
     def ensure_docker_repo_unprotected
       unless unprotected
         errors.add(:base, _("Container Image Repositories are not protected at this time. " \
-                             "They need to be published via http to be available to containers."))
+                            "They need to be published via http to be available to containers."))
       end
     end
 
@@ -387,10 +389,6 @@ module Katello
       Katello::RepositoryTypeManager.generic_repository_types(false).values.map(&:id).map(&:to_s).flatten.include? self.content_type
     end
 
-    def metadata_generate_needed?
-      (%w(unprotected checksum_type container_repsoitory_name) & previous_changes.keys).any?
-    end
-
     def using_mirrored_content?
       self.mirroring_policy != Katello::RootRepository::MIRRORING_POLICY_ADDITIVE
     end

data/app/models/katello/trace_status.rb

@@ -36,7 +36,7 @@ module Katello
     end
 
     def to_status(_options = {})
-      traces = host.host_traces.pluck(:app_type)
+      traces = host.host_traces.reload.pluck(:app_type)
       traces.delete(Katello::HostTracer::TRACE_APP_TYPE_SESSION)
 
       if traces.include?(Katello::HostTracer::TRACE_APP_TYPE_STATIC)

data/app/services/katello/content_unit_indexer.rb

@@ -55,6 +55,15 @@ module Katello
       end
     end
 
+    def reimport_units
+      units_from_pulp.each do |units|
+        to_update = units.map do |unit|
+          @service_class.generate_model_row(unit)
+        end
+        @model_class.upsert_all(to_update, unique_by: :pulp_id)
+      end
+    end
+
     def import_associations(units)
       pulp_id_to_id = self.class.pulp_id_to_id_map(@content_type, units.map { |unit| unit[@service_class.unit_identifier] })
       @service_class.insert_child_associations(units, pulp_id_to_id) if @service_class.respond_to?(:insert_child_associations)

data/app/services/katello/pulp3/alternate_content_source.rb

@@ -88,18 +88,16 @@ module Katello
       end
 
       def get_remote(href = smart_proxy_acs.remote_href)
-        acs.base_url&.start_with?('uln') ? api.remotes_uln_api.read(href) : api.remotes_api.read(href)
+        api.get_remotes_api(href: href).read(href)
       end
 
-      def update_remote
-        api.remotes_api.partial_update(smart_proxy_acs.remote_href, remote_options)
+      def update_remote(href = smart_proxy_acs.remote_href)
+        api.get_remotes_api(href: href).partial_update(href, remote_options)
       end
 
-      # The old repo URL is needed to determine which remote API to use.
       def delete_remote(options = {})
         options[:href] ||= smart_proxy_acs.remote_href
-        options[:old_url] ||= remote_options[:url]
-        ignore_404_exception { options[:old_url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
+        ignore_404_exception { api.get_remotes_api(href: options[:href]).delete(options[:href]) } if options[:href]
       end
 
       def create

data/app/services/katello/pulp3/api/core.rb

@@ -47,6 +47,11 @@ module Katello
           fail NotImplementedError
         end
 
+        # Method is called with either :url or :href parameters for the sake of yum content.
+        def get_remotes_api(*)
+          remotes_api
+        end
+
         def publications_api
           repository_type.publications_api_class.new(api_client) #Optional
         end
@@ -136,6 +141,10 @@ module Katello
           client
         end
 
+        def repair_api
+          PulpcoreClient::RepairApi.new(core_api_client)
+        end
+
         def uploads_api
           PulpcoreClient::UploadsApi.new(core_api_client)
         end
@@ -167,6 +176,14 @@ module Katello
           nil
         end
 
+        def purge_class
+          PulpcoreClient::Purge
+        end
+
+        def purge_completed_tasks
+          tasks_api.purge(purge_class.new(finished_before: DateTime.now - Setting[:completed_pulp_task_protection_days]))
+        end
+
         def delete_orphans
           [orphans_api.cleanup(PulpcoreClient::OrphansCleanup.new(orphan_protection_time: (smart_proxy.pulp_mirror? ? 0 : Setting[:orphan_protection_time])))]
         end
@@ -230,6 +247,10 @@ module Katello
           end
         end
 
+        def repair
+          repair_api.post(PulpcoreClient::Repair.new(verify_checksums: true))
+        end
+
         def self.fetch_from_list
           page_size = Setting[:bulk_load_size]
           page_opts = { "offset" => 0, limit: page_size }

data/app/services/katello/pulp3/api/docker.rb

@@ -15,6 +15,10 @@ module Katello
         def recursive_add_api
           PulpContainerClient::ContainerRecursiveAddApi.new(api_client)
         end
+
+        def container_push_api
+          PulpContainerClient::RepositoriesContainerPushApi.new(api_client)
+        end
       end
     end
   end

data/app/services/katello/pulp3/api/yum.rb

@@ -32,6 +32,17 @@ module Katello
           PulpRpmClient::RemotesUlnApi.new(api_client)
         end
 
+        def get_remotes_api(href: nil, url: nil)
+          fail 'Provide exactly one of href or url for yum remote selection!' if url.blank? && href.blank?
+          fail 'The href must be a pulp_rpm remote href!' if href && !href.start_with?('/pulp/api/v3/remotes/rpm/')
+
+          if href&.start_with?('/pulp/api/v3/remotes/rpm/uln/') || url&.start_with?('uln')
+            remotes_uln_api
+          else
+            remotes_api
+          end
+        end
+
         def copy_api
           PulpRpmClient::RpmCopyApi.new(api_client)
         end

data/app/services/katello/pulp3/docker_manifest.rb

@@ -26,7 +26,11 @@ module Katello
         {
           schema_version: unit['schema_version'],
           digest: unit['digest'],
-          pulp_id: unit[unit_identifier]
+          pulp_id: unit[unit_identifier],
+          annotations: unit['annotations'],
+          labels: unit['labels'],
+          is_bootable: unit['is_bootable'],
+          is_flatpak: unit['is_flatpak']
         }
       end
     end

data/app/services/katello/pulp3/repository/generic.rb

@@ -3,7 +3,7 @@ module Katello
     class Repository
       class Generic < ::Katello::Pulp3::Repository
         def copy_content_for_source(source_repository, _options = {})
-          copy_units_by_href(source_repository.files.pluck(:pulp_id))
+          copy_units_by_href(source_repository.generic_content_units&.pluck(:pulp_id))
         end
 
         def distribution_options(path)

data/app/services/katello/pulp3/repository/yum.rb

@@ -22,12 +22,7 @@ module Katello
 
         def publication_options(repository_version)
           options = super(repository_version)
-          options.merge(
-            {
-              metadata_checksum_type: root.checksum_type,
-              package_checksum_type: root.checksum_type
-            }
-          )
+          options.merge(checksum_type: root.checksum_type)
         end
 
         def specific_create_options

data/app/services/katello/pulp3/repository.rb

@@ -91,16 +91,21 @@ module Katello
       end
 
       def remote_partial_update
-        if remote_options[:url]&.start_with?('uln')
-          api.remotes_uln_api.partial_update(repo.remote_href, remote_options)
-        else
-          api.remotes_api.partial_update(repo.remote_href, remote_options)
+        url_type = remote_options[:url]&.start_with?('uln') ? 'uln' : 'default'
+        remote_type = repo.remote_href.start_with?('/pulp/api/v3/remotes/rpm/uln/') ? 'uln' : 'default'
+        href = repo.remote_href
+
+        if url_type == remote_type
+          api.get_remotes_api(href: href).partial_update(href, remote_options)
+        else # We need to recreate a remote of the correct type!
+          create_remote
+          delete_remote(href: href)
         end
       end
 
       def delete_remote(options = {})
         options[:href] ||= repo.remote_href
-        ignore_404_exception { remote_options[:url]&.start_with?('uln') ? api.remotes_uln_api.delete(options[:href]) : api.remotes_api.delete(options[:href]) } if options[:href]
+        ignore_404_exception { api.get_remotes_api(href: options[:href]).delete(options[:href]) } if options[:href]
       end
 
       def self.instance_for_type(repo, smart_proxy)
@@ -139,7 +144,7 @@ module Katello
       end
 
       def get_remote(href = repo.remote_href)
-        repo.url&.start_with?('uln') ? api.remotes_uln_api.read(href) : api.remotes_api.read(href)
+        api.get_remotes_api(href: href).read(href)
       end
 
       def get_distribution(href = distribution_reference.href)
@@ -210,6 +215,10 @@ module Katello
         api.publications_api.create(publication_data)
       end
 
+      def delete_publication
+        ignore_404_exception { api.publications_api.delete(repo.publication_href) } if repo.publication_href
+      end
+
       def publication_options(repository_version)
         {
           repository_version: repository_version
@@ -329,6 +338,17 @@ module Katello
 
       def delete_version
         ignore_404_exception { api.repository_versions_api.delete(repo.version_href) } unless version_zero?
+      rescue api.api_exception_class => e
+        if e.message.include?("are currently being used to distribute content")
+          Rails.logger.warn "Exception when calling repository_versions_api->delete: #{e}"
+          publication_href = repo.publication_href
+          Rails.logger.warn "Trying to delete publication #{publication_href} for repository #{repo.id}}"
+          Rails.logger.error "Could not delete version: #{repo.version_href} because conflicting publication could not be looked up" unless publication_href
+          if publication_href
+            ignore_404_exception { api.publications_api.delete(publication_href) }
+            ignore_404_exception { api.repository_versions_api.delete(repo.version_href) }
+          end
+        end
       end
 
       def create_version(options = {})

data/app/services/katello/pulp3/repository_mirror.rb

@@ -24,11 +24,9 @@ module Katello
       def refresh_entities
         href = remote_href
         if href
-          if remote_options[:url]&.start_with?('uln')
-            [api.remotes_uln_api.partial_update(href, remote_options)]
-          else
-            [api.remotes_api.partial_update(href, remote_options)]
-          end
+          # Do not consider remotes_uln_api, since the Katello server is not a ULN server. Even if the sync
+          # to Katello used ULN, the sync from Katello server to smart proxy will use a normal RPM remote!
+          [api.remotes_api.partial_update(href, remote_options)]
         else
           create_remote
           []
@@ -111,13 +109,10 @@ module Katello
       end
 
       def create_remote
-        if remote_options[:url]&.start_with?('uln')
-          remote_file_data = @repo_service.api.class.remote_uln_class.new(remote_options)
-          api.remotes_uln_api.create(remote_file_data)
-        else
-          remote_file_data = @repo_service.api.remote_class.new(remote_options)
-          api.remotes_api.create(remote_file_data)
-        end
+        # Do not consider remotes_uln_api, since the Katello server is not a ULN server. Even if the sync
+        # to Katello used ULN, the sync from Katello server to smart proxy will use a normal RPM remote!
+        remote_file_data = @repo_service.api.remote_class.new(remote_options)
+        api.remotes_api.create(remote_file_data)
       end
 
       def compute_remote_options
@@ -240,6 +235,12 @@ module Katello
         distribution_data = api.distribution_class.new(distribution_options(path))
         repo_service.distributions_api.create(distribution_data)
       end
+
+      def repair
+        data = api.repair_class.new
+        fail "Could not lookup a version_href for repo #{repo_service.repo.id}" if version_href.nil?
+        api.repository_versions_api.repair(version_href, data)
+      end
     end
   end
 end

data/app/services/katello/pulp3/service_common.rb

@@ -9,11 +9,7 @@ module Katello
           remote_file_data = api.remote_class.new(remote_options)
         end
         reformat_api_exception do
-          if remote_options[:url]&.start_with?('uln')
-            response = api.remotes_uln_api.create(remote_file_data)
-          else
-            response = api.remotes_api.create(remote_file_data)
-          end
+          response = api.get_remotes_api(url: remote_options[:url]).create(remote_file_data)
         end
         response
       end
@@ -37,11 +33,7 @@ module Katello
         end
 
         reformat_api_exception do
-          if remote_options[:url]&.start_with?('uln')
-            response = api.remotes_uln_api.create(remote_file_data)
-          else
-            response = api.remotes_api.create(remote_file_data)
-          end
+          response = api.get_remotes_api(url: remote_options[:url]).create(remote_file_data)
           #delete is async, but if its not properly deleted, orphan cleanup will take care of it later
           delete_remote(href: response.pulp_href)
         end

data/app/services/katello/pulp3/smart_proxy_repository.rb

@@ -35,13 +35,11 @@ module Katello
 
       def delete_orphan_repository_versions
         tasks = []
-
         orphan_repository_versions.each do |api, version_hrefs|
           tasks << version_hrefs.collect do |href|
             api.repository_versions_api.delete(href)
           end
         end
-
         tasks.flatten
       end
 

data/app/services/katello/ui_notifications/subscriptions/manifest_expire_soon_warning.rb

@@ -0,0 +1,75 @@
+module Katello
+  module UINotifications
+    module Subscriptions
+      class ManifestExpireSoonWarning
+        class << self
+          def deliver!
+            ::Organization.unscoped.all.each do |organization|
+              if (notification = existing_notification(organization))
+                days_remaining = organization.manifest_expire_days_remaining
+                if days_remaining == 0 || days_remaining > Setting[:expire_soon_days].to_i
+                  # if the manifest has already expired, delete the notification;
+                  # user will have a ManifestExpiredWarning instead.
+                  # If user changes the expire_soon_days setting, remove notifications
+                  # that are no longer relevant.
+                  Rails.logger.debug("ManifestExpireSoonWarning: deleting notification for #{organization.name}")
+                  notification.destroy
+                  next
+                end
+                # don't update if the message hasn't changed
+                next unless message(organization).to_s !=
+                  notification.message.to_s
+                notification.update(
+                  :message => message(organization),
+                  :actions => actions
+                )
+              else
+                next unless organization.manifest_expiring_soon?
+                ::Notification.create!(
+                  :subject => organization,
+                  :initiator => User.anonymous_admin,
+                  :audience => Notification::AUDIENCE_SUBJECT,
+                  :message => message(organization),
+                  :actions => actions,
+                  :notification_blueprint => blueprint
+                )
+              end
+            end
+          end
+
+          def existing_notification(subject)
+            matching_notification = Notification.unscoped.find_by(:subject => subject, :notification_blueprint => blueprint)
+            return false if matching_notification.blank?
+            matching_notification
+          end
+
+          def message(organization)
+            ::UINotifications::StringParser.new(
+              blueprint.message,
+              :manifest_expire_date => organization.manifest_expiration_date&.to_date,
+              :subject => organization,
+              :days_remaining => organization.manifest_expire_days_remaining
+            )
+          end
+
+          def actions
+            {
+              :links => [
+                {
+                  :href => "/subscriptions",
+                  :title => _('Subscriptions'),
+                  :external => false
+                }
+              ]
+            }
+          end
+
+          def blueprint
+            @blueprint ||= NotificationBlueprint.unscoped.find_by(
+              :name => 'manifest_expire_soon_warning')
+          end
+        end
+      end
+    end
+  end
+end

data/app/views/foreman/job_templates/update_package_-_katello_ansible_default.erb

@@ -13,4 +13,8 @@ provider_type: Ansible
 kind: job_template
 %>
 
-<%= render_template('Run Command - Ansible Default', :command => "yum -y update #{input('package')}") %>
+<%
+pkgs = input('package')
+pkgs = '"*"' if pkgs.empty?
+-%>
+<%= render_template('Package Action - Ansible Default', :state => 'latest', :name => pkgs) %>

data/app/views/foreman/job_templates/update_packages_by_search_query_-_katello_ansible_default.erb

@@ -3,7 +3,7 @@ kind: job_template
 name: Update packages by search query - Katello Ansible Default
 job_category: Katello
 description_format: 'Update package(s) %{Packages search query}'
-feature: katello_package_remove_by_search
+feature: katello_package_update_by_search
 provider_type: Ansible
 template_inputs:
 - name: Packages search query
@@ -22,7 +22,7 @@ template_inputs:
   versions: input('Selected update versions')
 ) -%>
 <% if package_names.empty? -%>
-<%= render_template('Run Command - Ansible Default', :command => "yum -y update") %>
+<%= render_template('Package Action - Ansible Default', :state => 'latest', :name => '"*"') %>
 <% else -%>
 ---
 - hosts: all

data/app/views/foreman/job_templates/upload_profile.erb

@@ -0,0 +1,16 @@
+<%#
+kind: job_template
+name: Upload profile - Katello Script Default
+job_category: Katello
+model: JobTemplate
+provider_type: script
+description_format: Upload package profile for a host
+feature: katello_upload_profile
+%>
+#!/bin/sh
+<% if @host.operatingsystem.family == 'Redhat' -%>
+dnf uploadprofile --force-upload
+<% else -%>
+package-profile-upload --force-upload
+<% end -%>
+subscription-manager repos

data/app/views/foreman/smart_proxies/_content_tab.html.erb

@@ -1,3 +1,5 @@
-<%= javascript_include_tag *webpack_asset_paths('katello', extension: 'js') %>
+<% content_for(:javascripts) do %>
+    <%= webpacked_plugins_js_for :katello %>
+<% end %>
 <% @smartProxyId= @smart_proxy.id %>
 <%= react_component('Content', smartProxyId: @smartProxyId, organizationId: Organization.current&.id,)  %>

data/app/views/katello/api/v2/content_view_filter_rules/show.json.rabl

@@ -15,4 +15,13 @@ attributes :architecture, :if => lambda { |rule| rule.respond_to?(:architecture)
 attributes :types, :if => lambda { |rule| rule.respond_to?(:types) && !rule.types.blank? }
 attributes :date_type, :if => lambda { |rule| rule.respond_to?(:date_type) }
 attributes :module_stream_id, :if => lambda { |rule| rule.respond_to?(:module_stream_id) && !rule.module_stream_id.blank? }
+if @resource&.try(:module_stream)
+  node :module_stream do |rule|
+    {
+      :module_stream_id => rule.module_stream.id,
+      :module_stream_name => rule.module_stream.name,
+      :module_stream_stream => rule.module_stream.stream
+    }
+  end
+end
 extends 'katello/api/v2/common/timestamps'

data/app/views/katello/api/v2/docker_manifests/show.json.rabl

@@ -1,6 +1,7 @@
 object @resource
 
 attributes :id, :schema_version, :digest, :manifest_type
+attributes :annotations, :labels, :is_bootable, :is_flatpak
 
 child :docker_tags => :tags do
   attributes :associated_meta_tag_identifier => :id

data/app/views/katello/api/v2/organizations/show.json.rabl

@@ -3,10 +3,18 @@ object @organization
 extends "api/v2/taxonomies/show"
 
 attributes :task_id, :label, :redhat_repository_url
-
+attributes :manifest_expiration_date, :manifest_expire_days_remaining
 attributes :system_purposes, :system_purposes
 attributes :service_levels, :service_level
 
+node :manifest_expiring_soon do |org|
+  org.manifest_expiring_soon?
+end
+
+node :manifest_expired do |org|
+  org.manifest_expired?
+end
+
 node :simple_content_access do |org|
   org.simple_content_access?
 end

data/app/views/overrides/activation_keys/_host_environment_select.html.erb

@@ -8,7 +8,7 @@
 
 <% if edit_action? && !using_hostgroups_page? && !using_discovered_hosts_page? %>
   <div style="margin-left: 270px">
-    <%= link_to _("Change content source"), "/change_host_content_source?host_id=#{@host.id}" %>
+    <%= link_to _("Change content source"), "/change_host_content_source?fromPage=hostEdit&host_id=#{@host.id}&initialContentSourceId=#{@host.content_source_id}" %>
   </div>
 <% end %>