katello 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a68ac9c26daead6a548c835d225d8dbebe0ce33ca5b6315b6505ef64660d9ee6
-  data.tar.gz: '09d66a70cef13109aff1e38a9e409862c5cae6263c5c7aea415399c7cd0e08b0'
+  metadata.gz: 02ff428534ded98d248efb23e9684364f8930cb1175c25a70ad3529963e435a0
+  data.tar.gz: 680e66df3f38df78a5feface53e92cc4129cc4a847fbd5e344ab63e5dd6bfd40
 SHA512:
-  metadata.gz: cd9715ea64ef37bf05df3ae49a209e15f0ffe0c51d8f69d4b3fcba119909349fea1fe1ab4f464240515057e1529304850176ec5a48cbbfb25cd04fb1ee788137
-  data.tar.gz: 6ab0c5c117c51017b37ca1366fc05f64efcc7a17375e2bb4ac9ee3633990baedf68ba505cc29292295f5a01e31ac6fd4bef209d9272e2f5cfce0f3f9c7f29a65
+  metadata.gz: 32ccbc77851e99ff2e16ebb36219871a0e086f86f60da1d3265e26cddbfdb7563c1363fc72ba5d3db34fb8360eb01ddb678fc882a41a16e635397631ab6b767e
+  data.tar.gz: 882219885a8aa1dd105a6b24a630092984d462fcac3a2030fa8fb3e6c112225ac48b00e746f7194bbb4e5ff2fb1d8766610b65434e0555958b4b251668be4cf6

data/app/controllers/katello/api/registry/registry_proxies_controller.rb

@@ -167,11 +167,9 @@ module Katello
       end
 
       if (manifest_response = redirect_client { Resources::Registry::Proxy.get(@_request.fullpath, headers) })
-        #when pulp 2 is removed, this should no longer be needed, and all clients should be redirected
-        logger.debug filter_sensitive_data(manifest_response)
+        #for some requests, we get a redirect, but for others we get the actual manifest in response
         results = JSON.parse(manifest_response)
-
-        response.header['Docker-Content-Digest'] = "sha256:#{Digest::SHA256.hexdigest(manifest_response)}"
+        response.header['Docker-Content-Digest'] = manifest_response.headers[:docker_content_digest]
         # https://docs.docker.com/registry/spec/manifest-v2-2/
         # If its v2 schema 2 only the mediaType attribute will be present in the manifest
         media_type = results['mediaType']

data/app/controllers/katello/api/v2/repositories_controller.rb

@@ -52,9 +52,9 @@ module Katello
       param :upstream_password, String, :desc => N_("Password of the upstream repository user used for authentication")
       param :ostree_upstream_sync_policy, ::Katello::RootRepository::OSTREE_UPSTREAM_SYNC_POLICIES, :desc => N_("policies for syncing upstream ostree repositories")
       param :ostree_upstream_sync_depth, :number, :desc => N_("if a custom sync policy is chosen for ostree repositories then a 'depth' value must be provided")
-      param :deb_releases, String, :desc => N_("comma-separated list of releases to be synced from deb-archive")
-      param :deb_components, String, :desc => N_("comma-separated list of repo components to be synced from deb-archive")
-      param :deb_architectures, String, :desc => N_("comma-separated list of architectures to be synced from deb-archive")
+      param :deb_releases, String, :desc => N_("whitespace-separated list of releases to be synced from deb-archive")
+      param :deb_components, String, :desc => N_("whitespace-separated list of repo components to be synced from deb-archive")
+      param :deb_architectures, String, :desc => N_("whitespace-separated list of architectures to be synced from deb-archive")
       param :ignorable_content, Array, :desc => N_("List of content units to ignore while syncing a yum repository. Must be subset of %s") % RootRepository::IGNORABLE_CONTENT_UNIT_TYPES.join(",")
       param :ansible_collection_requirements, String, :desc => N_("Contents of requirement yaml file to sync from URL")
       param :http_proxy_policy, ::Katello::RootRepository::HTTP_PROXY_POLICIES, :desc => N_("policies for HTTP proxy for content sync")

data/app/lib/actions/katello/agent_action.rb

@@ -15,22 +15,15 @@ module Actions
       def plan(host, options)
         action_subject(host, :hostname => host.name, :content => options[:content])
 
-        # if already dispatched by bulk action use the provided history ID
-        dispatch_history_id = options.dig(:dispatch_histories, host.id.to_s)
-
-        unless dispatch_history_id
-          histories = ::Katello::Agent::Dispatcher.dispatch(
-            self.class.agent_message,
-            [host.id],
-            content: options[:content]
-          )
-
-          dispatch_history_id = histories.first.id
-        end
+        dispatch_history_id = options.dig(:dispatch_histories, host.id.to_s) || ::Katello::Agent::Dispatcher.create_histories(
+          host_ids: [host.id]
+        ).first.id
 
         plan_self(
           host_id: host.id,
-          dispatch_history_id: dispatch_history_id
+          dispatch_history_id: dispatch_history_id,
+          content: options[:content],
+          bulk: options[:bulk]
         )
       end
 
@@ -38,20 +31,23 @@ module Actions
         case event
         when nil
           history = dispatch_history
+          timeout = accept_timeout
 
           if history.finished?
             fail_on_errors
             return
           elsif history.accepted?
-            schedule_timeout(finish_timeout)
-          else
-            schedule_timeout(accept_timeout)
+            timeout = finish_timeout
           end
 
           suspend do |suspended_action|
             history.dynflow_execution_plan_id = suspended_action.execution_plan_id
             history.dynflow_step_id = suspended_action.step_id
             history.save!
+
+            dispatch_message(history) unless input[:bulk]
+
+            schedule_timeout(timeout)
           end
         when Dynflow::Action::Timeouts::Timeout
           process_timeout
@@ -63,6 +59,14 @@ module Actions
         end
       end
 
+      def dispatch_message(history)
+        ::Katello::Agent::Dispatcher.dispatch(
+          self.class.agent_message,
+          [history],
+          content: input[:content]
+        )
+      end
+
       def accept_timeout
         Setting['content_action_accept_timeout']
       end
@@ -79,7 +83,12 @@ module Actions
         end
 
         unless history.finished?
-          fail _("Host did not finish content action in %s seconds.  The task has been cancelled.") % finish_timeout
+          # we could be processing the accept_timeout here
+          # only fail for finish_timeout unless the actual duration has elapsed
+          finish_limit = history.accepted_at + finish_timeout
+          if finish_limit < DateTime.now
+            fail _("Host did not finish content action in %s seconds.  The task has been cancelled.") % finish_timeout
+          end
         end
       end
 

data/app/lib/actions/katello/bulk_agent_action.rb

@@ -3,19 +3,32 @@ module Actions
     class BulkAgentAction < Actions::BulkAction
       def plan(agent_action, hosts, args)
         host_ids = hosts.map(&:id)
-        dispatch_args = {
-          content: args[:content]
-        }
-        histories = ::Katello::Agent::Dispatcher.dispatch(agent_action.agent_message, host_ids, dispatch_args)
+
+        histories = ::Katello::Agent::Dispatcher.create_histories(
+          host_ids: host_ids
+        )
 
         grouped_histories = {}
         histories.each { |h| grouped_histories[h.host_id] = h.id }
         options = {
           dispatch_histories: grouped_histories,
-          content: args[:content]
+          type: agent_action.agent_message,
+          content: args[:content],
+          bulk: true
         }
         super(agent_action, hosts, options)
       end
+
+      def spawn_plans
+        args = input[:args].first
+        histories = ::Katello::Agent::DispatchHistory.where(id: args[:dispatch_histories].slice(*current_batch.map(&:to_s)).values)
+        ::Katello::Agent::Dispatcher.dispatch(
+          args[:type].to_sym,
+          histories,
+          content: args[:content]
+        )
+        super
+      end
     end
   end
 end

data/app/lib/actions/katello/content_view/promote.rb

@@ -2,6 +2,9 @@ module Actions
   module Katello
     module ContentView
       class Promote < Actions::EntryAction
+        extend ApipieDSL::Class
+        include ::Actions::ObservableAction
+
         def plan(version, environments, is_force = false, description = nil, incremental_update = false)
           action_subject(version.content_view)
           version.check_ready_to_promote!(environments)
@@ -18,6 +21,28 @@ module Actions
             end
           end
         end
+
+        def environments
+          input['environments']
+        end
+
+        apipie :class, "A class representing #{self} object" do
+          desc 'This object is available as **@object** variable in
+                webhook templates when a corresponding event occures.
+                The following properties can be used to retrieve the needed information.'
+          name "#{class_scope}"
+          refs "#{class_scope}"
+          sections only: %w[all webhooks]
+          property :task, object_of: 'Task', desc: 'Returns the task to which this action belongs'
+          property :environments, array_of: String, desc: 'Returns the list of environments the content view was promoted to'
+        end
+        include Actions::Katello::JailConcern::Organization
+        include Actions::Katello::JailConcern::ContentView
+        class Jail < ::Actions::ObservableAction::Jail
+          allow :organization_id, :organization_name, :organization_label,
+                :content_view_id, :content_view_name, :content_view_label,
+                :environments
+        end
       end
     end
   end

data/app/lib/actions/katello/content_view/publish.rb

@@ -3,7 +3,9 @@ module Actions
   module Katello
     module ContentView
       class Publish < Actions::EntryAction
+        extend ApipieDSL::Class
         include ::Katello::ContentViewHelper
+        include ::Actions::ObservableAction
         attr_accessor :version
         # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity
         def plan(content_view, description = "", options = {importing: false}) # rubocop:disable Metrics/PerceivedComplexity
@@ -145,6 +147,14 @@ module Actions
         rescue ::Katello::Errors::CapsuleCannotBeReached # skip any capsules that cannot be connected to
         end
 
+        def content_view_version_id
+          input['content_view_version_id']
+        end
+
+        def content_view_version_name
+          input['content_view_version_name']
+        end
+
         private
 
         def include_other_components(override_components, content_view)
@@ -197,6 +207,25 @@ module Actions
             end
           end
         end
+
+        apipie :class, "A class representing #{self} object" do
+          desc 'This object is available as **@object** variable in
+                webhook templates when a corresponding event occures.
+                The following properties can be used to retrieve the needed information.'
+          name "#{class_scope}"
+          refs "#{class_scope}"
+          sections only: %w[all webhooks]
+          property :task, object_of: 'Task', desc: 'Returns the task to which this action belongs'
+          property :content_view_version_id, Integer, desc: 'Returns published content view version id'
+          property :content_view_version_name, String, desc: 'Returns published content view version name'
+        end
+        include Actions::Katello::JailConcern::Organization
+        include Actions::Katello::JailConcern::ContentView
+        class Jail < ::Actions::ObservableAction::Jail
+          allow :organization_id, :organization_name, :organization_label,
+                :content_view_id, :content_view_name, :content_view_label,
+                :content_view_version_id, :content_view_version_name
+        end
       end
     end
   end

data/app/lib/actions/katello/jail_concern/content_view.rb

@@ -0,0 +1,30 @@
+module Actions
+  module Katello
+    module JailConcern
+      module ContentView
+        def content_view_id
+          input['content_view']['id']
+        end
+
+        def content_view_name
+          input['content_view']['name']
+        end
+
+        def content_view_label
+          input['content_view']['label']
+        end
+
+        def self.included(base)
+          super
+          base.instance_eval do
+            apipie :class do
+              property :content_view_id, Integer, desc: 'Returns the id of the content view'
+              property :content_view_name, String, desc: 'Returns the name of the content view'
+              property :content_view_label, String, desc: 'Returns the label of the content view'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/jail_concern/organization.rb

@@ -0,0 +1,30 @@
+module Actions
+  module Katello
+    module JailConcern
+      module Organization
+        def organization_id
+          input['organization']['id']
+        end
+
+        def organization_name
+          input['organization']['name']
+        end
+
+        def organization_label
+          input['organization']['label']
+        end
+
+        def self.included(base)
+          super
+          base.instance_eval do
+            apipie :class do
+              property :organization_id, Integer, desc: 'Returns the id of the organization'
+              property :organization_name, String, desc: 'Returns the name of the organization'
+              property :organization_label, String, desc: 'Returns the label of the organization'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/katello/repository/sync.rb

@@ -3,8 +3,10 @@ module Actions
   module Katello
     module Repository
       class Sync < Actions::EntryAction
+        extend ApipieDSL::Class
         include Helpers::Presenter
         include Actions::Katello::PulpSelector
+        include ::Actions::ObservableAction
         middleware.use Actions::Middleware::ExecuteIfContentsChanged
 
         input_format do
@@ -124,6 +126,63 @@ module Actions
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end
+
+        def repository_id
+          input['repository']['id']
+        end
+
+        def repository_name
+          input['repository']['name']
+        end
+
+        def repository_label
+          input['repository']['label']
+        end
+
+        def product_id
+          input['product']['id']
+        end
+
+        def product_name
+          input['product']['name']
+        end
+
+        def product_label
+          input['product']['label']
+        end
+
+        def contents_changed
+          input['contents_changed']
+        end
+
+        def sync_result
+          input['sync_result']
+        end
+
+        apipie :class, "A class representing #{self} object" do
+          desc 'This object is available as **@object** variable in
+                webhook templates when a corresponding event occures.
+                The following properties can be used to retrieve the needed information.'
+          name "#{class_scope}"
+          refs "#{class_scope}"
+          sections only: %w[all webhooks]
+          property :task, object_of: 'Task', desc: 'Returns the task to which this action belongs'
+          property :repository_id, Integer, desc: 'Returns synced repository id'
+          property :repository_name, String, desc: 'Returns synced repository name'
+          property :repository_label, String, desc: 'Returns synced repository label'
+          property :product_id, Integer, desc: 'Returns product id the synced repository belongs to'
+          property :product_name, String, desc: 'Returns product name the synced repository belongs to'
+          property :product_label, String, desc: 'Returns product label the synced repository belongs to'
+          property :sync_result, Hash, desc: 'Returns Hash object with sync result'
+          property :contents_changed, one_of: [true, false], desc: 'Returns true if repository content was changed due to sync, false otherwise'
+        end
+        include Actions::Katello::JailConcern::Organization
+        class Jail < ::Actions::ObservableAction::Jail
+          allow :organization_id, :organization_name, :organization_label,
+                :repository_id, :repository_name, :repository_label,
+                :product_id, :product_name, :product_label,
+                :sync_result, :contents_changed
+        end
       end
     end
   end

data/app/lib/katello/agent/client_message_handler.rb

@@ -8,6 +8,7 @@ module Katello
         @dispatch_history = Katello::Agent::DispatchHistory.find_by_id(dispatch_history_id)
 
         unless @dispatch_history
+          logger.error("Invalid client message: #{@json}")
           fail("No valid dispatch history in client message")
         end
       end
@@ -33,9 +34,17 @@ module Katello
       private
 
       def handle_dynflow_event
-        task_exists = ForemanTasks::Task.exists?(external_id: @dispatch_history.dynflow_execution_plan_id, result: 'pending')
-        unless task_exists
-          logger.warn("Couldn't find pending task with external_id=#{@dispatch_history.dynflow_execution_plan_id} dispatch_history_id=#{@dispatch_history.id}")
+        return unless accepted? || result
+
+        task = ForemanTasks::Task.find_by_external_id(@dispatch_history.dynflow_execution_plan_id)
+
+        unless task
+          logger.info("Task external_id=#{@dispatch_history.dynflow_execution_plan_id} wasn't found. Skipping dynflow event dispatch")
+          return
+        end
+
+        unless task.result == 'pending'
+          logger.info("Task is no longer pending. Skipping dynflow event dispatch. task_result=#{task.result} external_id=#{@dispatch_history.dynflow_execution_plan_id} dispatch_history_id=#{@dispatch_history.id}")
           return
         end
 

data/app/lib/katello/event_daemon/monitor.rb

@@ -38,6 +38,7 @@ module Katello
             begin
               service_class.close
               service_class.run
+              sleep 0.1
               @service_statuses[service_name] = service_class.status
             rescue => error
               Rails.logger.error("Error occurred while starting #{service_class}")

data/app/lib/katello/event_daemon/services/agent_event_receiver.rb

@@ -2,8 +2,6 @@ module Katello
   module EventDaemon
     module Services
       class AgentEventReceiver
-        STATUS_CACHE_KEY = 'katello_agent_events'.freeze
-
         class Handler
           attr_accessor :processed, :failed
 
@@ -48,14 +46,12 @@ module Katello
           @agent_connection&.open? && @thread&.status.present?
         end
 
-        def self.status(refresh: true)
-          Rails.cache.fetch(STATUS_CACHE_KEY, force: refresh) do
-            {
-              running: running?,
-              processed_count: @handler&.processed || 0,
-              failed_count: @handler&.failed || 0
-            }
-          end
+        def self.status
+          {
+            running: running?,
+            processed_count: @handler&.processed || 0,
+            failed_count: @handler&.failed || 0
+          }
         end
       end
     end

data/app/models/katello/agent/dispatch_history.rb

@@ -3,6 +3,8 @@ module Katello
     class DispatchHistory < Katello::Model
       self.table_name = 'katello_agent_dispatch_histories'
 
+      belongs_to :host, :class_name => "::Host::Managed"
+
       serialize :result, Hash
 
       def accepted?

data/app/models/katello/concerns/hostgroup_extensions.rb

@@ -126,7 +126,7 @@ module Katello
           facet_model = Facets.registered_facets[facet].hostgroup_configuration.model
           value = facet_model.where.not(attribute => nil).joins(:hostgroup).merge(
             ::Hostgroup.where(id: self.ancestor_ids).reorder(ancestry: :desc)
-          ).limit(1).pluck(attribute)
+          ).limit(1).pluck(attribute).first
         end
         value
       end

data/app/models/katello/concerns/redhat_extensions.rb

@@ -51,17 +51,14 @@ module Katello
         end
       end
 
-      def variant_repo(host, variant)
+      def variant_repos(host, variant)
         if variant && host.content_source
           product_id = host.try(:content_facet).try(:kickstart_repository).try(:product_id) || host.try(:kickstart_repository).try(:product_id)
-          distro = distribution_repositories(host)
+          distros = distribution_repositories(host)
             .joins(:product)
-            .where(
-              distribution_variant: variant,
-              "#{Katello::Product.table_name}.id": product_id
-            ).first
-
-          distro&.to_hash(host.content_source)
+            .where("#{Katello::Repository.table_name}.distribution_variant LIKE :variant", { variant: "%#{variant}%" })
+            .where("#{Katello::Product.table_name}.id": product_id).collect { |repo| repo.to_hash(host.content_source, true) }
+          distros
         end
       end
 

data/app/models/katello/content_view.rb

@@ -6,6 +6,7 @@ module Katello
     include Ext::LabelFromName
     include Katello::Authorization::ContentView
     include ForemanTasks::Concerns::ActionSubject
+    include Foreman::ObservableModel
 
     CONTENT_DIR = "content_views".freeze
     IMPORT_LIBRARY = "Import-Library".freeze
@@ -90,6 +91,17 @@ module Katello
     scoped_search :on => :label, :complete_value => true
     scoped_search :on => :composite, :complete_value => true
 
+    set_crud_hooks :content_view
+
+    apipie :class, desc: "A class representing #{model_name.human} object" do
+      name 'Content View'
+      refs 'ContentView'
+      sections only: %w[all additional]
+      prop_group :katello_idname_props, Katello::Model, meta: { friendly_name: 'Content View' }
+      property :label, String, desc: 'Returns label of the Content View'
+      property :organization, 'Organization', desc: 'Returns organization object'
+    end
+
     def self.in_environment(env)
       joins(:content_view_environments).
         where("#{Katello::ContentViewEnvironment.table_name}.environment_id = ?", env.id)
@@ -757,14 +769,8 @@ module Katello
       self.organization
     end
 
-    apipie :class, desc: "A class representing #{model_name.human} object" do
-      name 'Content View'
-      refs 'ContentView'
-      sections only: %w[all additional]
-      prop_group :katello_basic_props, Katello::Model, meta: { friendly_name: 'Content View' }
-    end
     class Jail < ::Safemode::Jail
-      allow :name, :label, :version
+      allow :id, :name, :label, :version, :organization
     end
   end
 end

data/app/models/katello/ping.rb

@@ -72,7 +72,7 @@ module Katello
 
       def ping_katello_agent(result)
         exception_watch(result) do
-          status = Katello::EventDaemon::Services::AgentEventReceiver.status(refresh: false)
+          status = Katello::EventDaemon::Runner.service_status(:katello_agent_events)
           event_daemon_status(status, result)
         end
       end

data/app/models/katello/repository.rb

@@ -395,8 +395,8 @@ module Katello
       all_instances
     end
 
-    def to_hash(content_source = nil)
-      {id: id, name: label, url: full_path(content_source)}
+    def to_hash(content_source = nil, force_http = false)
+      {id: id, name: label, url: full_path(content_source, force_http)}
     end
 
     #is the repo cloned in the specified environment

data/app/models/katello/root_repository.rb

@@ -139,7 +139,7 @@ module Katello
     def ensure_compatible_download_policy
       if !url.blank? && URI(url).scheme == 'file' &&
           [::Runcible::Models::YumImporter::DOWNLOAD_ON_DEMAND, ::Runcible::Models::YumImporter::DOWNLOAD_BACKGROUND].include?(download_policy)
-        errors.add(:download_policy, _("Cannot sync file:// repositories with On Demand or Background Download Policies"))
+        errors.add(:download_policy, _("Cannot sync file:// repositories with the On Demand Download Policy"))
       end
     end
 

data/app/services/katello/agent/dispatcher.rb

@@ -14,37 +14,31 @@ module Katello
       register_message(:install_package_group, Katello::Agent::InstallPackageGroupMessage)
       register_message(:remove_package_group, Katello::Agent::RemovePackageGroupMessage)
 
-      def self.dispatch(message_type, host_ids, args)
+      def self.dispatch(message_type, histories, args)
         message_class = @supported_messages[message_type]
-
-        fail("Unsupported message type") unless message_class
-
-        uuid_data = ::Katello::Host::ContentFacet.where(host_id: host_ids).pluck(:host_id, :uuid)
-        fail("Couldn't find all hosts specified") unless host_ids.size == uuid_data.size
-
-        host_data = uuid_data.map do |host_id, consumer_id|
-          {
-            host_id: host_id,
-            consumer_id: consumer_id,
-            history: Katello::Agent::DispatchHistory.new(host_id: host_id),
-            message: message_class.new(**args.merge(consumer_id: consumer_id))
-          }
+        fail("Unsupported message type: #{message_type}") unless message_class
+
+        messages = histories.map do |history|
+          message = message_class.new(**args.merge(consumer_id: history.host.subscription_facet.uuid))
+          message.dispatch_history_id = history.id
+          message.recipient_address = settings[:client_queue_format] % history.host.subscription_facet.uuid
+          message.reply_to = settings[:event_queue_name]
+          message
         end
 
-        histories = host_data.map { |attrs| attrs[:history] }
-        ActiveRecord::Base.transaction do
-          Katello::Agent::DispatchHistory.import(histories)
+        connection = Connection.new
+        connection.send_messages(messages)
 
-          host_data.each do |d|
-            d[:message].dispatch_history_id = d[:history].id
-            d[:message].recipient_address = settings[:client_queue_format] % [d[:consumer_id]]
-            d[:message].reply_to = settings[:event_queue_name]
-          end
+        histories
+      end
 
-          connection = Connection.new
-          connection.send_messages(host_data.map { |d| d[:message] })
+      def self.create_histories(host_ids:)
+        histories = host_ids.map do |id|
+          Katello::Agent::DispatchHistory.new(host_id: id)
         end
 
+        Katello::Agent::DispatchHistory.import(histories)
+
         histories
       end
 

data/app/services/katello/applicability/applicable_content_helper.rb

@@ -122,20 +122,22 @@ module Katello
       end
 
       def applicable_differences
-        consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
-        content_ids = fetch_content_ids
+        ActiveRecord::Base.connection.uncached do
+          consumer_ids = content_facet.send(applicable_units).pluck("#{content_unit_class.table_name}.id")
+          content_ids = fetch_content_ids
 
-        to_remove = consumer_ids - content_ids
-        to_add = content_ids - consumer_ids
+          to_remove = consumer_ids - content_ids
+          to_add = content_ids - consumer_ids
 
-        [to_add, to_remove]
+          [to_add, to_remove]
+        end
       end
 
       def insert(applicable_ids)
         unless applicable_ids.empty?
           inserts = applicable_ids.map { |applicable_id| "(#{applicable_id.to_i}, #{content_facet.id.to_i})" }
           sql = "INSERT INTO #{content_facet_association_class.table_name} (#{content_unit_association_id}, content_facet_id) VALUES #{inserts.join(', ')}"
-          ActiveRecord::Base.connection.execute(sql)
+          ActiveRecord::Base.connection.exec_insert(sql)
         end
       end
 

data/app/services/katello/managed_content_medium_provider.rb

@@ -14,11 +14,11 @@ module Katello
       URI.parse(url)
     end
 
-    # If there is an 'AppStream' variant, we need to make it
+    # If there are any 'AppStream' variants, we need to make them
     # available to Anaconda
     def additional_media
-      appstream = entity.operatingsystem.variant_repo(entity, 'AppStream')
-      super + (appstream ? [appstream] : [])
+      appstream_repos = entity.operatingsystem.variant_repos(entity, 'AppStream')
+      super + (appstream_repos.present? ? appstream_repos : [])
     end
 
     def unique_id

data/app/services/katello/pulp3/migration.rb

@@ -293,10 +293,11 @@ module Katello
               to_import[[errata_id, repo_id]] ||= {erratum_id: errata_id, erratum_pulp3_href: pulp3_href, repository_id: repo_id}
             end
           end
-        end
 
-        Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
-                                          on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+          Katello::RepositoryErratum.import([:erratum_id, :erratum_pulp3_href, :repository_id], to_import.values, :validate => false,
+                                            on_duplicate_key_update: {conflict_target: [:erratum_id, :repository_id], columns: [:erratum_pulp3_href]})
+          to_import = {}
+        end
       end
 
       def import_content_type(content_type)

data/app/services/katello/pulp3/repository.rb

@@ -222,11 +222,38 @@ module Katello
       end
 
       def refresh_distributions
-        dist_ref = distribution_reference
-        if dist_ref
-          update_distribution
+        if repo.docker?
+          dist = lookup_distributions(base_path: repo.container_repository_name).first
         else
+          dist = lookup_distributions(base_path: repo.relative_path).first
+        end
+
+        # First check if the distribution exists
+        if dist
+          dist_ref = distribution_reference
+          # If we have a DistributionReference, update the distribution
+          if dist_ref
+            return update_distribution
+          # If no DistributionReference, create a DistributionReference and return
+          else
+            save_distribution_references([dist.pulp_href])
+            return []
+          end
+        end
+
+        # So far, it looks like there is no distribution. Try to create one.
+        begin
           create_distribution(relative_path)
+        rescue api.class.client_module::ApiError => e
+          # Now it seems there is a distribution. Fetch it and save the reference.
+          if e.message.include?("\"base_path\":[\"This field must be unique.\"]") ||
+              e.message.include?("\"base_path\":[\"Overlaps with existing distribution\"")
+            dist = lookup_distributions(base_path: repo.relative_path).first
+            save_distribution_references([dist.pulp_href])
+            return []
+          else
+            raise e
+          end
         end
       end
 
@@ -276,7 +303,8 @@ module Katello
           pulp3_distribution_data = api.get_distribution(href)
           path, content_guard_href = pulp3_distribution_data&.base_path, pulp3_distribution_data&.content_guard
           unless distribution_reference
-            DistributionReference.create!(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href)
+            # Ensure that duplicates won't be created in the case of a race condition
+            DistributionReference.where(path: path, href: href, repository_id: repo.id, content_guard_href: content_guard_href).first_or_create!
           end
         end
       end

data/app/services/katello/pulp3/repository/apt.rb

@@ -27,8 +27,7 @@ module Katello
           popts = super(repository_version)
           popts.merge!(
             {
-              # structured is not necessary for subscription-manager
-              #structured: true, # publish real suites (e.g. 'stable')
+              structured: true, # publish real suites (e.g. 'stable')
               simple: true # publish all into 'default'-suite
             }
           )

data/app/services/katello/pulp3/repository/yum.rb

@@ -61,7 +61,7 @@ module Katello
             )
             unless distribution.results.first.variants.empty?
               unless distribution.results.first.variants.first.name.nil?
-                repo.update!(:distribution_variant => distribution.results.first.variants.first.name)
+                repo.update!(:distribution_variant => distribution.results.first.variants.map(&:name).join(','))
               end
             end
           end

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/bulk/views/products-bulk-advanced-sync-modal.html

@@ -12,7 +12,7 @@
 
     <div class="help-block">
       <span translate>
-        Selecting "Complete Sync" will cause only Yum repositories of the selected product to be synced. Selecting "Validate Content" will cause only Yum repositories using the "Immediate" or "Background" download policies will be synced.
+        Selecting "Complete Sync" will cause only Yum repositories of the selected product to be synced. Selecting "Validate Content" will cause only Yum repositories using the "Immediate" download policy to be synced.
       </span>
     </div>
 
@@ -23,4 +23,4 @@
       <span translate>Sync</span>
     </button>
   </span>
-</div>
+</div>

data/engines/bastion_katello/app/assets/javascripts/bastion_katello/products/details/repositories/new/views/new-repository.html

@@ -108,7 +108,7 @@
                ng-model="repository.deb_releases"
                type="text"/>
         <p class="help-block" translate>
-          Comma separated list of releases (suite or codename) to sync from. Default: stable
+          Whitespace-separated list of releases (suite or codename) to sync from. Default: stable
         </p>
       </div>
       <div ng-show="repository.content_type === 'deb'" bst-form-group label="{{ 'Components' | translate }}">
@@ -117,7 +117,7 @@
                ng-model="repository.deb_components"
                type="text"/>
         <p class="help-block" translate>
-          Comma separated list of components to sync from (leave clear to sync all). Example: main
+          Whitespace-separated list of components to sync from (leave clear to sync all). Example: main
         </p>
       </div>
       <div ng-show="repository.content_type === 'deb'" bst-form-group label="{{ 'Architectures' | translate }}">
@@ -126,7 +126,7 @@
                ng-model="repository.deb_architectures"
                type="text"/>
         <p class="help-block" translate>
-          Comma separated list of processor architectures to filter the sync by. Example: amd64
+          Whitespace-separated list of processor architectures to filter the sync by. Example: amd64
         </p>
       </div>
 
@@ -216,7 +216,6 @@
         <p class="help-block" translate>
           For On Demand synchronization, only the metadata is downloaded during sync and packages are fetched and stored on the filesystem when clients request them.
           On Demand is not recommended for custom repositories unless the upstream repository maintains older versions of packages within the repository.
-          For Background synchronization, a background task will download all packages after the initial sync (Deprecated).
           The Immediate option will download all metadata and packages immediately during the sync.
         </p>
       </div>

data/lib/katello/engine.rb

@@ -79,16 +79,6 @@ module Katello
       require 'katello/apipie/validators'
     end
 
-    # make sure the Katello plugin is initialized before `after_initialize`
-    # hook so that the resumed Dynflow tasks can rely on everything ready.
-    initializer 'katello.register_plugin', :before => :finisher_hook, :after => 'foreman_remote_execution.register_plugin' do
-      ::Foreman::AccessControl::Permission.prepend ::Katello::Concerns::PermissionExtensions
-      require 'katello/plugin'
-
-      # extend builtin permissions from core with new actions
-      require 'katello/permissions'
-    end
-
     initializer "katello.register_actions", :before => :finisher_hook do |_app|
       ForemanTasks.dynflow.require!
       if (Setting.table_exists? rescue(false)) && Setting['host_tasks_workers_pool_size'].to_i > 0
@@ -99,6 +89,17 @@ module Katello
                         #{Katello::Engine.root}/app/lib/headpin/actions
                         #{Katello::Engine.root}/app/lib/katello/actions)
       ForemanTasks.dynflow.config.eager_load_paths.concat(action_paths)
+      ForemanTasks.dynflow.eager_load_actions!
+    end
+
+    # make sure the Katello plugin is initialized before `after_initialize`
+    # hook so that the resumed Dynflow tasks can rely on everything ready.
+    initializer 'katello.register_plugin', :before => :finisher_hook, :after => 'foreman_remote_execution.register_plugin' do
+      ::Foreman::AccessControl::Permission.prepend ::Katello::Concerns::PermissionExtensions
+      require 'katello/plugin'
+
+      # extend builtin permissions from core with new actions
+      require 'katello/permissions'
     end
 
     initializer "katello.set_dynflow_middlewares", :before => :finisher_hook do |_app|

data/lib/katello/plugin.rb

@@ -3,7 +3,7 @@ require 'katello/repository_types.rb'
 require 'katello/host_status_manager.rb'
 # rubocop:disable Metrics/BlockLength
 Foreman::Plugin.register :katello do
-  requires_foreman '>= 2.3'
+  requires_foreman '>= 2.4'
 
   sub_menu :top_menu, :content_menu, :caption => N_('Content'),
            :icon => 'fa fa-book', :after => :monitor_menu do
@@ -229,7 +229,8 @@ Foreman::Plugin.register :katello do
   apipie_documented_controllers ["#{Katello::Engine.root}/app/controllers/katello/api/v2/*.rb"]
   apipie_ignored_controllers %w(::Api::V2::OrganizationsController)
   ApipieDSL.configuration.dsl_classes_matchers.concat [
-    "#{Katello::Engine.root}/app/models/katello/**/*.rb"
+    "#{Katello::Engine.root}/app/models/katello/**/*.rb",
+    "#{Katello::Engine.root}/app/lib/actions/**/*.rb"
   ]
 
   parameter_filter ::Host::Managed, :host_collection_ids => [],
@@ -413,4 +414,6 @@ Foreman::Plugin.register :katello do
   precompile.concat(bastion_locale_files)
 
   precompile_assets(precompile)
+
+  extend_observable_events(::Dynflow::Action.descendants.select { |klass| klass <= ::Actions::ObservableAction }.map(&:namespaced_event_names))
 end

data/lib/katello/tasks/clean_backend_objects.rake

@@ -4,27 +4,18 @@ namespace :katello do
     class BackendCleaner
       def initialize
         @candlepin_uuids = []
-        @pulp_uuids = []
         @katello_candlepin_uuids = []
-        @katello_pulp_uuids = []
       end
 
       def populate!
         @candlepin_uuids = Katello::Resources::Candlepin::Consumer.all_uuids
         @katello_candlepin_uuids = Katello::Host::SubscriptionFacet.pluck(:uuid).compact
-
-        @pulp_uuids = ::Katello.pulp_server.extensions.consumer.retrieve_all.map { |consumer| consumer['id'] }
-        @katello_pulp_uuids = Katello::Host::ContentFacet.pluck(:uuid).compact
       end
 
       def hosts_with_no_subscriptions
         ::Host.where(:id => Katello::Host::SubscriptionFacet.where(:uuid => @katello_candlepin_uuids - @candlepin_uuids).select(:host_id))
       end
 
-      def hosts_with_no_content
-        ::Host.where(:id => Katello::Host::ContentFacet.where(:uuid => @katello_pulp_uuids - @pulp_uuids).select(:host_id))
-      end
-
       def hosts_with_nil_facets
         nil_sub = Katello::Host::SubscriptionFacet.where(:uuid => nil).select(:host_id).to_sql
         ::Host.where(" id in (#{nil_sub})")
@@ -33,10 +24,6 @@ namespace :katello do
       def cp_orphaned_host_uuids
         @candlepin_uuids - @katello_candlepin_uuids
       end
-
-      def pulp_orphaned_host_uuids
-        @pulp_uuids - @katello_pulp_uuids
-      end
     end
 
     def cleanup_hosts(cleaner)
@@ -49,11 +36,6 @@ namespace :katello do
         print "Host #{host.id} #{host.name} #{host.subscription_facet.try(:uuid)} is partially missing subscription information.  Un-registering\n"
         execute("Failed to delete host") { Katello::RegistrationManager.unregister_host(host, host_unregister_options(host)) }
       end
-
-      cleaner.hosts_with_no_content.each do |host|
-        print "Host #{host.id} #{host.name} #{host.content_facet.try(:uuid)} is partially missing content information.  Un-registering\n"
-        execute("Failed to delete host") { Katello::RegistrationManager.unregister_host(host, host_unregister_options(host)) }
-      end
     end
 
     def clean_backend_orphans(cleaner)
@@ -63,13 +45,6 @@ namespace :katello do
       cp_uuids.each do |consumer_id|
         execute("exception when destroying candlepin consumer #{consumer_id}") { Katello::Resources::Candlepin::Consumer.destroy(consumer_id) }
       end
-
-      pulp_uuids = cleaner.pulp_orphaned_host_uuids
-      print "#{pulp_uuids.count} orphaned consumer id(s) found in pulp.\n"
-      print "Pulp orphaned consumers: #{pulp_uuids}\n"
-      pulp_uuids.each do |consumer_id|
-        execute("exception when destroying pulp consumer #{consumer_id}") { Katello.pulp_server.extensions.consumer.delete(consumer_id) }
-      end
     end
 
     def host_unregister_options(host)

data/lib/katello/version.rb

@@ -1,3 +1,3 @@
 module Katello
-  VERSION = "4.0.0".freeze
+  VERSION = "4.0.1".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: katello
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - N/A
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-19 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -140,16 +140,16 @@ dependencies:
   name: qpid_proton
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.34'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.34'
 - !ruby/object:Gem::Dependency
   name: stomp
   requirement: !ruby/object:Gem::Requirement
@@ -354,20 +354,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.9.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.9.0
+        version: 2.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.9.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 2.9.0
+        version: 2.10.0
 - !ruby/object:Gem::Dependency
   name: pulp_rpm_client
   requirement: !ruby/object:Gem::Requirement
@@ -930,6 +930,8 @@ files:
 - app/lib/actions/katello/host/update_system_purpose.rb
 - app/lib/actions/katello/host/upload_package_profile.rb
 - app/lib/actions/katello/host/upload_profiles.rb
+- app/lib/actions/katello/jail_concern/content_view.rb
+- app/lib/actions/katello/jail_concern/organization.rb
 - app/lib/actions/katello/organization/create.rb
 - app/lib/actions/katello/organization/destroy.rb
 - app/lib/actions/katello/organization/manifest_delete.rb
@@ -5194,7 +5196,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Content and Subscription Management plugin for Foreman