katello 3.18.0.rc1 → 4.0.0.rc1

data/app/lib/actions/pulp3/content_migration_presenter.rb

@@ -0,0 +1,59 @@
+require 'katello/content_migration_progress'
+
+module Actions
+  module Pulp3
+    class ContentMigrationPresenter < Helpers::Presenter::Base
+      def initialize(migration_action)
+        @migration_action = migration_action
+      end
+
+      def humanized_output
+        if !@migration_action.done?
+          ContentMigrationTaskPresenter.new(@migration_action).humanized_output
+        else
+          task = ForemanTasks::Task.find_by(:external_id => @migration_action.execution_plan_id)
+          ::Katello::ContentMigrationProgress.find_by(:task_id => task.id)&.progress_message
+        end
+      end
+
+      class ContentMigrationTaskPresenter
+        def initialize(action)
+          @action = action
+        end
+
+        def task_progress_reports
+          if @action.pulp_tasks.empty?
+            []
+          else
+            @action.pulp_tasks[0].progress_reports
+          end
+        end
+
+        def task_group_progress_reports
+          if @action.task_groups.empty?
+            []
+          else
+            @action.task_groups[0].group_progress_reports
+          end
+        end
+
+        def humanized_output
+          report = task_progress_reports.find { |current| current['state'] == 'running' && current['total'] != 0 }
+          report ||= task_group_progress_reports.find { |current| current['total'] != 0 && current['done'] != current['total'] }
+
+          if !report.blank? && report['total'] != 0
+            "#{report['message']} #{report['done']}/#{report['total']}"
+          elsif report
+            report['message']
+          elsif task_progress_reports.empty?
+            "Content migration starting. "
+          else
+            "Initial Migration steps complete."
+          end
+        rescue
+          ""
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/content_view/delete_repository_references.rb

@@ -14,7 +14,7 @@ module Actions
           content_view.repository_references.each do |repository_reference|
             repo = repository_reference.root_repository.library_instance
             #force pulp3 in case we've done migrations, but haven't switched over yet
-            tasks << repo.backend_service(smart_proxy, true).delete(repository_reference.repository_href)
+            tasks << repo.backend_service(smart_proxy, true).delete_repository(repository_reference)
           end
           content_view.repository_references.destroy_all
 

data/app/lib/actions/pulp3/content_view_version/export.rb

@@ -4,6 +4,7 @@ module Actions
       class Export < Pulp3::AbstractAsyncTask
         input_format do
           param :content_view_version_id, Integer
+          param :from_content_view_version_id, Integer
           param :smart_proxy_id, Integer
           param :exporter_data, Hash
           param :chunk_size, Integer
@@ -11,8 +12,12 @@ module Actions
 
         def invoke_external_task
           cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+          from_cvv = ::Katello::ContentViewVersion.find(input[:from_content_view_version_id]) unless input[:from_content_view_version_id].blank?
           ::Katello::Pulp3::ContentViewVersion::Export.new(smart_proxy: smart_proxy,
-                                                   content_view_version: cvv).create_export(input[:exporter_data][:pulp_href], input[:chunk_size])
+                                                   content_view_version: cvv,
+                                                   from_content_view_version: from_cvv)
+                                                   .create_export(input[:exporter_data][:pulp_href],
+                                                                        chunk_size: input[:chunk_size])
         end
       end
     end

data/app/lib/actions/pulp3/import_migration.rb

@@ -6,9 +6,14 @@ module Actions
       end
 
       def run
-        migration_service = ::Katello::Pulp3::Migration.new(SmartProxy.pulp_primary)
+        task_id = ForemanTasks::Task.find_by(external_id: self.execution_plan_id)&.id
+        migration_service = ::Katello::Pulp3::Migration.new(SmartProxy.pulp_primary, input.merge(task_id: task_id))
         migration_service.import_pulp3_content
       end
+
+      def humanized_output
+        output[:status]
+      end
     end
   end
 end

data/app/lib/actions/pulp3/orchestration/content_view_version/copy_version_units_to_library.rb

@@ -9,7 +9,8 @@ module Actions
                 sequence do
                   copy_action = plan_action(Actions::Pulp3::Repository::CopyContent, repo, SmartProxy.pulp_primary!,
                                                     repo.library_instance,
-                                                    copy_all: true)
+                                                    copy_all: true,
+                                                    mirror: content_view_version.content_view.library_import?)
                   plan_action(Actions::Pulp3::Repository::SaveVersion, repo.library_instance,
                                 tasks: copy_action.output[:pulp_tasks])
                   plan_action(Katello::Repository::IndexContent, id: repo.library_instance_id)

data/app/lib/actions/pulp3/orchestration/content_view_version/export.rb

@@ -6,6 +6,7 @@ module Actions
           input_format do
             param :smart_proxy_id, Integer
             param :content_view_version_id, Integer
+            param :from_content_view_version_id, Integer
             param :export_history_id, Integer
             param :exporter_data, Hash
             param :destination_server, String
@@ -16,7 +17,11 @@ module Actions
             param :export_path, String
           end
 
-          def plan(content_view_version, destination_server:, chunk_size: nil)
+          # rubocop:disable Metrics/MethodLength
+          def plan(content_view_version:, destination_server: nil,
+                   chunk_size: nil, from_history: nil,
+                   validate_incremental: true,
+                   fail_on_missing_content: false)
             action_subject(content_view_version)
             unless File.directory?(Setting['pulpcore_export_destination'])
               fail ::Foreman::Exception, N_("Unable to export. 'pulpcore_export_destination' setting is not set to a valid directory.")
@@ -24,6 +29,15 @@ module Actions
 
             sequence do
               smart_proxy = SmartProxy.pulp_primary!
+              from_content_view_version = from_history&.content_view_version
+              export_service = ::Katello::Pulp3::ContentViewVersion::Export.new(
+                                                     smart_proxy: smart_proxy,
+                                                     content_view_version: content_view_version,
+                                                     destination_server: destination_server,
+                                                     from_content_view_version: from_content_view_version)
+              export_service.validate!(fail_on_missing_content: fail_on_missing_content,
+                                       validate_incremental: validate_incremental)
+
               action_output = plan_action(::Actions::Pulp3::ContentViewVersion::CreateExporter,
                                      content_view_version_id: content_view_version.id,
                                      smart_proxy_id: smart_proxy.id,
@@ -33,12 +47,13 @@ module Actions
                                      content_view_version_id: content_view_version.id,
                                      smart_proxy_id: smart_proxy.id,
                                      exporter_data: action_output[:exporter_data],
-                                     chunk_size: chunk_size
-                                     )
+                                     chunk_size: chunk_size,
+                                     from_content_view_version_id: from_content_view_version&.id)
 
               plan_self(exporter_data: action_output[:exporter_data], smart_proxy_id: smart_proxy.id,
                         destination_server: destination_server,
-                        content_view_version_id: content_view_version.id)
+                        content_view_version_id: content_view_version.id,
+                        from_content_view_version_id: from_content_view_version&.id)
 
               plan_action(::Actions::Pulp3::ContentViewVersion::DestroyExporter,
                             smart_proxy_id: smart_proxy.id,
@@ -55,17 +70,23 @@ module Actions
             path = File.dirname(file_name.to_s)
             output[:export_path] = path
             cvv = ::Katello::ContentViewVersion.find(input[:content_view_version_id])
+            from_cvv = ::Katello::ContentViewVersion.find(input[:from_content_view_version_id]) unless input[:from_content_view_version_id].blank?
+
+            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(
+                                                       content_view_version: cvv,
+                                                       smart_proxy: smart_proxy,
+                                                       from_content_view_version: from_cvv).generate_metadata
+
+            toc_path_info = output[:exported_file_checksum].find { |item| item.first.end_with?("toc.json") }
+            export_metadata[:toc] = File.basename(toc_path_info.first)
 
-            export_metadata = ::Katello::Pulp3::ContentViewVersion::Export.new(:content_view_version => cvv,
-                                                               :smart_proxy => smart_proxy).generate_metadata
-            toc = Dir.glob("#{path}/*toc.json").first
-            export_metadata[:toc] = File.basename(toc) if toc
-            ::Katello::ContentViewVersionExportHistory.create!(
+            history = ::Katello::ContentViewVersionExportHistory.create!(
               content_view_version_id: input[:content_view_version_id],
               destination_server: input[:destination_server],
               path: path,
               metadata: export_metadata
             )
+            output[:export_history_id] = history.id
           end
 
           def humanized_name

data/app/lib/actions/pulp3/orchestration/content_view_version/export_library.rb

@@ -0,0 +1,60 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module ContentViewVersion
+        class ExportLibrary < Actions::EntryAction
+          def plan(organization, destination_server: nil,
+                                 chunk_size: nil,
+                                 from_history: nil,
+                                 fail_on_missing_content: false)
+            action_subject(organization)
+            validate_repositories_immediate!(organization) if fail_on_missing_content
+
+            content_view = ::Katello::Pulp3::ContentViewVersion::Export.find_library_export_view(destination_server: destination_server,
+                                                                           organization: organization,
+                                                                           create_by_default: true)
+            repo_ids_in_library = organization.default_content_view_version.repositories.yum_type.immediate.pluck(:id)
+            content_view.update!(repository_ids: repo_ids_in_library)
+
+            sequence do
+              publish_action = plan_action(::Actions::Katello::ContentView::Publish, content_view, '')
+              export_action = plan_action(::Actions::Pulp3::Orchestration::ContentViewVersion::Export,
+                                          content_view_version: publish_action.version,
+                                          destination_server: destination_server,
+                                          chunk_size: chunk_size,
+                                          from_history: from_history,
+                                          validate_incremental: false,
+                                          fail_on_missing_content: fail_on_missing_content)
+              plan_self(export_action_output: export_action.output)
+            end
+          end
+
+          def run
+            output[:export_history_id] = input[:export_action_output][:export_history_id]
+          end
+
+          def humanized_name
+            _("Export Library")
+          end
+
+          def rescue_strategy
+            Dynflow::Action::Rescue::Skip
+          end
+
+          def validate_repositories_immediate!(organization)
+            non_immediate_repos = organization.default_content_view_version.repositories.yum_type.non_immediate
+            if non_immediate_repos.any?
+              fail _("NOTE: Unable to fully export '%{organization}' organization's library because"\
+                     " it contains repositories without the 'immediate' download policy."\
+                     " Update the download policy and sync affected repositories to include them in the export."\
+                     " \n %{repos}" %
+                     { organization: organization.name,
+                       repos: ::Katello::Pulp3::ContentViewVersion::Export
+                              .generate_product_repo_strings(repositories: non_immediate_repos)})
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/pulp3/orchestration/content_view_version/import.rb

@@ -36,10 +36,6 @@ module Actions
           def humanized_name
             _("Import")
           end
-
-          def rescue_strategy
-            Dynflow::Action::Rescue::Skip
-          end
         end
       end
     end

data/app/lib/actions/pulp3/orchestration/repository/generate_metadata.rb

@@ -8,7 +8,7 @@ module Actions
             publication_content_type = !::Katello::RepositoryTypeManager.find(repository.content_type).pulp3_skip_publication
             sequence do
               if options[:source_repository] && publication_content_type
-                plan_self(source_repository_id: options[:source_repository].id, target_repository_id: repository.id)
+                plan_self(source_repository_id: options[:source_repository].id, target_repository_id: repository.id, smart_proxy_id: smart_proxy.id)
               elsif publication_content_type
                 plan_action(Actions::Pulp3::Repository::CreatePublication, repository, smart_proxy, options)
               end
@@ -21,6 +21,9 @@ module Actions
             #we don't have to actually generate a publication, we can reuse the old one
             target_repo = ::Katello::Repository.find(input[:target_repository_id])
             source_repo = ::Katello::Repository.find(input[:source_repository_id])
+            if (target_repo.publication_href != source_repo.publication_href && smart_proxy.pulp_primary?)
+              target_repo.clear_smart_proxy_sync_histories
+            end
             target_repo.update!(publication_href: source_repo.publication_href)
           end
         end

data/app/lib/actions/pulp3/orchestration/repository/import_upload.rb

@@ -1,3 +1,4 @@
+# rubocop:disable Metrics/MethodLength
 module Actions
   module Pulp3
     module Orchestration
@@ -6,9 +7,19 @@ module Actions
           def plan(repository, smart_proxy, args)
             file = {:filename => args.dig(:unit_key, :name)}
             content_unit_href = args.dig(:unit_key, :content_unit_id)
+            docker_tag = (args.dig(:unit_type_id) == "docker_tag")
             sequence do
               if content_unit_href
                 plan_self(:commit_output => [], :content_unit_href => content_unit_href)
+                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
+              elsif docker_tag
+                tag_manifest_output = plan_action(Pulp3::Repository::UploadTag,
+                            repository,
+                            smart_proxy,
+                            args).output
+                plan_self(:commit_output => tag_manifest_output[:pulp_tasks])
+                plan_action(Pulp3::Repository::SaveVersion, repository, {force_fetch_version: true, tasks: tag_manifest_output[:pulp_tasks]})
               else
                 commit_output = plan_action(Pulp3::Repository::CommitUpload,
                                             repository,
@@ -24,9 +35,9 @@ module Actions
                                               args.dig(:unit_type_id)).output
                 content_unit_href = artifact_output[:pulp_tasks]
                 plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output[:pulp_tasks])
+                action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
+                plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
               end
-              action_output = plan_action(Pulp3::Repository::ImportUpload, content_unit_href, repository, smart_proxy).output
-              plan_action(Pulp3::Repository::SaveVersion, repository, tasks: action_output[:pulp_tasks]).output
             end
           end
 
@@ -34,8 +45,10 @@ module Actions
             output[:pulp_tasks] = input[:commit_output]
             if input[:content_unit_href]
               output[:content_unit_href] = input[:content_unit_href]
-            else
+            elsif input[:artifact_output]
               output[:content_unit_href] = input[:artifact_output].last[:created_resources].first
+            else
+              output[:content_unit_href] = nil
             end
           end
         end

data/app/lib/actions/pulp3/orchestration/repository/refresh_repos.rb

@@ -4,12 +4,7 @@ module Actions
       module Repository
         class RefreshRepos < Pulp3::AbstractAsyncTask
           include ::Actions::Katello::CapsuleContent::RefreshRepos
-          input_format do
-            param :smart_proxy_id
-            param :environment_id
-            param :content_view_id
-            param :repository_id
-          end
+
           def fetch_proxy_service(smart_proxy)
             ::Katello::Pulp3::SmartProxyRepository.instance_for_type(smart_proxy)
           end

data/app/lib/actions/pulp3/repository/copy_content.rb

@@ -13,7 +13,7 @@ module Actions
           target = ::Katello::Repository.find(input[:target_repository_id] || input[:target_repository])
           service = target.backend_service(smart_proxy)
           output[:pulp_tasks] = if input[:copy_all]
-                                  service.copy_all(source)
+                                  service.copy_all(source, mirror: input[:mirror] || false)
                                 else
                                   service.copy_content_for_source(source, input)
                                 end

data/app/lib/actions/pulp3/repository/delete.rb

@@ -8,7 +8,7 @@ module Actions
 
         def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
-          output[:response] = repo.backend_service(smart_proxy).delete
+          output[:response] = repo.backend_service(smart_proxy).delete_repository
         end
       end
     end

data/app/lib/actions/pulp3/repository/save_version.rb

@@ -2,7 +2,7 @@ module Actions
   module Pulp3
     module Repository
       class SaveVersion < Pulp3::Abstract
-        def plan(repository, options)
+        def plan(repository, options = {})
           fail "Cannot accept tasks and repository_details into Save Version." if options[:tasks].present? && options[:repository_details].present?
           plan_self(:repository_id => repository.id, :tasks => options[:tasks], :repository_details => options[:repository_details], :force_fetch_version => options.fetch(:force_fetch_version, false))
         end

data/app/lib/actions/pulp3/repository/upload_tag.rb

@@ -0,0 +1,18 @@
+module Actions
+  module Pulp3
+    module Repository
+      class UploadTag < Pulp3::AbstractAsyncTask
+        def plan(repository, smart_proxy, args)
+          plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :args => args)
+        end
+
+        def invoke_external_task
+          repo = ::Katello::Repository.find(input[:repository_id])
+          tag_name = input[:args].dig(:unit_key, :name)
+          manifest_digest = input[:args].dig(:unit_key, :digest)
+          output[:pulp_tasks] = [repo.backend_service(smart_proxy).tag_manifest(tag_name, manifest_digest)]
+        end
+      end
+    end
+  end
+end

data/app/lib/katello/concerns/base_template_scope_extensions.rb

@@ -19,6 +19,11 @@ module Katello
         host.subscriptions.redhat.pluck(:name)
       end
 
+      def host_redhat_subscriptions_consumed(host)
+        presenter = ::Katello::HostSubscriptionsPresenter.new(host)
+        presenter.subscriptions.select(&:redhat?).sum(&:quantity_consumed)
+      end
+
       def host_content_facet(host)
         host.content_facet
       end
@@ -87,6 +92,9 @@ module Katello
       end
 
       # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
       def load_errata_applications(filter_errata_type: nil, include_last_reboot: 'yes', since: nil, up_to: nil, status: nil, host_filter: nil)
         result = []
 

data/app/lib/katello/event_daemon/monitor.rb

@@ -0,0 +1,53 @@
+module Katello
+  module EventDaemon
+    class Monitor
+      def initialize(services)
+        @services = services
+        @service_statuses = {}
+        @services.keys.each do |service_name|
+          @service_statuses[service_name] = { running: 'starting' }
+        end
+      end
+
+      def start
+        write_statuses_to_cache
+        loop do
+          Rails.application.executor.wrap do
+            check_services
+          end
+          sleep 15
+        end
+      end
+
+      def write_statuses_to_cache
+        Rails.cache.write(
+          Katello::EventDaemon::Runner::STATUS_CACHE_KEY,
+          @service_statuses
+        )
+      end
+
+      def check_services
+        @services.each do |service_name, service_class|
+          @service_statuses[service_name] = service_class.status
+        rescue => error
+          Rails.logger.error("Error occurred while pinging #{service_class}")
+          Rails.logger.error(error.message)
+          Rails.logger.error(error.backtrace.join("\n"))
+        ensure
+          if error || !@service_statuses.dig(service_name, :running)
+            begin
+              service_class.close
+              service_class.run
+              @service_statuses[service_name] = service_class.status
+            rescue => error
+              Rails.logger.error("Error occurred while starting #{service_class}")
+              Rails.logger.error(error.message)
+              Rails.logger.error(error.backtrace.join("\n"))
+            end
+          end
+        end
+        write_statuses_to_cache
+      end
+    end
+  end
+end