foreman_rh_cloud 4.0.32 → 5.0.28

data/db/migrate/20211027000001_create_task_output.foreman_rh_cloud.rb

@@ -0,0 +1,18 @@
+class CreateTaskOutput < ActiveRecord::Migration[5.2]
+  def change
+    create_table :task_output_lines do |t|
+      t.string :label
+      t.string :line
+      t.timestamps
+
+      t.index :label
+    end
+
+    create_table :task_output_statuses do |t|
+      t.string :label
+      t.string :status
+
+      t.index :label, unique: true
+    end
+  end
+end

data/lib/foreman_inventory_upload/async/generate_all_reports_job.rb

@@ -1,7 +1,9 @@
 module ForemanInventoryUpload
   module Async
-    class GenerateAllReportsJob < ::ApplicationJob
-      def perform
+    class GenerateAllReportsJob < ::Actions::EntryAction
+      include ::Actions::RecurringAction
+
+      def plan
         unless Setting[:allow_auto_inventory_upload]
           logger.debug(
             'The scheduled process is disabled due to the "allow_auto_inventory_upload"
@@ -16,17 +18,19 @@ module ForemanInventoryUpload
           total_hosts = ForemanInventoryUpload::Generators::Queries.for_org(organization.id, use_batches: false).count
 
           if total_hosts <= ForemanInventoryUpload.max_org_size
-            GenerateReportJob.perform_later(ForemanInventoryUpload.generated_reports_folder, organization.id)
+            plan_generate_report(ForemanInventoryUpload.generated_reports_folder, organization)
           else
             logger.info("Skipping automatic uploads for organization #{organization.name}, too many hosts (#{total_hosts}/#{ForemanInventoryUpload.max_org_size})")
           end
         end.compact
-      ensure
-        self.class.set(:wait => 24.hours).perform_later
       end
 
-      def self.singleton_job_name
-        name
+      def rescue_strategy_for_self
+        Dynflow::Action::Rescue::Fail
+      end
+
+      def plan_generate_report(folder, organization)
+        plan_action(ForemanInventoryUpload::Async::GenerateReportJob, folder, organization.id)
       end
     end
   end

data/lib/foreman_inventory_upload/async/generate_report_job.rb

@@ -5,17 +5,21 @@ module ForemanInventoryUpload
         "report_for_#{label}"
       end
 
-      def perform(base_folder, organization)
-        @base_folder = base_folder
-        @organization = organization
+      def plan(base_folder, organization_id)
+        sequence do
+          super(
+            GenerateReportJob.output_label(organization_id),
+            organization_id: organization_id,
+            base_folder: base_folder
+          )
 
-        super(GenerateReportJob.output_label(organization))
-
-        QueueForUploadJob.perform_later(
-          base_folder,
-          ForemanInventoryUpload.facts_archive_name(organization),
-          organization
-        )
+          plan_action(
+            QueueForUploadJob,
+            base_folder,
+            ForemanInventoryUpload.facts_archive_name(organization_id),
+            organization_id
+          )
+        end
       end
 
       def rake_prefix
@@ -28,10 +32,18 @@ module ForemanInventoryUpload
 
       def env
         super.merge(
-          'target' => @base_folder,
-          'organization_id' => @organization
+          'target' => base_folder,
+          'organization_id' => organization_id
         )
       end
+
+      def base_folder
+        input[:base_folder]
+      end
+
+      def organization_id
+        input[:organization_id]
+      end
     end
   end
 end

data/lib/foreman_inventory_upload/async/progress_output.rb

@@ -6,6 +6,7 @@ module ForemanInventoryUpload
       end
 
       def self.register(label)
+        TaskOutputLine.where(label: @label).delete_all
         ProgressOutput.new(label, :writer)
       end
 
@@ -14,47 +15,23 @@ module ForemanInventoryUpload
         @mode = mode
       end
 
-      def buffer
-        @buffer ||= begin
-                      File.open(file_name, file_mode)
-                    rescue Errno::ENOENT
-                      StringIO.new
-                    end
-      end
-
       def full_output
-        buffer.read
+        TaskOutputLine.where(label: @label).order(:created_at).pluck(:line).join("\n")
       end
 
       def write_line(line)
-        buffer << line
-        buffer.fsync
+        TaskOutputLine.create!(label: @label, line: line)
       end
 
       def close
-        @buffer&.close
       end
 
       def status
-        File.read(file_name(:status))
-      rescue Errno::ENOENT
-        ''
+        TaskOutputStatus.where(label: @label).pluck(:status).first || ''
       end
 
       def status=(status)
-        File.atomic_write(file_name(:status)) do |status_file|
-          status_file.write(status)
-        end
-      end
-
-      private
-
-      def file_mode
-        (@mode == :reader) ? 'r' : 'w'
-      end
-
-      def file_name(type = 'out')
-        File.join(ForemanInventoryUpload.outputs_folder, "#{@label}.#{type}")
+        TaskOutputStatus.upsert({ label: @label, status: status }, unique_by: :label)
       end
     end
   end

data/lib/foreman_inventory_upload/async/queue_for_upload_job.rb

@@ -1,9 +1,12 @@
 module ForemanInventoryUpload
   module Async
-    class QueueForUploadJob < ::ApplicationJob
-      def perform(base_folder, report_file, organization_id)
-        @base_folder = base_folder
-        @report_file = report_file
+    class QueueForUploadJob < ::Actions::EntryAction
+      def plan(base_folder, report_file, organization_id)
+        enqueue_task = plan_self(base_folder: base_folder, report_file: report_file)
+        plan_upload_report(enqueue_task.output[:enqueued_file_name], organization_id)
+      end
+
+      def run
         logger.debug('Ensuring objects')
         ensure_ouput_folder
         ensure_output_script
@@ -12,7 +15,7 @@ module ForemanInventoryUpload
         FileUtils.mv(File.join(base_folder, report_file), enqueued_file_name)
         logger.debug("Done copying #{report_file} to #{enqueued_file_name}")
 
-        UploadReportJob.perform_later(enqueued_file_name, organization_id)
+        output[:enqueued_file_name] = enqueued_file_name
       end
 
       def uploads_folder
@@ -47,6 +50,18 @@ module ForemanInventoryUpload
       def logger
         Foreman::Logging.logger('background')
       end
+
+      def base_folder
+        input[:base_folder]
+      end
+
+      def report_file
+        input[:report_file]
+      end
+
+      def plan_upload_report(enqueued_file_name, organization_id)
+        plan_action(UploadReportJob, enqueued_file_name, organization_id)
+      end
     end
   end
 end

data/lib/foreman_inventory_upload/async/shell_process.rb

@@ -2,13 +2,18 @@ require 'open3'
 
 module ForemanInventoryUpload
   module Async
-    class ShellProcess < ::ApplicationJob
+    class ShellProcess < ::Actions::EntryAction
       include AsyncHelpers
 
-      def perform(instance_label)
+      def plan(instance_label, more_inputs = {})
+        inputs = more_inputs.merge(instance_label: instance_label)
+        plan_self(inputs)
+      end
+
+      def run
         klass_name = self.class.name
         logger.debug("Starting #{klass_name} with label #{instance_label}")
-        progress_output_for(instance_label) do |progress_output|
+        progress_output do |progress_output|
           Open3.popen2e(hash_to_s(env), *preprocess_command(command)) do |_stdin, stdout_stderr, wait_thread|
             progress_output.status = "Running in pid #{wait_thread.pid}"
 
@@ -25,7 +30,7 @@ module ForemanInventoryUpload
       def command
       end
 
-      def progress_output_for(instance_label)
+      def progress_output
         progress_output = ProgressOutput.register(instance_label)
         yield(progress_output)
       ensure
@@ -40,11 +45,19 @@ module ForemanInventoryUpload
         Foreman::Logging.logger('background')
       end
 
+      def rescue_strategy_for_self
+        Dynflow::Action::Rescue::Fail
+      end
+
       private
 
       def preprocess_command(command)
         command.kind_of?(Array) ? command : [command]
       end
+
+      def instance_label
+        input[:instance_label]
+      end
     end
   end
 end

data/lib/foreman_inventory_upload/async/upload_report_job.rb

@@ -7,44 +7,45 @@ module ForemanInventoryUpload
         "upload_for_#{label}"
       end
 
-      def perform(filename, organization_id)
+      def plan(filename, organization_id)
         label = UploadReportJob.output_label(organization_id)
-        @filename = filename
-        @organization = Organization.find(organization_id)
+        super(label, filename: filename, organization_id: organization_id)
+      end
 
+      def run
         if Setting[:content_disconnected]
-          progress_output_for(label) do |progress_output|
+          progress_output do |progress_output|
             progress_output.write_line('Upload was stopped since disconnected mode setting is enabled for content on this instance.')
             progress_output.status = "Task aborted, exit 1"
           end
           return
         end
 
-        unless @organization.owner_details&.fetch('upstreamConsumer')&.fetch('idCert')
-          logger.info("Skipping organization '#{@organization}', no candlepin certificate defined.")
-          progress_output_for(label) do |progress_output|
-            progress_output.write_line("Skipping organization #{@organization}, no candlepin certificate defined.")
+        unless organization.owner_details&.fetch('upstreamConsumer')&.fetch('idCert')
+          logger.info("Skipping organization '#{organization}', no candlepin certificate defined.")
+          progress_output do |progress_output|
+            progress_output.write_line("Skipping organization #{organization}, no candlepin certificate defined.")
             progress_output.status = "Task aborted, exit 1"
           end
           return
         end
 
-        Tempfile.create([@organization.name, '.pem']) do |cer_file|
+        Tempfile.create([organization.name, '.pem']) do |cer_file|
           cer_file.write(rh_credentials[:cert])
           cer_file.write(rh_credentials[:key])
           cer_file.flush
           @cer_path = cer_file.path
-          super(label)
+          super
         end
       end
 
       def command
-        ['/bin/bash', File.join(File.dirname(@filename), ForemanInventoryUpload.upload_script_file)]
+        ['/bin/bash', File.join(File.dirname(filename), ForemanInventoryUpload.upload_script_file)]
       end
 
       def env
         env_vars = super.merge(
-          'FILES' => @filename,
+          'FILES' => filename,
           'CER_PATH' => @cer_path
         )
 
@@ -58,13 +59,21 @@ module ForemanInventoryUpload
 
       def rh_credentials
         @rh_credentials ||= begin
-          candlepin_id_certificate = @organization.owner_details['upstreamConsumer']['idCert']
+          candlepin_id_certificate = organization.owner_details['upstreamConsumer']['idCert']
           {
             cert: candlepin_id_certificate['cert'],
             key: candlepin_id_certificate['key'],
           }
         end
       end
+
+      def filename
+        input[:filename]
+      end
+
+      def organization
+        @organization ||= Organization.find(input[:organization_id])
+      end
     end
   end
 end

data/lib/foreman_inventory_upload/generators/fact_helpers.rb

@@ -104,9 +104,7 @@ module ForemanInventoryUpload
       end
 
       def obfuscate_ip(ip, ips_dict)
-        max_obfuscated = ips_dict.values.map { |v| IPAddr.new(v).to_i }.max || IPAddr.new('10.230.230.0').to_i
-
-        IPAddr.new(max_obfuscated + 1, Socket::AF_INET).to_s
+        "10.230.230.#{ips_dict.count + 1}"
       end
 
       def bios_uuid(host)

data/lib/foreman_inventory_upload/generators/queries.rb

@@ -34,7 +34,6 @@ module ForemanInventoryUpload
 
       def self.for_slice(base)
         base
-          .search_for("not params.#{InsightsCloud.enable_client_param} = f")
           .joins(:subscription_facet)
           .preload(
             :interfaces,

data/lib/foreman_inventory_upload/generators/tags.rb

@@ -13,7 +13,7 @@ module ForemanInventoryUpload
           organizations +
           content_data +
           satellite_server_data
-        ).reject { |key, value| value.empty? }.map { |key, value| [key, truncated_value(value)] }
+        ).reject { |key, value| value.empty? }
       end
 
       def generate_parameters
@@ -21,7 +21,6 @@ module ForemanInventoryUpload
 
         (@host.host_params || {})
           .select { |_name, value| value.present? || value.is_a?(FalseClass) }
-          .map { |key, value| [key, truncated_value(value)] }
       end
 
       private
@@ -58,12 +57,6 @@ module ForemanInventoryUpload
           ['organization_id', @host.organization_id.to_s],
         ]
       end
-
-      def truncated_value(value)
-        return 'Original value exceeds 250 characters' if value.to_s.length > 250
-
-        value
-      end
     end
   end
 end

data/lib/foreman_rh_cloud/engine.rb

@@ -115,15 +115,6 @@ module ForemanRhCloud
       ::Host::Managed.include RhCloudHost
     end
 
-    initializer "foreman_rh_cloud.set_dynflow.config.on_init", :before => :finisher_hook do |_app|
-      unless Rails.env.test?
-        ForemanTasks.dynflow.config.on_init do |world|
-          ForemanInventoryUpload::Async::GenerateAllReportsJob.spawn_if_missing(world)
-          InsightsCloud::Async::InsightsScheduledSync.spawn_if_missing(world)
-        end
-      end
-    end
-
     rake_tasks do
       Rake::Task['db:seed'].enhance do
         ForemanRhCloud::Engine.load_seed
@@ -139,7 +130,6 @@ module ForemanRhCloud
     config.to_prepare do
       # skip database manipulations while tables do not exist, like in migrations
       if ActiveRecord::Base.connection.data_source_exists?(ForemanTasks::Task.table_name) &&
-
         RemoteExecutionFeature.register(
           :rh_cloud_remediate_hosts,
           N_('Apply Insights recommendations'),
@@ -149,7 +139,9 @@ module ForemanRhCloud
         # skip object creation when admin user is not present, for example in test DB
         if User.unscoped.find_by_login(User::ANONYMOUS_ADMIN).present?
           ::ForemanTasks.dynflow.config.on_init(false) do |world|
+            ForemanRhCloud::Engine.register_scheduled_task(ForemanInventoryUpload::Async::GenerateAllReportsJob, '0 0 * * *')
             ForemanRhCloud::Engine.register_scheduled_task(InventorySync::Async::InventoryScheduledSync, '0 0 * * *')
+            ForemanRhCloud::Engine.register_scheduled_task(InsightsCloud::Async::InsightsScheduledSync, '0 0 * * *')
             ForemanRhCloud::Engine.register_scheduled_task(InsightsCloud::Async::InsightsClientStatusAging, '0 0 * * *')
           end
         end

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '4.0.32'.freeze
+  VERSION = '5.0.28'.freeze
 end

data/lib/insights_cloud/async/insights_scheduled_sync.rb

@@ -1,7 +1,9 @@
 module InsightsCloud
   module Async
-    class InsightsScheduledSync < ::ApplicationJob
-      def perform
+    class InsightsScheduledSync < ::Actions::EntryAction
+      include ::Actions::RecurringAction
+
+      def plan
         unless Setting[:allow_auto_insights_sync]
           logger.debug(
             'The scheduled process is disabled due to the "allow_auto_insights_sync"
@@ -10,13 +12,15 @@ module InsightsCloud
           return
         end
 
-        ForemanTasks.async_task InsightsFullSync
-      ensure
-        self.class.set(:wait => 24.hours).perform_later
+        plan_full_sync
+      end
+
+      def plan_full_sync
+        plan_action InsightsFullSync
       end
 
-      def self.singleton_job_name
-        name
+      def rescue_strategy_for_self
+        Dynflow::Action::Rescue::Fail
       end
     end
   end

data/lib/inventory_sync/async/host_result.rb

@@ -17,6 +17,7 @@ module InventorySync
         @sub_ids.map do |sub_id|
           host_id = host_id(sub_id)
           if host_id
+            touched << host_id
             {
               host_id: host_id,
               status: InventorySync::InventoryStatus::SYNC,
@@ -27,6 +28,10 @@ module InventorySync
         end.compact
       end
 
+      def touched
+        @touched ||= []
+      end
+
       def host_id(sub_id)
         hosts[sub_id]
       end

data/lib/inventory_sync/async/inventory_full_sync.rb

@@ -14,7 +14,11 @@ module InventorySync
       end
 
       def setup_statuses
-        @subscribed_hosts_ids = Set.new(affected_host_ids)
+        @subscribed_hosts_ids = Set.new(
+          ForemanInventoryUpload::Generators::Queries.for_slice(
+            Host.unscoped.where(organization: input[:organization_id])
+          ).pluck(:id)
+        )
 
         InventorySync::InventoryStatus.transaction do
           InventorySync::InventoryStatus.where(host_id: @subscribed_hosts_ids).delete_all
@@ -31,10 +35,8 @@ module InventorySync
       def update_statuses_batch
         results = yield
 
-        existing_hosts = results.status_hashes.select { |hash| @subscribed_hosts_ids.include?(hash[:host_id]) }
-
-        update_hosts_status(existing_hosts)
-        host_statuses[:sync] += existing_hosts.size
+        update_hosts_status(results.status_hashes, results.touched)
+        host_statuses[:sync] += results.touched.size
       end
 
       def rescue_strategy_for_self
@@ -43,10 +45,9 @@ module InventorySync
 
       private
 
-      def update_hosts_status(status_hashes)
+      def update_hosts_status(status_hashes, touched)
         InventorySync::InventoryStatus.create(status_hashes)
-        updated_ids = status_hashes.map { |hash| hash[:host_id] }
-        @subscribed_hosts_ids.subtract(updated_ids)
+        @subscribed_hosts_ids.subtract(touched)
       end
 
       def add_missing_hosts_statuses(hosts_ids)
@@ -67,12 +68,6 @@ module InventorySync
           disconnect: 0,
         }
       end
-
-      def affected_host_ids
-        ForemanInventoryUpload::Generators::Queries.for_slice(
-          Host.unscoped.where(organization: input[:organization_id])
-        ).pluck(:id)
-      end
     end
   end
 end

data/lib/inventory_sync/async/inventory_hosts_sync.rb

@@ -35,14 +35,18 @@ module InventorySync
       private
 
       def add_missing_insights_facets(uuids_hash)
-        all_facets = uuids_hash.map do |host_id, uuid|
+        existing_facets = InsightsFacet.where(host_id: uuids_hash.keys).pluck(:host_id, :uuid)
+        missing_facets = uuids_hash.except(*existing_facets.map(&:first)).map do |host_id, uuid|
           {
             host_id: host_id,
             uuid: uuid,
           }
         end
+        InsightsFacet.create(missing_facets)
 
-        InsightsFacet.upsert_all(all_facets, unique_by: :host_id) unless all_facets.empty?
+        existing_facets.select { |host_id, uuid| uuid.empty? }.each do |host_id, _uuid|
+          InsightsFacet.where(host_id: host_id).update_all(uuid: uuids_hash[host_id])
+        end
       end
 
       def plan_self_host_sync

data/lib/tasks/rh_cloud_inventory.rake

@@ -12,7 +12,7 @@ namespace :rh_cloud_inventory do
 
       User.as_anonymous_admin do
         organizations.each do |organization|
-          ForemanInventoryUpload::Async::GenerateReportJob.perform_now(ForemanInventoryUpload.generated_reports_folder, organization.id)
+          ForemanTasks.async_task(ForemanInventoryUpload::Async::GenerateReportJob, ForemanInventoryUpload.generated_reports_folder, organization.id)
           puts "Generated and uploaded inventory report for organization '#{organization.name}'"
         end
       end
@@ -47,7 +47,7 @@ namespace :rh_cloud_inventory do
       base_folder = ENV['target'] || ForemanInventoryUpload.generated_reports_folder
       organization_id = ENV['organization_id']
       report_file = ForemanInventoryUpload.facts_archive_name(organization_id)
-      ForemanInventoryUpload::Async::QueueForUploadJob.perform_now(base_folder, report_file, organization_id)
+      ForemanTasks.sync_task(ForemanInventoryUpload::Async::QueueForUploadJob, base_folder, report_file, organization_id)
       puts "Uploaded #{report_file}"
     end
   end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "4.0.32",
+  "version": "5.0.28",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {
@@ -22,27 +22,22 @@
     "url": "http://projects.theforeman.org/projects/foreman_rh_cloud/issues"
   },
   "peerDependencies": {
-    "@theforeman/vendor": "~4.14.0"
+    "@theforeman/vendor": "~8.16.0"
   },
   "devDependencies": {
     "@babel/core": "~7.7.0",
-    "@redhat-cloud-services/frontend-components": "^2.5.0",
-    "@theforeman/builder": "~4.14.0",
-    "@theforeman/stories": "~4.14.0",
-    "@theforeman/test": "~4.14.0",
-    "@theforeman/vendor-dev": "~4.14.0",
-    "@theforeman/eslint-plugin-foreman": "~4.14.0",
+    "@theforeman/builder": "~8.16.0",
+    "@theforeman/stories": "~8.16.0",
+    "@theforeman/test": "~8.16.0",
+    "@theforeman/eslint-plugin-foreman": "~8.16.0",
     "babel-eslint": "~10.0.0",
     "eslint": "~6.7.2",
     "eslint-plugin-spellcheck": "~0.0.17",
+    "jed": "~1.1.1",
     "prettier": "~1.19.1",
     "stylelint": "~9.3.0",
     "stylelint-config-standard": "~18.0.0",
     "surge": "~0.20.3",
     "redux-mock-store": "~1.2.2"
-  },
-  "dependencies": {
-    "jed": "~1.1.1",
-    "react-intl": "~2.8.0"
   }
 }

data/test/controllers/inventory_upload/cloud_status_controller_test.rb

@@ -29,8 +29,6 @@ class CloudStatusControllerTest < ActionController::TestCase
       RestClient::Response.new('TEST RESPONSE ORG 1')
     )
 
-    Katello::UpstreamConnectionChecker.any_instance.expects(:assert_connection).twice.returns(true)
-
     get :index, session: set_session_user
 
     assert_response :success