foreman_rh_cloud 5.0.30 → 5.0.33

data/lib/insights_cloud/async/connector_playbook_execution_reporter_task.rb

@@ -0,0 +1,193 @@
+module InsightsCloud
+  module Async
+    class ConnectorPlaybookExecutionReporterTask < ::Actions::EntryAction
+      include Dynflow::Action::Polling
+      include ForemanRhCloud::CloudAuth
+
+      def self.subscribe
+        Actions::RemoteExecution::RunHostsJob
+      end
+
+      def self.connector_feature_id
+        @connector_feature_id ||= RemoteExecutionFeature.feature!(:rh_cloud_connector_run_playbook).id
+      end
+
+      def plan(job_invocation)
+        return unless connector_playbook_job?(job_invocation)
+
+        @job_invocation = job_invocation
+
+        invocation_inputs = invocation_inputs(job_invocation)
+        report_url = invocation_inputs['report_url']
+        report_interval = [invocation_inputs['report_interval'].to_i, 5].max
+        correlation_id = invocation_inputs['correlation_id']
+
+        plan_self(
+          report_url: report_url,
+          report_interval: report_interval,
+          job_invocation_id: job_invocation.id,
+          correlation_id: correlation_id
+        )
+      end
+
+      def run(event = nil)
+        # Handle skip events
+        return if event == Dynflow::Action::Skip
+
+        super
+      end
+
+      def rescue_strategy_for_self
+        Dynflow::Action::Rescue::Skip
+      end
+
+      def done?(current_status = invocation_status)
+        job_invocation.finished? || current_status.map { |_host_id, task_status| task_status['report_done'] }.all?
+      end
+
+      # noop, we don't want to do anything when the polling task starts
+      def invoke_external_task
+        poll_external_task
+      end
+
+      def poll_external_task
+        current_status = invocation_status
+        report_job_progress(current_status)
+        # record the current state and increment the sequence for the next invocation
+        {
+          invocation_status: current_status,
+        }
+      end
+
+      def poll_intervals
+        [report_interval]
+      end
+
+      private
+
+      def connector_playbook_job?(job_invocation)
+        puts "Job invocation id: #{job_invocation&.remote_execution_feature_id}, feature id: #{connector_feature_id}"
+        job_invocation&.remote_execution_feature_id == connector_feature_id
+      end
+
+      def connector_feature_id
+        self.class.connector_feature_id
+      end
+
+      def invocation_inputs(job_invocation)
+        Hash[
+          job_invocation.pattern_template_invocations.first.input_values.map do |input_value|
+            [input_value.template_input.name, input_value.value]
+          end
+        ]
+      end
+
+      def job_invocation
+        @job_invocation ||= JobInvocation.find(input['job_invocation_id'])
+      end
+
+      def report_interval
+        @report_interval ||= input['report_interval']
+      end
+
+      def correlation_id
+        @correlation_id ||= input['correlation_id']
+      end
+
+      def host_status(host)
+        external_task&.dig('invocation_status', host)
+      end
+
+      def sequence(host)
+        host_status(host)&.fetch('sequence', 0).to_i
+      end
+
+      def host_done?(host)
+        ActiveModel::Type::Boolean.new.cast(host_status(host)&.fetch('report_done', nil))
+      end
+
+      def report_url
+        input['report_url']
+      end
+
+      def invocation_status
+        Hash[job_invocation.targeting.hosts.map do |host|
+          next unless host.insights&.uuid
+          [
+            host.insights.uuid,
+            task_status(job_invocation.sub_task_for_host(host), host.insights.uuid),
+          ]
+        end.compact]
+      end
+
+      def task_status(host_task, host_name)
+        unless host_task
+          return { 'state' => 'unknown' }
+        end
+
+        {
+          'state' => host_task.state,
+          'output' => host_task.main_action.live_output.map { |line| line['output'] }.join("\n"),
+          'exit_status' => host_task.main_action.exit_status,
+          'sequence' => sequence(host_name),
+          'report_done' => host_done?(host_name),
+        }
+      end
+
+      def report_job_progress(invocation_status)
+        generator = InsightsCloud::Generators::PlaybookProgressGenerator.new(correlation_id)
+
+        invocation_status.each do |host_name, status|
+          # skip host if the host already reported that it's finished
+          next if status['report_done']
+
+          unless status['state'] == 'unknown'
+            sequence = status['sequence']
+            generator.host_progress_message(host_name, status['output'], sequence)
+            status['sequence'] = sequence + 1 # increase the sequence for each host
+          end
+
+          if status['state'] == 'stopped'
+            generator.host_finished_message(host_name, status['exit_status'])
+            status['report_done'] = true
+          end
+        end
+        generator.job_finished_message if done?(invocation_status)
+
+        send_report(generator.generate)
+      end
+
+      def send_report(report)
+        execute_cloud_request(
+          method: :post,
+          url: report_url,
+          content_type: 'application/vnd.redhat.playbook-sat.v3+jsonl',
+          payload: {
+            file: wrap_report(report),
+            multipart: true,
+          }
+        )
+      end
+
+      # RestClient has to accept an object that responds to :read, :path and :content_type methods
+      # to properly generate a multipart message.
+      # see: https://github.com/rest-client/rest-client/blob/2c72a2e77e2e87d25ff38feba0cf048d51bd5eca/lib/restclient/payload.rb#L161
+      def wrap_report(report)
+        obj = StringIO.new(report)
+        def obj.content_type
+          'application/vnd.redhat.playbook-sat.v3+jsonl'
+        end
+
+        def obj.path
+          'file'
+        end
+
+        obj
+      end
+
+      def logger
+        action_logger
+      end
+    end
+  end
+end

data/lib/insights_cloud/generators/playbook_progress_generator.rb

@@ -0,0 +1,49 @@
+module InsightsCloud
+  module Generators
+    class PlaybookProgressGenerator
+      attr_reader :correlation_id
+      def initialize(correlation_id)
+        @messages = []
+        @correlation_id = correlation_id
+      end
+
+      def host_progress_message(host_name, output, sequence)
+        @messages << {
+          "type": "playbook_run_update",
+          "version": 3,
+          "correlation_id": correlation_id,
+          "sequence": sequence,
+          "host": host_name,
+          "console": output,
+        }
+      end
+
+      def host_finished_message(host_name, exit_code)
+        @messages << {
+          "type": "playbook_run_finished",
+          "version": 3,
+          "correlation_id": correlation_id,
+          "host": host_name,
+          "status": exit_code == 0 ? 'success' : 'failure',
+          "connection_code": 0,
+          "execution_code": exit_code,
+        }
+      end
+
+      def job_finished_message
+        @messages << {
+          "type": "playbook_run_completed",
+          "version": 3,
+          "correlation_id": correlation_id,
+          "status": "success",
+        }
+      end
+
+      def generate
+        @messages.map do |message|
+          message.to_json
+        end.join("\n")
+      end
+    end
+  end
+end

data/lib/tasks/insights.rake

@@ -19,4 +19,17 @@ namespace :rh_cloud_insights do
 
     puts "Deleted #{deleted_count} insights statuses"
   end
+
+  desc "Re-announce all organizations into Sources on RH cloud."
+  task announce_to_sources: [:environment] do
+    logger = Logging::Logger.new(STDOUT)
+    Organization.unscoped.each do |org|
+      presence = ForemanRhCloud::CloudPresence.new(org, logger)
+      presence.announce_to_sources
+    rescue StandardError => ex
+      logger.warn(ex)
+    end
+
+    logger.info('Reannounced all organizations')
+  end
 end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "5.0.30",
+  "version": "5.0.33",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {

data/test/controllers/insights_cloud/api/cloud_request_controller_test.rb

@@ -0,0 +1,78 @@
+require 'json'
+require 'test_plugin_helper'
+
+module InsightsCloud::Api
+  class CloudRequestControllerTest < ActionController::TestCase
+    tests Api::V2::RhCloud::CloudRequestController
+
+    setup do
+      @test_org = FactoryBot.create(:organization)
+    end
+
+    test 'Fails for unknown directives' do
+      request_params = run_playbook_request
+      request_params['directive'] = 'not-valid'
+
+      post :update, params: request_params
+
+      assert_response :bad_request
+    end
+
+    test 'Starts playbook run for correct directive' do
+      Setting[:rh_cloud_token] = 'MOCK_TOKEN'
+      host1 = FactoryBot.create(:host, :with_insights_hits)
+      host1.insights.uuid = 'TEST_UUID1'
+      host1.insights.save!
+      host2 = FactoryBot.create(:host, :with_insights_hits)
+      host2.insights.uuid = 'TEST_UUID2'
+      host2.insights.save!
+
+      mock_composer = mock('composer')
+      ::JobInvocationComposer.expects(:for_feature).with do |feature, host_ids, params|
+        feature == :rh_cloud_connector_run_playbook &&
+        host_ids.first == host1.id &&
+        host_ids.last == host2.id
+      end.returns(mock_composer)
+      mock_composer.expects(:trigger!)
+      mock_composer.expects(:job_invocation)
+
+      post :update, params: run_playbook_request
+
+      assert_response :success
+    end
+
+    private
+
+    def run_playbook_request
+      request_json = <<-REQUEST
+      {
+        "type": "data",
+        "message_id": "a6a7d866-7de0-409a-84e0-3c56c4171bb7",
+        "version": 1,
+        "sent": "2021-01-12T15:30:08+00:00",
+        "directive": "playbook-sat",
+        "metadata": {
+            "operation": "run",
+            "return_url": "https://cloud.redhat.com/api/v1/ingres/upload",
+            "correlation_id": "6684b9dd-0d16-42c1-b13a-9f45be59e3b6",
+            "playbook_run_name": "Human-readable playbook run name",
+            "playbook_run_url": "https://console.redhat.com/insights/remediations/1234",
+            "sat_id": "aa3b1faa-56f3-4d14-8258-615d11e20060",
+            "sat_org_id": "#{FactoryBot.create(:organization).id}",
+            "initiator_user_id": "4efca34c6d9ae05ef7c3d7a7424e6370d198159a841ae005084888a9a4529e27",
+            "hosts": "TEST_UUID1,TEST_UUID2",
+            "response_interval": "30",
+            "response_full": "false"
+        },
+        "content": ""
+      }
+      REQUEST
+
+      request = JSON.parse(request_json)
+
+      request['content'] = "\"#{Base64.encode64('https://cloud.redhat.com/api/v1/remediations/1234/playbook')}\""
+
+      request
+    end
+  end
+end

data/test/controllers/inventory_upload/cloud_status_controller_test.rb

@@ -29,6 +29,8 @@ class CloudStatusControllerTest < ActionController::TestCase
       RestClient::Response.new('TEST RESPONSE ORG 1')
     )
 
+    Katello::UpstreamConnectionChecker.any_instance.expects(:assert_connection).twice.returns(true)
+
     get :index, session: set_session_user
 
     assert_response :success

data/test/jobs/connector_playbook_execution_reporter_task_test.rb

@@ -0,0 +1,207 @@
+require 'json'
+require 'test_plugin_helper'
+require 'foreman_tasks/test_helpers'
+require "#{ForemanTasks::Engine.root}/test/support/dummy_dynflow_action"
+
+class ConnectorPlaybookExecutionReporterTaskTest < ActiveSupport::TestCase
+  include ForemanTasks::TestHelpers::WithInThreadExecutor
+
+  # override default send behavior for the test
+  class TestConnectorPlaybookExecutionReporterTask < InsightsCloud::Async::ConnectorPlaybookExecutionReporterTask
+    def send_report(report)
+      output[:saved_reports] = (output[:saved_reports] || []) << report
+    end
+  end
+
+  setup do
+    feature = RemoteExecutionFeature.register(
+      :rh_cloud_connector_run_playbook,
+      N_('Run RH Cloud playbook'),
+      description: N_('Run playbook genrated by Red Hat remediations app'),
+      host_action_button: false,
+      provided_inputs: ['playbook_url', 'report_url', 'correlation_id', 'report_interval']
+    )
+
+    puts "REX Register: #{feature.id}"
+
+    @job_invocation = generate_job_invocation
+
+    # reset connector feature ID cache
+    TestConnectorPlaybookExecutionReporterTask.instance_variable_set(:@connector_feature_id, nil)
+
+    puts RemoteExecutionFeature.all.to_a
+  end
+
+  test 'It reports finish playbook messages' do
+    TestConnectorPlaybookExecutionReporterTask.any_instance.stubs(:done?).returns(true)
+
+    actual = ForemanTasks.sync_task(TestConnectorPlaybookExecutionReporterTask, @job_invocation)
+
+    actual_report = actual.output[:saved_reports].first.to_s
+
+    assert_equal 1, actual.output[:saved_reports].size
+    assert_not_nil actual_report
+    actual_jsonl = read_jsonl(actual_report)
+
+    assert_not_nil actual_report_finished = actual_jsonl.find { |l| l['type'] == 'playbook_run_completed' }
+    assert_equal 'TEST_CORRELATION', actual_report_finished['correlation_id']
+    assert_equal 'success', actual_report_finished['status']
+
+    assert_not_nil actual_host_finished = actual_jsonl.find { |l| l['type'] == 'playbook_run_finished' && l['host'] == @host1.insights.uuid }
+    assert_equal 'TEST_CORRELATION', actual_host_finished['correlation_id']
+    assert_equal 'success', actual_host_finished['status']
+  end
+
+  test 'It reports single progress message for done host' do
+    TestConnectorPlaybookExecutionReporterTask.any_instance.stubs(:done?).returns(false, true)
+
+    actual = ForemanTasks.sync_task(TestConnectorPlaybookExecutionReporterTask, @job_invocation)
+
+    actual_report = actual.output[:saved_reports].first.to_s
+
+    assert_equal 1, actual.output[:saved_reports].size
+    assert_not_nil actual_report
+    actual_jsonl = read_jsonl(actual_report)
+
+    actual_host_updates = actual_jsonl
+      .select { |l| l['type'] == 'playbook_run_update' && l['host'] == @host1.insights.uuid }
+    assert_equal 1, actual_host_updates.size
+    assert_equal 0, actual_host_updates.first['sequence']
+  end
+
+  test 'It reports two progress messages for in progress host' do
+    TestConnectorPlaybookExecutionReporterTask.any_instance.stubs(:done?).returns(false, false, true)
+
+    host1_task = @job_invocation.template_invocations.joins(:host).where(hosts: {name: @host1.name}).first.run_host_job_task
+    host1_task.state = 'running'
+    host1_task.save!
+
+    actual = ForemanTasks.sync_task(TestConnectorPlaybookExecutionReporterTask, @job_invocation)
+
+    assert_equal 2, actual.output[:saved_reports].size
+
+    first_report = actual.output[:saved_reports].first.to_s
+    actual_jsonl = read_jsonl(first_report)
+
+    actual_host_updates = actual_jsonl
+      .select { |l| l['type'] == 'playbook_run_update' && l['host'] == @host1.insights.uuid }
+    assert_equal 1, actual_host_updates.size
+    assert_equal 0, actual_host_updates.first['sequence']
+
+    actual_host_updates = actual_jsonl
+      .select { |l| l['type'] == 'playbook_run_update' && l['host'] == @host2.insights.uuid }
+    assert_equal 1, actual_host_updates.size
+    assert_equal 0, actual_host_updates.first['sequence']
+
+    second_report = actual.output[:saved_reports].last.to_s
+    actual_jsonl = read_jsonl(second_report)
+
+    actual_host_updates = actual_jsonl
+      .select { |l| l['type'] == 'playbook_run_update' && l['host'] == @host1.insights.uuid }
+    assert_equal 1, actual_host_updates.size
+    assert_equal 1, actual_host_updates.first['sequence']
+
+    actual_host_updates = actual_jsonl
+      .select { |l| l['type'] == 'playbook_run_update' && l['host'] == @host2.insights.uuid }
+    assert_equal 0, actual_host_updates.size
+  end
+
+  private
+
+  def generate_job_invocation
+    job_template = FactoryBot.build(
+      :job_template,
+      :template => 'BLEH'
+    )
+    feature = RemoteExecutionFeature.feature!(:rh_cloud_connector_run_playbook).id
+
+    puts "Generated feature: #{feature}"
+
+    job_invocation = FactoryBot.create(
+      :job_invocation,
+      remote_execution_feature_id: feature,
+      task_id: FactoryBot.create(:dynflow_task).id
+    )
+
+    job_template.template_inputs << playbook_url_input = FactoryBot.build(:template_input,
+      :name => 'playbook_url',
+      :input_type => 'user',
+      :required => true)
+    job_template.template_inputs << report_url_input = FactoryBot.build(:template_input,
+      :name => 'report_url',
+      :input_type => 'user',
+      :required => true)
+    job_template.template_inputs << correlation_id_input = FactoryBot.build(:template_input,
+      :name => 'correlation_id',
+      :input_type => 'user',
+      :required => true)
+    job_template.template_inputs << report_interval_input = FactoryBot.build(:template_input,
+      :name => 'report_interval',
+      :input_type => 'user',
+      :required => true)
+
+    template_invocation = FactoryBot.build(:template_invocation,
+      :template => job_template,
+      :job_invocation => job_invocation
+    )
+
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => playbook_url_input,
+      :value => 'http://example.com/TEST_PLAYBOOK'
+    )
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => report_url_input,
+      :value => 'http://example.com/TEST_REPORT'
+    )
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => correlation_id_input,
+      :value => 'TEST_CORRELATION'
+    )
+    template_invocation.input_values << FactoryBot.create(
+      :template_invocation_input_value,
+      :template_invocation => template_invocation,
+      :template_input => report_interval_input,
+      :value => '1'
+    )
+
+    @host1 = FactoryBot.create(:host, :with_insights_hits, name: 'host1')
+    @host1.insights.uuid = 'TEST_UUID1'
+    @host1.insights.save!
+    @host2 = FactoryBot.create(:host, :with_insights_hits, name: 'host2')
+    @host2.insights.uuid = 'TEST_UUID2'
+    @host2.insights.save!
+
+    targeting = FactoryBot.create(:targeting, hosts: [@host1, @host2])
+    job_invocation.targeting = targeting
+    job_invocation.save!
+
+    job_invocation.template_invocations << FactoryBot.create(
+      :template_invocation,
+      run_host_job_task: FactoryBot.create(:dynflow_task),
+      host_id: @host1.id
+    )
+    job_invocation.template_invocations << FactoryBot.create(
+      :template_invocation,
+      run_host_job_task: FactoryBot.create(:dynflow_task),
+      host_id: @host2.id
+    )
+
+    fake_output = (1..5).map do |i|
+      { 'timestamp' => (Time.now - (5 - i)).to_f, 'output' => "#{i}\n" }
+    end
+    Support::DummyDynflowAction.any_instance.stubs(:live_output).returns(fake_output)
+    Support::DummyDynflowAction.any_instance.stubs(:exit_status).returns(0)
+
+    job_invocation
+  end
+
+  def read_jsonl(jsonl)
+    jsonl.lines.map { |l| JSON.parse(l) }
+  end
+end

data/test/unit/fact_helpers_test.rb

@@ -30,13 +30,13 @@ class FactHelpersTest < ActiveSupport::TestCase
   test 'obfuscates ips with insights-client data' do
     host = mock('host')
     @instance.expects(:fact_value).with(host, 'insights_client::ips').returns(
-      '[{"obfuscated": "10.230.230.1", "original": "224.0.0.1"}, {"obfuscated": "10.230.230.2", "original": "224.0.0.251"}]'
+      '[{"obfuscated": "10.230.230.1", "original": "224.0.0.1"}, {"obfuscated": "10.230.230.255", "original": "224.0.0.251"}]'
     )
 
     actual = @instance.obfuscated_ips(host)
 
     assert_equal '10.230.230.1', actual['224.0.0.1']
-    assert_equal '10.230.230.3', actual['224.0.0.2']
+    assert_equal '10.230.231.0', actual['224.0.0.2']
   end
 
   test 'obfuscates ips without insights-client data' do

data/test/unit/foreman_rh_cloud_self_host_test.rb

@@ -25,4 +25,11 @@ class ForemanRhCloudSelfHostTest < ActiveSupport::TestCase
 
     assert_not_nil actual
   end
+
+  test 'finds host by infrastructure facet' do
+    @host = FactoryBot.create(:host, :managed, :with_infrastructure_facet)
+    actual = ForemanRhCloud.foreman_host
+
+    assert_equal @host, actual
+  end
 end

data/test/unit/playbook_progress_generator_test.rb

@@ -0,0 +1,75 @@
+require 'test_plugin_helper'
+
+class PlaybookProgressGeneratorTest < ActiveSupport::TestCase
+  setup do
+    @correlation_id = 'CORRELATION_ID'
+    @generator = InsightsCloud::Generators::PlaybookProgressGenerator.new(@correlation_id)
+  end
+
+  test 'Outputs host progress message' do
+    @generator.host_progress_message('test_host', 'test_output', 0)
+
+    actual = @generator.generate
+    actual_message = JSON.parse(actual)
+
+    assert_equal "playbook_run_update", actual_message["type"]
+    assert_equal 3, actual_message["version"]
+    assert_equal @correlation_id, actual_message["correlation_id"]
+    assert_equal 0, actual_message["sequence"]
+    assert_equal 'test_host', actual_message["host"]
+    assert_equal 'test_output', actual_message["console"]
+  end
+
+  test 'Outputs host finished with error message' do
+    @generator.host_finished_message('test_host', 100)
+
+    actual = @generator.generate
+    actual_message = JSON.parse(actual)
+
+    assert_equal "playbook_run_finished", actual_message["type"]
+    assert_equal 3, actual_message["version"]
+    assert_equal @correlation_id, actual_message["correlation_id"]
+    assert_equal 'test_host', actual_message["host"]
+    assert_equal 'failure', actual_message["status"]
+    assert_equal 0, actual_message["connection_code"]
+    assert_equal 100, actual_message["execution_code"]
+  end
+
+  test 'Outputs host finished successfully message' do
+    @generator.host_finished_message('test_host', 0)
+
+    actual = @generator.generate
+    actual_message = JSON.parse(actual)
+
+    assert_equal "playbook_run_finished", actual_message["type"]
+    assert_equal 3, actual_message["version"]
+    assert_equal @correlation_id, actual_message["correlation_id"]
+    assert_equal 'test_host', actual_message["host"]
+    assert_equal 'success', actual_message["status"]
+    assert_equal 0, actual_message["connection_code"]
+    assert_equal 0, actual_message["execution_code"]
+  end
+
+  test 'Outputs job finished message' do
+    @generator.job_finished_message
+
+    actual = @generator.generate
+    actual_message = JSON.parse(actual)
+
+    assert_equal "playbook_run_completed", actual_message["type"]
+    assert_equal 3, actual_message["version"]
+    assert_equal @correlation_id, actual_message["correlation_id"]
+    assert_equal "success", actual_message["status"]
+  end
+
+  test 'Outputs a valid JSONL format' do
+    @generator.host_finished_message('test_host1', 0)
+    @generator.host_finished_message('test_host2', 0)
+
+    actual = @generator.generate.lines
+    actual_message1 = JSON.parse(actual[0])
+    assert_equal 'test_host1', actual_message1['host']
+    actual_message2 = JSON.parse(actual[1])
+    assert_equal 'test_host2', actual_message2['host']
+  end
+end