foreman_rh_cloud 5.0.32 → 5.0.35

data/lib/foreman_rh_cloud/engine.rb

@@ -6,15 +6,18 @@ module ForemanRhCloud
     engine_name 'foreman_rh_cloud'
 
     def self.register_scheduled_task(task_class, cronline)
-      return if ForemanTasks::RecurringLogic.joins(:tasks)
-                .merge(ForemanTasks::Task.where(label: task_class.name))
-                .exists?
-
-      User.as_anonymous_admin do
-        recurring_logic = ForemanTasks::RecurringLogic.new_from_cronline(cronline)
-        recurring_logic.save!
-        recurring_logic.start(task_class)
+      ForemanTasks::RecurringLogic.transaction(isolation: :serializable) do
+        return if ForemanTasks::RecurringLogic.joins(:tasks)
+                  .merge(ForemanTasks::Task.where(label: task_class.name))
+                  .exists?
+
+        User.as_anonymous_admin do
+          recurring_logic = ForemanTasks::RecurringLogic.new_from_cronline(cronline)
+          recurring_logic.save!
+          recurring_logic.start(task_class)
+        end
       end
+    rescue ActiveRecord::TransactionIsolationError
     end
 
     initializer 'foreman_rh_cloud.load_default_settings', :before => :load_config_initializers do
@@ -76,9 +79,13 @@ module ForemanRhCloud
             },
             :resource_type => ::InsightsHit.name
           )
+          permission(
+            :dispatch_cloud_requests,
+            'api/v2/rh_cloud/cloud_request': [:update]
+          )
         end
 
-        plugin_permissions = [:view_foreman_rh_cloud, :generate_foreman_rh_cloud, :view_insights_hits]
+        plugin_permissions = [:view_foreman_rh_cloud, :generate_foreman_rh_cloud, :view_insights_hits, :dispatch_cloud_requests]
 
         role 'ForemanRhCloud', plugin_permissions, 'Role granting permissions to view the hosts inventory,
                                                     generate a report, upload it to the cloud and download it locally'
@@ -116,7 +123,7 @@ module ForemanRhCloud
         end
 
         extend_template_helpers ForemanRhCloud::TemplateRendererHelper
-        allowed_template_helpers :remediations_playbook
+        allowed_template_helpers :remediations_playbook, :download_rh_playbook
       end
 
       ::Katello::UINotifications::Subscriptions::ManifestImportSuccess.include ForemanInventoryUpload::Notifications::ManifestImportSuccessNotificationOverride if defined?(Katello)
@@ -136,15 +143,29 @@ module ForemanRhCloud
       Foreman::Gettext::Support.add_text_domain locale_domain, locale_dir
     end
 
-    config.to_prepare do
+    initializer 'foreman_rh_cloud.register_rex_features', :before => :finisher_hook do |_app|
       # skip database manipulations while tables do not exist, like in migrations
-      if ActiveRecord::Base.connection.data_source_exists?(ForemanTasks::Task.table_name) &&
+      if ActiveRecord::Base.connection.data_source_exists?(ForemanTasks::Task.table_name)
         RemoteExecutionFeature.register(
           :rh_cloud_remediate_hosts,
           N_('Apply Insights recommendations'),
           description: N_('Run remediation playbook generated by Insights'),
           host_action_button: false
         )
+        RemoteExecutionFeature.register(
+          :rh_cloud_connector_run_playbook,
+          N_('Run RH Cloud playbook'),
+          description: N_('Run playbook genrated by Red Hat remediations app'),
+          host_action_button: false,
+          provided_inputs: ['playbook_url', 'report_url', 'correlation_id', 'report_interval']
+        )
+        RemoteExecutionFeature.register(
+          :ansible_configure_cloud_connector,
+          N_('Configure Cloud Connector on given hosts'),
+          :description => N_('Configure Cloud Connector on given hosts'),
+          :proxy_selector_override => ::RemoteExecutionProxySelector::INTERNAL_PROXY
+        )
+
         # skip object creation when admin user is not present, for example in test DB
         if User.unscoped.find_by_login(User::ANONYMOUS_ADMIN).present?
           ::ForemanTasks.dynflow.config.on_init(false) do |world|

data/lib/foreman_rh_cloud/version.rb

@@ -1,3 +1,3 @@
 module ForemanRhCloud
-  VERSION = '5.0.32'.freeze
+  VERSION = '5.0.35'.freeze
 end

data/lib/foreman_rh_cloud.rb

@@ -121,4 +121,8 @@ module ForemanRhCloud
   def self.legacy_insights_ca
     "#{ForemanRhCloud::Engine.root}/config/rh_cert-api_chain.pem"
   end
+
+  def self.cloud_url_validator
+    @cloud_url_validator ||= Regexp.new(ENV['SATELLITE_RH_CLOUD_VALIDATOR'] || 'redhat.com$')
+  end
 end

data/lib/insights_cloud/async/connector_playbook_execution_reporter_task.rb

@@ -0,0 +1,193 @@
+module InsightsCloud
+  module Async
+    class ConnectorPlaybookExecutionReporterTask < ::Actions::EntryAction
+      include Dynflow::Action::Polling
+      include ForemanRhCloud::CloudAuth
+
+      def self.subscribe
+        Actions::RemoteExecution::RunHostsJob
+      end
+
+      def self.connector_feature_id
+        @connector_feature_id ||= RemoteExecutionFeature.feature!(:rh_cloud_connector_run_playbook).id
+      end
+
+      def plan(job_invocation)
+        return unless connector_playbook_job?(job_invocation)
+
+        @job_invocation = job_invocation
+
+        invocation_inputs = invocation_inputs(job_invocation)
+        report_url = invocation_inputs['report_url']
+        report_interval = [invocation_inputs['report_interval'].to_i, 5].max
+        correlation_id = invocation_inputs['correlation_id']
+
+        plan_self(
+          report_url: report_url,
+          report_interval: report_interval,
+          job_invocation_id: job_invocation.id,
+          correlation_id: correlation_id
+        )
+      end
+
+      def run(event = nil)
+        # Handle skip events
+        return if event == Dynflow::Action::Skip
+
+        super
+      end
+
+      def rescue_strategy_for_self
+        Dynflow::Action::Rescue::Skip
+      end
+
+      def done?(current_status = invocation_status)
+        job_invocation.finished? || current_status.map { |_host_id, task_status| task_status['report_done'] }.all?
+      end
+
+      # noop, we don't want to do anything when the polling task starts
+      def invoke_external_task
+        poll_external_task
+      end
+
+      def poll_external_task
+        current_status = invocation_status
+        report_job_progress(current_status)
+        # record the current state and increment the sequence for the next invocation
+        {
+          invocation_status: current_status,
+        }
+      end
+
+      def poll_intervals
+        [report_interval]
+      end
+
+      private
+
+      def connector_playbook_job?(job_invocation)
+        puts "Job invocation id: #{job_invocation&.remote_execution_feature_id}, feature id: #{connector_feature_id}"
+        job_invocation&.remote_execution_feature_id == connector_feature_id
+      end
+
+      def connector_feature_id
+        self.class.connector_feature_id
+      end
+
+      def invocation_inputs(job_invocation)
+        Hash[
+          job_invocation.pattern_template_invocations.first.input_values.map do |input_value|
+            [input_value.template_input.name, input_value.value]
+          end
+        ]
+      end
+
+      def job_invocation
+        @job_invocation ||= JobInvocation.find(input['job_invocation_id'])
+      end
+
+      def report_interval
+        @report_interval ||= input['report_interval']
+      end
+
+      def correlation_id
+        @correlation_id ||= input['correlation_id']
+      end
+
+      def host_status(host)
+        external_task&.dig('invocation_status', host)
+      end
+
+      def sequence(host)
+        host_status(host)&.fetch('sequence', 0).to_i
+      end
+
+      def host_done?(host)
+        ActiveModel::Type::Boolean.new.cast(host_status(host)&.fetch('report_done', nil))
+      end
+
+      def report_url
+        input['report_url']
+      end
+
+      def invocation_status
+        Hash[job_invocation.targeting.hosts.map do |host|
+          next unless host.insights&.uuid
+          [
+            host.insights.uuid,
+            task_status(job_invocation.sub_task_for_host(host), host.insights.uuid),
+          ]
+        end.compact]
+      end
+
+      def task_status(host_task, host_name)
+        unless host_task
+          return { 'state' => 'unknown' }
+        end
+
+        {
+          'state' => host_task.state,
+          'output' => host_task.main_action.live_output.map { |line| line['output'] }.join("\n"),
+          'exit_status' => host_task.main_action.exit_status,
+          'sequence' => sequence(host_name),
+          'report_done' => host_done?(host_name),
+        }
+      end
+
+      def report_job_progress(invocation_status)
+        generator = InsightsCloud::Generators::PlaybookProgressGenerator.new(correlation_id)
+
+        invocation_status.each do |host_name, status|
+          # skip host if the host already reported that it's finished
+          next if status['report_done']
+
+          unless status['state'] == 'unknown'
+            sequence = status['sequence']
+            generator.host_progress_message(host_name, status['output'], sequence)
+            status['sequence'] = sequence + 1 # increase the sequence for each host
+          end
+
+          if status['state'] == 'stopped'
+            generator.host_finished_message(host_name, status['exit_status'])
+            status['report_done'] = true
+          end
+        end
+        generator.job_finished_message if done?(invocation_status)
+
+        send_report(generator.generate)
+      end
+
+      def send_report(report)
+        execute_cloud_request(
+          method: :post,
+          url: report_url,
+          content_type: 'application/vnd.redhat.playbook-sat.v3+jsonl',
+          payload: {
+            file: wrap_report(report),
+            multipart: true,
+          }
+        )
+      end
+
+      # RestClient has to accept an object that responds to :read, :path and :content_type methods
+      # to properly generate a multipart message.
+      # see: https://github.com/rest-client/rest-client/blob/2c72a2e77e2e87d25ff38feba0cf048d51bd5eca/lib/restclient/payload.rb#L161
+      def wrap_report(report)
+        obj = StringIO.new(report)
+        def obj.content_type
+          'application/vnd.redhat.playbook-sat.v3+jsonl'
+        end
+
+        def obj.path
+          'file'
+        end
+
+        obj
+      end
+
+      def logger
+        action_logger
+      end
+    end
+  end
+end

data/lib/insights_cloud/async/insights_scheduled_sync.rb

@@ -22,6 +22,10 @@ module InsightsCloud
       def rescue_strategy_for_self
         Dynflow::Action::Rescue::Fail
       end
+
+      def logger
+        action_logger
+      end
     end
   end
 end

data/lib/insights_cloud/generators/playbook_progress_generator.rb

@@ -0,0 +1,49 @@
+module InsightsCloud
+  module Generators
+    class PlaybookProgressGenerator
+      attr_reader :correlation_id
+      def initialize(correlation_id)
+        @messages = []
+        @correlation_id = correlation_id
+      end
+
+      def host_progress_message(host_name, output, sequence)
+        @messages << {
+          "type": "playbook_run_update",
+          "version": 3,
+          "correlation_id": correlation_id,
+          "sequence": sequence,
+          "host": host_name,
+          "console": output,
+        }
+      end
+
+      def host_finished_message(host_name, exit_code)
+        @messages << {
+          "type": "playbook_run_finished",
+          "version": 3,
+          "correlation_id": correlation_id,
+          "host": host_name,
+          "status": exit_code == 0 ? 'success' : 'failure',
+          "connection_code": 0,
+          "execution_code": exit_code,
+        }
+      end
+
+      def job_finished_message
+        @messages << {
+          "type": "playbook_run_completed",
+          "version": 3,
+          "correlation_id": correlation_id,
+          "status": "success",
+        }
+      end
+
+      def generate
+        @messages.map do |message|
+          message.to_json
+        end.join("\n")
+      end
+    end
+  end
+end

data/lib/tasks/insights.rake

@@ -19,4 +19,17 @@ namespace :rh_cloud_insights do
 
     puts "Deleted #{deleted_count} insights statuses"
   end
+
+  desc "Re-announce all organizations into Sources on RH cloud."
+  task announce_to_sources: [:environment] do
+    logger = Logging::Logger.new(STDOUT)
+    Organization.unscoped.each do |org|
+      presence = ForemanRhCloud::CloudPresence.new(org, logger)
+      presence.announce_to_sources
+    rescue StandardError => ex
+      logger.warn(ex)
+    end
+
+    logger.info('Reannounced all organizations')
+  end
 end

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foreman_rh_cloud",
-  "version": "5.0.32",
+  "version": "5.0.35",
   "description": "Inventory Upload =============",
   "main": "index.js",
   "scripts": {

data/test/controllers/insights_cloud/api/cloud_request_controller_test.rb

@@ -0,0 +1,78 @@
+require 'json'
+require 'test_plugin_helper'
+
+module InsightsCloud::Api
+  class CloudRequestControllerTest < ActionController::TestCase
+    tests Api::V2::RhCloud::CloudRequestController
+
+    setup do
+      @test_org = FactoryBot.create(:organization)
+    end
+
+    test 'Fails for unknown directives' do
+      request_params = run_playbook_request
+      request_params['directive'] = 'not-valid'
+
+      post :update, params: request_params
+
+      assert_response :bad_request
+    end
+
+    test 'Starts playbook run for correct directive' do
+      Setting[:rh_cloud_token] = 'MOCK_TOKEN'
+      host1 = FactoryBot.create(:host, :with_insights_hits)
+      host1.insights.uuid = 'TEST_UUID1'
+      host1.insights.save!
+      host2 = FactoryBot.create(:host, :with_insights_hits)
+      host2.insights.uuid = 'TEST_UUID2'
+      host2.insights.save!
+
+      mock_composer = mock('composer')
+      ::JobInvocationComposer.expects(:for_feature).with do |feature, host_ids, params|
+        feature == :rh_cloud_connector_run_playbook &&
+        host_ids.first == host1.id &&
+        host_ids.last == host2.id
+      end.returns(mock_composer)
+      mock_composer.expects(:trigger!)
+      mock_composer.expects(:job_invocation)
+
+      post :update, params: run_playbook_request
+
+      assert_response :success
+    end
+
+    private
+
+    def run_playbook_request
+      request_json = <<-REQUEST
+      {
+        "type": "data",
+        "message_id": "a6a7d866-7de0-409a-84e0-3c56c4171bb7",
+        "version": 1,
+        "sent": "2021-01-12T15:30:08+00:00",
+        "directive": "playbook-sat",
+        "metadata": {
+            "operation": "run",
+            "return_url": "https://cloud.redhat.com/api/v1/ingres/upload",
+            "correlation_id": "6684b9dd-0d16-42c1-b13a-9f45be59e3b6",
+            "playbook_run_name": "Human-readable playbook run name",
+            "playbook_run_url": "https://console.redhat.com/insights/remediations/1234",
+            "sat_id": "aa3b1faa-56f3-4d14-8258-615d11e20060",
+            "sat_org_id": "#{FactoryBot.create(:organization).id}",
+            "initiator_user_id": "4efca34c6d9ae05ef7c3d7a7424e6370d198159a841ae005084888a9a4529e27",
+            "hosts": "TEST_UUID1,TEST_UUID2",
+            "response_interval": "30",
+            "response_full": "false"
+        },
+        "content": ""
+      }
+      REQUEST
+
+      request = JSON.parse(request_json)
+
+      request['content'] = "\"#{Base64.encode64('https://cloud.redhat.com/api/v1/remediations/1234/playbook')}\""
+
+      request
+    end
+  end
+end

data/test/controllers/insights_cloud/api/machine_telemetries_controller_test.rb

@@ -60,6 +60,18 @@ module InsightsCloud::Api
         assert_equal x_rh_insights_request_id, @response.headers['x_rh_insights_request_id']
       end
 
+      test "should set etag header to response from cloud" do
+        etag = '12345'
+        req = RestClient::Request.new(:method => 'GET', :url => 'http://test.theforeman.org', :headers => { "If-None-Match": etag})
+        net_http_resp = Net::HTTPResponse.new(1.0, 200, "OK")
+        net_http_resp[Rack::ETAG] = etag
+        res = RestClient::Response.create(@body, net_http_resp, req)
+        ::ForemanRhCloud::CloudRequestForwarder.any_instance.stubs(:forward_request).returns(res)
+
+        get :forward_request, params: { "path" => "static/v1/release/insights-core.egg" }
+        assert_equal etag, @response.headers[Rack::ETAG]
+      end
+
       test "should handle failed authentication to cloud" do
         net_http_resp = Net::HTTPResponse.new(1.0, 401, "Unauthorized")
         res = RestClient::Response.create(@body, net_http_resp, @http_req)
@@ -116,6 +128,10 @@ module InsightsCloud::Api
       test 'should get branch info' do
         get :branch_info
 
+        res = JSON.parse(@response.body)
+
+        assert_not_equal 0, res['labels'].find { |l| l['key'] == 'hostgroup' }.count
+
         assert_response :success
       end