foreman_remote_execution 7.2.2 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7d32320885b2dc2343cf6efc7c1f7b0f4ac1e0668ea3d0c2dc817c691f1e7d2
-  data.tar.gz: 0655a041907bc30339ac53479220adb5434b20f7e6f38d5ce843dfb675f375cd
+  metadata.gz: 9a4cbf1cdf74f0bd20c4729a8bc05a7a3bb2a958bace18d30755fb600d0a999c
+  data.tar.gz: 7eac3c732b1c9b05e9c0909d952b859010cdeef677324353901859232d8ab3b0
 SHA512:
-  metadata.gz: 307bb3022d6fca4c61897443448638d84069ce943abb08d63b4e7de99cebdcf4f9956b9aed33c6e3effe2dad9a897c8981e80c775866ea5f32081ddf8950606d
-  data.tar.gz: b32a13df6917c8d41832b66dce88ecdc55383e296e250cbdecfb935f162b5e6e42fb694d57a7338ef6c5c102be9c6a6d2c2e95e1b04df9ac7bfa115b469364c1
+  metadata.gz: 42280d6aadec2d5ede443a1c81f688ce838bf388a65b4abdd30ab3d03953c33986023f0aee445e06cf908b5d6040e716a7bc8582b6059f553f400a4328ace60a
+  data.tar.gz: 3248ccc1bc2f48d61d3538f171d8f6c005b241c654ca2949833993896489bbc4d2ca2c2e0f7f0776dbd6d352f63d0fc2a292edb29f1df3cf6d402ffc16feb1c8

data/.github/workflows/ruby_ci.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.5
+          ruby-version: 2.7
           bundler-cache: true
           cache-version: 1
           rubygems: 3.0.0
@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
       matrix:
         foreman-core-branch: [develop]
-        ruby-version: [2.5, 2.7]
+        ruby-version: [2.7]
         node-version: [12]
     steps:
       - run: sudo apt-get update

data/app/controllers/job_invocations_controller.rb

@@ -45,8 +45,7 @@ class JobInvocationsController < ApplicationController
     if @composer.trigger
       redirect_to job_invocation_path(@composer.job_invocation)
     else
-      @composer.job_invocation.description_format = nil if params.fetch(:job_invocation, {}).key?(:description_override)
-      render :action => 'new'
+      redirect_to new_job_invocation_path({:inputs => params[:inputs], :feature => params[:feature], :host_ids => params[:host_ids]})
     end
   end
 

data/app/controllers/job_templates_controller.rb

@@ -34,7 +34,7 @@ class JobTemplatesController < ::TemplatesController
   def import
     contents = params.fetch(:imported_template, {}).fetch(:template, nil).try(:read)
 
-    @template = JobTemplate.import_raw(contents, :update => Foreman::Cast.to_bool(params[:imported_template][:overwrite]))
+    @template = JobTemplate.import_raw(contents, :update => ActiveRecord::Type::Boolean.new.deserialize(params[:imported_template][:overwrite]))
     if @template&.save
       flash[:success] = _('Job template imported successfully.')
       redirect_to job_templates_path(:search => "name = \"#{@template.name}\"")

data/app/controllers/ui_job_wizard_controller.rb

@@ -57,4 +57,19 @@ class UiJobWizardController < ApplicationController
     render :json => { :results =>
       resource_list.sort_by { |r| r[:name] }.take(100), :subtotal => resource_list.count}
   end
+
+  def job_invocation
+    job = JobInvocation.authorized.find(params[:id])
+    composer = JobInvocationComposer.from_job_invocation(job, params).params
+    job_template_inputs = JobTemplate.authorized.find(composer[:template_invocations][0][:template_id]).template_inputs_with_foreign
+    inputs = Hash[job_template_inputs.map { |input| ["inputs[#{input[:name]}]", {:advanced => input[:advanced], :value => (composer[:template_invocations][0][:input_values].find { |value| value[:template_input_id] == input[:id] }).try(:[], :value)}] }]
+    job_organization = Taxonomy.find_by(id: job.task.input[:current_organization_id])
+    job_location = Taxonomy.find_by(id: job.task.input[:current_location_id])
+    render :json => {
+      :job => composer,
+      :job_organization => job_organization,
+      :job_location => job_location,
+      :inputs => inputs,
+    }
+  end
 end

data/app/helpers/job_invocations_helper.rb

@@ -13,13 +13,6 @@ module JobInvocationsHelper
     end
   end
 
-  def job_invocations_buttons
-    [
-      documentation_button_rex('3.2ExecutingaJob'),
-      display_link_if_authorized(_('Run Job'), hash_for_new_job_invocation_path),
-    ]
-  end
-
   def template_name_and_provider_link(template)
     template_name = content_tag(:strong, template.name)
     provider = template.provider.humanized_name

data/app/helpers/remote_execution_helper.rb

@@ -56,7 +56,7 @@ module RemoteExecutionHelper
   def job_invocations_buttons
     [
       documentation_button_rex('3.2ExecutingaJob'),
-      new_link(_('Run Job')),
+      display_link_if_authorized(_('Run Job'), hash_for_new_job_invocation_path, {:class => "btn btn-primary"}),
     ]
   end
 
@@ -246,7 +246,7 @@ module RemoteExecutionHelper
   def job_report_template
     template = ReportTemplate.where(name: Setting['remote_execution_job_invocation_report_template']).first
 
-    template if template.template_inputs.where(name: 'job_id').exists?
+    template if template && template.template_inputs.where(name: 'job_id').exists?
   end
 
   def job_report_template_parameters(job_invocation, template)

data/app/lib/actions/remote_execution/proxy_action.rb

@@ -0,0 +1,46 @@
+module Actions
+  module RemoteExecution
+    class ProxyAction < ::Actions::ProxyAction
+      include Actions::RemoteExecution::TemplateInvocationProgressLogging
+
+      def on_data(data, meta = {})
+        super
+        process_proxy_data(output[:proxy_output])
+      end
+
+      def run(event = nil)
+        with_template_invocation_error_logging { super }
+      end
+
+      private
+
+      def get_proxy_data(response)
+        data = super
+        process_proxy_data(data)
+        data
+      end
+
+      def process_proxy_data(data)
+        events = data['result'].map do |update|
+          {
+            template_invocation_id: template_invocation.id,
+            event: update['output'],
+            timestamp: Time.at(update['timestamp']).getlocal,
+            event_type: update['output_type'],
+          }
+        end
+        if data['exit_status']
+          events << {
+            template_invocation_id: template_invocation.id,
+            event: data['exit_status'],
+            timestamp: events.last[:timestamp] + 0.0001,
+            event_type: 'exit',
+          }
+        end
+        events.each_slice(1000) do |batch|
+          TemplateInvocationEvent.upsert_all(batch, unique_by: [:template_invocation_id, :timestamp, :event_type]) # rubocop:disable Rails/SkipsModelValidations
+        end
+      end
+    end
+  end
+end

data/app/lib/actions/remote_execution/run_host_job.rb

@@ -4,6 +4,7 @@ module Actions
       include ::Actions::Helpers::WithContinuousOutput
       include ::Actions::Helpers::WithDelegatedAction
       include ::Actions::ObservableAction
+      include ::Actions::RemoteExecution::TemplateInvocationProgressLogging
 
       execution_plan_hooks.use :emit_feature_event, :on => :success
 
@@ -19,6 +20,13 @@ module Actions
       end
 
       def plan(job_invocation, host, template_invocation, proxy_selector = ::RemoteExecutionProxySelector.new, options = {})
+        with_template_invocation_error_logging do
+          inner_plan(job_invocation, host, template_invocation, proxy_selector, options)
+        end
+      end
+
+      def inner_plan(job_invocation, host, template_invocation, proxy_selector, options)
+        raise _('Could not use any template used in the job invocation') if template_invocation.blank?
         features = template_invocation.template.remote_execution_features.pluck(:label).uniq
         action_subject(host,
           :job_category => job_invocation.job_category,
@@ -35,8 +43,6 @@ module Actions
 
         verify_permissions(host, template_invocation)
 
-        raise _('Could not use any template used in the job invocation') if template_invocation.blank?
-
         provider = template_invocation.template.provider
         proxy_selector = provider.required_proxy_selector_for(template_invocation.template) || proxy_selector
 
@@ -63,13 +69,15 @@ module Actions
         action_options = provider.proxy_command_options(template_invocation, host)
                                  .merge(additional_options)
 
-        plan_delegated_action(proxy, provider.proxy_action_class, action_options)
-        plan_self
+        plan_delegated_action(proxy, provider.proxy_action_class, action_options, proxy_action_class: ::Actions::RemoteExecution::ProxyAction)
+        plan_self :with_event_logging => true
       end
 
       def finalize(*args)
-        update_host_status
-        check_exit_status
+        with_template_invocation_error_logging do
+          update_host_status
+          check_exit_status
+        end
       end
 
       def self.feature_job_event_name(label, suffix = :success)
@@ -128,6 +136,11 @@ module Actions
       end
 
       def fill_continuous_output(continuous_output)
+        if input[:with_event_logging]
+          continuous_output_from_template_invocation_events(continuous_output)
+          return
+        end
+
         delegated_output.fetch('result', []).each do |raw_output|
           continuous_output.add_raw_output(raw_output)
         end
@@ -148,8 +161,26 @@ module Actions
         continuous_output.add_exception(_('Error loading data from proxy'), e)
       end
 
+      def continuous_output_from_template_invocation_events(continuous_output)
+        begin
+          # Trigger reload
+          delegated_output unless task.state == 'stopped'
+        rescue => e
+          # This is enough, the error will get shown using add_exception at the end of the method
+        end
+
+        task.template_invocation.template_invocation_events.find_each do |output|
+          if output.event_type == 'exit'
+            continuous_output.add_output(_('Exit status: %s') % output.event, 'stdout', output.timestamp)
+          else
+            continuous_output.add_raw_output(output.as_raw_continuous_output)
+          end
+        end
+        continuous_output.add_exception(_('Error loading data from proxy'), e) if e
+      end
+
       def exit_status
-        delegated_output[:exit_status]
+        input[:with_event_logging] ? task.template_invocation.template_invocation_events.find_by(event_type: 'exit').event : delegated_output[:exit_status]
       end
 
       def host_id
@@ -172,10 +203,6 @@ module Actions
         @host ||= ::Host.authorized.find(host_id)
       end
 
-      def self.cleanup_after
-        '90d'
-      end
-
       private
 
       def update_host_status

data/app/lib/actions/remote_execution/run_hosts_job.rb

@@ -47,7 +47,6 @@ module Actions
           # composer creates just "pattern" for template_invocations because target is evaluated
           # during actual run (here) so we build template invocations from these patterns
           template_invocation = job_invocation.pattern_template_invocation_for_host(host).deep_clone
-          template_invocation.host_id = host.id
           trigger(RunHostJob, job_invocation, host, template_invocation, proxy_selector, { :use_concurrency_control => uses_concurrency_control })
         end
       end
@@ -76,7 +75,7 @@ module Actions
         job_invocation.save!
 
         Rails.logger.debug "cleaning cache for keys that begin with 'job_invocation_#{job_invocation.id}'"
-        Rails.cache.delete_matched(/\A#{JobInvocation::CACHE_PREFIX}_#{job_invocation.id}/)
+        Rails.cache.delete_matched(cache_deletion_query(job_invocation.id))
       end
 
       def notify_on_success(_plan)
@@ -153,15 +152,17 @@ module Actions
         input[:proxy_batch_size]
       end
 
-      def self.cleanup_after
-        '90d'
-      end
-
       private
 
       def mail_notification_preference
         UserMailNotification.where(mail_notification_id: RexMailNotification.first, user_id: User.current.id).first
       end
+
+      def cache_deletion_query(job_invocation_id)
+        return "#{JobInvocation::CACHE_PREFIX}_#{job_invocation_id}*" if Rails.cache.kind_of? ActiveSupport::Cache::RedisCacheStore
+
+        /\A#{JobInvocation::CACHE_PREFIX}_#{job_invocation_id}/
+      end
     end
   end
 end

data/app/lib/actions/remote_execution/template_invocation_progress_logging.rb

@@ -0,0 +1,27 @@
+module Actions
+  module RemoteExecution
+    module TemplateInvocationProgressLogging
+      def template_invocation
+        @template_invocation ||= TemplateInvocation.find_by(:run_host_job_task_id => task.id)
+      end
+
+      def log_template_invocation_exception(exception)
+        template_invocation.template_invocation_events.create!(
+          :event_type => 'debug',
+          :event => "#{exception.class}: #{exception.message}",
+          :timestamp => Time.zone.now
+        )
+      end
+
+      def with_template_invocation_error_logging
+        unless catch(::Dynflow::Action::ERROR) { yield || true }
+          log_template_invocation_exception(error.exception)
+          throw ::Dynflow::Action::ERROR
+        end
+      rescue => e # rubocop:disable Style/RescueStandardError
+        log_template_invocation_exception(e)
+        raise e
+      end
+    end
+  end
+end

data/app/models/job_invocation.rb

@@ -213,13 +213,7 @@ class JobInvocation < ApplicationRecord
 
   def pattern_template_invocation_for_host(host)
     providers = available_providers(host)
-    providers.each do |provider|
-      pattern_template_invocations.each do |template_invocation|
-        if template_invocation.template.provider_type == provider
-          return template_invocation
-        end
-      end
-    end
+    pattern_template_invocations.find { |pti| providers.include? pti.template.provider_type }
   end
 
   # TODO: determine from the host and job_invocation details

data/app/models/job_invocation_composer.rb

@@ -22,6 +22,7 @@ class JobInvocationComposer
         :effective_user_password => blank_to_nil(job_invocation_base[:effective_user_password]),
         :concurrency_control => concurrency_control_params,
         :execution_timeout_interval => execution_timeout_interval,
+        :time_to_pickup => job_invocation_base[:time_to_pickup],
         :template_invocations => template_invocations_params }.with_indifferent_access
     end
 
@@ -129,6 +130,7 @@ class JobInvocationComposer
         :key_passphrase => api_params[:key_passphrase],
         :concurrency_control => concurrency_control_params,
         :execution_timeout_interval => api_params[:execution_timeout_interval] || template.execution_timeout_interval,
+        :time_to_pickup => api_params[:time_to_pickup],
         :template_invocations => template_invocations_params }.with_indifferent_access
     end
 
@@ -245,6 +247,7 @@ class JobInvocationComposer
         :execution_timeout_interval => job_invocation.execution_timeout_interval,
         :remote_execution_feature_id => job_invocation.remote_execution_feature_id,
         :template_invocations => template_invocations_params,
+        :time_to_pickup => job_invocation.time_to_pickup,
         :reruns => job_invocation.id }.with_indifferent_access
     end
 
@@ -415,6 +418,7 @@ class JobInvocationComposer
     job_invocation.key_passphrase = params[:key_passphrase]
     job_invocation.effective_user_password = params[:effective_user_password]
     job_invocation.ssh_user = params[:ssh_user]
+    job_invocation.time_to_pickup = params[:time_to_pickup]
 
     if @reruns && job_invocation.targeting.static?
       job_invocation.targeting.assign_host_ids(JobInvocation.find(@reruns).targeting.host_ids)
@@ -441,8 +445,11 @@ class JobInvocationComposer
   end
 
   def valid?
-    targeting.valid? & job_invocation.valid? & !pattern_template_invocations.map(&:valid?).include?(false) &
-      triggering.valid?
+    unless triggering.valid?
+      job_invocation.errors.add(:triggering, 'is invalid')
+      return false
+    end
+    targeting.valid? & job_invocation.valid? & !pattern_template_invocations.map(&:valid?).include?(false)
   end
 
   def save

data/app/models/remote_execution_provider.rb

@@ -33,7 +33,10 @@ class RemoteExecutionProvider
     end
 
     def proxy_command_options(template_invocation, host)
-      {:proxy_operation_name => proxy_operation_name}.merge(proxy_command_provider_inputs(template_invocation))
+      {
+        :proxy_operation_name => proxy_operation_name,
+        :time_to_pickup => time_to_pickup(template_invocation.job_invocation, host),
+      }.merge(proxy_command_provider_inputs(template_invocation))
     end
 
     def secrets(_host)
@@ -152,5 +155,10 @@ class RemoteExecutionProvider
     def alternative_names(host)
       { :fqdn => find_fqdn(effective_interfaces(host)) }
     end
+
+    def time_to_pickup(job_invocation, host)
+      time = job_invocation.time_to_pickup || host_setting(host, 'remote_execution_time_to_pickup')
+      Integer(time) if time
+    end
   end
 end

data/app/models/ssh_execution_provider.rb

@@ -45,6 +45,7 @@ class ScriptExecutionProvider < RemoteExecutionProvider
         :ssh_port => ssh_port(host),
         :ssh_password => ssh_password(host),
         :ssh_key_passphrase => ssh_key_passphrase(host),
+        :effective_user_password => effective_user_password(host),
       }
     end
 

data/app/models/template_invocation.rb

@@ -17,6 +17,7 @@ class TemplateInvocation < ApplicationRecord
   has_one :host_group, :through => :host, :source => :hostgroup
   belongs_to :run_host_job_task, :class_name => 'ForemanTasks::Task'
   has_many :remote_execution_features, :through => :template
+  has_many :template_invocation_events, :dependent => :destroy
 
   validates_associated :input_values
   validate :provides_required_input_values

data/app/models/template_invocation_event.rb

@@ -0,0 +1,11 @@
+class TemplateInvocationEvent < ::ApplicationRecord
+  belongs_to :template_invocation
+
+  def as_raw_continuous_output
+    raw = attributes
+    raw['output_type'] = raw.delete('event_type')
+    raw['output'] = raw.delete('event')
+    raw['timestamp'] = raw['timestamp'].to_f
+    raw
+  end
+end

data/app/views/job_invocations/_form.html.erb

@@ -102,6 +102,10 @@
       </fieldset>
     <% end %>
 
+    <div class="advanced hidden">
+      <%= number_f f, :time_to_pickup, :value => f.object.time_to_pickup, :label => _('Time to pickup'), :label_help => N_('Interval in seconds, if the job is not picked up by a client within this interval it will be cancelled.') %>
+    </div>
+
     <div class="advanced hidden">
       <%= password_f f, :password, :placeholder => '*****', :label => _('Password'), :label_help => N_('Password is stored encrypted in DB until the job finishes. For future or recurring executions, it is removed after the last execution.') %>
       <%= password_f f, :key_passphrase, :placeholder => '*****', :label => _('Private key passphrase'), :label_help => N_('Key passhprase is only applicable for SSH provider. Other providers ignore this field. <br> Passphrase is stored encrypted in DB until the job finishes. For future or recurring executions, it is removed after the last execution.') %>

data/app/views/job_invocations/new.html.erb

@@ -1,4 +1,9 @@
 <% javascript 'foreman_remote_execution/template_invocation' %>
 <% stylesheet 'foreman_remote_execution/foreman_remote_execution' %>
 <% title _('Job invocation') %>
+<% if(request.path_parameters[:action] == "rerun") %>
+  <%= display_link_if_authorized(_('Use new job wizard'), hash_for_rerun_job_invocation_path({:id => request.path_parameters[:id]}).merge(request.query_parameters)) %>
+<% else %>
+  <%= display_link_if_authorized(_('Use new job wizard'), hash_for_new_job_invocation_path().merge(request.query_parameters)) %>
+<%end%>
 <%= render :partial => 'form' %>

data/app/views/templates/script/package_action.erb

@@ -16,6 +16,11 @@ template_inputs:
   input_type: user
   required: true
   options: "install\nupdate\nremove\ngroup install\ngroup remove"
+- name: options
+  description: Additional options for the package manager
+  input_type: user
+  required: false
+  advanced: true
 - name: package
   description: The name of the package, if any
   input_type: user
@@ -87,7 +92,7 @@ handle_zypp_res_codes () {
 
 # Action
 <% if package_manager == 'yum' -%>
-  yum -y <%= action %> <%= input("package") %>
+  yum -y <%= input("options") %> <%= action %> <%= input("package") %>
 <% elsif package_manager == 'apt' -%>
   <%-
     action = 'install' if action == 'group install'
@@ -103,7 +108,7 @@ handle_zypp_res_codes () {
   [ -x "$(command -v subscription-manager)" ] && subscription-manager refresh
   export DEBIAN_FRONTEND=noninteractive
   apt-get -y update
-  apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y <%= action %> <%= input("package") %>
+  apt-get -o APT::Get::Upgrade-Allow-New="true" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y <%= input("options") %> <%= action %> <%= input("package") %>
 <% elsif package_manager == 'zypper' -%>
   <%-
     if action == "group install"
@@ -113,7 +118,7 @@ handle_zypp_res_codes () {
     end
   -%>
   zypper refresh
-  zypper -n <%= action %> <%= input("package") %>
+  zypper -n <%= action %> <%= input("options") %> <%= input("package") %>
   handle_zypp_res_codes $?
 <% end -%>
 RETVAL=$?

data/config/routes.rb

@@ -16,7 +16,11 @@ Rails.application.routes.draw do
     end
   end
 
-  match 'job_invocations/new', to: 'job_invocations#new', via: [:get, :post], as: 'new_job_invocation'
+  match 'job_invocations/new', to: 'react#index', :via => [:get], as: 'new_job_invocation'
+  match 'job_invocations/new', to: 'job_invocations#create', via: [:post], as: 'create_job_invocation'
+  match 'job_invocations/:id/rerun', to: 'react#index', :via => [:get], as: 'rerun_job_invocation'
+  match 'old/job_invocations/new', to: 'job_invocations#new', via: [:get], as: 'form_new_job_invocation'
+  match 'old/job_invocations/:id/rerun', to: 'job_invocations#rerun', via: [:get, :post], as: 'form_rerun_job_invocation'
   resources :job_invocations, :only => [:create, :show, :index] do
     collection do
       post 'refresh'
@@ -25,7 +29,6 @@ Rails.application.routes.draw do
       get 'auto_complete_search'
     end
     member do
-      get 'rerun'
       post 'cancel'
     end
   end
@@ -46,8 +49,7 @@ Rails.application.routes.draw do
   get 'ui_job_wizard/categories', to: 'ui_job_wizard#categories'
   get 'ui_job_wizard/template/:id', to: 'ui_job_wizard#template'
   get 'ui_job_wizard/resources', to: 'ui_job_wizard#resources'
-
-  match '/experimental/job_wizard', to: 'react#index', :via => [:get]
+  get 'ui_job_wizard/job_invocation', to: 'ui_job_wizard#job_invocation'
 
   namespace :api, :defaults => {:format => 'json'} do
     scope '(:apiv)', :module => :v2, :defaults => {:apiv => 'v2'}, :apiv => /v1|v2/, :constraints => ApiConstraints.new(:version => 2, :default => true) do

data/db/migrate/20220713095705_create_template_invocation_events.rb

@@ -0,0 +1,17 @@
+class CreateTemplateInvocationEvents < ActiveRecord::Migration[6.1]
+  def change
+    # rubocop:disable Rails/CreateTableWithTimestamps
+    create_table :template_invocation_events do |t|
+      t.references :template_invocation, :null => false
+      t.timestamp :timestamp, :null => false
+      t.string :event_type, :null => false
+      t.string :event, :null => false
+      t.string :meta
+
+      t.index [:template_invocation_id, :timestamp, :event_type],
+        unique: true,
+        name: 'unique_template_invocation_events_index'
+    end
+    # rubocop:enable Rails/CreateTableWithTimestamps
+  end
+end

data/db/migrate/20220822155946_add_time_to_pickup_to_job_invocation.rb

@@ -0,0 +1,5 @@
+class AddTimeToPickupToJobInvocation < ActiveRecord::Migration[6.0]
+  def change
+    add_column :job_invocations, :time_to_pickup, :integer
+  end
+end