foreman_remote_execution 4.6.0 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14dc89216ec220750d38149b86e0ecded26e13d99678505302a5a157b0c7c9e2
-  data.tar.gz: b2091af78be3ed0fec0d5acea056a49e601e6d1ef96954c49976fd1c0da39ecb
+  metadata.gz: cd91f2539393453fe888022502baba8ad7597da50682a4d87a8cf78dcaaebbfa
+  data.tar.gz: 2c97670fa1b3555f9660ea8adee03e48ae83779aff3324d977eae65a70b86956
 SHA512:
-  metadata.gz: 10d901fec16519db44d05d9b9ecb2b15923e1a2e439f1e0032778b9272ed4c48a446436b4ef7375bb5113f8067d5596035b5a7fa593a522fc37e56a472d51c31
-  data.tar.gz: cb7686bf72b55bf65bbd35cee82910925f12b9dd29ec573dc96bb340fb1cc629833a80239e96317dcef2163d17d00ba93db92f5f50a10c5a4f4363d77647f729
+  metadata.gz: 7fbc26b3be07128f7dd015db03b8e46d34faee8f13f8b7e7f3d6ff54f360afd70cdd6f4143862695342591be55cfed9a74f315ae77ef5231cd20bee5f9bf8565
+  data.tar.gz: 851d5fd53efb928e8a6ce15468af83bc992b98265bc289f23fed310834e1c83753113e981a673f89e3183efaa7d6f76ec270f06f1f41072f79c045947a78b6c8

data/.github/workflows/ruby_ci.yml

@@ -71,3 +71,10 @@ jobs:
         run: |
           bundle exec rake test:foreman_remote_execution
           bundle exec rake test TEST="test/unit/foreman/access_permissions_test.rb"
+      - name: 'Upload logs'
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: log/*.log
+          retention-days: 5

data/.rubocop_todo.yml

@@ -220,6 +220,7 @@ Naming/FileName:
     - 'db/seeds.d/60-ssh_proxy_feature.rb'
     - 'db/seeds.d/70-job_templates.rb'
     - 'db/seeds.d/90-bookmarks.rb'
+    - 'db/seeds.d/95-mail_notifications.rb'
 
 # Offense count: 1
 # Configuration parameters: ForbiddenDelimiters.

data/app/controllers/api/v2/job_invocations_controller.rb

@@ -41,12 +41,18 @@ module Api
             param :effective_user, String,
               :required => false,
               :desc => N_('What user should be used to run the script (using sudo-like mechanisms). Defaults to a template parameter or global setting.')
+            param :effective_user_password, String,
+              :required => false,
+              :desc => N_('Set password for effective user (using sudo-like mechanisms)')
           end
+          param :password, String, :required => false, :desc => N_('Set SSH password')
+          param :key_passphrase, String, :required => false, :desc => N_('Set SSH key passphrase')
 
           param :recurrence, Hash, :desc => N_('Create a recurring job') do
             param :cron_line, String, :required => false, :desc => N_('How often the job should occur, in the cron format')
             param :max_iteration, :number, :required => false, :desc => N_('Repeat a maximum of N times')
             param :end_time, DateTime, :required => false, :desc => N_('Perform no more executions after this time')
+            param :purpose, String, :required => false, :desc => N_('Designation of a special purpose')
           end
 
           param :scheduling, Hash, :desc => N_('Schedule the job to start at a later time') do
@@ -162,6 +168,15 @@ module Api
         render :json => { :outputs => outputs }
       end
 
+      def resource_name(resource = controller_name)
+        case resource
+        when 'organization', 'location'
+          nil
+        else
+          'job_invocation'
+        end
+      end
+
       private
 
       def allowed_nested_id
@@ -197,7 +212,7 @@ module Api
         end
 
         if job_invocation_params.key?(:ssh)
-          job_invocation_params.merge!(job_invocation_params.delete(:ssh).permit(:effective_user))
+          job_invocation_params.merge!(job_invocation_params.delete(:ssh).permit(:effective_user, :effective_user_password))
         end
 
         job_invocation_params[:inputs] ||= {}

data/app/controllers/job_invocations_controller.rb

@@ -67,7 +67,7 @@ class JobInvocationsController < ApplicationController
   end
 
   def index
-    @job_invocations = resource_base_search_and_page.with_task.order('job_invocations.id DESC')
+    @job_invocations = resource_base_search_and_page.preload(:task, :targeting).order('job_invocations.id DESC')
   end
 
   # refreshes the form

data/app/controllers/ui_job_wizard_controller.rb

@@ -1,25 +1,37 @@
-class UiJobWizardController < ::Api::V2::BaseController
+class UiJobWizardController < ApplicationController
+  include FiltersHelper
   def categories
     job_categories = resource_scope
                      .search_for("job_category ~ \"#{params[:search]}\"")
                      .select(:job_category).distinct
                      .reorder(:job_category)
                      .pluck(:job_category)
-    render :json => {:job_categories =>job_categories}
+    render :json => {:job_categories =>job_categories, :with_katello => with_katello}
   end
 
   def template
     job_template = JobTemplate.authorized.find(params[:id])
+    advanced_template_inputs, template_inputs = map_template_inputs(job_template.template_inputs_with_foreign).partition { |x| x["advanced"] }
     render :json => {
       :job_template => job_template,
       :effective_user => job_template.effective_user,
+      :template_inputs => template_inputs,
+      :advanced_template_inputs => advanced_template_inputs,
     }
   end
 
+  def map_template_inputs(template_inputs_with_foreign)
+    template_inputs_with_foreign.map { |input| input.attributes.merge({:resource_type_tableize => input.resource_type&.tableize }) }
+  end
+
   def resource_name(nested_resource = nil)
     nested_resource || 'job_template'
   end
 
+  def with_katello
+    !!defined?(::Katello)
+  end
+
   def resource_class
     JobTemplate
   end
@@ -27,4 +39,11 @@ class UiJobWizardController < ::Api::V2::BaseController
   def action_permission
     :view_job_templates
   end
+
+  def resources
+    resource_type = params[:resource]
+    resource_list = resource_type.constantize.authorized("view_#{resource_type.underscore.pluralize}").all.map { |r| {:name => r.to_s, :id => r.id } }.select { |v| v[:name] =~ /#{params[:name]}/ }
+    render :json => { :results =>
+      resource_list.sort_by { |r| r[:name] }.take(100), :subtotal => resource_list.count}
+  end
 end

data/app/graphql/mutations/job_invocations/create.rb

@@ -0,0 +1,43 @@
+module Mutations
+  module JobInvocations
+    class Create < ::Mutations::CreateMutation
+      description 'Create a new job invocation'
+      graphql_name 'CreateJobInvocationMutation'
+
+      argument :job_invocation, ::Types::JobInvocationInput, required: true
+      field :job_invocation, ::Types::JobInvocation, 'The new job invocation', null: true
+
+      def resolve(params)
+        begin
+          composer = JobInvocationComposer.from_api_params(params[:job_invocation].to_h)
+          job_invocation = composer.job_invocation
+          authorize!(job_invocation, :create)
+          errors = []
+          composer.trigger!
+        rescue ActiveRecord::RecordNotSaved
+          errors = map_all_errors(job_invocation)
+        rescue JobInvocationComposer::JobTemplateNotFound, JobInvocationComposer::FeatureNotFound => e
+          errors = [{ path => ['job_template'], :message => e.message }]
+        end
+
+        {
+          result_key => job_invocation,
+          :errors => errors,
+        }
+      end
+
+      def map_all_errors(job_invocation)
+        map_errors_to_path(job_invocation) + map_errors('triggering', job_invocation.triggering) + map_errors('targeting', job_invocation.targeting)
+      end
+
+      def map_errors(attr_name, resource)
+        resource.errors.map do |attribute, message|
+          {
+            path: [attr_name, attribute.to_s.camelize(:lower)],
+            message: message,
+          }
+        end
+      end
+    end
+  end
+end

data/app/graphql/types/job_invocation.rb

@@ -0,0 +1,16 @@
+module Types
+  class JobInvocation < BaseObject
+    description 'A Job Invocation'
+
+    global_id_field :id
+    field :job_category, String
+    field :description, String
+    field :time_span, Integer
+    field :start_at, GraphQL::Types::ISO8601DateTime
+    field :status_label, String
+
+    belongs_to :triggering, Types::Triggering
+    belongs_to :task, Types::Task
+    field :recurring_logic, Types::RecurringLogic
+  end
+end

data/app/graphql/types/job_invocation_input.rb

@@ -0,0 +1,13 @@
+module Types
+  class JobInvocationInput < BaseInputObject
+    argument :job_category, String, required: false
+    argument :job_template_id, Integer, required: false
+    argument :feature, String, required: false
+    argument :targeting_type, ::Types::TargetingEnum, required: false
+    argument :recurrence, ::Types::RecurrenceInput, required: false
+    argument :scheduling, ::Types::SchedulingInput, required: false
+    argument :search_query, String, required: false
+    argument :host_ids, [Integer], required: false
+    argument :inputs, ::Types::RawJson, required: false
+  end
+end

data/app/graphql/types/recurrence_input.rb

@@ -0,0 +1,8 @@
+module Types
+  class RecurrenceInput < BaseInputObject
+    argument :cron_line, String, required: false
+    argument :max_iteration, Integer, required: false
+    argument :end_time, String, required: false
+    argument :purpose, String, required: false
+  end
+end

data/app/graphql/types/scheduling_input.rb

@@ -0,0 +1,6 @@
+module Types
+  class SchedulingInput < BaseInputObject
+    argument :start_at, String, required: false
+    argument :start_before, String, required: false
+  end
+end

data/app/graphql/types/targeting_enum.rb

@@ -0,0 +1,7 @@
+module Types
+  class TargetingEnum < Types::BaseEnum
+    Targeting::TYPES.each_key do |key|
+      value key
+    end
+  end
+end

data/app/helpers/concerns/foreman_remote_execution/hosts_helper_extensions.rb

@@ -7,7 +7,9 @@ module ForemanRemoteExecution
     end
 
     def multiple_actions
-      super + [ [_('Schedule Remote Job'), new_job_invocation_path, false] ]
+      res = super
+      res += [ [_('Schedule Remote Job'), new_job_invocation_path, false] ] if authorized_for(controller: :job_invocations, action: :new)
+      res
     end
 
     def schedule_job_multi_button(*args)
@@ -21,12 +23,14 @@ module ForemanRemoteExecution
     end
 
     def rex_host_features(*args)
+      return unless authorized_for(controller: :job_invocations, action: :create)
       RemoteExecutionFeature.with_host_action_button.order(:label).map do |feature|
         link_to(_('%s') % feature.name, job_invocations_path(:host_ids => [args.first.id], :feature => feature.label), :method => :post)
       end
     end
 
     def schedule_job_button(*args)
+      return unless authorized_for(controller: :job_invocations, action: :new)
       link_to(_('Schedule Remote Job'), new_job_invocation_path(:host_ids => [args.first.id]), :id => :run_button, :class => 'btn btn-default')
     end
 

data/app/helpers/remote_execution_helper.rb

@@ -7,6 +7,10 @@ module RemoteExecutionHelper
     @job_hosts_authorizer ||= Authorizer.new(User.current, :collection => @hosts)
   end
 
+  def host_tasks_authorizer
+    @host_tasks_authorizer ||= Authorizer.new(User.current, :collection => @job_invocation.sub_tasks)
+  end
+
   def host_counter(label, count)
     content_tag(:div, :class => 'host_counter') do
       content_tag(:div, label, :class => 'header') + content_tag(:div, count.to_s, :class => 'count')
@@ -36,7 +40,7 @@ module RemoteExecutionHelper
                   'data-method': 'get', id: "#{host.name}-actions-rerun" } }
     end
 
-    if host_task.present? && authorized_for(hash_for_foreman_tasks_task_path(host_task).merge(auth_object: host_task, permission: :view_foreman_tasks))
+    if host_task.present? && authorized_for(hash_for_foreman_tasks_task_path(host_task).merge(auth_object: host_task, permission: :view_foreman_tasks, authorizer: host_tasks_authorizer))
       links << { title: _('Host task'),
         action: { href: foreman_tasks_task_path(host_task),
                   'data-method': 'get', id: "#{host.name}-actions-task" } }
@@ -109,12 +113,14 @@ module RemoteExecutionHelper
         :class => 'btn btn-danger',
         :title => _('Try to cancel the job on a host'),
         :disabled => !task.cancellable?,
-        :method => :post)
+        :method => :post,
+        :remote => true)
       buttons << link_to(_('Abort Job'), abort_foreman_tasks_task_path(task),
         :class => 'btn btn-danger',
         :title => _('Try to abort the job on a host without waiting for its result'),
         :disabled => !task.cancellable?,
-        :method => :post)
+        :method => :post,
+        :remote => true)
     end
     buttons
   end

data/app/lib/actions/remote_execution/run_host_job.rb

@@ -47,12 +47,17 @@ module Actions
         script = renderer.render
         raise _('Failed rendering template: %s') % renderer.error_message unless script
 
+        first_execution = host.executed_through_proxies.where(:id => proxy.id).none?
+        host.executed_through_proxies << proxy if first_execution
+
         additional_options = { :hostname => provider.find_ip_or_hostname(host),
                                :script => script,
                                :execution_timeout_interval => job_invocation.execution_timeout_interval,
                                :secrets => secrets(host, job_invocation, provider),
                                :use_batch_triggering => true,
-                               :use_concurrency_control => options[:use_concurrency_control]}
+                               :use_concurrency_control => options[:use_concurrency_control],
+                               :first_execution => first_execution,
+                               :alternative_names => provider.alternative_names(host) }
         action_options = provider.proxy_command_options(template_invocation, host)
                                  .merge(additional_options)
 
@@ -191,6 +196,10 @@ module Actions
       def verify_permissions(host, template_invocation)
         raise _('User can not execute job on host %s') % host.name unless User.current.can?(:view_hosts, host)
         raise _('User can not execute this job template') unless User.current.can?(:view_job_templates, template_invocation.template)
+        infra_facet = host.infrastructure_facet
+        if (infra_facet&.foreman_instance || infra_facet&.smart_proxy_id) && !User.current.can?(:execute_jobs_on_infrastructure_hosts)
+          raise _('User can not execute job on infrastructure host %s') % host.name
+        end
 
         # we don't want to load all template_invocations to verify so we construct Authorizer object manually and set
         # the base collection to current template

data/app/lib/actions/remote_execution/run_hosts_job.rb

@@ -9,7 +9,9 @@ module Actions
       middleware.use Actions::Middleware::BindJobInvocation
       middleware.use Actions::Middleware::RecurringLogic
       middleware.use Actions::Middleware::WatchDelegatedProxySubTasks
-      middleware.use Actions::Middleware::ProxyBatchTriggering
+
+      execution_plan_hooks.use :notify_on_success, :on => :success
+      execution_plan_hooks.use :notify_on_failure, :on => :failure
 
       class CheckOnProxyActions; end
 
@@ -32,6 +34,10 @@ module Actions
         set_up_concurrency_control job_invocation
         input.update(:job_category => job_invocation.job_category)
         plan_self(:job_invocation_id => job_invocation.id)
+        provider = job_invocation.pattern_template_invocations.first&.template&.provider
+        input[:proxy_batch_size] ||= provider&.proxy_batch_size || Setting['foreman_tasks_proxy_batch_size']
+        trigger_action = plan_action(Actions::TriggerProxyBatch, batch_size: proxy_batch_size, total_count: hosts.count)
+        input[:trigger_run_step_id] = trigger_action.run_step_id
       end
 
       def create_sub_plans
@@ -46,14 +52,47 @@ module Actions
         end
       end
 
+      def spawn_plans
+        super
+      ensure
+        trigger_remote_batch
+      end
+
+      def trigger_remote_batch
+        batches_ready = (output[:planned_count] - output[:remote_triggered_count]) / proxy_batch_size
+        return unless batches_ready > 0
+
+        plan_event(Actions::TriggerProxyBatch::TriggerNextBatch[batches_ready], nil, step_id: input[:trigger_run_step_id])
+        output[:remote_triggered_count] += proxy_batch_size * batches_ready
+      end
+
+      def on_planning_finished
+        plan_event(Actions::TriggerProxyBatch::TriggerLastBatch, nil, step_id: input[:trigger_run_step_id])
+        super
+      end
+
       def finalize
         job_invocation.password = job_invocation.key_passphrase = job_invocation.effective_user_password = nil
         job_invocation.save!
 
         Rails.logger.debug "cleaning cache for keys that begin with 'job_invocation_#{job_invocation.id}'"
         Rails.cache.delete_matched(/\A#{JobInvocation::CACHE_PREFIX}_#{job_invocation.id}/)
-        # creating the success notification should be the very last thing this tasks do
+      end
+
+      def notify_on_success(_plan)
         job_invocation.build_notification.deliver!
+
+        if [RexMailNotification::SUCCEEDED_JOBS, RexMailNotification::ALL_JOBS].include?(mail_notification_preference&.interval)
+          RexJobMailer.job_finished(job_invocation, subject: _("REX job has succeeded - %s") % job_invocation.to_s).deliver_now
+        end
+      end
+
+      def notify_on_failure(_plan)
+        job_invocation.build_notification.deliver!
+
+        if [RexMailNotification::FAILED_JOBS, RexMailNotification::ALL_JOBS].include?(mail_notification_preference&.interval)
+          RexJobMailer.job_finished(job_invocation, subject: _("REX job has failed - %s") % job_invocation.to_s).deliver_now
+        end
       end
 
       def job_invocation
@@ -67,12 +106,13 @@ module Actions
 
       def initiate
         output[:host_count] = total_count
+        output[:remote_triggered_count] = 0
         super
       end
 
       def total_count
         # For compatibility with already existing tasks
-        return output[:total_count] unless output.has_key?(:host_count) || task.pending?
+        return output[:total_count] || hosts.count unless output.has_key?(:host_count) || task.pending?
 
         output[:host_count] || hosts.count
       end
@@ -94,7 +134,11 @@ module Actions
       end
 
       def run(event = nil)
-        super unless event == Dynflow::Action::Skip
+        if event == Dynflow::Action::Skip
+          plan_event(Dynflow::Action::Skip, nil, step_id: input[:trigger_run_step_id])
+        else
+          super
+        end
       end
 
       def humanized_input
@@ -104,6 +148,16 @@ module Actions
       def humanized_name
         '%s:' % _(super)
       end
+
+      def proxy_batch_size
+        input[:proxy_batch_size]
+      end
+
+      private
+
+      def mail_notification_preference
+        UserMailNotification.where(mail_notification_id: RexMailNotification.first, user_id: User.current.id).first
+      end
     end
   end
 end

data/app/mailers/rex_job_mailer.rb

@@ -0,0 +1,15 @@
+class RexJobMailer < ApplicationMailer
+  add_template_helper(ApplicationHelper)
+
+  def job_finished(job, opts = {})
+    @job = job
+    @subject = opts[:subject] || _('REX job has finished - %s') % @job.to_s
+
+    if @job.user.nil?
+      Rails.logger.warn 'Job user no longer exist, skipping email notification'
+      return
+    end
+
+    mail(to: @job.user.mail, subject: @subject)
+  end
+end

data/app/models/concerns/foreman_remote_execution/host_extensions.rb

@@ -6,6 +6,8 @@ module ForemanRemoteExecution
         has_many :template_invocations, :dependent => :destroy, :foreign_key => 'host_id'
         has_one :execution_status_object, :class_name => 'HostStatus::ExecutionStatus', :foreign_key => 'host_id', :dependent => :destroy
         has_many :run_host_job_tasks, :through => :template_invocations
+        has_many :host_proxy_invocations, :foreign_key => 'host_id', :dependent => :destroy
+        has_many :executed_through_proxies, :through => :host_proxy_invocations, :source => 'smart_proxy'
 
         scoped_search :relation => :run_host_job_tasks, :on => :result, :rename => 'job_invocation.result',
           :ext_method => :search_by_job_invocation,
@@ -18,6 +20,14 @@ module ForemanRemoteExecution
         scoped_search :relation => :execution_status_object, :on => :status, :rename => :execution_status,
           :complete_value => { :ok => HostStatus::ExecutionStatus::OK, :error => HostStatus::ExecutionStatus::ERROR }
 
+        scope :execution_scope, lambda {
+          if User.current&.can?('execute_jobs_on_infrastructure_hosts')
+            self
+          else
+            search_for('not (infrastructure_facet.foreman = true or set? infrastructure_facet.smart_proxy_id)')
+          end
+        }
+
         def search_by_job_invocation(key, operator, value)
           if key == 'job_invocation.result'
             operator = operator == '=' ? 'IN' : 'NOT IN'

data/app/models/concerns/foreman_remote_execution/smart_proxy_extensions.rb

@@ -1,5 +1,11 @@
 module ForemanRemoteExecution
   module SmartProxyExtensions
+    def self.prepended(base)
+      base.instance_eval do
+        has_many :host_proxy_invocations, :dependent => :destroy
+      end
+    end
+
     def pubkey
       self[:pubkey] || update_pubkey
     end

data/app/models/host_proxy_invocation.rb

@@ -0,0 +1,4 @@
+class HostProxyInvocation < ApplicationRecord
+  belongs_to :host, :class_name => 'Host::Managed', :inverse_of => :host_proxy_invocations
+  belongs_to :smart_proxy
+end

data/app/models/host_status/execution_status.rb

@@ -64,11 +64,11 @@ class HostStatus::ExecutionStatus < HostStatus::Status
 
       case status
         when OK
-          [ "foreman_tasks_tasks.state = 'stopped' AND result = 'success'" ]
+          [ "foreman_tasks_tasks.state = 'stopped' AND foreman_tasks_tasks.result = 'success'" ]
         when CANCELLED
-          [ "foreman_tasks_tasks.state = 'stopped' AND result = 'cancelled'" ]
+          [ "foreman_tasks_tasks.state = 'stopped' AND foreman_tasks_tasks.result = 'cancelled'" ]
         when ERROR
-          [ "foreman_tasks_tasks.state = 'stopped' AND (result = 'error' OR result = 'warning')" ]
+          [ "foreman_tasks_tasks.state = 'stopped' AND (foreman_tasks_tasks.result = 'error' OR foreman_tasks_tasks.result = 'warning')" ]
         when QUEUED
           [ "foreman_tasks_tasks.state = 'scheduled' OR foreman_tasks_tasks.state IS NULL" ]
         when RUNNING

data/app/models/job_invocation.rb

@@ -65,7 +65,7 @@ class JobInvocation < ApplicationRecord
   scoped_search :on => 'pattern_template_name', :rename => 'pattern_template_name', :operators => ['= '],
     :complete_value => false, :only_explicit => true, :ext_method => :search_by_pattern_template
 
-  scope :with_task, -> { references(:task) }
+  scoped_search :relation => :recurring_logic, :on => 'purpose', :rename => 'recurring_logic.purpose'
 
   scoped_search :relation => :recurring_logic, :on => 'id', :rename => 'recurring_logic.id'
 
@@ -75,6 +75,8 @@ class JobInvocation < ApplicationRecord
 
   default_scope -> { order('job_invocations.id DESC') }
 
+  scope :latest_jobs, -> { unscoped.joins(:task).order('foreman_tasks_tasks.start_at DESC').authorized(:view_job_invocations).limit(5) }
+
   validates_lengths_from_database :only => [:description]
 
   attr_accessor :start_before, :description_format
@@ -99,7 +101,7 @@ class JobInvocation < ApplicationRecord
   def self.search_by_status(key, operator, value)
     conditions = HostStatus::ExecutionStatus::ExecutionTaskStatusMapper.sql_conditions_for(value)
     conditions[0] = "NOT (#{conditions[0]})" if operator == '<>'
-    { :conditions => sanitize_sql_for_conditions(conditions), :include => :task }
+    { :conditions => sanitize_sql_for_conditions(conditions), :joins => :task }
   end
 
   def self.search_by_recurring_logic(key, operator, value)
@@ -193,11 +195,16 @@ class JobInvocation < ApplicationRecord
   end
 
   def total_hosts_count
+    count = _('N/A')
+
     if targeting.resolved?
-      task&.main_action&.total_count || targeting.hosts.count
-    else
-      _('N/A')
+      count = if task&.main_action.respond_to?(:total_count)
+                task.main_action.total_count
+              else
+                targeting.hosts.count
+              end
     end
+    count
   end
 
   def pattern_template_invocation_for_host(host)