foreman_remote_execution 4.5.5 → 4.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99202dff05c36d40eac36950a23298ca1cf2ca4510e2506787bace85c72f0fd9
-  data.tar.gz: '059cd0dd0b427bb2fc1a80c59a47ec26e2fedb4d0e27f989c1ec859e52e1191a'
+  metadata.gz: c7565defbf0881174acb2a48e274aacb3d790d2ee8b9de3119307c2afabd7355
+  data.tar.gz: acffdb2df04d0c1caeff65513e1a63e5cf66276897206affd606c5136b7afba8
 SHA512:
-  metadata.gz: 19e978232f3ee68a30d65ceb7c1b799b180894c16f5060c80ed4c5b85870fbe1f0f7ed320b44924cbb7177e280deb32309a061ab2737ff9bc2cc944092a48b63
-  data.tar.gz: 9ac86095eb5c6cdf2bed666e911895462e44b0cbf96ddf40023c062604f0de947ebf4851e48d556332ed6624a59d6748608bdb472ce63c3e593ebcc3e5a21d52
+  metadata.gz: 7064c4e3984fd4ef3ef463fc10cb27e911d1f24ac0b74ccf5fb3b7682ba3d4c3418e130f5b6e0f1920ed83406dbf0787585ef1e1b83cfd0c0e412219c059309c
+  data.tar.gz: ab22827c2750d67893621267a43095c2b845937b6d72d5a665bf800ab97542a06cda1480187174943077c9365caac0076d67f765b6c43c4e01650dca1f7b9010

data/.github/workflows/ruby_ci.yml

@@ -71,3 +71,10 @@ jobs:
         run: |
           bundle exec rake test:foreman_remote_execution
           bundle exec rake test TEST="test/unit/foreman/access_permissions_test.rb"
+      - name: 'Upload logs'
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: log/*.log
+          retention-days: 5

data/.rubocop_todo.yml

@@ -220,6 +220,7 @@ Naming/FileName:
     - 'db/seeds.d/60-ssh_proxy_feature.rb'
     - 'db/seeds.d/70-job_templates.rb'
     - 'db/seeds.d/90-bookmarks.rb'
+    - 'db/seeds.d/95-mail_notifications.rb'
 
 # Offense count: 1
 # Configuration parameters: ForbiddenDelimiters.

data/app/controllers/api/v2/job_invocations_controller.rb

@@ -41,12 +41,18 @@ module Api
             param :effective_user, String,
               :required => false,
               :desc => N_('What user should be used to run the script (using sudo-like mechanisms). Defaults to a template parameter or global setting.')
+            param :effective_user_password, String,
+              :required => false,
+              :desc => N_('Set password for effective user (using sudo-like mechanisms)')
           end
+          param :password, String, :required => false, :desc => N_('Set SSH password')
+          param :key_passphrase, String, :required => false, :desc => N_('Set SSH key passphrase')
 
           param :recurrence, Hash, :desc => N_('Create a recurring job') do
             param :cron_line, String, :required => false, :desc => N_('How often the job should occur, in the cron format')
             param :max_iteration, :number, :required => false, :desc => N_('Repeat a maximum of N times')
             param :end_time, DateTime, :required => false, :desc => N_('Perform no more executions after this time')
+            param :purpose, String, :required => false, :desc => N_('Designation of a special purpose')
           end
 
           param :scheduling, Hash, :desc => N_('Schedule the job to start at a later time') do
@@ -197,7 +203,7 @@ module Api
         end
 
         if job_invocation_params.key?(:ssh)
-          job_invocation_params.merge!(job_invocation_params.delete(:ssh).permit(:effective_user))
+          job_invocation_params.merge!(job_invocation_params.delete(:ssh).permit(:effective_user, :effective_user_password))
         end
 
         job_invocation_params[:inputs] ||= {}

data/app/graphql/types/job_invocation.rb

@@ -0,0 +1,16 @@
+module Types
+  class JobInvocation < BaseObject
+    description 'A Job Invocation'
+
+    global_id_field :id
+    field :job_category, String
+    field :description, String
+    field :time_span, Integer
+    field :start_at, GraphQL::Types::ISO8601DateTime
+    field :status_label, String
+
+    belongs_to :triggering, Types::Triggering
+    belongs_to :task, Types::Task
+    field :recurring_logic, Types::RecurringLogic
+  end
+end

data/app/lib/actions/remote_execution/run_host_job.rb

@@ -56,7 +56,8 @@ module Actions
                                :secrets => secrets(host, job_invocation, provider),
                                :use_batch_triggering => true,
                                :use_concurrency_control => options[:use_concurrency_control],
-                               :first_execution => first_execution }
+                               :first_execution => first_execution,
+                               :alternative_names => provider.alternative_names(host) }
         action_options = provider.proxy_command_options(template_invocation, host)
                                  .merge(additional_options)
 

data/app/lib/actions/remote_execution/run_hosts_job.rb

@@ -9,7 +9,9 @@ module Actions
       middleware.use Actions::Middleware::BindJobInvocation
       middleware.use Actions::Middleware::RecurringLogic
       middleware.use Actions::Middleware::WatchDelegatedProxySubTasks
-      middleware.use Actions::Middleware::ProxyBatchTriggering
+
+      execution_plan_hooks.use :notify_on_success, :on => :success
+      execution_plan_hooks.use :notify_on_failure, :on => :failure
 
       class CheckOnProxyActions; end
 
@@ -32,6 +34,10 @@ module Actions
         set_up_concurrency_control job_invocation
         input.update(:job_category => job_invocation.job_category)
         plan_self(:job_invocation_id => job_invocation.id)
+        provider = job_invocation.pattern_template_invocations.first&.template&.provider
+        input[:proxy_batch_size] ||= provider&.proxy_batch_size || Setting['foreman_tasks_proxy_batch_size']
+        trigger_action = plan_action(Actions::TriggerProxyBatch, batch_size: proxy_batch_size, total_count: hosts.count)
+        input[:trigger_run_step_id] = trigger_action.run_step_id
       end
 
       def create_sub_plans
@@ -46,14 +52,47 @@ module Actions
         end
       end
 
+      def spawn_plans
+        super
+      ensure
+        trigger_remote_batch
+      end
+
+      def trigger_remote_batch
+        batches_ready = (output[:planned_count] - output[:remote_triggered_count]) / proxy_batch_size
+        return unless batches_ready > 0
+
+        plan_event(Actions::TriggerProxyBatch::TriggerNextBatch[batches_ready], nil, step_id: input[:trigger_run_step_id])
+        output[:remote_triggered_count] += proxy_batch_size * batches_ready
+      end
+
+      def on_planning_finished
+        plan_event(Actions::TriggerProxyBatch::TriggerLastBatch, nil, step_id: input[:trigger_run_step_id])
+        super
+      end
+
       def finalize
         job_invocation.password = job_invocation.key_passphrase = job_invocation.effective_user_password = nil
         job_invocation.save!
 
         Rails.logger.debug "cleaning cache for keys that begin with 'job_invocation_#{job_invocation.id}'"
         Rails.cache.delete_matched(/\A#{JobInvocation::CACHE_PREFIX}_#{job_invocation.id}/)
-        # creating the success notification should be the very last thing this tasks do
+      end
+
+      def notify_on_success(_plan)
         job_invocation.build_notification.deliver!
+
+        if [RexMailNotification::SUCCEEDED_JOBS, RexMailNotification::ALL_JOBS].include?(mail_notification_preference&.interval)
+          RexJobMailer.job_finished(job_invocation, subject: _("REX job has succeeded - %s") % job_invocation.to_s).deliver_now
+        end
+      end
+
+      def notify_on_failure(_plan)
+        job_invocation.build_notification.deliver!
+
+        if [RexMailNotification::FAILED_JOBS, RexMailNotification::ALL_JOBS].include?(mail_notification_preference&.interval)
+          RexJobMailer.job_finished(job_invocation, subject: _("REX job has failed - %s") % job_invocation.to_s).deliver_now
+        end
       end
 
       def job_invocation
@@ -67,6 +106,7 @@ module Actions
 
       def initiate
         output[:host_count] = total_count
+        output[:remote_triggered_count] = 0
         super
       end
 
@@ -94,7 +134,11 @@ module Actions
       end
 
       def run(event = nil)
-        super unless event == Dynflow::Action::Skip
+        if event == Dynflow::Action::Skip
+          plan_event(Dynflow::Action::Skip, nil, step_id: input[:trigger_run_step_id])
+        else
+          super
+        end
       end
 
       def humanized_input
@@ -104,6 +148,16 @@ module Actions
       def humanized_name
         '%s:' % _(super)
       end
+
+      def proxy_batch_size
+        input[:proxy_batch_size]
+      end
+
+      private
+
+      def mail_notification_preference
+        UserMailNotification.where(mail_notification_id: RexMailNotification.first, user_id: User.current.id).first
+      end
     end
   end
 end

data/app/mailers/rex_job_mailer.rb

@@ -0,0 +1,15 @@
+class RexJobMailer < ApplicationMailer
+  add_template_helper(ApplicationHelper)
+
+  def job_finished(job, opts = {})
+    @job = job
+    @subject = opts[:subject] || _('REX job has finished - %s') % @job.to_s
+
+    if @job.user.nil?
+      Rails.logger.warn 'Job user no longer exist, skipping email notification'
+      return
+    end
+
+    mail(to: @job.user.mail, subject: @subject)
+  end
+end

data/app/models/job_invocation.rb

@@ -65,6 +65,8 @@ class JobInvocation < ApplicationRecord
   scoped_search :on => 'pattern_template_name', :rename => 'pattern_template_name', :operators => ['= '],
     :complete_value => false, :only_explicit => true, :ext_method => :search_by_pattern_template
 
+  scoped_search :relation => :recurring_logic, :on => 'purpose', :rename => 'recurring_logic.purpose'
+
   scoped_search :relation => :recurring_logic, :on => 'id', :rename => 'recurring_logic.id'
 
   scoped_search :relation => :recurring_logic, :on => 'id', :rename => 'recurring',
@@ -73,6 +75,8 @@ class JobInvocation < ApplicationRecord
 
   default_scope -> { order('job_invocations.id DESC') }
 
+  scope :latest_jobs, -> { unscoped.joins(:task).order('foreman_tasks_tasks.start_at DESC').authorized(:view_job_invocations).limit(5) }
+
   validates_lengths_from_database :only => [:description]
 
   attr_accessor :start_before, :description_format

data/app/models/job_invocation_composer.rb

@@ -121,7 +121,10 @@ class JobInvocationComposer
         :targeting => targeting_params,
         :triggering => triggering_params,
         :description_format => api_params[:description_format],
+        :password => api_params[:password],
         :remote_execution_feature_id => remote_execution_feature_id,
+        :effective_user_password => api_params[:effective_user_password],
+        :key_passphrase => api_params[:key_passphrase],
         :concurrency_control => concurrency_control_params,
         :execution_timeout_interval => api_params[:execution_timeout_interval] || template.execution_timeout_interval,
         :template_invocations => template_invocations_params }.with_indifferent_access
@@ -138,17 +141,11 @@ class JobInvocationComposer
     end
 
     def triggering_params
-      raise ::Foreman::Exception, _('Cannot specify both recurrence and scheduling') if api_params[:recurrence].present? && api_params[:scheduling].present?
-
-      if api_params[:recurrence].present?
-        {
-          :mode => :recurring,
-          :cronline => api_params[:recurrence][:cron_line],
-          :end_time => format_datetime(api_params[:recurrence][:end_time]),
-          :input_type => :cronline,
-          :max_iteration => api_params[:recurrence][:max_iteration],
-        }
-      elsif api_params[:scheduling].present?
+      if api_params[:recurrence].present? && api_params[:scheduling].present?
+        recurring_mode_params.merge :start_at_raw => format_datetime(api_params[:scheduling][:start_at])
+      elsif api_params[:recurrence].present? && api_params[:scheduling].empty?
+        recurring_mode_params
+      elsif api_params[:recurrence].empty? && api_params[:scheduling].present?
         {
           :mode => :future,
           :start_at_raw => format_datetime(api_params[:scheduling][:start_at]),
@@ -160,6 +157,17 @@ class JobInvocationComposer
       end
     end
 
+    def recurring_mode_params
+      {
+        :mode => :recurring,
+        :cronline => api_params[:recurrence][:cron_line],
+        :end_time => format_datetime(api_params[:recurrence][:end_time]),
+        :input_type => :cronline,
+        :max_iteration => api_params[:recurrence][:max_iteration],
+        :purpose => api_params[:recurrence][:purpose],
+      }
+    end
+
     def concurrency_control_params
       {
         :level => api_params.fetch(:concurrency_control, {})[:concurrency_level],
@@ -513,7 +521,7 @@ class JobInvocationComposer
   end
 
   def available_bookmarks
-    Bookmark.authorized(:view_bookmarks).my_bookmarks.where(:controller => ['hosts', 'dashboard'])
+    Bookmark.my_bookmarks.where(:controller => ['hosts', 'dashboard'])
   end
 
   def targeted_hosts
@@ -636,7 +644,7 @@ class JobInvocationComposer
     setting_value = Setting['remote_execution_form_job_template']
     return default_value unless setting_value
 
-    form_template = JobTemplate.find_by :name => setting_value
+    form_template = JobTemplate.authorized(:view_job_templates).find_by :name => setting_value
     return default_value unless form_template
 
     if block_given?

data/app/models/job_template.rb

@@ -83,7 +83,7 @@ class JobTemplate < ::Template
     end
 
     def acceptable_template_input_types
-      [ :user, :fact, :variable, :puppet_parameter ]
+      [ :user, :fact, :variable ]
     end
 
     def default_render_scope_class

data/app/models/remote_execution_provider.rb

@@ -80,7 +80,14 @@ class RemoteExecutionProvider
 
     def find_ip(host, interfaces)
       if host_setting(host, :remote_execution_connect_by_ip)
-        interfaces.find { |i| i.ip.present? }.try(:ip)
+        ip4_address = interfaces.find { |i| i.ip.present? }.try(:ip)
+        ip6_address = interfaces.find { |i| i.ip6.present? }.try(:ip6)
+
+        if host_setting(host, :remote_execution_connect_by_ip_prefer_ipv6)
+          ip6_address || ip4_address
+        else
+          ip4_address || ip6_address
+        end
       end
     end
 
@@ -112,7 +119,11 @@ class RemoteExecutionProvider
     end
 
     def proxy_action_class
-      'ForemanRemoteExecutionCore::Actions::RunScript'
+      'Proxy::RemoteExecution::Ssh::Actions::RunScript'
+    end
+
+    def proxy_batch_size
+      Setting['foreman_tasks_proxy_batch_size']
     end
 
     # Return a specific proxy selector to use for running a given template
@@ -124,5 +135,9 @@ class RemoteExecutionProvider
         ::DefaultProxyProxySelector.new
       end
     end
+
+    def alternative_names(host)
+      { :fqdn => find_fqdn(effective_interfaces(host)) }
+    end
   end
 end

data/app/models/rex_mail_notification.rb

@@ -0,0 +1,13 @@
+class RexMailNotification < MailNotification
+  FAILED_JOBS = N_("Subscribe to my failed jobs")
+  SUCCEEDED_JOBS = N_("Subscribe to my succeeded jobs")
+  ALL_JOBS = N_("Subscribe to all my jobs")
+
+  def subscription_options
+    [
+      FAILED_JOBS,
+      SUCCEEDED_JOBS,
+      ALL_JOBS,
+    ]
+  end
+end

data/app/models/setting/remote_execution.rb

@@ -39,9 +39,15 @@ class Setting::RemoteExecution < Setting
       self.set('remote_execution_connect_by_ip',
         N_('Should the ip addresses on host interfaces be preferred over the fqdn? '\
         'It is useful when DNS not resolving the fqdns properly. You may override this per host by setting a parameter called remote_execution_connect_by_ip. '\
-        'This setting only applies to IPv4. When the host has only an IPv6 address on the interface used for remote execution, hostname will be used even if this setting is set to true.'),
+        'For dual-stacked hosts you should consider the remote_execution_connect_by_ip_prefer_ipv6 setting'),
         false,
         N_('Connect by IP')),
+      self.set('remote_execution_connect_by_ip_prefer_ipv6',
+        N_('When connecting using ip address, should the IPv6 addresses be preferred? '\
+        'If no IPv6 address is set, it falls back to IPv4 automatically. You may override this per host by setting a parameter called remote_execution_connect_by_ip_prefer_ipv6. '\
+        'By default and for compatibility, IPv4 will be preferred over IPv6 by default'),
+        false,
+        N_('Prefer IPv6 over IPv4')),
       self.set('remote_execution_ssh_password',
         N_('Default password to use for SSH. You may override per host by setting a parameter called remote_execution_ssh_password'),
         nil,

data/app/services/ui_notifications/remote_execution_jobs/base_job_finish.rb

@@ -20,7 +20,8 @@ module UINotifications
       end
 
       def blueprint
-        @blueprint ||= NotificationBlueprint.unscoped.find_by(:name => 'rex_job_succeeded')
+        blueprint = @subject.status == HostStatus::ExecutionStatus::ERROR ? 'rex_job_failed' : 'rex_job_succeeded'
+        @blueprint ||= NotificationBlueprint.unscoped.find_by(:name => blueprint)
       end
 
       def message

data/app/views/dashboard/_latest-jobs.html.erb

@@ -0,0 +1,21 @@
+<h4 class="header">
+  <%= link_to _("Latest Jobs"), job_invocations_path(:order=>'start_at DESC') %>
+</h4>
+<% if JobInvocation.latest_jobs.any? %>
+  <table class="<%= table_css_classes('table-fixed') %>">
+    <tr>
+      <th class="col-md-5"><%= _("Name") %></th>
+      <th class="col-md-2"><%= _("State") %></th>
+      <th class="col-md-3"><%= _("Started") %></th>
+    </tr>
+    <% JobInvocation.latest_jobs.each do |invocation| %>
+      <tr>
+        <td class="ellipsis"><%= link_to_if_authorized invocation_description(invocation), hash_for_job_invocation_path(invocation).merge(:auth_object => invocation, :permission => :view_job_invocations, :authorizer => authorizer) %></td>
+        <td><%= link_to_invocation_task_if_authorized(invocation) %></td>
+        <td><%= time_in_words_span(invocation.start_at) %></td>
+      </tr>
+    <% end %>
+  </table>
+<% else %>
+  <p class="ca"><%= _("No jobs available") %></p>
+<% end %>

data/app/views/rex_job_mailer/job_finished.html.erb

@@ -0,0 +1,24 @@
+<p>
+  <%= _("A job '%{job_name}' has %{status} at %{time}") % {
+      :job_name => @job.to_s, :status => @job.status_label, :time => date_time_absolute_value(@job.task.ended_at)
+  } %>
+</p>
+
+<div class="dashboard">
+  <table>
+    <tr>
+      <td width="18%" class="hosts-rows"><b><%= _("Job result") %></b></td>
+      <td width="82%" class="hosts-rows"><%= @job.status_label %></td>
+    </tr>
+    <tr>
+      <td width="18%" class="hosts-rows"><b><%= _("Total hosts") %></b></td>
+      <td width="82%" class="hosts-rows"><%= @job.total_hosts_count %></td>
+    </tr>
+    <tr>
+      <td width="18%" class="hosts-rows"><b><%= _("Failed hosts") %></b></td>
+      <td width="82%" class="hosts-rows"><%= @job.failed_hosts.count %></td>
+    </tr>
+  </table>
+</div>
+
+<%= link_to 'More details', job_invocation_url(@job) %>

data/app/views/rex_job_mailer/job_finished.text.erb

@@ -0,0 +1,9 @@
+<%= _("A job '%{job_name}' has %{status} at %{time}") % {
+    :job_name => @job.to_s, :status => @job.status_label, :time => date_time_absolute_value(@job.task.ended_at)
+} %>
+
+<%= _('Job result') %>: <%= @job.status_label %>
+<%= _('Total hosts') %>: <%= @job.total_hosts_count %>
+<%= _('Failed hosts') %>: <%= @job.failed_hosts.count %>
+
+<%= _('See more details at %s') % job_invocation_url(@job) %>