foreman-tasks 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fae15bedc8d294cc8486d2d99b0843d53840d1e6
-  data.tar.gz: af614f0e462ca233cc5615f2f47793caa184f380
+  metadata.gz: ffd1f0c3a1276172b47fc40f58b34d92641fe2b1
+  data.tar.gz: e9faaecd202c22b32c13de8de2ba8bea5ddaee49
 SHA512:
-  metadata.gz: bd813948f1ca5b2305e06dd090e631bd1c598f8b83e52a37ae7e50071e9cfd31a73ae52cec3ed0822033089ead74109d222adc35b718d4e29021536264ef2cce
-  data.tar.gz: 9ce2326641602979ada0e7c56dc957923eb579791ceee0c74b0e06346d2007a792472e5cc9d216ec59f65576427adcfaab276b3023995e045410016554dbcf39
+  metadata.gz: 5f057fff909d5f4cfc7a107f91dcba2654618dca2615fa23b819e0cbe6867f6ad7cdaae9b4de8bbbde832c9783d8c0b1d729151a9a7dacefdb42d76c15f5e597
+  data.tar.gz: 3f306778e7796e788f5f251bd2c4be7ef31ae94094a6b29d9ab7e1972650eae9c0c9c85590ae10d173be2b46101cb2c7d77da43cdaed09de038db478246f00b0

data/Gemfile

@@ -3,7 +3,8 @@ source "https://rubygems.org"
 # Declare your gem's dependencies in tasks.gemspec.
 # Bundler will treat runtime dependencies like base dependencies, and
 # development dependencies will be added by default to the :development group.
-gemspec
+
+gemspec :name => 'foreman-tasks'
 
 # Declare any dependencies that are still in development here instead of in
 # your gemspec. These might include edge Rails or gems from your path or

data/app/helpers/foreman_tasks/foreman_tasks_helper.rb

@@ -116,7 +116,7 @@ module ForemanTasks
       tags = []
       tags << text_f(f, :start_at_raw, :label => _('Start at'), :placeholder => 'YYYY-mm-dd HH:MM')
       tags << text_f(f, :start_before_raw, :label => _('Start before'), :placeholder => 'YYYY-mm-dd HH:MM',
-                     :label_help => _('Indicates that the action should be cancelled if it cannot be started before this time.'))
+                                           :label_help => _('Indicates that the action should be cancelled if it cannot be started before this time.'))
       content_tag(:fieldset, nil, :id => 'trigger_mode_future', :class => "trigger_mode_form #{'hidden' unless triggering.future?}") do
         tags.join.html_safe
       end

data/app/lib/actions/bulk_action.rb

@@ -1,6 +1,5 @@
 module Actions
   class BulkAction < Actions::ActionWithSubPlans
-
     include Dynflow::Action::WithBulkSubPlans
 
     # == Parameters:

data/app/lib/actions/middleware/keep_current_user.rb

@@ -47,7 +47,7 @@ module Actions
       end
 
       def restore_curent_user
-        User.current = User.find(action.input[:current_user_id])
+        User.current = User.unscoped.find(action.input[:current_user_id])
         yield
       ensure
         User.current = nil

data/app/models/foreman_tasks/concerns/action_triggering.rb

@@ -108,13 +108,13 @@ module ForemanTasks
         @_dynflow_task_wrapped = true
 
         @_dynflow_hook_action = case method
-                 when :save
-                   new_record? ? create_action : update_action
-                 when :destroy
-                   destroy_action
-                 else
-                   raise 'unexpected method'
-                 end
+                                when :save
+                                  new_record? ? create_action : update_action
+                                when :destroy
+                                  destroy_action
+                                else
+                                  raise 'unexpected method'
+                                end
         ensure_not_in_transaction! if @_dynflow_hook_action
         yield.tap do |result|
           execute_planned_action if result

data/app/models/foreman_tasks/task.rb

@@ -38,10 +38,10 @@ module ForemanTasks
     scoped_search :on => :parent_task_id, :complete_value => true
     scoped_search :relation => :locks,  :on => :resource_type, :complete_value => true, :rename => 'resource_type', :ext_method => :search_by_generic_resource
     scoped_search :relation => :locks,  :on => :resource_id, :complete_value => false, :rename => 'resource_id', :ext_method => :search_by_generic_resource
-    scoped_search :relation => :owners,  
-                  :on => :id, 
-                  :complete_value => true, 
-                  :rename => 'owner.id', 
+    scoped_search :relation => :owners,
+                  :on => :id,
+                  :complete_value => true,
+                  :rename => 'owner.id',
                   :ext_method => :search_by_owner,
                   :validator => ->(value) { ScopedSearch::Validators::INTEGER.call(value) || value == 'current_user' }
     scoped_search :relation => :owners,  :on => :login, :complete_value => true, :rename => 'owner.login', :ext_method => :search_by_owner
@@ -172,7 +172,9 @@ module ForemanTasks
     end
 
     def sub_tasks_counts
-      result = %w(cancelled error pending success warning).zip([0].cycle).to_h
+      result = %w(cancelled error pending success warning).inject({}) do |hash, state|
+        hash.update(state => 0)
+      end
       result.update sub_tasks.group(:result).count
       sum = result.values.reduce(:+)
       if respond_to?(:main_action) && main_action.respond_to?(:total_count)

data/app/models/foreman_tasks/task/dynflow_task.rb

@@ -2,7 +2,6 @@ module ForemanTasks
   class Task::DynflowTask < ForemanTasks::Task
     include Algebrick::TypeCheck
 
-    delegate :cancellable?, :progress, to: :execution_plan
     scope :for_action, ->(action_class) { where(label: action_class.name) }
 
     def update_from_dynflow(data)
@@ -19,26 +18,50 @@ module ForemanTasks
                                   DynflowTask.where(:external_id => main_action.caller_execution_plan_id).first!.id
                                 end
                               end
-      self.label          ||= main_action.class.name
+      self.label          ||= main_action && main_action.class.name
       changes = self.changes
       save!
       changes
     end
 
+    def cancellable?
+      execution_plan.try(:cancellable?)
+    end
+
     def cancel
-      execution_plan.cancel.any?
+      execution_plan!.cancel.any?
     end
 
     def resumable?
-      execution_plan.state == :paused
+      execution_plan.try(:state) == :paused
     end
 
     def cancellable_action?(action)
       action.is_a?(::Dynflow::Action::Cancellable)
     end
 
-    def execution_plan
-      @execution_plan ||= ForemanTasks.dynflow.world.persistence.load_execution_plan(external_id)
+    def progress
+      execution_plan.try(:progress) || 0
+    end
+
+    def execution_plan(silence_exception = true)
+      return @execution_plan if defined?(@execution_plan)
+      execution_plan = ForemanTasks.dynflow.world.persistence.load_execution_plan(external_id)
+      # don't use invalid execution plans for our purposes
+      if execution_plan.respond_to?(:valid?) && !execution_plan.valid?
+        raise execution_plan.exception
+      else
+        @execution_plan = execution_plan
+      end
+      @execution_plan
+    rescue => e
+      Foreman::Logging.exception("Could not load execution plan #{external_id} for task #{id}", e, :logger => 'foreman-tasks')
+      raise e unless silence_exception
+      nil
+    end
+
+    def execution_plan!
+      execution_plan(false)
     end
 
     def input
@@ -50,11 +73,11 @@ module ForemanTasks
     end
 
     def failed_steps
-      execution_plan.steps_in_state(:skipped, :skipping, :error)
+      execution_plan.try(:steps_in_state, :skipped, :skipping, :error) || []
     end
 
     def running_steps
-      execution_plan.steps_in_state(:running, :suspended)
+      execution_plan.try(:steps_in_state, :running, :suspended) || []
     end
 
     def humanized
@@ -69,8 +92,8 @@ module ForemanTasks
     end
 
     def main_action
-      return @main_action if @main_action
-      execution_plan.root_plan_step.action execution_plan
+      return @main_action if defined?(@main_action)
+      @main_action = execution_plan && execution_plan.root_plan_step.try(:action, execution_plan)
     end
 
     def get_humanized(method)
@@ -79,7 +102,8 @@ module ForemanTasks
         method = "humanized_#{method}".to_sym
       end
       Match! method, :humanized_name, :humanized_input, :humanized_output, :humanized_errors
-      return N_('N/A') if method != :humanized_name && main_action.execution_plan.state == :scheduled
+      return N_('N/A') if method != :humanized_name && (main_action.nil? || main_action.execution_plan.state == :scheduled)
+      return N_(label) if method == :humanized_name && main_action.nil?
       @humanized_cache[method] ||= begin
                                      if main_action.respond_to? method
                                        begin
@@ -102,6 +126,10 @@ module ForemanTasks
             logger.warn('Task %s updated at consistency check: %s' % [task.id, changes.inspect])
           end
         rescue => e
+          # if we fail updating the data from dynflow, it usually means there is something
+          # odd with the data consistency and at this point it is not possible to resume, switching
+          # the task to stopped/error
+          task.update_attributes(:state => 'stopped', :result => 'error')
           Foreman::Logging.exception("Failed at consistency check for task #{task.id}", e, :logger => 'foreman-tasks')
         end
       end
@@ -126,7 +154,7 @@ module ForemanTasks
     end
 
     def cancelled?
-      execution_plan.errors.map(&:exception).any? { |exception| exception.class == ::ForemanTasks::Task::TaskCancelledException }
+      execution_plan.errors.any? { |error| error.exception_class == ::ForemanTasks::Task::TaskCancelledException }
     end
   end
 end

data/app/views/foreman_tasks/tasks/_errors.html.erb

@@ -1,7 +1,11 @@
 <% if @task.is_a? ForemanTasks::Task::DynflowTask %>
   <div>
     <% failed_steps = @task.failed_steps %>
-    <% if failed_steps.empty? %>
+    <% if @task.execution_plan.nil? %>
+      <div class="alert alert-danger">
+        <%= _("Execution plan data not available ") %>
+      </div>
+    <% elsif failed_steps.empty? %>
       <div class="alert alert-success">
         <%= _("No errors") %>
       </div>

data/bin/dynflow-executor

@@ -2,45 +2,70 @@
 
 require 'optparse'
 
-options = { foreman_root: Dir.pwd }
+class ArgvParser
+  attr_reader :options, :command
 
-opts = OptionParser.new do |opts|
-  opts.banner = <<BANNER
+  def initialize(argv, file)
+    @options = { foreman_root: Dir.pwd }
+
+    opts = OptionParser.new do |opts|
+      opts.banner = banner(file)
+
+      opts.on('-h', '--help', 'Show this message') do
+        puts opts
+        exit 1
+      end
+      opts.on('-f', '--foreman-root=PATH', "Path to Foreman Rails root path. By default '#{@options[:foreman_root]}'") do |path|
+        @options[:foreman_root] = path
+      end
+      opts.on('-c', '--executors-count=COUNT', 'Number of parallel executors to spawn. Overrides EXECUTORS_COUNT environment varaible.') do |count|
+        @options[:executors_count] = count.to_i
+      end
+      opts.on('-m', '--memory-limit=SIZE', 'Limits the amount of memory an executor can consume. Overrides EXECUTOR_MEMORY_LIMIT environment varaible. You can use kb, mb, gb') do |size|
+        @options[:memory_limit] = size
+      end
+      opts.on('--executor-memory-init-delay=SECONDS', 'Start memory polling after SECONDS. Overrides EXECUTOR_MEMORY_MONITOR_DELAY environment varaible.') do |seconds|
+        @options[:memory_init_delay] = seconds.to_i
+      end
+      opts.on('--executor-memory-polling-interval=SECONDS', 'Check for memory useage every SECONDS sec. Overrides EXECUTOR_MEMORY_MONITOR_INTERVAL environment varaible.') do |seconds|
+        @options[:memory_polling_interval] = seconds.to_i
+      end
+    end
+
+    args = opts.parse!(argv)
+    @command = args.first || 'run'
+  end
+
+  def banner(file)
+    banner = <<BANNER
 Run Dynflow executor for Foreman tasks.
 
-Usage: #{File.basename($PROGRAM_NAME)} [options] ACTION"
+Usage: #{File.basename(file)} [options] ACTION"
 
 ACTION can be one of:
 
-  * start   - start the executor on background. It creates these files
-              in tmp/pid directory:
+* start   - start the executor on background. It creates these files
+          in tmp/pid directory:
 
-                * dynflow_executor_monitor.pid - pid of monitor ensuring
-                                                 the executor keeps running
-                * dynflow_executor.pid         - pid of the executor itself
-                * dynflow_executor.output      - stdout of the executor
-  * stop    - stops the running executor
-  * restart - restarts the running executor
-  * run     - run the executor in foreground
+            * dynflow_executor_monitor.pid - pid of monitor ensuring
+                                             the executor keeps running
+            * dynflow_executor.pid         - pid of the executor itself
+            * dynflow_executor.output      - stdout of the executor
+* stop    - stops the running executor
+* restart - restarts the running executor
+* run     - run the executor in foreground
 
 BANNER
-
-  opts.on('-h', '--help', 'Show this message') do
-    puts opts
-    exit 1
-  end
-  opts.on('-f', '--foreman-root=PATH', "Path to Foreman Rails root path. By default '#{options[:foreman_root]}'") do |path|
-    options[:foreman_root] = path
-  end
-  opts.on('-c', '--executors-count=COUNT', 'Number of parallel executors to spawn. Overrides EXECUTORS_COUNT environment varaible.') do |count|
-    options[:executors_count] = count.to_i
+    banner
   end
 end
 
-args = opts.parse!(ARGV)
-command = args.first || 'run'
+# run the script if it's executed explicitly
+if $PROGRAM_NAME == __FILE__
+  parser = ArgvParser.new(ARGV, $PROGRAM_NAME)
 
-app_file = File.expand_path('./config/application', options[:foreman_root])
-require app_file
+  app_file = File.expand_path('./config/application', parser.options[:foreman_root])
+  require app_file
 
-ForemanTasks::Dynflow::Daemon.new.run_background(command, options)
+  ForemanTasks::Dynflow::Daemon.new.run_background(parser.command, parser.options)
+end

data/deploy/foreman-tasks.sysconfig

@@ -12,3 +12,15 @@ RUBY_GC_MALLOC_LIMIT_MAX=16000100
 RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR=1.1
 RUBY_GC_OLDMALLOC_LIMIT=16000100
 RUBY_GC_OLDMALLOC_LIMIT_MAX=16000100
+
+#Set the number of executors you want to run
+#EXECUTORS_COUNT=1
+
+#Set memory limit for executor process, before it's restarted automatically
+#EXECUTOR_MEMORY_LIMIT=2gb
+
+#Set delay before first memory polling to let executor initialize (in sec)
+#EXECUTOR_MEMORY_MONITOR_DELAY=7200 #default: 2 hours
+
+#Set memory polling interval, process memory will be checked every N seconds.
+#EXECUTOR_MEMORY_MONITOR_INTERVAL=60

data/foreman-tasks.gemspec

@@ -29,9 +29,10 @@ DESC
   s.extra_rdoc_files = Dir['README*', 'LICENSE']
 
   s.add_dependency "foreman-tasks-core"
-  s.add_dependency "dynflow", '~> 0.8.22'
+  s.add_dependency "dynflow", '~> 0.8.24'
   s.add_dependency "sequel" # for Dynflow process persistence
   s.add_dependency "sinatra" # for Dynflow web console
   s.add_dependency "daemons" # for running remote executor
   s.add_dependency "parse-cron", '~> 0.1.4'
+  s.add_dependency "get_process_mem" # for memory polling
 end

data/lib/foreman_tasks/dynflow.rb

@@ -9,8 +9,9 @@ module ForemanTasks
     require 'foreman_tasks/dynflow/daemon'
     require 'foreman_tasks/dynflow/console_authorizer'
 
-    def initialize
+    def initialize(world_class = nil)
       @required = false
+      @world_class = world_class
     end
 
     def config
@@ -37,7 +38,7 @@ module ForemanTasks
       if config.lazy_initialization && defined?(PhusionPassenger)
         config.dynflow_logger.warn('ForemanTasks: lazy loading with PhusionPassenger might lead to unexpected results')
       end
-      config.initialize_world.tap do |world|
+      init_world.tap do |world|
         @world = world
 
         unless config.remote?
@@ -119,5 +120,12 @@ module ForemanTasks
     def loaded_paths
       @loaded_paths ||= Set.new
     end
+
+    private
+
+    def init_world
+      return config.initialize_world(@world_class) if @world_class
+      config.initialize_world
+    end
   end
 end

data/lib/foreman_tasks/dynflow/daemon.rb

@@ -1,18 +1,47 @@
 require 'fileutils'
+require 'daemons'
+require 'get_process_mem'
+require 'dynflow/watchers/memory_consumption_watcher'
 
 module ForemanTasks
   class Dynflow::Daemon
+    attr_reader :dynflow_memory_watcher_class,
+                :daemons_class
+
+    # make Daemon dependency injection ready for testing purposes
+    def initialize(
+      dynflow_memory_watcher_class = ::Dynflow::Watchers::MemoryConsumptionWatcher,
+      daemons_class = ::Daemons
+    )
+      @dynflow_memory_watcher_class = dynflow_memory_watcher_class
+      @daemons_class = daemons_class
+    end
+
     # load the Rails environment and initialize the executor
     # in this thread.
-    def run(foreman_root = Dir.pwd)
+    def run(foreman_root = Dir.pwd, options = {})
       STDERR.puts('Starting Rails environment')
       foreman_env_file = File.expand_path('./config/environment.rb', foreman_root)
       unless File.exist?(foreman_env_file)
         raise "#{foreman_root} doesn't seem to be a foreman root directory"
       end
+
+      STDERR.puts("Starting dynflow with the following options: #{options}")
+
       ForemanTasks.dynflow.executor!
+
+      if options[:memory_limit] > 0
+        ForemanTasks.dynflow.config.on_init do |world|
+          memory_watcher = initialize_memory_watcher(world, options[:memory_limit], options)
+          world.terminated.on_completion do
+            STDERR.puts("World has been terminated")
+            memory_watcher = nil # the object can be disposed
+          end
+        end
+      end
+
       require foreman_env_file
-      STDERR.puts('Everything ready')
+      STDERR.puts("Everything ready for world: #{(ForemanTasks.dynflow.initialized? ? ForemanTasks.dynflow.world.id : nil)}")
       sleep
     ensure
       STDERR.puts('Exiting')
@@ -20,13 +49,6 @@ module ForemanTasks
 
     # run the executor as a daemon
     def run_background(command = 'start', options = {})
-      default_options = { foreman_root: Dir.pwd,
-                          process_name: 'dynflow_executor',
-                          pid_dir: "#{Rails.root}/tmp/pids",
-                          log_dir: File.join(Rails.root, 'log'),
-                          wait_attempts: 300,
-                          wait_sleep: 1,
-                          executors_count: (ENV['EXECUTORS_COUNT'] || 1).to_i }
       options = default_options.merge(options)
       FileUtils.mkdir_p(options[:pid_dir])
       begin
@@ -42,18 +64,13 @@ module ForemanTasks
       STDERR.puts("Dynflow Executor: #{command} in progress")
 
       options[:executors_count].times do
-        Daemons.run_proc(options[:process_name],
-                         :multiple => true,
-                         :dir => options[:pid_dir],
-                         :log_dir => options[:log_dir],
-                         :dir_mode => :normal,
-                         :monitor => true,
-                         :log_output => true,
-                         :log_output_syslog => true,
-                         :ARGV => [command]) do |*_args|
+        daemons_class.run_proc(
+          options[:process_name],
+          daemons_options(command, options)
+        ) do |*_args|
           begin
             ::Logging.reopen
-            run(options[:foreman_root])
+            run(options[:foreman_root], options)
           rescue => e
             STDERR.puts e.message
             Foreman::Logging.exception('Failed running foreman-tasks daemon', e)
@@ -68,5 +85,59 @@ module ForemanTasks
     def world
       ForemanTasks.dynflow.world
     end
+
+    private
+
+    def daemons_options(command, options)
+      {
+        :multiple => true,
+        :dir => options[:pid_dir],
+        :log_dir => options[:log_dir],
+        :dir_mode => :normal,
+        :monitor => true,
+        :log_output => true,
+        :log_output_syslog => true,
+        :monitor_interval => [options[:memory_polling_interval] / 2, 30].min,
+        :ARGV => [command]
+      }
+    end
+
+    def default_options
+      {
+        foreman_root: Dir.pwd,
+        process_name: 'dynflow_executor',
+        pid_dir: "#{Rails.root}/tmp/pids",
+        log_dir: File.join(Rails.root, 'log'),
+        wait_attempts: 300,
+        wait_sleep: 1,
+        executors_count: (ENV['EXECUTORS_COUNT'] || 1).to_i,
+        memory_limit: begin
+                        (ENV['EXECUTOR_MEMORY_LIMIT'] || '').to_gb.gigabytes
+                      rescue RuntimeError
+                        ENV['EXECUTOR_MEMORY_LIMIT'].to_i
+                      end,
+        memory_init_delay: (ENV['EXECUTOR_MEMORY_MONITOR_DELAY'] || 7200).to_i, # 2 hours
+        memory_polling_interval: (ENV['EXECUTOR_MEMORY_MONITOR_INTERVAL'] || 60).to_i
+      }
+    end
+
+    def initialize_memory_watcher(world, memory_limit, options)
+      watcher_options = {}
+      watcher_options[:polling_interval] = options[:memory_polling_interval]
+      watcher_options[:initial_wait] = options[:memory_init_delay]
+      watcher_options[:memory_checked_callback] = ->(current_memory, memory_limit) { log_memory_within_limit(current_memory, memory_limit) }
+      watcher_options[:memory_limit_exceeded_callback] = ->(current_memory, memory_limit) { log_memory_limit_exceeded(current_memory, memory_limit) }
+      dynflow_memory_watcher_class.new(world, memory_limit, watcher_options)
+    end
+
+    def log_memory_limit_exceeded(current_memory, memory_limit)
+      message = "Memory level exceeded, registered #{current_memory} bytes, which is greater than #{memory_limit} limit."
+      Foreman::Logging.logger('foreman-tasks').error(message)
+    end
+
+    def log_memory_within_limit(current_memory, memory_limit)
+      message = "Memory level OK, registered #{current_memory} bytes, which is less than #{memory_limit} limit."
+      Foreman::Logging.logger('foreman-tasks').debug(message)
+    end
   end
 end