foreman-tasks 0.10.2 → 0.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e10fb3f1d62ca016568197df4df727b25c73b68
-  data.tar.gz: 506d7b4cc63c11fbd931667258122ee48f03a504
+  metadata.gz: ae915bf659a976d2dfcac1f2e51cb0a447d080a8
+  data.tar.gz: 41bf2fb543e2ba405f828be97d0727ff9ec73c17
 SHA512:
-  metadata.gz: 4fe503d8cbfce8eec1ea48f0add2442d1df0acb2729f53d391dfd5068ebf975722f69ffd0801780fae770c02958b3909c466668d368ad39ca345ca6acb9dd2d9
-  data.tar.gz: 910f71926669c46ef32a8c55a1b666c2a74ac810d176e27c0dfcf451e49b814af871ebf0639d70dc1378d01be48167c0d53c4bf3bd514e02d8302f6c3bb49355
+  metadata.gz: 3b641565b6e1360887774df9489089c2e5e960d299b976140f307b15e2b8e7d92815900541124265eead2a6294a57fae798371bab3074f4026f71cc6298ce6f4
+  data.tar.gz: b9304f40c6ab2f54a37bc92f93d68e6fb81f3cae68e63d5299994f9e5a5c1bc466d3d081e6047067cac2901e3664e58734421549c3892f2fce400347f871d7ac

data/config/foreman-tasks.yaml.example

@@ -9,18 +9,43 @@
 #     :action:
 #       :enabled: true
 
+  # Task backup configuration can be changed by altering the values in
+  # the backup section
+  #
+  :backup:
+    #
+    # Whether to back up tasks when they are removed
+    #
+    :backup_deleted_tasks: true
+    #
+    # Where to put the tasks which were backed up
+    #
+    :backup_dir: /var/lib/foreman/tasks-backup
 
 # Cleaning configuration: how long should the actions be kept before deleted
 # by `rake foreman_tasks:clean` task
 #
-#  :cleanup:
+  :cleanup:
 #
 # the period after which to delete all the tasks (by default all tasks are not being deleted after some period)
+# will be deprecated in Foreman 1.18 and the use of rules is recommended.
 #
-#    :after: 365d
+#     :after: 30d
 #
 # per action settings to override the default defined in the actions (self.cleanup_after method)
 #
 #    :actions:
 #      - :name: Actions::Foreman::Host::ImportFacts
 #        :after: 10d
+#
+# Rules defined in this section by default don't operate
+# on tasks specified in the actions section. This behavior
+# can be overriden by setting the override_actions to true
+   :rules:
+     # Delete successful tasks after a month
+     - :filter: result = success
+       :after: 30d
+     # Delete everything (any action, any state) after one year
+     - :states: all # Either list of state names or all
+       :after: 1y
+       :override_actions: true

data/lib/foreman_tasks/cleaner.rb

@@ -1,3 +1,5 @@
+require 'csv'
+
 module ForemanTasks
   # Represents the cleanup mechanism for tasks
   class Cleaner
@@ -11,11 +13,16 @@ module ForemanTasks
           end
         end
         if cleanup_settings[:after]
+          Foreman::Deprecation.deprecation_warning('1.18', _(':after setting in tasks cleanup section is deprecated, use :after in :rules section to set the value. to cleanup rules'))
           new(options.merge(:filter => '', :after => cleanup_settings[:after])).delete
         end
-        actions_with_default_cleanup.each do |action_class, period|
+        with_periods = actions_with_default_cleanup
+        with_periods.each do |action_class, period|
           new(options.merge(:filter => "label = #{action_class.name}", :after => period)).delete
         end
+        actions_by_rules(with_periods).each do |hash|
+          new(options.merge(hash)).delete
+        end
       end
     end
 
@@ -44,6 +51,19 @@ module ForemanTasks
       @cleanup_settings = SETTINGS[:'foreman-tasks'] && SETTINGS[:'foreman-tasks'][:cleanup] || {}
     end
 
+    def self.actions_by_rules(actions_with_periods)
+      disable_actions_with_periods = "label !^ (#{actions_with_periods.keys.join(', ')})"
+      cleanup_settings.fetch(:rules, []).map do |hash|
+        next if hash[:after].nil?
+        conditions = []
+        conditions << disable_actions_with_periods unless hash[:override_actions]
+        conditions << hash[:filter] if hash[:filter]
+        hash[:states] = [] if hash[:states] == 'all'
+        hash[:filter] = conditions.map { |condition| "(#{condition})" }.join(' AND ')
+        hash
+      end.compact
+    end
+
     attr_reader :filter, :after, :states, :verbose, :batch_size, :noop, :full_filter
 
     # @param filter [String] scoped search matching the tasks to be deleted
@@ -56,7 +76,8 @@ module ForemanTasks
                           :verbose      => false,
                           :batch_size   => 1000,
                           :noop         => false,
-                          :states       => ['stopped'] }
+                          :states       => ['stopped'],
+                          :backup_dir   => ForemanTasks.dynflow.world.persistence.current_backup_dir }
       options         = default_options.merge(options)
 
       @filter         = options[:filter]
@@ -65,6 +86,7 @@ module ForemanTasks
       @verbose        = options[:verbose]
       @batch_size     = options[:batch_size]
       @noop           = options[:noop]
+      @backup_dir     = options[:backup_dir]
 
       raise ArgumentError, 'filter not speficied' if @filter.nil?
 
@@ -91,12 +113,33 @@ module ForemanTasks
     end
 
     def delete_tasks(chunk)
-      ForemanTasks::Task.where(:id => chunk.map(&:id)).delete_all
+      tasks = ForemanTasks::Task.where(:id => chunk.map(&:id))
+      tasks_to_csv(tasks, @backup_dir, 'foreman_tasks.csv') if @backup_dir
+      tasks.delete_all
+    end
+
+    def tasks_to_csv(dataset, backup_dir, file_name)
+      with_backup_file(backup_dir, file_name) do |csv, appending|
+        csv << ForemanTasks::Task.attribute_names.to_csv unless appending
+        dataset.each do |row|
+          csv << row.attributes.values.to_csv
+        end
+      end
+      dataset
+    end
+
+    def with_backup_file(backup_dir, file_name)
+      FileUtils.mkdir_p(backup_dir) unless File.directory?(backup_dir)
+      csv_file = File.join(backup_dir, file_name)
+      appending = File.exist?(csv_file)
+      File.open(csv_file, 'a') do |f|
+        yield f, appending
+      end
     end
 
     def delete_dynflow_plans(chunk)
       dynflow_ids = chunk.find_all { |task| task.is_a? Task::DynflowTask }.map(&:external_id)
-      ForemanTasks.dynflow.world.persistence.delete_execution_plans({ 'uuid' => dynflow_ids }, batch_size)
+      ForemanTasks.dynflow.world.persistence.delete_execution_plans({ 'uuid' => dynflow_ids }, batch_size, @backup_dir)
     end
 
     def prepare_filter

data/lib/foreman_tasks/dynflow/configuration.rb

@@ -6,6 +6,37 @@ require 'foreman_tasks/dynflow/persistence'
 module ForemanTasks
   # Import all Dynflow configuration from Foreman, and add our own for Tasks
   class Dynflow::Configuration < ::Foreman::Dynflow::Configuration
+    def world_config
+      super.tap do |config|
+        config.backup_deleted_plans = backup_settings[:backup_deleted_plans]
+        config.backup_dir           = backup_settings[:backup_dir]
+      end
+    end
+
+    def backup_settings
+      return @backup_settings if @backup_settings
+      backup_options = {
+        :backup_deleted_plans => true,
+        :backup_dir => default_backup_dir
+      }
+      settings = SETTINGS[:'foreman-tasks'] && SETTINGS[:'foreman-tasks'][:backup]
+      backup_options.merge!(settings) if settings
+      @backup_settings = with_environment_override backup_options
+    end
+
+    def default_backup_dir
+      File.join(Rails.root, 'tmp', 'task-backup')
+    end
+
+    def with_environment_override(options)
+      env_var = ENV['TASK_BACKUP']
+      unless env_var.nil?
+        # Everything except 0, n, no, false is considered to be a truthy value
+        options[:backup_deleted_plans] = !%w[0 n no false].include?(env_var.downcase)
+      end
+      options
+    end
+
     def initialize_persistence
       ForemanTasks::Dynflow::Persistence.new(default_sequel_adapter_options)
     end

data/lib/foreman_tasks/tasks/cleanup.rake

@@ -5,10 +5,11 @@ namespace :foreman_tasks do
 
         * TASK_SEARCH : scoped search filter (example: 'label = "Actions::Foreman::Host::ImportFacts"')
         * AFTER       : delete tasks created after *AFTER* period. Expected format is a number followed by the time unit (s,h,m,y), such as '10d' for 10 days
-        * STATES      : comma separated list of task states to touch with the cleanup, by default only stopped tasks are covered
-        * NOOP        : set to "true" if the task should not actuall perform the deletion
+        * STATES      : comma separated list of task states to touch with the cleanup, by default only stopped tasks are covered, special value all can be used to clean the tasks, disregarding their states
+        * NOOP        : set to "true" if the task should not actually perform the deletion
         * VERBOSE     : set to "true" for more verbose output
         * BATCH_SIZE  : the size of batches the tasks get processed in (1000 by default)
+        * TASK_BACKUP : set to "true" or "false" to enable/disable task backup
 
       If none of TASK_SEARCH, BEFORE, STATES is specified, the tasks will be cleaned based
       configuration in settings
@@ -21,6 +22,7 @@ namespace :foreman_tasks do
       options[:after] = ENV['AFTER'] if ENV['AFTER']
 
       options[:states] = ENV['STATES'].to_s.split(',') if ENV['STATES']
+      options[:states] = [] if options[:states] == ['all']
 
       options[:noop] = true if ENV['NOOP']
 
@@ -52,6 +54,24 @@ namespace :foreman_tasks do
           printf("%-50s %s\n", action.name, after)
         end
       end
+      puts
+      by_rules = ForemanTasks::Cleaner.actions_by_rules(ForemanTasks::Cleaner.actions_with_default_cleanup)
+      if by_rules.empty?
+        puts _('No cleanup rules are configured')
+      else
+        printf("%-50s %-15s %s\n", _('states'), _('delete after'), _('filter'))
+        by_rules.each do |hash|
+          state = case hash[:states]
+                  when []
+                    _('ANY')
+                  when nil
+                    'stopped'
+                  else
+                    hash[:states]
+                  end
+          printf("%-50s %-15s %s\n", state, hash[:after], hash[:filter])
+        end
+      end
     end
   end
 

data/lib/foreman_tasks/version.rb

@@ -1,3 +1,3 @@
 module ForemanTasks
-  VERSION = '0.10.2'.freeze
+  VERSION = '0.10.3'.freeze
 end

data/locale/en/foreman_tasks.po

@@ -38,6 +38,9 @@ msgstr ""
 msgid "%{value} is wrong format"
 msgstr ""
 
+msgid ":after setting in tasks cleanup section is deprecated, use :after in :rules section to set the value. to cleanup rules"
+msgstr ""
+
 msgid "Action"
 msgstr ""
 
@@ -207,6 +210,9 @@ msgstr ""
 msgid "Last occurrence"
 msgstr ""
 
+msgid "Last start time"
+msgstr ""
+
 msgid "Latest Warning/Error Tasks"
 msgstr ""
 
@@ -268,9 +274,6 @@ msgstr ""
 msgid "Parent task"
 msgstr ""
 
-msgid "Play Ansible roles"
-msgstr ""
-
 msgid "Please inspect their state, fix their errors and resume them."
 msgstr ""
 
@@ -394,6 +397,9 @@ msgstr ""
 msgid "Task Status"
 msgstr ""
 
+msgid "Task aborted: the task might be still running on the proxy"
+msgstr ""
+
 msgid "Task count"
 msgstr ""
 
@@ -424,6 +430,9 @@ msgstr ""
 msgid "The targets are of different types"
 msgstr ""
 
+msgid "The task cannot be aborted at the moment."
+msgstr ""
+
 msgid "The task cannot be cancelled at the moment."
 msgstr ""
 
@@ -451,6 +460,9 @@ msgstr ""
 msgid "Triggered by"
 msgstr ""
 
+msgid "Trying to abort the task"
+msgstr ""
+
 msgid "Trying to cancel step %s"
 msgstr ""
 

data/locale/foreman_tasks.pot

@@ -8,8 +8,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: foreman_tasks 1.0.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2017-07-19 17:30+0200\n"
-"PO-Revision-Date: 2017-07-19 17:30+0200\n"
+"POT-Creation-Date: 2017-09-21 20:56+0200\n"
+"PO-Revision-Date: 2017-09-21 20:56+0200\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -90,19 +90,19 @@ msgstr ""
 msgid "Data to be sent to the action"
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:181
+#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:182
 msgid "User search_params requires user_id to be specified"
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:190
+#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:191
 msgid "Resource search_params requires resource_type and resource_id to be specified"
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:197
+#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:198
 msgid "Task search_params requires task_id to be specified"
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:201
+#: ../app/controllers/foreman_tasks/api/tasks_controller.rb:202
 msgid "Type %s for search_params is not supported"
 msgstr ""
 
@@ -118,30 +118,40 @@ msgstr ""
 msgid "The task cannot be cancelled at the moment."
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/tasks_controller.rb:43
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:42
+msgid "Trying to abort the task"
+msgstr ""
+
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:44
+msgid "The task cannot be aborted at the moment."
+msgstr ""
+
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:53
 msgid "The execution was resumed."
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/tasks_controller.rb:45
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:55
 msgid "The execution has to be resumable."
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/tasks_controller.rb:55
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:65
 msgid "The task resources were unlocked."
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/tasks_controller.rb:57
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:67
 msgid "The execution has to be paused."
 msgstr ""
 
-#: ../app/controllers/foreman_tasks/tasks_controller.rb:66
+#: ../app/controllers/foreman_tasks/tasks_controller.rb:76
 msgid "The task resources were unlocked with force."
 msgstr ""
 
 #: ../app/helpers/foreman_tasks/foreman_tasks_helper.rb:38
 #: ../app/helpers/foreman_tasks/foreman_tasks_helper.rb:47
 #: ../app/models/foreman_tasks/recurring_logic.rb:101
-#: ../app/models/foreman_tasks/task/dynflow_task.rb:105
+#: ../app/models/foreman_tasks/task/dynflow_task.rb:109
+#: ../app/views/foreman_tasks/tasks/dashboard/_latest_tasks_in_error_warning.html.erb:14
+#: ../app/views/foreman_tasks/tasks/dashboard/_tasks_status.html.erb:14
 msgid "N/A"
 msgstr ""
 
@@ -151,6 +161,7 @@ msgstr ""
 
 #: ../app/helpers/foreman_tasks/foreman_tasks_helper.rb:40
 #: ../app/views/foreman_tasks/tasks/dashboard/_latest_tasks_in_error_warning.html.erb:14
+#: ../app/views/foreman_tasks/tasks/dashboard/_tasks_status.html.erb:14
 msgid "%s ago"
 msgstr ""
 
@@ -303,11 +314,11 @@ msgstr ""
 msgid "The targets are of different types"
 msgstr ""
 
-#: ../app/lib/actions/foreman/host/import_facts.rb:38 action_names.rb:3
+#: ../app/lib/actions/foreman/host/import_facts.rb:38 action_names.rb:5
 msgid "Import facts"
 msgstr ""
 
-#: ../app/lib/actions/foreman/puppetclass/import.rb:18 action_names.rb:4
+#: ../app/lib/actions/foreman/puppetclass/import.rb:18 action_names.rb:3
 msgid "Import Puppet classes"
 msgstr ""
 
@@ -355,15 +366,19 @@ msgstr ""
 msgid "All proxies with the required feature are unavailable at the moment"
 msgstr ""
 
-#: ../app/lib/actions/proxy_action.rb:59
+#: ../app/lib/actions/proxy_action.rb:61
 msgid "The smart proxy task %s failed."
 msgstr ""
 
-#: ../app/lib/actions/proxy_action.rb:73
+#: ../app/lib/actions/proxy_action.rb:75
 msgid "Cancel enforced: the task might be still running on the proxy"
 msgstr ""
 
-#: ../app/lib/actions/proxy_action.rb:114
+#: ../app/lib/actions/proxy_action.rb:85
+msgid "Task aborted: the task might be still running on the proxy"
+msgstr ""
+
+#: ../app/lib/actions/proxy_action.rb:121
 msgid "Initialization error: %s"
 msgstr ""
 
@@ -718,6 +733,10 @@ msgstr ""
 msgid "No. of Tasks"
 msgstr ""
 
+#: ../app/views/foreman_tasks/tasks/dashboard/_tasks_status.html.erb:7
+msgid "Last start time"
+msgstr ""
+
 #: ../app/views/foreman_tasks/tasks/index.html.erb:1
 #: ../lib/foreman_tasks/engine.rb:48
 msgid "Tasks"
@@ -751,7 +770,13 @@ msgstr ""
 msgid "Raw"
 msgstr ""
 
-#: ../lib/foreman_tasks/dynflow.rb:102
+#: ../lib/foreman_tasks/cleaner.rb:16
+msgid ""
+":after setting in tasks cleanup section is deprecated, use :after in :rules se"
+"ction to set the value. to cleanup rules"
+msgstr ""
+
+#: ../lib/foreman_tasks/dynflow.rb:19
 msgid "Back to tasks"
 msgstr ""
 
@@ -763,10 +788,6 @@ msgstr ""
 msgid "Action with sub plans"
 msgstr ""
 
-#: action_names.rb:5
+#: action_names.rb:4
 msgid "Remote action:"
 msgstr ""
-
-#: action_names.rb:6
-msgid "Play Ansible roles"
-msgstr ""

data/test/unit/cleaner_test.rb

@@ -1,6 +1,11 @@
 require 'foreman_tasks_test_helper'
 
 class TasksTest < ActiveSupport::TestCase
+  before do
+    # To stop dynflow from backing up actions, execution_plans and steps
+    ForemanTasks.dynflow.world.persistence.adapter.stubs(:backup_to_csv)
+  end
+
   describe ForemanTasks::Cleaner do
     it 'is able to delete tasks (including the dynflow plans) based on filter' do
       cleaner = ForemanTasks::Cleaner.new(:filter => 'label = "Actions::User::Create"', :after => '10d')
@@ -12,6 +17,7 @@ class TasksTest < ActiveSupport::TestCase
                            task.save
                          end,
                          FactoryGirl.create(:dynflow_task, :product_create_task)]
+      cleaner.expects(:tasks_to_csv)
       cleaner.delete
       ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
       ForemanTasks::Task.where(id: tasks_to_keep).order(:id).map(&:id).must_equal tasks_to_keep.map(&:id).sort
@@ -32,6 +38,7 @@ class TasksTest < ActiveSupport::TestCase
                          end]
 
       tasks_to_keep   = [FactoryGirl.create(:dynflow_task, :product_create_task)]
+      cleaner.expects(:tasks_to_csv)
       cleaner.delete
       ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
       ForemanTasks::Task.where(id: tasks_to_keep).must_equal tasks_to_keep
@@ -46,11 +53,32 @@ class TasksTest < ActiveSupport::TestCase
                            task.started_at = task.ended_at = Time.zone.now
                            task.save
                          end]
+      cleaner.expects(:tasks_to_csv)
       cleaner.delete
       ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
       ForemanTasks::Task.where(id: tasks_to_keep).must_equal tasks_to_keep
     end
 
+    it 'backs tasks up before deleting' do
+      dir = '/tmp'
+      cleaner = ForemanTasks::Cleaner.new(:filter => '', :after => '10d', :backup_dir => dir)
+      tasks_to_delete = [FactoryGirl.create(:dynflow_task, :user_create_task),
+                         FactoryGirl.create(:dynflow_task, :product_create_task)]
+
+      r, w = IO.pipe
+      cleaner.expects(:with_backup_file)
+             .with(dir, 'foreman_tasks.csv')
+             .yields(w, false)
+      cleaner.delete
+      w.close
+      header, *data = r.readlines.map(&:chomp)
+      header.must_equal ForemanTasks::Task.attribute_names.join(',')
+      expected_lines = tasks_to_delete.map { |task| task.attributes.values.join(',') }
+      data.count.must_equal expected_lines.count
+      expected_lines.each { |line| data.must_include line }
+      ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
+    end
+
     class ActionWithCleanup < Actions::Base
       def self.cleanup_after
         '15d'
@@ -68,6 +96,29 @@ class TasksTest < ActiveSupport::TestCase
                                      { :actions => [{ :name => ActionWithCleanup.name, :after => '5d' }] })
         ForemanTasks::Cleaner.actions_with_default_cleanup[ActionWithCleanup].must_equal '5d'
       end
+
+      it 'deprecates the usage of :after' do
+        Foreman::Deprecation.expects(:deprecation_warning)
+        ForemanTasks::Cleaner.any_instance.expects(:delete)
+        ForemanTasks::Cleaner.stubs(:cleanup_settings =>
+                                    { :after => '1d' })
+        ForemanTasks::Cleaner.stubs(:actions_with_default_cleanup).returns({})
+        ForemanTasks::Cleaner.run({})
+      end
+
+      it 'generates filters from rules properly' do
+        actions_with_default = { 'action1' => nil, 'action2' => nil }
+        rules = [{ :after => nil },
+                 { :after => '10d', :filter => 'label = something', :states => %w[stopped paused] },
+                 { :after => '15d', :filter => 'label = something_else',
+                   :override_actions => true, :states => 'all' }]
+        ForemanTasks::Cleaner.stubs(:cleanup_settings).returns(:rules => rules)
+        r1, r2 = ForemanTasks::Cleaner.actions_by_rules actions_with_default
+        r1[:filter].must_equal '(label !^ (action1, action2)) AND (label = something)'
+        r1[:states].must_equal %w[stopped paused]
+        r2[:filter].must_equal '(label = something_else)'
+        r2[:states].must_equal []
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: foreman-tasks
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.10.3
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-19 00:00:00.000000000 Z
+date: 2017-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: foreman-tasks-core