foreman-tasks 0.9.5 → 0.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2fbc8738b9259442215b12e07307cb605dbdc299
-  data.tar.gz: e172529a5fd0c0950a163e403a8667dab0f8538c
+  metadata.gz: e1e9246d6fdeb8fc99d6e42f7ed41e87e398341a
+  data.tar.gz: a67dd1ec1cbad7e37a168b8495ed48242a862bf2
 SHA512:
-  metadata.gz: 16557fbad3ca19db7cef81f6ee905ebcde440c5c397a85e2e5c77bdf1ed97a7984d8e9eb09610d257bb95202cd872241394cd1fcb45365e5bd2204648f8b434c
-  data.tar.gz: 898dba76fa6a6cb556555ea5fbc8403df8070ebe156b37410ada39a4bd58a853d1716f6eb1f4cfde4424e1ab6285086e75d519857db873b850cc4283ddd41586
+  metadata.gz: d3f758c1445380c7b28ee3e132fdfc2f467c1c7b907286e18a56b6fdfeb62abcb0f81753e9c3809f7d8c8342ff7e35860dbdb34d491acc71b7107aac10fbde79
+  data.tar.gz: c814a2d7dd11fadd76cce7a11a7f38e469529301ab6debd6e3fdc7763a5fc8793acc20291bf11c69ba65ad837670f8a3cc027d320641f83e7b5633e9384df03b

data/config/foreman-tasks.yaml.example

@@ -9,18 +9,43 @@
 #     :action:
 #       :enabled: true
 
+  # Task backup configuration can be changed by altering the values in
+  # the backup section
+  #
+  :backup:
+    #
+    # Whether to back up tasks when they are removed
+    #
+    :backup_deleted_tasks: true
+    #
+    # Where to put the tasks which were backed up
+    #
+    :backup_dir: /var/lib/foreman/tasks-backup
 
 # Cleaning configuration: how long should the actions be kept before deleted
 # by `rake foreman_tasks:clean` task
 #
-#  :cleanup:
+  :cleanup:
 #
 # the period after which to delete all the tasks (by default all tasks are not being deleted after some period)
+# will be deprecated in Foreman 1.18 and the use of rules is recommended.
 #
-#    :after: 365d
+#     :after: 30d
 #
 # per action settings to override the default defined in the actions (self.cleanup_after method)
 #
 #    :actions:
 #      - :name: Actions::Foreman::Host::ImportFacts
 #        :after: 10d
+#
+# Rules defined in this section by default don't operate
+# on tasks specified in the actions section. This behavior
+# can be overriden by setting the override_actions to true
+   :rules:
+     # Delete successful tasks after a month
+     - :filter: result = success
+       :after: 30d
+     # Delete everything (any action, any state) after one year
+     - :states: all # Either list of state names or all
+       :after: 1y
+       :override_actions: true

data/foreman-tasks.gemspec

@@ -29,7 +29,7 @@ same resource. It also optionally provides Dynflow infrastructure for using it f
   s.extra_rdoc_files = Dir['README*', 'LICENSE']
 
   s.add_dependency "foreman-tasks-core"
-  s.add_dependency "dynflow", '~> 0.8.26'
+  s.add_dependency "dynflow", '~> 0.8.29'
   s.add_dependency "sequel" # for Dynflow process persistence
   s.add_dependency "sinatra" # for Dynflow web console
   s.add_dependency "daemons" # for running remote executor

data/lib/foreman_tasks/cleaner.rb

@@ -1,3 +1,5 @@
+require 'csv'
+
 module ForemanTasks
   # Represents the cleanup mechanism for tasks
   class Cleaner
@@ -11,11 +13,16 @@ module ForemanTasks
           end
         end
         if cleanup_settings[:after]
+          Foreman::Deprecation.deprecation_warning('1.18', _(':after setting in tasks cleanup section is deprecated, use :after in :rules section to set the value. to cleanup rules'))
           new(options.merge(:filter => '', :after => cleanup_settings[:after])).delete
         end
-        actions_with_default_cleanup.each do |action_class, period|
+        with_periods = actions_with_default_cleanup
+        with_periods.each do |action_class, period|
           new(options.merge(:filter => "label = #{action_class.name}", :after => period)).delete
         end
+        actions_by_rules(with_periods).each do |hash|
+          new(options.merge(hash)).delete
+        end
       end
     end
 
@@ -44,6 +51,19 @@ module ForemanTasks
       @cleanup_settings = SETTINGS[:'foreman-tasks'] && SETTINGS[:'foreman-tasks'][:cleanup] || {}
     end
 
+    def self.actions_by_rules(actions_with_periods)
+      disable_actions_with_periods = "label !^ (#{actions_with_periods.keys.join(', ')})"
+      cleanup_settings.fetch(:rules, []).map do |hash|
+        next if hash[:after].nil?
+        conditions = []
+        conditions << disable_actions_with_periods unless hash[:override_actions]
+        conditions << hash[:filter] if hash[:filter]
+        hash[:states] = [] if hash[:states] == 'all'
+        hash[:filter] = conditions.map { |condition| "(#{condition})" }.join(' AND ')
+        hash
+      end.compact
+    end
+
     attr_reader :filter, :after, :states, :verbose, :batch_size, :noop, :full_filter
 
     # @param filter [String] scoped search matching the tasks to be deleted
@@ -56,7 +76,8 @@ module ForemanTasks
                           :verbose      => false,
                           :batch_size   => 1000,
                           :noop         => false,
-                          :states       => ['stopped'] }
+                          :states       => ['stopped'],
+                          :backup_dir   => ForemanTasks.dynflow.world.persistence.current_backup_dir }
       options         = default_options.merge(options)
 
       @filter         = options[:filter]
@@ -65,6 +86,7 @@ module ForemanTasks
       @verbose        = options[:verbose]
       @batch_size     = options[:batch_size]
       @noop           = options[:noop]
+      @backup_dir     = options[:backup_dir]
 
       raise ArgumentError, 'filter not speficied' if @filter.nil?
 
@@ -91,12 +113,33 @@ module ForemanTasks
     end
 
     def delete_tasks(chunk)
-      ForemanTasks::Task.where(:id => chunk.map(&:id)).delete_all
+      tasks = ForemanTasks::Task.where(:id => chunk.map(&:id))
+      tasks_to_csv(tasks, @backup_dir, 'foreman_tasks.csv') if @backup_dir
+      tasks.delete_all
+    end
+
+    def tasks_to_csv(dataset, backup_dir, file_name)
+      with_backup_file(backup_dir, file_name) do |csv, appending|
+        csv << ForemanTasks::Task.attribute_names.to_csv unless appending
+        dataset.each do |row|
+          csv << row.attributes.values.to_csv
+        end
+      end
+      dataset
+    end
+
+    def with_backup_file(backup_dir, file_name)
+      FileUtils.mkdir_p(backup_dir) unless File.directory?(backup_dir)
+      csv_file = File.join(backup_dir, file_name)
+      appending = File.exist?(csv_file)
+      File.open(csv_file, 'a') do |f|
+        yield f, appending
+      end
     end
 
     def delete_dynflow_plans(chunk)
       dynflow_ids = chunk.find_all { |task| task.is_a? Task::DynflowTask }.map(&:external_id)
-      ForemanTasks.dynflow.world.persistence.delete_execution_plans({ 'uuid' => dynflow_ids }, batch_size)
+      ForemanTasks.dynflow.world.persistence.delete_execution_plans({ 'uuid' => dynflow_ids }, batch_size, @backup_dir)
     end
 
     def prepare_filter

data/lib/foreman_tasks/dynflow/configuration.rb

@@ -103,6 +103,8 @@ module ForemanTasks
         config.transaction_adapter = transaction_adapter
         config.executor            = ->(world, _) { initialize_executor(world) }
         config.connector           = ->(world, _) { initialize_connector(world) }
+        config.backup_deleted_plans = backup_settings[:backup_deleted_plans]
+        config.backup_dir           = backup_settings[:backup_dir]
 
         # we can't do any operation until the ForemanTasks.dynflow.world is set
         config.auto_execute        = false
@@ -144,5 +146,29 @@ module ForemanTasks
     def initialize_persistence
       ForemanTasks::Dynflow::Persistence.new(default_sequel_adapter_options)
     end
+
+    def backup_settings
+      return @backup_settings if @backup_settings
+      backup_options = {
+        :backup_deleted_plans => true,
+        :backup_dir => default_backup_dir
+      }
+      settings = SETTINGS[:'foreman-tasks'] && SETTINGS[:'foreman-tasks'][:backup]
+      backup_options.merge!(settings) if settings
+      @backup_settings = with_environment_override backup_options
+    end
+
+    def default_backup_dir
+      File.join(Rails.root, 'tmp', 'task-backup')
+    end
+
+    def with_environment_override(options)
+      env_var = ENV['TASK_BACKUP']
+      unless env_var.nil?
+        # Everything except 0, n, no, false is considered to be a truthy value
+        options[:backup_deleted_plans] = !%w[0 n no false].include?(env_var.downcase)
+      end
+      options
+    end
   end
 end

data/lib/foreman_tasks/tasks/cleanup.rake

@@ -5,10 +5,11 @@ namespace :foreman_tasks do
 
         * TASK_SEARCH : scoped search filter (example: 'label = "Actions::Foreman::Host::ImportFacts"')
         * AFTER       : delete tasks created after *AFTER* period. Expected format is a number followed by the time unit (s,h,m,y), such as '10d' for 10 days
-        * STATES      : comma separated list of task states to touch with the cleanup, by default only stopped tasks are covered
-        * NOOP        : set to "true" if the task should not actuall perform the deletion
+        * STATES      : comma separated list of task states to touch with the cleanup, by default only stopped tasks are covered, special value all can be used to clean the tasks, disregarding their states
+        * NOOP        : set to "true" if the task should not actually perform the deletion
         * VERBOSE     : set to "true" for more verbose output
         * BATCH_SIZE  : the size of batches the tasks get processed in (1000 by default)
+        * TASK_BACKUP : set to "true" or "false" to enable/disable task backup
 
       If none of TASK_SEARCH, BEFORE, STATES is specified, the tasks will be cleaned based
       configuration in settings
@@ -21,6 +22,7 @@ namespace :foreman_tasks do
       options[:after] = ENV['AFTER'] if ENV['AFTER']
 
       options[:states] = ENV['STATES'].to_s.split(',') if ENV['STATES']
+      options[:states] = [] if options[:states] == ['all']
 
       options[:noop] = true if ENV['NOOP']
 
@@ -52,6 +54,24 @@ namespace :foreman_tasks do
           printf("%-50s %s\n", action.name, after)
         end
       end
+      puts
+      by_rules = ForemanTasks::Cleaner.actions_by_rules(ForemanTasks::Cleaner.actions_with_default_cleanup)
+      if by_rules.empty?
+        puts _('No cleanup rules are configured')
+      else
+        printf("%-50s %-15s %s\n", _('states'), _('delete after'), _('filter'))
+        by_rules.each do |hash|
+          state = case hash[:states]
+                  when []
+                    _('ANY')
+                  when nil
+                    'stopped'
+                  else
+                    hash[:states]
+                  end
+          printf("%-50s %-15s %s\n", state, hash[:after], hash[:filter])
+        end
+      end
     end
   end
 

data/lib/foreman_tasks/version.rb

@@ -1,3 +1,3 @@
 module ForemanTasks
-  VERSION = '0.9.5'.freeze
+  VERSION = '0.9.6'.freeze
 end

data/test/unit/cleaner_test.rb

@@ -1,6 +1,11 @@
 require 'foreman_tasks_test_helper'
 
 class TasksTest < ActiveSupport::TestCase
+  before do
+    # To stop dynflow from backing up actions, execution_plans and steps
+    ForemanTasks.dynflow.world.persistence.adapter.stubs(:backup_to_csv)
+  end
+
   describe ForemanTasks::Cleaner do
     it 'is able to delete tasks (including the dynflow plans) based on filter' do
       cleaner = ForemanTasks::Cleaner.new(:filter => 'label = "Actions::User::Create"', :after => '10d')
@@ -12,6 +17,7 @@ class TasksTest < ActiveSupport::TestCase
                            task.save
                          end,
                          FactoryGirl.create(:dynflow_task, :product_create_task)]
+      cleaner.expects(:tasks_to_csv)
       cleaner.delete
       ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
       ForemanTasks::Task.where(id: tasks_to_keep).order(:id).map(&:id).must_equal tasks_to_keep.map(&:id).sort
@@ -32,6 +38,7 @@ class TasksTest < ActiveSupport::TestCase
                          end]
 
       tasks_to_keep   = [FactoryGirl.create(:dynflow_task, :product_create_task)]
+      cleaner.expects(:tasks_to_csv)
       cleaner.delete
       ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
       ForemanTasks::Task.where(id: tasks_to_keep).must_equal tasks_to_keep
@@ -46,11 +53,32 @@ class TasksTest < ActiveSupport::TestCase
                            task.started_at = task.ended_at = Time.zone.now
                            task.save
                          end]
+      cleaner.expects(:tasks_to_csv)
       cleaner.delete
       ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
       ForemanTasks::Task.where(id: tasks_to_keep).must_equal tasks_to_keep
     end
 
+    it 'backs tasks up before deleting' do
+      dir = '/tmp'
+      cleaner = ForemanTasks::Cleaner.new(:filter => '', :after => '10d', :backup_dir => dir)
+      tasks_to_delete = [FactoryGirl.create(:dynflow_task, :user_create_task),
+                         FactoryGirl.create(:dynflow_task, :product_create_task)]
+
+      r, w = IO.pipe
+      cleaner.expects(:with_backup_file)
+             .with(dir, 'foreman_tasks.csv')
+             .yields(w, false)
+      cleaner.delete
+      w.close
+      header, *data = r.readlines.map(&:chomp)
+      header.must_equal ForemanTasks::Task.attribute_names.join(',')
+      expected_lines = tasks_to_delete.map { |task| task.attributes.values.join(',') }
+      data.count.must_equal expected_lines.count
+      expected_lines.each { |line| data.must_include line }
+      ForemanTasks::Task.where(id: tasks_to_delete).must_be_empty
+    end
+
     class ActionWithCleanup < Actions::Base
       def self.cleanup_after
         '15d'
@@ -68,6 +96,29 @@ class TasksTest < ActiveSupport::TestCase
                                      { :actions => [{ :name => ActionWithCleanup.name, :after => '5d' }] })
         ForemanTasks::Cleaner.actions_with_default_cleanup[ActionWithCleanup].must_equal '5d'
       end
+
+      it 'deprecates the usage of :after' do
+        Foreman::Deprecation.expects(:deprecation_warning)
+        ForemanTasks::Cleaner.any_instance.expects(:delete)
+        ForemanTasks::Cleaner.stubs(:cleanup_settings =>
+                                    { :after => '1d' })
+        ForemanTasks::Cleaner.stubs(:actions_with_default_cleanup).returns({})
+        ForemanTasks::Cleaner.run({})
+      end
+
+      it 'generates filters from rules properly' do
+        actions_with_default = { 'action1' => nil, 'action2' => nil }
+        rules = [{ :after => nil },
+                 { :after => '10d', :filter => 'label = something', :states => %w[stopped paused] },
+                 { :after => '15d', :filter => 'label = something_else',
+                   :override_actions => true, :states => 'all' }]
+        ForemanTasks::Cleaner.stubs(:cleanup_settings).returns(:rules => rules)
+        r1, r2 = ForemanTasks::Cleaner.actions_by_rules actions_with_default
+        r1[:filter].must_equal '(label !^ (action1, action2)) AND (label = something)'
+        r1[:states].must_equal %w[stopped paused]
+        r2[:filter].must_equal '(label = something_else)'
+        r2[:states].must_equal []
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: foreman-tasks
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.6
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-03 00:00:00.000000000 Z
+date: 2017-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: foreman-tasks-core
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.26
+        version: 0.8.29
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.26
+        version: 0.8.29
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -286,7 +286,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: Foreman plugin for showing tasks information for resoruces and users