gat 0.2.8 → 0.2.11

data/lib/gat/version.rb

@@ -1,20 +1,22 @@
 =begin
   This file is part of GAT: http://gat.rubyforge.org
-  
+
   (c) 2008-2009 A.C. Gnoxys info@gnoxys.net
-   
+
   GAT (Gnoxys Administration Tools) is released under the GPL License 3 or higher.
   You can get a copy of the license easily at http://www.gnu.org/licenses/gpl.html.
-  
+
   THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 =end
 
 module Gat
-  VERSION = '0.2.8'
+
+  VERSION = '0.2.11'
+
 end

data/lib/gatgets/dar_backup/dar_backup.rb

@@ -16,38 +16,38 @@
 =end
 
 require 'gat'
+require 'gatgets/dar_backup/system_backup'
 
 GATGET_DAR_BACKUP_ROOT =  File.expand_path(File.dirname(__FILE__))
 
 class GatgetDarBackup < Gat::Base
 
+  ####################
+  ## Gatget Helpers ##
+  ####################
 
-  # helpers
   def set_flag_mysql_dumps
     set_flag("active_mysql_dumps", get_dependence_value("static", "mysql_dumps"))
   end
 
-  def set_flag_catalog_exists
-    catalog_name = "bbdd_#{ self.config['server_name'] || 'backup' }"
-    set_dependence_variable("catalog_name", catalog_name)
-    set_dependence_variable("catalog_path", "#{ get_dependence_value("folders", "backups_folder_catalog") }/#{ catalog_name }")
-    set_flag("catalog_exists", File.exists?(get_dependence_variable("catalog_path")))
-  end
-
-  # get all the system backups and catalogs backups
+  # get all the system backups
   def get_all_backups
-    get_system_backups
-    get_catalog_backups
+    backups_path = get_dependence_value("folders", "backups_folder_data_local")
+    set_dependence_variable("system_backups", SystemBackup.find(:all, backups_path))
   end
 
   # We are going to set the backups variables. Which ones must be deleted and the name of the current backup
   def set_do_backup_variables
-    # if we dont have a backup
-    if not get_current_complete_backup
+
+    backups_path        = get_dependence_value("folders", "backups_folder_data_local")
+    last_complet_backup = SystemBackup.find(:last, backups_path, { :current => true, :type => 'complet'})
+
+
+    if not last_complet_backup
 
       # * perform a complet backup
       # * set the minimum space required at 1 kb
-      # * mark backups to delete, if nothing to delete,system will get it
+      # * mark backups to delete, if nothing to delete, system will get it
       # * create new dir and link it to current through new_link_current flag
 
       set_variables_to_complete_backup
@@ -98,12 +98,13 @@ class GatgetDarBackup < Gat::Base
 
   end
 
-  # ruby methods
-  def list_backups_data
-  end
+  ################################
+  ## Gatget RubyMethods Actions ##
+  ################################
 
-  def show_nice_search
-    get_catalog_backups
+
+  # list_backups_data is empty. SystemBackup.find(:all) is launched at required helper get_all_backups
+  def list_backups_data
   end
 
   # We need to have enough space to do backup
@@ -111,13 +112,56 @@ class GatgetDarBackup < Gat::Base
 
     last_backup_size = get_dependence_variable("last_backup_size")
     backup_folder    = get_dependence_value("folders", "backups_folder_data_local")
-    thresold       = get_dependence_value("static", "space_thresold").to_i * 1024 * 1024 # to Bytes
+    thresold         = get_dependence_value("static", "space_thresold").to_i * 1024 * 1024 # to Bytes
     # check it
     if not check_enough_space({'where' => backup_folder, 'space_needed' => last_backup_size}) and not check_enough_space({'where' => backup_folder, 'space_needed' => thresold})
       raise Gat::GatgetProcessException.new("System doesnt have enough space", "check_enough_space_for_backup")
     end
   end
 
+  def search_in_backups
+
+    backup_folder = get_dependence_value("folders", "backups_folder_data_local")
+    all_backups   = get_dependence_variable("system_backups")
+    search_what   = get_dependence_value("arguments", "search_pattern")
+
+    something_found = false
+
+    all_backups.each do |backup|
+
+      # do we have already the dumped content??
+      unless backup.dump
+        self.logger.log("warning", "process_action", "Backup #{ backup.name } doesnt have a valid dump content. Dumping content")
+        backup_path = "#{ backup_folder }/#{ backup.folder }/#{ backup.name }"
+        set_dependence_variable("backup_path", backup_path)
+
+        dumping_action = Gat::Action::ShellCommand.new('dump_backup_content_to_file', get_element_config('actions', 'dump_backup_content_to_file'), self.operation)
+
+        begin
+          dumping_action.execute
+        rescue => error
+          raise Gat::GatgetProcessException.new("Unable to dump content for backup #{ backup_path}", "search_in_backups")
+        end
+      end
+
+      # now, search in backup the patern
+      results_for_backup = backup.search_pattern(search_what)
+
+      if results_for_backup.blank?
+        self.logger.log("trace", "process_action", "Backup #{ backup.name } doesnt contain pattern #{ search_what }")
+      else
+        something_found = true
+        self.logger.log("direct", "process_action", "\n\nBackup #{ backup.name } contain pattern '#{ search_what }'\n")
+        self.logger.log("direct", "process_action", "#{ results_for_backup.gsub(/^\[/, " » [") }")
+      end
+    end
+
+    unless something_found
+      self.logger.log("direct", "process_action", "\n Sorry. Pattern #{ search_what } not found in stored backups.\n Try to simplify search, for example, seach 'bin' instead of 'bin/ls'\n\n")
+    end
+  end
+
+
   # failed action launch_backup_rescue callback
   # Dar, when the backup is so big, usually end with a 2 status, that means some files changed during backup operations. That
   # is normal while is working over logs, mysql, etc...
@@ -154,7 +198,7 @@ class GatgetDarBackup < Gat::Base
   def onfail
 
     unless self.config['email_config']
-      raise Gat::GatgetConfigException.new("Default email config is empty at gatget.config['email_config']", "run_output_email")
+      raise Gat::GatgetConfigException.new("Default email config is empty at gatget.config['email_config']", "dar_backup:onfail")
     end
 
     email_subject = "[GAT] [#{ self.class.name }] [#{ self.operation.name.camelize }] [FAILED]"
@@ -170,82 +214,16 @@ class GatgetDarBackup < Gat::Base
 
   end
 
-  # nothing to delete if catalog is empty
-  def catalog_is_empty
-    set_flag("catalog_is_not_empty", get_dependence_variable("catalog_backups").any?)
-  end
-
   def print_status
     if check_i_am_running
       ret = "running"
     else
       ret = "not running"
     end
-    puts ret
     Gat::Action::Base.get_current_action.output = ret
   end
 
   private
-  def get_system_backups
-    backup_entries    = []
-    local_data_folder = get_dependence_value("folders", "backups_folder_data_local")
-
-    # get the current folder
-    current_folder = nil
-    if File.exists?("#{ local_data_folder }/current")
-      # on some S.O, readlink return folder with last slash. remove it anyway.
-      current_folder = File.readlink("#{ local_data_folder }/current").gsub(/\//,'')
-    end
-
-    (Dir.entries(local_data_folder) - ['current', '.', '..']).each do |backup_folder|
-      backup_folder_path = "#{ local_data_folder }/#{ backup_folder }"
-      (Dir.entries(backup_folder_path) - ['..', '.' ]).each do |backup_file|
-        backup_entries << { 'type'    => backup_file.include?('complet') ? 'complet' : 'diff',
-                            'name'    => backup_file,
-                            'path'    => "#{ local_data_folder }/#{ backup_folder }/#{ backup_file }",
-                            'dar_name'=> backup_file.split('.').first,
-                            'isolate' => backup_file.include?('isolate'),
-                            'folder'  => backup_folder,
-                            'current' => backup_folder == current_folder }
-      end
-    end
-    set_dependence_variable("system_backups", backup_entries)
-  end
-
-  def get_catalog_backups
-    backup_entries    = []
-    # define a new action to get all the catalogs in database
-    action = Gat::Action::ShellCommand.new('identify_all_backups_in_catalog', get_element_config('actions', 'identify_backups_in_catalog'), self.operation)
-    action.execute
-
-    backup_in_catalog = action.output
-    backup_in_catalog.each_line do |line|
-      if line.include?('complet') or line.include?('diff')
-        line = line.strip
-        backup_entries << { 'type'    => line.include?('complet') ? 'complet' : 'diff',
-                            'dar_name'=> line.split(' ')[2],
-                            'id'      => line.split(' ')[0],
-                            'folder'  => line.split(' ')[1] }
-      end
-    end
-    set_dependence_variable("catalog_backups", backup_entries)
-  end
-
-  def get_current_complete_backup
-    stored_backups      = get_dependence_variable("system_backups")
-
-    # select complete and current
-    complet_and_current = stored_backups.select { |backup| backup['type'] == 'complet' and backup['current'] and !backup['isolate']}
-    if complet_and_current.empty?
-      complet_backup_file = false
-    else
-      set_dependence_variable("backup_complet_folder", complet_and_current.first['folder'] )
-      set_dependence_variable("backup_complet_file", complet_and_current.first['name'].split('.').first)
-      complet_backup_file = true
-    end
-    complet_backup_file
-  end
-
   def set_variables_to_complete_backup
     backups_folder = get_dependence_value("folders", "backups_folder_data_local")
     backup_folder = "#{ Date.today }"
@@ -258,62 +236,43 @@ class GatgetDarBackup < Gat::Base
     set_dependence_variable("backup_path", backup_path)
   end
 
+
   def set_variables_to_diff_backup
-    backups_folder = get_dependence_value("folders", "backups_folder_data_local")
-    backup_folder  = get_dependence_variable("backup_complet_folder")
-    backup_name    = "#{ Date.today }_#{ self.config['server_name'] || 'backup' }_diff"
-    backup_path    = "#{ backups_folder }/#{ backup_folder }/#{ backup_name }"
-    backup_complet = "#{ backups_folder }/#{ backup_folder }/#{ get_dependence_variable("backup_complet_file") }"
+    backups_path     = get_dependence_value("folders", "backups_folder_data_local")
+    complete_backup  = SystemBackup.find(:last, backups_path, { :current => true, :type => 'complet'})
+    diff_backup_name = "#{ Date.today }_#{ self.config['server_name'] || 'backup' }_diff"
+    diff_backup_path = "#{ backups_path }/#{ complete_backup.folder }/#{ diff_backup_name }"
     # set the variables
-    set_dependence_variable("backup_launch_argument", "-c #{ backup_path } -A #{ backup_complet }")
-    set_dependence_variable("backup_folder", backup_folder)
-    set_dependence_variable("backup_name", backup_name)
-    set_dependence_variable("backup_path", backup_path)
+    set_dependence_variable("backup_launch_argument", "-c #{ diff_backup_path } -A #{ complete_backup.full_dar_path }")
+    set_dependence_variable("backup_folder", complete_backup.folder)
+    set_dependence_variable("backup_name", diff_backup_name)
+    set_dependence_variable("backup_path", diff_backup_path)
   end
 
   def set_backups_to_delete
-    folders_backups_to_delete   = []
-    catalog_backups_to_delete   = []
-    number_stored_backups       = get_dependence_value("static", "number_backup_stored")
-    stored_entries              = Dir.entries(get_dependence_value("folders", "backups_folder_data_local")) - [".","..","current"]
-    catalog_backups             = get_dependence_variable("catalog_backups")
-    (stored_entries.sort.reverse[number_stored_backups..stored_entries.size] || []).each do |backup|
-      folders_backups_to_delete << backup
-      cbtd = catalog_backups.select { |b| b['folder'] == "#{ get_dependence_value("folders", "backups_folder_data_local") }/#{ backup }" }
-      if cbtd.any?
-        catalog_backups_to_delete << cbtd
-      end
-    end
-    # set dependence_variable to delete system folders
-    set_dependence_variable("backup_folders_to_delete", folders_backups_to_delete)
-
-    # anyway, need to delete?
-    set_flag("delete_system_backups_if_any",  folders_backups_to_delete.any?)
+    number_stored_backups   = get_dependence_value("static", "number_backup_stored")
+    backups_folder          = get_dependence_value("folders", "backups_folder_data_local")
 
-    # set depedendece variable to delete catalog files
-    set_dependence_variable("backup_catalog_to_delete", catalog_backups_to_delete.flatten.uniq.map{ |b| '1' })
-    # any way, need to delete?
-    set_flag("delete_catalog_backups_if_any", catalog_backups_to_delete.any?)
+    # get all complete folders backups, in reverse order, olders as first
+    complete_backup_folders = SystemBackup.find(:all, backups_folder, { :type => 'complet' }).map(&:folder).sort.reverse || []
 
+    if complete_backup_folders.any?
+      delete_complete_folders = complete_backup_folders[number_stored_backups..complete_backup_folders.count] || []
+      set_dependence_variable("backup_folders_to_delete", delete_complete_folders)
+      set_flag("delete_system_backups_if_any", delete_complete_folders.any?)
+    else
+      set_dependence_variable("backup_folders_to_delete", [])
+      set_flag("delete_system_backups_if_any", false)
+    end
   end
 
   def get_last_backup_size(backup_type)
     size           = 0
-    system_backups = get_dependence_variable("system_backups")
-    if backup_type == 'complet'
-      complet_and_current = system_backups.select { |backup| backup['type'] == 'complet' and backup['current']}
-      complet_and_current.each do |backup_entry|
-        size += File.size("#{ get_dependence_value("folders","backups_folder_data_local") }/current/#{ backup_entry['name'] }")
-      end
-    elsif backup_type == 'diff'
-      # get the last diff backups
-      last_diff_backups = system_backups.select{ |backup| backup['type'] == 'diff' and not backup['isolate'] }
-      # select all the backup files with the same dar_name and calculate size if we have a last_diff_backup
-      if last_diff_backups and last_diff_backups.any?
-        system_backups.select { |backup| backup['dar_name'] == last_diff_backups.last['dar_name'] }.each do |backup_entry|
-          size += File.size("#{ get_dependence_value("folders","backups_folder_data_local") }/#{ backup_entry ['folder'] }/#{ backup_entry['name'] }")
-        end
-      end
+    backups_folder = get_dependence_value("folders", "backups_folder_data_local")
+    last_backup    = SystemBackup.find(:last, backups_folder, { :type => backup_type })
+
+    if last_backup
+      size = last_backup.size
     end
     size
   end
@@ -355,7 +314,7 @@ module Gat
             status = 99 if status == 0
             add_next_line = true
           else
-            ret_output = line + "\n"
+            ret_output << line + "\n"
           end
         end
         return ret_output, ret_error.join(". \n"), status

data/lib/gatgets/dar_backup/dar_backup.yml

@@ -113,6 +113,9 @@ parameters:
   backups_folder_restore:
     type: dependence
     value: folders/backups_folder_restore
+  backup_name:
+    type: dependence
+    value: variable/backup_name
   backup_path:
     type: dependence
     value: variable/backup_path
@@ -120,13 +123,9 @@ parameters:
     type: dependence
     value: variable/system_backups
     array_method: "select{|b| b['isolate'] }.map{|b| b['folder'] + '/' + b['dar_name'].gsub(/_isolate$/,'') }"
-  catalog_path:
-    type: dependence
-    value: variable/catalog_path
-  catalog_entries:
+  catalog_folder:
     type: dependence
-    value: variable/catalog_backups
-    array_method: "map{|b| '1' }"
+    value: folders/backups_folder_catalog
   complete_folder:
     type: dependence
     value: variable/backup_folder
@@ -190,9 +189,6 @@ parameters:
   pattern_to_restore:
     type: dependence
     value: arguments/pattern_to_restore
-  old_backups_at_catalog_ids:
-    type: dependence
-    value: variable/backup_catalog_to_delete
   old_system_backups:
     type: dependence
     value: variable/backup_folders_to_delete
@@ -229,11 +225,6 @@ actions:
     flags: [ delete_old_backups, delete_system_backups_if_any ]
     syntax: 'rm -rf {{backups_folder_data_local}}/{{old_system_backups}}'
     multiple: [ old_system_backups ]
-  remove_old_catalog_backups:
-    type: shell_command
-    flags: [ delete_old_backups, catalog_exists, delete_catalog_backups_if_any ]
-    syntax: '{{dar_manager_program}} -B {{catalog_path}} -D {{old_backups_at_catalog_ids}}'
-    multiple: [ old_backups_at_catalog_ids ]
   remove_old_current_link:
     type: shell_command
     flags: [ delete_current_link ]
@@ -258,34 +249,8 @@ actions:
   create_isolation_file:
     type: shell_command
     syntax: "{{dar_program}} -w -Q -C '{{backup_path}}_isolate' -A '{{backup_path}}' -J {{dar_decrypt_password}}"
-  identify_backups_in_catalog:
-    type: shell_command
-    flags: [ catalog_exists ]
-    syntax: '{{dar_manager_program}} -B {{catalog_path}} -l'
-  create_catalog:
-    type: shell_command
-    flags: [ '!catalog_exists' ]
-    syntax: '{{dar_manager_program}} -C {{catalog_path}}'
-  insert_in_catalog:
-    type: shell_command
-    syntax: "{{dar_manager_program}} -Q -B {{catalog_path}} -A '{{backup_path}}_isolate' '{{backup_path}}'"
   list_backups_data:
     type: ruby_method
-  delete_all_backups_at_catalog:
-    type: shell_command
-    flags: [ catalog_exists, catalog_is_not_empty ]
-    syntax: '{{dar_manager_program}} -B {{catalog_path}} -D {{catalog_entries}}'
-    multiple:  [ catalog_entries ]
-  insert_all_backups_at_catalog:
-    type: shell_command
-    flags: [ catalog_exists ]
-    syntax: "{{dar_manager_program}} -Q -B {{catalog_path}} -A '{{backups_folder_data_local}}/{{backups_path_for_dar}}_isolate' '{{backups_folder_data_local}}/{{backups_path_for_dar}}'"
-    multiple:  [ backups_path_for_dar ]
-  search_simple:
-    type: shell_command
-    flags: [ catalog_exists ]
-    syntax: "{{dar_manager_program}} -B {{catalog_path}} -f {{pattern_simple_for_search}}"
-    onfail: continue
   show_nice_search:
     type: ruby_method
   show_backup_content:
@@ -309,6 +274,11 @@ actions:
     onfail_message: 'I am already running, I will exit now'
   print_status:
     type: ruby_method
+  dump_backup_content_to_file:
+    type: shell_command
+    syntax: "{{dar_program}} -l {{backup_path}} -K {{dar_decrypt_password}} 1> {{backup_path}}_dump"
+  search_in_backups:
+    type: ruby_method
 
 operations:
   do_backup:
@@ -317,46 +287,35 @@ operations:
     programs:   [ dar, dar_manager, mysql, mysqldump, gzip ]
     arguments:  [ ]
     folders:    [ backups_folder ]
-    helpers:    [ exit_if_i_am_running, set_flag_mysql_dumps, set_flag_catalog_exists, get_all_backups, set_do_backup_variables ]
+    helpers:    [ exit_if_i_am_running, set_flag_mysql_dumps, get_all_backups, set_do_backup_variables ]
     actions:    [ delete_old_mysql_dumps, get_mysql_databases, do_mysql_dumps, compress_mysql_dumps,
-                  remove_old_catalog_backups, remove_old_system_backups, remove_old_current_link,
+                  remove_old_system_backups, remove_old_current_link,
                   check_enough_space_for_backup, create_new_complet_folder, set_new_current_link,
-                  launch_backup, test_backup,
-                  create_isolation_file, create_catalog, insert_in_catalog ]
-    outputs:    [ ]
+                  launch_backup, test_backup, create_isolation_file, dump_backup_content_to_file  ]
+    outputs:    [ email ]
 
   list_backups:
-    description: 'List avalaible backups at System && Catalog'
+    description: 'List avalaible backups at System'
     users:      [ root ]
     programs:   [ dar_manager ]
     arguments:  [  ]
     folders:    [ backups_folder ]
-    helpers:    [ set_flag_catalog_exists, get_all_backups ]
+    helpers:    [ get_all_backups ]
     conditions: [ ]
     actions:    [ list_backups_data ]
     onfail:     [ ]
     outputs:    [ template ]
 
-  syncronize_backups:
-    description: 'Syncronize catalog with avalaible system backups'
-    users:      [ root ]
-    programs:   [ dar_manager ]
-    arguments:  [  ]
-    folders:    [ backups_folder ]
-    helpers:    [ exit_if_i_am_running, set_flag_catalog_exists, get_all_backups, catalog_is_empty ]
-    conditions: [ ]
-    actions:    [ delete_all_backups_at_catalog, insert_all_backups_at_catalog ]
-
   search:
     description: 'Search files to restore in backups'
     users:      [ root ]
-    programs:   [ dar_manager ]
+    programs:   [ dar ]
     arguments:  [ search_pattern ]
     folders:    [ backups_folder ]
-    helpers:    [ set_flag_catalog_exists ]
-    outputs:    [ template, yml ]
+    helpers:    [ get_all_backups ]
+    outputs:    [  ]
     conditions: [ ]
-    actions:    [ search_simple, show_nice_search ]
+    actions:    [ search_in_backups ]
 
   show_backup_content:
     description: 'Show all the backup contents. Redirect to a file if your backup content has many information'