sweety_backy 0.0.19 → 0.0.21

data/Gemfile.lock

@@ -1,28 +1,30 @@
 PATH
   remote: .
   specs:
-    sweety_backy (0.0.15)
+    sweety_backy (0.0.19)
       aws-sdk
 
 GEM
   remote: http://rubygems.org/
   specs:
-    aws-sdk (1.1.1)
+    aws-sdk (1.7.1)
       httparty (~> 0.7)
       json (~> 1.4)
       nokogiri (>= 1.4.4)
       uuidtools (~> 2.1)
     chronic (0.6.2)
-    crack (0.1.8)
     delorean (1.1.0)
       chronic
-    httparty (0.7.8)
-      crack (= 0.1.8)
-    json (1.5.3)
+    httparty (0.9.0)
+      multi_json (~> 1.0)
+      multi_xml
+    json (1.7.5)
     mocha (0.9.12)
-    nokogiri (1.5.0)
+    multi_json (1.3.7)
+    multi_xml (0.5.1)
+    nokogiri (1.5.5)
     rake (0.9.2)
-    uuidtools (2.1.2)
+    uuidtools (2.1.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -15,11 +15,11 @@ Please take a look of other **Ruby backups gems**:
 ## How install
 
     gem install 'sweety_backy'
-    
+
 ## How to use it
-    
+
     sweety_backy <config_file>
-    
+
 ### Config file
 
 It is a _yaml_ file with all this attributes
@@ -30,6 +30,7 @@ It is a _yaml_ file with all this attributes
     monthly: <quantity of monthly backups>
     weekly: <quantity of weekly backups>
     daily: <quantity of daily backups>
+    slices_size: <in MB, if present the compressed files will be sliced in pieces>
     database_user: <database user with read privileges of all datases>
     database_pass: <database user password>
     storage_system: { 's3' | 'local' }
@@ -37,16 +38,16 @@ It is a _yaml_ file with all this attributes
       path: <absoulte path to the root folder of the backups>
     s3_opts: (only if the storage_system is 's3')
       bucket: <bucket name>
-      path: <bucket path where the backups will be stored><
+      path: <bucket path where the backups will be stored>
       passwd_file: <path to the S3 credentials>
-    
+
 ### S3 credentials file
 
 It is a _yaml_ file with two keys with the S3 credentials:
 
     access_key_id: "XXX"
     secret_access_key: "YYY"
-    
+
 ### Example
 
 #### S3 config example
@@ -56,7 +57,7 @@ It is a _yaml_ file with two keys with the S3 credentials:
     secret_access_key: "YYY"
 
 
-#### SweetyBacky config example    
+#### SweetyBacky config example
 
     # ~/.sweety_backy.conf
     paths: [ "/Users/fguillen/Develop/Brico", "/Users/fguillen/Develop/Arduino" ]
@@ -65,57 +66,86 @@ It is a _yaml_ file with two keys with the S3 credentials:
     monthly: 2
     weekly: 3
     daily: 4
+    slices_size: 100
     database_user: 'root'
     database_pass: ''
     storage_system: 's3'
-    s3_opts: 
+    s3_opts:
       bucket: 'sweety_backy'
       path: 'fguillen'
       passwd_file: '~/.s3.passwd'
-      
+
 #### Execute
 
     sweety_backy ~/.sweety_backy.conf
-    
+
 #### Result
 
 This will generate a bunch of backups in the _sweety_backy_ bucket like these ones:
 
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110626.weekly.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110703.weekly.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110704.daily.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110705.daily.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110706.daily.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110707.daily.tar.gz
-
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110626.weekly.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110703.weekly.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110704.daily.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110705.daily.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110706.daily.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110707.daily.tar.gz
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110626.weekly.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110626.weekly.tar.gz.part_ab
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110626.weekly.tar.gz.part_ac
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110626.weekly.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110703.weekly.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110703.weekly.tar.gz.part_ab
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110703.weekly.tar.gz.part_ac
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110703.weekly.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110704.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110704.daily.tar.gz.part_ab
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110704.daily.tar.gz.part_ac
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110704.daily.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110705.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110705.daily.tar.gz.part_ab
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110705.daily.tar.gz.part_ac
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110705.daily.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110706.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110706.daily.tar.gz.part_ab
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110706.daily.tar.gz.part_ac
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110706.daily.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110707.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110707.daily.tar.gz.part_ab
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110707.daily.tar.gz.part_ac
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Arduino.20110707.daily.tar.gz.md5
+
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110626.weekly.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110626.weekly.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110703.weekly.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110703.weekly.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110704.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110704.daily.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110705.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110705.daily.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110706.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110706.daily.tar.gz.md5
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110707.daily.tar.gz.part_aa
+    https://s3.amazonaws.com/sweety_backy/fguillen/files/Users.fguillen.Develop.Brico.20110707.daily.tar.gz.md5
 
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110626.weekly.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110703.weekly.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110704.daily.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110705.daily.sql.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110706.daily.sql.tar.gz    
+    https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110706.daily.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/test.20110707.daily.sql.tar.gz
 
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110626.weekly.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110703.weekly.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110704.daily.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110705.daily.sql.tar.gz
-    https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110706.daily.sql.tar.gz    
+    https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110706.daily.sql.tar.gz
     https://s3.amazonaws.com/sweety_backy/fguillen/databases/mysql.20110707.daily.sql.tar.gz
-    
+
 ... and so on.
 
 ### Cron execution example
-    
+
     # every day at 02:00 am
     00 02 * * * sweety_backy /home/fguillen/.sweety_backy.conf >> /var/log/sweety_backy.log 2>&1
 
+### State of development
+
+I'm using it in my prouduction servers, but it can fails and damage your data.. use it at your own risk.
+
 ## License
 
 MIT License. (c) 2011 Fernando Guillen (http://fernandoguillen.info).

data/bin/sweety_backy

@@ -4,23 +4,23 @@
 #     sweety_backy /path/to/sweety_backy.conf
 
 begin
-  require 'sweety_backy'
+  require "sweety_backy"
 rescue LoadError
-  require 'rubygems'
-  require 'sweety_backy'
+  require "rubygems"
+  require "sweety_backy"
 end
 
-require 'benchmark'
+require "benchmark"
 
 if( ARGV[0].nil? )
   SweetyBacky::Utils.log "use: $ sweety_backy <config_file_path>"
   exit 1
 end
 
-lapsus_time = 
+lapsus_time =
   Benchmark.realtime do
     SweetyBacky::Utils.log "--------------------"
-    SweetyBacky::Utils.log "Starting SweetyBacky"
+    SweetyBacky::Utils.log "Starting SweetyBacky v #{SweetyBacky::VERSION}"
     sb = SweetyBacky::Runner.new( ARGV[0] )
     sb.run
   end

data/lib/sweety_backy.rb

@@ -1,11 +1,11 @@
 begin
-  require 'aws-sdk'
+  require "aws-sdk"
 rescue LoadError
-  require 'rubygems'
-  require 'aws-sdk'
+  require "rubygems"
+  require "aws-sdk"
 end
 
-require 'digest/md5'
+require "digest/md5"
 
 require "#{File.dirname(__FILE__)}/sweety_backy/version"
 require "#{File.dirname(__FILE__)}/sweety_backy/runner"

data/lib/sweety_backy/commander.rb

@@ -1,23 +1,23 @@
 module SweetyBacky
-  module Commander    
+  module Commander
     def self.do_files_backup( path, backup_path )
       SweetyBacky::Utils.log "doing files backup of #{path} to #{backup_path}"
-    
+
       FileUtils.mkdir_p( File.dirname( backup_path ) )
       SweetyBacky::Utils::command( "tar -cz --directory #{path} --same-permissions --file #{backup_path} ." )
     end
-  
+
     def self.do_database_backup( database_name, backup_path, opts )
       SweetyBacky::Utils.log "doing database backup #{database_name} on #{backup_path}"
-    
+
       FileUtils.mkdir_p( File.dirname( backup_path ) )
       tmp_sql_file_path = File.join( Dir::tmpdir, "#{File.basename( backup_path, '.tar.gz' )}" )
-      
+
       database_pass = opts[:database_pass].empty? ? '' : "-p'#{opts[:database_pass]}'"
 
       SweetyBacky::Utils::command( "mysqldump -u#{opts[:database_user]} #{database_pass} #{database_name} > #{tmp_sql_file_path}" )
       SweetyBacky::Utils::command( "tar -cz --same-permissions --file #{backup_path} --directory #{File.dirname(tmp_sql_file_path)} #{File.basename(tmp_sql_file_path)}" )
-      
+
       File.delete( tmp_sql_file_path )
     end
 
@@ -25,43 +25,54 @@ module SweetyBacky
       clean_files( opts )
       clean_databases( opts )
     end
-  
+
     def self.clean_files( opts )
       SweetyBacky::Utils.log "cleaning files on #{opts[:target_path]}/files/"
-      
+
+      suffix = opts[:slices_size] ? ".part_*" : ""           # suffix support in case of spliting activate
+      suffix_regex = opts[:slices_size] ? /\.part_.*/ : ""   # suffix support in case of spliting activate
+
       opts[:paths].each do |path|
         SweetyBacky::Utils.log "cleaning file #{path}"
-        
+
         [:yearly, :monthly, :weekly, :daily].each do |period|
-          paths_in( 
-            "#{opts[:target_path]}/files/#{SweetyBacky::Utils.namerize( path )}.*.#{period.to_s}.tar.gz",
+          paths_in(
+            "#{opts[:target_path]}/files/#{SweetyBacky::Utils.namerize( path )}.*.#{period.to_s}.tar.gz#{suffix}",
             opts
-          ).sort[0..(-1*(opts[period]+1))].each do |file_path|
-            remove_path( file_path, opts )
-            remove_path( "#{file_path}.md5", opts ) if exists?( "#{file_path}.md5", opts )
-          end      
+          ).map do |file_name|
+            file_name.match( "files\/#{SweetyBacky::Utils.namerize( path )}.(\\d{8}).#{period.to_s}.tar.gz#{suffix}" )[1]
+          end.uniq.sort[0..(-1*(opts[period]+1))].each do |date_to_remove|
+            paths_in(
+              "#{opts[:target_path]}/files/#{SweetyBacky::Utils.namerize( path )}.#{date_to_remove}.#{period.to_s}.tar.gz#{suffix}",
+              opts
+            ).each do |file_path|
+              Utils.log( "Removing file: #{file_path}" )
+              remove_path( file_path, opts )
+              remove_path( "#{file_path.gsub(suffix_regex, "")}.md5", opts ) if exists?( "#{file_path.gsub(suffix_regex, "")}.md5", opts )
+            end
+          end
         end
       end
     end
-    
+
     def self.clean_databases( opts )
       SweetyBacky::Utils.log "cleaning databases on #{opts[:target_path]}/databases/"
-      
+
       opts[:databases].each do |database_name|
         SweetyBacky::Utils.log "cleaning database #{database_name}"
-        
+
         [:yearly, :monthly, :weekly, :daily].each do |period|
-          paths_in( 
+          paths_in(
             "#{opts[:target_path]}/databases/#{database_name}.*.#{period.to_s}.sql.tar.gz",
             opts
           ).sort[0..(-1*(opts[period]+1))].each do |file_path|
             remove_path( file_path, opts )
             remove_path( "#{file_path}.md5", opts ) if exists?( "#{file_path}.md5", opts )
-          end      
+          end
         end
       end
     end
-    
+
     def self.exists?( path, opts )
       if( opts[:storage_system].to_sym == :s3 )
         return SweetyBacky::S3.exists?( path, opts[:s3_opts] )
@@ -69,7 +80,7 @@ module SweetyBacky
         return File.exists?( path )
       end
     end
-    
+
     def self.paths_in( path, opts )
       if( opts[:storage_system].to_sym == :s3 )
         return SweetyBacky::S3.paths_in( path, opts[:s3_opts] )
@@ -77,7 +88,7 @@ module SweetyBacky
         return Dir.glob( path )
       end
     end
-    
+
     def self.remove_path( path, opts )
       if( opts[:storage_system].to_sym == :s3 )
         SweetyBacky::Utils.log "cleaning: removing #{opts[:s3_opts][:bucket]}/#{path}"
@@ -87,19 +98,24 @@ module SweetyBacky
         File.delete( path )
       end
     end
-    
+
     def self.do_md5( path, md5_path )
       digest = Digest::MD5.new();
-      
+
       File.open( path, 'r' ) do |f|
         f.each_line { |line| digest << line }
       end
 
       result = digest.hexdigest
-      
+
       File.open( md5_path, 'w' ) { |f| f.write result }
-      
+
       return result
     end
+
+    def self.do_slices( file_path, size )
+      SweetyBacky::Utils::command( "split -b #{size}m #{file_path} #{file_path}.part_" )
+      File.delete( file_path )
+    end
   end
 end

data/lib/sweety_backy/runner.rb

@@ -7,15 +7,15 @@ require File.dirname(__FILE__) + "/utils.rb"
 module SweetyBacky
   class Runner
     attr_reader :opts, :results
-    
+
     def initialize( path = nil )
       if( !path.nil? )
         config( SweetyBacky::OptsReader.read_opts( path ) )
       end
-      
+
       @results = []
     end
-  
+
     def config( opts )
       @opts = {
         :paths          => [],
@@ -26,7 +26,7 @@ module SweetyBacky
         :daily          => 4,
         :storage_system => :local
       }.merge( opts )
-      
+
       if( @opts[:storage_system].to_sym == :s3 )
         @opts[:working_path] = File.join( Dir::tmpdir, "sweety_backy_#{Time.now.to_i}" )
         @opts[:target_path]  = @opts[:s3_opts][:path]
@@ -35,82 +35,83 @@ module SweetyBacky
         @opts[:target_path]  = @opts[:local_opts][:path]
       end
     end
-    
+
     def do_backup
       do_files_backup
       do_databases_backup
     end
-    
+
     def do_files_backup
       @opts[:paths].each do |path|
         success     = nil
         backup_path = "#{@opts[:working_path]}/files/#{SweetyBacky::Utils.namerize( path )}.#{Date.today.strftime('%Y%m%d')}.#{SweetyBacky::Utils.period}.tar.gz"
         md5_path    = "#{backup_path}.md5"
-        
+
         begin
           SweetyBacky::Commander.do_files_backup( path, backup_path )
           SweetyBacky::Commander.do_md5( backup_path, md5_path )
-        
+          SweetyBacky::Commander.do_slices( backup_path, opts[:slices_size] ) if opts[:slices_size]
+
           if( @opts[:storage_system].to_sym == :s3 )
             upload_files_backup_to_s3( backup_path, md5_path )
           end
-          
+
           success = true
-          
+
         rescue Exception => e
           Utils.log( "ERROR: backing up file: '#{path}', e: #{e.message}" )
           Utils.log( e.backtrace.join("\n") )
-          
+
           success = false
         end
-        
+
         @results << { :name => "file: #{path}", :success => success }
       end
     end
-    
+
     def do_databases_backup
       @opts[:databases].each do |database_name|
-        
+
         success     = nil
         backup_path = "#{@opts[:working_path]}/databases/#{database_name}.#{Date.today.strftime('%Y%m%d')}.#{SweetyBacky::Utils.period}.sql.tar.gz"
         md5_path    = "#{backup_path}.md5"
-        
+
         begin
           SweetyBacky::Commander.do_database_backup( database_name, backup_path, @opts)
           SweetyBacky::Commander.do_md5( backup_path, md5_path )
-        
+
           if( @opts[:storage_system].to_sym == :s3 )
             upload_databases_backup_to_s3( backup_path, md5_path )
           end
-          
+
           success = true
-          
+
         rescue Exception => e
           Utils.log( "ERROR: backing up database: '#{database_name}', e: #{e.message}" )
           Utils.log( e.backtrace.join("\n") )
-          
+
           success = false
         end
-        
+
         @results << { :name => "database: #{database_name}", :success => success }
       end
     end
-    
 
-    
+
+
     def clean
       SweetyBacky::Commander.clean( @opts )
     end
-    
+
     def print_results
       Utils.log( "RESULTS:" )
       Utils.log( "--------" )
-      
+
       @results.each do |result|
         Utils.log( "#{result[:name]} -> #{result[:success] ? 'OK' : 'ERROR'}" )
       end
     end
-    
+
     def run
       begin
         do_backup
@@ -122,40 +123,43 @@ module SweetyBacky
         SweetyBacky::Utils.log "I should send and email at this moment"
       end
     end
-  
-    private
-    
+
     def upload_databases_backup_to_s3( backup_path, md5_path )
       SweetyBacky::S3.upload(
         backup_path,
         "#{@opts[:target_path]}/databases/#{File.basename( backup_path )}",
         @opts[:s3_opts]
       )
-    
+
       SweetyBacky::S3.upload(
         md5_path,
         "#{@opts[:target_path]}/databases/#{File.basename( md5_path )}",
         @opts[:s3_opts]
       )
-    
+
       FileUtils.rm backup_path
       FileUtils.rm md5_path
     end
-    
+
     def upload_files_backup_to_s3( backup_path, md5_path )
-      SweetyBacky::S3.upload(
-        backup_path,
-        "#{@opts[:target_path]}/files/#{File.basename( backup_path )}",
-        @opts[:s3_opts]
-      )
-    
+      backup_paths = opts[:slices_size] ? Dir.glob( "#{backup_path}.part_*" ) : [backup_path]
+
+      backup_paths.each do |backup_path|
+        SweetyBacky::S3.upload(
+          backup_path,
+          "#{@opts[:target_path]}/files/#{File.basename( backup_path )}",
+          @opts[:s3_opts]
+        )
+
+        FileUtils.rm backup_path
+      end
+
       SweetyBacky::S3.upload(
         md5_path,
         "#{@opts[:target_path]}/files/#{File.basename( md5_path )}",
         @opts[:s3_opts]
       )
-    
-      FileUtils.rm backup_path
+
       FileUtils.rm md5_path
     end
 

data/lib/sweety_backy/s3.rb

@@ -2,34 +2,34 @@ module SweetyBacky
   class S3
     def self.upload( path, s3_path, opts )
       SweetyBacky::Utils.log( "S3 uploading: #{path} to #{opts[:bucket]}/#{s3_path}" )
-      
+
       s3 = AWS::S3.new( read_s3_password( opts[:passwd_file] ) )
       bucket = s3.buckets[ opts[:bucket] ]
-      
+
       if !bucket.exists?
         bucket = s3.buckets.create( opts[:bucket] )
       end
 
-      object = bucket.objects.create( s3_path )
+      object = bucket.objects[ s3_path ]
       object.write( :file => path )
     end
-    
+
     def self.object( path, opts )
       s3 = AWS::S3.new( read_s3_password( opts[:passwd_file] ) )
       bucket = s3.buckets[ opts[:bucket] ]
       object = bucket.objects[ path ]
-      
+
       object
     end
-    
+
     def self.exists?( path, opts )
       return object( path, opts ).exists?
     end
-    
+
     def self.paths_in( path, opts )
       s3 = AWS::S3.new( read_s3_password( opts[:passwd_file] ) )
       bucket = s3.buckets[ opts[:bucket] ]
-      
+
       regex = Regexp.escape( path ).gsub('\*', '.*').gsub('\?', '.')
 
       objects = bucket.objects.select { |e| e.key =~ /^#{regex}$/ }
@@ -37,22 +37,22 @@ module SweetyBacky
 
       return paths
     end
-    
+
     def self.read_s3_password( path )
       opts = YAML.load( File.read( File.expand_path path ) )
       new_opts = {}
-      
+
       # symbolize keys
       opts.keys.each do |key|
         new_opts[key.to_sym] = opts[key]
       end
-      
+
       return new_opts
     end
-    
+
     def self.delete( path, opts )
       SweetyBacky::S3.object( path, opts ).delete
     end
-    
+
   end
 end

data/lib/sweety_backy/version.rb

@@ -1,3 +1,3 @@
 module SweetyBacky
-  VERSION = "0.0.19"
+  VERSION = "0.0.21"
 end

data/sweety_backy.gemspec

@@ -12,13 +12,13 @@ Gem::Specification.new do |s|
   s.description = "Simple mechanism to configure and execute backups of folders and MySQL DBs and store them in local folder or S3 bucket"
 
   s.rubyforge_project = "SweetyBacky"
-  
+
   s.add_development_dependency "bundler", ">= 1.0.0.rc.6"
   s.add_development_dependency "mocha"
   s.add_development_dependency "delorean"
   s.add_development_dependency "rake"
-  
-  s.add_dependency "aws-sdk"
+
+  s.add_dependency "aws-sdk", "1.7.1"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/test/commander_test.rb

@@ -1,25 +1,25 @@
 require "#{File.dirname(__FILE__)}/test_helper"
 
 class CommanderTest < Test::Unit::TestCase
-  
+
   def setup
     SweetyBacky::Utils.stubs(:log)
-    
+
     # tmp dir
     @tmp_dir = File.join( Dir::tmpdir, "sweety_backy_#{Time.now.to_i}" )
     Dir.mkdir( @tmp_dir )  unless File.exists?(@tmp_dir)
   end
-  
+
   def teardown
     FileUtils.rm_rf @tmp_dir  if File.exists?(@tmp_dir)
   end
-  
+
   def test_do_files_backup
-    SweetyBacky::Commander.do_files_backup( 
-      "#{FIXTURES_PATH}/path", 
+    SweetyBacky::Commander.do_files_backup(
+      "#{FIXTURES_PATH}/path",
       "#{@tmp_dir}/back.tar.gz"
     )
-    
+
     result = %x(tar -tzvf #{@tmp_dir}/back.tar.gz)
 
     assert_match( "./", result )
@@ -27,23 +27,23 @@ class CommanderTest < Test::Unit::TestCase
     assert_match( "./a/", result )
     assert_match( "./b/file3.txt", result )
   end
-  
+
   def test_do_databases_backup
-    SweetyBacky::Commander.do_database_backup( 
-      "test",  
+    SweetyBacky::Commander.do_database_backup(
+      "test",
       "#{@tmp_dir}/back.sql.tar.gz",
       {
-        :database_user => "test", 
+        :database_user => "test",
         :database_pass => ""
       }
     )
-    
+
     result = %x(tar -tzvf #{@tmp_dir}/back.sql.tar.gz)
-    
+
     assert_match( /\sback.sql$/, result )
   end
 
-  
+
   def test_clean
     opts = {
       :paths        => [ 'name1', 'name2' ],
@@ -59,10 +59,10 @@ class CommanderTest < Test::Unit::TestCase
       :working_path => @tmp_dir,
       :target_path  => @tmp_dir
     }
-    
+
     Dir.mkdir( "#{@tmp_dir}/files" )  unless File.exists?( "#{@tmp_dir}/files" )
     Dir.mkdir( "#{@tmp_dir}/databases" )  unless File.exists?( "#{@tmp_dir}/databases" )
-    
+
     [
       'name1.20081231.yearly',
       'name1.20081232.yearly',
@@ -85,17 +85,18 @@ class CommanderTest < Test::Unit::TestCase
       'name2.20100726.daily'
     ].each do |file_part|
       File.open( "#{@tmp_dir}/files/#{file_part}.tar.gz", 'w' ) { |f| f.write 'wadus' }
+      File.open( "#{@tmp_dir}/files/#{file_part}.tar.gz.md5", 'w' ) { |f| f.write 'wadus' }
       File.open( "#{@tmp_dir}/databases/#{file_part}.sql.tar.gz", 'w' ) { |f| f.write 'wadus' }
     end
-    
+
     # puts @tmp_dir
     # exit 1
-    
+
     SweetyBacky::Commander.clean( opts )
-    
+
     files_keeped     = Dir.glob( "#{@tmp_dir}/files/*" ).join( "\n" )
     databases_keeped = Dir.glob( "#{@tmp_dir}/databases/*" ).join( "\n" )
-    
+
     # files to keep
     [
       'name1.20081232.yearly',
@@ -113,9 +114,10 @@ class CommanderTest < Test::Unit::TestCase
       'name2.20100726.daily'
     ].each do |file_part|
       assert_match( "#{file_part}.tar.gz", files_keeped )
+      assert_match( "#{file_part}.tar.gz.md5", files_keeped )
       assert_match( "#{file_part}.sql.tar.gz", databases_keeped )
     end
-    
+
     # files to deleted
     [
       'name1.20081231.yearly',
@@ -125,16 +127,109 @@ class CommanderTest < Test::Unit::TestCase
       'name2.20100721.daily'
     ].each do |file_part|
       assert_no_match( /#{file_part}.tar.gz/, files_keeped )
+      assert_no_match( /#{file_part}.tar.gz.md5/, files_keeped )
+      assert_no_match( /#{file_part}.sql.tar.gz/, databases_keeped )
+    end
+  end
+
+  def test_clean_with_part_extensions
+    opts = {
+      :paths        => [ 'name1', 'name2' ],
+      :databases    => [ 'name1', 'name2' ],
+      :yearly       => 1,
+      :monthly      => 2,
+      :weekly       => 3,
+      :daily        => 4,
+      :slices_size  => 10,
+      :storage_system => :local,
+      :local_opts => {
+        :path => @tmp_dir
+      },
+      :working_path => @tmp_dir,
+      :target_path  => @tmp_dir
+    }
+
+    Dir.mkdir( "#{@tmp_dir}/files" )  unless File.exists?( "#{@tmp_dir}/files" )
+    Dir.mkdir( "#{@tmp_dir}/databases" )  unless File.exists?( "#{@tmp_dir}/databases" )
+
+    [
+      'name1.20081231.yearly',
+      'name1.20081232.yearly',
+      'name2.20091231.yearly',
+      'name1.20100131.monthly',
+      'name1.20100228.monthly',
+      'name1.20100331.monthly',
+      'name2.20100430.monthly',
+      'name2.20100531.monthly',
+      'name2.20100630.monthly',
+      'name1.20100704.weekly',
+      'name1.20100711.weekly',
+      'name1.20100718.weekly',
+      'name1.20100725.weekly',
+      'name1.20100720.daily',
+      'name2.20100721.daily',
+      'name2.20100722.daily',
+      'name2.20100723.daily',
+      'name2.20100724.daily',
+      'name2.20100726.daily'
+    ].each do |file_part|
+      File.open( "#{@tmp_dir}/files/#{file_part}.tar.gz.part_aa", 'w' ) { |f| f.write 'wadus' }
+      File.open( "#{@tmp_dir}/files/#{file_part}.tar.gz.part_ab", 'w' ) { |f| f.write 'wadus' }
+      File.open( "#{@tmp_dir}/files/#{file_part}.tar.gz.md5", 'w' ) { |f| f.write 'wadus' }
+      File.open( "#{@tmp_dir}/databases/#{file_part}.sql.tar.gz", 'w' ) { |f| f.write 'wadus' }
+    end
+
+    # puts @tmp_dir
+    # exit 1
+
+    SweetyBacky::Commander.clean( opts )
+
+    files_keeped     = Dir.glob( "#{@tmp_dir}/files/*" ).join( "\n" )
+    databases_keeped = Dir.glob( "#{@tmp_dir}/databases/*" ).join( "\n" )
+
+    # files to keep
+    [
+      'name1.20081232.yearly',
+      'name2.20091231.yearly',
+      'name1.20100228.monthly',
+      'name1.20100331.monthly',
+      'name2.20100531.monthly',
+      'name2.20100630.monthly',
+      'name1.20100718.weekly',
+      'name1.20100725.weekly',
+      'name1.20100720.daily',
+      'name2.20100722.daily',
+      'name2.20100723.daily',
+      'name2.20100724.daily',
+      'name2.20100726.daily'
+    ].each do |file_part|
+      assert_match( "#{file_part}.tar.gz.part_aa", files_keeped )
+      assert_match( "#{file_part}.tar.gz.part_ab", files_keeped )
+      assert_match( "#{file_part}.tar.gz.md5", files_keeped )
+      assert_match( "#{file_part}.sql.tar.gz", databases_keeped )
+    end
+
+    # files to deleted
+    [
+      'name1.20081231.yearly',
+      'name1.20100131.monthly',
+      'name2.20100430.monthly',
+      'name1.20100704.weekly',
+      'name2.20100721.daily'
+    ].each do |file_part|
+      assert_no_match( /#{file_part}.tar.gz.part_aa/, files_keeped )
+      assert_no_match( /#{file_part}.tar.gz.part_ab/, files_keeped )
+      assert_no_match( /#{file_part}.tar.gz.md5/, files_keeped )
       assert_no_match( /#{file_part}.sql.tar.gz/, databases_keeped )
     end
   end
-  
+
   def test_do_md5
     md5_path = "#{@tmp_dir}/#{Time.now.to_i}.md5"
     fixture_file_path = "#{FIXTURES_PATH}/file_for_md5.txt"
-    
+
     md5 = SweetyBacky::Commander.do_md5( fixture_file_path, md5_path )
-    
+
     assert_equal( '75fca8d37b0f7d75d11ccc8d255debe5', md5 )
     assert_equal( '75fca8d37b0f7d75d11ccc8d255debe5', File.read( md5_path ) )
   end

data/test/fixtures/config_s3_slices.yml

@@ -0,0 +1,18 @@
+paths:
+  - "path1"
+  - "path2"
+databases:
+  - "db1"
+  - "db2"
+yearly: 1
+monthly: 2
+weekly: 3
+daily: 4
+slices_size: 10
+database_user: 'database_user'
+database_pass: 'database_pass'
+storage_system: 's3'
+s3_opts:
+  bucket: 'bucket_name'
+  path: 's3/path/path'
+  passwd_file: '/path/.s3.passwd'

data/test/fixtures/slices/file1.txt.md5

@@ -0,0 +1 @@
+wadus

data/test/fixtures/slices/file1.txt.part_aa

@@ -0,0 +1 @@
+wadus

data/test/fixtures/slices/file1.txt.part_ab

@@ -0,0 +1 @@
+wadus

data/test/runner_test.rb

@@ -1,14 +1,14 @@
 require "#{File.dirname(__FILE__)}/test_helper"
 
 class RunnerTest < Test::Unit::TestCase
-  
+
   def setup
     SweetyBacky::Utils.stubs(:log)
 
     # tmp dir
     @tmp_dir = File.join( Dir::tmpdir, "sweety_backy_#{Time.now.to_i}" )
     Dir.mkdir( @tmp_dir )  unless File.exists?(@tmp_dir)
-    
+
     # runner
     @opts = {
       :paths          => [ "#{FIXTURES_PATH}/path" ],
@@ -24,28 +24,28 @@ class RunnerTest < Test::Unit::TestCase
         :path => @tmp_dir
       }
     }
-    
+
     @runner = SweetyBacky::Runner.new
     @runner.config( @opts )
   end
-  
+
   def teardown
     FileUtils.rm_rf @tmp_dir  if File.exists?(@tmp_dir)
   end
-  
+
   def test_do_backup_yearly
     Delorean.time_travel_to( '2009-12-31' ) do
       @runner.do_backup
     end
-    
+
     assert( File.exists?( "#{@tmp_dir}/files/#{SweetyBacky::Utils.namerize( @opts[:paths][0] )}.20091231.yearly.tar.gz" ) )
     assert( File.exists?( "#{@tmp_dir}/databases/test.20091231.yearly.sql.tar.gz" ) )
 
     assert( File.exists?( "#{@tmp_dir}/files/#{SweetyBacky::Utils.namerize( @opts[:paths][0] )}.20091231.yearly.tar.gz.md5" ) )
     assert( File.exists?( "#{@tmp_dir}/databases/test.20091231.yearly.sql.tar.gz.md5" ) )
   end
-  
-  def test_do_backup_monthly    
+
+  def test_do_backup_monthly
     Delorean.time_travel_to( '2010-01-31' ) do
       @runner.do_backup
     end
@@ -53,21 +53,21 @@ class RunnerTest < Test::Unit::TestCase
     assert( File.exists?( "#{@tmp_dir}/files/#{SweetyBacky::Utils.namerize( @opts[:paths][0] )}.20100131.monthly.tar.gz") )
     assert( File.exists?( "#{@tmp_dir}/databases/test.20100131.monthly.sql.tar.gz") )
   end
-  
-  def test_do_backup_weekly    
+
+  def test_do_backup_weekly
     Delorean.time_travel_to( '2010-01-03' ) do
       @runner.do_backup
     end
-    
+
     assert( File.exists?( "#{@tmp_dir}/files/#{SweetyBacky::Utils.namerize( @opts[:paths][0] )}.20100103.weekly.tar.gz") )
     assert( File.exists?( "#{@tmp_dir}/databases/test.20100103.weekly.sql.tar.gz") )
   end
-  
+
   def test_do_backup_daily
     Delorean.time_travel_to( '2010-01-04' ) do
       @runner.do_backup
     end
-    
+
     assert( File.exists?( "#{@tmp_dir}/files/#{SweetyBacky::Utils.namerize( @opts[:paths][0] )}.20100104.daily.tar.gz") )
     assert( File.exists?( "#{@tmp_dir}/databases/test.20100104.daily.sql.tar.gz") )
   end
@@ -76,22 +76,22 @@ class RunnerTest < Test::Unit::TestCase
     @runner.expects(:do_backup)
     @runner.expects(:print_results)
     SweetyBacky::Commander.expects(:clean)
-    
+
     @runner.run
   end
-  
+
   def test_initialize_with_config_file
-    SweetyBacky::OptsReader.expects( :read_opts ).with( '/path/config.yml' ).returns( 
-      { 
+    SweetyBacky::OptsReader.expects( :read_opts ).with( '/path/config.yml' ).returns(
+      {
         :paths => [ 'pepe', 'juan' ],
         :local_opts => {
           :path => '/local/path'
         }
       }
     )
-    
+
     runner = SweetyBacky::Runner.new( "/path/config.yml" )
-    
+
     assert_equal( [ "pepe", "juan" ], runner.opts[:paths] )
     assert_equal( [], runner.opts[:databases] )
     assert_equal( 1, runner.opts[:yearly] )
@@ -103,6 +103,46 @@ class RunnerTest < Test::Unit::TestCase
     assert_equal( '/local/path', runner.opts[:working_path] )
     assert_equal( '/local/path', runner.opts[:target_path] )
   end
-    
+
+  def test_upload_files_backup_to_s3_with_slices
+    @runner.opts[:slices_size] = 10
+    @runner.opts[:s3_opts] = "s3_opts"
+
+    sequence_uploads = sequence( "uploads" )
+    sequence_removes = sequence( "removes" )
+
+    SweetyBacky::S3.expects(:upload).with(
+      "#{FIXTURES_PATH}/slices/file1.txt.part_aa",
+      "#{@tmp_dir}/files/file1.txt.part_aa",
+      @runner.opts[:s3_opts]
+    ).in_sequence( sequence_uploads )
+
+
+    SweetyBacky::S3.expects(:upload).with(
+      "#{FIXTURES_PATH}/slices/file1.txt.part_ab",
+      "#{@tmp_dir}/files/file1.txt.part_ab",
+      @runner.opts[:s3_opts]
+    ).in_sequence( sequence_uploads )
+
+    SweetyBacky::S3.expects(:upload).with(
+      "#{FIXTURES_PATH}/slices/file1.txt.md5",
+      "#{@tmp_dir}/files/file1.txt.md5",
+      @runner.opts[:s3_opts]
+    ).in_sequence( sequence_uploads )
+
+    FileUtils.expects(:rm).with("#{FIXTURES_PATH}/slices/file1.txt.part_aa").in_sequence( sequence_removes )
+    FileUtils.expects(:rm).with("#{FIXTURES_PATH}/slices/file1.txt.part_ab").in_sequence( sequence_removes )
+    FileUtils.expects(:rm).with("#{FIXTURES_PATH}/slices/file1.txt.md5").in_sequence( sequence_removes )
+
+    @runner.upload_files_backup_to_s3(
+      "#{FIXTURES_PATH}/slices/file1.txt",
+      "#{FIXTURES_PATH}/slices/file1.txt.md5"
+    )
+
+
+
+    # FileUtils.expects(:rm).with()
+  end
+
 end
 

data/test/s3/s3_test.rb

@@ -10,57 +10,57 @@ class S3Test < Test::Unit::TestCase
       :path               => 'test_path',
       :passwd_file        => '~/.s3.passwd'
     }
-    
+
     s3 = AWS::S3.new( SweetyBacky::S3.read_s3_password( @opts[:passwd_file] ) )
     @bucket = s3.buckets.create( @opts[:bucket] )
   end
-  
+
   def teardown
     @bucket.delete!
   end
-  
+
   def test_upload
     SweetyBacky::S3.upload(
       "#{FIXTURES_PATH}/file.txt",
       "test/path/file.txt",
       @opts
     )
-    
+
     assert_equal(
       File.read( "#{FIXTURES_PATH}/file.txt" ),
       SweetyBacky::S3.object( "test/path/file.txt", @opts ).read
     )
   end
-  
+
   def test_paths_in
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/file1.txt", @opts )
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/file2.txt", @opts )
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/file3.txt", @opts )
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/other_file.txt", @opts )
-    
+
     paths = SweetyBacky::S3.paths_in( "test/path/file*.txt", @opts )
-    
+
     assert_equal(3, paths.size)
     assert( ( paths.include? "test/path/file1.txt" ) )
     assert( ( paths.include? "test/path/file2.txt" ) )
     assert( ( paths.include? "test/path/file3.txt" ) )
     assert( ( !paths.include? "test/path/other_file.txt" ) )
   end
-  
+
   def test_delete
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/file1.txt", @opts )
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/file2.txt", @opts )
-    
+
     SweetyBacky::S3.delete( "test/path/file2.txt", @opts )
-    
+
     assert( @bucket.objects[ "test/path/file1.txt" ].exists? )
     assert( !@bucket.objects[ "test/path/file2.txt" ].exists? )
   end
-  
+
   def test_exists
     SweetyBacky::S3.upload( "#{FIXTURES_PATH}/file.txt", "test/path/file1.txt", @opts )
     assert_equal( true, SweetyBacky::S3.exists?( "test/path/file1.txt", @opts ) )
     assert_equal( false, SweetyBacky::S3.exists?( "test/path/file2.txt", @opts ) )
   end
-  
+
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: sweety_backy
 version: !ruby/object:Gem::Version 
-  hash: 57
-  prerelease: false
+  hash: 53
+  prerelease: 
   segments: 
   - 0
   - 0
-  - 19
-  version: 0.0.19
+  - 21
+  version: 0.0.21
 platform: ruby
 authors: 
 - Fernando Guillen
@@ -15,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-04 00:00:00 +02:00
-default_executable: 
+date: 2012-11-24 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: bundler
@@ -26,7 +25,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 15424057
+        hash: -2749782956
         segments: 
         - 1
         - 0
@@ -84,12 +83,14 @@ dependencies:
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - "="
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 9
         segments: 
-        - 0
-        version: "0"
+        - 1
+        - 7
+        - 1
+        version: 1.7.1
   type: :runtime
   version_requirements: *id005
 description: Simple mechanism to configure and execute backups of folders and MySQL DBs and store them in local folder or S3 bucket
@@ -119,12 +120,16 @@ files:
 - sweety_backy.gemspec
 - test/commander_test.rb
 - test/fixtures/config_s3.yml
+- test/fixtures/config_s3_slices.yml
 - test/fixtures/file.txt
 - test/fixtures/file_for_md5.txt
 - test/fixtures/path/a/file2.txt
 - test/fixtures/path/b/file3.txt
 - test/fixtures/path/file1.txt
 - test/fixtures/s3.passwd
+- test/fixtures/slices/file1.txt.md5
+- test/fixtures/slices/file1.txt.part_aa
+- test/fixtures/slices/file1.txt.part_ab
 - test/opts_reader_test.rb
 - test/runner_test.rb
 - test/s3/commander_s3_test.rb
@@ -132,7 +137,6 @@ files:
 - test/s3/s3_test.rb
 - test/test_helper.rb
 - test/utils_test.rb
-has_rdoc: true
 homepage: https://github.com/fguillen/SweetyBacky
 licenses: []
 
@@ -162,19 +166,23 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: SweetyBacky
-rubygems_version: 1.3.7
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Ruby backup mechanism
 test_files: 
 - test/commander_test.rb
 - test/fixtures/config_s3.yml
+- test/fixtures/config_s3_slices.yml
 - test/fixtures/file.txt
 - test/fixtures/file_for_md5.txt
 - test/fixtures/path/a/file2.txt
 - test/fixtures/path/b/file3.txt
 - test/fixtures/path/file1.txt
 - test/fixtures/s3.passwd
+- test/fixtures/slices/file1.txt.md5
+- test/fixtures/slices/file1.txt.part_aa
+- test/fixtures/slices/file1.txt.part_ab
 - test/opts_reader_test.rb
 - test/runner_test.rb
 - test/s3/commander_s3_test.rb