mogbak 0.1.2 → 0.2.0

data/CHANGES

@@ -0,0 +1,10 @@
+2012-04-03: Release version 0.2.0
+
+   * Graceful handling of SIGINT and SIGTERM have been added.  Upon receiving, mogbak will finish 
+     the files being transfered and will exit. (Jesse Angell <jesse.angell@firespring.com>)
+     
+   * Backup now has an optional --non-stop switch.  This will cause mogbak to perform backup after backup 
+     without exiting until SIGTERM or SIGINT is received. (Jesse Angell <jesse.angell@firespring.com>)
+
+   * --log-file option added to Restore, List, and Backup.  Optional parameter that will redirect
+     output to a log file.  Works well in conjunction with --non-stop. (Jesse Angell <jesse.angell@firespring.com>)

data/Gemfile.lock

@@ -1,20 +1,44 @@
 PATH
   remote: .
   specs:
-    mogbak (0.0.1)
+    mogbak (0.1.3)
+      activerecord-import (>= 0.2.9)
+      gli (>= 1.5.1)
+      mogilefs-client (>= 3.1.1)
+      mysql2 (>= 0.3.11)
+      sqlite3 (>= 1.3.5)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    json (1.6.1)
-    rake (0.9.2.2)
-    rdoc (3.11)
-      json (~> 1.4)
+    activemodel (3.2.2)
+      activesupport (= 3.2.2)
+      builder (~> 3.0.0)
+    activerecord (3.2.2)
+      activemodel (= 3.2.2)
+      activesupport (= 3.2.2)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activerecord-import (0.2.9)
+      activerecord (~> 3.0)
+      activerecord (~> 3.0)
+    activesupport (3.2.2)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    awesome_print (1.0.2)
+    builder (3.0.0)
+    gli (1.5.1)
+    i18n (0.6.0)
+    mogilefs-client (3.1.1)
+    multi_json (1.2.0)
+    mysql2 (0.3.11)
+    sqlite3 (1.3.5)
+    tzinfo (0.3.32)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  awesome_print
   mogbak!
-  rake
-  rdoc

data/bin/mogbak

@@ -26,6 +26,11 @@ require 'sqlite3'
 require 'yaml'
 require 'forkinator'
 require 'path_helper'
+require 'signal_handler'
+require 'log'
+
+#activate signal handler
+SignalHandler.instance
 
 
 #Set master process name
@@ -102,15 +107,25 @@ command :backup do |c|
   c.desc 'do not remove deleted files from the backup (faster)'
   c.switch ['no-delete']
 
+  c.desc 'after backup is complete, start another backup. This causes mogbak to run non-stop.  Send SIGINT or SIGTERM to exit cleanly'
+  c.switch ['non-stop']
+
   c.desc 'Number of worker processes'
   c.default_value 1
   c.flag :workers
 
+  c.desc 'send backup output to log file'
+  c.flag ['log-file']
+
   c.action do |global_options, options, args|
     raise '[backup_path] is required - see: mogbak help backup' unless args[0]
     $backup_path = args[0]
+
+    #setup a logger for output
+    Log.instance(options[:'log-file'])
+
     mog = Backup.new(:backup_path => args[0], :workers => options[:workers].to_i)
-    mog.backup(:no_delete => options[:"no-delete"])
+    mog.backup(:no_delete => options[:"no-delete"], :non_stop => options[:"non-stop"])
   end
 
 end
@@ -142,11 +157,17 @@ command :restore do |c|
   c.desc 'restore a single file by dkey'
   c.flag :"single-file"
 
+  c.desc 'send restore output to log file'
+  c.flag ['log-file']
+
   c.action do |global_options, options, args|
     raise 'domain parameter is required - see: mogbak help restore' unless options[:domain]
     raise '[backup_path] is required - see: mogbak help restore' unless args[0]
     $backup_path = args[0]
 
+    #setup a logger for output
+    Log.instance(options[:'log-file'])
+
     restore = Restore.new(:tracker_ip => options[:trackerip],
                           :tracker_port => options[:trackerport],
                           :domain => options[:domain],
@@ -162,18 +183,28 @@ Output is: fid,dkey,length,classname
 EOS
 arg_name '[backup_path]'
 command :list do |c|
+
+  c.desc 'send list output to log file'
+  c.flag ['log-file']
+
   c.action do |global_options, options, args|
     raise '[backup_path] is required - see: mogbak help list' unless args[0]
     $backup_path = args[0]
 
+    #setup a logger for output
+    Log.instance(options[:'log-file'])
+
     list = List.new(:backup_path => args[0])
     list.list
   end
 end
 
-#If the user passes in --debug we can detect it here.
+#Handle --debug
 pre do |global_options, command, options, args|
+
+  #This is used to enable some more verbose output
   $debug = global_options[:debug]
+
   true
 end
 

data/lib/backup.rb

@@ -22,7 +22,7 @@ class Backup
 
 
     #run validations and setup
-    raise  unless check_backup_path
+    raise unless check_backup_path
     create_sqlite_db
     connect_sqlite
     migrate_sqlite
@@ -43,9 +43,9 @@ class Backup
   def bak_file(file)
     saved = file.bak_it
     if saved
-      puts "Backed up: FID #{file.fid}"
+       Log.instance.info("Backed up: FID #{file.fid}")
     else
-      puts "Error - will try again on next run: FID #{file.fid}"
+       Log.instance.info("Error - will try again on next run: FID #{file.fid}")
     end
 
     return saved
@@ -72,11 +72,13 @@ class Backup
       SqliteActiveRecord.clear_active_connections!
     }
 
-    #This proc receives an array of BakFiles,  proccesses them,  and returns a result array to the parent proc
+    #This proc receives an array of BakFiles,  proccesses them,  and returns a result array to the parent proc. We will break
+    #from the files if the signal handler says so.
     child = Proc.new { |files|
       result = []
       files.each do |file|
         break if file.nil?
+        break if SignalHandler.instance.should_quit
         saved = bak_file(file)
         result << {:saved => saved, :file => file}
       end
@@ -91,16 +93,18 @@ class Backup
   #param [Array] files must be an array of BakFiles that need to be deleted
   def launch_delete_workers(fids)
 
-    #This proc receives an array of BakFiles, handles them,  and spits them back to the parent.
+    #This proc receives an array of BakFiles, handles them,  and spits them back to the parent, break from the fids if
+    #the signal handler says so.
     child = Proc.new { |fids|
       result = []
       fids.each do |fid|
         break if fid.nil?
+        break if SignalHandler.instance.should_quit
         deleted = BakFile.delete_from_fs(fid)
         if deleted
-          puts "Deleting from backup: FID #{fid}"
+          Log.instance.info("Deleting from backup: FID #{fid}")
         else
-          puts "Failed to delete from backup: FID #{fid}"
+          Log.instance.info("Failed to delete from backup: FID #{fid}")
         end
 
         result << fid
@@ -133,64 +137,75 @@ class Backup
   #@param [Hash] o if :no_delete then don't remove deleted files from the backup (intensive process)
   def backup(o = {})
 
-    files = []
-    #first we retry files that we haven't been able to backup successfully, if any.
-    BakFile.find_each(:conditions => ['saved = ?', false]) do |bak_file|
-      files << bak_file
-    end
+    #Loop over the main backup logic.  We'll break out at the end unless o[:non_stop] is set
+    loop do
+      files = []
+      #first we retry files that we haven't been able to backup successfully, if any.
+      BakFile.find_each(:conditions => ['saved = ?', false]) do |bak_file|
+        files << bak_file
+      end
 
-    launch_backup_workers(files)
+      launch_backup_workers(files)
 
-    #now back up any new files.  if they fail to be backed up we'll retry them the next time the backup
-    #command is ran.
-    dmid = Domain.find_by_namespace(self.domain)
-    results = Fid.find_in_batches(:conditions => ['dmid = ? AND fid > ?', dmid, BakFile.max_fid], :batch_size => 500 * self.workers.to_i, :include => [:domain, :fileclass]) do |batch|
+      #now back up any new files.  if they fail to be backed up we'll retry them the next time the backup
+      #command is ran.
+      dmid = Domain.find_by_namespace(self.domain)
+      results = Fid.find_in_batches(:conditions => ['dmid = ? AND fid > ?', dmid, BakFile.max_fid], :batch_size => 500 * self.workers.to_i, :include => [:domain, :fileclass]) do |batch|
+
+        #Insert all the files into our bak db with :saved false so that we don't think we backed up something that crashed
+        files = []
+        batch.each do |file|
+          files << BakFile.new(:fid => file.fid,
+                               :domain => file.domain.namespace,
+                               :dkey => file.dkey,
+                               :length => file.length,
+                               :classname => file.classname,
+                               :saved => false)
+        end
 
-      #Insert all the files into our bak db with :saved false so that we don't think we backed up something that crashed
-      files = []
-      batch.each do |file|
-        files << BakFile.new(:fid => file.fid,
-                             :domain => file.domain.namespace,
-                             :dkey => file.dkey,
-                             :length => file.length,
-                             :classname => file.classname,
-                             :saved => false)
-      end
+        #There is no way to do a bulk insert in sqlite so this generates a lot of inserts.  wrapping all of the inserts
+        #inside a single transaction makes it much much faster.
+        BakFile.transaction do
+          BakFile.import files, :validate => false
+        end
 
-      #There is no way to do a bulk insert in sqlite so this generates a lot of inserts.  wrapping all of the inserts
-      #inside a single transaction makes it much much faster.
-      BakFile.transaction do
-        BakFile.import files, :validate => false
+        #Fire up the workers now that we have work for them to do
+        launch_backup_workers(files)
+
+        #Terminate program if the signal handler says so and this is a clean place to do it
+        return true if SignalHandler.instance.should_quit
       end
 
-      #Fire up the workers now that we have work for them to do
-      launch_backup_workers(files)
+      #Delete files from the backup that no longer exist in the mogilefs domain.  Unfortunently there is no easy way to detect
+      #which files have been deleted from the MogileFS domain.  Our only option is to brute force our way through.  This is a bulk
+      #query that checks a thousand files in each query against the MogileFS database server.  The query is kind of tricky because
+      #I wanted to do this with nothing but SELECT privileges which meant I couldn't create a temporary table (which would require,
+      #create temporary table and insert privleges).  You might want to only run this operation every once and awhile if you have a
+      #very large domain.  In my testing,  it is able to get through domains with millions of files in a matter of a second.  So
+      #all in all it's not so bad
+      if !o[:no_delete]
 
-    end
+        BakFile.find_in_batches { |bak_files|
+          union = "SELECT #{bak_files.first.fid} as fid"
+          bak_files.shift
+          bak_files.each do |bakfile|
+            union = "#{union} UNION SELECT #{bakfile.fid}"
+          end
+          connection = ActiveRecord::Base.connection
+          files = connection.select_values("SELECT t1.fid FROM (#{union}) as t1 LEFT JOIN file on t1.fid = file.fid WHERE file.fid IS NULL")
+          launch_delete_workers(files)
 
-    #Delete files from the backup that no longer exist in the mogilefs domain.  Unfortunently there is no easy way to detect
-    #which files have been deleted from the MogileFS domain.  Our only option is to brute force our way through.  This is a bulk
-    #query that checks a thousand files in each query against the MogileFS database server.  The query is kind of tricky because
-    #I wanted to do this with nothing but SELECT privileges which meant I couldn't create a temporary table (which would require,
-    #create temporary table and insert privleges).  You might want to only run this operation every once and awhile if you have a
-    #very large domain.  In my testing,  it is able to get through domains with millions of files in a matter of a second.  So
-    #all in all it's not so bad
-    if !o[:no_delete]
-      files_to_delete = Array.new
-      BakFile.find_in_batches { |bak_files|
-
-        union = "SELECT #{bak_files.first.fid} as fid"
-        bak_files.shift
-        bak_files.each do |bakfile|
-          union = "#{union} UNION SELECT #{bakfile.fid}"
-        end
-        connection = ActiveRecord::Base.connection
-        files = connection.select_values("SELECT t1.fid FROM (#{union}) as t1 LEFT JOIN file on t1.fid = file.fid WHERE file.fid IS NULL")
-        files_to_delete += files
-      }
+          #Terminate program if the signal handler says so and this is a clean place to do it
+          return true if SignalHandler.instance.should_quit
+        }
 
-      launch_delete_workers(files_to_delete)
 
+
+      end
+
+      #Break out of infinite loop unless o[:non_stop] is set
+      break unless o[:non_stop]
+      sleep 1
     end
 
   end

data/lib/forkinator.rb

@@ -88,13 +88,6 @@ class Forkinator
     children = []
     qty.times { children << make_child(child_proc)}
 
-    #register signal handler so that children kill if program receives a SIGINT
-    #which will happen if the user ctrl c's the parent process
-    Signal.trap :SIGINT do
-      children.each { |child| Process.kill(:KILL, child[:pid]) if child[:pid]}
-      exit 1
-    end
-
     #For each worker
     qty.times do |i|
 

data/lib/list.rb

@@ -22,7 +22,8 @@ class List
   #fid,key,length,class
   def list
     files = BakFile.find_each(:conditions => ['saved = ?', true]) do |file|
-      puts "#{file.fid},#{file.dkey},#{file.length},#{file.classname}"
+      Log.instance.info("#{file.fid},#{file.dkey},#{file.length},#{file.classname}")
+      break if SignalHandler.instance.should_quit
     end
   end
 end

data/lib/log.rb

@@ -0,0 +1,17 @@
+require 'logger'
+#Create a Logger that is a singleton provided by the instance method
+class Log
+  def self.instance(log_file = nil)
+    @@instance ||= create_logger(log_file)
+  end
+
+  def self.create_logger(log_file)
+    log_file = STDOUT if log_file == nil
+    logger = Logger.new(log_file)
+    logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+    logger.formatter = proc do |severity, datetime, progname, msg|
+      "#{datetime}: #{msg}\n"
+    end
+    logger
+  end
+end

data/lib/mogbak_version.rb

@@ -1,3 +1,3 @@
 module Mogbak
-  VERSION = '0.1.2'
+  VERSION = '0.2.0'
 end

data/lib/restore.rb

@@ -28,9 +28,9 @@ class Restore
 
   def output_save(save, fid)
     if save
-      puts "Restored: FID #{fid}"
+      Log.instance.info("Restored: FID #{fid}")
     else
-      puts "Error: FID #{fid}"
+      Log.instance.info("Error: FID #{fid}")
     end
   end
 
@@ -39,6 +39,7 @@ class Restore
       results = []
       files.each do |file|
         break if file.nil?
+        break if SignalHandler.instance.should_quit
         save = file.restore
         output_save(save, file.fid)
         results << {:restored => save, :fid => file.fid}
@@ -64,6 +65,7 @@ class Restore
 
       BakFile.find_in_batches(:conditions => ['saved = ?', true], :batch_size => 2000) do |batch|
         launch_restore_workers(batch)
+        break if SignalHandler.instance.should_quit
       end
 
     end

data/lib/signal_handler.rb

@@ -0,0 +1,19 @@
+require 'singleton'
+
+#Singleton class used to intercept signals.  If a SIGINT or SIGTERM is received a message is outputted and @should_quit
+#is set to true
+class SignalHandler
+  include Singleton
+  attr_reader :should_quit
+
+  def handle_signal
+    puts "PID #{Process.pid} is gracefully shutting down..."
+    @should_quit = true
+  end
+
+  def initialize
+    @should_quit = false
+    Signal.trap("SIGINT") { handle_signal }
+    Signal.trap("SIGTERM") { handle_signal }
+  end
+end

data/lib/validations.rb

@@ -45,7 +45,7 @@ module Validations
   #@return [Bool]
   def connect_sqlite(raise_msg = nil)
     begin
-      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => "#{$backup_path}/db.sqlite", :timeout => 1000)
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => "#{$backup_path}/db.sqlite", :timeout => 10000)
     rescue Exception => e
       raise raise_msg if raise_msg
       raise e if $debug

data/mogbak.gemspec

@@ -19,9 +19,6 @@ spec = Gem::Specification.new do |s|
   s.add_runtime_dependency('gli', '>= 1.5.1')
   s.add_runtime_dependency('mysql2', '>= 0.3.11')
   s.add_runtime_dependency('mogilefs-client','>= 3.1.1')
-  s.add_runtime_dependency('json','>= 1.6.5')
   s.add_runtime_dependency('sqlite3','>=1.3.5')
   s.add_runtime_dependency('activerecord-import','>=0.2.9')
-
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mogbak
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-27 00:00:00.000000000 Z
+date: 2012-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -75,22 +75,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.1.1
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.6.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 1.6.5
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -131,6 +115,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGES
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -145,10 +130,12 @@ files:
 - lib/fileclass.rb
 - lib/forkinator.rb
 - lib/list.rb
+- lib/log.rb
 - lib/mogbak_version.rb
 - lib/monkey_patch.rb
 - lib/path_helper.rb
 - lib/restore.rb
+- lib/signal_handler.rb
 - lib/validations.rb
 - mogbak.gemspec
 homepage: http://www.github.com/firespring/mogbak