mogbak 0.1.2

data/.gitignore

@@ -0,0 +1 @@
+.idea/

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,20 @@
+PATH
+  remote: .
+  specs:
+    mogbak (0.0.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    json (1.6.1)
+    rake (0.9.2.2)
+    rdoc (3.11)
+      json (~> 1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mogbak!
+  rake
+  rdoc

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012 Firespring
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+Mogbak makes it easy to backup your MogileFS domain to a single self contained directory.  It has the ability to    
+update that directory again and again to match your MogileFS domain.  This makes it possible for you to use   
+LVM snapshots.  Mogbak can also fork worker processes to backup or restore files in parallel.
+
+##Need a backup?
+    mogbak create --db=mogilefs --dbhost=mysqlserver --dbpass=secret --dbuser=mogile --domain=awesomeapp \\
+    --trackerip=10.10.10.10 --workers=10 /backups/awesomeapp
+    mogbak backup /backups/awesomeapp
+    
+###Perhaps you want an incremental, no problem.
+    mogbak backup --workers=10 /backups/awesomeapp
+    
+###Maybe you need to see the files in the backup
+    mogbak list /backups/awesomeapp
+    
+###Backups suck if you can't restore,  well good thing we can
+    mogbak restore --domain=restoreawesomeapp --trackerip=10.10.10.10 --workers=10 /backups/awesomeapp
+    
+###Maybe you just want to restore one file?
+    mogbak restore --domain=restoreawesomeapp --trackerip=10.10.10.10 --single-file=abc1234file --workers=10 /backups/awesomeapp
+
+
+###Why does Mogbak need to connect to my database?
+MogileFS simply bumps its FID value in the files table when a new file is saved. This makes it quite simple
+for us to query and see what files need to be backed up since our last backup. The problem is that we also need
+to know what files have been deleted from MogileFS but still live within your backup.  Since MogileFS has no delete
+log for us to look at we need to query the database in a brute-force manner. This would be extremely painful without
+access to the database. We do this as efficiently as we can,  our cluster has about 3 million files and it takes less than a second.
+You can disable this feature with --no-delete switch.
+
+The good news is that mogbak only needs *SELECT access*.
+
+###What does the self contained backup directory look like?
+
+ * db.sqlite - holds the metadata of each file in the backup
+ * settings.yml - holds the settings to connect to the mysql database and the tracker
+ * Backup files hashed using the same scheme as MogileFS Server
+
+###Whats the catch?
+
+ * Space. Obviously with large clusters the ability to save a full backup onto one device probably isn't possible
+ * Database.  Right now only MySQL backed trackers are supported
+
+There are certainly things that could be done about the above issues.  Pull requests are welcome :)
+
+####Requirements
+
+ * Ruby 1.9 is what we test against.  It'll probably work under 1.8 but you'll be a ghiney pig.
+ * *nix
+ * mysql client development libraries (for mysql2 gem dependency)
+ * sqlite3 development libraries (for sqlite3 gem dependency)
+
+####How to install?
+    gem install mogbak
+
+####Syntax?
+See https://github.com/firespring/mogbak/wiki/Command-syntax

data/bin/mogbak

@@ -0,0 +1,193 @@
+#!/usr/bin/env ruby
+#Realpath didn't exist in Ruby 1.8 so we'll add it.
+unless File.respond_to? :realpath
+  class File #:nodoc:
+    def self.realpath path
+      return realpath(File.readlink(path)) if symlink?(path)
+      path
+    end
+  end
+end
+$: << File.expand_path(File.dirname(File.realpath(__FILE__)) + '/../lib')
+
+require 'monkey_patch'
+require 'rubygems'
+require 'gli'
+require 'mogbak_version'
+require 'validations'
+require 'create'
+require 'backup'
+require 'restore'
+require 'list'
+require 'active_record'
+require 'activerecord-import'
+require "mogilefs"
+require 'sqlite3'
+require 'yaml'
+require 'forkinator'
+require 'path_helper'
+
+
+#Set master process name
+$0 = "mogbak [master]"
+
+include GLI
+program_desc 'Backup a mogilefs domain to the filesystem.  mogbak needs SELECT access to your MogileFS tracker database, which must
+currently be MySQL.'
+
+desc 'enable debug mode'
+switch ['debug']
+
+version Mogbak::VERSION
+
+desc 'Create a new backup profile for a MogileFS domain'
+arg_name '[backup_path]'
+command :create do |c|
+  c.desc 'tracker ip'
+  c.default_value '127.0.0.1'
+  c.flag :trackerip
+
+  c.desc 'tracker port'
+  c.default_value 7001
+  c.flag :trackerport
+
+  c.desc 'mogilefs domain'
+  c.default_value 'test'
+  c.flag :domain
+
+
+  c.desc 'mysql db server host'
+  c.default_value 'localhost'
+  c.flag :dbhost
+
+  c.desc 'mysql db server port'
+  c.default_value 3306
+  c.flag :dbport
+
+  c.desc 'database name'
+  c.default_value 'mogilefs'
+  c.flag :db
+
+  c.desc 'database user'
+  c.default_value 'mogile'
+  c.flag :dbuser
+
+  c.desc 'database password (REQUIRED)'
+  c.flag :dbpass
+
+
+  c.action do |global_options, options, args|
+
+    #required options and arguments
+    raise '--dbpass is required - see: mogbak help creater' unless options[:dbpass]
+    raise '[backup_path] is required see mogadm - see: mogbak help create' unless args[0]
+    $backup_path = args[0]
+
+    mog = Create.new(:tracker_ip => options[:trackerip],
+                      :tracker_port => options[:trackerport],
+                      :domain => options[:domain],
+                      :db_host => options[:dbhost],
+                      :db_port => options[:dbport],
+                      :db => options[:db],
+                      :db_user => options[:dbuser],
+                      :db_pass => options[:dbpass],
+                      :backup_path => args[0])
+    puts "Backup profile successfully created. To start a backup run: mogbak backup #{args[0]}"
+  end
+end
+
+desc 'Start a backup using an existing backup profile'
+arg_name '[backup_path]'
+command :backup do |c|
+  c.desc 'do not remove deleted files from the backup (faster)'
+  c.switch ['no-delete']
+
+  c.desc 'Number of worker processes'
+  c.default_value 1
+  c.flag :workers
+
+  c.action do |global_options, options, args|
+    raise '[backup_path] is required - see: mogbak help backup' unless args[0]
+    $backup_path = args[0]
+    mog = Backup.new(:backup_path => args[0], :workers => options[:workers].to_i)
+    mog.backup(:no_delete => options[:"no-delete"])
+  end
+
+end
+
+desc 'Restore a backup to a new MogileFS domain.'
+long_desc <<EOS
+This function restores all files in backup to the new domain. It does not keep track of what has been restored, so the
+restore cannot be stopped and resumed.  If it fails to restore a file it will output an error.  It restores the files with
+the same class that they were created with.  It is expected that you will create those classes on the domain before restoring.
+EOS
+arg_name '[backup_path]'
+command :restore do |c|
+
+  c.desc 'restore dest tracker ip'
+  c.default_value '127.0.0.1'
+  c.flag :trackerip
+
+  c.desc 'restore dest tracker port'
+  c.default_value 7001
+  c.flag :trackerport
+
+  c.desc 'restore dest mogilefs domain (REQUIRED)'
+  c.flag :domain
+
+  c.desc 'Number of worker processes'
+  c.default_value 1
+  c.flag :workers
+
+  c.desc 'restore a single file by dkey'
+  c.flag :"single-file"
+
+  c.action do |global_options, options, args|
+    raise 'domain parameter is required - see: mogbak help restore' unless options[:domain]
+    raise '[backup_path] is required - see: mogbak help restore' unless args[0]
+    $backup_path = args[0]
+
+    restore = Restore.new(:tracker_ip => options[:trackerip],
+                          :tracker_port => options[:trackerport],
+                          :domain => options[:domain],
+                          :backup_path => args[0],
+                          :workers => options[:workers])
+    restore.restore(options[:"single-file"])
+  end
+end
+
+desc 'List the files in a backup'
+long_desc <<EOS
+Output is: fid,dkey,length,classname
+EOS
+arg_name '[backup_path]'
+command :list do |c|
+  c.action do |global_options, options, args|
+    raise '[backup_path] is required - see: mogbak help list' unless args[0]
+    $backup_path = args[0]
+
+    list = List.new(:backup_path => args[0])
+    list.list
+  end
+end
+
+#If the user passes in --debug we can detect it here.
+pre do |global_options, command, options, args|
+  $debug = global_options[:debug]
+  true
+end
+
+#If debug is enabled we'll spit out the exception and the entire backtrace.  Otherwise it will just output
+#the exception message
+on_error do |exception|
+  if $debug
+    puts exception
+    puts ''
+    puts 'Backtrace:'
+    puts exception.backtrace
+    return false
+  end
+  true
+end
+
+exit GLI.run(ARGV)

data/db/migrate/20120316200100_create_initial_schema.rb

@@ -0,0 +1,17 @@
+class CreateInitialSchema < ActiveRecord::Migration
+  def self.up
+    create_table :bak_files do |t|
+      t.integer :fid
+      t.string :domain, :limit => 255
+      t.string :dkey, :limit => 255
+      t.integer :length, :limit => 20
+      t.string :classname, :limit => 255
+      t.boolean :saved, :default => false
+    end
+    add_index :bak_files, :fid
+  end
+
+  def self.down
+    drop_table :bak_files
+  end
+end

data/lib/backup.rb

@@ -0,0 +1,197 @@
+#Used to backup a mogilefs domain using a backup profile.
+class Backup
+  attr_accessor :db, :db_host, :db_port, :db_pass, :db_user, :domain, :tracker_host, :tracker_port, :workers
+  include Validations
+
+  #Run validations and prepare the object for a backup
+  #@param [Hash] o hash containing the settings for the backup
+  def initialize(o={})
+
+    #Load up the settings file
+    check_settings_file
+    settings = YAML::load(File.open("#{$backup_path}/settings.yml"))
+    @db = settings['db']
+    @db_host = settings['db_host']
+    @db_port = settings['db_port']
+    @db_pass = settings['db_pass']
+    @db_user = settings['db_user']
+    @domain = settings['domain']
+    @tracker_ip = settings['tracker_ip']
+    @tracker_port = settings['tracker_port']
+    @workers = o[:workers] if o[:workers]
+
+
+    #run validations and setup
+    raise  unless check_backup_path
+    create_sqlite_db
+    connect_sqlite
+    migrate_sqlite
+    mogile_db_connect
+    mogile_tracker_connect
+    check_mogile_domain(domain)
+
+    require ('domain')
+    require('file')
+    require('bakfile')
+    require('fileclass')
+  end
+
+
+  #Create a backup of a file using a BakFile object
+  #@param [BakFile] file file that needs to be backed up
+  #@return [Bool] file save result
+  def bak_file(file)
+    saved = file.bak_it
+    if saved
+      puts "Backed up: FID #{file.fid}"
+    else
+      puts "Error - will try again on next run: FID #{file.fid}"
+    end
+
+    return saved
+  end
+
+  #Launch workers to backup an array of BakFiles
+  #@param [Array] files must be an array of BakFiles
+  def launch_backup_workers(files)
+
+    #This proc will process the results of the child proc
+    parent = Proc.new { |results|
+      fids = []
+
+      results.each do |result|
+        file = result[:file]
+        saved = result[:saved]
+        fids << file.fid if saved
+      end
+
+      #bulk update all the fids.  much faster then doing it one at a time
+      BakFile.update_all({:saved => true}, {:fid => fids})
+
+      #release the connection from the connection pool
+      SqliteActiveRecord.clear_active_connections!
+    }
+
+    #This proc receives an array of BakFiles,  proccesses them,  and returns a result array to the parent proc
+    child = Proc.new { |files|
+      result = []
+      files.each do |file|
+        break if file.nil?
+        saved = bak_file(file)
+        result << {:saved => saved, :file => file}
+      end
+      result
+    }
+
+    #launch workers using the above procs and files
+    Forkinator.hybrid_fork(self.workers.to_i, files, parent, child)
+  end
+
+  #Launch workers to delete an array of files
+  #param [Array] files must be an array of BakFiles that need to be deleted
+  def launch_delete_workers(fids)
+
+    #This proc receives an array of BakFiles, handles them,  and spits them back to the parent.
+    child = Proc.new { |fids|
+      result = []
+      fids.each do |fid|
+        break if fid.nil?
+        deleted = BakFile.delete_from_fs(fid)
+        if deleted
+          puts "Deleting from backup: FID #{fid}"
+        else
+          puts "Failed to delete from backup: FID #{fid}"
+        end
+
+        result << fid
+      end
+      result
+    }
+
+    #This proc will process the results of the child proc
+    parent = Proc.new { |results|
+      fids = []
+
+      results.each do |result|
+        fids << result
+      end
+
+      BakFile.delete_all({:fid => fids})
+
+      #release the connection from the connection pool
+      SqliteActiveRecord.clear_active_connections!
+    }
+
+    #launch workers using the above procs and files
+    Forkinator.hybrid_fork(self.workers.to_i, fids, parent, child)
+
+  end
+
+  #The real logic for backing the domain up.  It is pretty careful about making sure that it doesn't report a file
+  #as backed up unless it actually was.  Supports the ability to remove deleted files from the backup as well.  We grab files
+  #from the mogilefs mysql server in groups of 500 * number of workers (default is 1 worker)
+  #@param [Hash] o if :no_delete then don't remove deleted files from the backup (intensive process)
+  def backup(o = {})
+
+    files = []
+    #first we retry files that we haven't been able to backup successfully, if any.
+    BakFile.find_each(:conditions => ['saved = ?', false]) do |bak_file|
+      files << bak_file
+    end
+
+    launch_backup_workers(files)
+
+    #now back up any new files.  if they fail to be backed up we'll retry them the next time the backup
+    #command is ran.
+    dmid = Domain.find_by_namespace(self.domain)
+    results = Fid.find_in_batches(:conditions => ['dmid = ? AND fid > ?', dmid, BakFile.max_fid], :batch_size => 500 * self.workers.to_i, :include => [:domain, :fileclass]) do |batch|
+
+      #Insert all the files into our bak db with :saved false so that we don't think we backed up something that crashed
+      files = []
+      batch.each do |file|
+        files << BakFile.new(:fid => file.fid,
+                             :domain => file.domain.namespace,
+                             :dkey => file.dkey,
+                             :length => file.length,
+                             :classname => file.classname,
+                             :saved => false)
+      end
+
+      #There is no way to do a bulk insert in sqlite so this generates a lot of inserts.  wrapping all of the inserts
+      #inside a single transaction makes it much much faster.
+      BakFile.transaction do
+        BakFile.import files, :validate => false
+      end
+
+      #Fire up the workers now that we have work for them to do
+      launch_backup_workers(files)
+
+    end
+
+    #Delete files from the backup that no longer exist in the mogilefs domain.  Unfortunently there is no easy way to detect
+    #which files have been deleted from the MogileFS domain.  Our only option is to brute force our way through.  This is a bulk
+    #query that checks a thousand files in each query against the MogileFS database server.  The query is kind of tricky because
+    #I wanted to do this with nothing but SELECT privileges which meant I couldn't create a temporary table (which would require,
+    #create temporary table and insert privleges).  You might want to only run this operation every once and awhile if you have a
+    #very large domain.  In my testing,  it is able to get through domains with millions of files in a matter of a second.  So
+    #all in all it's not so bad
+    if !o[:no_delete]
+      files_to_delete = Array.new
+      BakFile.find_in_batches { |bak_files|
+
+        union = "SELECT #{bak_files.first.fid} as fid"
+        bak_files.shift
+        bak_files.each do |bakfile|
+          union = "#{union} UNION SELECT #{bakfile.fid}"
+        end
+        connection = ActiveRecord::Base.connection
+        files = connection.select_values("SELECT t1.fid FROM (#{union}) as t1 LEFT JOIN file on t1.fid = file.fid WHERE file.fid IS NULL")
+        files_to_delete += files
+      }
+
+      launch_delete_workers(files_to_delete)
+
+    end
+
+  end
+end

data/lib/bakfile.rb

@@ -0,0 +1,75 @@
+#This is kind of awkward but is the only good way to support multiple database connections using ActiveRecord
+class SqliteActiveRecord < ActiveRecord::Base
+  establish_connection(:adapter => 'sqlite3', :database => "#{$backup_path}/db.sqlite")
+  self.abstract_class = true
+end
+
+#Represents files that are either backed up,  about to be backed up,  or failed to be backed up.
+class BakFile < SqliteActiveRecord
+
+  #get the max fid that is backed up
+  def self.max_fid
+    last_backed_file = BakFile.order("fid").last
+    if last_backed_file
+      max_fid = last_backed_file.fid
+    else
+      max_fid = 0
+    end
+    max_fid
+  end
+
+  #Restore a file back to a MogileFS domain
+  #@return [Bool]
+  def restore
+    path = PathHelper.path(self.fid)
+    begin
+      $mg.store_file(self.dkey, self.classname, path)
+    rescue Exception => e
+      if $debug
+        raise e
+      end
+    end
+  end
+
+
+  #Get a file from MogileFS and save it to the destination path.
+  #@return [Bool]
+  def bak_it
+    begin
+      path = PathHelper.path(self.fid)
+      $mg.get_file_data(self.dkey, path)
+    rescue Exception => e
+      if $debug
+        raise e
+      end
+      return false
+    end
+    true
+  end
+
+
+  #Delete from filesystem using just a fid
+  #@return [Bool]
+  def self.delete_from_fs(delete_fid)
+    begin
+      File.delete(PathHelper.path(delete_fid))
+    rescue Exception => e
+      if $debug
+        raise e
+      end
+    end
+  end
+
+
+  #Delete file from filesystem if someone deletes this object through ActiveRecord somehow.
+  before_destroy do
+    path = PathHelper.path(self.fid)
+    begin
+      File.delete(path)
+    rescue Exception => e
+      if $debug
+        raise e
+      end
+    end
+  end
+end

data/lib/create.rb

@@ -0,0 +1,56 @@
+#Creates a backup profile to be used by Backup
+class Create
+  attr_accessor :db, :db_host, :db_port, :db_pass, :db_user, :domain, :tracker_host, :tracker_port, :workers
+  include Validations
+
+  #Run validations and create the backup profile
+  #@param [Hash] o hash containing the settings for the backup profile
+  def initialize(o={})
+    @db = o[:db] if o[:db]
+    @db_host = o[:db_host] if o[:db_host]
+    @db_port = o[:db_port] if o[:db_port]
+    @db_pass = o[:db_pass] if o[:db_pass]
+    @db_user = o[:db_user] if o[:db_user]
+    @domain = o[:domain] if o[:domain]
+    @tracker_ip = o[:tracker_ip] if o[:tracker_ip]
+    @tracker_port = o[:tracker_port] if o[:tracker_port]
+
+    #If settings.yml exists then this is an existing backup and you cannot run a create on top of it
+    raise 'Cannot run create on an existing backup.  Try: mogbak backup #{$backup_path} to backup.  If you want
+        to change settings on this backup profile you will have to edit #{$backup_path}/settings.yml manually.' if check_settings_file(nil)
+
+    check_backup_path
+    create_sqlite_db
+    connect_sqlite
+    migrate_sqlite
+    mogile_db_connect
+    mogile_tracker_connect
+    check_mogile_domain(@domain)
+
+    #Save settings
+    save_settings
+  end
+
+  #Save the settings for the backup into a yaml file (settings.yaml) so that an incremental can be ran without so many parameters
+  #@return [Bool] true or false
+  def save_settings
+    require ('yaml')
+    settings = {
+        'db' => @db,
+        'db_host' => @db_host,
+        'db_port' => @db_port,
+        'db_pass' => @db_pass,
+        'db_user' => @db_user,
+        'domain' => @domain,
+        'tracker_ip' => @tracker_ip,
+        'tracker_port' => @tracker_port,
+        'backup_path' => $backup_path
+    }
+
+    File.open("#{$backup_path}/settings.yml", "w") do |file|
+      file.write settings.to_yaml
+    end
+
+    true
+  end
+end

data/lib/domain.rb

@@ -0,0 +1,5 @@
+#Represents domains in a MogileFS database server
+class Domain < ActiveRecord::Base
+  self.primary_key = "dmid"
+  self.table_name = "domain"
+end

data/lib/file.rb

@@ -0,0 +1,21 @@
+#represents files that are in the MogileFS database.  This model is used for talking to the MogileFS database via
+#ActiveRecord
+class Fid < ActiveRecord::Base
+  self.primary_key = "fid"
+  self.table_name = "file"
+  belongs_to :domain,
+             :foreign_key => "dmid"
+  belongs_to :fileclass,
+             :foreign_key => "classid"
+
+  #If there is no fileclass then it is the default class
+  #@return [String] name of class file belongs to
+  def classname
+    if fileclass
+      fileclass.classname
+    else
+      return 'default'
+    end
+  end
+
+end

data/lib/fileclass.rb

@@ -0,0 +1,5 @@
+#Represents classes in a MogileFS database server
+class Fileclass < ActiveRecord::Base
+    self.primary_key = "classid"
+    self.table_name = "class"
+end

data/lib/forkinator.rb

@@ -0,0 +1,132 @@
+#The Forkinator makes it easy to fork workers,  pass a list of jobs for them to work on,  and listen for the results back from
+#the child process.  It uses a combination of threading and forking to accomplish this.  Marshal is used to pass objects back
+#and forth between the child and parent via IO.pipe.
+class Forkinator
+
+  #Wait for threads
+  #@param [Array] threads an array containing threads we need to wait on
+  def self.wait_for_threads(threads)
+    threads.compact.each do |t|
+      begin
+        t.join
+      rescue Interrupt
+        # no reason to wait on dead threads
+      end
+    end
+  end
+
+  #Fork a child.  Provide a proc of code to run inside child.  The child proc expects to be sent an array of jobs
+  #@param [Proc] child_proc code for the child to run
+  #@return [Hash] :write = pipe for writing to the child, :read = pipe for reading from the child, :pid = pid of the child
+  def self.make_child(child_proc)
+
+    #open pipes for two way communication between the parent and child
+    child_read, parent_write = IO.pipe
+    parent_read, child_write = IO.pipe
+
+    #remove our database connection,  we don't want it inside the child,  as it'll get closed when the child shuts down
+    mog_config = ActiveRecord::Base.remove_connection
+
+    #fork, code inside this block is only ran inside the child
+    pid = Process.fork do
+      begin
+
+        #Since we're the child now,  we'll close the parent's r/w pipes as we don't need them
+        parent_write.close
+        parent_read.close
+
+        #child loops through IO pipe,  listening for data from the parent,  if the parent closes the pipe then we're
+        #done
+        while !child_read.eof? do
+          #rename the process to make it clear that it's a worker in idle status
+          $0 = "mogbak [idle]"
+          #this call blocks until it receives something from the parent via the pipe
+          job = Marshal.load(child_read)
+          #since we're working now we'll rename the process
+          $0 = "mogbak [working]"
+          #call the child proc
+          result = child_proc.call(job)
+          #hand the child proc response back to the parent
+          Marshal.dump(result, child_write)
+        end
+
+      #no matter what happens..make sure we get the pipes closed
+      ensure
+        child_read.close
+        child_write.close
+      end
+    end
+
+    #This is the parent executing this -- reconnect to the database we just dropped above.
+    ActiveRecord::Base.establish_connection(mog_config)
+
+    #close the child's handle on the pipes since the parent won't need them
+    child_read.close
+    child_write.close
+
+    {:write => parent_write, :read => parent_read, :pid => pid}
+  end
+
+  #Forks children,  makes threads for two-way communication,  and evenly distributes jobs to each child.
+  #@param [Integer] qty number of workers to launch
+  #@param [Array] jobs array containing jobs for each child
+  #@param [Proc] parent_proc code to be ran in the thread used to communicate with the child
+  #@param [Proc] child_proc code to be ran in the forked child
+  def self.hybrid_fork(qty, jobs, parent_proc, child_proc)
+    threads = []
+
+    #mutex is used to ensure that some operations in the threads don't have the potential of happening at the same time
+    #in another thread
+    semaphore = Mutex.new
+
+    require('thread')
+
+    #split the jobs up
+    jobs = jobs.in_groups(qty)
+
+    #spawn the children
+    children = []
+    qty.times { children << make_child(child_proc)}
+
+    #register signal handler so that children kill if program receives a SIGINT
+    #which will happen if the user ctrl c's the parent process
+    Signal.trap :SIGINT do
+      children.each { |child| Process.kill(:KILL, child[:pid]) if child[:pid]}
+      exit 1
+    end
+
+    #For each worker
+    qty.times do |i|
+
+      #start a thread
+      threads[i] = Thread.new do
+        Thread.current.abort_on_exception = true
+
+        child = {}
+        semaphore.synchronize { child = children.pop }
+
+        pid = child[:pid]
+        njobs = jobs[i - 1]
+
+        #pass jobs to child
+        Marshal.dump(njobs, child[:write])
+
+        #wait for result
+        result = Marshal.load(child[:read])
+
+        #process result
+        semaphore.synchronize { parent_proc.call(result) }
+
+        #close the pipe
+        child[:write].close
+
+        #wait for process to finish before terminating this thread
+        Process.wait(pid)
+
+        #close db connection
+        SqliteActiveRecord.connection.close
+      end
+    end
+    wait_for_threads(threads)
+  end
+end

data/lib/list.rb

@@ -0,0 +1,28 @@
+#List the files in a MogileFS backup with their metadata
+class List
+  attr_accessor :backup_path
+  include Validations
+
+
+  #initialize the list object
+  #@param[Hash] o :backup_path is required
+  def initialize(o={})
+
+    #If settings file does not exist then this is not a valid mogilefs backup
+    check_settings_file('settings.yml not found in path.  This must not be a backup profile. Cannot list')
+
+    connect_sqlite
+    migrate_sqlite
+
+    #Now that database is all setup load the model class
+    require('bakfile')
+  end
+
+  #Outputs a list of files in CSV format
+  #fid,key,length,class
+  def list
+    files = BakFile.find_each(:conditions => ['saved = ?', true]) do |file|
+      puts "#{file.fid},#{file.dkey},#{file.length},#{file.classname}"
+    end
+  end
+end

data/lib/mogbak_version.rb

@@ -0,0 +1,3 @@
+module Mogbak
+  VERSION = '0.1.2'
+end

data/lib/monkey_patch.rb

@@ -0,0 +1,28 @@
+#pretty useful rails method.  Splits an array into groups
+class Array
+  def in_groups(number, fill_with = nil)
+    # size / number gives minor group size;
+    # size % number gives how many objects need extra accommodation;
+    # each group hold either division or division + 1 items.
+    division = size / number
+    modulo = size % number
+
+    # create a new array avoiding dup
+    groups = []
+    start = 0
+
+    number.times do |index|
+      length = division + (modulo > 0 && modulo > index ? 1 : 0)
+      padding = fill_with != false &&
+          modulo > 0 && length == division ? 1 : 0
+      groups << slice(start, length).concat([fill_with] * padding)
+      start += length
+    end
+
+    if block_given?
+      groups.each { |g| yield(g) }
+    else
+      groups
+    end
+  end
+end

data/lib/path_helper.rb

@@ -0,0 +1,20 @@
+class PathHelper
+  #This produces a hashed path very similar to mogilefs just without the device id.  It also recursively creates the
+  #directory inside the backup
+  def self.path(sfid)
+    sfid = "#{sfid}"
+    length = sfid.length
+    if length < 10
+      nfid = "0" * (10 - length) + sfid
+    else
+      nfid = fid
+    end
+    /(?<b>\d)(?<mmm>\d{3})(?<ttt>\d{3})(?<hto>\d{3})/ =~ nfid
+
+    #create the directory
+    directory_path = "#{$backup_path}/#{b}/#{mmm}/#{ttt}"
+    FileUtils.mkdir_p(directory_path)
+
+    return "#{directory_path}/#{nfid}.fid"
+  end
+end

data/lib/restore.rb

@@ -0,0 +1,71 @@
+#Restore a mogbak backup to a MogileFS domain
+class Restore
+  attr_accessor :domain, :tracker_host, :tracker_port, :backup_path, :workers
+  include Validations
+
+  #
+  def initialize(o={})
+    @domain = o[:domain] if o[:domain]
+    @tracker_ip = o[:tracker_ip] if o[:tracker_ip]
+    @tracker_port = o[:tracker_port] if o[:tracker_port]
+    @backup_path = o[:backup_path] if o[:backup_path]
+    @workers = o[:workers] if o[:workers]
+
+
+    #If settings file does not exist then this is not a valid mogilefs backup
+    check_settings_file('settings.yml not found in path.  This must not be a backup profile. Cannot restore')
+
+    connect_sqlite
+    migrate_sqlite
+    mogile_tracker_connect
+
+    #Now that database is all setup load the model classes
+    require ('domain')
+    require('file')
+    require('bakfile')
+    require('fileclass')
+  end
+
+  def output_save(save, fid)
+    if save
+      puts "Restored: FID #{fid}"
+    else
+      puts "Error: FID #{fid}"
+    end
+  end
+
+  def launch_restore_workers(files)
+    child = Proc.new { |files|
+      results = []
+      files.each do |file|
+        break if file.nil?
+        save = file.restore
+        output_save(save, file.fid)
+        results << {:restored => save, :fid => file.fid}
+      end
+      results
+    }
+
+    parent = Proc.new { |results|
+      SqliteActiveRecord.clear_active_connections!
+    }
+
+    Forkinator.hybrid_fork(self.workers.to_i, files, parent, child)
+
+  end
+
+  def restore(dkey = false)
+    if dkey
+      file = BakFile.find_by_dkey(dkey)
+      raise 'file not found in backup' unless file
+      save = file.restore
+      output_save(save, file.fid)
+    else
+
+      BakFile.find_in_batches(:conditions => ['saved = ?', true], :batch_size => 2000) do |batch|
+        launch_restore_workers(batch)
+      end
+
+    end
+  end
+end

data/lib/validations.rb

@@ -0,0 +1,120 @@
+#Common methods used for setting up or verifing
+module Validations
+
+  #Check that the settings.yml file exists in the backup_path
+  #@param [String] raise_msg exception to raise if the file is missing.  if set to nil no exception will be rasied
+  #@return [Bool]
+  def check_settings_file(raise_msg = 'settings.yml not found in path.  This must not be a backup profile. See: mogbak help create')
+    if File.exists?("#{$backup_path}/settings.yml")
+      return true
+    else
+      raise raise_msg if raise_msg
+      return false
+    end
+  end
+
+
+  #Check that the backup_path is valid
+  #@param [String] raise_msg exception to raise if the backup_path is not a valid direcotry. if set to nil no exception will be rasied
+  #@return [Bool]
+  def check_backup_path(raise_msg = 'backup_path is not a valid directory')
+    if !File.directory?($backup_path)
+      raise raise_msg if raise_msg
+      return false
+    end
+    true
+  end
+
+  #Create database for metadata
+  #@param [String] raise_msg exception to raise if database cannot be created.  nil will raise no exception
+  #@return [Bool]
+  def create_sqlite_db(raise_msg = "Could not create #{$backup_path}/db.sqlite - check permissions")
+    begin
+      if !File.exists?("#{$backup_path}/db.sqlite")
+        SQLite3::Database.new("#{$backup_path}/db.sqlite")
+      end
+    rescue Exception => e
+      raise raise_msg if raise_msg
+      return false
+    end
+    true
+  end
+
+  #Connect to sqlite metadata db
+  #@param [String] raise_msg exception to raise if we cannot connect. if set to nil no exception will be rasied
+  #@return [Bool]
+  def connect_sqlite(raise_msg = nil)
+    begin
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => "#{$backup_path}/db.sqlite", :timeout => 1000)
+    rescue Exception => e
+      raise raise_msg if raise_msg
+      raise e if $debug
+      return false
+    end
+    true
+  end
+
+  #Run ActiveRecord migrations on the sqlite database
+  #@param [String] raise_msg exception to raise if migrations fail. if set to nil no exception will be rasied
+  #@return [Bool]
+  def migrate_sqlite(raise_msg = "could not run migrations on #{$backup_path}/db.sqlite")
+    #run migrations
+    begin
+      ActiveRecord::Migrator.up(File.expand_path(File.dirname(__FILE__)) + '/../db/migrate/')
+    rescue
+      raise raise_msg if raise_msg
+      return false
+    end
+    true
+  end
+
+  #Connect to MogileFS mysql server
+  #@param [String] raise_msg exception to raise if we cannot connect. if set to nil no exception will be rasied
+  #@return [Bool]
+  def mogile_db_connect(raise_msg = 'Could not connect to MySQL database')
+    #Verify that we can connect to the mogilefs mysql server
+    begin
+      ActiveRecord::Base.establish_connection({:adapter => "mysql2",
+                                               :host => @db_host,
+                                               :port => @db_port,
+                                               :username => @db_user,
+                                               :password => @db_pass,
+                                               :database => @db,
+                                               :reconnect => true})
+    rescue Exception => e
+      raise raise_msg if raise_msg
+      return false
+    end
+    true
+  end
+
+  #Connect to mogile tracker
+  #@param [String] raise_msg exception to raise if we cannot connect. if set to nil no exception will be raised
+  #@return [Bool]
+  def mogile_tracker_connect(raise_msg = 'Could not connect to MogileFS tracker')
+    host = ["#{@tracker_ip}:#{@tracker_port}"]
+    begin
+    $mg = MogileFS::MogileFS.new(:domain => @domain, :hosts => host)
+    rescue Exception => e
+      if $debug
+        raise e
+      end
+      raise raise_msg if raise_msg
+      return false
+    end
+  end
+
+  #Check if mogile domain is valid
+  #@param [String] raise_msg exception to raise if domain does not exist. if set to nil no exception will be raised
+  #@return [Bool]
+  def check_mogile_domain(domain, raise_msg = 'Domain does not exist in MogileFS')
+    require('domain')
+    domain = Domain.find_by_namespace(self.domain)
+    if !domain
+      raise raise_msg if raise_msg
+      return false
+    end
+    true
+  end
+
+end

data/mogbak.gemspec

@@ -0,0 +1,27 @@
+# Ensure we require the local version and not one we might have installed already
+require File.join([File.dirname(__FILE__),'lib','mogbak_version.rb'])
+spec = Gem::Specification.new do |s| 
+  s.name = 'mogbak'
+  s.license = "MIT"
+  s.version = Mogbak::VERSION
+  s.author = 'Jesse Angell'
+  s.email = 'jesse.angell@firespring.com'
+  s.homepage = 'http://www.github.com/firespring/mogbak'
+  s.platform = Gem::Platform::RUBY
+  s.summary = 'Backup utility for MogileFS'
+  s.description = 'mogbak makes it easy to backup and restore mogilefs domains'
+  s.files         = `git ls-files`.split("\n")
+  s.require_paths << 'lib'
+  s.bindir = 'bin'
+  s.executables << 'mogbak'
+  s.add_development_dependency('awesome_print')
+
+  s.add_runtime_dependency('gli', '>= 1.5.1')
+  s.add_runtime_dependency('mysql2', '>= 0.3.11')
+  s.add_runtime_dependency('mogilefs-client','>= 3.1.1')
+  s.add_runtime_dependency('json','>= 1.6.5')
+  s.add_runtime_dependency('sqlite3','>=1.3.5')
+  s.add_runtime_dependency('activerecord-import','>=0.2.9')
+
+
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: mogbak
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+  prerelease: 
+platform: ruby
+authors:
+- Jesse Angell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+- !ruby/object:Gem::Dependency
+  name: mogilefs-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.1.1
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.5
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.5
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.9
+description: mogbak makes it easy to backup and restore mogilefs domains
+email: jesse.angell@firespring.com
+executables:
+- mogbak
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/mogbak
+- db/migrate/20120316200100_create_initial_schema.rb
+- lib/backup.rb
+- lib/bakfile.rb
+- lib/create.rb
+- lib/domain.rb
+- lib/file.rb
+- lib/fileclass.rb
+- lib/forkinator.rb
+- lib/list.rb
+- lib/mogbak_version.rb
+- lib/monkey_patch.rb
+- lib/path_helper.rb
+- lib/restore.rb
+- lib/validations.rb
+- mogbak.gemspec
+homepage: http://www.github.com/firespring/mogbak
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Backup utility for MogileFS
+test_files: []