heroku-mongo-backup-pm 0.1

data/.gitignore

@@ -0,0 +1,4 @@
+pkg
+*.swp
+*.swo
+.DS_Store

data/README.md

@@ -0,0 +1,67 @@
+## heroku-mongo-backup *— backup mongodb on Heroku and push it to S3 or FTP storage*
+
+**heroku-mongo-backup** does:
+
+1. Backup mongodb collections to one file;
+2. Compress backup file with gzip;
+3. Push backup to the specified S3 bucket or FTP server;
+
+> Why not mongodump command?
+
+*mongodump* command is not available on Heroku side. If you don't want to setup third party backup service for every project *heroku-mongo-backup* may be helpful.
+
+
+## Configuration
+
+Add gem to the ```Gemfile```: ```gem "heroku-mongo-backup"``` - if everything's okay ```rake -T``` command should show ```rake mongo:backup``` rake tasks.
+
+For S3 support **heroku-mongo-backup** requires ```s3``` or ```aws-s3``` or ```fog``` library. One of those should be in ```Gemfile```, if you don't care add ```fog``` it's seems to be the most advanced.
+
+Configure heroku scheduler to run ```mongo:backup``` rake task. Or if cron is used add backup task to ```/lib/tasks/cron.rake``` file:
+
+```
+desc "This task is called by the Heroku cron add-on"
+task :cron => :environment do
+  Rake::Task['mongo:backup'].invoke
+end
+```
+
+Set Heroku environment variables:
+
+```heroku config:add S3_BACKUPS_BUCKET=_value_ S3_KEY_ID=_value_ S3_SECRET_KEY=_value_ MONGO_URL=_value_```
+
+On MONGO_URL place anyone of these is assaptable: *MONGOHQ_URI* or *MONGOLAB_URI*.
+
+For FTP set these variables:
+
+```heroku config:add UPLOAD_TYPE=ftp FTP_HOST=_host_ FTP_PASSWORD=_pass_ FTP_USERNAME=_user_```
+
+
+## Rake Commands
+
+* ```heroku run rake mongo:backup```
+
+If you want to automatically remove old backup files pass ```MAX_BACKUPS``` parameter to the rake command:
+
+* ```heroku run rake mongo:backup MAX_BACKUPS=7```
+
+Restore from backup:
+
+* ```heroku run rake mongo:restore FILE=backup-file-name.gz```
+
+If you want to restore from local file run:
+
+* ```rake mongo:restore LOCAL=/absolute/path/to/<backup-file.gz>```
+
+For Rails 2 add this to your Rakefile to import rake tasks:
+
+```import File.expand_path(File.join(Gem.datadir('heroku-mongo-backup'), '..', '..', 'lib', 'tasks', 'heroku_mongo_backup.rake'))```
+
+## Gem Contributors
+
+1. [alexkravets - slatestudio.com](http://slatestudio.com "Slate Studio") - gem itself with S3 support
+2. [matyi](https://github.com/matyi "Matyi - GitHub Profile") - FTP support
+3. [stefl - stef.io](http://stef.io "Stef Lewandowski") - Rails is not required for production
+4. [moonhouse - moonhouse.se](http://www.moonhouse.se/ "David Hall") - default config improvement
+5. [wolfpakz](https://github.com/wolfpakz "Dan Porter") - Rails2 support
+6. [solacreative](http://sola-la.com/creative "Taro Murao") - Max backups feature for aws/s3 and s3 gems

data/Rakefile

@@ -0,0 +1,7 @@
+require 'rubygems/package_task'
+
+spec = Gem::Specification.load(Dir['*.gemspec'].first)
+gem = Gem::PackageTask.new(spec)
+gem.define()
+
+#gem push pkg/heroku-mongo-backup-version.gem

data/heroku-mongo-backup.gemspec

@@ -0,0 +1,20 @@
+Gem::Specification.new do |s|
+  s.name    = 'heroku-mongo-backup-pm'
+  s.version = '0.1'
+  s.summary = 'Rake task backups mongo database on Heroku and push gzipped file to Amazon S3 or FTP.'
+  s.description = 'Rake task for backing up mongo database on heroku and push it to S3 or FTP. Library can be used as rake task or be easily integrated into daily cron job.'
+
+  s.authors  = ['Alex Kravets', 'matyi', 'Stef Lewandowski', 'David Hall', 'Dan Porter']
+  s.email    = 'santyor@gmail.com'
+  s.homepage = 'https://github.com/alexkravets/heroku-mongo-backup'
+
+  s.require_paths = ["lib"]
+  s.files         = `git ls-files`.split($\)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+
+  # Supress the warning about no rubyforge project
+  s.rubyforge_project = 'nowarning'
+
+  s.add_runtime_dependency 'mongo'
+end

data/lib/heroku-mongo-backup.rb

@@ -0,0 +1,239 @@
+# encoding: UTF-8
+
+require 'mongo'
+require 'json'
+require 'zlib'
+require 'uri'
+require 'yaml'
+require 'rubygems'
+require 'net/ftp'
+
+module HerokuMongoBackup
+
+  if defined?(Rails::Railtie)
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load "tasks/heroku_mongo_backup.rake"
+      end
+    end
+  end
+
+  require 's3_helpers'
+
+  class Backup
+    def chdir
+      Dir.chdir("/tmp")
+      begin
+        Dir.mkdir("dump")
+      rescue
+      end
+      Dir.chdir("dump")
+    end
+
+    def store
+      backup = {}
+  
+      @db.collections.each do |col|
+        backup['system.indexes.db.name'] = col.db.name if col.name == "system.indexes"
+    
+        records = []
+    
+        col.find().each do |record|
+          records << record
+        end
+
+        backup[col.name] = records
+      end
+  
+      marshal_dump = Marshal.dump(backup)
+  
+      file = File.new(@file_name, 'w')
+      file.binmode
+      file = Zlib::GzipWriter.new(file)
+      file.write marshal_dump
+      file.close
+    end
+
+    def load
+      file = Zlib::GzipReader.open(@file_name)
+      obj = Marshal.load file.read
+      file.close
+
+      obj.each do |col_name, records|
+        next if col_name =~ /^system\./
+    
+        @db.drop_collection(col_name)
+        dest_col = @db.create_collection(col_name)
+    
+        records.each do |record|
+          dest_col.insert record
+        end
+      end
+  
+      # Load indexes here
+      col_name = "system.indexes"
+      dest_index_col = @db.collection(col_name)
+      obj[col_name].each do |index|
+        if index['_id']
+          index['ns'] = index['ns'].sub(obj['system.indexes.db.name'], dest_index_col.db.name)
+          dest_index_col.insert index
+        end
+      end
+    end
+
+    def db_connect
+      uri = URI.parse(@url)
+      connection = ::Mongo::Connection.new(uri.host, uri.port)
+      @db = connection.db(uri.path.gsub(/^\//, ''))
+      @db.authenticate(uri.user, uri.password) if uri.user
+    end
+    
+    def ftp_connect
+      @ftp = Net::FTP.new(ENV['FTP_HOST'])
+      @ftp.passive = true
+      @ftp.login(ENV['FTP_USERNAME'], ENV['FTP_PASSWORD'])
+    end
+    
+    def ftp_upload
+      @ftp.putbinaryfile(@file_name)
+    end
+    
+    def ftp_download
+      open(@file_name, 'w') do |file|
+        file_content = @ftp.getbinaryfile(@file_name)
+        file.binmode
+        file.write file_content
+      end
+    end
+    
+    def s3_connect
+      bucket            = ENV['S3_BACKUPS_BUCKET']
+      if bucket.nil?
+        bucket          = ENV['S3_BACKUP_BUCKET']
+      end
+      if bucket.nil?
+        bucket          = ENV['S3_BACKUP']
+      end
+      if bucket.nil?
+        bucket          = ENV['S3_BUCKET']
+      end
+
+      access_key_id     = ENV['S3_KEY_ID']
+      if access_key_id.nil?
+        access_key_id   = ENV['S3_KEY']
+      end
+      if access_key_id.nil?
+        access_key_id   = ENV['AWS_ACCESS_KEY_ID']
+      end
+
+      secret_access_key = ENV['S3_SECRET_KEY']
+      if secret_access_key.nil?
+        secret_access_key = ENV['S3_SECRET']
+      end
+      if secret_access_key.nil?
+        secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
+      end
+
+      @bucket = HerokuMongoBackup::s3_connect(bucket, access_key_id, secret_access_key)
+    end
+
+    def s3_upload
+      HerokuMongoBackup::s3_upload(@bucket, @file_name)
+    end
+
+    def s3_download
+      open(@file_name, 'w') do |file|
+        file_content = HerokuMongoBackup::s3_download(@bucket, @file_name)
+        file.binmode
+        file.write file_content
+      end
+    end
+
+    def initialize connect = true
+      @file_name = Time.now.strftime("%Y-%m-%d.gz")
+  
+      if( ['production', 'staging'].include?(ENV['RAILS_ENV'] || ENV['RACK_ENV']) )
+
+        #config_template = ERB.new(IO.read("config/mongoid.yml"))
+        #uri = YAML.load(config_template.result)['production']['uri']
+        uri = ENV['MONGO_URL']
+
+        if uri.nil?
+          uri = ENV['MONGOHQ_URL']
+        end
+        if uri.nil?
+          uri = ENV['MONGOLAB_URI']
+        end          
+      
+      else
+        mongoid_config  = YAML.load_file("config/mongoid.yml")
+        config = {}
+        defaults        = mongoid_config['defaults']
+        dev_config      = mongoid_config['development']
+
+        config.merge!(defaults) unless defaults.nil?
+        config.merge!(dev_config)
+
+        host            = config['host']
+        port            = config['port']
+        database        = config['database']
+        uri = "mongodb://#{host}:#{port}/#{database}"
+
+        if uri == 'mongodb://:/' # new mongoid version 3.x
+          mongoid_config  = YAML.load_file("config/mongoid.yml")
+          dev_config      = mongoid_config['development']['sessions']['default']
+          host_port       = dev_config['hosts'].first
+          database        = dev_config['database']
+          uri = "mongodb://#{host_port}/#{database}"
+        end
+      end
+  
+      @url = uri
+  
+      puts "Using database: #{@url}"
+  
+      self.db_connect
+
+      if connect
+        if ENV['UPLOAD_TYPE'] == 'ftp'
+          self.ftp_connect
+        else
+          self.s3_connect
+        end
+      end
+    end
+
+    def backup files_number_to_leave=0
+      self.chdir    
+      self.store
+
+      if ENV['UPLOAD_TYPE'] == 'ftp'
+        self.ftp_upload
+        @ftp.close
+      else
+        self.s3_upload
+      end
+
+      if files_number_to_leave > 0
+        HerokuMongoBackup::remove_old_backup_files(@bucket, files_number_to_leave)
+      end
+    end
+    
+    def restore file_name, download_file = true
+      @file_name = file_name
+  
+      self.chdir
+      
+      if download_file
+        if ENV['UPLOAD_TYPE'] == 'ftp'
+          self.ftp_download
+          @ftp.close
+        else
+          self.s3_download
+        end
+      end
+
+      self.load
+    end
+  end
+end

data/lib/s3_helpers.rb

@@ -0,0 +1,147 @@
+begin
+  require 's3'
+  
+rescue LoadError
+    #
+    # There is no 's3' gem in Gmefile
+    #
+    #puts "There is no 's3' gem in Gemfile."
+end
+
+if defined?(S3)
+    #
+    # Using 's3' gem an Amazon S3 interface
+    #
+    #puts "Using \'s3\' gem as Amazon S3 interface."
+    def HerokuMongoBackup::s3_connect(bucket, key, secret)
+      service = S3::Service.new(:access_key_id     => key,
+                                :secret_access_key => secret)
+      bucket  = service.buckets.find(bucket)
+      return bucket
+    end
+
+    def HerokuMongoBackup::s3_upload(bucket, filename)
+      object = bucket.objects.build("backups/#{filename}")
+      object.content = open(filename)
+      object.save
+    end
+
+    def HerokuMongoBackup::s3_download(bucket, filename)
+      object  = bucket.objects.find("backups/#{filename}")
+      content = object.content(reload=true)
+
+      puts "Backup file:"
+      puts "  name: #{filename}"
+      puts "  type: #{object.content_type}"
+      puts "  size: #{content.size} bytes"
+      puts "\n"
+
+      return content
+    end
+
+    def HerokuMongoBackup::remove_old_backup_files(bucket, files_number_to_leave)
+      excess = ( object_keys = bucket.objects.find_all(:prefix => "backups/").map { |o| o.key }.sort ).count - files_number_to_leave
+      (0..excess-1).each { |i| bucket.objects.find(object_keys[i]).destroy } if excess > 0
+    end
+
+end
+
+
+
+begin
+  require 'aws/s3'
+rescue LoadError
+  #
+  # There is no 'aws/s3' in Gemfile
+  #
+  #puts "There is no 'aws/s3' gem in Gemfile."
+end
+
+if defined?(AWS)
+  #
+  # Using 'aws/s3' gem as Amazon S3 interface
+  #
+  #puts "Using \'aws/s3\' gem as Amazon S3 interface."
+  def HerokuMongoBackup::s3_connect(bucket, key, secret)
+    AWS::S3::Base.establish_connection!(:access_key_id     => key,
+                                        :secret_access_key => secret)
+    return bucket
+  end
+
+  def HerokuMongoBackup::s3_upload(bucket, filename)
+    AWS::S3::S3Object.store("backups/#{filename}", open(filename), bucket)
+  end
+
+  def HerokuMongoBackup::s3_download(bucket, filename)
+    content = AWS::S3::S3Object.value("backups/#{filename}", bucket)
+    return content
+  end
+
+  def HerokuMongoBackup::remove_old_backup_files(bucket, files_number_to_leave)
+    excess = ( object_keys = AWS::S3::Bucket.find(bucket).objects(:prefix => 'backups/').map { |o| o.key }.sort ).count - files_number_to_leave
+    (0..excess-1).each { |i| AWS::S3::S3Object.find(object_keys[i], bucket).delete } if excess > 0
+  end
+
+end
+
+
+
+
+begin
+  require 'fog'
+rescue LoadError
+  #
+  # There is no 'fog' in Gemfile
+  #
+  #puts "There is no 'fog' gem in Gemfile."
+end
+
+if defined?(Fog)
+  #
+  # Using 'aws/s3' gem as Amazon S3 interface
+  #
+  #puts "Using \'aws/s3\' gem as Amazon S3 interface."
+  def HerokuMongoBackup::s3_connect(bucket, key, secret)
+    connection = Fog::Storage.new({
+      :provider                 => 'AWS',
+      :aws_access_key_id        => key,
+      :aws_secret_access_key    => secret
+    })
+    directory = connection.directories.new(:key => bucket)
+    return directory
+  end
+
+  def HerokuMongoBackup::s3_upload(directory, filename)
+    file = directory.files.create(
+      :key    => "backups/#{filename}",
+      :body   => open(filename),
+      :public => true
+    )    
+  end
+
+  def HerokuMongoBackup::s3_download(directory, filename)
+    file = directory.files.get("backups/#{filename}")
+    return file.body
+  end
+
+  def HerokuMongoBackup::remove_old_backup_files(directory, files_number_to_leave)
+    total_backups = directory.files.all.size
+    
+    if total_backups > files_number_to_leave
+      
+      files_to_destroy = (0..total_backups-files_number_to_leave-1).collect{|i| directory.files.all[i] }
+      
+      files_to_destroy.each do |f|
+        f.destroy
+      end
+    end
+  end
+
+else
+  logging = Logger.new(STDOUT)
+  logging.error "\n\nheroku-mongo-backup: Please include 's3', 'aws/s3' or 'fog' gem in applications Gemfile for uploading backup to S3 bucket. (ignore this if using FTP)\n\n"
+end
+
+
+
+

data/lib/tasks/heroku_mongo_backup.rake

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+namespace :mongo do
+  desc "Backup prodution database and store it on S3.\n
+        Example of usage: rake mongo:backup OR rake mongo:backup MAX_BACKUPS=7"
+  task :backup => :environment do
+    if ENV['MAX_BACKUPS']
+      HerokuMongoBackup::Backup.new.backup(ENV['MAX_BACKUPS'].to_i)
+    else
+      HerokuMongoBackup::Backup.new.backup
+    end
+  end
+  
+  desc "Restore command gets backup file from S3 server or local file and pushes data to production db.\n
+        Example of usage: rake mongo:restore FILE=<backup-file.gz>"
+  task :restore => :environment do
+    if ENV['FILE']
+      HerokuMongoBackup::Backup.new.restore ENV['FILE']
+    elsif ENV['LOCAL']
+      HerokuMongoBackup::Backup.new(false).restore ENV['LOCAL'], false
+    else
+    	latest = Time.new.strftime("%Y-%m-%d.gz")
+    	 HerokuMongoBackup::Backup.new.restore latest
+    end
+  end
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: heroku-mongo-backup-pm
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Alex Kravets
+- matyi
+- Stef Lewandowski
+- David Hall
+- Dan Porter
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rake task for backing up mongo database on heroku and push it to S3 or
+  FTP. Library can be used as rake task or be easily integrated into daily cron job.
+email: santyor@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- Rakefile
+- heroku-mongo-backup.gemspec
+- lib/heroku-mongo-backup.rb
+- lib/s3_helpers.rb
+- lib/tasks/heroku_mongo_backup.rake
+homepage: https://github.com/alexkravets/heroku-mongo-backup
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: nowarning
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Rake task backups mongo database on Heroku and push gzipped file to Amazon
+  S3 or FTP.
+test_files: []