mys3ql 0.0.2 → 0.0.4

data/README.md

@@ -32,7 +32,7 @@ Second, create your config file:
 
     mysql:
       # Database to back up
-      database: aircms_production
+      database:
       # MySql credentials
       user:
       password:
@@ -41,7 +41,7 @@ Second, create your config file:
       # If you are using MySql binary logging:
       # Path to the binary logs, should match the bin_log option in your my.cnf.
       # Comment out if you are not using mysql binary logging
-      bin_log: /Users/andy/Desktop/mysql-bin
+      bin_log: /var/lib/mysql/binlog/mysql-bin
 
     s3:
       # S3 credentials
@@ -63,6 +63,10 @@ The MySQL user needs to have the RELOAD and the SUPER privileges, these can be g
     GRANT RELOAD ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password';
     GRANT SUPER ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password';
 
+You may need to run mys3ql's incremental backup with special permissions (sudo), depending on the ownership of the binlogs directory.
+
+N.B. the binary logs contain updates to all the databases on the server.  This means you can only switch on incremental backups for one database per server, because the logs will be purged each time a database is dumped.
+
 
 ## Inspiration
 
@@ -72,8 +76,8 @@ Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s
 ## To Do
 
 - tests ;)
-- restore (pull latest dump, pull bin files, pipe dump into mysql, pipe binfiles into mysql)
 - remove old dump files (s3)
+- restore from non-latest dump
 
 
 ## Questions, Problems, Feedback

data/bin/mys3ql

@@ -1,63 +1,35 @@
 #!/usr/bin/env ruby-local-exec
 
-# Hmm, is there a better way which doesn't force rubygems?
-require 'rubygems'
-require 'bundler/setup'
-
 lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
 $LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
 
 require 'mys3ql'
-require 'choice'
-
-Choice.options do
-
-  header ''
-  header 'Specific options:'
+require 'main'
 
-  option :full do
-    short '-f'
-    long '--full'
-    desc 'Dump database, push to S3, and clean up bin logs on S3'
+Main do
+  # consider using modes as/when we need command-specific arguments (e.g. restore specific backup)
+  argument 'command' do
+    validate { |command| %w[ full incremental restore ].include? command }
+    description 'full | incremental | restore'
   end
 
-  option :incremental do
-    short '-i'
-    long '--incremental'
-    desc "Push mysql's bin logs to S3"
+  option 'config', 'c' do
+    argument :required
+    description 'load configuration from YAML file'
+    defaults '~/.mys3ql'
   end
 
-  option :config do
-    short '-c'
-    long '--config=FILE'
-    desc 'Load configuration from YAML file (default: ~/.mys3ql)'
-    default '~/.mys3ql'
+  option 'debug', 'd' do
+    description 'be verbose'
   end
 
-  separator ''
-  separator 'General options:'
-
-  option :debug do
-    short '-d'
-    long '--debug'
-    desc 'Turn on debugging mode'
-  end
+  option 'version', 'v'
 
-  option :help do
-    long '--help'
-    desc 'Show this message'
-  end
-
-  option :version do
-    short '-v'
-    long '--version'
-    desc 'Show version'
-    action do
+  def run
+    if params[:version]
       puts "mys3ql v#{Mys3ql::VERSION}"
       exit
     end
+    Mys3ql::Conductor.run params[:command].value, params[:config].value, params[:debug].value
   end
-
 end
-
-Mys3ql::Conductor.run Choice.choices

data/lib/mys3ql/conductor.rb

@@ -1,3 +1,4 @@
+require 'tempfile'
 require 'mys3ql/config'
 require 'mys3ql/mysql'
 require 'mys3ql/s3'
@@ -5,15 +6,10 @@ require 'mys3ql/s3'
 module Mys3ql
   class Conductor
 
-    def self.run(options)
-      conductor = Conductor.new(options['config'])
-      conductor.debug = options[:debug]
-
-      if options['full']
-        conductor.full
-      elsif options['incremental']
-        conductor.incremental
-      end
+    def self.run(command, config, debug)
+      conductor = Conductor.new(config)
+      conductor.debug = debug
+      conductor.send command
     end
 
     def initialize(config_file = nil)
@@ -24,17 +20,49 @@ module Mys3ql
 
     def full
       @mysql.dump
-      @s3.push_dump_to_s3 @mysql.dump_file
-      @mysql.clean_up_dump
-      @s3.delete_bin_logs_on_s3
+      @s3.store @mysql.dump_file
+      @mysql.delete_dump
+      @s3.delete_bin_logs
     end
 
     def incremental
-      @s3.push_bin_logs_to_s3
+      @mysql.each_bin_log do |log|
+        @s3.store log, false
+      end
+    end
+
+    # for now only restore from latest
+    def restore
+      # get latest dump
+      with_temp_file do |file|
+        @s3.retrieve :latest, file
+        @mysql.restore file
+      end
+
+      # apply subsequent bin logs
+      @s3.each_bin_log do |log|
+        with_temp_file do |file|
+          @s3.retrieve log, file
+          @mysql.apply_bin_log file
+        end
+      end
+
+      # NOTE: not sure about this:
+      puts "You might want to flush mysql's logs..."
     end
 
     def debug=(val)
       @config.debug = val
     end
+
+    private
+
+    def with_temp_file(&block)
+      file = Tempfile.new 'mys3ql-sql'
+      yield file.path
+      nil
+    ensure
+      file.close!
+    end
   end
 end

data/lib/mys3ql/mysql.rb

@@ -8,22 +8,57 @@ module Mys3ql
       @config = config
     end
 
+    #
+    # dump
+    #
+
     def dump
-      cmd  = "#{@config.bin_path}mysqldump -u'#{@config.user}'"
-      cmd += " -p'#{@config.password}'" if @config.password
-      cmd += " --quick --single-transaction --create-options"
+      cmd  = "#{@config.bin_path}mysqldump"
+      cmd += ' --quick --single-transaction --create-options'
       cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging?
-      cmd += " #{@config.database} | gzip > #{dump_file}"
-      execute cmd
+      cmd += cli_options
+      cmd += " | gzip > #{dump_file}"
+      run cmd
+    end
+
+    def dump_file
+      @dump_file ||= "#{timestamp}.sql.gz"
     end
 
-    def clean_up_dump
+    def delete_dump
       File.delete dump_file
-      log "deleted #{dump_file}"
+      log "mysql: deleted #{dump_file}"
     end
 
-    def dump_file
-      @dump_file ||= "#{timestamp}.sql.gz"
+    #
+    # bin_logs
+    #
+
+    # flushes logs, yields each bar the last to the block
+    def each_bin_log(&block)
+      execute 'flush logs'
+      logs = Dir.glob("#{@config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
+      logs_to_backup = logs[0..-2]  # all logs except the last, which is in use
+      logs_to_backup.each do |log_file|
+        yield log_file
+      end
+      # delete binlogs from file system
+      #execute "purge master logs to '#{File.basename(logs[-1])}'"
+    end
+
+    #
+    # restore
+    #
+
+    def restore(file)
+      run "gunzip -c #{file} | #{@config.bin_path}mysql #{cli_options}"
+    end
+
+    def apply_bin_log(file)
+      cmd  = "#{@config.bin_path}mysqlbinlog --database=#{@config.database} #{file}"
+      cmd += " | #{@config.bin_path}mysql -u'#{@config.user}'"
+      cmd += " -p'#{@config.password}'" if @config.password
+      run cmd
     end
 
     private
@@ -36,5 +71,15 @@ module Mys3ql
       @config.bin_log && @config.bin_log.length > 0
     end
 
+    def cli_options
+      cmd  = " -u'#{@config.user}'"
+      cmd += " -p'#{@config.password}'" if @config.password
+      cmd += " #{@config.database}"
+    end
+
+    def execute(sql)
+      run %Q(#{@config.bin_path}mysql -e "#{sql}" #{cli_options})
+    end
+
   end
 end

data/lib/mys3ql/s3.rb

@@ -9,35 +9,66 @@ module Mys3ql
       @config = config
     end
 
-    def push_dump_to_s3(dump_file)
-      key      = "#{dumps_prefix}/#{dump_file}"
-      copy_key = "#{dumps_prefix}/latest.sql.gz"
-      s3_file  = push_to_s3 dump_file, key
-      if s3_file
+    def store(file, dump = true)
+      key = key_for(dump ? :dump : :bin_log, file)
+      s3_file = save file, key
+      if dump && s3_file
+        copy_key = key_for :latest
         s3_file.copy @config.bucket, copy_key
         log "copied #{key} to #{copy_key}"
       end
     end
 
-    def push_bin_logs_to_s3
-      if bin_logs_exist?
-        Dir["#{@config.bin_log}/*"].each do |file|
-          name = File.basename file
-          key = "#{bin_logs_prefix}/#{name}"
-          push_to_s3 file, key
-        end
+    def delete_bin_logs
+      each_bin_log do |file|
+        file.destroy
+        log "s3: destroyed #{file.key}"
       end
     end
 
-    def delete_bin_logs_on_s3
-      bucket.files.all(:prefix => "#{bin_logs_prefix}").each do |file|
-        file.destroy
-        log "destroyed #{file.key}"
+    def each_bin_log(&block)
+      bucket.files.all(:prefix => "#{bin_logs_prefix}").sort_by { |file| file.key[/\d+/].to_i }.each do |file|
+        yield file
       end
     end
 
+    def retrieve(s3_file, local_file)
+      key = (s3_file == :latest) ? key_for(:latest) : s3_file.key
+      get key, local_file
+    end
+
     private
 
+    def get(s3_key, local_file_name)
+      s3_file = bucket.files.get s3_key
+      File.open(local_file_name, 'wb') do |file|
+        file.write s3_file.body
+      end
+      log "s3: pulled #{s3_key} to #{local_file_name}"
+    end
+
+    # returns Fog::Storage::AWS::File if we pushed, nil otherwise.
+    def save(local_file_name, s3_key)
+      unless bucket.files.head(s3_key)
+        s3_file = bucket.files.create(
+          :key    => s3_key,
+          :body   => File.open(local_file_name),
+          :public => false
+        )
+        log "s3: pushed #{local_file_name} to #{s3_key}"
+        s3_file
+      end
+    end
+
+    def key_for(kind, file = nil)
+      name = File.basename file if file
+      case kind
+        when :dump;    "#{dumps_prefix}/#{name}"
+        when :bin_log; "#{bin_logs_prefix}/#{name}"
+        when :latest;  "#{dumps_prefix}/latest.sql.gz"
+      end
+    end
+
     def s3
       @s3 ||= begin
         s = Fog::Storage.new(
@@ -45,32 +76,20 @@ module Mys3ql
           :aws_secret_access_key => @config.secret_access_key,
           :aws_access_key_id     => @config.access_key_id
         )
-        log 'connected to s3'
+        log 's3: connected'
         s
       end
     end
 
     def bucket
       @directory ||= begin
-        d = s3.directories.get @config.bucket  # assume bucket exists
-        log "opened bucket #{@config.bucket}"
+        d = s3.directories.get @config.bucket
+        raise "S3 bucket #{@config.bucket} not found" unless d  # create bucket instead?
+        log "s3: opened bucket #{@config.bucket}"
         d
       end
     end
 
-    # returns Fog::Storage::AWS::File if we pushed, nil otherwise.
-    def push_to_s3(local_file_name, s3_key)
-      unless bucket.files.head(s3_key)
-        s3_file = bucket.files.create(
-          :key    => s3_key,
-          :body   => File.open(local_file_name),
-          :public => false
-        )
-        log "pushed #{local_file_name} to #{s3_key}"
-        s3_file
-      end
-    end
-
     def dumps_prefix
       "#{@config.database}/dumps"
     end

data/lib/mys3ql/shell.rb

@@ -2,7 +2,7 @@ module Mys3ql
   class ShellCommandError < RuntimeError ; end
 
   module Shell
-    def execute(command)
+    def run(command)
       log command
       result = `#{command}`
       log "==> #{result}"

data/lib/mys3ql/version.rb

@@ -1,3 +1,3 @@
 module Mys3ql
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

data/mys3ql.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency 'choice', '~> 0.1.4'
+  s.add_dependency 'main', '~> 4.8.0'
   s.add_dependency 'fog', '~> 1.0.0'
   s.add_development_dependency 'rake'
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mys3ql
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: 
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Andy Stewart
@@ -15,29 +15,29 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-31 00:00:00 +01:00
+date: 2011-11-03 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: choice
+  name: main
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 19
+        hash: 31
         segments: 
-        - 0
-        - 1
         - 4
-        version: 0.1.4
+        - 8
+        - 0
+        version: 4.8.0
   type: :runtime
-  version_requirements: *id001
+  requirement: *id001
 - !ruby/object:Gem::Dependency 
   name: fog
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -49,11 +49,11 @@ dependencies:
         - 0
         version: 1.0.0
   type: :runtime
-  version_requirements: *id002
+  requirement: *id002
 - !ruby/object:Gem::Dependency 
   name: rake
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -63,7 +63,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id003
+  requirement: *id003
 description: Simple backup of your MySql database onto Amazon S3.
 email: 
 - boss@airbladesoftware.com