sqlup 0.0.1

data/lib/mysql_backup/server.rb

@@ -0,0 +1,237 @@
+require 'rubygems'
+require 'named_arguments'
+require 'active_record/vendor/mysql'
+require 'mysql_backup/entity'
+require 'mysql_backup/entity/logs'
+require 'mysql_backup/entity/files/innodb'
+require 'mysql_backup/entity/files/myisam'
+
+module MysqlBackup; end
+
+# To get the current log position:
+#
+#   show_master_status => {:file => 'logfilename_0004', :position => 386}
+#
+# To get a list of the available log files:
+#
+#   show_binary_logs => %w(logfilename_0003 logfilename_0004)
+# 
+# To get a list of closed log files:
+#
+#   completed_logs => %w(logfilename_0003)
+#
+# To get a MysqlBackup::InnodbFiles object:
+# 
+#   
+class MysqlBackup::Server
+  include NamedArguments 
+  
+  # The MySQL connection
+  attr_accessor :connection
+  
+  # Returns just the base element of innodb_data_file_path.
+  # Given 
+  #   ibdata1:10M:autoextend
+  # Return 
+  #   ibdata
+  def innodb_data_file_path
+    v = show_variables
+    path = v[:innodb_data_file_path]
+    path = path.split(':').first
+    path =~ /(.*)\d+$/
+    $1
+  end
+  
+  def innodb_data_home_dir
+    innodb_data_home_dir = show_variables[:innodb_data_home_dir]
+    return innodb_data_home_dir unless innodb_data_home_dir.empty? 
+    datadir
+  end
+  
+  def datadir
+    show_variables[:datadir]
+  end
+  
+  def log_bin_dir
+    @log_bin_dir || datadir
+  end
+  
+  # Attempts to recreate the option passed
+  # to log-bin.
+  def log_bin
+    logs = show_binary_logs
+    logs.last =~ /(.*)\.\d+$/
+    $1
+  end
+  
+  # Returns an array of log file names.
+  #
+  # If you need the file sizes, call 
+  # query_to_array 'show binary logs'
+  # directly.
+  #
+  # Returns [] if there are no logs.
+  def show_binary_logs
+    query_to_list 'show binary logs', 'Log_name'
+  end
+  
+  def create_innodb_files_obj
+    MysqlBackup::Entity::Files::Innodb.new :datadir => datadir, :innodb_data_home_dir => innodb_data_home_dir, :innodb_data_file_path => innodb_data_file_path
+  end
+  
+  def create_myisam_files_obj
+    MysqlBackup::Entity::Files::Myisam.new :datadir => datadir
+  end
+  
+  def create_logs_obj
+    result = nil
+    with_lock do |log_identifier|
+      i = log_identifier.merge :log_bin_dir => log_bin_dir, :completed_logs => completed_logs
+      result = MysqlBackup::Entity::Logs.new i
+    end
+    result
+  end
+  
+  # Return a hash of the mysql system variables.
+  # 
+  # The keys are lower case symbols.
+  def show_variables
+    query_to_hash 'show variables', 'Variable_name', "Value"
+  end
+  
+  # Returns a hash containing:
+  #
+  #   :file => the name of the current log file
+  #   :position => the position in the current log file where the next entry will be written
+  def show_master_status
+    r = query_to_array_of_symbolic_hashes 'show master status'
+    r.first
+  end
+  
+  # Returns true if any entries have been written to the current log file.
+  def current_log_has_entries?
+    m = show_master_status
+    return false if m.empty?
+    m['Position'].to_i > 4
+  end
+  
+  # Returns a list of all log files except the open log.
+  def completed_logs
+    result = show_binary_logs
+    result.pop
+    result
+  end
+  
+  ######################
+  # Internal methods below this point
+  ######################
+  
+  # Call <tt>flush logs</tt>.
+  def flush_logs!
+    query_to_array 'flush logs'
+  end
+  
+  def with_lock
+    lock_tables
+    m = show_master_status
+    raise RuntimeError, "No results from show master status.  Is binary logging enabled?" unless m
+    yield :log_file => m[:file], :log_position => m[:position]
+  ensure
+    unlock_tables
+  end
+  
+  def lock_tables
+    query_to_array 'FLUSH TABLES WITH READ LOCK'
+  end
+  
+  def unlock_tables
+    query_to_array 'UNLOCK TABLES'
+  end
+  
+  # Required arguments:
+  #
+  #   :connection => The MySQL connection
+  def initialize args = {}
+    super
+  end
+  
+  # Run the query given in +q+ and
+  # return the value in the
+  # result identified by +field+.
+  # 
+  # Return nil if no rows are returned.
+  # There is no way to distinguish
+  # between no rows and a nil
+  # value.  Call query_to_list
+  # if you need that functionality.
+  def query_to_value q, field
+    result = query_to_list q, 'Value'
+    result.first
+  end
+  
+  # Return a list of values
+  # from the column +field+
+  # from the query +q+. 
+  #
+  # Return [] if no rows
+  # are returned.
+  def query_to_list q, field
+    r = query_to_array(q).map {|l| l[field.downcase.to_sym]}
+    r
+  end
+  
+  # Return the rows of the query as
+  # an array of 
+  def query_to_array q
+    result = []
+    a = @connection.query(q)
+    a && a.each_hash do |l|
+      result << hash_to_symbolic_hash(l)
+    end
+    result
+  end
+  
+  def query_to_array_of_symbolic_hashes q #:nodoc:
+    result = query_to_array q
+    result.map {|r| hash_to_symbolic_hash r}
+  end
+  
+  # Run the query and return all rows
+  # as key/value pairs in a hash.
+  def query_to_hash q, k_name, v_name #:nodoc:
+    result = {}
+    @connection.query(q).each_hash do |l|
+      k = l[k_name]
+      result[k.to_s.downcase.to_sym] = normalized_value l[v_name]
+    end
+    result
+  end
+  
+  # Run the query given in +q+ and
+  # return the first row as a hash.
+  # 
+  # Return nil if no rows are returned.
+  def query_to_single_hash q #:nodoc:
+    result = query_to_array q
+    hash_to_symbolic_hash result.first
+  end
+  
+  def hash_to_symbolic_hash h #:nodoc:
+    result = {}
+    h.each_pair do |k,v|
+      result[k.to_s.downcase.to_sym] = normalized_value v
+    end
+    result
+  end
+  
+  def normalized_value v #:nodoc:
+    i = v.to_i
+    v = i if i.to_s == v.to_s
+    v = case v
+    when /^(on|yes)$/i: true
+    when /^(off|no)$/i: nil
+    else v
+    end
+    v
+  end
+end

data/lib/mysql_backup/storage/s3.rb

@@ -0,0 +1,110 @@
+require 'aws/s3'
+require 'mysql_backup/storage'
+
+class MysqlBackup::Storage::S3 < MysqlBackup::Storage
+  # The name of the bucket storing backup objects
+  attr_accessor :bucket
+  
+  # The AWS::S3::Bucket object
+  attr_writer :bucket_obj
+  
+  # Takes:
+  #
+  #   :access_key_id     => 'abc',
+  #   :secret_access_key => '123'
+  #   :bucket => 'name_of_the_backup_bucket'
+  def initialize args = {}
+    super
+    connect! args unless args[:no_connect]
+  end
+  
+  # +identifier+ is a MysqlBackup::Entity::Identifier
+  # object.
+  def save args
+    identifier = args[:identifier]
+    log && log.info("saving to S3(#{@bucket}): #{identifier}")
+    AWS::S3::S3Object.store identifier.to_s, args[:file].open, @bucket
+    mark_as_existing identifier.to_s
+  end
+  
+  # +identifier+ is a MysqlBackup::Entity::Identifier
+  # object.
+  def conditional_save args
+    identifier = args[:identifier]
+    if include? identifier
+      log && log.info("not saving; object exists in (#{@bucket}): #{identifier}")
+    else
+      save args
+    end
+  end
+  
+  def rm identifier
+    AWS::S3::S3Object.delete identifier.to_s, bucket
+  end
+  
+  def include? key
+    @existing_objects ||= {}
+    @existing_objects.include? key.to_s
+  end
+  
+  def mark_as_existing key
+    @existing_objects ||= {}
+    @existing_objects[key.to_s] = nil
+  end
+  
+  def read_existing_objects
+    yield_objects do |o|
+      mark_as_existing s3_object_to_identifier(o).to_s
+      true
+    end
+  end
+  
+  def yield_objects n_keys = 1000 #:nodoc:
+    objs = []
+    while objs
+      objs = bucket_obj.objects :max_keys => n_keys, :marker => (objs.empty? ? nil : objs.last.key)
+      objs.each do |o|
+        yield o
+      end
+      objs = nil if objs.empty? || objs.length < n_keys
+    end
+  end
+  
+  def yield_backup_objects
+    yield_objects do |o|
+      i = MysqlBackup::Entity::Identifier.build_object :string
+      yield o if i
+    end
+  end
+  
+  def yield_identifiers
+    yield_objects do |o|
+      i = s3_object_to_identifier o
+      yield i if i
+    end
+  end
+  
+  def retrieve_backup_and_then_yield_file group
+    Tempfile.open 'backup_retrieval' do |f|
+      group.identifiers.each do |i|
+        AWS::S3::S3Object.stream i.to_s, bucket do |chunk|
+          f.write chunk
+        end
+      end
+      f.seek 0
+      yield f
+    end
+  end
+  
+  def s3_object_to_identifier o
+    MysqlBackup::Entity::Identifier.build_object :string => o.key, :timestamp => o.last_modified
+  end
+  
+  def bucket_obj
+    @bucket_obj ||= AWS::S3::Bucket.find bucket
+  end
+  
+  def connect! args
+    AWS::S3::Base.establish_connection! :access_key_id => args[:access_key_id], :secret_access_key => args[:secret_access_key]
+  end
+end

data/lib/mysql_backup/storage.rb

@@ -0,0 +1,13 @@
+require 'rubygems'
+require 'named_arguments'
+
+require 'mysql_backup/entity/identifier'
+
+module MysqlBackup; end
+
+class MysqlBackup::Storage
+  include NamedArguments
+  
+  # The Logger object
+  attr_accessor :log
+end

data/lib/mysql_backup/utilities/factory_create_method.rb

@@ -0,0 +1,51 @@
+module FactoryCreateMethod
+  def self.included target
+    target.class_eval do
+      include NamedArguments::MethodExtensions
+      extend ClassMethods
+      define_method_with_value :factory_create_base_class, target
+    end
+  end
+  
+  class NoMatchingFactory < NoMethodError; end
+  
+  module ClassMethods
+    # Creates the appropriate object.
+    # 
+    # Throws NoMethodError if no factory method
+    # responded to the arguments.
+    # 
+    # Use build_object to return nil instead
+    # of throwing an exception.
+    def create_object args = {}
+      factory_create_base_class.factory_methods.each do |f|
+        result = f.call args
+        return result if result
+      end
+      raise FactoryCreateMethod::NoMatchingFactory, "No factory method matched #{args.inspect}"
+    end
+    
+    # Note that this method will be replaced on the
+    # first call to append_factory_method.
+    def factory_methods # nodoc
+      return []
+    end
+    
+    def build_object args = {}
+      create_object args
+    rescue
+      nil
+    end
+    
+    def append_factory_method &block
+      orig = factory_create_base_class
+      orig.define_method_with_value :factory_methods, [block] + factory_methods
+    end
+    
+    def new_if_class klass, field
+      append_factory_method do |args|
+        klass === args[field] && new(args)
+      end
+    end
+  end
+end

data/lib/mysql_backup.rb

@@ -0,0 +1,8 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'mysql_backup/librarian'
+require 'mysql_backup/storage/s3'
+
+module MysqlBackup
+  VERSION = '0.0.1'
+end

data/lib/sqlup.rb

@@ -0,0 +1,2 @@
+#$:.unshift File.expand_path(File.dirname(__FILE__))
+#require 'pp'

data/test/unit/mysql_backup/entity/files/files_test.rb

@@ -0,0 +1,45 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'mysql_backup/entity/files'
+
+class MysqlBackup::Entity::FilesTest < Test::Unit::TestCase
+  def test_class_create_tar_files
+    build_result = {:files => ['log.001'], :log_position => 12, :n_parts => 1, :log_file => 'log.001', :log_position => 0}
+    MysqlBackup::Entity::Files.expects(:build_files).times(0..100).returns(build_result)
+    expects(:must_call)
+    MysqlBackup::Entity::Files.create_tar_files do |args|
+      must_call
+      assert_kind_of MysqlBackup::Entity::Identifier, args[:identifier]
+    end
+  end
+  
+  def test_class_build_files
+    o1 = stub(:required_path_strings => ['/tmp/a'], :confirm_required_paths_are_readable => nil)
+    o2 = stub(:required_path_strings => ['/tmp/b'], :confirm_required_paths_are_readable => nil)
+    ms = stub('mysqlserver')
+    ms.expects(:with_lock).yields(:log_file => 'snark', :log_position => 3)
+     (MysqlBackup::Entity::Files.expects(:tar_files).with {|t| t == %w(/tmp/a /tmp/b)}).returns([])
+    MysqlBackup::Entity::Files.build_files :mysql_server => ms, :mysql_files => [o1, o2] 
+  end
+  
+  def test_class_do_tar
+  end
+  
+  def test_class_tar_files
+    MysqlBackup::Entity::Files.expects(:do_tar).with {|x| x[:cmd] =~ /tar.*tmp.a.*gzip.*split/}
+    files = MysqlBackup::Entity::Files.tar_files ['/tmp/a']
+  end
+  
+  def test_required_path_strings
+    f = MysqlBackup::Entity::Files.new
+    f.expects(:required_paths).returns([])
+    f.required_path_strings
+  end
+  
+  def test_set_path_vars
+    f = MysqlBackup::Entity::Files.new
+    f.expects(:foo=)
+    Pathname.any_instance.expects(:readable?).returns true
+    f.instance_variable_set(:@foo, '/tmp/foo')
+    f.set_path_vars [:foo], :foo => '/tmp/foo'
+  end
+end

data/test/unit/mysql_backup/entity/files/innodb_test.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'mysql_backup/entity/files/innodb'
+
+class MysqlBackup::Entity::Files::InnodbTest  < Test::Unit::TestCase
+  def test_required_paths
+    paths = std_innodb.required_paths
+    assert_equal ["/tmp/fakeinnodb1", "/tmp/mock_database"].sort, (paths.map {|px| px.cleanpath.to_s}).sort
+  end
+  
+  def test_innodb_database_dirs
+    s = std_innodb
+    result = s.innodb_database_dirs
+    assert_equal [Pathname.new('/tmp/mock_database')], result
+  end
+  
+  def test_innodb_files
+    assert_equal [Pathname.new('/tmp/fakeinnodb1')], std_innodb.innodb_files
+  end
+  
+  protected
+  
+  def std_innodb
+    @innodb = MysqlBackup::Entity::Files::Innodb.new :datadir => @basedir, :innodb_data_home_dir => @basedir, :innodb_data_file_path => 'fakeinnodb'
+  end
+  
+  def setup
+    @basedir = '/tmp'
+    @basepath = Pathname.new @basedir
+    p = @basepath + 'mock_database'
+    p.mkdir unless p.directory?
+    q = p + 'foo.frm'
+    q.open 'w' do |qf|
+      qf << 'testing'
+    end
+    # Build fake innodb files
+    i = @basepath + 'fakeinnodb1'
+    i.open 'w' do |innofile|
+      innofile << 'testing'
+    end
+  end
+  
+  def teardown
+    p = @basepath + 'mock_database'
+    Pathname.glob(p + '*').each do |f|
+      f.unlink
+    end
+    p.rmdir
+    Pathname.glob(@basepath + 'fakeinnodb*').each {|pr| pr.delete}
+  end
+end

data/test/unit/mysql_backup/entity/files/myisam_test.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'mysql_backup/entity/files/myisam'
+
+class MysqlBackup::Entity::Files::MyisamTest < Test::Unit::TestCase
+  def test_required_paths
+    paths = std_myisam.required_paths
+    assert_equal ["/tmp/mock_myisam_database"].sort, (paths.map {|px| px.cleanpath.to_s}).sort
+  end
+  
+  def test_myisam_database_dirs
+    s = std_myisam
+    assert_equal s.datadir, '/tmp'
+    result = s.myisam_database_dirs
+    assert_equal [Pathname.new('/tmp/mock_myisam_database')], result
+  end
+  
+  protected
+  
+  def std_myisam
+    MysqlBackup::Entity::Files::Myisam.new :datadir => @basedir
+  end
+  
+  def setup
+    @basedir = '/tmp'
+    @basepath = Pathname.new @basedir
+    p = @basepath + 'mock_myisam_database'
+    p.mkdir
+    q = p + 'foo.MYD'
+    q.open 'w' do |qf|
+      qf << 'testing'
+    end
+  end
+  
+  def teardown
+    p = @basepath + 'mock_myisam_database'
+    Pathname.glob(p + '*').each do |f|
+      f.unlink
+    end
+    p.rmdir
+    Pathname.glob(@basepath + 'fakeinnodb*').each {|pr| pr.delete}
+  end
+end

data/test/unit/mysql_backup/entity/identifier_test.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../test_helper'
+require 'mysql_backup/entity/identifier'
+
+class MysqlBackup::Entity::IdentifierTest < Test::Unit::TestCase
+  def test_initialize
+    n = MysqlBackup::Entity::Identifier.create_object :category => :full, :type => :mysqldump
+    assert_kind_of MysqlBackup::Entity::Identifier, n
+    assert_equal "full:type_mysqldump:log_file_:log_position_0000000000:n_parts_0000000000:part_number_0000000000", n.to_s
+  end
+  
+  def test_object_creation
+    n = MysqlBackup::Entity::Identifier.create_object :category => :full, :type => :binary
+    assert_kind_of MysqlBackup::Entity::Identifier::Full::Binary, n
+  end
+  
+  def test_create_object_with_string
+    # Current files
+    s = 'log:type_current:log_file_thelog.0000000006:log_position_0000000311:n_parts_0000000001:part_number_0000000000'
+    n = MysqlBackup::Entity::Identifier.create_object :string => s
+    assert_equal 'log:type_current:log_file_thelog.0000000006:log_position_0000000311', n.to_s
+    assert_kind_of MysqlBackup::Entity::Identifier::Log::Current, n
+    assert_no_match(/n_parts/, n.to_s)
+    
+    # Complete files
+    s = 'log:type_complete:log_file_thelog.0000000006'
+    n = MysqlBackup::Entity::Identifier.create_object :string => s
+    assert_equal s, n.to_s
+    assert_kind_of MysqlBackup::Entity::Identifier::Log::Complete, n
+  end
+  
+  def test_class_string_to_args
+    a = MysqlBackup::Entity::Identifier.string_to_args('log:type_complete:log_file_thelog.000001')
+    assert_equal :log, a[:category]
+    assert_equal :complete, a[:type]
+    assert_equal 'thelog.000001', a[:log_file]
+
+    a = MysqlBackup::Entity::Identifier.string_to_args('log:type_current:log_file_thelog.0000000006:log_position_0000000311:n_parts_0000000001:part_number_0000000000')
+    assert_equal 311, a[:log_position]
+    assert_equal 0, a[:part_number]
+
+    # log:type_current:log_file_thelog.0000000006:log_position_0000000311:n_parts_0000000001:part_number_0000000000
+    # full:type_mysqldump:log_file_thelog.0000000006:log_position_0000000382:n_parts_0000000001:part_number_0000000000
+    # full:type_binary:log_file_thelog.0000000006:log_position_0000000311:n_parts_0000000001:part_number_0000000000
+  end
+  
+  def test_log_file_number
+    t = MysqlBackup::Entity::Identifier.new :log_file => 'testing.003'
+    assert_equal 3, t.log_file_number
+  end
+end

data/test/unit/mysql_backup/entity/logs_test.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'mysql_backup/entity/logs'
+
+class MysqlBackup::Entity::LogsTest < Test::Unit::TestCase
+  def test_save
+    l = MysqlBackup::Entity::Logs.new :log_file => 'foo.0001', :log_position => 3, :completed_logs => ['foo.0001'], :log_bin_dir => '/tmp'
+    expects(:must_call).times(2)
+    l.save do |i|
+      must_call
+      assert_equal 'foo.0001', i[:identifier].log_file
+    end
+  end
+end

data/test/unit/mysql_backup/entity/mysqldump_test.rb

@@ -0,0 +1,64 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'mysql_backup/entity/mysqldump'
+
+class MysqlBackup::Entity::MysqldumpTest < Test::Unit::TestCase
+  def test_create
+    mysqldump_obj = MysqlBackup::Entity::Mysqldump.new
+     (mysqldump_obj.expects(:system).with {|m| m =~ /mysqldump --opt/ && m !~ /#</}).returns true
+    mysqldump_obj.expects(:get_log_position).returns(true)
+    mysqldump_obj.expects(:compress_and_split).returns([stub])
+    expects(:must_call)
+    mysqldump_obj.create do |args|
+      must_call
+      i = args[:identifier]
+      assert_equal 0, i.part_number
+      assert_equal 1, i.n_parts
+    end
+  end
+  
+  def test_get_log_position
+    m = MysqlBackup::Entity::Mysqldump.new
+    m.get_log_position mysqldump_sample_data
+    assert_equal 'thelog.000005', m.log_file
+    assert_equal 1234, m.log_position
+  end
+  
+  def mysqldump_sample_data
+    StringIO.new <<-EOS
+-- MySQL dump 10.10
+--
+-- Host: localhost    Database: test
+-- ------------------------------------------------------
+-- Server version       5.0.27-standard-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0
+*/;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Position to start replication or point-in-time recovery from
+--
+
+-- CHANGE MASTER TO MASTER_LOG_FILE='thelog.000005', MASTER_LOG_POS=1234;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2007-06-05  0:53:56
+EOS
+  end
+end