mysql_mirror 0.1.1

data/README.rdoc

@@ -0,0 +1,43 @@
+= Mysql Mirror 
+Use MysqlMirror to mirror data between databases. This can be useful when you want to update your
+development or staging environments with real data to work with. Or, if you do some heavy lifting
+calculations on another server, you might want to use a seperate database on another host, etc.
+ 
+=== General Approach
+- Mirror Across Hosts: performs a mysql_dump to an sql file, then imports file to target host
+- Mirror Same Host: uses CREATE TABLE ( SELECT ... ) style for mirroring. Much faster than mysql_dump
+
+Note:
+ALL information will be lost in the tables mirrored to the Target Database
+
+== Dependencies
+- Active Record
+- FileUtils
+
+== Usage
+Basic usage, copy production db to development
+    @m = MysqlMirror.new({
+     :source => :production,
+     :target => :development
+   })
+ 
+Choose what tables you want to bring over and how you want to scope them...
+   @m = MysqlMirror.new({
+     :source => :production,
+     :target => :development,
+     :tables => [:users, :widgets],
+     :where => {:users => "is_admin NOT NULL"},
+   })
+ 
+Database information not in your database.yml file? (Or Not Running Rails?) No Problem!
+   @m = MysqlMirror.new({
+     :source => { :database => "app_production", :user => ..., :password => ..., :hostname => ...},
+     :target => {:database => "app_development", :hostname => 'localhost'}
+   })
+ 
+Want to use everything in :production environment (user, pass, host) but need to change the database?
+   @m = MysqlMirror.new({
+     :source => :production,
+     :override => {:source => {:database => "heavy_calculations_database"}},
+     :target => :production
+   })

data/Rakefile

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "mysql_mirror" 
+    gemspec.summary = "Helps mirror MySql Databases"
+    gemspec.description = "Will mirror tables / databases between mysql databases and across hosts"
+    gemspec.email = "peterleonhardt@gmail.com"
+    gemspec.homepage = "http://github.com/pjleonhardt/mysql_mirror"
+    gemspec.authors = ["Peter Leonhardt", "Joe Goggins"]      
+    gemspec.files = FileList["[A-Z]*", "lib/mysql_mirror.rb"]
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Please install the jeweler gem."
+end
+
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/lib/mysql_mirror.rb

@@ -0,0 +1,190 @@
+require 'active_record'
+require 'fileutils'
+
+class MysqlMirror
+  class MysqlMirrorException < Exception; end
+  
+  class Source < ActiveRecord::Base
+  end
+  
+  class Target < ActiveRecord::Base
+  end
+  
+  attr_accessor :tables, :where
+  
+  def initialize(options = {})
+    unless ([:source, :target] - options.keys).blank?
+      # Need to specify a Source and Target database
+      raise MysqlMirrorException.new("You must specify both Source and Target connections")
+    end
+    
+    self.tables = options.delete(:tables)
+    self.where  = options.delete(:where)
+    
+    overrides = options.delete(:override)       || {}
+    source_override = overrides.delete(:source) || {}
+    target_override = overrides.delete(:target) || {}
+    
+    @source_config = get_configuration(options.delete(:source))
+    @target_config = get_configuration(options.delete(:target))
+    
+    @source_config.merge!(source_override)
+    @target_config.merge!(target_override)    
+    
+    # @commands is an array of methods to call
+    if mirroring_same_host?
+      @commands = commands_for_local_mirror
+    else
+      @commands = commands_for_remote_mirror
+    end
+  end
+  
+  def commands_for_local_mirror
+    [:local_copy]
+  end
+  
+  def commands_for_remote_mirror
+    [ 
+      :remote_mysqldump,
+      :remote_tmp_file_table_rename,
+      :remote_insert_command,
+      :remote_rename_tmp_tables,
+      :remote_remove_tmp_file 
+    ]
+  end
+  
+  def mirroring_same_host?
+    @source_config[:host] == @target_config[:host]
+  end
+  
+  def execute!
+    @start_time = Time.now
+    @source = connect_to(:source)
+    @target = connect_to(:target)
+    
+    @commands.each do |c|
+      self.send(c)
+    end
+  end
+  
+  
+private
+  #   e.g, connect_to(:source)
+  #      => MysqlMirror::Source.establish_connection(@source_config).connection
+  #
+  def connect_to(which)
+    "MysqlMirror::#{which.to_s.classify}".constantize.establish_connection(self.instance_variable_get("@#{which}_config")).connection
+  end
+
+  def local_copy
+    get_tables.each do |table|
+      target_db = @target_config[:database]
+      source_db = @source_config[:database]
+      target_table     = "#{target_db}.#{table}"
+      target_tmp_table = "#{target_db}.#{table}_MirrorTmp"
+      target_old_table = "#{target_db}.#{table}_OldMarkedToDelete"
+      source_table     = "#{source_db}.#{table}"
+      
+      
+      prime_statement_1  = "DROP TABLE IF EXISTS #{target_tmp_table}"
+      prime_statement_2  = "CREATE TABLE IF NOT EXISTS #{target_table} LIKE #{source_table}"
+      
+      create_statement = "CREATE TABLE #{target_tmp_table} LIKE #{source_table}"
+      
+        select_clause = "SELECT * FROM #{source_table}"
+        select_clause << " WHERE #{self.where[table]}" unless (self.where.blank? or self.where[table].blank?)
+      
+      insert_statement  = "INSERT INTO #{target_tmp_table} #{select_clause}"
+      rename_statement  = "RENAME TABLE #{target_table} TO #{target_old_table}, #{target_tmp_table} TO #{target_table}"
+      cleanup_statement = "DROP TABLE IF EXISTS #{target_old_table}"
+      
+      staments_to_run = [prime_statement_1, prime_statement_2, create_statement, insert_statement, rename_statement, cleanup_statement]
+      
+      staments_to_run.each do |statement|
+        @target.execute(statement)
+      end
+    end    
+  end
+  
+  def mysqldump_command_prefix
+    "mysqldump --compact=TRUE --max_allowed_packet=100663296 --extended-insert=TRUE --lock-tables=FALSE --add-locks=FALSE --add-drop-table=FALSE"
+  end
+  
+  def remote_mysqldump
+    @tmp_file_name = "mysql_mirror_#{@start_time.to_i}.sql"
+    tables = get_tables.map(&:to_s).join(" ")
+    where  = self.where.blank? ? "" : "--where\"#{@source_config[:where]}\""
+    config = "-u#{@source_config[:username]} -p'#{@source_config[:password]}' -h #{@source_config[:host]} #{@source_config[:database]}"
+    
+    the_cmd = "#{mysqldump_command_prefix} #{where} #{config} #{tables} > #{@tmp_file_name}"
+    puts the_cmd
+    `#{the_cmd}`
+  end
+  
+  def remote_tmp_file_table_rename
+    create_or_insert_regex = Regexp.new('(^CREATE TABLE|^INSERT INTO)( `)(.+?)(`)(.+)')
+    new_file_name = @tmp_file_name + ".replaced.sql"
+    
+    new_file = File.new(new_file_name, "w")
+    IO.foreach(@tmp_file_name) do |line|
+      if match_data = line.match(create_or_insert_regex)
+        table_name = match_data[3]
+        new_table_name = "#{table_name}_#{@start_time.to_i}"
+        new_file.puts match_data[1] + match_data[2] + new_table_name + match_data[4]+ match_data[5]
+      else
+        new_file.puts line
+      end
+    end
+    new_file.close
+    # replace dump'd sql file with this gsub'd one
+    FileUtils.move(new_file_name, @tmp_file_name)
+  end
+  
+  def remote_insert_command
+    config = "-u#{@target_config[:username]} -p'#{@target_config[:password]}' -h #{@target_config[:host]} #{@target_config[:database]}"
+    the_cmd = "mysql #{config} < #{@tmp_file_name}"
+    `#{the_cmd}`
+  end
+  
+  def remote_rename_tmp_tables
+    get_tables.each do |table|
+      tmp_table_name = "#{table}_#{@start_time.to_i}"
+      old_table_name = "#{table}_OldMarkedToDelete"
+      
+      @target.transaction do 
+        @target.execute("DROP TABLE IF EXISTS #{old_table_name}")
+        @target.execute("RENAME TABLE #{table} TO #{old_table_name}, #{tmp_table_name} TO #{table}")
+        @target.execute("DROP TABLE IF EXISTS #{old_table_name}")        
+      end
+    end 
+  end
+  
+  def remote_remove_tmp_file
+    FileUtils.rm(@tmp_file_name)
+  end
+  
+  def get_tables
+    the_tables = self.tables.blank? ? @source.select_values("SHOW TABLES").map!(&:to_sym) : self.tables
+  end
+
+
+  def get_configuration(env_or_hash)
+    config = env_or_hash
+    
+    if(env_or_hash.is_a? Symbol)
+      config = ActiveRecord::Base.configurations[env_or_hash.to_s]
+    end
+    
+    config.symbolize_keys
+  end
+  
+end
+
+
+
+
+
+
+
+
+

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification 
+name: mysql_mirror
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
+platform: ruby
+authors: 
+- Peter Leonhardt
+- Joe Goggins
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-03-26 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: Will mirror tables / databases between mysql databases and across hosts
+email: peterleonhardt@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/mysql_mirror.rb
+has_rdoc: true
+homepage: http://github.com/pjleonhardt/mysql_mirror
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Helps mirror MySql Databases
+test_files: []
+