db-copier 0.0.1

data/README.textile

@@ -0,0 +1,75 @@
+h2. DbCopier
+
+DbCopier is a DSL around Sequel for quickly making back-ups/copies across various database systems with minimal fuss: 
+
+#my_copy_script.rb
+require 'rubygems'
+require 'db-copier'
+source_db => {:adapter => 'mysql', :host => 'localhost', :user => 'root', :password => '', :database => 'db_copier_test_src'} 
+target_db => {:adapter => 'mysql', :host => 'localhost', :user => 'root', :password => '', :database => 'db_copier_test_target'}
+DbCopier.app do
+  copy :from => source_db, :to => target_db
+end
+
+This would create tables and indexes in the target_db for you (if they do not exist) and copy over contents from the source and best of all it <notextile>*works with any database that works with Sequel*</notextile> (which is a pretty damn big set) -- so you can copy your stuff over from mysql over to postgres, no worries.
+
+h3. Installation
+
+sudo gem install db-copier
+
+h3. Options
+
+You can specify how many rows you would like to copy at a time via the <notextile>*rows_per_copy*</notextile> (default is 50) attribute.
+copy :from => source_db, :to => target_db, :rows_per_copy => 100 #Copies 100 rows at a time
+You can also specify the maximum number of open db connections you would like via the <notextile>*max_connections*</notextile> attribute (default is 5).
+
+copy :from => source_db, :to => target_db, :max_connections => 5 #Maintains a maximum of 5 connections
+
+h3. More Granular Control
+
+If you would like to copy only certain tables, you can do that as well using the 'only' and 'except' methods.
+Example: Copy only the users table
+
+  copy :from => source_db, :to => target_db do
+    only => 'users'
+  end
+
+Example: Copy only the users and projects tables
+
+  copy :from => source_db, :to => target_db do
+    only => 'users', 'projects'
+  end
+
+Example: Copy everything but the users and projects tables
+
+  copy :from => source_db, :to => target_db do
+    except => 'users', 'projects'
+  end
+
+You can also choose to copy only over certain columns of a table using the for_table method.
+
+Example: Copy only the name and id fields of the users table
+
+  copy :from => source_db, :to => target_db do
+    for_table 'users', :copy_columns => ['name','id']
+  end
+
+You can mix-and-match to your heart's content.
+
+  copy :from => source_db, :to => target_db, :rows_per_copy => 1000, :max_connections => 10 do
+    except => 'departments', 'projects'
+    for_table 'users', :copy_columns => ['name','id']
+    for_table 'employees', :copy_columns => ['id','age','name']
+  end
+
+Finally, if you have already created the schema of the target_db db-copier only tries to copy columns that are present in the target_db's schema.
+
+h4. LICENSE
+
+(The MIT License)
+Copyright (c) 2009:
+"Santosh Kumar"
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+0:0

data/lib/db-copier.rb

@@ -0,0 +1,9 @@
+libdir = File.dirname(__FILE__)
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+
+require 'rubygems'
+require 'sequel'
+require 'term/ansicolor'
+include Term::ANSIColor
+require 'db-copier/db-copier'
+require 'db-copier/worker'

data/lib/db-copier/db-copier.rb

@@ -0,0 +1,113 @@
+
+
+module DbCopier
+  class Application
+    attr_reader :tables_to_copy
+
+    def initialize &block
+      @tables_to_copy        = []
+      @index_to_copy         = []
+      @only_tables_to_copy   = []
+      @except_tables_to_copy = []
+      instance_eval &block
+    end
+
+    def copy options = {}
+      begin
+        from, to, @rows_per_copy, max_connections = options[:from], options[:to], (options[:rows_per_copy] || 50), (options[:max_connections] || 5)
+        raise ArgumentError unless from && to && from.is_a?(Hash) && to.is_a?(Hash) && from.size > 0 && to.size > 0
+        @source_db, @target_db = Sequel.connect(from.merge(:max_connections => max_connections, :single_threaded => false)),
+                Sequel.connect(to.merge(:max_connections => max_connections, :single_threaded => false))
+        @source_db.test_connection && @target_db.test_connection #test connections
+        @tables_to_copy = @source_db.tables
+        @target_db.tables
+        instance_eval { yield } if block_given?
+        copy_tables
+      ensure
+        self.close_connections
+      end
+    end
+
+    def copy_columns
+      #Hash of the form {:table => [:col1, :col2]} -- maintaing record of columns to keep for a table
+      @copy_columns ||= {}
+    end
+
+    protected :copy_columns
+
+    def for_table(table, options = {})
+      raise ArgumentError, "missing required copy_cols attribute" unless (copy_columns = options[:copy_columns])
+      table, copy_columns = table.to_sym, copy_columns.map {|col| col.to_sym}
+      raise ArgumentError, "columns do not exist" unless (@source_db.schema(table).map {|cols| cols.first} & copy_columns) == copy_columns
+      self.copy_columns[table] = copy_columns
+    end
+
+    def index(ind)
+      @index_to_copy << ind
+    end
+
+    def only(*tabs)
+      @tables_to_copy = Array(tabs)
+    end
+
+    def except(*tabs)
+      @tables_to_copy -= Array(tabs).map { |tb| tb.to_sym }
+    end
+
+    def copy_tables
+      threads = []
+      multi_threaded = !(@source_db.single_threaded? || @target_db.single_threaded?)
+      @tables_to_copy.each do |tab|
+        db_src_conn, db_target_con, table_to_copy, copy_columns_for_table =
+                @source_db, @target_db, tab.to_sym, self.copy_columns[tab.to_sym]
+        if multi_threaded
+          t = Thread.new(table_to_copy, db_src_conn, db_target_con, copy_columns_for_table) do
+            $stdout.print green, bold, "starting Thread: #{Thread.current.object_id}", reset, "\n"
+            w = Worker.new :src_db_conn => db_src_conn, :target_db_conn => db_target_con,
+                           :table_name => table_to_copy, :copy_columns => copy_columns_for_table
+            w.copy_table
+          end
+          threads << t
+        else
+          $stdout.print green, bold, "running in single threaded mode", reset, "\n"
+          w = Worker.new :src_db_conn => db_src_conn, :target_db_conn => db_target_con,
+                         :table_name => table_to_copy, :copy_columns => copy_columns_for_table
+          w.copy_table
+        end
+      end
+      threads.each {|t| t.join} if multi_threaded
+    end
+
+    protected :copy_tables
+
+    def close_connections
+      @source_db.disconnect if defined?(@source_db) && @source_db
+      @target_db.disconnect if defined?(@target_db) && @target_db
+    end
+
+    protected :close_connections
+
+  end
+
+  def self.app &block
+    Application.new &block
+  end
+
+  def self.generate_create_table_ddl(db_conn, table_to_create_schema_from, new_table_name, only_copy_columns = [])
+    ret = String.new
+    db_conn.schema(table_to_create_schema_from.to_sym).each do |col|
+      col_name, col_type = col[0].to_sym, col[1][:type]
+      #Skip creating this column if it was not specified in the columns to copy for this table
+      next if only_copy_columns.count > 0 && !only_copy_columns.include?(col_name)
+      if col[1][:primary_key]
+        ret << "primary_key #{col_name.inspect}, #{col_type.to_sym.inspect}, :default => #{col[1][:default].inspect}, :null => #{col[1][:allow_null].inspect};\n"
+      else
+        ret << "#{col_type.to_s.capitalize} #{col_name.inspect}, :default => #{col[1][:default].inspect}, :null => #{col[1][:allow_null].inspect};\n"
+      end
+    end
+    ret = (".create_table #{new_table_name.to_sym.inspect} do \n" + ret)
+    ret << "end\n"
+  end
+
+end
+

data/lib/db-copier/worker.rb

@@ -0,0 +1,51 @@
+module DbCopier
+  class Worker
+    def initialize(options = {})
+      @src_db_conn, @target_db_conn, @table_name, @rows_per_copy, @copy_columns = options[:src_db_conn], options[:target_db_conn],
+              options[:table_name], (options[:rows_per_copy] || 1000), (options[:copy_columns] || [])
+      raise ArgumentError unless @src_db_conn && @target_db_conn && @table_name
+      @copy_columns ||= []
+    end
+
+    def copy_table
+      start = Time.now
+      $stdout.print green, bold, "Thread: #{Thread.current.object_id} is copying table: #{@table_name}", reset, "\n"
+      tab_to_copy = @table_name.to_sym
+      num_rows = @src_db_conn[tab_to_copy].count
+      i = 0
+      #Create the table if it does not already exist
+      unless @target_db_conn.table_exists?(tab_to_copy)
+        table_creation_ddl = DbCopier.generate_create_table_ddl(@src_db_conn, tab_to_copy, tab_to_copy, @copy_columns)
+        table_creation_ddl = ("@target_db_conn" + table_creation_ddl)
+        eval table_creation_ddl
+      end
+
+      #This is the intersection of columns specified via the +copy_columns+ argumnent in the +for_table+ method
+      #and those that actually exist in the target table.
+      columns_in_target_db = @target_db_conn.schema(tab_to_copy).map {|cols| cols.first}
+      columns_to_copy =
+              if @copy_columns.count > 0
+                @copy_columns & columns_in_target_db
+              else
+                columns_in_target_db
+              end
+
+      while i < num_rows
+        rows_to_copy = @src_db_conn[tab_to_copy].select(*columns_to_copy).limit(@rows_per_copy, i).all
+        #Special handling of datetime columns
+        rows_to_copy.each { |col_name, col_val| rows_to_copy[col_name] = DateTime.parse(col_val) if col_val.class == Time }
+        i += rows_to_copy.count
+        @target_db_conn[tab_to_copy].multi_insert(rows_to_copy)
+      end
+
+      #copy indexes now
+      @src_db_conn.indexes(tab_to_copy).each do |index_name, index_info|
+        #Make sure we are adding an index to a column that is going to be there
+        next unless (columns_to_copy & index_info[:columns] == index_info[:columns])
+        @target_db_conn.add_index(tab_to_copy, index_info[:columns])
+      end
+      $stdout.print green,bold, "Thread: #{Thread.current.object_id} has COMPLETED copying table: #{@table_name} in #{Time.now - start} seconds", reset, "\n"
+    end
+
+  end
+end

data/spec/basic_spec.rb

@@ -0,0 +1,254 @@
+require File.dirname(__FILE__) + "/spec_helper.rb"
+
+describe DbCopier do
+  source_db = {:adapter => 'mysql', :host => 'localhost', :user => 'root', :password => '', :database => 'db_copier_test_src'}
+  target_db = source_db.merge(:database => 'db_copier_test_target')
+  fake_cred_db = target_db.merge(:user => 'rootsss')
+
+  def create_and_populate_table(conn, tbl_name, num_rows = 1000)
+    conn.create_table tbl_name do
+      primary_key :id, :integer, :null => false
+      String :nombre, :null => true
+      DateTime :created_at, :null => false, :default => DateTime.now
+    end
+    rows = []
+    num_rows.times {|i| rows << {:id => (i+1), :nombre => Faker::Name.name, :created_at => DateTime.now}}
+    conn[tbl_name].multi_insert rows
+  end
+
+  before(:all) do
+    #create some connections
+    @source_db_conn, @fake_cred_db_conn, @target_db_conn = Sequel.connect(source_db), Sequel.connect(fake_cred_db),
+            Sequel.connect(target_db)
+
+    create_and_populate_table @source_db_conn, :uno
+    create_and_populate_table @source_db_conn, :dos
+    create_and_populate_table @source_db_conn, :tres
+    create_and_populate_table @source_db_conn, :quatro
+    create_and_populate_table @source_db_conn, :quinto
+  end
+
+  after(:each) do
+    @target_db_conn.tables.each { |tbl| @target_db_conn.drop_table(tbl) } #clear the target tables
+  end
+
+  after(:all) do
+    @source_db_conn.tables.each { |tab| @source_db_conn.drop_table tab } #clear the source tables
+  end
+
+  def create_target_tables
+    @target_db_conn.create_table :uno do
+      primary_key :id, :integer, :null => false
+      String :nombre, :null => true
+      DateTime :created_at, :null => false
+    end
+    @target_db_conn.create_table :dos do
+      primary_key :id, :integer, :null => false
+      String :nombre, :null => true
+      DateTime :created_at, :null => false
+    end
+    @target_db_conn.create_table :tres do
+      primary_key :id, :integer, :null => false
+      String :nombre, :null => true
+      DateTime :created_at, :null => false
+    end
+  end
+
+  def create_target_with_one_less_column
+    @target_db_conn.create_table :uno do
+      primary_key :id, :integer, :null => false
+      String :nombre, :null => true
+    end
+  end
+
+  it "should throw an argument error if the +:from+ and +:to+ arguments are not provided for the +copy+ method" do
+    create_target_tables
+    begin
+      DbCopier.app do
+        copy
+      end
+      raise RuntimeError, "shouldn't be here"
+    rescue ArgumentError
+    end
+
+    begin
+      DbCopier.app do
+        copy :from => {:adapter => 'mysql'}
+      end
+      raise RuntimeError, "shouldn't be here"
+    rescue ArgumentError
+    end
+
+    begin
+      DbCopier.app do
+        copy :to => {:adapter => 'mysql'}
+      end
+      raise RuntimeError, "shouldn't be here"
+    rescue ArgumentError
+    end
+  end
+
+  it "should throw an argument error if the +:from+ and +:to+ arguments are NOT hashes" do
+    create_target_tables
+    begin
+      DbCopier.app do
+        copy :from => 'foo', :to => 'bar'
+      end
+      raise RuntimeError, "shouldn't be here"
+    rescue ArgumentError
+    end
+  end
+
+  it "should throw an Sequel::DatabaseConnectionError for invalid credentials" do
+    create_target_tables
+    begin
+      DbCopier.app do
+        copy :from => source_db, :to => fake_cred_db
+      end
+      raise RuntimeError, "shouldn't be here"
+    rescue Sequel::DatabaseConnectionError
+    end
+  end
+
+  it "should copy all tables if no #table methods are called in the dsl" do
+    create_target_tables
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db
+    end
+    @source_db_conn.tables.map{|tbl| tbl.inspect}.sort.should == @target_db_conn.tables.map{|tbl| tbl.inspect}.sort
+  end
+
+  it "should copy only select tables if they are provided as an #only method in the dsl" do
+    create_target_tables
+    only_tables_to_copy = ['uno', 'dos'].sort
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        only *only_tables_to_copy
+      end
+    end
+    only_tables_to_copy.each { |tbl| @source_db_conn[tbl.to_sym].count.should == @target_db_conn[tbl.to_sym].count }
+  end
+
+  it "should not copy tables specified in the #except dsl method" do
+    create_target_tables
+    original_tres_count = @target_db_conn[:tres].count
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        except 'tres'
+      end
+    end
+    @target_db_conn[:tres].count.should == original_tres_count
+  end
+
+  it "should throw an ArgumentError if the for_table method is called without appropriate args" do
+    create_target_tables
+    begin
+      app = DbCopier.app do
+        copy :from => source_db, :to => target_db do
+          except 'uno', 'dos'
+          for_table :custom_src_query_ratings #Missing :copy_columns argument
+        end
+      end
+      raise RuntimeError, "Shouldn't be here"
+    rescue ArgumentError
+    end
+  end
+
+  it "should only copy the columns specified in the +copy_columns+ argument in the for_table method" do
+    create_target_tables
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        except 'tres'
+        for_table :uno, :copy_columns => ['id', 'created_at']
+      end
+    end
+    @target_db_conn[:uno].all.each do |row|
+      row[:nombre].should == nil
+    end
+  end
+
+  it "should throw an ArgumentError if the columns specified in the for_table method does not exist do not exist in the source table" do
+    create_target_tables
+    begin
+      app = DbCopier.app do
+        copy :from => source_db, :to => target_db do
+          except 'tres'
+          for_table :uno, :copy_columns => ['id', 'foo']
+        end
+      end
+      raise RuntimeError, "Shouldn't be here"
+    rescue ArgumentError
+    end
+  end
+
+  it "should copy only columns that are there in the target db" do
+    create_target_with_one_less_column
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        only 'uno'
+      end
+    end
+    @source_db_conn[:uno].count.should == @target_db_conn[:uno].count
+  end
+
+  it "should create tables in the target db if they do not exist" do
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+      end
+    end
+    @target_db_conn.tables.map{|tbl| tbl.to_s}.sort.should == @source_db_conn.tables.map{|tbl| tbl.to_s}.sort
+    @source_db_conn.tables.each do |tbl|
+      @source_db_conn[tbl].count.should == @target_db_conn[tbl].count
+    end
+  end
+
+  it "should create tables in the target db if they do not exist and it should support a +for_table+ argument" do
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        for_table :uno, :copy_columns => ['id', 'created_at']
+      end
+    end
+    @target_db_conn.tables.map{|tbl| tbl.to_s}.sort.should == @source_db_conn.tables.map{|tbl| tbl.to_s}.sort
+    @source_db_conn.tables.each do |tbl|
+      @source_db_conn[tbl].count.should == @target_db_conn[tbl].count
+    end
+  end
+
+  it "should copy indexes" do
+    #Create an index
+    @source_db_conn.add_index :uno, :nombre
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        only 'uno'
+      end
+    end
+
+    @target_db_conn.indexes(:uno).count.should > 0
+    @target_db_conn.indexes(:uno).each { |key, val| val[:columns].should include(:nombre) }
+    @source_db_conn.drop_index :uno, :nombre #clean-up
+
+    @source_db_conn.add_index :dos, [:nombre, :created_at]
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        only 'dos'
+      end
+    end
+    @target_db_conn.indexes(:dos).count.should > 0
+    @target_db_conn.indexes(:dos).each { |key, val| val[:columns].should == [:nombre,:created_at] }
+  end
+
+  it "should not copy indexes of columns that are not to be copied" do
+    #Create an index
+    @source_db_conn.add_index :uno, :nombre
+    app = DbCopier.app do
+      copy :from => source_db, :to => target_db do
+        only 'uno'
+        for_table :uno, :copy_columns => [:id,:created_at]
+      end
+    end
+    @target_db_conn.indexes(:uno).count.should == 0
+  end
+
+  it "should contain useful log msgs"
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require "faker"
+require File.dirname(__FILE__) + "/../lib/db-copier"
+

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification 
+name: db-copier
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Santosh Kumar
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-18 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 3.8.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: term-ansicolor
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 1.0.4
+    version: 
+description: 
+email: santosh79@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/db-copier.rb
+- lib/db-copier/db-copier.rb
+- lib/db-copier/worker.rb
+- README.textile
+- spec/spec_helper.rb
+- spec/basic_spec.rb
+has_rdoc: true
+homepage: http://github.com/santosh79/db-copier
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A DSL around Sequel to aid in copying or replicating databases.
+test_files: []
+