merge_db 0.0.2

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+tmp/
+rerun.txt
+log/*

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.2-p290@merge-db

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in merge_db.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard 'cucumber', :cli => '-c --drb --no-profile --format pretty' do
+  watch(%r{^features/.+\.feature$})
+  watch(%r{^features/support/.+$})          { 'features' }
+  watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
+end

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/bin/merge_db

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.dirname(__FILE__) + '/../lib') unless $:.include?(File.dirname(__FILE__) + '/../lib')
+
+require "rubygems"
+require "thor"
+require 'merge_db'
+
+class Db < Thor
+  desc "merge", "merge two databases"
+  method_option :source, :aliases => "-s", :required => true, :desc => "Records will be copied from this database"
+  method_option :target, :aliases => "-t", :required => true, :desc => "Records will be inserted into this database"
+
+  def merge
+    source = options[:source]
+    target = options[:target]
+
+    MergeDb::Merger.new(:source => source, :target => target).merge
+  end
+
+  desc "restore", "restore associations"
+  method_option :target, :aliases => "-t", :required => true, :desc => "Database for restoring"
+  def restore
+    target = options[:target]
+
+    MergeDb::Merger.new(:target => target).restore_associations
+  end
+
+  desc "prepare", "creates columns to store ids from source database"
+  method_option :target, :aliases => "-t", :required => true, :desc => "database into which data will be copied"
+  def prepare
+    target = options[:target]
+
+    MergeDb::Merger.new(:target => target).prepare
+  end
+end
+
+Db.start

data/config/database.yml

@@ -0,0 +1,29 @@
+defaults: &defaults
+  adapter: mysql2
+  encoding: utf8
+  host: localhost
+  reconnect: false
+
+db_first:
+  database: db_first
+  username: root
+  password: ''
+  <<: *defaults
+
+db_second:
+  database: db_second
+  username: root
+  password: ''
+  <<: *defaults
+
+db_third:
+  database: db_third
+  username: root
+  password: ''
+  <<: *defaults
+
+db_target:
+  database: db_target
+  username: root
+  password: ''
+  <<: *defaults

data/config/merge.rb

@@ -0,0 +1,3 @@
+MergeDb::Configuration.scope_name = 'studio'
+
+MergeDb::Configuration.scoped_tables = %w(users groups)

data/db.thor

@@ -0,0 +1,24 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/lib')
+
+require 'merge_db'
+
+class Db < Thor
+  desc "merge", "merge two databases"
+  method_option :source, :aliases => "-s", :required => true, :desc => "Records will be copied from this database"
+  method_option :target, :aliases => "-t", :required => true, :desc => "Records will be inserted into this database"
+
+  def merge
+    source = options[:source]
+    target = options[:target]
+
+    MergeDb::Merger.new(:source => source, :target => target).merge
+  end
+
+  desc "prepare", "creates columns to store ids from source database"
+  method_option :target, :aliases => "-t", :required => true, :desc => "database into which data will be copied"
+  def prepare
+    target = options[:target]
+
+    MergeDb::Merger.new(:target => target).prepare
+  end
+end

data/features/merge.feature

@@ -0,0 +1,94 @@
+Feature: Background
+  In order to migrate from multiple databases to single database in my application
+  As a developer
+  I want to merge all databases
+
+  Scenario:
+    Given a schema "old.rb":
+      """
+      Schema.define do
+        create_table "users" do |t|
+          t.integer  "id"
+          t.string   "name"
+          t.integer  "group_id"
+        end
+
+        create_table "groups", :force => true do |t|
+          t.integer  "id"
+          t.string   "name"
+        end
+      end
+      """
+    Given a schema "new.rb":
+      """
+      Schema.define do
+        create_table "users" do |t|
+          t.integer  "id"
+          t.string   "name"
+          t.integer  "group_id"
+        end
+
+        create_table "groups", :force => true do |t|
+          t.integer  "id"
+          t.string   "name"
+        end
+
+        create_table "studios", :force => true do |t|
+          t.integer  "id"
+          t.string   "db_name"
+        end
+      end
+      """
+    Given a database named "db_first" with schema "old.rb"
+    And a database named "db_second" with schema "old.rb"
+    And a database named "db_third" with schema "old.rb"
+    And a database named "db_target" with schema "new.rb"
+    And a table "users" in "db_first" with:
+      | id | name  | group_id |
+      | 1  | John  | 1        |
+      | 2  | Marry | 2        |
+      | 3  | Jane  | 2        |
+    And a table "groups" in "db_first" with:
+      | id | name   |
+      | 1  | First  |
+      | 2  | Second |
+    And a table "users" in "db_second" with:
+      | id | name  | group_id |
+      | 1  | Jeff  | 1        |
+      | 2  | Frank | 1        |
+      | 3  | Adam  | 2        |
+    And a table "groups" in "db_second" with:
+      | id | name     |
+      | 1  | First 2  |
+      | 2  | Second 2 |
+    And a table "users" in "db_third" with:
+      | id | name  | group_id |
+      | 1  | Piter | 1        |
+      | 2  | James | 2        |
+      | 3  | Bill  | 3        |
+    And a table "groups" in "db_third" with:
+      | id | name  |
+      | 1  | One   |
+      | 2  | Two   |
+      | 3  | Three |
+    When I prepare "db_target"
+    When I merge "db_first" into "db_target"
+    And I merge "db_second" into "db_target"
+    And I merge "db_third" into "db_target"
+    # When I run in shell "thor db:merge -s db_source -t db_target"
+    # Then I should see "Databases were merged."
+    When I connected to "db_target"
+    Then 9 users should exist
+    And  7 groups should exist
+    Then the following users should exist:
+      | name  | groups.name | studios.db_name |
+      | Piter | One         | db_third        |
+      | James | Two         | db_third        |
+      | Bill  | Three       | db_third        |
+      | John  | First       | db_first        |
+      | Marry | Second      | db_first        |
+      | Jane  | Second      | db_first        |
+      | Jeff  | First 2     | db_second       |
+      | Frank | First 2     | db_second       |
+      | Adam  | Second 2    | db_second       |
+

data/features/step_definitions/database_steps.rb

@@ -0,0 +1,77 @@
+Given /^a schema "([^"]*)":$/ do |schema_name, string|
+  Given %|a file named "temp/#{schema_name}" with:|, string
+end
+
+Given /^a database named "([^"]*)" with schema "([^"]*)"$/ do |db_name, schema_name|
+  connection.drop_database db_name
+  connection.create_database db_name
+
+  created_databases << db_name
+
+  connection.execute("use #{db_name}")
+
+  schema = File.read("./tmp/aruba/temp/#{schema_name}")
+
+  instance_eval schema
+end
+
+When /^I run in shell "([^"]*)"$/ do |command|
+  @output = `#{command}`
+end
+
+When /^I prepare "([^"]*)"$/ do |target|
+  MergeDb::Merger.new(:target => target).prepare
+end
+
+When /^I merge "([^"]*)" into "([^"]*)"$/ do |source, target|
+  MergeDb::Merger.new(:source => source, :target => target).merge
+end
+
+Then /^I should see "([^"]*)"$/ do |str|
+  @output.should include(str)
+end
+
+Then /^I should see the output$/ do
+  puts @output
+end
+
+Given /^a table "([^"]*)" in "([^"]*)" with:$/ do |table_name, db_name, data|
+  connection.execute("use #{db_name}")
+
+  data.hashes.each do |fixture|
+    connection.insert_fixture(fixture, table_name)
+  end
+end
+
+When /^I use "([^"]*)"$/ do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+When /^I connected to "([^"]*)"$/ do |db_name|
+  connection.execute("use #{db_name}")
+end
+
+Then /^(\d+) (\w+)?s should exist$/ do |count, model_name|
+  model = model_name.camelize.constantize
+  model.count.should == count.to_i
+end
+
+Then /^the following (\w+)?s should exist:$/ do |model_name, table|
+  model = model_name.camelize.constantize
+
+  join_tables = table.column_names.collect { |name| name.split(".").first.singularize.to_sym if name =~ /\w+\.\w+/ }.delete_if(&:nil?)
+
+  table.hashes.each do |attributes|
+    model.joins(join_tables).where(attributes).should_not be_empty, "expected to find a record with attributes: #{attributes}"
+  end
+end
+
+Then /^the following groups should exist:$/ do |table|
+  # table is a Cucumber::Ast::Table
+  pending # express the regexp above with the code you wish you had
+end
+
+Then /debug/ do
+  debugger
+  a = 1
+end

data/features/support/env.rb

@@ -0,0 +1,27 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'aruba/cucumber'
+require 'active_record'
+require 'ruby-debug'
+require 'merge_db'
+
+module CustomData
+  def connection
+    ActiveRecord::Base.establish_connection(connection_params) unless ActiveRecord::Base.connected?
+    ActiveRecord::Base.connection
+  end
+
+  def connection_params
+    {
+      :adapter => 'mysql2',
+      :host => 'localhost',
+      :username => 'root',
+      :password => '',
+    }
+  end
+
+  def created_databases
+    @databases ||= []
+  end
+end
+
+World(CustomData)

data/features/support/hooks.rb

@@ -0,0 +1,5 @@
+After do
+  created_databases.each do |db_name|
+    connection.drop_database db_name
+  end
+end

data/features/support/models.rb

@@ -0,0 +1,11 @@
+class User < ActiveRecord::Base
+  belongs_to :group
+  belongs_to :studio
+end
+
+class Studio < ActiveRecord::Base
+end
+
+class Group < ActiveRecord::Base
+  has_many :users
+end

data/features/support/schema.rb

@@ -0,0 +1,8 @@
+class Schema < ActiveRecord::Migration
+  def self.define(&block)
+    schema = new
+    schema.verbose = false
+    schema.instance_eval(&block)
+  end
+end
+

data/lib/merge_db/configuration.rb

@@ -0,0 +1,16 @@
+module MergeDb
+  class Configuration
+    class << self
+      attr_accessor :scope_name, :scoped_tables
+
+      def scoped_tables
+        @scoped_tables || []
+      end
+
+      def database
+        require 'erb'
+        YAML::load(ERB.new(IO.read('./config/database.yml')).result)
+      end
+    end
+  end
+end

data/lib/merge_db/merger.rb

@@ -0,0 +1,190 @@
+module MergeDb
+  class Source < ActiveRecord::Base
+  end
+
+  class Target < ActiveRecord::Base
+  end
+
+  class Merger
+
+    def initialize(params)
+      @source_name = params[:source]
+      @target_name = params[:target]
+    end
+
+    def prepare
+      prepare_tables_in_target
+    end
+
+    def merge
+      add_db_to_scope
+      copy_data_from_source_to_target
+      restore_association_references
+      clean_backedup_primary_keys
+    end
+
+    def restore_associations
+      restore_association_references
+      clean_backedup_primary_keys
+    end
+
+    private
+
+    def prepare_tables_in_target
+      puts "Prepare target"
+
+      pbar = ProgressBar.new("tables", targte.tables.size)
+
+      target.tables.each do |table|
+
+        pbar.inc
+
+        add_scope_id_column(table) if Configuration.scoped_tables.include? table
+
+        target.columns(table)
+        .find_all {|column| column.name =~ /id$/}
+        .each do |column|
+          add_backup_id_column(table, column)
+        end
+      end
+
+      pbar.finish
+    end
+
+    def add_db_to_scope
+      @scope_id = target.insert("insert into #{Configuration.scope_name.pluralize} (db_name) values ('#{@source_name}')")
+    end
+
+    def copy_data_from_source_to_target
+      puts "Copy data into target"
+
+      source.tables.each do |table|
+        puts "\nCopy #{table} records "
+        query = "select * from #{table}"
+
+        records = source.select_all(query)
+
+        pbar = ProgressBar.new("#{table}", records.size)
+
+        records.each do |fixture|
+
+          pbar.inc
+
+          fixture_with_saved_ids = prepare_for_merge(fixture)
+
+          if Configuration.scoped_tables.include? table
+            fixture_with_saved_ids[scope_id_column] = @scope_id
+          end
+
+          target.insert_fixture(fixture_with_saved_ids, table)
+        end
+
+        pbar.finish
+      end
+    end
+
+    def restore_association_references
+      puts "Restore association references"
+
+      target.tables.each do |table|
+        query = "select * from #{table}"
+
+        restored_columns = Hash.new {|hash, key| hash[key] = [] }
+
+        records = target.select_all(query)
+
+        pbar = ProgressBar.new("#{table}", records.size)
+
+        records.each do |record|
+          pbar.inc
+          record.each do |column, old_id|
+            if column =~ /__id$/ && !old_id.nil? && !restored_columns[column].include?(old_id)
+
+              restored_columns[column] << old_id
+
+              if association = find_association_by_old_id(column, old_id)
+
+                new_id = association["id"]
+
+                update_query = "update #{table} set #{normalize_column_name(column)} = #{new_id} where #{column} = #{old_id}"
+                target.update(update_query)
+              else
+                ActiveRecord::Base.logger.error "Can't find #{column} with old id: #{old_id}"
+              end
+            end
+          end
+        end
+
+        pbar.finish
+
+        unless restored_columns.empty?
+          columns_with_null = restored_columns.keys.collect {|column| "#{column} = NULL"}.join(", ")
+          query = "update #{table} set #{columns_with_null}"
+          target.execute(query)
+        end
+      end
+    end
+
+    def clean_backedup_primary_keys
+      source.tables.each do |table|
+        query = "update #{table} set _id = NULL"
+
+        target.execute(query)
+      end
+    end
+
+    def add_backup_id_column(table, column)
+      target.add_column(table, backup_column_name(column.name), column.type)  
+    end
+
+    def add_scope_id_column(table)
+      target.add_column(table, scope_id_column, :integer)  
+    end
+
+    def scope_id_column
+      Configuration.scope_name + "_id"
+    end
+
+    def find_association_by_old_id(column, value)
+      association_table = column.split("__").first.pluralize
+      query = "select * from #{association_table} where _id = #{value}"
+      association = target.select_one(query)
+    end
+
+    def prepare_for_merge(fixture)
+      {}.tap do |new_fixture|
+        fixture.each do |key, value|
+          new_fixture[backup_column_name(key)] = value
+        end
+      end
+    end
+
+    def backup_column_name(name)
+      if name =~ /id$/
+        name = name.gsub(/id$/, "_id")
+      else
+        name
+      end
+    end
+
+    def normalize_column_name(name)
+      name.gsub(/__id$/, '_id')
+    end
+
+    def target
+      @target_connection ||= connect(Target, @target_name)
+    end
+
+    def source
+      @source_connection ||= connect(Source, @source_name)
+    end
+
+    def connect(constant, db_name)
+      if db_name
+        constant.establish_connection(Configuration.database[db_name])
+        constant.connection
+      end
+    end
+  end
+end
+

data/lib/merge_db/version.rb

@@ -0,0 +1,3 @@
+module MergeDb
+  VERSION = "0.0.2"
+end

data/lib/merge_db.rb

@@ -0,0 +1,19 @@
+require "rubygems"
+require "bundler/setup"
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'active_record'
+# require 'ruby-debug'
+require 'logger'
+require 'progressbar'
+
+require 'merge_db/configuration'
+require 'merge_db/merger'
+
+ActiveRecord::Base.logger = Logger.new('log.txt')
+
+module MergeDb
+end
+
+require "./config/merge.rb"

data/merge_db.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "merge_db/version"
+
+Gem::Specification.new do |s|
+  s.name        = "merge_db"
+  s.version     = MergeDb::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Pavel Gabriel"]
+  s.email       = ["alovak@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Merge your Rails databases}
+  s.description = %q{With merge db you can easily merge your databases and save all associations}
+
+  s.rubyforge_project = "merge_db"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "cucumber"
+  s.add_development_dependency "aruba"
+  s.add_development_dependency "guard"
+  s.add_development_dependency "guard-cucumber"
+  s.add_development_dependency "ruby-debug19"
+
+  s.add_runtime_dependency "thor"
+  s.add_runtime_dependency "activerecord"
+  s.add_runtime_dependency "mysql2"
+  s.add_runtime_dependency "ruby-progressbar"
+end

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: merge_db
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Pavel Gabriel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-11-04 00:00:00.000000000 +02:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: &2157140920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157140920
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: &2157140480 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157140480
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: &2157140060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157140060
+- !ruby/object:Gem::Dependency
+  name: guard-cucumber
+  requirement: &2157139640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157139640
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &2157139180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2157139180
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &2157138760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157138760
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &2157138280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157138280
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: &2157137760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157137760
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: &2157137300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2157137300
+description: With merge db you can easily merge your databases and save all associations
+email:
+- alovak@gmail.com
+executables:
+- merge_db
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Guardfile
+- README
+- Rakefile
+- bin/merge_db
+- config/database.yml
+- config/merge.rb
+- db.thor
+- features/merge.feature
+- features/step_definitions/database_steps.rb
+- features/support/env.rb
+- features/support/hooks.rb
+- features/support/models.rb
+- features/support/schema.rb
+- lib/merge_db.rb
+- lib/merge_db/configuration.rb
+- lib/merge_db/merger.rb
+- lib/merge_db/version.rb
+- merge_db.gemspec
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: merge_db
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Merge your Rails databases
+test_files:
+- features/merge.feature
+- features/step_definitions/database_steps.rb
+- features/support/env.rb
+- features/support/hooks.rb
+- features/support/models.rb
+- features/support/schema.rb