xmysql2psql 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 68e6a0393134773b9525654b5b9f943634a3616c
+  data.tar.gz: ac5b3c5ec00fbc003a276d0bb41c5e407d67a57e
+SHA512:
+  metadata.gz: cffc0de72ddc657ae62f74200a3bd6a96037fec2bcee6d5606dadacdd937c20beaa26987709405f65d90e8c25a0120f5b6f1e870a282a66e6759f49802da2a8a
+  data.tar.gz: f224a0393a3da4496902176d2e8bfff9b84ac9c3bb54f610549c99f9e1a2af447a77b474c6312a877c279232eb08ab8c404f47d6db81be5d00fbc56d511ec0bd

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009-2010 name <email> 
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,123 @@
+= XMYSQL2PSQL - MySQL to PostgreSQL Migration Tool
+
+Tool for converting mysql database to postgresql.
+It can create postgresql dump from mysql database or directly load data from mysql to
+postgresql (at about 100 000 records per minute).
+
+It can translate now most data types and indexes, but if you experience some problems, feel free
+to contact me, I'll help you.
+
+== Installation
+Xmysql2psql is packaged as a gem. Install as usual (use sudo if required).
+
+  $ gem install mysql2psql
+  NB: the gem hasn't been formally released on http://rubygems.org/ yet. For now you need to clone the git repo at http://github.com/tardate/mysql2postgres and 'rake install' 
+
+After installation, the "mysql2psql" command will be available. 
+When run, it will generate a default configuration file in the current directory if a config
+file is not already found. The default configuration filename is mysql2psql.yml.
+
+  $ xmysql2psql
+
+You can use an alternate config file by passing the filename/path as a parameter:
+
+  $ xmysql2psql ../another/world.yml
+
+== Running Migrations
+See the configuration file for documentation about settings. Key choices include:
+* whether to dump to a file, or migrate direct to PostgreSQL
+* migrate only the schema definition, only the data, or both schema and data
+
+
+After editing the configuration file and launching tool, you will see something like..
+
+  Creating table friendships...
+  Created table friendships
+  Loading friendships...
+  620000 of 638779 rows loaded. [ETA: 2010/01/21 21:32 (00h:00m:01s)]
+  638779 rows loaded in 1min 3s
+  Indexing table friendships...
+  Indexed table friendships
+  Table creation 0 min, loading 1 min, indexing 0 min, total 1 min
+
+
+== Database driver dependencies
+Xmysql2psql uses the 'mysql2' and 'pg' gems for database connectivity.
+Xmysql2psql gem installation should automatically install these too.
+
+If you encounter any issues with db connectivity, verify that the 'mysql' and 'pg' gems are installed and working correctly first.
+The 'mysql' gem in particular may need a hint on where to find the mysql headers and libraries:
+
+  $ [sudo] gem install mysql -- --with-mysql-dir=/usr/local/mysql
+
+== Getting the source
+Xmysql2psql's source is located at http://github.com/kmizu/xmysql2psql
+
+== Running tests
+If you fork/clone the project, you will want to run the test suite (and add to it if you are developing new features or fixing bugs).
+ 
+A suite of tests are provided and are run using rake (rake -T for more info)
+  rake test:units
+  rake test:integration
+  rake test 
+
+Rake without parameters (or "rake test") will run both the unit and integration tests.
+
+Unit tests are small standalone tests of the xmysql2psql codebase that have no external dependencies (like a database)
+
+Integration tests require suitable mysql and postgres databases to be setup in advance. 
+Running the integration tests *will* rewrite data and schema for the "xmysql2psql_test" databases in mysql and postgres. Don't use this database for anything else!
+
+mysql on localhost:3306
+- database created called "xmysql2psql_test"
+- user setup for "xmysql2psql" with no password
+- e.g. 
+  mysqladmin -uroot -p create xmysql2psql_test
+  mysql -uroot -p -e "grant all on xmysql2psql_test.* to 'mysql2psql'@'localhost';"
+  # verify connecction:
+  mysql -uxmysql2psql -e "select database(),'yes' as connected" mysql2psql_test
+  
+postgres on localhost:5432
+- database created called "xmysql2psql_test" 
+- role (user) access setup for "xmysql2psql" with no password
+- e.g.
+  psql postgres -c "create role xmysql2psql with login"
+  psql postgres -c "create database xmysql2psql_test with owner xmysql2psql encoding='UTF8'"
+  # verify connection:
+  psql xmysql2psql_test -U xmysql2psql -c "\c"
+
+== Notes, Limitations, Outstanding Issues..
+
+Todo:
+- more tests
+- release gem
+- a windows cmd shim
+
+=== note from mgkimsal
+I'm still having trouble with bit(1)/boolean fields
+workaround I've found is to put output in file
+then in VIM on file, search/replace the true/false binary fields with t/f
+specifically
+
+(reversed on 3/23 - should be ^A gets f)
+:%s/^@/t/g
+:%s/^A/f/g
+keystrokes are ctrl-v ctrl-shift-@ to get the 'true' binary field
+keystrokes are ctrl-v ctrl-shift-A to get the 'false' binary field
+
+== Contributors
+Project founded by Max Lapshin <max@maxidoors.ru>
+
+Other contributors (in git log order):
+- Anton Ageev <anton@ageev.name>
+- Samuel Tribehou <cracoucax@gmail.com>
+- Marco Nenciarini <marco.nenciarini@devise.it>
+- James Nobis <jnobis@jnobis.controldocs.com>
+- quel <github@quelrod.net>
+- Holger Amann <keeney@fehu.org>
+- Maxim Dobriakov <closer.main@gmail.com>
+- Michael Kimsal <mgkimsal@gmail.com>
+- Jacob Coby <jcoby@portallabs.com>
+- Neszt Tibor <neszt@tvnetwork.hu>
+- Miroslav Kratochvil <exa.exa@gmail.com>
+- Paul Gallagher <gallagher.paul@gmail.com>

data/Rakefile

@@ -0,0 +1,79 @@
+require 'rubygems'
+require 'rake'
+
+$LOAD_PATH.unshift('lib')
+require 'xmysql2psql/version'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "xmysql2psql"
+    gem.version = Xmysql2psql::Version::STRING
+    gem.summary = %Q{Tool for converting mysql database to postgresql}
+    gem.description = %Q{It can create postgresql dump from mysql database or directly load data from mysql to
+    postgresql (at about 100 000 records per minute). Translates most data types and indexes.}
+    gem.email = "mizukota@gmail.com"
+    gem.homepage = "http://github.com/kmizu/xmysql2psql"
+    gem.authors = [
+      "Kota Mizushima <mizukota@gmail.com>"
+    ]
+    gem.add_dependency "mysql", "= 2.8.1"
+    gem.add_dependency "pg", ">= 0.15.0"
+    gem.add_development_dependency "test-unit", ">= 2.1.1"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/tasklib'
+require 'rake/testtask'
+require 'rdoc/task'
+namespace :test do
+  Rake::TestTask.new(:units) do |test|
+    test.libs << 'lib' << 'test/lib'
+    test.pattern = 'test/units/*test.rb'
+    test.verbose = true
+  end
+
+  Rake::TestTask.new(:integration) do |test|
+    test.libs << 'lib' << 'test/lib'
+    test.pattern = 'test/integration/*test.rb'
+    test.verbose = true
+  end
+end
+
+desc "Run all tests"
+task :test do
+  Rake::Task['test:units'].invoke
+  Rake::Task['test:integration'].invoke
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+task :default => :test
+
+require 'rake/tasklib'
+require 'rake/testtask'
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = Xmysql2psql::Version::STRING
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "xmysql2psql #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/bin/xmysql2psql

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'xmysql2psql'
+
+exit Xmysql2psql.new(ARGV).convert

data/lib/xmysql2psql/config.rb

@@ -0,0 +1,100 @@
+require 'xmysql2psql/config_base'
+
+class Xmysql2psql
+  
+  class Config < ConfigBase
+    
+    def initialize(configfilepath, generate_default_if_not_found = true)
+      unless File.exists?(configfilepath)
+        reset_configfile(configfilepath) if generate_default_if_not_found
+        if File.exists?(configfilepath) 
+          raise Xmysql2psql::ConfigurationFileInitialized.new("\n
+No configuration file found.
+A new file has been initialized at: #{configfilepath}
+Please review the configuration and retry..\n\n\n")
+        else
+          raise Xmysql2psql::ConfigurationFileNotFound.new("cannot load config file #{configfilepath}")
+        end
+      end
+      super(configfilepath)
+    end
+
+    def reset_configfile(filepath)
+      file = File.new(filepath,'w')
+      self.class.template.each_line do | line|
+        file.puts line
+      end
+      file.close
+    end
+
+    def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false)
+      configtext = <<EOS
+mysql:
+ hostname: localhost
+ port: 3306
+ socket: /tmp/mysql.sock
+ username: xmysql2psql
+ password: 
+ database: xmysql2psql_test
+
+destination:
+ # if file is given, output goes to file, else postgres
+ file: #{ to_filename ? to_filename : ''}
+ postgres:
+  hostname: localhost
+  port: 5432
+  username: xmysql2psql
+  password: 
+  database: xmysql2psql_test
+
+# if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
+#tables:
+#- table1
+#- table2
+EOS
+      if include_tables.length>0
+        configtext += "\ntables:\n"
+        include_tables.each do |t|
+          configtext += "- #{t}\n"
+        end
+      end
+      configtext += <<EOS
+# if exclude_tables is given, exclude the listed tables from the conversion.
+#exclude_tables:
+#- table3
+#- table4
+
+EOS
+      if exclude_tables.length>0
+        configtext += "\nexclude_tables:\n"
+        exclude_tables.each do |t|
+          configtext += "- #{t}\n"
+        end
+      end
+      if !supress_data.nil?
+        configtext += <<EOS
+
+# if supress_data is true, only the schema definition will be exported/migrated, and not the data
+supress_data: #{supress_data}
+EOS
+      end
+      if !supress_ddl.nil?
+        configtext += <<EOS
+
+# if supress_ddl is true, only the data will be exported/imported, and not the schema
+supress_ddl: #{supress_ddl}
+EOS
+      end
+      if !force_truncate.nil?
+        configtext += <<EOS
+
+# if force_truncate is true, forces a table truncate before table loading
+force_truncate: #{force_truncate}
+EOS
+      end
+      configtext
+    end
+    
+  end
+
+end

data/lib/xmysql2psql/config_base.rb

@@ -0,0 +1,39 @@
+require 'yaml'
+require 'xmysql2psql/errors'
+
+class Xmysql2psql
+  
+  class ConfigBase
+    attr_reader :config, :filepath
+
+    def initialize(configfilepath)
+      @filepath=configfilepath
+      @config = YAML::load(File.read(filepath))
+    end
+    def [](key)
+      self.send( key )
+    end
+    def method_missing(name, *args)
+      token=name.to_s
+      default = args.length>0 ? args[0] : ''
+      must_be_defined = default == :none
+      case token
+      when /mysql/i
+        key=token.sub( /^mysql/, '' )
+        value=config["mysql"][key]
+      when /pg/i
+        key=token.sub( /^pg/, '' )
+        value=config["destination"]["postgres"][key]
+      when /dest/i
+        key=token.sub( /^dest/, '' )
+        value=config["destination"][key]
+      when /only_tables/i
+        value=config["tables"] 
+      else
+        value=config[token]
+      end
+      value.nil? ? ( must_be_defined ? (raise Xmysql2psql::UninitializedValueError.new("no value and no default for #{name}")) : default ) : value
+    end
+  end
+  
+end

data/lib/xmysql2psql/converter.rb

@@ -0,0 +1,55 @@
+class Xmysql2psql
+
+  class Converter
+    attr_reader :reader, :writer, :options
+    attr_reader :exclude_tables, :only_tables, :supress_data, :supress_ddl, :force_truncate
+  
+    def initialize(reader, writer, options)
+      @reader = reader
+      @writer = writer
+      @options = options
+      @exclude_tables = options.exclude_tables([])
+      @only_tables = options.only_tables(nil)
+      @supress_data = options.supress_data(false)
+      @supress_ddl = options.supress_ddl(false)
+      @force_truncate = options.force_truncate(false)
+    end
+  
+    def convert
+      _time1 = Time.now
+
+      tables = reader.tables.
+        reject {|table| @exclude_tables.include?(table.name)}.
+        select {|table| @only_tables ? @only_tables.include?(table.name) : true}
+
+
+      tables.each do |table|
+        writer.write_table(table)
+      end unless @supress_ddl
+ 
+      _time2 = Time.now
+      tables.each do |table|
+        writer.truncate(table) if force_truncate && supress_ddl
+        writer.write_contents(table, reader)
+      end unless @supress_data
+ 
+      _time3 = Time.now
+      tables.each do |table|
+        writer.write_indexes(table)
+      end unless @supress_ddl
+      tables.each do |table|
+        writer.write_constraints(table)
+      end unless @supress_ddl
+ 
+ 
+      writer.close
+      _time4 = Time.now
+      puts "Table creation #{((_time2 - _time1) / 60).round} min, loading #{((_time3 - _time2) / 60).round} min, indexing #{((_time4 - _time3) / 60).round} min, total #{((_time4 - _time1) / 60).round} min"
+      return 0
+    rescue => e
+      $stderr.puts "Xmysql2psql: conversion failed: #{e.to_s}"
+      return -1
+    end
+  end
+
+end

data/lib/xmysql2psql/errors.rb

@@ -0,0 +1,16 @@
+
+class Xmysql2psql
+  
+  class GeneralError < StandardError
+	end
+
+  class ConfigurationError < StandardError
+	end
+  class UninitializedValueError < ConfigurationError
+	end
+  class ConfigurationFileNotFound < ConfigurationError
+	end
+  class ConfigurationFileInitialized < ConfigurationError
+	end	
+	
+end

data/lib/xmysql2psql/mysql_reader.rb

@@ -0,0 +1,190 @@
+require 'mysql'
+
+class Xmysql2psql
+
+  class MysqlReader
+    class Field
+    end
+  
+    class Table
+      attr_reader :name
+    
+      def initialize(reader, name)
+        @reader = reader
+        @name = name
+      end
+    
+      @@types = %w(tiny enum decimal short long float double null timestamp longlong int24 date time datetime year set blob string var_string char).inject({}) do |list, type|
+        list[eval("::Mysql::Field::TYPE_#{type.upcase}")] = type
+        list
+      end
+    
+      @@types[246] = "decimal"
+    
+      def columns
+        @columns ||= load_columns
+      end
+    
+      def convert_type(type)
+        case type
+        when /int.* unsigned/
+          "bigint"
+        when /bigint/
+          "bigint"
+        when "bit(1)"
+          "boolean"
+        when "tinyint(1)"
+          "boolean"
+        when /tinyint/
+          "tinyint"
+        when /int/
+          "integer"
+        when /varchar/
+          "varchar"
+        when /char/
+          "char"
+        when /(float|decimal)/
+          "decimal"
+        when /double/
+           "double precision"
+        else
+          type
+        end 
+      end
+    
+      def load_columns
+        @reader.reconnect
+        result = @reader.mysql.list_fields(name)
+        mysql_flags = ::Mysql::Field.constants.select {|c| c =~ /FLAG/}
+        fields = []
+        @reader.mysql.query("EXPLAIN `#{name}`") do |res|
+          while field = res.fetch_row do
+            length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/
+            length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/
+            desc = {
+              :name => field[0],
+              :table_name => name,
+              :type => convert_type(field[1]),
+              :length => length && length.to_i,
+              :decimals => field[1][/\((\d+),(\d+)\)/, 2],
+              :null => field[2] == "YES",
+              :primary_key => field[3] == "PRI",
+              :auto_increment => field[5] == "auto_increment"
+              }
+            desc[:default] = field[4] unless field[4].nil?
+            fields << desc
+          end
+        end
+ 
+        fields.select {|field| field[:auto_increment]}.each do |field|
+          @reader.mysql.query("SELECT max(`#{field[:name]}`) FROM `#{name}`") do |res|
+            field[:maxval] = res.fetch_row[0].to_i
+          end
+        end
+        fields
+      end
+    
+    
+      def indexes
+        load_indexes unless @indexes
+        @indexes 
+      end
+ 
+      def foreign_keys
+        load_indexes unless @foreign_keys
+        @foreign_keys
+      end
+    
+      def load_indexes
+        @indexes = []
+        @foreign_keys = []
+      
+        @reader.mysql.query("SHOW CREATE TABLE `#{name}`") do |result|
+          explain = result.fetch_row[1]
+          explain.split(/\n/).each do |line|
+            next unless line =~ / KEY /
+            index = {}
+            if match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \(`(\w+)`\) REFERENCES `(\w+)` \(`(\w+)`\)/.match(line)
+              index[:name] = match_data[1]
+              index[:column] = match_data[2]
+              index[:ref_table] = match_data[3]
+              index[:ref_column] = match_data[4]
+              @foreign_keys << index
+            elsif match_data = /KEY `(\w+)` \((.*)\)/.match(line)
+              index[:name] = match_data[1]
+              index[:columns] = match_data[2].split(",").map {|col| col[/`(\w+)`/, 1]}
+              index[:unique] = true if line =~ /UNIQUE/
+              @indexes << index
+            elsif match_data = /PRIMARY KEY .*\((.*)\)/.match(line)
+              index[:primary] = true
+              index[:columns] = match_data[1].split(",").map {|col| col.strip.gsub(/`/, "")}
+              @indexes << index
+            end
+          end
+        end
+      end
+    
+      def count_rows
+        @reader.mysql.query("SELECT COUNT(*) FROM `#{name}`")  do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+    
+      def has_id?
+        !!columns.find {|col| col[:name] == "id"} 
+      end
+    
+      def count_for_pager
+        query = has_id? ? 'MAX(id)' : 'COUNT(*)'
+        @reader.mysql.query("SELECT #{query} FROM `#{name}`") do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+ 
+      def query_for_pager
+        query = has_id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?'
+        "SELECT #{columns.map{|c| "`"+c[:name]+"`"}.join(", ")} FROM `#{name}` #{query}"
+      end
+    end
+  
+    def connect
+      @mysql = ::Mysql.connect(@host, @user, @passwd, @db, @port, @sock, @flag)
+      @mysql.query("SET NAMES utf8")
+    end
+  
+    def reconnect
+      @mysql.close rescue false
+      connect
+    end
+  
+    def initialize(options)
+      @host, @user, @passwd, @db, @port, @sock, @flag = 
+        options.mysqlhostname('localhost'), options.mysqlusername, 
+        options.mysqlpassword, options.mysqldatabase, 
+        options.mysqlport, options.mysqlsocket, Mysql::CLIENT_COMPRESS
+      connect
+    end
+  
+    attr_reader :mysql
+  
+    def tables
+      @tables ||= @mysql.list_tables.map {|table| Table.new(self, table)}
+    end
+  
+    def paginated_read(table, page_size)
+      count = table.count_for_pager
+      return if count < 1
+      statement = @mysql.prepare(table.query_for_pager)
+      counter = 0
+      0.upto((count + page_size)/page_size) do |i|
+        statement.execute(i*page_size, table.has_id? ? (i+1)*page_size : page_size)
+        while row = statement.fetch
+          counter += 1
+          yield(row, counter)
+        end
+      end
+      counter
+    end
+  end
+
+end