mysql2psql 0.1.0

data/lib/mysql2psql/postgres_writer.rb

@@ -0,0 +1,143 @@
+require 'pg'
+
+require 'mysql2psql/writer'
+
+class Mysql2psql
+
+  class PostgresWriter < Writer
+    def column_description(column)
+      "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}"
+    end
+  
+    def column_type(column)
+      column_type_info(column).split(" ").first
+    end
+  
+    def column_type_info(column)
+      if column[:auto_increment]
+        return "integer DEFAULT nextval('#{column[:table_name]}_#{column[:name]}_seq'::regclass) NOT NULL"
+      end
+    
+      default = column[:default] ? " DEFAULT #{column[:default] == nil ? 'NULL' : "'"+PGconn.escape(column[:default])+"'"}" : nil
+      null = column[:null] ? "" : " NOT NULL"
+      type = 
+      case column[:type]
+    
+      # String types
+      when "char"
+        default = default + "::char" if default
+        "character(#{column[:length]})"
+      when "varchar"
+        default = default + "::character varying" if default
+  #      puts "VARCHAR: #{column.inspect}"
+        "character varying(#{column[:length]})"
+      
+      # Integer and numeric types
+      when "integer"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default
+        "integer"
+      when "bigint"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default
+        "bigint"
+      when "tinyint"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_i}" if default
+        "smallint"
+    
+      when "boolean"
+        default = " DEFAULT #{column[:default].to_i == 1 ? 'true' : 'false'}" if default
+        "boolean"
+      when "float"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default
+        "real"
+      when "float unsigned"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default].to_f}" if default
+        "real"
+      when "decimal"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default
+        "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})"
+
+      when "double precision"
+        default = " DEFAULT #{column[:default].nil? ? 'NULL' : column[:default]}" if default
+        "double precision"
+
+      # Mysql datetime fields
+      when "datetime"
+        default = nil
+        "timestamp without time zone"
+      when "date"
+        default = nil
+        "date"
+      when "timestamp"
+        default = " DEFAULT CURRENT_TIMESTAMP" if column[:default] == "CURRENT_TIMESTAMP"
+        default = " DEFAULT '1970-01-01 00:00'" if column[:default] == "0000-00-00 00:00"
+        default = " DEFAULT '1970-01-01 00:00:00'" if column[:default] == "0000-00-00 00:00:00"
+        "timestamp without time zone"
+      when "time"
+        default = " DEFAULT NOW()" if default
+        "time without time zone"
+
+      when "tinyblob"
+        "bytea"
+      when "mediumblob"
+        "bytea"
+      when "longblob"
+        "bytea"
+      when "blob"
+        "bytea"
+      when "varbinary"
+        "bytea"
+      when "tinytext"
+        "text"
+      when "mediumtext"
+        "text"
+      when "longtext"
+        "text"
+      when "text"
+        "text"
+      when /^enum/
+        default = default + "::character varying" if default
+        enum = column[:type].gsub(/enum|\(|\)/, '')
+        max_enum_size = enum.split(',').map{ |check| check.size() -2}.sort[-1]
+        "character varying(#{max_enum_size}) check( #{column[:name]} in (#{enum}))"
+      else
+        puts "Unknown #{column.inspect}"
+        column[:type].inspect
+        return ""
+      end
+      "#{type}#{default}#{null}"
+    end
+  
+    def process_row(table, row)
+    	table.columns.each_with_index do |column, index|
+
+          if column[:type] == "time"
+            row[index] = "%02d:%02d:%02d" % [row[index].hour, row[index].minute, row[index].second]
+          end
+        
+          if row[index].is_a?(Mysql::Time)
+            row[index] = row[index].to_s.gsub('0000-00-00 00:00', '1970-01-01 00:00')
+            row[index] = row[index].to_s.gsub('0000-00-00 00:00:00', '1970-01-01 00:00:00')
+          end
+        
+          if column_type(column) == "boolean"
+            row[index] = row[index] == 1 ? 't' : row[index] == 0 ? 'f' : row[index]
+          end
+        
+          if row[index].is_a?(String)
+            if column_type(column) == "bytea"
+              row[index] = PGconn.escape_bytea(row[index])
+            else
+              row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r').gsub(/\0/, '')
+            end
+          end
+        
+          row[index] = '\N' if !row[index]
+        end
+    end
+  
+    def truncate(table)
+    end
+  
+  end
+
+end

data/lib/mysql2psql/version.rb

@@ -0,0 +1,9 @@
+class Mysql2psql
+  module Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 0
+
+    STRING = [MAJOR, MINOR, PATCH].compact.join('.')
+  end
+end

data/lib/mysql2psql/writer.rb

@@ -0,0 +1,6 @@
+class Mysql2psql
+
+  class Writer
+  end
+
+end

data/mysql2psql.gemspec

@@ -0,0 +1,91 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{mysql2psql}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Max Lapshin <max@maxidoors.ru>", "Anton Ageev <anton@ageev.name>", "Samuel Tribehou <cracoucax@gmail.com>", "Marco Nenciarini <marco.nenciarini@devise.it>", "James Nobis <jnobis@jnobis.controldocs.com>", "quel <github@quelrod.net>", "Holger Amann <keeney@fehu.org>", "Maxim Dobriakov <closer.main@gmail.com>", "Michael Kimsal <mgkimsal@gmail.com>", "Jacob Coby <jcoby@portallabs.com>", "Neszt Tibor <neszt@tvnetwork.hu>", "Miroslav Kratochvil <exa.exa@gmail.com>", "Paul Gallagher <gallagher.paul@gmail.com>"]
+  s.date = %q{2010-09-19}
+  s.default_executable = %q{mysql2psql}
+  s.description = %q{It can create postgresql dump from mysql database or directly load data from mysql to
+    postgresql (at about 100 000 records per minute). Translates most data types and indexes.}
+  s.email = %q{gallagher.paul@gmail.com}
+  s.executables = ["mysql2psql"]
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "MIT-LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "bin/mysql2psql",
+     "lib/mysql2psql.rb",
+     "lib/mysql2psql/config.rb",
+     "lib/mysql2psql/config_base.rb",
+     "lib/mysql2psql/converter.rb",
+     "lib/mysql2psql/errors.rb",
+     "lib/mysql2psql/mysql_reader.rb",
+     "lib/mysql2psql/postgres_db_writer.rb",
+     "lib/mysql2psql/postgres_file_writer.rb",
+     "lib/mysql2psql/postgres_writer.rb",
+     "lib/mysql2psql/version.rb",
+     "lib/mysql2psql/writer.rb",
+     "mysql2psql.gemspec",
+     "test/fixtures/config_all_options.yml",
+     "test/fixtures/seed_integration_tests.sql",
+     "test/integration/convert_to_db_test.rb",
+     "test/integration/convert_to_file_test.rb",
+     "test/integration/converter_test.rb",
+     "test/integration/mysql_reader_base_test.rb",
+     "test/integration/mysql_reader_test.rb",
+     "test/integration/postgres_db_writer_base_test.rb",
+     "test/lib/ext_test_unit.rb",
+     "test/lib/test_helper.rb",
+     "test/units/config_base_test.rb",
+     "test/units/config_test.rb",
+     "test/units/postgres_file_writer_test.rb"
+  ]
+  s.homepage = %q{http://github.com/tardate/mysql2postgresql}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Tool for converting mysql database to postgresql}
+  s.test_files = [
+    "test/integration/convert_to_db_test.rb",
+     "test/integration/convert_to_file_test.rb",
+     "test/integration/converter_test.rb",
+     "test/integration/mysql_reader_base_test.rb",
+     "test/integration/mysql_reader_test.rb",
+     "test/integration/postgres_db_writer_base_test.rb",
+     "test/lib/ext_test_unit.rb",
+     "test/lib/test_helper.rb",
+     "test/units/config_base_test.rb",
+     "test/units/config_test.rb",
+     "test/units/postgres_file_writer_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<mysql>, ["= 2.8.1"])
+      s.add_runtime_dependency(%q<pg>, ["= 0.9.0"])
+      s.add_development_dependency(%q<test-unit>, [">= 2.1.1"])
+    else
+      s.add_dependency(%q<mysql>, ["= 2.8.1"])
+      s.add_dependency(%q<pg>, ["= 0.9.0"])
+      s.add_dependency(%q<test-unit>, [">= 2.1.1"])
+    end
+  else
+    s.add_dependency(%q<mysql>, ["= 2.8.1"])
+    s.add_dependency(%q<pg>, ["= 0.9.0"])
+    s.add_dependency(%q<test-unit>, [">= 2.1.1"])
+  end
+end
+

data/test/fixtures/seed_integration_tests.sql

@@ -0,0 +1,24 @@
+-- seed data for integration tests
+
+DROP TABLE IF EXISTS numeric_types_basics;
+CREATE TABLE numeric_types_basics (
+  id int,
+	f_tinyint TINYINT,
+	f_smallint SMALLINT,
+	f_mediumint MEDIUMINT,
+	f_int INT,
+	f_integer INTEGER,
+	f_bigint BIGINT,
+	f_real REAL,
+	f_double DOUBLE,
+	f_float FLOAT,
+	f_decimal DECIMAL,
+	f_numeric NUMERIC
+);
+
+INSERT INTO numeric_types_basics VALUES
+(1,1,1,1,1,1,1,1,1,1,1,1),
+(2,2,2,2,2,2,2,2,2,2,2,2),
+(23,23,23,23,23,23,23,23,23,23,23,23);
+
+

data/test/integration/convert_to_db_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+require 'mysql2psql'
+
+class ConvertToDbTest < Test::Unit::TestCase
+
+  class << self
+    def startup
+      seed_test_database
+      @@options=get_test_config_by_label(:localmysql_to_db_convert_all)
+      @@mysql2psql = Mysql2psql.new([@@options.filepath])
+      @@mysql2psql.convert
+      @@mysql2psql.writer.open
+    end
+    def shutdown
+      @@mysql2psql.writer.close
+      delete_files_for_test_config(@@options)
+    end
+  end
+  def setup
+  end
+  def teardown
+  end
+
+  def test_table_creation
+    assert_true @@mysql2psql.writer.exists?('numeric_types_basics')
+  end
+
+end

data/test/integration/convert_to_file_test.rb

@@ -0,0 +1,66 @@
+require 'test_helper'
+
+require 'mysql2psql'
+
+class ConvertToFileTest < Test::Unit::TestCase
+
+  class << self
+    def startup
+      seed_test_database
+      @@options=get_test_config_by_label(:localmysql_to_file_convert_all)
+      @@mysql2psql = Mysql2psql.new([@@options.filepath])
+      @@mysql2psql.convert
+      @@content = IO.read(@@mysql2psql.options.destfile)
+    end
+    def shutdown
+      delete_files_for_test_config(@@options)
+    end
+  end
+  def setup
+  end
+  def teardown
+  end
+  def content
+    @@content
+  end
+  
+  def test_table_creation
+    assert_not_nil content.match('DROP TABLE IF EXISTS "numeric_types_basics" CASCADE')
+    assert_not_nil content.match(/CREATE TABLE "numeric_types_basics"/)
+  end
+
+  def test_basic_numerics_tinyint
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_smallint
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_mediumint
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_int
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_integer
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_bigint
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_real
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_double
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_float
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" numeric\(20, 0\),.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_decimal
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match( content )
+  end
+  def test_basic_numerics_numeric
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match( content )
+  end
+
+end

data/test/integration/converter_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+require 'mysql2psql/converter'
+
+class ConverterTest < Test::Unit::TestCase
+
+  class << self
+    def startup
+      seed_test_database
+      @@options=get_test_config_by_label(:localmysql_to_file_convert_nothing)
+    end
+    def shutdown
+      delete_files_for_test_config(@@options)
+    end
+  end
+  def setup
+  end
+  def teardown
+  end
+  def options
+    @@options
+  end
+  
+  def test_new_converter
+    assert_nothing_raised do
+      reader=get_test_reader(options)
+      writer=get_test_file_writer(options)
+      converter=Mysql2psql::Converter.new(reader,writer,options)
+      assert_equal 0,converter.convert
+    end
+  end
+
+
+end

data/test/integration/mysql_reader_base_test.rb

@@ -0,0 +1,35 @@
+require 'test_helper'
+
+require 'mysql2psql/mysql_reader'
+
+class MysqlReaderBaseTest < Test::Unit::TestCase
+
+  class << self
+    def startup
+      seed_test_database
+      @@options = get_test_config_by_label(:localmysql_to_file_convert_nothing)
+    end
+    def shutdown
+      delete_files_for_test_config(@@options)
+    end
+  end
+  def setup
+  end
+  def teardown
+  end
+  def options
+    @@options
+  end
+  
+  def test_mysql_connection
+    assert_nothing_raised do
+      reader = Mysql2psql::MysqlReader.new(options)
+    end
+  end
+  def test_mysql_reconnect
+    assert_nothing_raised do
+      reader = Mysql2psql::MysqlReader.new(options)
+      reader.reconnect
+    end
+  end
+end