mysql2psql 0.1.0

data/lib/mysql2psql/errors.rb

@@ -0,0 +1,16 @@
+
+class Mysql2psql
+  
+  class GeneralError < StandardError
+	end
+
+  class ConfigurationError < StandardError
+	end
+  class UninitializedValueError < ConfigurationError
+	end
+  class ConfigurationFileNotFound < ConfigurationError
+	end
+  class ConfigurationFileInitialized < ConfigurationError
+	end	
+	
+end

data/lib/mysql2psql/mysql_reader.rb

@@ -0,0 +1,191 @@
+require 'mysql'
+
+class Mysql2psql
+
+  class MysqlReader
+    class Field
+    end
+  
+    class Table
+      attr_reader :name
+    
+      def initialize(reader, name)
+        @reader = reader
+        @name = name
+      end
+    
+      @@types = %w(tiny enum decimal short long float double null timestamp longlong int24 date time datetime year set blob string var_string char).inject({}) do |list, type|
+        list[eval("::Mysql::Field::TYPE_#{type.upcase}")] = type
+        list
+      end
+    
+      @@types[246] = "decimal"
+    
+      def columns
+        @columns ||= load_columns
+      end
+    
+      def convert_type(type)
+        case type
+        when /int.* unsigned/
+          "bigint"
+        when /bigint/
+          "bigint"
+        when "bit(1)"
+          "boolean"
+        when "tinyint(1)"
+          "boolean"
+        when /tinyint/
+          "tinyint"
+        when /int/
+          "integer"
+        when /varchar/
+          "varchar"
+        when /char/
+          "char"
+        when /(float|decimal)/
+          "decimal"
+        when /double/
+           "double precision"
+        else
+          type
+        end 
+      end
+    
+      def load_columns
+        @reader.reconnect
+        result = @reader.mysql.list_fields(name)
+        mysql_flags = ::Mysql::Field.constants.select {|c| c =~ /FLAG/}
+        fields = []
+        @reader.mysql.query("EXPLAIN `#{name}`") do |res|
+          while field = res.fetch_row do
+            length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/
+            length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/
+            desc = {
+              :name => field[0],
+              :table_name => name,
+              :type => convert_type(field[1]),
+              :length => length && length.to_i,
+              :decimals => field[1][/\((\d+),(\d+)\)/, 2],
+              :null => field[2] == "YES",
+              :primary_key => field[3] == "PRI",
+              :auto_increment => field[5] == "auto_increment"
+              }
+            desc[:default] = field[4] unless field[4].nil?
+            fields << desc
+          end
+        end
+ 
+        fields.select {|field| field[:auto_increment]}.each do |field|
+          @reader.mysql.query("SELECT max(`#{field[:name]}`) FROM `#{name}`") do |res|
+            field[:maxval] = res.fetch_row[0].to_i
+          end
+        end
+        fields
+      end
+    
+    
+      def indexes
+        load_indexes unless @indexes
+        @indexes 
+      end
+ 
+      def foreign_keys
+        load_indexes unless @foreign_keys
+        @foreign_keys
+      end
+    
+      def load_indexes
+        @indexes = []
+        @foreign_keys = []
+      
+        @reader.mysql.query("SHOW CREATE TABLE `#{name}`") do |result|
+          explain = result.fetch_row[1]
+          explain.split(/\n/).each do |line|
+            next unless line =~ / KEY /
+            index = {}
+            if match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \(`(\w+)`\) REFERENCES `(\w+)` \(`(\w+)`\)/.match(line)
+              index[:name] = match_data[1]
+              index[:column] = match_data[2]
+              index[:ref_table] = match_data[3]
+              index[:ref_column] = match_data[4]
+              @foreign_keys << index
+            elsif match_data = /KEY `(\w+)` \((.*)\)/.match(line)
+              index[:name] = match_data[1]
+              index[:columns] = match_data[2].split(",").map {|col| col[/`(\w+)`/, 1]}
+              index[:unique] = true if line =~ /UNIQUE/
+              @indexes << index
+            elsif match_data = /PRIMARY KEY .*\((.*)\)/.match(line)
+              index[:primary] = true
+              index[:columns] = match_data[1].split(",").map {|col| col.strip.gsub(/`/, "")}
+              @indexes << index
+            end
+          end
+        end
+      end
+    
+      def count_rows
+        @reader.mysql.query("SELECT COUNT(*) FROM `#{name}`")  do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+    
+      def has_id?
+        !!columns.find {|col| col[:name] == "id"} 
+      end
+    
+      def count_for_pager
+        query = has_id? ? 'MAX(id)' : 'COUNT(*)'
+        @reader.mysql.query("SELECT #{query} FROM `#{name}`") do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+ 
+      def query_for_pager
+        query = has_id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?'
+        "SELECT #{columns.map{|c| "`"+c[:name]+"`"}.join(", ")} FROM `#{name}` #{query}"
+      end
+    end
+  
+    def connect
+      @mysql = ::Mysql.connect(@host, @user, @passwd, @db, @port, @sock, @flag)
+      @mysql.query("SET NAMES utf8")
+      @mysql.query("SET SESSION query_cache_type = OFF")
+    end
+  
+    def reconnect
+      @mysql.close rescue false
+      connect
+    end
+  
+    def initialize(options)
+      @host, @user, @passwd, @db, @port, @sock, @flag = 
+        options.mysqlhostname('localhost'), options.mysqlusername, 
+        options.mysqlpassword, options.mysqldatabase, 
+        options.mysqlport, options.mysqlsocket
+      connect
+    end
+  
+    attr_reader :mysql
+  
+    def tables
+      @tables ||= @mysql.list_tables.map {|table| Table.new(self, table)}
+    end
+  
+    def paginated_read(table, page_size)
+      count = table.count_for_pager
+      return if count < 1
+      statement = @mysql.prepare(table.query_for_pager)
+      counter = 0
+      0.upto((count + page_size)/page_size) do |i|
+        statement.execute(i*page_size, table.has_id? ? (i+1)*page_size : page_size)
+        while row = statement.fetch
+          counter += 1
+          yield(row, counter)
+        end
+      end
+      counter
+    end
+  end
+
+end

data/lib/mysql2psql/postgres_db_writer.rb

@@ -0,0 +1,179 @@
+require 'pg'
+
+require 'mysql2psql/postgres_writer'
+
+class Mysql2psql
+
+class PostgresDbWriter < PostgresWriter
+  attr_reader :conn, :hostname, :login, :password, :database, :schema, :port
+  
+  def initialize(options)
+    @hostname, @login, @password, @database, @port =  
+      options.pghostname('localhost'), options.pgusername, 
+      options.pgpassword, options.pgdatabase, options.pgport(5432).to_s
+    @database, @schema = database.split(":")
+    open
+  end
+
+  def open
+    @conn = PGconn.new(hostname, port, '', '', database, login, password)
+    @conn.exec("SET search_path TO #{PGconn.quote_ident(schema)}") if schema
+    @conn.exec("SET client_encoding = 'UTF8'")
+    @conn.exec("SET standard_conforming_strings = off") if @conn.server_version >= 80200
+    @conn.exec("SET check_function_bodies = false")
+    @conn.exec("SET client_min_messages = warning")    
+  end
+  
+  def close
+    @conn.close
+  end
+
+  def exists?(relname)
+    rc = @conn.exec("SELECT COUNT(*) FROM pg_class WHERE relname = '#{relname}'")
+    (!rc.nil?) && (rc.to_a.length==1) && (rc.first.count.to_i==1)
+  end
+  
+  def write_table(table)
+    puts "Creating table #{table.name}..."
+    primary_keys = []
+    serial_key = nil
+    maxval = nil
+    
+    columns = table.columns.map do |column|
+      if column[:auto_increment]
+        serial_key = column[:name]
+        maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1
+      end
+      if column[:primary_key]
+        primary_keys << column[:name]
+      end
+      "  " + column_description(column)
+    end.join(",\n")
+    
+    if serial_key
+      if @conn.server_version < 80200
+        serial_key_seq = "#{table.name}_#{serial_key}_seq"
+        @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq)
+      else
+        @conn.exec("DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE")
+      end
+      @conn.exec <<-EOF
+        CREATE SEQUENCE #{table.name}_#{serial_key}_seq
+        INCREMENT BY 1
+        NO MAXVALUE
+        NO MINVALUE
+        CACHE 1
+      EOF
+    
+      @conn.exec "SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true)"
+    end
+    
+    if @conn.server_version < 80200
+      @conn.exec "DROP TABLE #{PGconn.quote_ident(table.name)} CASCADE;" if exists?(table.name)
+    else
+      @conn.exec "DROP TABLE IF EXISTS #{PGconn.quote_ident(table.name)} CASCADE;"
+    end
+    create_sql = "CREATE TABLE #{PGconn.quote_ident(table.name)} (\n" + columns + "\n)\nWITHOUT OIDS;"
+    begin
+      @conn.exec(create_sql)
+    rescue Exception => e
+      puts "Error: \n#{create_sql}"
+      raise
+    end
+    puts "Created table #{table.name}"
+ 
+  end
+  
+  def write_indexes(table)
+    puts "Indexing table #{table.name}..."
+    if primary_index = table.indexes.find {|index| index[:primary]}
+      @conn.exec("ALTER TABLE #{PGconn.quote_ident(table.name)} ADD CONSTRAINT \"#{table.name}_pkey\" PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})")
+    end
+    
+    table.indexes.each do |index|
+      next if index[:primary]
+      unique = index[:unique] ? "UNIQUE " : nil
+      
+      #MySQL allows an index name which could be equal to a table name, Postgres doesn't
+      indexname = index[:name]
+      if indexname.eql?(table.name)
+        indexnamenew = "#{indexname}_index"
+        puts "WARNING: index \"#{indexname}\" equals table name. This is not allowed by postgres and will be renamed to \"#{indexnamenew}\""
+        indexname = indexnamenew
+      end
+      
+      if @conn.server_version < 80200
+        @conn.exec("DROP INDEX #{PGconn.quote_ident(indexname)} CASCADE;") if exists?(indexname)
+      else
+        @conn.exec("DROP INDEX IF EXISTS #{PGconn.quote_ident(indexname)} CASCADE;")
+      end
+      @conn.exec("CREATE #{unique}INDEX #{PGconn.quote_ident(indexname)} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});")
+    end
+    
+    
+    #@conn.exec("VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)}")
+    puts "Indexed table #{table.name}"
+  rescue Exception => e
+    puts "Couldn't create indexes on #{table} (#{table.indexes.inspect})"
+    puts e
+    puts e.backtrace[0,3].join("\n")
+  end
+  
+  def write_constraints(table)
+    table.foreign_keys.each do |key|
+      key_sql = "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{PGconn.quote_ident(key[:column])}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{PGconn.quote_ident(key[:ref_column])})"
+      begin
+        @conn.exec(key_sql)
+      rescue Exception => e
+        puts "Error: \n#{key_sql}\n#{e}"
+      end
+    end
+  end
+  
+  def format_eta (t)
+    t = t.to_i
+    sec = t % 60
+    min = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02dh:%02dm:%02ds", hour, min, sec)
+  end
+  
+  def write_contents(table, reader)
+    _time1 = Time.now
+    copy_line = "COPY #{PGconn.quote_ident(table.name)} (#{table.columns.map {|column| PGconn.quote_ident(column[:name])}.join(", ")}) FROM stdin;"
+    @conn.exec(copy_line)
+    puts "Counting rows of #{table.name}... "
+    STDOUT.flush
+    rowcount = table.count_rows
+    puts "Rows counted"
+    puts "Loading #{table.name}..."
+    STDOUT.flush
+    _counter = reader.paginated_read(table, 1000) do |row, counter|
+      line = []
+      process_row(table, row)
+      @conn.put_copy_data(row.join("\t") + "\n")
+       
+      if counter != 0 && counter % 20000 == 0
+        elapsedTime = Time.now - _time1
+        eta = elapsedTime * rowcount / counter - elapsedTime
+        etaf = self.format_eta(eta)
+        etatimef = (Time.now + eta).strftime("%Y/%m/%d %H:%M")
+        printf "\r#{counter} of #{rowcount} rows loaded. [ETA: #{etatimef} (#{etaf})]"
+        STDOUT.flush
+      end
+      
+      if counter % 5000 == 0
+        @conn.put_copy_end
+        @conn.exec(copy_line)
+      end
+       
+    end
+    _time2 = Time.now
+    puts "\n#{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s"
+#    @conn.putline(".\n")
+    @conn.put_copy_end
+  end
+  
+end
+
+end

data/lib/mysql2psql/postgres_file_writer.rb

@@ -0,0 +1,142 @@
+require 'mysql2psql/postgres_writer'
+
+class Mysql2psql
+
+class PostgresFileWriter < PostgresWriter
+  def initialize(file)
+    @f = File.open(file, "w+")
+    @f << <<-EOF
+-- MySQL 2 PostgreSQL dump\n
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+ 
+EOF
+  end
+  
+  def truncate(table)
+    serial_key = nil
+    maxval = nil
+    
+    table.columns.map do |column|
+      if column[:auto_increment]
+        serial_key = column[:name]
+        maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1
+      end
+    end
+
+    @f << <<-EOF
+-- TRUNCATE #{table.name};
+TRUNCATE #{PGconn.quote_ident(table.name)} CASCADE;
+
+EOF
+    if serial_key
+    @f << <<-EOF
+SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true);
+EOF
+    end
+  end
+  
+  def write_table(table)
+    primary_keys = []
+    serial_key = nil
+    maxval = nil
+    
+    columns = table.columns.map do |column|
+      if column[:auto_increment]
+        serial_key = column[:name]
+        maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1
+      end
+      if column[:primary_key]
+        primary_keys << column[:name]
+      end
+      "  " + column_description(column)
+    end.join(",\n")
+    
+    if serial_key
+      
+      @f << <<-EOF
+--
+-- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public
+--
+ 
+DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE;
+ 
+CREATE SEQUENCE #{table.name}_#{serial_key}_seq
+    INCREMENT BY 1
+    NO MAXVALUE
+    NO MINVALUE
+    CACHE 1;
+    
+    
+SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true);
+ 
+      EOF
+    end
+    
+    @f << <<-EOF
+-- Table: #{table.name}
+ 
+-- DROP TABLE #{table.name};
+DROP TABLE IF EXISTS #{PGconn.quote_ident(table.name)} CASCADE;
+ 
+CREATE TABLE #{PGconn.quote_ident(table.name)} (
+EOF
+  
+    @f << columns
+ 
+    if primary_index = table.indexes.find {|index| index[:primary]}
+      @f << ",\n  CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})"
+    end
+    
+    @f << <<-EOF
+\n)
+WITHOUT OIDS;
+EOF
+  
+    table.indexes.each do |index|
+      next if index[:primary]
+      unique = index[:unique] ? "UNIQUE " : nil
+      @f << <<-EOF
+DROP INDEX IF EXISTS #{PGconn.quote_ident(index[:name])} CASCADE;
+CREATE #{unique}INDEX #{PGconn.quote_ident(index[:name])} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});
+EOF
+    end
+ 
+  end
+  
+  def write_indexes(table)
+  end
+  
+  def write_constraints(table)
+    table.foreign_keys.each do |key|
+      @f << "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{PGconn.quote_ident(key[:column])}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{PGconn.quote_ident(key[:ref_column])});\n"
+    end
+  end
+  
+  
+  def write_contents(table, reader)
+    @f << <<-EOF
+--
+-- Data for Name: #{table.name}; Type: TABLE DATA; Schema: public
+--
+
+COPY "#{table.name}" (#{table.columns.map {|column| PGconn.quote_ident(column[:name])}.join(", ")}) FROM stdin;
+EOF
+    
+    reader.paginated_read(table, 1000) do |row, counter|
+      line = []
+      process_row(table, row)
+      @f << row.join("\t") + "\n"
+    end
+    @f << "\\.\n\n"
+    #@f << "VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)};\n\n"
+  end
+  
+  def close
+    @f.close
+  end
+end
+
+end