benhutton-mysql2psql 0.2.0

data/lib/mysql2psql/mysql_reader.rb

@@ -0,0 +1,230 @@
+require 'mysql'
+require 'csv'
+
+class Mysql2psql
+
+  class MysqlReader
+    class Field
+    end
+  
+    class Table
+      attr_reader :name
+    
+      def initialize(reader, name)
+        @reader = reader
+        @name = name
+      end
+    
+      @@types = %w(tiny enum decimal short long float double null timestamp longlong int24 date time datetime year set blob string var_string char).inject({}) do |list, type|
+        list[eval("::Mysql::Field::TYPE_#{type.upcase}")] = type
+        list
+      end
+    
+      @@types[246] = "decimal"
+    
+      def columns
+        @columns ||= load_columns
+      end
+    
+      def convert_type(type)
+        case type
+        when /^int.* unsigned/
+          "bigint"
+        when /bigint/
+          "bigint"
+        when "bit(1)"
+          "boolean"
+        when /smallint.* unsigned/
+          "integer"
+        when /smallint/
+          "smallint"
+        when "tinyint(1)"
+          "boolean"
+        when /tinyint/
+          "tinyint"
+        when /int/
+          "integer"
+        when /varchar/
+          "varchar"
+        when /char/
+          "char"
+        when /decimal/
+          "decimal"
+        when /float/
+          "float"
+        when /real|double/
+          "double precision"
+        else
+          type
+        end
+      end
+    
+      def load_columns
+        @reader.reconnect
+        result = @reader.mysql.list_fields(name)
+        mysql_flags = ::Mysql::Field.constants.select {|c| c =~ /FLAG/}
+        fields = []
+        @reader.mysql.query("EXPLAIN `#{name}`") do |res|
+          while field = res.fetch_row do
+            length = -1
+            length = field[1][/\((\d+)\)/, 1] if field[1] =~ /\((\d+)\)/
+            length = field[1][/\((\d+),(\d+)\)/, 1] if field[1] =~ /\((\d+),(\d+)\)/
+            desc = {
+              :name => field[0],
+              :table_name => name,
+              :type => convert_type(field[1]),
+              :length => length && length.to_i,
+              :decimals => field[1][/\((\d+),(\d+)\)/, 2],
+              :null => field[2] == "YES",
+              :primary_key => field[3] == "PRI",
+              :auto_increment => field[5] == "auto_increment"
+              }
+            desc[:default] = field[4] unless field[4].nil?
+            fields << desc
+          end
+        end
+ 
+        fields.select {|field| field[:auto_increment]}.each do |field|
+          @reader.mysql.query("SELECT max(`#{field[:name]}`) FROM `#{name}`") do |res|
+            field[:maxval] = res.fetch_row[0].to_i
+          end
+        end
+        fields
+      end
+    
+    
+      def indexes
+        load_indexes unless @indexes
+        @indexes
+      end
+ 
+      def foreign_keys
+        load_indexes unless @foreign_keys
+        @foreign_keys
+      end
+    
+      def load_indexes
+        @indexes = []
+        @foreign_keys = []
+      
+        @reader.mysql.query("SHOW CREATE TABLE `#{name}`") do |result|
+          explain = result.fetch_row[1]
+          explain.split(/\n/).each do |line|
+            next unless line =~ / KEY /
+            index = {}
+            if match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line)
+              index[:name] = match_data[1]
+              index[:column] = match_data[2].parse_csv(:quote_char => '`',:col_sep => ', ')
+              index[:ref_table] = match_data[3]
+              index[:ref_column] = match_data[4].parse_csv(:quote_char => '`',:col_sep => ', ')
+              
+              the_rest = match_data[5]
+
+              if match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest)
+                index[:on_delete] = match_data[1]
+              else
+                index[:on_delete] ||= 'RESTRICT'
+              end
+
+              if match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest)
+                index[:on_update] = match_data[1]
+              else
+                index[:on_update] ||= 'RESTRICT'
+              end
+              
+              @foreign_keys << index
+            elsif match_data = /KEY `(\w+)` \((.*)\)/.match(line)
+              index[:name] = match_data[1]
+              index[:columns] = match_data[2].split(",").map {|col| col[/`(\w+)`/, 1]}
+              index[:unique] = true if line =~ /UNIQUE/
+              @indexes << index
+            elsif match_data = /PRIMARY KEY .*\((.*)\)/.match(line)
+              index[:primary] = true
+              index[:columns] = match_data[1].split(",").map {|col| col.strip.gsub(/`/, "")}
+              @indexes << index
+            end
+          end
+        end
+      end
+    
+      def count_rows
+        @reader.mysql.query("SELECT COUNT(*) FROM `#{name}`")  do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+    
+      def has_id?
+        !!columns.find {|col| col[:name] == "id"}
+      end
+    
+      def count_for_pager
+        query = has_id? ? 'MAX(id)' : 'COUNT(*)'
+        @reader.mysql.query("SELECT #{query} FROM `#{name}`") do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+ 
+      def query_for_pager
+        query = has_id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?'
+        "SELECT #{columns.map{|c| "`"+c[:name]+"`"}.join(", ")} FROM `#{name}` #{query}"
+      end
+    end
+  
+    def connect
+      @mysql = ::Mysql.connect(@host, @user, @passwd, @db, @port, @sock, @flag)
+      @mysql.query("SET NAMES utf8")
+      @mysql.query("SET SESSION query_cache_type = OFF")
+    end
+  
+    def reconnect
+      @mysql.close rescue false
+      connect
+    end
+  
+    def initialize(options)
+      @host, @user, @passwd, @db, @port, @sock, @flag =
+        options.mysqlhostname('localhost'), options.mysqlusername,
+        options.mysqlpassword, options.mysqldatabase,
+        options.mysqlport, options.mysqlsocket
+      @port = nil if @port == ""  # for things like Amazon's RDS you don't have a port and connect fails with "" for a value
+      @sock = nil if @sock == ""
+      @flag = nil if @flag == ""
+      connect
+    end
+  
+    attr_reader :mysql
+    
+    def views
+      unless defined? @views
+        @mysql.query("SELECT t.TABLE_NAME FROM INFORMATION_SCHEMA.TABLES t WHERE t.TABLE_SCHEMA = '#{@db}' AND t.TABLE_TYPE = 'VIEW';") do |res|
+          @views = []
+          res.each { |row| @views << row[0] }
+        end
+      end
+      
+      @views
+    end
+    
+    def tables
+      @tables ||= (@mysql.list_tables - views).map do |table|
+        Table.new(self, table)
+      end
+    end
+  
+    def paginated_read(table, page_size)
+      count = table.count_for_pager
+      return if count < 1
+      statement = @mysql.prepare(table.query_for_pager)
+      counter = 0
+      0.upto((count + page_size)/page_size) do |i|
+        statement.execute(i*page_size, table.has_id? ? (i+1)*page_size : page_size)
+        while row = statement.fetch
+          counter += 1
+          yield(row, counter)
+        end
+      end
+      counter
+    end
+  end
+
+end

data/lib/mysql2psql/postgres_db_writer.rb

@@ -0,0 +1,201 @@
+require 'pg'
+
+require 'mysql2psql/postgres_writer'
+
+class Mysql2psql
+
+class PostgresDbWriter < PostgresWriter
+  attr_reader :conn, :hostname, :login, :password, :database, :schema, :port
+  
+  def initialize(options)
+    @hostname, @login, @password, @database, @port =
+      options.pghostname('localhost'), options.pgusername,
+      options.pgpassword, options.pgdatabase, options.pgport(5432).to_s
+    @database, @schema = database.split(":")
+    open
+  end
+
+  def open
+    @conn = PGconn.new(hostname, port, '', '', database, login, password)
+    @conn.exec("SET search_path TO #{PGconn.quote_ident(schema)}") if schema
+    @conn.exec("SET client_encoding = 'UTF8'")
+    @conn.exec("SET standard_conforming_strings = off") if @conn.server_version >= 80200
+    @conn.exec("SET check_function_bodies = false")
+    @conn.exec("SET client_min_messages = warning")
+  end
+  
+  def close
+    @conn.close
+  end
+  
+  def exists?(relname)
+    rc = @conn.exec("SELECT COUNT(*) FROM pg_class WHERE relname = '#{relname}'")
+    (!rc.nil?) && (rc.to_a.length==1) && (rc.first.count.to_i==1)
+  end
+  
+  def write_sequence_update(table, options)
+    serial_key_column = table.columns.detect do |column|
+      column[:auto_increment]
+    end
+    
+    if serial_key_column
+      serial_key = serial_key_column[:name]
+      max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1
+      serial_key_seq = "#{table.name}_#{serial_key}_seq"
+      
+      if !options.supress_ddl
+        if @conn.server_version < 80200
+          @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq)
+        else
+          @conn.exec("DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE")
+        end
+        @conn.exec <<-EOF
+          CREATE SEQUENCE #{serial_key_seq}
+          INCREMENT BY 1
+          NO MAXVALUE
+          NO MINVALUE
+          CACHE 1
+        EOF
+      end
+      
+      if !options.supress_sequence_update
+        puts "Updated sequence #{serial_key_seq} to current value of #{max_value}"
+        @conn.exec sqlfor_set_serial_sequence(table, serial_key_seq, max_value)
+      end
+    end
+  end
+  
+  def write_table(table, options)
+    puts "Creating table #{table.name}..."
+    primary_keys = []
+    
+    columns = table.columns.map do |column|
+      if column[:primary_key]
+        primary_keys << column[:name]
+      end
+      "  " + column_description(column, options)
+    end.join(",\n")
+    
+    if @conn.server_version < 80200
+      @conn.exec "DROP TABLE #{PGconn.quote_ident(table.name)} CASCADE;" if exists?(table.name)
+    else
+      @conn.exec "DROP TABLE IF EXISTS #{PGconn.quote_ident(table.name)} CASCADE;"
+    end
+    create_sql = "CREATE TABLE #{PGconn.quote_ident(table.name)} (\n" + columns + "\n)\nWITHOUT OIDS;"
+    begin
+      @conn.exec(create_sql)
+    rescue Exception => e
+      puts "Error: \n#{create_sql}"
+      raise
+    end
+    puts "Created table #{table.name}"
+ 
+  end
+
+  def write_indexes(table)
+    puts "Indexing table #{table.name}..."
+    if primary_index = table.indexes.find {|index| index[:primary]}
+      index_sql = "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD CONSTRAINT \"#{table.name}_pkey\" PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})"
+      @conn.exec(index_sql)
+    end
+
+    table.indexes.each do |index|
+      next if index[:primary]
+      unique = index[:unique] ? "UNIQUE " : nil
+      
+      # MySQL allows an index name which could be equal to a table name, Postgres doesn't
+      indexname = index[:name]
+      indexname_quoted = ''
+
+      if indexname.eql?(table.name)
+        indexname = (@conn.server_version < 90000) ? "#{indexname}_index" : nil
+        puts "WARNING: index \"#{index[:name]}\" equals table name. This is not allowed in PostgreSQL and will be renamed."
+      end
+
+      if indexname
+        indexname_quoted = PGconn.quote_ident(indexname)
+        if @conn.server_version < 80200
+          @conn.exec("DROP INDEX #{PGconn.quote_ident(indexname)} CASCADE;") if exists?(indexname)
+        else
+          @conn.exec("DROP INDEX IF EXISTS #{PGconn.quote_ident(indexname)} CASCADE;")
+        end
+      end
+      
+      index_sql = "CREATE #{unique}INDEX #{indexname_quoted} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});"
+      @conn.exec(index_sql)
+    end
+
+    #@conn.exec("VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)}")
+    puts "Indexed table #{table.name}"
+  rescue Exception => e
+    puts "Couldn't create indexes on #{table} (#{table.indexes.inspect})"
+    puts e
+    puts e.backtrace[0,3].join("\n")
+  end
+
+  def write_constraints(table)
+    table.foreign_keys.each do |key|
+      key_sql = "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{key[:column].map{|c|PGconn.quote_ident(c)}.join(', ')}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{key[:ref_column].map{|c|PGconn.quote_ident(c)}.join(', ')}) ON UPDATE #{key[:on_update]} ON DELETE #{key[:on_delete]}"
+      begin
+        @conn.exec(key_sql)
+      rescue Exception => e
+        puts "Error: \n#{key_sql}\n#{e}"
+      end
+    end
+  end
+  
+  def format_eta (t)
+    t = t.to_i
+    sec = t % 60
+    min = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02dh:%02dm:%02ds", hour, min, sec)
+  end
+  
+  def write_contents(table, reader)
+    _time1 = Time.now
+    copy_line = "COPY #{PGconn.quote_ident(table.name)} (#{table.columns.map {|column| PGconn.quote_ident(column[:name])}.join(", ")}) FROM stdin;"
+    @conn.exec(copy_line)
+    puts "Counting rows of #{table.name}... "
+    STDOUT.flush
+    rowcount = table.count_rows
+    puts "Rows counted"
+    puts "Loading #{table.name}..."
+    STDOUT.flush
+    _counter = reader.paginated_read(table, 1000) do |row, counter|
+      process_row(table, row)
+      @conn.put_copy_data(row.join("\t") + "\n")
+       
+      if counter != 0 && counter % 20000 == 0
+        elapsedTime = Time.now - _time1
+        eta = elapsedTime * rowcount / counter - elapsedTime
+        etaf = self.format_eta(eta)
+        etatimef = (Time.now + eta).strftime("%Y/%m/%d %H:%M")
+        printf "\r#{counter} of #{rowcount} rows loaded. [ETA: #{etatimef} (#{etaf})]"
+        STDOUT.flush
+      end
+      
+      if counter % 5000 == 0
+        @conn.put_copy_end
+        res = @conn.get_result
+        if res.cmdtuples != 5000
+          puts "\nWARNING: #{table.name} expected 5000 tuple inserts got #{res.cmdtuples} at row #{counter}\n"
+        end
+        @conn.exec(copy_line)
+      end
+       
+    end
+    @conn.put_copy_end
+    if _counter && (_counter % 5000) > 0
+      res = @conn.get_result
+      if res.cmdtuples != (_counter % 5000)
+        puts "\nWARNING: table #{table.name} expected #{_counter % 5000} tuple inserts got #{res.cmdtuples}\n"
+      end
+    end
+    _time2 = Time.now
+    puts "\n#{table.name} #{_counter} rows loaded in #{((_time2 - _time1) / 60).round}min #{((_time2 - _time1) % 60).round}s"
+  end
+
+end
+
+end

data/lib/mysql2psql/postgres_file_writer.rb

@@ -0,0 +1,152 @@
+require 'mysql2psql/postgres_writer'
+
+class Mysql2psql
+
+class PostgresFileWriter < PostgresWriter
+  def initialize(file)
+    @f = File.open(file, "w+")
+    @f << <<-EOF
+-- MySQL 2 PostgreSQL dump\n
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+ 
+EOF
+  end
+  
+  def truncate(table)
+    serial_key = nil
+    maxval = nil
+    
+    table.columns.map do |column|
+      if column[:auto_increment]
+        serial_key = column[:name]
+        maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1
+      end
+    end
+
+    @f << <<-EOF
+-- TRUNCATE #{table.name};
+TRUNCATE #{PGconn.quote_ident(table.name)} CASCADE;
+
+EOF
+    if serial_key
+    @f << <<-EOF
+#{sqlfor_reset_serial_sequence(table,serial_key,maxval)}
+EOF
+    end
+  end
+  
+  def write_sequence_update(table, options)
+    serial_key_column = table.columns.detect do |column|
+      column[:auto_increment]
+    end
+    
+    if serial_key_column
+      serial_key = serial_key_column[:name]
+      serial_key_seq = "#{table.name}_#{serial_key}_seq"
+      max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1
+      
+      @f << <<-EOF
+--
+-- Name: #{serial_key_seq}; Type: SEQUENCE; Schema: public
+--
+EOF
+
+      if !options.supress_ddl
+        @f << <<-EOF
+DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE;
+ 
+CREATE SEQUENCE #{serial_key_seq}
+    INCREMENT BY 1
+    NO MAXVALUE
+    NO MINVALUE
+    CACHE 1;
+EOF
+      end
+      
+      if !options.supress_sequence_update
+        @f << <<-EOF
+#{sqlfor_set_serial_sequence(table, serial_key_seq, max_value)}
+EOF
+      end
+    end
+  end
+  
+  def write_table(table, options)
+    primary_keys = []
+    serial_key = nil
+    maxval = nil
+    
+    columns = table.columns.map do |column|
+      if column[:primary_key]
+        primary_keys << column[:name]
+      end
+      "  " + column_description(column, options)
+    end.join(",\n")
+    
+    @f << <<-EOF
+-- Table: #{table.name}
+ 
+-- DROP TABLE #{table.name};
+DROP TABLE IF EXISTS #{PGconn.quote_ident(table.name)} CASCADE;
+ 
+CREATE TABLE #{PGconn.quote_ident(table.name)} (
+EOF
+  
+    @f << columns
+ 
+    if primary_index = table.indexes.find {|index| index[:primary]}
+      @f << ",\n  CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{primary_index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")})"
+    end
+    
+    @f << <<-EOF
+\n)
+WITHOUT OIDS;
+EOF
+  
+    table.indexes.each do |index|
+      next if index[:primary]
+      unique = index[:unique] ? "UNIQUE " : nil
+      @f << <<-EOF
+DROP INDEX IF EXISTS #{PGconn.quote_ident(index[:name])} CASCADE;
+CREATE #{unique}INDEX #{PGconn.quote_ident(index[:name])} ON #{PGconn.quote_ident(table.name)} (#{index[:columns].map {|col| PGconn.quote_ident(col)}.join(", ")});
+EOF
+    end
+ 
+  end
+  
+  def write_indexes(table)
+  end
+  
+  def write_constraints(table)
+    table.foreign_keys.each do |key|
+      @f << "ALTER TABLE #{PGconn.quote_ident(table.name)} ADD FOREIGN KEY (#{key[:column].map{|c|PGconn.quote_ident(c)}.join(', ')}) REFERENCES #{PGconn.quote_ident(key[:ref_table])}(#{key[:ref_column].map{|c|PGconn.quote_ident(c)}.join(', ')}) ON UPDATE #{key[:on_update]} ON DELETE #{key[:on_delete]};\n"
+    end
+  end
+  
+  
+  def write_contents(table, reader)
+    @f << <<-EOF
+--
+-- Data for Name: #{table.name}; Type: TABLE DATA; Schema: public
+--
+
+COPY "#{table.name}" (#{table.columns.map {|column| PGconn.quote_ident(column[:name])}.join(", ")}) FROM stdin;
+EOF
+    
+    reader.paginated_read(table, 1000) do |row, counter|
+      process_row(table, row)
+      @f << row.join("\t") + "\n"
+    end
+    @f << "\\.\n\n"
+    #@f << "VACUUM FULL ANALYZE #{PGconn.quote_ident(table.name)};\n\n"
+  end
+  
+  def close
+    @f.close
+  end
+end
+
+end