mysql2postgres 0.3.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bd05dc2b701ccf002f167cc1ab6d605dd20e857a5fa0306c4c4423b91ce031d1
+  data.tar.gz: 72ccaec021e0cdf53f31d5f9ae47622fcc36bd518beb7ec93188d9d8df9875a2
+SHA512:
+  metadata.gz: c9c6b0bb6a3811020cc8be01000bc8375bd48b91ae65dd622f5a336c8953a7dd37ab2fe338ec3417462ea186905ed2afc65cf9fb7e416f704d1fa29d3efe4e6e
+  data.tar.gz: bae0e0acc099c166b3e0f0af3317c2c40cec6b2fff75816708eb0b8a099bb0fab92680c8dabaeb985701c7670b39ecb2d9a86eb6e9c9d50f7292c0522befb008

data/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+*.gem
+*.sql
+.bundle
+.rvmrc
+config/database.yml
+Gemfile.lock
+pkg
+test/fixtures/test*.sql

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012-2013 Regents of the University of Minnesota
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# mysql-to-postgres - MySQL to PostgreSQL Data Translation
+
+[![Run Linters](https://github.com/AlphaNodes/mysql2psql/workflows/Run%20Rubocop/badge.svg)](https://github.com/AlphaNodes/mysql2psql/actions/workflows/rubocop.yml) [![Run Tests](https://github.com/AlphaNodes/mysql2psql/workflows/Tests/badge.svg)](https://github.com/AlphaNodes/mysql2psql/actions/workflows/tests.yml)
+
+The minimum Ruby version supported in `main` branch is `2.7`,
+and the next release will have the same requirement.
+
+With a bit of a modified rails `database.yml` configuration, you can integrate `mysql-to-postgres`into a project.
+
+## Installation
+
+```sh
+git clone https://github.com/AlphaNodes/mysql2postgres.git
+cd mysql2postgres
+bundle install
+gem build mysql2postgres.gemspec
+sudo gem install mysql2postgres-0.3.2.gem
+```
+
+## Configuration
+
+Configuration is written in [YAML format](http://www.yaml.org/ "YAML Ain't Markup Language")
+and passed as the first argument on the command line.
+
+Configuration file has be provided with config/database.yml, see config/default.database.yml for
+an example.
+
+## Usage
+
+After providing settings, start migration with
+
+```sh
+
+bundle exec mysql2psql
+```
+
+## Tests
+
+```sh
+rake test
+```
+
+## License
+
+Licensed under [the MIT license](MIT-LICENSE).

data/Rakefile

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'rake'
+
+require_relative 'lib/mysql2psql/version'
+
+require 'rake/testtask'
+namespace :test do
+  Rake::TestTask.new :units do |test|
+    test.libs << 'lib' << 'test/lib'
+    test.pattern = 'test/units/*test.rb'
+    test.verbose = true
+  end
+
+  Rake::TestTask.new :integration do |test|
+    test.libs << 'lib' << 'test/lib'
+    test.pattern = 'test/integration/*test.rb'
+    test.verbose = true
+  end
+end
+
+desc 'Run all tests'
+task :test do
+  # Rake::Task['test:units'].invoke
+  Rake::Task['test:integration'].invoke
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort 'RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov'
+  end
+end
+
+task default: :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "mysql2psql #{Mysql2psql::VERSION}"
+  rdoc.rdoc_files.include 'README*'
+  rdoc.rdoc_files.include 'lib/**/*.rb'
+end

data/bin/mysql2psql

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.join(File.dirname(__dir__), 'lib')
+
+require 'rubygems'
+require 'bundler/setup'
+require 'mysql2psql'
+
+CONFIG_FILE = File.join File.dirname(__dir__), 'config', 'database.yml'
+
+file = if ARGV.length.positive?
+         ARGV[0]
+       else
+         CONFIG_FILE
+       end
+
+unless FileTest.exist?(CONFIG_FILE) || (ARGV.length.positive? && FileTest.exist?(File.expand_path(ARGV[0])))
+  raise "'#{file}' does not exist"
+end
+
+db_yaml = YAML.safe_load File.read(file)
+raise "'#{file}' does not contain a mysql configuration directive for conversion" unless db_yaml.key? 'mysql'
+raise "'#{file}' does not contain a destination configuration directive for conversion" unless db_yaml.key? 'destination'
+
+Mysql2psql.new(db_yaml).convert

data/lib/mysql2psql/connection.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+class Mysql2psql
+  class Connection
+    attr_reader :conn,
+                :adapter,
+                :hostname,
+                :login,
+                :password,
+                :database,
+                :schema,
+                :port,
+                :environment,
+                :copy_manager,
+                :stream,
+                :is_copying
+
+    def initialize(options)
+      @environment = (ENV['RAILS_ENV'] || 'development').to_sym
+
+      if options[:destination].nil? ||
+         options[:destination].empty? ||
+         options[:destination][environment].nil? ||
+         options[:destination][environment].empty?
+        raise 'Unable to locate PostgreSQL destination environment in the configuration file'
+      end
+
+      pg_options = options[:destination][environment]
+      @hostname = pg_options[:hostname] || 'localhost'
+      @login = pg_options[:username]
+      @password = pg_options[:password]
+      @database = pg_options[:database]
+      @port = (pg_options[:port] || 5432).to_s
+
+      @database, @schema = database.split ':'
+      @adapter = pg_options[:adapter] || 'jdbcpostgresql'
+
+      @conn = PG::Connection.open dbname: database,
+                                  user: login,
+                                  password: password,
+                                  host: hostname,
+                                  port: port
+
+      raise_nil_connection if conn.nil?
+
+      @is_copying = false
+      @current_statement = ''
+    end
+
+    # ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream.
+    def flush
+      conn.put_copy_end
+    rescue StandardError => e
+      warn e
+    ensure
+      @is_copying = false
+    end
+
+    def execute(sql)
+      if sql.match(/^COPY /) && !is_copying
+        # sql.chomp!   # cHomp! cHomp!
+        conn.exec sql
+        @is_copying = true
+      elsif sql.match(/^(ALTER|CREATE|DROP|SELECT|SET|TRUNCATE) /) && !is_copying
+        @current_statement = sql
+      elsif is_copying
+        if sql.chomp == '\.' || sql.chomp.match(/^$/)
+          flush
+        else
+          begin
+            until conn.put_copy_data sql
+              warn '  waiting for connection to be writable...'
+              sleep 0.1
+            end
+          rescue StandardError => e
+            @is_copying = false
+            warn e
+            raise e
+          end
+        end
+      elsif @current_statement.length.positive?
+        @current_statement << ' '
+        @current_statement << sql
+      end
+
+      return unless @current_statement.match?(/;$/)
+
+      run_statement @current_statement
+      @current_statement = ''
+    end
+
+    # we're done talking to the database, so close the connection cleanly.
+    def finish
+      @conn.finish
+    end
+
+    # given a file containing psql syntax at path, pipe it down to the database.
+    def load_file(path)
+      if @conn
+        File.open path, 'r:UTF-8' do |file|
+          file.each_line do |line|
+            execute line
+          end
+          flush
+        end
+        finish
+      else
+        raise_nil_connection
+      end
+    end
+
+    def clear_schema
+      statements = ['DROP SCHEMA PUBLIC CASCADE', 'CREATE SCHEMA PUBLIC']
+      statements.each do |statement|
+        run_statement statement
+      end
+    end
+
+    def raise_nil_connection
+      raise 'No Connection'
+    end
+
+    private
+
+    def run_statement(statement)
+      @conn.exec statement
+    end
+  end
+end

data/lib/mysql2psql/converter.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class Mysql2psql
+  class Converter
+    attr_reader :reader,
+                :writer,
+                :options,
+                :exclude_tables,
+                :only_tables,
+                :suppress_data,
+                :suppress_ddl,
+                :force_truncate,
+                :preserve_order,
+                :clear_schema
+
+    def initialize(reader, writer, options)
+      @reader = reader
+      @writer = writer
+      @options = options
+      @exclude_tables = options[:exclude_tables] || []
+      @only_tables = options[:tables]
+      @suppress_data = options[:suppress_data] || false
+      @suppress_ddl = options[:suppress_ddl] || false
+      @force_truncate = options[:force_truncate] || false
+      @preserve_order = options[:preserve_order] || false
+      @clear_schema = options[:clear_schema] || false
+    end
+
+    def convert
+      tables = reader.tables
+      tables.reject! { |table| exclude_tables.include?(table.name) }
+      tables.select! { |table| only_tables ? only_tables.include?(table.name) : true }
+
+      # preserve order only works, if only_tables are specified
+      if preserve_order && only_tables
+        reordered_tables = []
+
+        only_tables.each do |only_table|
+          idx = tables.index { |table| table.name == only_table }
+          if idx.nil?
+            warn "Specified source table '#{only_table}' does not exist, skiped by migration"
+          else
+            reordered_tables << tables[idx]
+          end
+        end
+
+        tables = reordered_tables
+      end
+
+      unless suppress_ddl
+        tables.each do |table|
+          puts "Writing DDL for #{table.name}"
+          writer.write_table table
+        end
+      end
+
+      unless suppress_data
+        if force_truncate && suppress_ddl
+          tables.each do |table|
+            puts "Truncate table #{table.name}"
+            writer.truncate table
+          end
+        end
+
+        tables.each do |table|
+          puts "Writing data for #{table.name}"
+          writer.write_contents table, reader
+        end
+      end
+
+      puts 'Writing indices and constraints'
+      unless suppress_ddl
+        tables.each do |table|
+          writer.write_indexes table
+        end
+      end
+
+      unless suppress_ddl
+        tables.each do |table|
+          writer.write_constraints table
+        end
+      end
+
+      writer.close
+      writer.clear_schema if clear_schema
+      writer.inload
+      0
+    rescue StandardError => e
+      warn "Mysql2psql: Conversion failed: #{e}"
+      warn e
+      warn e.backtrace[0, 3].join("\n")
+      -1
+    end
+  end
+end

data/lib/mysql2psql/mysql_reader.rb

@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+require 'bundler/setup'
+
+require 'mysql'
+require 'csv'
+
+class Mysql2psql
+  class MysqlReader
+    class Table
+      attr_reader :name
+
+      def initialize(reader, name)
+        @reader = reader
+        @name = name
+      end
+
+      def columns
+        @columns ||= load_columns
+      end
+
+      def convert_type(type)
+        case type
+        when /int.* unsigned/, /bigint/
+          'bigint'
+        when 'bit(1)', 'tinyint(1)'
+          'boolean'
+        when /tinyint/
+          'tinyint'
+        when /int/
+          'integer'
+        when /varchar/, /set/
+          'varchar'
+        when /char/
+          'char'
+        when /decimal/
+          'decimal'
+        when /(float|double)/
+          'double precision'
+        else
+          type
+        end
+      end
+
+      def load_columns
+        @reader.reconnect
+        # mysql_flags = ::Mysql::Field.constants.select { |c| c.to_s.include?('FLAG') }
+
+        fields = []
+        @reader.query "EXPLAIN `#{name}`" do |res|
+          while (field = res.fetch_row)
+            length = field[1][/\((\d+)\)/, 1] if field[1].match?(/\((\d+)\)/)
+            length = field[1][/\((\d+),(\d+)\)/, 1] if field[1].match?(/\((\d+),(\d+)\)/)
+            desc = {
+              name: field[0],
+              table_name: name,
+              type: convert_type(field[1]),
+              length: length&.to_i,
+              decimals: field[1][/\((\d+),(\d+)\)/, 2],
+              null: field[2] == 'YES',
+              primary_key: field[3] == 'PRI',
+              auto_increment: field[5] == 'auto_increment'
+            }
+            desc[:default] = field[4] unless field[4].nil?
+            fields << desc
+          end
+        end
+
+        fields.select { |field| field[:auto_increment] }.each do |field|
+          @reader.query "SELECT max(`#{field[:name]}`) FROM `#{name}`" do |res|
+            field[:maxval] = res.fetch_row[0].to_i
+          end
+        end
+        fields
+      end
+
+      def indexes
+        load_indexes unless @indexes
+        @indexes
+      end
+
+      def foreign_keys
+        load_indexes unless @foreign_keys
+        @foreign_keys
+      end
+
+      def load_indexes
+        @indexes = []
+        @foreign_keys = []
+
+        @reader.query "SHOW CREATE TABLE `#{name}`" do |result|
+          explain = result.fetch_row[1]
+          explain.split("\n").each do |line|
+            next unless line.include? ' KEY '
+
+            index = {}
+            if (match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line))
+              index[:name] = "fk_#{name}_#{match_data[1]}"
+              index[:column] = match_data[2].delete!('`').split(', ')
+              index[:ref_table] = match_data[3]
+              index[:ref_column] = match_data[4].delete!('`').split(', ')
+
+              the_rest = match_data[5]
+
+              if (match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest))
+                index[:on_delete] = match_data[1]
+              else
+                index[:on_delete] ||= 'RESTRICT'
+              end
+
+              if (match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest))
+                index[:on_update] = match_data[1]
+              else
+                index[:on_update] ||= 'RESTRICT'
+              end
+
+              @foreign_keys << index
+            elsif (match_data = /KEY `(\w+)` \((.*)\)/.match(line))
+              # index[:name] = 'idx_' + name + '_' + match_data[1]
+              # with redmine we do not want prefix idx_tablename_
+              index[:name] = match_data[1]
+              index[:columns] = match_data[2].split(',').map { |col| col[/`(\w+)`/, 1] }
+              index[:unique] = true if line.include? 'UNIQUE'
+              @indexes << index
+            elsif (match_data = /PRIMARY KEY .*\((.*)\)/.match(line))
+              index[:primary] = true
+              index[:columns] = match_data[1].split(',').map { |col| col.strip.delete('`') }
+              @indexes << index
+            end
+          end
+        end
+      end
+
+      def count_rows
+        @reader.query "SELECT COUNT(*) FROM `#{name}`" do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+
+      def id?
+        !!columns.find { |col| col[:name] == 'id' }
+      end
+
+      def count_for_pager
+        query = id? ? 'MAX(id)' : 'COUNT(*)'
+        @reader.query "SELECT #{query} FROM `#{name}`" do |res|
+          return res.fetch_row[0].to_i
+        end
+      end
+
+      def query_for_pager
+        query = id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?'
+
+        cols = columns.map do |c|
+          if c[:type] == 'multipolygon'
+            "AsWKT(`#{c[:name]}`) as `#{c[:name]}`"
+          else
+            "`#{c[:name]}`"
+          end
+        end
+
+        "SELECT #{cols.join ', '} FROM `#{name}` #{query}"
+      end
+    end
+
+    def connect
+      @mysql = ::Mysql.connect @host, @user, @passwd, @db, @port, @sock
+      # utf8_unicode_ci :: https://rubydoc.info/gems/ruby-mysql/Mysql/Charset
+      @mysql.charset = ::Mysql::Charset.by_number 192
+      @mysql.query 'SET NAMES utf8'
+
+      var_info = @mysql.query "SHOW VARIABLES LIKE 'query_cache_type'"
+      return if var_info.nil? || var_info.first.nil? || var_info.first[1] == 'OFF'
+
+      @mysql.query 'SET SESSION query_cache_type = OFF'
+    end
+
+    def reconnect
+      @mysql.close
+    rescue StandardError
+      warn 'could not close previous mysql connection'
+    ensure
+      connect
+    end
+
+    def query(*args, &block)
+      mysql.query(*args, &block)
+    rescue Mysql::Error => e
+      if e.message.match?(/gone away/i)
+        reconnect
+        retry
+      else
+        puts "MySQL Query failed '#{args.inspect}' #{e.inspect}"
+        puts e.backtrace[0, 5].join("\n")
+        []
+      end
+    end
+
+    def initialize(options)
+      @host = options[:mysql][:hostname]
+      @user = options[:mysql][:username]
+      @passwd = options[:mysql][:password]
+      @db = options[:mysql][:database]
+      @port = options[:mysql][:port] || 3306
+      @sock = options[:mysql][:socket] && !options[:mysql][:socket].empty? ? options[:mysql][:socket] : nil
+      @sock = options[:mysql][:flag] && !options[:mysql][:flag].empty? ? options[:mysql][:flag] : nil
+
+      connect
+    end
+
+    attr_reader :mysql
+
+    def tables
+      @tables ||= @mysql.query('SHOW TABLES').map { |row| Table.new(self, row.first) }
+    end
+
+    def paginated_read(table, page_size)
+      count = table.count_for_pager
+      return if count < 1
+
+      statement = @mysql.prepare table.query_for_pager
+      counter = 0
+      0.upto (count + page_size) / page_size do |i|
+        statement.execute(i * page_size, table.id? ? (i + 1) * page_size : page_size)
+        while (row = statement.fetch)
+          counter += 1
+          yield row, counter
+        end
+      end
+      counter
+    end
+  end
+end

data/lib/mysql2psql/postgres_db_writer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'mysql2psql/postgres_writer'
+require 'mysql2psql/connection'
+
+class Mysql2psql
+  class PostgresDbWriter < PostgresFileWriter
+    attr_reader :connection, :filename
+
+    def initialize(filename, options)
+      # NOTE: the superclass opens and truncates filename for writing
+      super filename
+
+      @filename = filename
+      @connection = Connection.new options
+    end
+
+    def inload(path = filename)
+      connection.load_file path
+    end
+
+    def clear_schema
+      connection.clear_schema
+    end
+  end
+end