mysql2postgres 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44c431dce8b1d09c5ebef95e6f1a56dbc2d7b79e67b594821d9a33c287d5d2b5
-  data.tar.gz: cf5352e91abcd255f2286a95a5a0c3c1dadae5cd3804f0914ea8dee999e27f4d
+  metadata.gz: cc905b9634e92298f5b5d192b6f4388294f6153a777b551b408bc5e944247b3f
+  data.tar.gz: 108d90760f0600e55f117df749147f8137d3044a7724dd901f76869db7b45caf
 SHA512:
-  metadata.gz: 34cce30cf4cffd22bc21e0be57267da80f5806ebeebfcd2934acb6f306af79ba5da512065a8acd5367964a5af00265cea9b3c5cfbacc1f6e7425412300613420
-  data.tar.gz: 72c40e74f83546cac80b33e8b22ecb7f8035ec15ace7a572d95373c83ce4055fde7809f52f7a0608fb7a426197d4c6a71cb39f815f3719f1a52adaf1ade266a2
+  metadata.gz: 6b08edc13623c942b61d03cecce1f577eca9fce3906e3c06ef17c848396dfebadb39017cb7ae7aecb5193e91cffc85d2a4cea2e49054f0e7dd3dffc8f53012eb
+  data.tar.gz: 5ef72f3a6ac5961102d282b38cf2eb5bc62d301f5e84b590dc4e9dd7d057fa51c43f5272b112478ab6b6d3513fc3ac67e8a24bae0d9f23d3f162c0ddd8f591e0

data/.gitignore

@@ -7,3 +7,5 @@ config/database.yml
 Gemfile.lock
 pkg
 test/fixtures/test*.sql
+.enable_dev
+.enable_test

data/README.md

@@ -2,8 +2,7 @@
 
 [![Run Linters](https://github.com/AlphaNodes/mysql2postgres/workflows/Run%20Rubocop/badge.svg)](https://github.com/AlphaNodes/mysql2postgres/actions/workflows/rubocop.yml) [![Run Tests](https://github.com/AlphaNodes/mysql2postgres/workflows/Tests/badge.svg)](https://github.com/AlphaNodes/mysql2postgres/actions/workflows/tests.yml)
 
-The minimum Ruby version supported in `main` branch is `2.7`,
-and the next release will have the same requirement.
+Convert MySQL database to PostgreSQL database.
 
 ## Requirements
 
@@ -22,16 +21,22 @@ gem 'mysql2postgres'
 Configuration is written in [YAML format](http://www.yaml.org/ "YAML Ain't Markup Language")
 and passed as the first argument on the command line.
 
-Configuration file has be provided with config/database.yml, see config/default.database.yml for
-an example.
+Configuration file has be provided with config/database.yml, see [config/default.database.yml](config/default.database.yml) for an example and for configuration information.
 
 ## Usage
 
 After providing settings, start migration with
 
 ```sh
+# set destination to use
+MYSQL2POSTGRES_ENV=test
+# use can also use (MYSQL2POSTGRES_ENV is used, if both are defined)
+RAILS_ENV=test
 
+# with default configuration, which use config/database.yml
 bundle exec mysql2postgres
+# OR with specified configuration file
+bundle exec mysql2postgres /home/you/mysql2postgres.yml
 ```
 
 ## Tests

data/Rakefile

@@ -22,7 +22,7 @@ end
 
 desc 'Run all tests'
 task :test do
-  # Rake::Task['test:units'].invoke
+  Rake::Task['test:units'].invoke
   Rake::Task['test:integration'].invoke
 end
 

data/bin/mysql2postgres

@@ -7,21 +7,13 @@ require 'rubygems'
 require 'bundler/setup'
 require 'mysql2postgres'
 
-CONFIG_FILE = File.join File.dirname(__dir__), 'config', 'database.yml'
+config_file = ARGV.empty? ? File.join(File.dirname(__dir__), 'config', 'database.yml') : File.expand_path(ARGV[0])
 
-file = if ARGV.length.positive?
-         ARGV[0]
-       else
-         CONFIG_FILE
-       end
+raise "'#{config_file}' does not exist" unless FileTest.exist? config_file
 
-unless FileTest.exist?(CONFIG_FILE) || (ARGV.length.positive? && FileTest.exist?(File.expand_path(ARGV[0])))
-  raise "'#{file}' does not exist"
-end
+db_yaml = YAML.safe_load File.read(config_file)
 
-db_yaml = YAML.safe_load File.read(file)
+raise "'#{config_file}' does not contain a mysql configuration directive for conversion" unless db_yaml.key? 'mysql'
+raise "'#{config_file}' does not contain destinations configuration directive for conversion" unless db_yaml.key? 'destinations'
 
-raise "'#{file}' does not contain a mysql configuration directive for conversion" unless db_yaml.key? 'mysql'
-raise "'#{file}' does not contain a destination configuration directive for conversion" unless db_yaml.key? 'destination'
-
-Mysql2postgres.new(db_yaml).convert
+Mysql2postgres.new(db_yaml, config_file).convert

data/lib/mysql2postgres/connection.rb

@@ -3,29 +3,17 @@
 class Mysql2postgres
   class Connection
     attr_reader :conn,
-                :adapter,
                 :hostname,
                 :login,
                 :password,
                 :database,
                 :schema,
                 :port,
-                :environment,
                 :copy_manager,
                 :stream,
                 :is_copying
 
-    def initialize(options)
-      @environment = (ENV['RAILS_ENV'] || 'development').to_sym
-
-      if options[:destination].nil? ||
-         options[:destination].empty? ||
-         options[:destination][environment].nil? ||
-         options[:destination][environment].empty?
-        raise 'Unable to locate PostgreSQL destination environment in the configuration file'
-      end
-
-      pg_options = options[:destination][environment]
+    def initialize(pg_options)
       @hostname = pg_options[:hostname] || 'localhost'
       @login = pg_options[:username]
       @password = pg_options[:password]
@@ -33,18 +21,20 @@ class Mysql2postgres
       @port = (pg_options[:port] || 5432).to_s
 
       @database, @schema = database.split ':'
-      @adapter = pg_options[:adapter] || 'jdbcpostgresql'
 
+      @conn = open
+      raise_nil_connection if conn.nil?
+
+      @is_copying = false
+      @current_statement = ''
+    end
+
+    def open
       @conn = PG::Connection.open dbname: database,
                                   user: login,
                                   password: password,
                                   host: hostname,
                                   port: port
-
-      raise_nil_connection if conn.nil?
-
-      @is_copying = false
-      @current_statement = ''
     end
 
     # ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream.
@@ -120,6 +110,16 @@ class Mysql2postgres
       raise 'No Connection'
     end
 
+    def tables
+      result = run_statement <<~SQL_TABLES
+        SELECT table_name
+        FROM information_schema.tables
+        WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
+      SQL_TABLES
+
+      result.map { |t| t['table_name'] }
+    end
+
     private
 
     def run_statement(statement)

data/lib/mysql2postgres/converter.rb

@@ -16,7 +16,6 @@ class Mysql2postgres
     def initialize(reader, writer, options)
       @reader = reader
       @writer = writer
-      @options = options
       @exclude_tables = options[:exclude_tables] || []
       @only_tables = options[:tables]
       @suppress_data = options[:suppress_data] || false

data/lib/mysql2postgres/mysql_reader.rb

@@ -164,6 +164,24 @@ class Mysql2postgres
       end
     end
 
+    attr_reader :mysql
+
+    def initialize(options)
+      @host = options[:mysql][:hostname]
+      @user = options[:mysql][:username]
+      @passwd = options[:mysql][:password]
+      @db = options[:mysql][:database]
+      @port = if options[:mysql][:port]
+                options[:mysql][:port] unless options[:mysql][:port].to_s.empty?
+              else
+                3306
+              end
+      @sock = options[:mysql][:socket] && !options[:mysql][:socket].empty? ? options[:mysql][:socket] : nil
+      @flag = options[:mysql][:flag] && !options[:mysql][:flag].empty? ? options[:mysql][:flag] : nil
+
+      connect
+    end
+
     def connect
       @mysql = ::Mysql.connect @host, @user, @passwd, @db, @port, @sock
       # utf8_unicode_ci :: https://rubydoc.info/gems/ruby-mysql/Mysql/Charset
@@ -197,20 +215,6 @@ class Mysql2postgres
       end
     end
 
-    def initialize(options)
-      @host = options[:mysql][:hostname]
-      @user = options[:mysql][:username]
-      @passwd = options[:mysql][:password]
-      @db = options[:mysql][:database]
-      @port = options[:mysql][:port] || 3306
-      @sock = options[:mysql][:socket] && !options[:mysql][:socket].empty? ? options[:mysql][:socket] : nil
-      @sock = options[:mysql][:flag] && !options[:mysql][:flag].empty? ? options[:mysql][:flag] : nil
-
-      connect
-    end
-
-    attr_reader :mysql
-
     def tables
       @tables ||= @mysql.query('SHOW TABLES').map { |row| Table.new(self, row.first) }
     end

data/lib/mysql2postgres/postgres_db_writer.rb

@@ -5,14 +5,13 @@ require 'mysql2postgres/connection'
 
 class Mysql2postgres
   class PostgresDbWriter < PostgresFileWriter
-    attr_reader :connection, :filename
+    attr_reader :connection
 
-    def initialize(filename, options)
+    def initialize(file, destination)
       # NOTE: the superclass opens and truncates filename for writing
-      super filename
+      super
 
-      @filename = filename
-      @connection = Connection.new options
+      @connection = Connection.new destination
     end
 
     def inload(path = filename)

data/lib/mysql2postgres/postgres_file_writer.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 require 'mysql2postgres/postgres_writer'
+require 'fileutils'
 
 class Mysql2postgres
   class PostgresFileWriter < PostgresWriter
-    def initialize(file)
+    def initialize(file, destination)
       super()
 
+      @filename = file
+      @destination = destination
+
       @f = File.open file, 'w+:UTF-8'
       @f << <<~SQL_HEADER
         -- MySQL 2 PostgreSQL dump\n
@@ -88,7 +92,7 @@ class Mysql2postgres
       @f << columns
 
       if (primary_index = table.indexes.find { |index| index[:primary] })
-        @f << ",\n  CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{primary_index[:columns].map { |col| PG::Connection.quote_ident(col) }.join(', ')})"
+        @f << ",\n  CONSTRAINT #{table.name}_pkey PRIMARY KEY(#{quoted_list primary_index[:columns]})"
       end
 
       @f << <<~SQL_OIDS
@@ -103,7 +107,7 @@ class Mysql2postgres
         @f << <<~SQL_INDEX
           DROP INDEX IF EXISTS #{PG::Connection.quote_ident index[:name]} CASCADE;
           CREATE #{unique}INDEX #{PG::Connection.quote_ident index[:name]}
-          ON #{PG::Connection.quote_ident table.name} (#{index[:columns].map { |col| PG::Connection.quote_ident(col) }.join(', ')});
+          ON #{PG::Connection.quote_ident table.name} (#{quoted_list index[:columns]});
         SQL_INDEX
       end
     end
@@ -113,8 +117,8 @@ class Mysql2postgres
     def write_constraints(table)
       table.foreign_keys.each do |key|
         @f << "ALTER TABLE #{PG::Connection.quote_ident table.name} " \
-              "ADD FOREIGN KEY (#{key[:column].map { |c| PG::Connection.quote_ident(c) }.join(', ')}) " \
-              "REFERENCES #{PG::Connection.quote_ident key[:ref_table]}(#{key[:ref_column].map { |c| PG::Connection.quote_ident(c) }.join(', ')}) " \
+              "ADD FOREIGN KEY (#{quoted_list key[:column]}) " \
+              "REFERENCES #{PG::Connection.quote_ident key[:ref_table]}(#{quoted_list key[:ref_column]}) " \
               "ON UPDATE #{key[:on_update]} ON DELETE #{key[:on_delete]};\n"
       end
     end
@@ -125,7 +129,7 @@ class Mysql2postgres
         -- Data for Name: #{table.name}; Type: TABLE DATA; Schema: public
         --
 
-        COPY "#{table.name}" (#{table.columns.map { |column| PG::Connection.quote_ident(column[:name]) }.join(', ')}) FROM stdin;
+        COPY "#{table.name}" (#{quoted_list(table.columns.map { |m| m[:name] })}) FROM stdin;
       SQL_COPY
 
       reader.paginated_read table, 1000 do |row, _counter|
@@ -139,5 +143,15 @@ class Mysql2postgres
     def close
       @f.close
     end
+
+    def inload
+      puts "\nSkip import to PostgreSQL DB. SQL file created successfully."
+    end
+
+    private
+
+    def quoted_list(list)
+      list.map { |c| PG::Connection.quote_ident(c) }.join(', ')
+    end
   end
 end

data/lib/mysql2postgres/postgres_writer.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 require 'zlib'
-require 'mysql2postgres/writer'
 
 class Mysql2postgres
-  class PostgresWriter < Writer
+  class PostgresWriter
+    attr_reader :filename, :destination
+
     def column_description(column)
       "#{PG::Connection.quote_ident column[:name]} #{column_type_info column}"
     end
@@ -57,9 +58,14 @@ class Mysql2postgres
                default = nil
                'date'
              when 'timestamp'
-               default = ' DEFAULT CURRENT_TIMESTAMP' if column[:default] == 'CURRENT_TIMESTAMP'
-               default = " DEFAULT '1970-01-01 00:00'" if column[:default] == '0000-00-00 00:00'
-               default = " DEFAULT '1970-01-01 00:00:00'" if column[:default] == '0000-00-00 00:00:00'
+               case column[:default]
+               when 'CURRENT_TIMESTAMP'
+                 default = ' DEFAULT CURRENT_TIMESTAMP'
+               when datetime_zero
+                 default = " DEFAULT '#{datetime_zero_fix}'"
+               when datetime_zero(with_seconds: true) # rubocop: disable Style/MethodCallWithArgsParentheses
+                 default = " DEFAULT '#{datetime_zero_fix with_seconds: true}'"
+               end
                'timestamp without time zone'
              when 'time'
                default = ' DEFAULT NOW()' if default
@@ -89,45 +95,74 @@ class Mysql2postgres
         row[index] = Time.at(row[index]).utc.strftime('%H:%M:%S') if column[:type] == 'time' && row[index]
 
         if row[index].is_a? Time
-          row[index] = row[index].to_s.gsub '0000-00-00 00:00', '1970-01-01 00:00'
-          row[index] = row[index].to_s.gsub '0000-00-00 00:00:00', '1970-01-01 00:00:00'
+          row[index] = row[index].to_s.gsub datetime_zero, datetime_zero_fix
+          row[index] = row[index].to_s.gsub datetime_zero(with_seconds: true), datetime_zero_fix(with_seconds: true)
         end
 
         if column_type(column) == 'boolean'
           row[index] = if row[index] == 1
                          't'
+                       elsif row[index].zero?
+                         'f'
                        else
-                         row[index].zero? ? 'f' : row[index]
+                         row[index]
                        end
         end
 
-        if row[index].is_a? String
-          row[index] = if column_type(column) == 'bytea'
-                         if column[:name] == 'data'
-                           with_gzip = false
-                           table.columns.each_with_index do |column_data, index_data|
-                             if column_data[:name] == 'compression' && row[index_data] == 'gzip'
-                               with_gzip = true
-                               break
-                             end
-                           end
-                           if with_gzip
-                             PG::Connection.escape_bytea(Zlib::Inflate.inflate(row[index])).gsub(/\\/, '\\\\\\').gsub(/''/, "'")
-                           else
-                             PG::Connection.escape_bytea(row[index]).gsub(/\\/, '\\\\\\').gsub(/''/, "'")
-                           end
-                         else
-                           PG::Connection.escape_bytea(row[index]).gsub(/\\/, '\\\\\\').gsub(/''/, "'")
-                         end
-                       else
-                         row[index].gsub(/\\/, '\\\\\\').gsub(/\n/, '\n').gsub(/\t/, '\t').gsub(/\r/, '\r').gsub(/\0/, '')
-                       end
-        end
+        row[index] = string_data row, index, column if row[index].is_a? String
 
         row[index] = '\N' unless row[index]
       end
     end
 
     def truncate(_table) end
+
+    def inload
+      raise "Method 'inload' needs to be overridden..."
+    end
+
+    private
+
+    def datetime_zero(with_seconds: false)
+      datetime_value date: '0000-00-00', with_seconds: with_seconds
+    end
+
+    def datetime_zero_fix(with_seconds: false)
+      datetime_value date: '1970-01-01', with_seconds: with_seconds
+    end
+
+    def datetime_value(date:, with_seconds: false)
+      value = ["#{date} 00:00"]
+      value << '00' if with_seconds
+      value.join ':'
+    end
+
+    def string_data(row, index, column)
+      if column_type(column) == 'bytea'
+        if column[:name] == 'data'
+          with_gzip = false
+          table.columns.each_with_index do |column_data, index_data|
+            if column_data[:name] == 'compression' && row[index_data] == 'gzip'
+              with_gzip = true
+              break
+            end
+          end
+
+          escape_bytea(with_gzip ? Zlib::Inflate.inflate(row[index]) : row[index])
+        else
+          escape_bytea row[index]
+        end
+      else
+        escape_data(row[index]).gsub(/\n/, '\n').gsub(/\t/, '\t').gsub(/\r/, '\r').gsub(/\0/, '')
+      end
+    end
+
+    def escape_bytea(data)
+      escape_data(PG::Connection.escape_bytea(data)).gsub(/''/, "'")
+    end
+
+    def escape_data(value)
+      value.gsub(/\\/, '\\\\\\')
+    end
   end
 end

data/lib/mysql2postgres/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Mysql2postgres
-  VERSION = '0.3.3'
+  VERSION = '0.4.0'
 end

data/lib/mysql2postgres.rb

@@ -12,7 +12,6 @@ require 'pg/result'
 require 'mysql2postgres/version'
 require 'mysql2postgres/converter'
 require 'mysql2postgres/mysql_reader'
-require 'mysql2postgres/writer'
 require 'mysql2postgres/postgres_writer'
 require 'mysql2postgres/postgres_file_writer'
 require 'mysql2postgres/postgres_db_writer'
@@ -20,42 +19,76 @@ require 'mysql2postgres/postgres_db_writer'
 require 'debug' if ENV.fetch('ENABLE_DEBUG', nil) == '1'
 
 class Mysql2postgres
-  attr_reader :options, :reader, :writer
+  attr_reader :options, :config_file, :reader, :writer
 
-  def initialize(yaml)
+  def initialize(yaml, config_file = nil)
+    @config_file = config_file
     @options = build_options yaml
   end
 
-  def build_options(yaml)
-    yaml.transform_keys(&:to_sym).tap do |opts|
-      opts[:mysql].transform_keys!(&:to_sym)
-      opts[:destination].transform_keys!(&:to_sym)
-      opts[:destination].each do |env, settings|
-        opts[:destination][env] = settings.transform_keys(&:to_sym)
-      end
-    end
-  end
+  def convert
+    @reader = MysqlReader.new options
+
+    puts "mysql2postgres #{Mysql2postgres::VERSION}"
+    puts "Config file: #{config_file}"
+    puts "Dumpfile: #{dump_file}"
+
+    @writer = if to_file?
+                puts 'Target: File'
+                PostgresFileWriter.new dump_file, options[:destination]
+              else
+                puts "Target: PostgreSQL DB (#{adapter})"
+                PostgresDbWriter.new dump_file, options[:destination]
+              end
 
-  def send_file_to_postgres(path)
-    connection = Connection.new options
-    connection.load_file path
+    Converter.new(reader, writer, options).convert
+    File.delete dump_file if options[:remove_dump_file] && File.exist?(dump_file)
   end
 
-  def convert
-    @reader = MysqlReader.new options
+  private
 
-    tag = Time.new.strftime '%Y%m%d-%H%M%S'
+  def adapter
+    if options[:destination][:adapter].nil? || options[:destination][:adapter].empty?
+      'postgresql'
+    else
+      options[:destination][:adapter]
+    end
+  end
 
-    path = './'
-    path = options[:dump_file_directory] if options[:dump_file_directory]
+  def environment
+    if ENV['MYSQL2POSTGRES_ENV']
+      ENV['MYSQL2POSTGRES_ENV']
+    elsif ENV['RAILS_ENV']
+      ENV['RAILS_ENV']
+    else
+      'development'
+    end
+  end
 
-    filename = File.expand_path File.join(path, "output_#{tag}.sql")
-    puts "Dumpfile: #{filename}"
+  def to_file?
+    adapter == 'file'
+  end
 
-    @writer = PostgresDbWriter.new filename, options
+  def build_options(yaml)
+    yaml.transform_keys(&:to_sym).tap do |opts|
+      opts[:mysql].transform_keys!(&:to_sym)
 
-    Converter.new(reader, writer, options).convert
+      destinations = opts.delete :destinations
+      opts[:destination] = destinations[environment]&.transform_keys(&:to_sym)
+
+      if opts[:destination].nil? || opts[:destination].empty?
+        raise "no configuration for environment '#{environment}' in destinations available. Use MYSQL2POSTGRES_ENV or RAILS_ENV."
+      end
+    end
+  end
 
-    File.delete filename if options[:remove_dump_file] && File.exist?(filename)
+  def dump_file
+    @dump_file ||= if to_file? && options[:destination][:filename] && options[:destination][:filename] != ''
+                     options[:destination][:filename]
+                   else
+                     tag = Time.new.strftime '%Y%m%d-%H%M%S'
+                     path = options[:dump_file_directory] || './'
+                     File.expand_path File.join(path, "output_#{tag}.sql")
+                   end
   end
 end

data/mysql2postgres.gemspec

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 lib = File.expand_path '../lib', __FILE__
-puts lib
 $LOAD_PATH.unshift lib unless $LOAD_PATH.include? lib
 require 'mysql2postgres/version'
 
@@ -50,31 +49,28 @@ Gem::Specification.new do |s|
     'lib/mysql2postgres/postgres_db_writer.rb',
     'lib/mysql2postgres/postgres_writer.rb',
     'lib/mysql2postgres/version.rb',
-    'lib/mysql2postgres/writer.rb',
     'mysql2postgres.gemspec',
     'test/fixtures/config_all_options.yml',
+    'test/fixtures/config_min_options.yml',
+    'test/fixtures/config_to_file.yml',
     'test/fixtures/seed_integration_tests.sql',
     'test/integration/convert_to_db_test.rb',
     'test/integration/convert_to_file_test.rb',
     'test/integration/converter_test.rb',
-    'test/integration/mysql_reader_base_test.rb',
+    'test/integration/mysql_reader_connection_test.rb',
     'test/integration/mysql_reader_test.rb',
-    'test/integration/postgres_db_writer_base_test.rb',
+    'test/integration/postgres_db_writer_test.rb',
+    'test/units/mysql_test.rb',
     'test/units/option_test.rb',
     'test/units/postgres_file_writer_test.rb',
     'test/test_helper.rb'
   ]
-  s.homepage = 'https://code.alphanodes.com/alphanodes/mysql2postgres'
+  s.homepage = 'https://github.com/AlphaNodes/mysql2postgres'
   s.rdoc_options = ['--charset=UTF-8']
   s.require_paths = ['lib']
   s.summary = 'MySQL to PostgreSQL Data Translation'
 
-  s.add_dependency 'rake'
   s.add_runtime_dependency 'pg', '~> 1.2.2'
-  s.add_runtime_dependency 'postgres-pr', '~> 0.7'
+  s.add_runtime_dependency 'rake'
   s.add_runtime_dependency 'ruby-mysql', '~> 3.0'
-  s.add_development_dependency 'debug'
-  s.add_development_dependency 'rubocop-minitest'
-  s.add_development_dependency 'rubocop-performance'
-  s.add_development_dependency 'test-unit', '~> 3.5.3'
 end

data/test/fixtures/config_all_options.yml

@@ -1,20 +1,21 @@
 mysql:
-  hostname: localhost
+  hostname: 127.0.0.1
   port: 3306
-  socket: /tmp/mysql.sock
-  username: somename
-  password: secretpassword
+  username: root
+  password: BestPasswordEver
   database: somename
+  encoding: utf8mb4
 
-
-destination:
-  # if file is given, output goes to file, else postgres
-  file: somefile
+# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used!
+destinations:
+  development:
+    adapter: file
+    filename: /tmp/exported_file.sql
   test:
     hostname: localhost
     port: 5432
-    username: somename
-    password: secretpassword
+    username: postgres
+    password: postgres
     database: somename
 
 # if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
@@ -29,8 +30,10 @@ exclude_tables:
 - table5
 - table6
 
+dump_file_directory: /tmp
+
 # if suppress_data is true, only the schema definition will be exported/migrated, and not the data
-suppress_data: true
+suppress_data: false
 
 # if suppress_ddl is true, only the data will be exported/imported, and not the schema
 suppress_ddl: false

data/test/fixtures/config_min_options.yml

@@ -0,0 +1,16 @@
+mysql:
+  hostname: localhost
+  username: root
+  password: BestPasswordEver
+  database: somename
+
+# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used!
+destinations:
+  test:
+    hostname: localhost
+    port: 5432
+    username: somename
+    password: secretpassword
+    database: somename
+
+# all options are false as default

data/test/fixtures/config_to_file.yml

@@ -0,0 +1,31 @@
+mysql:
+  hostname: 127.0.0.1
+  port: 3306
+  username: root
+  password: BestPasswordEver
+  database: somename
+  encoding: utf8mb4
+
+# if MYSQL2POSTGRES_ENV and RAILS_ENV is missing, development is used!
+destinations:
+  test:
+    adapter: file
+    filename: /tmp/exported_file.sql
+
+# if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
+tables:
+- table1
+- table2
+- table3
+- table4
+
+dump_file_directory: /tmp
+
+# if suppress_data is true, only the schema definition will be exported/migrated, and not the data
+suppress_data: false
+
+# if suppress_ddl is true, only the data will be exported/imported, and not the schema
+suppress_ddl: false
+
+# if force_truncate is true, forces a table truncate before table loading
+force_truncate: false

data/test/integration/convert_to_db_test.rb

@@ -6,18 +6,25 @@ class ConvertToDbTest < Test::Unit::TestCase
   class << self
     def startup
       seed_test_database
-      @options = get_test_config_by_label :localmysql_to_db_convert_all
-      @mysql2postgres = Mysql2postgres.new @options
-      @mysql2postgres.convert
-      @mysql2postgres.writer.open
     end
+  end
 
-    def shutdown
-      @@mysql2postgres.writer.close
-    end
+  def setup
+    @mysql2postgres = instance_from_file
+    @options = @mysql2postgres.options
+    @options[:force_truncate] = true
+    @options.delete :tables # convert all available tables
+
+    @mysql2postgres.convert
+    @mysql2postgres.writer.connection.open
+  end
+
+  def teardown
+    @mysql2postgres&.writer&.connection&.finish
   end
 
   def test_table_creation
-    assert_true @mysql2postgres.writer.exists?('numeric_types_basics')
+    tables = @mysql2postgres.writer.connection.tables
+    assert tables.include?('numeric_types_basics')
   end
 end

data/test/integration/convert_to_file_test.rb

@@ -3,18 +3,22 @@
 require File.expand_path '../test_helper', __dir__
 
 class ConvertToFileTest < Test::Unit::TestCase
+  attr_reader :content
+
   class << self
     def startup
-      seed_test_database
-      @options = get_test_config_by_label :localmysql_to_file_convert_all
-      @mysql2postgres = Mysql2postgres.new @options
-      @mysql2postgres.convert
-      @content = File.read @mysql2postgres.options[:destfile]
+      seed_test_database option_file: 'config_to_file'
     end
   end
 
-  def content
-    @@content
+  def setup
+    @mysql2postgres = instance_from_file 'config_to_file'
+    @options = @mysql2postgres.options
+    @options[:force_truncate] = true
+    @options.delete :tables # convert all available tables
+
+    @mysql2postgres.convert
+    @content = File.read @mysql2postgres.options[:destination][:filename]
   end
 
   def test_table_creation
@@ -23,46 +27,57 @@ class ConvertToFileTest < Test::Unit::TestCase
   end
 
   def test_basic_numerics_tinyint
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_tinyint" smallint,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_smallint
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_smallint" integer,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_mediumint
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_mediumint" integer,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_int
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_int" integer,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_integer
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_integer" integer,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_bigint
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_bigint" bigint,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_real
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_real" double precision,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_double
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_double" double precision,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_float
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision,.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_float" double precision,.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_decimal
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_decimal" numeric\(10, 0\),.*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 
   def test_basic_numerics_numeric
-    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE).match(content)
+    assert_not_nil Regexp.new('CREATE TABLE "numeric_types_basics".*"f_numeric" numeric\(10, 0\)[\w\n]*\)', Regexp::MULTILINE)
+                         .match(content)
   end
 end

data/test/integration/converter_test.rb

@@ -5,15 +5,22 @@ require File.expand_path '../test_helper', __dir__
 class ConverterTest < Test::Unit::TestCase
   class << self
     def startup
-      seed_test_database
-      @options = get_test_config_by_label :localmysql_to_file_convert_nothing
+      seed_test_database option_file: 'config_to_file'
     end
   end
 
+  def setup
+    @options = options_from_file 'config_to_file'
+    @options[:suppress_data] = true
+    @options[:suppress_ddl] = true
+
+    @destfile = get_temp_file 'mysql2postgres_test'
+  end
+
   def test_new_converter
     assert_nothing_raised do
       reader = get_test_reader @options
-      writer = get_test_file_writer @options
+      writer = Mysql2postgres::PostgresFileWriter.new @destfile, @options[:destination]
       converter = Mysql2postgres::Converter.new reader, writer, @options
       assert_equal 0, converter.convert
     end

data/test/integration/{mysql_reader_base_test.rb → mysql_reader_connection_test.rb}

@@ -2,23 +2,26 @@
 
 require File.expand_path '../test_helper', __dir__
 
-class MysqlReaderBaseTest < Test::Unit::TestCase
+class MysqlReaderConnectionTest < Test::Unit::TestCase
   class << self
     def startup
-      seed_test_database
-      @options = get_test_config_by_label :localmysql_to_file_convert_nothing
+      seed_test_database option_file: 'config_to_file'
     end
   end
 
+  def setup
+    @options = options_from_file 'config_to_file'
+  end
+
   def test_mysql_connection
     assert_nothing_raised do
-      mysql2postgres::MysqlReader.new @options
+      Mysql2postgres::MysqlReader.new @options
     end
   end
 
   def test_mysql_reconnect
     assert_nothing_raised do
-      reader = mysql2postgres::MysqlReader.new @options
+      reader = Mysql2postgres::MysqlReader.new @options
       reader.reconnect
     end
   end
@@ -27,7 +30,7 @@ class MysqlReaderBaseTest < Test::Unit::TestCase
     assert_nothing_raised do
       @options[:mysql][:port] = ''
       @options[:mysql][:socket] = ''
-      mysql2postgres::MysqlReader.new @options
+      Mysql2postgres::MysqlReader.new @options
     end
   end
 end

data/test/integration/mysql_reader_test.rb

@@ -5,12 +5,15 @@ require File.expand_path '../test_helper', __dir__
 class MysqlReaderTest < Test::Unit::TestCase
   class << self
     def startup
-      seed_test_database
-      @options = get_test_config_by_label :localmysql_to_file_convert_nothing
-      @reader = get_test_reader @options
+      seed_test_database option_file: 'config_to_file'
     end
   end
 
+  def setup
+    @options = options_from_file 'config_to_file'
+    @reader = get_test_reader @options
+  end
+
   def test_db_connection
     assert_nothing_raised do
       @reader.mysql.ping
@@ -26,7 +29,7 @@ class MysqlReaderTest < Test::Unit::TestCase
   def test_paginated_read
     expected_rows = 3
     page_size = 2
-    expected_pages = (1.0 * expected_rows / page_size).ceil
+    # expected_pages = (1.0 * expected_rows / page_size).ceil
 
     row_count = my_row_count = 0
     table = @reader.tables.find { |t| t.name == 'numeric_types_basics' }

data/test/integration/postgres_db_writer_test.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require File.expand_path '../test_helper', __dir__
+
+class PostgresDbWriterTest < Test::Unit::TestCase
+  class << self
+    def startup
+      seed_test_database
+    end
+  end
+
+  def setup
+    @options = options_from_file
+    @options[:suppress_data] = true
+    @options[:suppress_ddl] = true
+  end
+
+  def test_pg_connection
+    assert_nothing_raised do
+      Mysql2postgres::PostgresDbWriter.new Tempfile.new('mysql2postgres_test_').path,
+                                           @options[:destination]
+    end
+  end
+end

data/test/test_helper.rb

@@ -4,47 +4,37 @@ require 'rubygems'
 require 'test/unit'
 require 'debug' if ENV.fetch('ENABLE_DEBUG', nil) == '1'
 
-require File.expand_path('../lib/mysql2postgres/version', __dir__)
-require File.expand_path('../lib/mysql2postgres/converter', __dir__)
-require File.expand_path('../lib/mysql2postgres/mysql_reader', __dir__)
-require File.expand_path('../lib/mysql2postgres/writer', __dir__)
-require File.expand_path('../lib/mysql2postgres/postgres_writer', __dir__)
-require File.expand_path('../lib/mysql2postgres/postgres_file_writer', __dir__)
-require File.expand_path('../lib/mysql2postgres/postgres_db_writer', __dir__)
+require File.expand_path('lib/mysql2postgres')
 
-def seed_test_database
-  options = get_test_config_by_label :localmysql_to_file_convert_nothing
-  seedfilepath = "#{__dir__}/../fixtures/seed_integration_tests.sql"
-  rc = system "mysql -u#{options[:mysql][:username]} #{options[:mysql][:database]} < #{seedfilepath}"
-  raise StandardError unless rc
+def load_yaml_file(file = 'config_all_options')
+  YAML.load_file "#{__dir__}/fixtures/#{file}.yml"
+end
 
-  true
-rescue StandardError
-  raise 'Failed to seed integration test db. See README for setup requirements.'
+def instance_from_file(file = 'config_all_options')
+  Mysql2postgres.new load_yaml_file(file)
 end
 
-def get_test_reader(options)
-  require 'mysql2postgres/mysql_reader'
-  mysql2postgres::MysqlReader.new options
-rescue StandardError
-  raise 'Failed to initialize integration test db. See README for setup requirements.'
+def options_from_file(file = 'config_all_options')
+  instance_from_file(file).options
 end
 
-def get_test_file_writer(options)
-  require 'mysql2postgres/postgres_file_writer'
-  mysql2postgres::PostgresFileWriter.new options[:destfile]
-rescue StandardError => e
-  puts e.inspect
-  raise "Failed to initialize file writer from #{options.inspect}. See README for setup requirements."
+def seed_test_database(option_file: 'config_all_options', sql_file: 'seed_integration_tests.sql')
+  options = options_from_file option_file
+  seedfilepath = File.expand_path "test/fixtures/#{sql_file}"
+  system 'mysql ' \
+         "--host #{options[:mysql][:hostname]} " \
+         "--port #{options[:mysql][:port]} " \
+         "-u#{options[:mysql][:username]} " \
+         "-p#{options[:mysql][:password]} " \
+         "#{options[:mysql][:database]} < #{seedfilepath}", exception: true
+rescue StandardError
+  raise 'Failed to seed integration test db. See README for setup requirements.'
 end
 
-def get_test_converter(options)
-  require 'mysql2postgres/converter'
-  reader = get_test_reader options
-  writer = get_test_file_writer options
-  mysql2postgres::Converter.new reader, writer, options
+def get_test_reader(options)
+  Mysql2postgres::MysqlReader.new options
 rescue StandardError
-  raise "Failed to initialize converter from #{options.inspect}. See README for setup requirements."
+  raise 'Failed to initialize integration test db. See README for setup requirements.'
 end
 
 def get_temp_file(basename)
@@ -54,38 +44,3 @@ def get_temp_file(basename)
   f.close!
   path
 end
-
-def get_new_test_config(include_tables: [], exclude_tables: [], to_file: false,
-                        suppress_data: false, suppress_ddl: false, force_truncate: false)
-
-  to_filename = to_file ? get_temp_file(['mysql2postgres_tmp_output_', '.yml']) : nil
-  configtext = Mysql2postgres::Config.template to_filename, include_tables, exclude_tables, suppress_data, suppress_ddl, force_truncate
-  configfile = get_temp_file 'mysql2postgres_tmp_config'
-  File.open(configfile, 'w:UTF-8') { |f| f.write(configtext) }
-  yaml = YAML.load_file configfile
-  Mysql2postgres::ConfigBase.new yaml
-rescue StandardError
-  raise "Failed to initialize options from #{configfile}. See README for setup requirements."
-end
-
-def get_test_config_by_label(name)
-  case name
-  when :localmysql_to_file_convert_nothing
-    get_new_test_config to_file: true,
-                        tables: ['unobtainium'],
-                        exclude_tables: ['kryptonite'],
-                        suppress_data: true,
-                        suppress_ddl: true
-  when :localmysql_to_file_convert_all
-    get_new_test_config to_file: true
-  when :localmysql_to_db_convert_all
-    get_new_test_config
-  when :localmysql_to_db_convert_nothing
-    get_new_test_config tables: ['unobtainium'],
-                        exclude_tables: ['kryptonite'],
-                        suppress_data: true,
-                        suppress_ddl: true
-  else
-    raise "Invalid label: #{name}"
-  end
-end

data/test/units/mysql_test.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require File.expand_path '../test_helper', __dir__
+
+class MysqlTest < Test::Unit::TestCase
+  def test_mysql_charset
+    charset = ::Mysql::Charset.by_number 192
+    assert_equal 'utf8mb3', charset.name
+  end
+end

data/test/units/option_test.rb

@@ -4,14 +4,11 @@ require File.expand_path '../test_helper', __dir__
 require 'yaml'
 
 class SettingTest < Test::Unit::TestCase
-  attr_reader :config_all_opts
-
-  def setup
-    @config_all_opts = YAML.load_file "#{__dir__}/../fixtures/config_all_options.yml"
-  end
-
   def test_options_loaded
-    instance = Mysql2postgres.new @config_all_opts
-    assert_not_nil instance.options
+    options = options_from_file
+
+    assert_equal false, options[:suppress_data]
+    assert_equal 'postgres', options[:destination][:username]
+    assert_equal 'somename', options[:destination][:database]
   end
 end

data/test/units/postgres_file_writer_test.rb

@@ -15,10 +15,12 @@ class PostgresFileWriterTest < Test::Unit::TestCase
     File.delete destfile
   end
 
-  def test_basic_write
-    writer = Mysql2postgres::PostgresFileWriter.new destfile
+  def test_file_writer
+    destination = { filename: '/tmp/test.sql' }
+    writer = Mysql2postgres::PostgresFileWriter.new @destfile, destination
     writer.close
     content = File.read destfile
+
     assert_not_nil content.match("SET client_encoding = 'UTF8'")
     assert_nil content.match('unobtanium')
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mysql2postgres
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Max Lapshin <max@maxidoors.ru>
@@ -24,22 +24,8 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-07-31 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -55,19 +41,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.2.2
 - !ruby/object:Gem::Dependency
-  name: postgres-pr
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-mysql
   requirement: !ruby/object:Gem::Requirement
@@ -82,62 +68,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: debug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-performance
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: test-unit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.5.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.5.3
 description: Translates MySQL -> PostgreSQL
 email: a.meindl@alphanodes.com
 executables:
@@ -158,20 +88,22 @@ files:
 - lib/mysql2postgres/postgres_file_writer.rb
 - lib/mysql2postgres/postgres_writer.rb
 - lib/mysql2postgres/version.rb
-- lib/mysql2postgres/writer.rb
 - mysql2postgres.gemspec
 - test/fixtures/config_all_options.yml
+- test/fixtures/config_min_options.yml
+- test/fixtures/config_to_file.yml
 - test/fixtures/seed_integration_tests.sql
 - test/integration/convert_to_db_test.rb
 - test/integration/convert_to_file_test.rb
 - test/integration/converter_test.rb
-- test/integration/mysql_reader_base_test.rb
+- test/integration/mysql_reader_connection_test.rb
 - test/integration/mysql_reader_test.rb
-- test/integration/postgres_db_writer_base_test.rb
+- test/integration/postgres_db_writer_test.rb
 - test/test_helper.rb
+- test/units/mysql_test.rb
 - test/units/option_test.rb
 - test/units/postgres_file_writer_test.rb
-homepage: https://code.alphanodes.com/alphanodes/mysql2postgres
+homepage: https://github.com/AlphaNodes/mysql2postgres
 licenses:
 - MIT
 metadata:

data/lib/mysql2postgres/writer.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-class Mysql2postgres
-  class Writer
-    def inload
-      raise "Method 'inload' needs to be overridden..."
-    end
-  end
-end

data/test/integration/postgres_db_writer_base_test.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require File.expand_path '../test_helper', __dir__
-
-class PostgresDbWriterBaseTest < Test::Unit::TestCase
-  class << self
-    def startup
-      seed_test_database
-      @options = get_test_config_by_label :localmysql_to_db_convert_nothing
-    end
-  end
-
-  def test_pg_connection
-    assert_nothing_raised do
-      mysql2postgres::PostgresDbWriter.new Tempfile.new('mysql2postgres_test_').path, @options
-    end
-  end
-end