mysql2postgres 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/MIT-LICENSE +20 -0
- data/README.md +45 -0
- data/Rakefile +50 -0
- data/bin/mysql2psql +26 -0
- data/lib/mysql2psql/connection.rb +129 -0
- data/lib/mysql2psql/converter.rb +95 -0
- data/lib/mysql2psql/mysql_reader.rb +234 -0
- data/lib/mysql2psql/postgres_db_writer.rb +26 -0
- data/lib/mysql2psql/postgres_file_writer.rb +143 -0
- data/lib/mysql2psql/postgres_writer.rb +133 -0
- data/lib/mysql2psql/version.rb +5 -0
- data/lib/mysql2psql/writer.rb +9 -0
- data/lib/mysql2psql.rb +59 -0
- data/mysql2postgres.gemspec +80 -0
- data/test/fixtures/config_all_options.yml +39 -0
- data/test/fixtures/seed_integration_tests.sql +24 -0
- data/test/integration/convert_to_db_test.rb +23 -0
- data/test/integration/convert_to_file_test.rb +68 -0
- data/test/integration/converter_test.rb +21 -0
- data/test/integration/mysql_reader_base_test.rb +33 -0
- data/test/integration/mysql_reader_test.rb +40 -0
- data/test/integration/postgres_db_writer_base_test.rb +18 -0
- data/test/test_helper.rb +92 -0
- data/test/units/option_test.rb +17 -0
- data/test/units/postgres_file_writer_test.rb +25 -0
- metadata +199 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: bd05dc2b701ccf002f167cc1ab6d605dd20e857a5fa0306c4c4423b91ce031d1
|
4
|
+
data.tar.gz: 72ccaec021e0cdf53f31d5f9ae47622fcc36bd518beb7ec93188d9d8df9875a2
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c9c6b0bb6a3811020cc8be01000bc8375bd48b91ae65dd622f5a336c8953a7dd37ab2fe338ec3417462ea186905ed2afc65cf9fb7e416f704d1fa29d3efe4e6e
|
7
|
+
data.tar.gz: bae0e0acc099c166b3e0f0af3317c2c40cec6b2fff75816708eb0b8a099bb0fab92680c8dabaeb985701c7670b39ecb2d9a86eb6e9c9d50f7292c0522befb008
|
data/.gitignore
ADDED
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012-2013 Regents of the University of Minnesota
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# mysql-to-postgres - MySQL to PostgreSQL Data Translation
|
2
|
+
|
3
|
+
[](https://github.com/AlphaNodes/mysql2psql/actions/workflows/rubocop.yml) [](https://github.com/AlphaNodes/mysql2psql/actions/workflows/tests.yml)
|
4
|
+
|
5
|
+
The minimum Ruby version supported in `main` branch is `2.7`,
|
6
|
+
and the next release will have the same requirement.
|
7
|
+
|
8
|
+
With a bit of a modified rails `database.yml` configuration, you can integrate `mysql-to-postgres`into a project.
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
```sh
|
13
|
+
git clone https://github.com/AlphaNodes/mysql2postgres.git
|
14
|
+
cd mysql2postgres
|
15
|
+
bundle install
|
16
|
+
gem build mysql2postgres.gemspec
|
17
|
+
sudo gem install mysql2postgres-0.3.2.gem
|
18
|
+
```
|
19
|
+
|
20
|
+
## Configuration
|
21
|
+
|
22
|
+
Configuration is written in [YAML format](http://www.yaml.org/ "YAML Ain't Markup Language")
|
23
|
+
and passed as the first argument on the command line.
|
24
|
+
|
25
|
+
Configuration file has be provided with config/database.yml, see config/default.database.yml for
|
26
|
+
an example.
|
27
|
+
|
28
|
+
## Usage
|
29
|
+
|
30
|
+
After providing settings, start migration with
|
31
|
+
|
32
|
+
```sh
|
33
|
+
|
34
|
+
bundle exec mysql2psql
|
35
|
+
```
|
36
|
+
|
37
|
+
## Tests
|
38
|
+
|
39
|
+
```sh
|
40
|
+
rake test
|
41
|
+
```
|
42
|
+
|
43
|
+
## License
|
44
|
+
|
45
|
+
Licensed under [the MIT license](MIT-LICENSE).
|
data/Rakefile
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'rake'
|
5
|
+
|
6
|
+
require_relative 'lib/mysql2psql/version'
|
7
|
+
|
8
|
+
require 'rake/testtask'
|
9
|
+
namespace :test do
|
10
|
+
Rake::TestTask.new :units do |test|
|
11
|
+
test.libs << 'lib' << 'test/lib'
|
12
|
+
test.pattern = 'test/units/*test.rb'
|
13
|
+
test.verbose = true
|
14
|
+
end
|
15
|
+
|
16
|
+
Rake::TestTask.new :integration do |test|
|
17
|
+
test.libs << 'lib' << 'test/lib'
|
18
|
+
test.pattern = 'test/integration/*test.rb'
|
19
|
+
test.verbose = true
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
desc 'Run all tests'
|
24
|
+
task :test do
|
25
|
+
# Rake::Task['test:units'].invoke
|
26
|
+
Rake::Task['test:integration'].invoke
|
27
|
+
end
|
28
|
+
|
29
|
+
begin
|
30
|
+
require 'rcov/rcovtask'
|
31
|
+
Rcov::RcovTask.new do |test|
|
32
|
+
test.libs << 'test'
|
33
|
+
test.pattern = 'test/**/*test.rb'
|
34
|
+
test.verbose = true
|
35
|
+
end
|
36
|
+
rescue LoadError
|
37
|
+
task :rcov do
|
38
|
+
abort 'RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov'
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
task default: :test
|
43
|
+
|
44
|
+
require 'rdoc/task'
|
45
|
+
Rake::RDocTask.new do |rdoc|
|
46
|
+
rdoc.rdoc_dir = 'rdoc'
|
47
|
+
rdoc.title = "mysql2psql #{Mysql2psql::VERSION}"
|
48
|
+
rdoc.rdoc_files.include 'README*'
|
49
|
+
rdoc.rdoc_files.include 'lib/**/*.rb'
|
50
|
+
end
|
data/bin/mysql2psql
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift File.join(File.dirname(__dir__), 'lib')
|
5
|
+
|
6
|
+
require 'rubygems'
|
7
|
+
require 'bundler/setup'
|
8
|
+
require 'mysql2psql'
|
9
|
+
|
10
|
+
CONFIG_FILE = File.join File.dirname(__dir__), 'config', 'database.yml'
|
11
|
+
|
12
|
+
file = if ARGV.length.positive?
|
13
|
+
ARGV[0]
|
14
|
+
else
|
15
|
+
CONFIG_FILE
|
16
|
+
end
|
17
|
+
|
18
|
+
unless FileTest.exist?(CONFIG_FILE) || (ARGV.length.positive? && FileTest.exist?(File.expand_path(ARGV[0])))
|
19
|
+
raise "'#{file}' does not exist"
|
20
|
+
end
|
21
|
+
|
22
|
+
db_yaml = YAML.safe_load File.read(file)
|
23
|
+
raise "'#{file}' does not contain a mysql configuration directive for conversion" unless db_yaml.key? 'mysql'
|
24
|
+
raise "'#{file}' does not contain a destination configuration directive for conversion" unless db_yaml.key? 'destination'
|
25
|
+
|
26
|
+
Mysql2psql.new(db_yaml).convert
|
@@ -0,0 +1,129 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Mysql2psql
|
4
|
+
class Connection
|
5
|
+
attr_reader :conn,
|
6
|
+
:adapter,
|
7
|
+
:hostname,
|
8
|
+
:login,
|
9
|
+
:password,
|
10
|
+
:database,
|
11
|
+
:schema,
|
12
|
+
:port,
|
13
|
+
:environment,
|
14
|
+
:copy_manager,
|
15
|
+
:stream,
|
16
|
+
:is_copying
|
17
|
+
|
18
|
+
def initialize(options)
|
19
|
+
@environment = (ENV['RAILS_ENV'] || 'development').to_sym
|
20
|
+
|
21
|
+
if options[:destination].nil? ||
|
22
|
+
options[:destination].empty? ||
|
23
|
+
options[:destination][environment].nil? ||
|
24
|
+
options[:destination][environment].empty?
|
25
|
+
raise 'Unable to locate PostgreSQL destination environment in the configuration file'
|
26
|
+
end
|
27
|
+
|
28
|
+
pg_options = options[:destination][environment]
|
29
|
+
@hostname = pg_options[:hostname] || 'localhost'
|
30
|
+
@login = pg_options[:username]
|
31
|
+
@password = pg_options[:password]
|
32
|
+
@database = pg_options[:database]
|
33
|
+
@port = (pg_options[:port] || 5432).to_s
|
34
|
+
|
35
|
+
@database, @schema = database.split ':'
|
36
|
+
@adapter = pg_options[:adapter] || 'jdbcpostgresql'
|
37
|
+
|
38
|
+
@conn = PG::Connection.open dbname: database,
|
39
|
+
user: login,
|
40
|
+
password: password,
|
41
|
+
host: hostname,
|
42
|
+
port: port
|
43
|
+
|
44
|
+
raise_nil_connection if conn.nil?
|
45
|
+
|
46
|
+
@is_copying = false
|
47
|
+
@current_statement = ''
|
48
|
+
end
|
49
|
+
|
50
|
+
# ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream.
|
51
|
+
def flush
|
52
|
+
conn.put_copy_end
|
53
|
+
rescue StandardError => e
|
54
|
+
warn e
|
55
|
+
ensure
|
56
|
+
@is_copying = false
|
57
|
+
end
|
58
|
+
|
59
|
+
def execute(sql)
|
60
|
+
if sql.match(/^COPY /) && !is_copying
|
61
|
+
# sql.chomp! # cHomp! cHomp!
|
62
|
+
conn.exec sql
|
63
|
+
@is_copying = true
|
64
|
+
elsif sql.match(/^(ALTER|CREATE|DROP|SELECT|SET|TRUNCATE) /) && !is_copying
|
65
|
+
@current_statement = sql
|
66
|
+
elsif is_copying
|
67
|
+
if sql.chomp == '\.' || sql.chomp.match(/^$/)
|
68
|
+
flush
|
69
|
+
else
|
70
|
+
begin
|
71
|
+
until conn.put_copy_data sql
|
72
|
+
warn ' waiting for connection to be writable...'
|
73
|
+
sleep 0.1
|
74
|
+
end
|
75
|
+
rescue StandardError => e
|
76
|
+
@is_copying = false
|
77
|
+
warn e
|
78
|
+
raise e
|
79
|
+
end
|
80
|
+
end
|
81
|
+
elsif @current_statement.length.positive?
|
82
|
+
@current_statement << ' '
|
83
|
+
@current_statement << sql
|
84
|
+
end
|
85
|
+
|
86
|
+
return unless @current_statement.match?(/;$/)
|
87
|
+
|
88
|
+
run_statement @current_statement
|
89
|
+
@current_statement = ''
|
90
|
+
end
|
91
|
+
|
92
|
+
# we're done talking to the database, so close the connection cleanly.
|
93
|
+
def finish
|
94
|
+
@conn.finish
|
95
|
+
end
|
96
|
+
|
97
|
+
# given a file containing psql syntax at path, pipe it down to the database.
|
98
|
+
def load_file(path)
|
99
|
+
if @conn
|
100
|
+
File.open path, 'r:UTF-8' do |file|
|
101
|
+
file.each_line do |line|
|
102
|
+
execute line
|
103
|
+
end
|
104
|
+
flush
|
105
|
+
end
|
106
|
+
finish
|
107
|
+
else
|
108
|
+
raise_nil_connection
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def clear_schema
|
113
|
+
statements = ['DROP SCHEMA PUBLIC CASCADE', 'CREATE SCHEMA PUBLIC']
|
114
|
+
statements.each do |statement|
|
115
|
+
run_statement statement
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def raise_nil_connection
|
120
|
+
raise 'No Connection'
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
def run_statement(statement)
|
126
|
+
@conn.exec statement
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Mysql2psql
|
4
|
+
class Converter
|
5
|
+
attr_reader :reader,
|
6
|
+
:writer,
|
7
|
+
:options,
|
8
|
+
:exclude_tables,
|
9
|
+
:only_tables,
|
10
|
+
:suppress_data,
|
11
|
+
:suppress_ddl,
|
12
|
+
:force_truncate,
|
13
|
+
:preserve_order,
|
14
|
+
:clear_schema
|
15
|
+
|
16
|
+
def initialize(reader, writer, options)
|
17
|
+
@reader = reader
|
18
|
+
@writer = writer
|
19
|
+
@options = options
|
20
|
+
@exclude_tables = options[:exclude_tables] || []
|
21
|
+
@only_tables = options[:tables]
|
22
|
+
@suppress_data = options[:suppress_data] || false
|
23
|
+
@suppress_ddl = options[:suppress_ddl] || false
|
24
|
+
@force_truncate = options[:force_truncate] || false
|
25
|
+
@preserve_order = options[:preserve_order] || false
|
26
|
+
@clear_schema = options[:clear_schema] || false
|
27
|
+
end
|
28
|
+
|
29
|
+
def convert
|
30
|
+
tables = reader.tables
|
31
|
+
tables.reject! { |table| exclude_tables.include?(table.name) }
|
32
|
+
tables.select! { |table| only_tables ? only_tables.include?(table.name) : true }
|
33
|
+
|
34
|
+
# preserve order only works, if only_tables are specified
|
35
|
+
if preserve_order && only_tables
|
36
|
+
reordered_tables = []
|
37
|
+
|
38
|
+
only_tables.each do |only_table|
|
39
|
+
idx = tables.index { |table| table.name == only_table }
|
40
|
+
if idx.nil?
|
41
|
+
warn "Specified source table '#{only_table}' does not exist, skiped by migration"
|
42
|
+
else
|
43
|
+
reordered_tables << tables[idx]
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
tables = reordered_tables
|
48
|
+
end
|
49
|
+
|
50
|
+
unless suppress_ddl
|
51
|
+
tables.each do |table|
|
52
|
+
puts "Writing DDL for #{table.name}"
|
53
|
+
writer.write_table table
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
unless suppress_data
|
58
|
+
if force_truncate && suppress_ddl
|
59
|
+
tables.each do |table|
|
60
|
+
puts "Truncate table #{table.name}"
|
61
|
+
writer.truncate table
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
tables.each do |table|
|
66
|
+
puts "Writing data for #{table.name}"
|
67
|
+
writer.write_contents table, reader
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
puts 'Writing indices and constraints'
|
72
|
+
unless suppress_ddl
|
73
|
+
tables.each do |table|
|
74
|
+
writer.write_indexes table
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
unless suppress_ddl
|
79
|
+
tables.each do |table|
|
80
|
+
writer.write_constraints table
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
writer.close
|
85
|
+
writer.clear_schema if clear_schema
|
86
|
+
writer.inload
|
87
|
+
0
|
88
|
+
rescue StandardError => e
|
89
|
+
warn "Mysql2psql: Conversion failed: #{e}"
|
90
|
+
warn e
|
91
|
+
warn e.backtrace[0, 3].join("\n")
|
92
|
+
-1
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,234 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler/setup'
|
5
|
+
|
6
|
+
require 'mysql'
|
7
|
+
require 'csv'
|
8
|
+
|
9
|
+
class Mysql2psql
|
10
|
+
class MysqlReader
|
11
|
+
class Table
|
12
|
+
attr_reader :name
|
13
|
+
|
14
|
+
def initialize(reader, name)
|
15
|
+
@reader = reader
|
16
|
+
@name = name
|
17
|
+
end
|
18
|
+
|
19
|
+
def columns
|
20
|
+
@columns ||= load_columns
|
21
|
+
end
|
22
|
+
|
23
|
+
def convert_type(type)
|
24
|
+
case type
|
25
|
+
when /int.* unsigned/, /bigint/
|
26
|
+
'bigint'
|
27
|
+
when 'bit(1)', 'tinyint(1)'
|
28
|
+
'boolean'
|
29
|
+
when /tinyint/
|
30
|
+
'tinyint'
|
31
|
+
when /int/
|
32
|
+
'integer'
|
33
|
+
when /varchar/, /set/
|
34
|
+
'varchar'
|
35
|
+
when /char/
|
36
|
+
'char'
|
37
|
+
when /decimal/
|
38
|
+
'decimal'
|
39
|
+
when /(float|double)/
|
40
|
+
'double precision'
|
41
|
+
else
|
42
|
+
type
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def load_columns
|
47
|
+
@reader.reconnect
|
48
|
+
# mysql_flags = ::Mysql::Field.constants.select { |c| c.to_s.include?('FLAG') }
|
49
|
+
|
50
|
+
fields = []
|
51
|
+
@reader.query "EXPLAIN `#{name}`" do |res|
|
52
|
+
while (field = res.fetch_row)
|
53
|
+
length = field[1][/\((\d+)\)/, 1] if field[1].match?(/\((\d+)\)/)
|
54
|
+
length = field[1][/\((\d+),(\d+)\)/, 1] if field[1].match?(/\((\d+),(\d+)\)/)
|
55
|
+
desc = {
|
56
|
+
name: field[0],
|
57
|
+
table_name: name,
|
58
|
+
type: convert_type(field[1]),
|
59
|
+
length: length&.to_i,
|
60
|
+
decimals: field[1][/\((\d+),(\d+)\)/, 2],
|
61
|
+
null: field[2] == 'YES',
|
62
|
+
primary_key: field[3] == 'PRI',
|
63
|
+
auto_increment: field[5] == 'auto_increment'
|
64
|
+
}
|
65
|
+
desc[:default] = field[4] unless field[4].nil?
|
66
|
+
fields << desc
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
fields.select { |field| field[:auto_increment] }.each do |field|
|
71
|
+
@reader.query "SELECT max(`#{field[:name]}`) FROM `#{name}`" do |res|
|
72
|
+
field[:maxval] = res.fetch_row[0].to_i
|
73
|
+
end
|
74
|
+
end
|
75
|
+
fields
|
76
|
+
end
|
77
|
+
|
78
|
+
def indexes
|
79
|
+
load_indexes unless @indexes
|
80
|
+
@indexes
|
81
|
+
end
|
82
|
+
|
83
|
+
def foreign_keys
|
84
|
+
load_indexes unless @foreign_keys
|
85
|
+
@foreign_keys
|
86
|
+
end
|
87
|
+
|
88
|
+
def load_indexes
|
89
|
+
@indexes = []
|
90
|
+
@foreign_keys = []
|
91
|
+
|
92
|
+
@reader.query "SHOW CREATE TABLE `#{name}`" do |result|
|
93
|
+
explain = result.fetch_row[1]
|
94
|
+
explain.split("\n").each do |line|
|
95
|
+
next unless line.include? ' KEY '
|
96
|
+
|
97
|
+
index = {}
|
98
|
+
if (match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line))
|
99
|
+
index[:name] = "fk_#{name}_#{match_data[1]}"
|
100
|
+
index[:column] = match_data[2].delete!('`').split(', ')
|
101
|
+
index[:ref_table] = match_data[3]
|
102
|
+
index[:ref_column] = match_data[4].delete!('`').split(', ')
|
103
|
+
|
104
|
+
the_rest = match_data[5]
|
105
|
+
|
106
|
+
if (match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest))
|
107
|
+
index[:on_delete] = match_data[1]
|
108
|
+
else
|
109
|
+
index[:on_delete] ||= 'RESTRICT'
|
110
|
+
end
|
111
|
+
|
112
|
+
if (match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest))
|
113
|
+
index[:on_update] = match_data[1]
|
114
|
+
else
|
115
|
+
index[:on_update] ||= 'RESTRICT'
|
116
|
+
end
|
117
|
+
|
118
|
+
@foreign_keys << index
|
119
|
+
elsif (match_data = /KEY `(\w+)` \((.*)\)/.match(line))
|
120
|
+
# index[:name] = 'idx_' + name + '_' + match_data[1]
|
121
|
+
# with redmine we do not want prefix idx_tablename_
|
122
|
+
index[:name] = match_data[1]
|
123
|
+
index[:columns] = match_data[2].split(',').map { |col| col[/`(\w+)`/, 1] }
|
124
|
+
index[:unique] = true if line.include? 'UNIQUE'
|
125
|
+
@indexes << index
|
126
|
+
elsif (match_data = /PRIMARY KEY .*\((.*)\)/.match(line))
|
127
|
+
index[:primary] = true
|
128
|
+
index[:columns] = match_data[1].split(',').map { |col| col.strip.delete('`') }
|
129
|
+
@indexes << index
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def count_rows
|
136
|
+
@reader.query "SELECT COUNT(*) FROM `#{name}`" do |res|
|
137
|
+
return res.fetch_row[0].to_i
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def id?
|
142
|
+
!!columns.find { |col| col[:name] == 'id' }
|
143
|
+
end
|
144
|
+
|
145
|
+
def count_for_pager
|
146
|
+
query = id? ? 'MAX(id)' : 'COUNT(*)'
|
147
|
+
@reader.query "SELECT #{query} FROM `#{name}`" do |res|
|
148
|
+
return res.fetch_row[0].to_i
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def query_for_pager
|
153
|
+
query = id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?'
|
154
|
+
|
155
|
+
cols = columns.map do |c|
|
156
|
+
if c[:type] == 'multipolygon'
|
157
|
+
"AsWKT(`#{c[:name]}`) as `#{c[:name]}`"
|
158
|
+
else
|
159
|
+
"`#{c[:name]}`"
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
"SELECT #{cols.join ', '} FROM `#{name}` #{query}"
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
def connect
|
168
|
+
@mysql = ::Mysql.connect @host, @user, @passwd, @db, @port, @sock
|
169
|
+
# utf8_unicode_ci :: https://rubydoc.info/gems/ruby-mysql/Mysql/Charset
|
170
|
+
@mysql.charset = ::Mysql::Charset.by_number 192
|
171
|
+
@mysql.query 'SET NAMES utf8'
|
172
|
+
|
173
|
+
var_info = @mysql.query "SHOW VARIABLES LIKE 'query_cache_type'"
|
174
|
+
return if var_info.nil? || var_info.first.nil? || var_info.first[1] == 'OFF'
|
175
|
+
|
176
|
+
@mysql.query 'SET SESSION query_cache_type = OFF'
|
177
|
+
end
|
178
|
+
|
179
|
+
def reconnect
|
180
|
+
@mysql.close
|
181
|
+
rescue StandardError
|
182
|
+
warn 'could not close previous mysql connection'
|
183
|
+
ensure
|
184
|
+
connect
|
185
|
+
end
|
186
|
+
|
187
|
+
def query(*args, &block)
|
188
|
+
mysql.query(*args, &block)
|
189
|
+
rescue Mysql::Error => e
|
190
|
+
if e.message.match?(/gone away/i)
|
191
|
+
reconnect
|
192
|
+
retry
|
193
|
+
else
|
194
|
+
puts "MySQL Query failed '#{args.inspect}' #{e.inspect}"
|
195
|
+
puts e.backtrace[0, 5].join("\n")
|
196
|
+
[]
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def initialize(options)
|
201
|
+
@host = options[:mysql][:hostname]
|
202
|
+
@user = options[:mysql][:username]
|
203
|
+
@passwd = options[:mysql][:password]
|
204
|
+
@db = options[:mysql][:database]
|
205
|
+
@port = options[:mysql][:port] || 3306
|
206
|
+
@sock = options[:mysql][:socket] && !options[:mysql][:socket].empty? ? options[:mysql][:socket] : nil
|
207
|
+
@sock = options[:mysql][:flag] && !options[:mysql][:flag].empty? ? options[:mysql][:flag] : nil
|
208
|
+
|
209
|
+
connect
|
210
|
+
end
|
211
|
+
|
212
|
+
attr_reader :mysql
|
213
|
+
|
214
|
+
def tables
|
215
|
+
@tables ||= @mysql.query('SHOW TABLES').map { |row| Table.new(self, row.first) }
|
216
|
+
end
|
217
|
+
|
218
|
+
def paginated_read(table, page_size)
|
219
|
+
count = table.count_for_pager
|
220
|
+
return if count < 1
|
221
|
+
|
222
|
+
statement = @mysql.prepare table.query_for_pager
|
223
|
+
counter = 0
|
224
|
+
0.upto (count + page_size) / page_size do |i|
|
225
|
+
statement.execute(i * page_size, table.id? ? (i + 1) * page_size : page_size)
|
226
|
+
while (row = statement.fetch)
|
227
|
+
counter += 1
|
228
|
+
yield row, counter
|
229
|
+
end
|
230
|
+
end
|
231
|
+
counter
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'mysql2psql/postgres_writer'
|
4
|
+
require 'mysql2psql/connection'
|
5
|
+
|
6
|
+
class Mysql2psql
|
7
|
+
class PostgresDbWriter < PostgresFileWriter
|
8
|
+
attr_reader :connection, :filename
|
9
|
+
|
10
|
+
def initialize(filename, options)
|
11
|
+
# NOTE: the superclass opens and truncates filename for writing
|
12
|
+
super filename
|
13
|
+
|
14
|
+
@filename = filename
|
15
|
+
@connection = Connection.new options
|
16
|
+
end
|
17
|
+
|
18
|
+
def inload(path = filename)
|
19
|
+
connection.load_file path
|
20
|
+
end
|
21
|
+
|
22
|
+
def clear_schema
|
23
|
+
connection.clear_schema
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|