mysql2postgres 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: bd05dc2b701ccf002f167cc1ab6d605dd20e857a5fa0306c4c4423b91ce031d1
4
+ data.tar.gz: 72ccaec021e0cdf53f31d5f9ae47622fcc36bd518beb7ec93188d9d8df9875a2
5
+ SHA512:
6
+ metadata.gz: c9c6b0bb6a3811020cc8be01000bc8375bd48b91ae65dd622f5a336c8953a7dd37ab2fe338ec3417462ea186905ed2afc65cf9fb7e416f704d1fa29d3efe4e6e
7
+ data.tar.gz: bae0e0acc099c166b3e0f0af3317c2c40cec6b2fff75816708eb0b8a099bb0fab92680c8dabaeb985701c7670b39ecb2d9a86eb6e9c9d50f7292c0522befb008
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ .DS_Store
2
+ *.gem
3
+ *.sql
4
+ .bundle
5
+ .rvmrc
6
+ config/database.yml
7
+ Gemfile.lock
8
+ pkg
9
+ test/fixtures/test*.sql
data/MIT-LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2012-2013 Regents of the University of Minnesota
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,45 @@
1
+ # mysql-to-postgres - MySQL to PostgreSQL Data Translation
2
+
3
+ [![Run Linters](https://github.com/AlphaNodes/mysql2psql/workflows/Run%20Rubocop/badge.svg)](https://github.com/AlphaNodes/mysql2psql/actions/workflows/rubocop.yml) [![Run Tests](https://github.com/AlphaNodes/mysql2psql/workflows/Tests/badge.svg)](https://github.com/AlphaNodes/mysql2psql/actions/workflows/tests.yml)
4
+
5
+ The minimum Ruby version supported in `main` branch is `2.7`,
6
+ and the next release will have the same requirement.
7
+
8
+ With a bit of a modified rails `database.yml` configuration, you can integrate `mysql-to-postgres`into a project.
9
+
10
+ ## Installation
11
+
12
+ ```sh
13
+ git clone https://github.com/AlphaNodes/mysql2postgres.git
14
+ cd mysql2postgres
15
+ bundle install
16
+ gem build mysql2postgres.gemspec
17
+ sudo gem install mysql2postgres-0.3.2.gem
18
+ ```
19
+
20
+ ## Configuration
21
+
22
+ Configuration is written in [YAML format](http://www.yaml.org/ "YAML Ain't Markup Language")
23
+ and passed as the first argument on the command line.
24
+
25
+ Configuration file has be provided with config/database.yml, see config/default.database.yml for
26
+ an example.
27
+
28
+ ## Usage
29
+
30
+ After providing settings, start migration with
31
+
32
+ ```sh
33
+
34
+ bundle exec mysql2psql
35
+ ```
36
+
37
+ ## Tests
38
+
39
+ ```sh
40
+ rake test
41
+ ```
42
+
43
+ ## License
44
+
45
+ Licensed under [the MIT license](MIT-LICENSE).
data/Rakefile ADDED
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rubygems'
4
+ require 'rake'
5
+
6
+ require_relative 'lib/mysql2psql/version'
7
+
8
+ require 'rake/testtask'
9
+ namespace :test do
10
+ Rake::TestTask.new :units do |test|
11
+ test.libs << 'lib' << 'test/lib'
12
+ test.pattern = 'test/units/*test.rb'
13
+ test.verbose = true
14
+ end
15
+
16
+ Rake::TestTask.new :integration do |test|
17
+ test.libs << 'lib' << 'test/lib'
18
+ test.pattern = 'test/integration/*test.rb'
19
+ test.verbose = true
20
+ end
21
+ end
22
+
23
+ desc 'Run all tests'
24
+ task :test do
25
+ # Rake::Task['test:units'].invoke
26
+ Rake::Task['test:integration'].invoke
27
+ end
28
+
29
+ begin
30
+ require 'rcov/rcovtask'
31
+ Rcov::RcovTask.new do |test|
32
+ test.libs << 'test'
33
+ test.pattern = 'test/**/*test.rb'
34
+ test.verbose = true
35
+ end
36
+ rescue LoadError
37
+ task :rcov do
38
+ abort 'RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov'
39
+ end
40
+ end
41
+
42
+ task default: :test
43
+
44
+ require 'rdoc/task'
45
+ Rake::RDocTask.new do |rdoc|
46
+ rdoc.rdoc_dir = 'rdoc'
47
+ rdoc.title = "mysql2psql #{Mysql2psql::VERSION}"
48
+ rdoc.rdoc_files.include 'README*'
49
+ rdoc.rdoc_files.include 'lib/**/*.rb'
50
+ end
data/bin/mysql2psql ADDED
@@ -0,0 +1,26 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ $LOAD_PATH.unshift File.join(File.dirname(__dir__), 'lib')
5
+
6
+ require 'rubygems'
7
+ require 'bundler/setup'
8
+ require 'mysql2psql'
9
+
10
+ CONFIG_FILE = File.join File.dirname(__dir__), 'config', 'database.yml'
11
+
12
+ file = if ARGV.length.positive?
13
+ ARGV[0]
14
+ else
15
+ CONFIG_FILE
16
+ end
17
+
18
+ unless FileTest.exist?(CONFIG_FILE) || (ARGV.length.positive? && FileTest.exist?(File.expand_path(ARGV[0])))
19
+ raise "'#{file}' does not exist"
20
+ end
21
+
22
+ db_yaml = YAML.safe_load File.read(file)
23
+ raise "'#{file}' does not contain a mysql configuration directive for conversion" unless db_yaml.key? 'mysql'
24
+ raise "'#{file}' does not contain a destination configuration directive for conversion" unless db_yaml.key? 'destination'
25
+
26
+ Mysql2psql.new(db_yaml).convert
@@ -0,0 +1,129 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Mysql2psql
4
+ class Connection
5
+ attr_reader :conn,
6
+ :adapter,
7
+ :hostname,
8
+ :login,
9
+ :password,
10
+ :database,
11
+ :schema,
12
+ :port,
13
+ :environment,
14
+ :copy_manager,
15
+ :stream,
16
+ :is_copying
17
+
18
+ def initialize(options)
19
+ @environment = (ENV['RAILS_ENV'] || 'development').to_sym
20
+
21
+ if options[:destination].nil? ||
22
+ options[:destination].empty? ||
23
+ options[:destination][environment].nil? ||
24
+ options[:destination][environment].empty?
25
+ raise 'Unable to locate PostgreSQL destination environment in the configuration file'
26
+ end
27
+
28
+ pg_options = options[:destination][environment]
29
+ @hostname = pg_options[:hostname] || 'localhost'
30
+ @login = pg_options[:username]
31
+ @password = pg_options[:password]
32
+ @database = pg_options[:database]
33
+ @port = (pg_options[:port] || 5432).to_s
34
+
35
+ @database, @schema = database.split ':'
36
+ @adapter = pg_options[:adapter] || 'jdbcpostgresql'
37
+
38
+ @conn = PG::Connection.open dbname: database,
39
+ user: login,
40
+ password: password,
41
+ host: hostname,
42
+ port: port
43
+
44
+ raise_nil_connection if conn.nil?
45
+
46
+ @is_copying = false
47
+ @current_statement = ''
48
+ end
49
+
50
+ # ensure that the copy is completed, in case we hadn't seen a '\.' in the data stream.
51
+ def flush
52
+ conn.put_copy_end
53
+ rescue StandardError => e
54
+ warn e
55
+ ensure
56
+ @is_copying = false
57
+ end
58
+
59
+ def execute(sql)
60
+ if sql.match(/^COPY /) && !is_copying
61
+ # sql.chomp! # cHomp! cHomp!
62
+ conn.exec sql
63
+ @is_copying = true
64
+ elsif sql.match(/^(ALTER|CREATE|DROP|SELECT|SET|TRUNCATE) /) && !is_copying
65
+ @current_statement = sql
66
+ elsif is_copying
67
+ if sql.chomp == '\.' || sql.chomp.match(/^$/)
68
+ flush
69
+ else
70
+ begin
71
+ until conn.put_copy_data sql
72
+ warn ' waiting for connection to be writable...'
73
+ sleep 0.1
74
+ end
75
+ rescue StandardError => e
76
+ @is_copying = false
77
+ warn e
78
+ raise e
79
+ end
80
+ end
81
+ elsif @current_statement.length.positive?
82
+ @current_statement << ' '
83
+ @current_statement << sql
84
+ end
85
+
86
+ return unless @current_statement.match?(/;$/)
87
+
88
+ run_statement @current_statement
89
+ @current_statement = ''
90
+ end
91
+
92
+ # we're done talking to the database, so close the connection cleanly.
93
+ def finish
94
+ @conn.finish
95
+ end
96
+
97
+ # given a file containing psql syntax at path, pipe it down to the database.
98
+ def load_file(path)
99
+ if @conn
100
+ File.open path, 'r:UTF-8' do |file|
101
+ file.each_line do |line|
102
+ execute line
103
+ end
104
+ flush
105
+ end
106
+ finish
107
+ else
108
+ raise_nil_connection
109
+ end
110
+ end
111
+
112
+ def clear_schema
113
+ statements = ['DROP SCHEMA PUBLIC CASCADE', 'CREATE SCHEMA PUBLIC']
114
+ statements.each do |statement|
115
+ run_statement statement
116
+ end
117
+ end
118
+
119
+ def raise_nil_connection
120
+ raise 'No Connection'
121
+ end
122
+
123
+ private
124
+
125
+ def run_statement(statement)
126
+ @conn.exec statement
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Mysql2psql
4
+ class Converter
5
+ attr_reader :reader,
6
+ :writer,
7
+ :options,
8
+ :exclude_tables,
9
+ :only_tables,
10
+ :suppress_data,
11
+ :suppress_ddl,
12
+ :force_truncate,
13
+ :preserve_order,
14
+ :clear_schema
15
+
16
+ def initialize(reader, writer, options)
17
+ @reader = reader
18
+ @writer = writer
19
+ @options = options
20
+ @exclude_tables = options[:exclude_tables] || []
21
+ @only_tables = options[:tables]
22
+ @suppress_data = options[:suppress_data] || false
23
+ @suppress_ddl = options[:suppress_ddl] || false
24
+ @force_truncate = options[:force_truncate] || false
25
+ @preserve_order = options[:preserve_order] || false
26
+ @clear_schema = options[:clear_schema] || false
27
+ end
28
+
29
+ def convert
30
+ tables = reader.tables
31
+ tables.reject! { |table| exclude_tables.include?(table.name) }
32
+ tables.select! { |table| only_tables ? only_tables.include?(table.name) : true }
33
+
34
+ # preserve order only works, if only_tables are specified
35
+ if preserve_order && only_tables
36
+ reordered_tables = []
37
+
38
+ only_tables.each do |only_table|
39
+ idx = tables.index { |table| table.name == only_table }
40
+ if idx.nil?
41
+ warn "Specified source table '#{only_table}' does not exist, skiped by migration"
42
+ else
43
+ reordered_tables << tables[idx]
44
+ end
45
+ end
46
+
47
+ tables = reordered_tables
48
+ end
49
+
50
+ unless suppress_ddl
51
+ tables.each do |table|
52
+ puts "Writing DDL for #{table.name}"
53
+ writer.write_table table
54
+ end
55
+ end
56
+
57
+ unless suppress_data
58
+ if force_truncate && suppress_ddl
59
+ tables.each do |table|
60
+ puts "Truncate table #{table.name}"
61
+ writer.truncate table
62
+ end
63
+ end
64
+
65
+ tables.each do |table|
66
+ puts "Writing data for #{table.name}"
67
+ writer.write_contents table, reader
68
+ end
69
+ end
70
+
71
+ puts 'Writing indices and constraints'
72
+ unless suppress_ddl
73
+ tables.each do |table|
74
+ writer.write_indexes table
75
+ end
76
+ end
77
+
78
+ unless suppress_ddl
79
+ tables.each do |table|
80
+ writer.write_constraints table
81
+ end
82
+ end
83
+
84
+ writer.close
85
+ writer.clear_schema if clear_schema
86
+ writer.inload
87
+ 0
88
+ rescue StandardError => e
89
+ warn "Mysql2psql: Conversion failed: #{e}"
90
+ warn e
91
+ warn e.backtrace[0, 3].join("\n")
92
+ -1
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,234 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rubygems'
4
+ require 'bundler/setup'
5
+
6
+ require 'mysql'
7
+ require 'csv'
8
+
9
+ class Mysql2psql
10
+ class MysqlReader
11
+ class Table
12
+ attr_reader :name
13
+
14
+ def initialize(reader, name)
15
+ @reader = reader
16
+ @name = name
17
+ end
18
+
19
+ def columns
20
+ @columns ||= load_columns
21
+ end
22
+
23
+ def convert_type(type)
24
+ case type
25
+ when /int.* unsigned/, /bigint/
26
+ 'bigint'
27
+ when 'bit(1)', 'tinyint(1)'
28
+ 'boolean'
29
+ when /tinyint/
30
+ 'tinyint'
31
+ when /int/
32
+ 'integer'
33
+ when /varchar/, /set/
34
+ 'varchar'
35
+ when /char/
36
+ 'char'
37
+ when /decimal/
38
+ 'decimal'
39
+ when /(float|double)/
40
+ 'double precision'
41
+ else
42
+ type
43
+ end
44
+ end
45
+
46
+ def load_columns
47
+ @reader.reconnect
48
+ # mysql_flags = ::Mysql::Field.constants.select { |c| c.to_s.include?('FLAG') }
49
+
50
+ fields = []
51
+ @reader.query "EXPLAIN `#{name}`" do |res|
52
+ while (field = res.fetch_row)
53
+ length = field[1][/\((\d+)\)/, 1] if field[1].match?(/\((\d+)\)/)
54
+ length = field[1][/\((\d+),(\d+)\)/, 1] if field[1].match?(/\((\d+),(\d+)\)/)
55
+ desc = {
56
+ name: field[0],
57
+ table_name: name,
58
+ type: convert_type(field[1]),
59
+ length: length&.to_i,
60
+ decimals: field[1][/\((\d+),(\d+)\)/, 2],
61
+ null: field[2] == 'YES',
62
+ primary_key: field[3] == 'PRI',
63
+ auto_increment: field[5] == 'auto_increment'
64
+ }
65
+ desc[:default] = field[4] unless field[4].nil?
66
+ fields << desc
67
+ end
68
+ end
69
+
70
+ fields.select { |field| field[:auto_increment] }.each do |field|
71
+ @reader.query "SELECT max(`#{field[:name]}`) FROM `#{name}`" do |res|
72
+ field[:maxval] = res.fetch_row[0].to_i
73
+ end
74
+ end
75
+ fields
76
+ end
77
+
78
+ def indexes
79
+ load_indexes unless @indexes
80
+ @indexes
81
+ end
82
+
83
+ def foreign_keys
84
+ load_indexes unless @foreign_keys
85
+ @foreign_keys
86
+ end
87
+
88
+ def load_indexes
89
+ @indexes = []
90
+ @foreign_keys = []
91
+
92
+ @reader.query "SHOW CREATE TABLE `#{name}`" do |result|
93
+ explain = result.fetch_row[1]
94
+ explain.split("\n").each do |line|
95
+ next unless line.include? ' KEY '
96
+
97
+ index = {}
98
+ if (match_data = /CONSTRAINT `(\w+)` FOREIGN KEY \((.*?)\) REFERENCES `(\w+)` \((.*?)\)(.*)/.match(line))
99
+ index[:name] = "fk_#{name}_#{match_data[1]}"
100
+ index[:column] = match_data[2].delete!('`').split(', ')
101
+ index[:ref_table] = match_data[3]
102
+ index[:ref_column] = match_data[4].delete!('`').split(', ')
103
+
104
+ the_rest = match_data[5]
105
+
106
+ if (match_data = /ON DELETE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest))
107
+ index[:on_delete] = match_data[1]
108
+ else
109
+ index[:on_delete] ||= 'RESTRICT'
110
+ end
111
+
112
+ if (match_data = /ON UPDATE (SET NULL|SET DEFAULT|RESTRICT|NO ACTION|CASCADE)/.match(the_rest))
113
+ index[:on_update] = match_data[1]
114
+ else
115
+ index[:on_update] ||= 'RESTRICT'
116
+ end
117
+
118
+ @foreign_keys << index
119
+ elsif (match_data = /KEY `(\w+)` \((.*)\)/.match(line))
120
+ # index[:name] = 'idx_' + name + '_' + match_data[1]
121
+ # with redmine we do not want prefix idx_tablename_
122
+ index[:name] = match_data[1]
123
+ index[:columns] = match_data[2].split(',').map { |col| col[/`(\w+)`/, 1] }
124
+ index[:unique] = true if line.include? 'UNIQUE'
125
+ @indexes << index
126
+ elsif (match_data = /PRIMARY KEY .*\((.*)\)/.match(line))
127
+ index[:primary] = true
128
+ index[:columns] = match_data[1].split(',').map { |col| col.strip.delete('`') }
129
+ @indexes << index
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ def count_rows
136
+ @reader.query "SELECT COUNT(*) FROM `#{name}`" do |res|
137
+ return res.fetch_row[0].to_i
138
+ end
139
+ end
140
+
141
+ def id?
142
+ !!columns.find { |col| col[:name] == 'id' }
143
+ end
144
+
145
+ def count_for_pager
146
+ query = id? ? 'MAX(id)' : 'COUNT(*)'
147
+ @reader.query "SELECT #{query} FROM `#{name}`" do |res|
148
+ return res.fetch_row[0].to_i
149
+ end
150
+ end
151
+
152
+ def query_for_pager
153
+ query = id? ? 'WHERE id >= ? AND id < ?' : 'LIMIT ?,?'
154
+
155
+ cols = columns.map do |c|
156
+ if c[:type] == 'multipolygon'
157
+ "AsWKT(`#{c[:name]}`) as `#{c[:name]}`"
158
+ else
159
+ "`#{c[:name]}`"
160
+ end
161
+ end
162
+
163
+ "SELECT #{cols.join ', '} FROM `#{name}` #{query}"
164
+ end
165
+ end
166
+
167
+ def connect
168
+ @mysql = ::Mysql.connect @host, @user, @passwd, @db, @port, @sock
169
+ # utf8_unicode_ci :: https://rubydoc.info/gems/ruby-mysql/Mysql/Charset
170
+ @mysql.charset = ::Mysql::Charset.by_number 192
171
+ @mysql.query 'SET NAMES utf8'
172
+
173
+ var_info = @mysql.query "SHOW VARIABLES LIKE 'query_cache_type'"
174
+ return if var_info.nil? || var_info.first.nil? || var_info.first[1] == 'OFF'
175
+
176
+ @mysql.query 'SET SESSION query_cache_type = OFF'
177
+ end
178
+
179
+ def reconnect
180
+ @mysql.close
181
+ rescue StandardError
182
+ warn 'could not close previous mysql connection'
183
+ ensure
184
+ connect
185
+ end
186
+
187
+ def query(*args, &block)
188
+ mysql.query(*args, &block)
189
+ rescue Mysql::Error => e
190
+ if e.message.match?(/gone away/i)
191
+ reconnect
192
+ retry
193
+ else
194
+ puts "MySQL Query failed '#{args.inspect}' #{e.inspect}"
195
+ puts e.backtrace[0, 5].join("\n")
196
+ []
197
+ end
198
+ end
199
+
200
+ def initialize(options)
201
+ @host = options[:mysql][:hostname]
202
+ @user = options[:mysql][:username]
203
+ @passwd = options[:mysql][:password]
204
+ @db = options[:mysql][:database]
205
+ @port = options[:mysql][:port] || 3306
206
+ @sock = options[:mysql][:socket] && !options[:mysql][:socket].empty? ? options[:mysql][:socket] : nil
207
+ @sock = options[:mysql][:flag] && !options[:mysql][:flag].empty? ? options[:mysql][:flag] : nil
208
+
209
+ connect
210
+ end
211
+
212
+ attr_reader :mysql
213
+
214
+ def tables
215
+ @tables ||= @mysql.query('SHOW TABLES').map { |row| Table.new(self, row.first) }
216
+ end
217
+
218
+ def paginated_read(table, page_size)
219
+ count = table.count_for_pager
220
+ return if count < 1
221
+
222
+ statement = @mysql.prepare table.query_for_pager
223
+ counter = 0
224
+ 0.upto (count + page_size) / page_size do |i|
225
+ statement.execute(i * page_size, table.id? ? (i + 1) * page_size : page_size)
226
+ while (row = statement.fetch)
227
+ counter += 1
228
+ yield row, counter
229
+ end
230
+ end
231
+ counter
232
+ end
233
+ end
234
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'mysql2psql/postgres_writer'
4
+ require 'mysql2psql/connection'
5
+
6
+ class Mysql2psql
7
+ class PostgresDbWriter < PostgresFileWriter
8
+ attr_reader :connection, :filename
9
+
10
+ def initialize(filename, options)
11
+ # NOTE: the superclass opens and truncates filename for writing
12
+ super filename
13
+
14
+ @filename = filename
15
+ @connection = Connection.new options
16
+ end
17
+
18
+ def inload(path = filename)
19
+ connection.load_file path
20
+ end
21
+
22
+ def clear_schema
23
+ connection.clear_schema
24
+ end
25
+ end
26
+ end