sqlite2mysql 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/sqlite2mysql +2 -75
- data/lib/sqlite2mysql.rb +37 -1
- data/lib/sqlite2mysql/services/arguments.rb +71 -0
- data/lib/sqlite2mysql/services/bound_finder.rb +23 -0
- data/lib/sqlite2mysql/services/mysql.rb +64 -0
- data/lib/sqlite2mysql/services/sqlite.rb +58 -0
- data/lib/sqlite2mysql/services/type_inferrer.rb +85 -0
- data/lib/sqlite2mysql/version.rb +1 -1
- data/readme.md +29 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4459055881f5d71cbb6f70ba64ac6370f414b6b9
|
4
|
+
data.tar.gz: c7b60bb262fa636a7d53efd98a3d54bf9bd13f88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4a0b2dbf1af68fe5c2e52029e54f9c71d1133d25808401a83764fddde1b6d2490781affc9a56592b349b1e0471432c9524e540b49f305d18a601bb8e1e548616
|
7
|
+
data.tar.gz: 120080c429c5d492778e2a8d7b587e5f5762c90dcf578c273845c2856155aff7239502c90293567ef0ac66de3740c897d37047e7ad3859f5061794c2f99b6a10
|
data/exe/sqlite2mysql
CHANGED
@@ -1,78 +1,5 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require '
|
4
|
-
require 'sqlite3'
|
3
|
+
require 'sqlite2mysql'
|
5
4
|
|
6
|
-
|
7
|
-
|
8
|
-
DATABASE = ARGV.first
|
9
|
-
SQL_DB_NAME = ARGV[1] || DATABASE.gsub(/[^0-9a-z]/i, '')
|
10
|
-
|
11
|
-
puts 'Collecting Sqlit3 Info' # ===============================================
|
12
|
-
|
13
|
-
db = SQLite3::Database.new DATABASE
|
14
|
-
|
15
|
-
schema = {}
|
16
|
-
|
17
|
-
tables = db.execute 'SELECT name FROM sqlite_master WHERE type="table"'
|
18
|
-
|
19
|
-
tables.flatten.each do |t|
|
20
|
-
columns = db.execute("pragma table_info(#{t})")
|
21
|
-
|
22
|
-
formatted_columns = []
|
23
|
-
columns.each do |col|
|
24
|
-
formatted_columns << { name: col[1],
|
25
|
-
type: col[2],
|
26
|
-
notnull: col[3],
|
27
|
-
default: col[4] }
|
28
|
-
end
|
29
|
-
|
30
|
-
schema[t] = formatted_columns
|
31
|
-
end
|
32
|
-
|
33
|
-
puts "Creating MySQL DB: #{SQL_DB_NAME}" # ====================================
|
34
|
-
|
35
|
-
RESERVED_WORDS = %w(key int)
|
36
|
-
|
37
|
-
def create_table_query(table, columns)
|
38
|
-
query = "CREATE TABLE #{table} ("
|
39
|
-
cols = []
|
40
|
-
columns.each do |col|
|
41
|
-
col[:name] += '_1' if RESERVED_WORDS.include?(col[:name])
|
42
|
-
if col[:type] == ''
|
43
|
-
col[:type] = 'varchar(255)'
|
44
|
-
elsif col[:type].start_with?('float')
|
45
|
-
col[:type] = 'float'
|
46
|
-
end
|
47
|
-
cols << "#{col[:name]} #{col[:type]} #{'NOT NULL' if col[:notnull]}"
|
48
|
-
end
|
49
|
-
query + "#{cols.join(', ')})"
|
50
|
-
end
|
51
|
-
|
52
|
-
client = Mysql2::Client.new(host: 'localhost', username: 'root')
|
53
|
-
|
54
|
-
client.query("DROP DATABASE IF EXISTS #{SQL_DB_NAME}")
|
55
|
-
client.query("CREATE DATABASE #{SQL_DB_NAME}")
|
56
|
-
client.query("USE #{SQL_DB_NAME}")
|
57
|
-
|
58
|
-
schema.keys.each do |table|
|
59
|
-
puts "Creating table: #{table}"
|
60
|
-
client.query(create_table_query(table, schema[table]))
|
61
|
-
end
|
62
|
-
|
63
|
-
print 'Grab a ☕' # ============================================================
|
64
|
-
|
65
|
-
schema.keys.each do |table|
|
66
|
-
puts "\nInserting data: #{table}"
|
67
|
-
data = db.execute("select * from #{table}")
|
68
|
-
data.each_slice(1000) do |slice|
|
69
|
-
slice.each do |row|
|
70
|
-
cleaned_row = row.map do |val|
|
71
|
-
val.is_a?(String) ? client.escape(val) : val
|
72
|
-
end
|
73
|
-
client.query("INSERT INTO #{table} VALUES (\"#{cleaned_row.join('", "')}\")")
|
74
|
-
end
|
75
|
-
print '.'
|
76
|
-
end
|
77
|
-
end
|
78
|
-
puts ''
|
5
|
+
Sqlite2Mysql.run(ARGV)
|
data/lib/sqlite2mysql.rb
CHANGED
@@ -1,3 +1,39 @@
|
|
1
1
|
require 'sqlite2mysql/version'
|
2
|
+
require 'sqlite2mysql/services/arguments'
|
3
|
+
require 'sqlite2mysql/services/bound_finder'
|
4
|
+
require 'sqlite2mysql/services/mysql'
|
5
|
+
require 'sqlite2mysql/services/sqlite'
|
6
|
+
require 'sqlite2mysql/services/type_inferrer'
|
2
7
|
|
3
|
-
|
8
|
+
class Sqlite2Mysql
|
9
|
+
class << self
|
10
|
+
def run(args)
|
11
|
+
arguments = Arguments.new(args)
|
12
|
+
|
13
|
+
puts 'Collecting Sqlite3 Info'
|
14
|
+
|
15
|
+
db = SqliteClient.new(arguments.sqlite_db, infer_column_types: arguments.infer_types)
|
16
|
+
|
17
|
+
schema = db.build_schema
|
18
|
+
|
19
|
+
puts "Creating MySQL DB: #{arguments.mysql_db}"
|
20
|
+
|
21
|
+
mysql = MysqlClient.new(
|
22
|
+
host: arguments.mysql_host,
|
23
|
+
username: arguments.username,
|
24
|
+
password: arguments.password,
|
25
|
+
port: arguments.mysql_port)
|
26
|
+
mysql.recreate(arguments.mysql_db)
|
27
|
+
mysql.build_from_schema(schema)
|
28
|
+
|
29
|
+
print 'Grab a ☕'
|
30
|
+
|
31
|
+
schema.keys.each do |table|
|
32
|
+
puts "\nInserting data: #{table}"
|
33
|
+
data = db.get_data(table)
|
34
|
+
mysql.insert_table(table, data)
|
35
|
+
end
|
36
|
+
puts ''
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
class Arguments
|
2
|
+
attr_accessor :mysql_host, :username, :password, :mysql_port, :infer_types,
|
3
|
+
:mysql_db, :sqlite_db
|
4
|
+
|
5
|
+
def initialize(args)
|
6
|
+
help(args) if args.size == 0
|
7
|
+
|
8
|
+
set_defaults
|
9
|
+
unmodified_args = args.dup
|
10
|
+
unmodified_args.each do |arg|
|
11
|
+
if arg.start_with?('--')
|
12
|
+
send(arg[2..-1], args)
|
13
|
+
args.delete(arg)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
@sqlite_db = args.first
|
18
|
+
@mysql_db = args[1] || @sqlite_db.gsub(/[^0-9a-z]/i, '')
|
19
|
+
end
|
20
|
+
|
21
|
+
def infer(_)
|
22
|
+
@infer_types = true
|
23
|
+
end
|
24
|
+
|
25
|
+
def user(args)
|
26
|
+
@username = get_value_for_flag('--user', args)
|
27
|
+
end
|
28
|
+
|
29
|
+
def pass(args)
|
30
|
+
@password = get_value_for_flag('--pass', args)
|
31
|
+
end
|
32
|
+
|
33
|
+
def port(args)
|
34
|
+
@mysql_port = get_value_for_flag('--port', args)
|
35
|
+
end
|
36
|
+
|
37
|
+
def host(args)
|
38
|
+
@mysql_host = get_value_for_flag('--host', args)
|
39
|
+
end
|
40
|
+
|
41
|
+
def help(_)
|
42
|
+
puts <<-HELP
|
43
|
+
Usage:
|
44
|
+
sqlite2mysql sqlite.db [mysql_name]
|
45
|
+
|
46
|
+
Options:
|
47
|
+
--help Show this message
|
48
|
+
--infer Infer types for columns
|
49
|
+
--user MySQL username (default: root)
|
50
|
+
--host MySQL host (default: localhost)
|
51
|
+
--pass MySQL password
|
52
|
+
--port MySQL port
|
53
|
+
HELP
|
54
|
+
exit 0
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def get_value_for_flag(flag, args)
|
60
|
+
index = args.index(flag)
|
61
|
+
args.delete_at(index + 1)
|
62
|
+
end
|
63
|
+
|
64
|
+
def set_defaults
|
65
|
+
@username = 'root'
|
66
|
+
@password = nil
|
67
|
+
@host = 'localhost'
|
68
|
+
@port = nil
|
69
|
+
@infer = false
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
class BoundFinder
|
2
|
+
def initialize(client, table, column)
|
3
|
+
@client = client
|
4
|
+
@table = table
|
5
|
+
@column = column
|
6
|
+
end
|
7
|
+
|
8
|
+
def max
|
9
|
+
@client.select("MAX(#{@column})", @table)
|
10
|
+
end
|
11
|
+
|
12
|
+
def min
|
13
|
+
@client.select("MIN(#{@column})", @table)
|
14
|
+
end
|
15
|
+
|
16
|
+
def max_length
|
17
|
+
@client.select("MAX(LENGTH(#{@column}))", @table)
|
18
|
+
end
|
19
|
+
|
20
|
+
def min_length
|
21
|
+
@client.select("MIN(LENGTH(#{@column}))", @table)
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'mysql2'
|
2
|
+
|
3
|
+
class MysqlClient
|
4
|
+
def initialize(*args)
|
5
|
+
@client = Mysql2::Client.new(*args)
|
6
|
+
end
|
7
|
+
|
8
|
+
def recreate(name)
|
9
|
+
@client.query("DROP DATABASE IF EXISTS #{name}")
|
10
|
+
@client.query("CREATE DATABASE #{name}")
|
11
|
+
@client.query("USE #{name}")
|
12
|
+
end
|
13
|
+
|
14
|
+
def build_from_schema(schema)
|
15
|
+
schema.keys.each do |table|
|
16
|
+
puts "Creating table: #{table}"
|
17
|
+
create_table(table, schema[table])
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_table(table, fields)
|
22
|
+
puts create_table_query(table, fields)
|
23
|
+
@client.query(create_table_query(table, fields))
|
24
|
+
end
|
25
|
+
|
26
|
+
def insert_table(table, data)
|
27
|
+
data.each_slice(1000) do |slice|
|
28
|
+
@client.query(chunk_sql(table, slice))
|
29
|
+
print '.'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def chunk_sql(table, chunk)
|
36
|
+
values = []
|
37
|
+
chunk.each do |row|
|
38
|
+
values << "#{row_sql(row)}"
|
39
|
+
end
|
40
|
+
"INSERT INTO #{table} VALUES #{values.join(', ')}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def row_sql(row)
|
44
|
+
values = row.map do |val|
|
45
|
+
if val.is_a?(String)
|
46
|
+
(val.empty? || val.nil? || val == '') ? nil : @client.escape(val)
|
47
|
+
else
|
48
|
+
val
|
49
|
+
end
|
50
|
+
end
|
51
|
+
"(\"#{values.join('", "')}\")"
|
52
|
+
end
|
53
|
+
|
54
|
+
def create_table_query(table, fields)
|
55
|
+
reserved_words = %w(key int)
|
56
|
+
query = "CREATE TABLE #{table} ("
|
57
|
+
cols = []
|
58
|
+
fields.each do |col|
|
59
|
+
col[:name] += '_1' if reserved_words.include?(col[:name])
|
60
|
+
cols << "#{col[:name]} #{col[:type]} #{'NOT NULL' if col[:notnull]}"
|
61
|
+
end
|
62
|
+
query + "#{cols.join(', ')})"
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'sqlite3'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
class SqliteClient
|
5
|
+
def initialize(filename, infer_column_types: false)
|
6
|
+
@db = SQLite3::Database.new(filename)
|
7
|
+
@infer = infer_column_types
|
8
|
+
end
|
9
|
+
|
10
|
+
def build_schema
|
11
|
+
schema = {}
|
12
|
+
tables = @db.execute 'SELECT name FROM sqlite_master WHERE type="table"'
|
13
|
+
|
14
|
+
tables.flatten.each do |t|
|
15
|
+
schema[t] = column_formatter(t)
|
16
|
+
end
|
17
|
+
|
18
|
+
schema
|
19
|
+
end
|
20
|
+
|
21
|
+
def get_data(table)
|
22
|
+
@db.execute("select * from #{table}")
|
23
|
+
end
|
24
|
+
|
25
|
+
def column_formatter(table)
|
26
|
+
columns = @db.execute("pragma table_info(#{table})")
|
27
|
+
|
28
|
+
formatted_columns = []
|
29
|
+
columns.each do |col|
|
30
|
+
formatted_columns << { name: col[1],
|
31
|
+
type: type_getter(col[2], table, col[1]),
|
32
|
+
notnull: col[3],
|
33
|
+
default: col[4] }
|
34
|
+
end
|
35
|
+
formatted_columns
|
36
|
+
end
|
37
|
+
|
38
|
+
def type_getter(type, table, column)
|
39
|
+
if @infer
|
40
|
+
samples = @db.execute("SELECT #{column} FROM #{table} WHERE #{column} IS NOT NULL AND #{column} != '' ORDER BY RANDOM() LIMIT 100").flatten
|
41
|
+
type = TypeInferrer.new(samples, BoundFinder.new(self, table, column)).make_inference
|
42
|
+
puts "Inferring type of #{column} as #{type}"
|
43
|
+
return type
|
44
|
+
else
|
45
|
+
if type == '' || type.nil?
|
46
|
+
return 'varchar(255)'
|
47
|
+
elsif type.start_with?('float')
|
48
|
+
return 'float'
|
49
|
+
else
|
50
|
+
return type
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def select(column, table)
|
56
|
+
@db.execute("SELECT #{column} FROM #{table}").flatten.first
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
class TypeInferrer
|
2
|
+
def initialize(samples, bound_finder)
|
3
|
+
@samples = samples
|
4
|
+
@bound_finder = bound_finder
|
5
|
+
end
|
6
|
+
|
7
|
+
def make_inference
|
8
|
+
possibilities = weigh_possibilities
|
9
|
+
|
10
|
+
case possibilities.max_by(&:last).first
|
11
|
+
when :int
|
12
|
+
return get_integer_type
|
13
|
+
when :float
|
14
|
+
return 'FLOAT'
|
15
|
+
when :date
|
16
|
+
return 'DATE'
|
17
|
+
when :datetime
|
18
|
+
return 'DATETIME'
|
19
|
+
when :string
|
20
|
+
return get_varchar_type
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def weigh_possibilities
|
25
|
+
{
|
26
|
+
float: 0,
|
27
|
+
int: 0,
|
28
|
+
date: 0,
|
29
|
+
datetime: 0,
|
30
|
+
string: 0
|
31
|
+
}.tap do |possibilities|
|
32
|
+
@samples.each do |sample|
|
33
|
+
if date_or_time?(sample)
|
34
|
+
if sample.is_a?(Date) || Date.parse(sample).to_time == Time.parse(sample)
|
35
|
+
possibilities[:date] += 1
|
36
|
+
else
|
37
|
+
possibilities[:datetime] += 1
|
38
|
+
end
|
39
|
+
elsif sample.is_a?(Float) || sample.to_i > 0 && sample.to_f != sample.to_i.to_f
|
40
|
+
possibilities[:float] += 1
|
41
|
+
elsif sample.is_a?(Integer) || sample.to_i > 0 || sample == '0'
|
42
|
+
possibilities[:int] += 1
|
43
|
+
else
|
44
|
+
possibilities[:string] += 1
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def get_integer_type
|
51
|
+
max = @bound_finder.max.to_i
|
52
|
+
min = @bound_finder.min.to_i
|
53
|
+
if min > -128 && max < 127
|
54
|
+
'TINYINT'
|
55
|
+
elsif min > -32768 && max < 32767
|
56
|
+
'SMALLINT'
|
57
|
+
elsif min > -8388608 && max < 8388607
|
58
|
+
'MEDIUMINT'
|
59
|
+
elsif min > -2147483648 && max < 2147483647
|
60
|
+
'INT'
|
61
|
+
else
|
62
|
+
'BIGINT'
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def get_varchar_type
|
67
|
+
max_length = @bound_finder.max_length
|
68
|
+
max_length = 1 if max_length == 0 || max_length.nil?
|
69
|
+
|
70
|
+
return 'TEXT' if max_length > 255
|
71
|
+
|
72
|
+
"VARCHAR(#{max_length})"
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
def date_or_time?(sample)
|
78
|
+
sample.is_a?(Date) ||
|
79
|
+
sample.is_a?(Time) ||
|
80
|
+
sample.is_a?(String) &&
|
81
|
+
(%r((\d{1,2}[-\/]\d{1,2}[-\/]\d{4})|(\d{4}[-\/]\d{1,2}[-\/]\d{1,2})).match(sample) &&
|
82
|
+
Date.parse(sample) rescue false &&
|
83
|
+
Time.parse(sample) rescue false)
|
84
|
+
end
|
85
|
+
end
|
data/lib/sqlite2mysql/version.rb
CHANGED
data/readme.md
CHANGED
@@ -0,0 +1,29 @@
|
|
1
|
+
# sqlite2mysql
|
2
|
+
|
3
|
+
### Installation
|
4
|
+
|
5
|
+
gem install sqlite2mysql
|
6
|
+
|
7
|
+
Don't include it in your projects, that's not what it's for. This is a command line tool.
|
8
|
+
|
9
|
+
### Usage
|
10
|
+
|
11
|
+
Run like this:
|
12
|
+
|
13
|
+
sqlite2mysql test.db
|
14
|
+
|
15
|
+
This will create a database called testdb in mysql with the exact schema and data as was in `test.db`. You can optionally name it something else in mysql like this:
|
16
|
+
|
17
|
+
sqlite2mysql test.db my_awesome_database
|
18
|
+
|
19
|
+
Isn't that handy?
|
20
|
+
|
21
|
+
This assumes you can log in as root to your localhost mysql database without a password.
|
22
|
+
|
23
|
+
### Contributing
|
24
|
+
|
25
|
+
Do.
|
26
|
+
|
27
|
+
### License
|
28
|
+
|
29
|
+
MIT.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqlite2mysql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexander Standke
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -100,6 +100,11 @@ files:
|
|
100
100
|
- bin/setup
|
101
101
|
- exe/sqlite2mysql
|
102
102
|
- lib/sqlite2mysql.rb
|
103
|
+
- lib/sqlite2mysql/services/arguments.rb
|
104
|
+
- lib/sqlite2mysql/services/bound_finder.rb
|
105
|
+
- lib/sqlite2mysql/services/mysql.rb
|
106
|
+
- lib/sqlite2mysql/services/sqlite.rb
|
107
|
+
- lib/sqlite2mysql/services/type_inferrer.rb
|
103
108
|
- lib/sqlite2mysql/version.rb
|
104
109
|
- readme.md
|
105
110
|
- sqlite2mysql.gemspec
|
@@ -128,3 +133,4 @@ signing_key:
|
|
128
133
|
specification_version: 4
|
129
134
|
summary: Simple tool to convert sqlite3 to mysql
|
130
135
|
test_files: []
|
136
|
+
has_rdoc:
|