fast_innodb_import 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/fast_innodb_import +1 -1
- data/lib/fast_innodb_import.rb +84 -63
- data/spec/fast_innodb_import_spec.rb +43 -4
- metadata +3 -3
data/bin/fast_innodb_import
CHANGED
data/lib/fast_innodb_import.rb
CHANGED
@@ -3,36 +3,27 @@ $:.unshift(File.dirname(__FILE__)) unless
|
|
3
3
|
|
4
4
|
require "mysql2"
|
5
5
|
class FastInnodbImport
|
6
|
-
VERSION = '0.0.
|
6
|
+
VERSION = '0.0.2'
|
7
7
|
CMD_MAPPING = { "u" => :username, "h" => :host, "d" => :database, "p" => "password" }
|
8
8
|
|
9
|
-
|
10
|
-
|
9
|
+
attr_accessor :options, :stream
|
10
|
+
|
11
|
+
def initialize(options = {})
|
12
|
+
self.options = options
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.execute(argv, stream = nil)
|
16
|
+
importer = self.new(options_from_argv(argv))
|
17
|
+
importer.stream = stream
|
18
|
+
importer.execute
|
11
19
|
end
|
12
20
|
|
13
21
|
def execute
|
14
|
-
|
22
|
+
print_usage_and_exit_if_missing_options
|
15
23
|
file_paths.each do |path|
|
16
24
|
import_data_from_file_path(path)
|
17
25
|
end
|
18
26
|
end
|
19
|
-
|
20
|
-
private
|
21
|
-
def index_definitions_from_table(table)
|
22
|
-
query("SHOW CREATE TABLE #{table}").to_a.first["Create Table"].split("\n").map do |line|
|
23
|
-
if line.match(/^\s*KEY \`(.*?)\` \((.*?)\)/)
|
24
|
-
{ :name => $1, :columns => $2 }
|
25
|
-
end
|
26
|
-
end.compact
|
27
|
-
end
|
28
|
-
|
29
|
-
def index_drop_statement_for_table(table_name)
|
30
|
-
"ALTER TABLE #{table_name} " + index_definitions_from_table(table_name).map { |config| "\nDROP INDEX #{config[:name]}" }.join(", ")
|
31
|
-
end
|
32
|
-
|
33
|
-
def index_create_statement_for_table(table_name)
|
34
|
-
"ALTER TABLE #{table_name} " + index_definitions_from_table(table_name).map { |config| "\nADD INDEX #{config[:name]} (#{config[:columns]})" }.join(", ")
|
35
|
-
end
|
36
27
|
|
37
28
|
def import_data_from_file_path(path)
|
38
29
|
file_path = File.expand_path(path)
|
@@ -40,54 +31,57 @@ private
|
|
40
31
|
table_name = table_name_from_file_path(file_path)
|
41
32
|
wc = `wc -l #{file_path}`.strip.to_i
|
42
33
|
started = Time.now
|
43
|
-
|
44
|
-
|
34
|
+
if stream
|
35
|
+
stream.print "import %d rows from %s: " % [wc, file_path] if stream
|
36
|
+
stream.flush
|
37
|
+
end
|
45
38
|
|
46
39
|
# create drop and create statements as long as we can
|
47
40
|
drop_statement = index_drop_statement_for_table(table_name)
|
48
41
|
create_statement = index_create_statement_for_table(table_name)
|
49
42
|
|
50
43
|
query("TRUNCATE `#{table_name}`")
|
44
|
+
query(drop_statement) if drop_keys?
|
45
|
+
`#{self.class.import_command_prefix_from_options(options)} #{file_path}`
|
46
|
+
query(create_statement) if drop_keys?
|
51
47
|
|
52
|
-
#
|
53
|
-
if
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
query("LOAD DATA INFILE '#{file_path}' INTO TABLE `#{table_name}`")
|
58
|
-
|
59
|
-
# recreate keys if asked
|
60
|
-
if drop_keys?
|
61
|
-
query(create_statement)
|
48
|
+
# print stats if stream given
|
49
|
+
if stream
|
50
|
+
diff = Time.now - started
|
51
|
+
per_sec = wc / diff
|
52
|
+
stream.puts "%d (%d/sec)" % [diff, per_sec]
|
62
53
|
end
|
63
|
-
|
64
|
-
# print stats
|
65
|
-
diff = Time.now - started
|
66
|
-
per_sec = wc / diff
|
67
|
-
puts "%d (%d/sec)" % [diff, per_sec]
|
68
54
|
end
|
69
55
|
|
70
|
-
def
|
71
|
-
|
56
|
+
def options
|
57
|
+
@options || {}
|
72
58
|
end
|
73
59
|
|
74
|
-
def
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
60
|
+
def self.options_from_argv(argv)
|
61
|
+
options = { :db => {} }
|
62
|
+
%w(u h d p).each do |key|
|
63
|
+
if idx = argv.index("-#{key}")
|
64
|
+
value = argv.at(idx + 1)
|
65
|
+
options[:db][CMD_MAPPING[key]] = value if !value.match(/^\-/)
|
66
|
+
argv.delete_at(idx)
|
67
|
+
argv.delete_at(idx)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
options[:drop_keys] = argv.delete("--without-drop-keys").nil?
|
71
|
+
options[:file_paths] = argv
|
72
|
+
options
|
80
73
|
end
|
81
|
-
|
82
|
-
|
83
|
-
|
74
|
+
|
75
|
+
private
|
76
|
+
def print_usage_and_exit_if_missing_options
|
77
|
+
self.class.print_usage_and_exit if missing_options?
|
84
78
|
end
|
85
79
|
|
86
|
-
def
|
87
|
-
|
80
|
+
def missing_options?
|
81
|
+
!file_paths.respond_to?(:empty?) || file_paths.empty? || db_options[:username].nil? || db_options[:database].nil?
|
88
82
|
end
|
89
83
|
|
90
|
-
def print_usage_and_exit
|
84
|
+
def self.print_usage_and_exit
|
91
85
|
puts "Usage: #{File.basename(__FILE__)} [OPTIONS] [dumpfile]"
|
92
86
|
puts " -h Database Host"
|
93
87
|
puts " -u Database User"
|
@@ -97,19 +91,46 @@ private
|
|
97
91
|
exit
|
98
92
|
end
|
99
93
|
|
100
|
-
def
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
value = argv.at(idx + 1)
|
105
|
-
options[:db][CMD_MAPPING[key]] = value if !value.match(/^\-/)
|
106
|
-
argv.delete_at(idx)
|
107
|
-
argv.delete_at(idx)
|
94
|
+
def index_definitions_from_table(table)
|
95
|
+
query("SHOW CREATE TABLE #{table}").to_a.first["Create Table"].split("\n").map do |line|
|
96
|
+
if line.match(/^\s*KEY \`(.*?)\` \((.*?)\)/)
|
97
|
+
{ :name => $1, :columns => $2 }
|
108
98
|
end
|
99
|
+
end.compact
|
100
|
+
end
|
101
|
+
|
102
|
+
def index_drop_statement_for_table(table_name)
|
103
|
+
"ALTER TABLE #{table_name} " + index_definitions_from_table(table_name).map { |config| "\nDROP INDEX #{config[:name]}" }.join(", ")
|
104
|
+
end
|
105
|
+
|
106
|
+
def index_create_statement_for_table(table_name)
|
107
|
+
"ALTER TABLE #{table_name} " + index_definitions_from_table(table_name).map { |config| "\nADD INDEX #{config[:name]} (#{config[:columns]})" }.join(", ")
|
108
|
+
end
|
109
|
+
|
110
|
+
def db_options
|
111
|
+
options[:db] || {}
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.import_command_prefix_from_options(options)
|
115
|
+
cmd = "mysqlimport -L"
|
116
|
+
if db_options = options[:db]
|
117
|
+
cmd << " -u #{db_options[:username]}" if db_options[:username]
|
118
|
+
cmd << " -p#{db_options[:password]}" if db_options[:password]
|
119
|
+
cmd << " #{db_options[:database]}" if db_options[:database]
|
109
120
|
end
|
110
|
-
|
111
|
-
|
112
|
-
|
121
|
+
cmd
|
122
|
+
end
|
123
|
+
|
124
|
+
def table_name_from_file_path(path)
|
125
|
+
File.basename(path).gsub(".txt", "")
|
126
|
+
end
|
127
|
+
|
128
|
+
def file_paths
|
129
|
+
options[:file_paths]
|
130
|
+
end
|
131
|
+
|
132
|
+
def drop_keys?
|
133
|
+
options[:drop_keys]
|
113
134
|
end
|
114
135
|
|
115
136
|
def client
|
@@ -1,11 +1,50 @@
|
|
1
|
-
require File.dirname(__FILE__) + '
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper.rb'
|
2
|
+
require "fileutils"
|
2
3
|
|
3
4
|
# Time to add your specs!
|
4
5
|
# http://rspec.info/
|
5
6
|
describe "Place your specs here" do
|
6
|
-
|
7
|
-
|
8
|
-
|
7
|
+
{ { :username => "root" } => "-u root",
|
8
|
+
{ :username => "root", :database => "some_db" } => "-u root some_db",
|
9
|
+
{ :username => "root", :password => "secret", :database => "some_db" } => "-u root -psecret some_db",
|
10
|
+
}.each do |db_options, string|
|
11
|
+
it "should constract the correct mysqlimport command prefix for db_options #{db_options.inspect}" do
|
12
|
+
FastInnodbImport.send(:import_command_prefix_from_options, { :db => db_options }).should == "mysqlimport -L #{string}"
|
13
|
+
end
|
9
14
|
end
|
10
15
|
|
16
|
+
describe "import from a file" do
|
17
|
+
before(:each) do
|
18
|
+
@database = "fast_innodb_import_test"
|
19
|
+
@username = "root"
|
20
|
+
@client = Mysql2::Client.new(:database => @database, :username => @username)
|
21
|
+
@client.query("TRUNCATE albums")
|
22
|
+
violated if @client.query("SELECT * FROM albums").to_a.length != 0
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should import a dummy file" do
|
26
|
+
importer = FastInnodbImport.new(:db => { :username => @username, :database => @database })
|
27
|
+
importer.import_data_from_file_path("spec/fixtures/albums.txt")
|
28
|
+
@client.query("SELECT * FROM albums ORDER BY id").to_a.should == [
|
29
|
+
{ "id" => 1, "artist_name" => "Jay-Z", "title" => "Black Album" },
|
30
|
+
{ "id" => 2, "artist_name" => "Mos Def", "title" => "Black on Both Sides" },
|
31
|
+
]
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should write log to stream" do
|
35
|
+
FileUtils.rm_f("tmp/import.log")
|
36
|
+
stream = File.open("tmp/import.log", "w")
|
37
|
+
importer = FastInnodbImport.new(:db => { :username => @username, :database => @database })
|
38
|
+
importer.stream = stream
|
39
|
+
importer.import_data_from_file_path("spec/fixtures/albums.txt")
|
40
|
+
stream.close
|
41
|
+
File.read("tmp/import.log").should match(/import 2 rows .*?\/spec\/fixtures\/albums\.txt: \d+ \(\d+\/sec\)/)
|
42
|
+
FileUtils.rm_f("tmp/import.log")
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should import data when called with ARGV options" do
|
46
|
+
FastInnodbImport.execute(["-u", @username, "-d", @database, "spec/fixtures/albums.txt"])
|
47
|
+
@client.query("SELECT * FROM albums").to_a.length.should == 2
|
48
|
+
end
|
49
|
+
end
|
11
50
|
end
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: 0.0.
|
8
|
+
- 2
|
9
|
+
version: 0.0.2
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Tobias Schwab
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-09-
|
17
|
+
date: 2010-09-23 00:00:00 +02:00
|
18
18
|
default_executable: fast_innodb_import
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|