large-hadron-migrator 0.1.4 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +5 -0
- data/CHANGELOG.md +22 -0
- data/README.md +1 -8
- data/Rakefile +3 -1
- data/VERSION +1 -1
- data/large-hadron-migrator.gemspec +6 -6
- data/lib/large-hadron-migrator.rb +26 -7
- data/spec/{large_hadron_migration_spec.rb → large_hadron_migrator_spec.rb} +96 -86
- data/spec/migrate/add_new_column.rb +2 -2
- data/spec/spec_helper.rb +10 -8
- metadata +64 -7
- data/CHANGES.md +0 -12
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# 0.2.0 (November 26, 2011)
|
2
|
+
|
3
|
+
* Add Ruby 1.8 compatibility
|
4
|
+
* Setup travis continuous integration
|
5
|
+
* Fix record lose issue
|
6
|
+
* Fix and speed up specs
|
7
|
+
|
8
|
+
# 0.1.4
|
9
|
+
|
10
|
+
* Merged [Pullrequest #9](https://github.com/soundcloud/large-hadron-migrator/pull/9)
|
11
|
+
|
12
|
+
# 0.1.3
|
13
|
+
|
14
|
+
* code cleanup
|
15
|
+
* Merged [Pullrequest #8](https://github.com/soundcloud/large-hadron-migrator/pull/8)
|
16
|
+
* Merged [Pullrequest #7](https://github.com/soundcloud/large-hadron-migrator/pull/7)
|
17
|
+
* Merged [Pullrequest #4](https://github.com/soundcloud/large-hadron-migrator/pull/4)
|
18
|
+
* Merged [Pullrequest #1](https://github.com/soundcloud/large-hadron-migrator/pull/1)
|
19
|
+
|
20
|
+
# 0.1.2
|
21
|
+
|
22
|
+
* Initial Release
|
data/README.md
CHANGED
@@ -163,7 +163,7 @@ site. We rolled out the fix and restarted the app servers in mid migration.
|
|
163
163
|
This was not a good idea.
|
164
164
|
|
165
165
|
TL;DR: Never restart during migrations when removing columns with LHM.
|
166
|
-
You can restart while adding migrations as long as active record reads column
|
166
|
+
You can restart while adding migrations as long as active record reads column
|
167
167
|
definitions from the slave.
|
168
168
|
|
169
169
|
The information below is only relevant if you want to restart your app servers
|
@@ -199,9 +199,6 @@ while migrating in a master slave setup.
|
|
199
199
|
|
200
200
|
## Todos
|
201
201
|
|
202
|
-
~~Load data into outfile instead of `INSERT INTO... SELECT`. Avoid contention and
|
203
|
-
increase speed.~~ It showed that this wont result in major speed up.
|
204
|
-
|
205
202
|
Handle invalidation of 'update' entries in journal while replaying. Avoid
|
206
203
|
stale update replays.
|
207
204
|
|
@@ -214,10 +211,6 @@ problems with this, but the chunked copy could be sped up by factoring this
|
|
214
211
|
in. Currently a copy range may be completely empty, but there will still be
|
215
212
|
a `INSERT INTO... SELECT`.
|
216
213
|
|
217
|
-
Records inserted after the last insert id is retrieved and before the triggers
|
218
|
-
are created are currently lost. The table should be briefly locked while id is
|
219
|
-
read and triggers are applied.
|
220
|
-
|
221
214
|
## Contributing
|
222
215
|
|
223
216
|
We'll check out your contribution if you:
|
data/Rakefile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -14,11 +14,11 @@ Gem::Specification.new do |s|
|
|
14
14
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
15
15
|
s.require_paths = ["lib"]
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
end
|
17
|
+
s.add_dependency "activerecord", "~> 2.3.8"
|
18
|
+
s.add_dependency "activesupport", "~> 2.3.8"
|
20
19
|
|
21
|
-
|
22
|
-
|
23
|
-
|
20
|
+
# this should be a real dependency, but we're using a different gem in our code
|
21
|
+
s.add_development_dependency "mysql", "~> 2.8.1"
|
22
|
+
s.add_development_dependency "rspec", "=1.3.1"
|
23
|
+
s.add_development_dependency "rake"
|
24
24
|
end
|
@@ -100,14 +100,11 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
100
100
|
|
101
101
|
raise "chunk_size must be >= 1" unless chunk_size >= 1
|
102
102
|
|
103
|
-
started = Time.now
|
103
|
+
started = format_time(Time.now)
|
104
104
|
new_table = "lhmn_%s" % curr_table
|
105
105
|
old_table = "lhmo_%s_%s" % [started, curr_table]
|
106
106
|
journal_table = "lhmj_%s_%s" % [started, curr_table]
|
107
107
|
|
108
|
-
last_insert_id = last_insert_id(curr_table)
|
109
|
-
say "last inserted id in #{curr_table}: #{last_insert_id}"
|
110
|
-
|
111
108
|
begin
|
112
109
|
# clean tables. old tables are never deleted to guard against rollbacks.
|
113
110
|
execute %Q{drop table if exists %s} % new_table
|
@@ -120,6 +117,9 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
120
117
|
add_trigger_on_action(curr_table, journal_table, "update")
|
121
118
|
add_trigger_on_action(curr_table, journal_table, "delete")
|
122
119
|
|
120
|
+
last_insert_id = last_smaller_insert_id(curr_table, journal_table)
|
121
|
+
say "last inserted id in #{curr_table}: #{last_insert_id}"
|
122
|
+
|
123
123
|
# alter new table
|
124
124
|
default_values = {}
|
125
125
|
yield new_table, default_values
|
@@ -135,7 +135,7 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
135
135
|
curr_table,
|
136
136
|
wait
|
137
137
|
|
138
|
-
rename_tables curr_table
|
138
|
+
rename_tables [[ curr_table, old_table ], [ new_table, curr_table ]]
|
139
139
|
cleanup(curr_table)
|
140
140
|
|
141
141
|
# replay changes from the changes jornal
|
@@ -226,15 +226,26 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
226
226
|
end
|
227
227
|
end
|
228
228
|
|
229
|
+
# to satisfy spec "does not lose records during table clone step"
|
230
|
+
def self.last_smaller_insert_id(curr_table, journal_table)
|
231
|
+
with_master do
|
232
|
+
select_value(%{
|
233
|
+
SELECT max(#{curr_table}.id) as max_id from #{curr_table}
|
234
|
+
LEFT OUTER JOIN #{journal_table} ON #{journal_table}.id = #{curr_table}.id
|
235
|
+
WHERE #{journal_table}.id IS NULL
|
236
|
+
} % [ curr_table, journal_table ]).to_i
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
229
240
|
def self.last_insert_id(curr_table)
|
230
241
|
with_master do
|
231
|
-
|
242
|
+
select_value("SELECT max(id) FROM #{curr_table}").to_i
|
232
243
|
end
|
233
244
|
end
|
234
245
|
|
235
246
|
def self.table_column_names(table_name)
|
236
247
|
with_master do
|
237
|
-
|
248
|
+
select_values %Q{
|
238
249
|
select column_name
|
239
250
|
from information_schema.columns
|
240
251
|
where table_name = "%s"
|
@@ -382,4 +393,12 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
382
393
|
def self.tick(col)
|
383
394
|
"`#{ col }`"
|
384
395
|
end
|
396
|
+
|
397
|
+
def self.format_time(time)
|
398
|
+
if RUBY_VERSION < "1.9"
|
399
|
+
time.strftime("%Y_%m_%d_%H_%M_%S_#{"%03d" % (time.usec / 1000)}")
|
400
|
+
else
|
401
|
+
time.strftime("%Y_%m_%d_%H_%M_%S_%3N")
|
402
|
+
end
|
403
|
+
end
|
385
404
|
end
|
@@ -6,13 +6,12 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
|
6
6
|
|
7
7
|
require "migrate/add_new_column"
|
8
8
|
|
9
|
-
describe "
|
9
|
+
describe "LargeHadronMigrator", "integration" do
|
10
10
|
include SpecHelper
|
11
11
|
|
12
12
|
before(:each) { recreate }
|
13
13
|
|
14
14
|
it "should add new column" do
|
15
|
-
|
16
15
|
table("addscolumn") do |t|
|
17
16
|
t.string :title
|
18
17
|
t.integer :rating
|
@@ -40,7 +39,7 @@ describe "LargeHadronMigration", "integration" do
|
|
40
39
|
t.timestamps
|
41
40
|
end
|
42
41
|
|
43
|
-
|
42
|
+
420.times do |i|
|
44
43
|
random_string = (0...rand(25)).map{65.+(rand(25)).chr}.join
|
45
44
|
sql "INSERT INTO `addscolumn` SET
|
46
45
|
`id` = #{i+1},
|
@@ -52,12 +51,57 @@ describe "LargeHadronMigration", "integration" do
|
|
52
51
|
|
53
52
|
ghost = AddNewColumn.up
|
54
53
|
|
55
|
-
truthiness_rows "addscolumn", ghost
|
54
|
+
truthiness_rows "addscolumn", ghost, 0, 420
|
56
55
|
end
|
57
|
-
end
|
58
56
|
|
57
|
+
context "trigger creation" do
|
58
|
+
before do
|
59
|
+
class LargeHadronMigrator
|
60
|
+
class << self
|
61
|
+
alias_method :orig_clone_table, :clone_table
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.clone_table(*args)
|
65
|
+
sleep 0.3
|
66
|
+
orig_clone_table(*args)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
after do
|
72
|
+
class LargeHadronMigrator
|
73
|
+
class << self
|
74
|
+
alias_method :clone_table, :orig_clone_table
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
it "does not lose records during trigger creation" do
|
80
|
+
table "addscolumn" do |t|
|
81
|
+
t.string :data
|
82
|
+
t.timestamps
|
83
|
+
end
|
84
|
+
|
85
|
+
parallel_insert = Thread.new do
|
86
|
+
20.times do |i|
|
87
|
+
sql %{
|
88
|
+
INSERT INTO addscolumn (data, created_at, updated_at)
|
89
|
+
VALUES ('#{ActiveSupport::SecureRandom.base64}', NOW(), NOW())
|
90
|
+
}
|
91
|
+
sleep 0.1
|
92
|
+
end
|
93
|
+
end
|
59
94
|
|
60
|
-
|
95
|
+
ghost = AddNewColumn.up
|
96
|
+
|
97
|
+
parallel_insert.join
|
98
|
+
|
99
|
+
select_value("SELECT count(*) FROM addscolumn").should == "20"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
describe "LargeHadronMigrator", "rename" do
|
61
105
|
include SpecHelper
|
62
106
|
|
63
107
|
before(:each) do
|
@@ -73,7 +117,7 @@ describe "LargeHadronMigration", "rename" do
|
|
73
117
|
t.integer :number
|
74
118
|
end
|
75
119
|
|
76
|
-
|
120
|
+
LargeHadronMigrator.rename_tables("renameme" => "renameme_new", "renamemetoo" => "renameme")
|
77
121
|
|
78
122
|
truthiness_column "renameme", "number", "int"
|
79
123
|
truthiness_column "renameme_new", "text", "varchar"
|
@@ -81,7 +125,7 @@ describe "LargeHadronMigration", "rename" do
|
|
81
125
|
|
82
126
|
end
|
83
127
|
|
84
|
-
describe "
|
128
|
+
describe "LargeHadronMigrator", "triggers" do
|
85
129
|
include SpecHelper
|
86
130
|
|
87
131
|
before(:each) do
|
@@ -93,7 +137,7 @@ describe "LargeHadronMigration", "triggers" do
|
|
93
137
|
t.timestamps
|
94
138
|
end
|
95
139
|
|
96
|
-
|
140
|
+
LargeHadronMigrator.clone_table_for_changes \
|
97
141
|
"triggerme",
|
98
142
|
"triggerme_changes"
|
99
143
|
end
|
@@ -104,18 +148,16 @@ describe "LargeHadronMigration", "triggers" do
|
|
104
148
|
end
|
105
149
|
|
106
150
|
it "should trigger on insert" do
|
107
|
-
|
151
|
+
LargeHadronMigrator.add_trigger_on_action \
|
108
152
|
"triggerme",
|
109
153
|
"triggerme_changes",
|
110
154
|
"insert"
|
111
155
|
|
112
156
|
# test
|
113
157
|
sql("insert into triggerme values (111, 'hallo', 5, NOW(), NOW())")
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
row['text'].should == 'hallo'
|
118
|
-
end
|
158
|
+
select_one("select * from triggerme_changes where id = 111").tap do |row|
|
159
|
+
row["hadron_action"].should == "insert"
|
160
|
+
row["text"].should == "hallo"
|
119
161
|
end
|
120
162
|
end
|
121
163
|
|
@@ -123,18 +165,16 @@ describe "LargeHadronMigration", "triggers" do
|
|
123
165
|
|
124
166
|
# setup
|
125
167
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
126
|
-
|
168
|
+
LargeHadronMigrator.add_trigger_on_action \
|
127
169
|
"triggerme",
|
128
170
|
"triggerme_changes",
|
129
171
|
"update"
|
130
172
|
|
131
173
|
# test
|
132
174
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
row['text'].should == 'goodbye'
|
137
|
-
end
|
175
|
+
select_one("select * from triggerme_changes where id = 111").tap do |row|
|
176
|
+
row["hadron_action"].should == "update"
|
177
|
+
row["text"].should == "goodbye"
|
138
178
|
end
|
139
179
|
end
|
140
180
|
|
@@ -142,28 +182,26 @@ describe "LargeHadronMigration", "triggers" do
|
|
142
182
|
|
143
183
|
# setup
|
144
184
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
145
|
-
|
185
|
+
LargeHadronMigrator.add_trigger_on_action \
|
146
186
|
"triggerme",
|
147
187
|
"triggerme_changes",
|
148
188
|
"delete"
|
149
189
|
|
150
190
|
# test
|
151
191
|
sql("delete from triggerme where id = '111'")
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
row['text'].should == 'hallo'
|
156
|
-
end
|
192
|
+
select_one("select * from triggerme_changes where id = 111").tap do |row|
|
193
|
+
row["hadron_action"].should == "delete"
|
194
|
+
row["text"].should == "hallo"
|
157
195
|
end
|
158
196
|
end
|
159
197
|
|
160
198
|
it "should trigger on create and update" do
|
161
|
-
|
199
|
+
LargeHadronMigrator.add_trigger_on_action \
|
162
200
|
"triggerme",
|
163
201
|
"triggerme_changes",
|
164
202
|
"insert"
|
165
203
|
|
166
|
-
|
204
|
+
LargeHadronMigrator.add_trigger_on_action \
|
167
205
|
"triggerme",
|
168
206
|
"triggerme_changes",
|
169
207
|
"update"
|
@@ -172,16 +210,12 @@ describe "LargeHadronMigration", "triggers" do
|
|
172
210
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
173
211
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
174
212
|
|
175
|
-
|
176
|
-
res.fetch_hash.tap do |row|
|
177
|
-
row['cnt'].should == '1'
|
178
|
-
end
|
179
|
-
end
|
213
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "1"
|
180
214
|
end
|
181
215
|
|
182
216
|
it "should trigger on multiple update" do
|
183
217
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
184
|
-
|
218
|
+
LargeHadronMigrator.add_trigger_on_action \
|
185
219
|
"triggerme",
|
186
220
|
"triggerme_changes",
|
187
221
|
"update"
|
@@ -190,25 +224,21 @@ describe "LargeHadronMigration", "triggers" do
|
|
190
224
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
191
225
|
sql("update triggerme set text = 'hallo again' where id = '111'")
|
192
226
|
|
193
|
-
|
194
|
-
res.fetch_hash.tap do |row|
|
195
|
-
row['cnt'].should == '1'
|
196
|
-
end
|
197
|
-
end
|
227
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "1"
|
198
228
|
end
|
199
229
|
|
200
230
|
it "should trigger on inser, update and delete" do
|
201
|
-
|
231
|
+
LargeHadronMigrator.add_trigger_on_action \
|
202
232
|
"triggerme",
|
203
233
|
"triggerme_changes",
|
204
234
|
"insert"
|
205
235
|
|
206
|
-
|
236
|
+
LargeHadronMigrator.add_trigger_on_action \
|
207
237
|
"triggerme",
|
208
238
|
"triggerme_changes",
|
209
239
|
"update"
|
210
240
|
|
211
|
-
|
241
|
+
LargeHadronMigrator.add_trigger_on_action \
|
212
242
|
"triggerme",
|
213
243
|
"triggerme_changes",
|
214
244
|
"delete"
|
@@ -218,38 +248,30 @@ describe "LargeHadronMigration", "triggers" do
|
|
218
248
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
219
249
|
sql("delete from triggerme where id = '111'")
|
220
250
|
|
221
|
-
|
222
|
-
res.fetch_hash.tap do |row|
|
223
|
-
row['cnt'].should == '1'
|
224
|
-
end
|
225
|
-
end
|
251
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "1"
|
226
252
|
end
|
227
253
|
|
228
254
|
it "should cleanup triggers" do
|
229
255
|
%w(insert update delete).each do |action|
|
230
|
-
|
256
|
+
LargeHadronMigrator.add_trigger_on_action \
|
231
257
|
"triggerme",
|
232
258
|
"triggerme_changes",
|
233
259
|
action
|
234
260
|
end
|
235
261
|
|
236
|
-
|
262
|
+
LargeHadronMigrator.cleanup "triggerme"
|
237
263
|
|
238
264
|
# test
|
239
265
|
sql("insert into triggerme values (111, 'hallo', 5, NOW(), NOW())")
|
240
266
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
241
267
|
sql("delete from triggerme where id = '111'")
|
242
268
|
|
243
|
-
|
244
|
-
res.fetch_hash.tap do |row|
|
245
|
-
row['cnt'].should == '0'
|
246
|
-
end
|
247
|
-
end
|
269
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "0"
|
248
270
|
end
|
249
271
|
|
250
272
|
end
|
251
273
|
|
252
|
-
describe "
|
274
|
+
describe "LargeHadronMigrator", "replaying changes" do
|
253
275
|
include SpecHelper
|
254
276
|
|
255
277
|
before(:each) do
|
@@ -285,22 +307,12 @@ describe "LargeHadronMigration", "replaying changes" do
|
|
285
307
|
values (3, 'goodbye', 5, NOW(), NOW(), 'delete')
|
286
308
|
}
|
287
309
|
|
288
|
-
|
310
|
+
LargeHadronMigrator.replay_insert_changes("source", "source_changes")
|
289
311
|
|
290
|
-
|
291
|
-
|
292
|
-
row['text'].should == 'goodbye'
|
293
|
-
end
|
294
|
-
end
|
295
|
-
|
296
|
-
sql("select count(*) as cnt from source where id = 3").tap do |res|
|
297
|
-
res.fetch_hash.tap do |row|
|
298
|
-
row['cnt'].should == '0'
|
299
|
-
end
|
300
|
-
end
|
312
|
+
select_value("select text from source where id = 2").should == "goodbye"
|
313
|
+
select_value("select count(*) from source where id = 3").should == "0"
|
301
314
|
end
|
302
315
|
|
303
|
-
|
304
316
|
it "should replay updates" do
|
305
317
|
sql %Q{
|
306
318
|
insert into source (id, text, number, created_at, updated_at)
|
@@ -312,13 +324,9 @@ describe "LargeHadronMigration", "replaying changes" do
|
|
312
324
|
values (1, 'goodbye', 5, NOW(), NOW(), 'update')
|
313
325
|
}
|
314
326
|
|
315
|
-
|
327
|
+
LargeHadronMigrator.replay_update_changes("source", "source_changes")
|
316
328
|
|
317
|
-
|
318
|
-
res.fetch_hash.tap do |row|
|
319
|
-
row['text'].should == 'goodbye'
|
320
|
-
end
|
321
|
-
end
|
329
|
+
select_value("select text from source where id = 1").should == "goodbye"
|
322
330
|
end
|
323
331
|
|
324
332
|
it "should replay deletes" do
|
@@ -333,23 +341,19 @@ describe "LargeHadronMigration", "replaying changes" do
|
|
333
341
|
values (1, 'goodbye', 5, NOW(), NOW(), 'delete')
|
334
342
|
}
|
335
343
|
|
336
|
-
|
344
|
+
LargeHadronMigrator.replay_delete_changes("source", "source_changes")
|
337
345
|
|
338
|
-
|
339
|
-
res.fetch_hash.tap do |row|
|
340
|
-
row['cnt'].should == '1'
|
341
|
-
end
|
342
|
-
end
|
346
|
+
select_value("select count(*) from source").should == "1"
|
343
347
|
end
|
344
348
|
|
345
349
|
it "doesn't replay delete if there are any" do
|
346
|
-
|
347
|
-
|
350
|
+
LargeHadronMigrator.should_receive(:execute).never
|
351
|
+
LargeHadronMigrator.replay_delete_changes("source", "source_changes")
|
348
352
|
end
|
349
353
|
|
350
354
|
end
|
351
355
|
|
352
|
-
describe "
|
356
|
+
describe "LargeHadronMigrator", "units" do
|
353
357
|
include SpecHelper
|
354
358
|
|
355
359
|
it "should return correct schema" do
|
@@ -366,11 +370,17 @@ describe "LargeHadronMigration", "units" do
|
|
366
370
|
(2, 'schmu', 5, NOW(), NOW())
|
367
371
|
}
|
368
372
|
|
369
|
-
schema =
|
373
|
+
schema = LargeHadronMigrator.schema_sql("source", "source_changes", 1000)
|
370
374
|
|
371
375
|
schema.should_not include('`source`')
|
372
376
|
schema.should include('`source_changes`')
|
373
377
|
schema.should include('1003')
|
374
378
|
end
|
375
|
-
end
|
376
379
|
|
380
|
+
describe ".format_time" do
|
381
|
+
it "formats a given Time object" do
|
382
|
+
time = Time.parse("2011-10-09 08:07:56")
|
383
|
+
LargeHadronMigrator.format_time(time).should == "2011_10_09_08_07_56_000"
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|
@@ -2,9 +2,9 @@
|
|
2
2
|
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek
|
3
3
|
#
|
4
4
|
|
5
|
-
class AddNewColumn <
|
5
|
+
class AddNewColumn < LargeHadronMigrator
|
6
6
|
def self.up
|
7
|
-
large_hadron_migrate "addscolumn", :chunk_size => 100 do |table_name|
|
7
|
+
large_hadron_migrate "addscolumn", :chunk_size => 100 do |table_name, _|
|
8
8
|
execute %Q{
|
9
9
|
alter table %s add column spam tinyint(1)
|
10
10
|
} % table_name
|
data/spec/spec_helper.rb
CHANGED
@@ -6,30 +6,32 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
6
6
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
7
7
|
|
8
8
|
require 'active_record'
|
9
|
-
require '
|
9
|
+
require 'large-hadron-migrator'
|
10
10
|
require 'spec'
|
11
11
|
require 'spec/autorun'
|
12
12
|
|
13
13
|
ActiveRecord::Base.establish_connection(
|
14
14
|
:adapter => 'mysql',
|
15
|
-
:database => '
|
16
|
-
:username => '
|
17
|
-
:password => '',
|
15
|
+
:database => 'large_hadron_migrator',
|
16
|
+
:username => '',
|
18
17
|
:host => 'localhost'
|
19
18
|
)
|
19
|
+
ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? true : false
|
20
20
|
|
21
21
|
module SpecHelper
|
22
|
+
delegate :select_one, :select_value, :to => :connection
|
23
|
+
|
22
24
|
def connection
|
23
25
|
ActiveRecord::Base.connection
|
24
26
|
end
|
25
27
|
|
26
|
-
def sql(args)
|
27
|
-
connection.execute(args)
|
28
|
+
def sql(*args)
|
29
|
+
connection.execute(*args)
|
28
30
|
end
|
29
31
|
|
30
32
|
def recreate
|
31
|
-
sql "drop database
|
32
|
-
sql "create database
|
33
|
+
sql "drop database large_hadron_migrator"
|
34
|
+
sql "create database large_hadron_migrator character set = 'UTF8'"
|
33
35
|
|
34
36
|
ActiveRecord::Base.connection.reconnect!
|
35
37
|
end
|
metadata
CHANGED
@@ -1,8 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: large-hadron-migrator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
4
5
|
prerelease:
|
5
|
-
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 0
|
10
|
+
version: 0.2.0
|
6
11
|
platform: ruby
|
7
12
|
authors:
|
8
13
|
- SoundCloud
|
@@ -12,7 +17,7 @@ autorequire:
|
|
12
17
|
bindir: bin
|
13
18
|
cert_chain: []
|
14
19
|
|
15
|
-
date: 2011-
|
20
|
+
date: 2011-11-26 00:00:00 Z
|
16
21
|
dependencies:
|
17
22
|
- !ruby/object:Gem::Dependency
|
18
23
|
name: activerecord
|
@@ -22,6 +27,11 @@ dependencies:
|
|
22
27
|
requirements:
|
23
28
|
- - ~>
|
24
29
|
- !ruby/object:Gem::Version
|
30
|
+
hash: 19
|
31
|
+
segments:
|
32
|
+
- 2
|
33
|
+
- 3
|
34
|
+
- 8
|
25
35
|
version: 2.3.8
|
26
36
|
type: :runtime
|
27
37
|
version_requirements: *id001
|
@@ -33,20 +43,60 @@ dependencies:
|
|
33
43
|
requirements:
|
34
44
|
- - ~>
|
35
45
|
- !ruby/object:Gem::Version
|
46
|
+
hash: 19
|
47
|
+
segments:
|
48
|
+
- 2
|
49
|
+
- 3
|
50
|
+
- 8
|
36
51
|
version: 2.3.8
|
37
52
|
type: :runtime
|
38
53
|
version_requirements: *id002
|
39
54
|
- !ruby/object:Gem::Dependency
|
40
|
-
name:
|
55
|
+
name: mysql
|
41
56
|
prerelease: false
|
42
57
|
requirement: &id003 !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ~>
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
hash: 45
|
63
|
+
segments:
|
64
|
+
- 2
|
65
|
+
- 8
|
66
|
+
- 1
|
67
|
+
version: 2.8.1
|
68
|
+
type: :development
|
69
|
+
version_requirements: *id003
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rspec
|
72
|
+
prerelease: false
|
73
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
43
74
|
none: false
|
44
75
|
requirements:
|
45
76
|
- - "="
|
46
77
|
- !ruby/object:Gem::Version
|
78
|
+
hash: 25
|
79
|
+
segments:
|
80
|
+
- 1
|
81
|
+
- 3
|
82
|
+
- 1
|
47
83
|
version: 1.3.1
|
48
84
|
type: :development
|
49
|
-
version_requirements: *
|
85
|
+
version_requirements: *id004
|
86
|
+
- !ruby/object:Gem::Dependency
|
87
|
+
name: rake
|
88
|
+
prerelease: false
|
89
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
hash: 3
|
95
|
+
segments:
|
96
|
+
- 0
|
97
|
+
version: "0"
|
98
|
+
type: :development
|
99
|
+
version_requirements: *id005
|
50
100
|
description: Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.
|
51
101
|
email: rany@soundcloud.com, tobi@soundcloud.com
|
52
102
|
executables: []
|
@@ -57,7 +107,8 @@ extra_rdoc_files: []
|
|
57
107
|
|
58
108
|
files:
|
59
109
|
- .gitignore
|
60
|
-
-
|
110
|
+
- .travis.yml
|
111
|
+
- CHANGELOG.md
|
61
112
|
- Gemfile
|
62
113
|
- Gemfile.lock
|
63
114
|
- LICENSE
|
@@ -66,7 +117,7 @@ files:
|
|
66
117
|
- VERSION
|
67
118
|
- large-hadron-migrator.gemspec
|
68
119
|
- lib/large-hadron-migrator.rb
|
69
|
-
- spec/
|
120
|
+
- spec/large_hadron_migrator_spec.rb
|
70
121
|
- spec/migrate/add_new_column.rb
|
71
122
|
- spec/spec_helper.rb
|
72
123
|
homepage: http://github.com/soundcloud/large-hadron-migrator
|
@@ -82,12 +133,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
82
133
|
requirements:
|
83
134
|
- - ">="
|
84
135
|
- !ruby/object:Gem::Version
|
136
|
+
hash: 3
|
137
|
+
segments:
|
138
|
+
- 0
|
85
139
|
version: "0"
|
86
140
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
141
|
none: false
|
88
142
|
requirements:
|
89
143
|
- - ">="
|
90
144
|
- !ruby/object:Gem::Version
|
145
|
+
hash: 3
|
146
|
+
segments:
|
147
|
+
- 0
|
91
148
|
version: "0"
|
92
149
|
requirements: []
|
93
150
|
|
@@ -97,6 +154,6 @@ signing_key:
|
|
97
154
|
specification_version: 3
|
98
155
|
summary: online schema changer for mysql
|
99
156
|
test_files:
|
100
|
-
- spec/
|
157
|
+
- spec/large_hadron_migrator_spec.rb
|
101
158
|
- spec/migrate/add_new_column.rb
|
102
159
|
- spec/spec_helper.rb
|
data/CHANGES.md
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
# 0.1.4
|
2
|
-
* Merged [Pullrequest #9](https://github.com/soundcloud/large-hadron-migrator/pull/9)
|
3
|
-
|
4
|
-
# 0.1.3
|
5
|
-
* code cleanup
|
6
|
-
* Merged [Pullrequest #8](https://github.com/soundcloud/large-hadron-migrator/pull/8)
|
7
|
-
* Merged [Pullrequest #7](https://github.com/soundcloud/large-hadron-migrator/pull/7)
|
8
|
-
* Merged [Pullrequest #4](https://github.com/soundcloud/large-hadron-migrator/pull/4)
|
9
|
-
* Merged [Pullrequest #1](https://github.com/soundcloud/large-hadron-migrator/pull/1)
|
10
|
-
|
11
|
-
# 0.1.2
|
12
|
-
* Initial Release
|