large-hadron-migrator 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +5 -0
- data/CHANGELOG.md +22 -0
- data/README.md +1 -8
- data/Rakefile +3 -1
- data/VERSION +1 -1
- data/large-hadron-migrator.gemspec +6 -6
- data/lib/large-hadron-migrator.rb +26 -7
- data/spec/{large_hadron_migration_spec.rb → large_hadron_migrator_spec.rb} +96 -86
- data/spec/migrate/add_new_column.rb +2 -2
- data/spec/spec_helper.rb +10 -8
- metadata +64 -7
- data/CHANGES.md +0 -12
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# 0.2.0 (November 26, 2011)
|
2
|
+
|
3
|
+
* Add Ruby 1.8 compatibility
|
4
|
+
* Setup travis continuous integration
|
5
|
+
* Fix record lose issue
|
6
|
+
* Fix and speed up specs
|
7
|
+
|
8
|
+
# 0.1.4
|
9
|
+
|
10
|
+
* Merged [Pullrequest #9](https://github.com/soundcloud/large-hadron-migrator/pull/9)
|
11
|
+
|
12
|
+
# 0.1.3
|
13
|
+
|
14
|
+
* code cleanup
|
15
|
+
* Merged [Pullrequest #8](https://github.com/soundcloud/large-hadron-migrator/pull/8)
|
16
|
+
* Merged [Pullrequest #7](https://github.com/soundcloud/large-hadron-migrator/pull/7)
|
17
|
+
* Merged [Pullrequest #4](https://github.com/soundcloud/large-hadron-migrator/pull/4)
|
18
|
+
* Merged [Pullrequest #1](https://github.com/soundcloud/large-hadron-migrator/pull/1)
|
19
|
+
|
20
|
+
# 0.1.2
|
21
|
+
|
22
|
+
* Initial Release
|
data/README.md
CHANGED
@@ -163,7 +163,7 @@ site. We rolled out the fix and restarted the app servers in mid migration.
|
|
163
163
|
This was not a good idea.
|
164
164
|
|
165
165
|
TL;DR: Never restart during migrations when removing columns with LHM.
|
166
|
-
You can restart while adding migrations as long as active record reads column
|
166
|
+
You can restart while adding migrations as long as active record reads column
|
167
167
|
definitions from the slave.
|
168
168
|
|
169
169
|
The information below is only relevant if you want to restart your app servers
|
@@ -199,9 +199,6 @@ while migrating in a master slave setup.
|
|
199
199
|
|
200
200
|
## Todos
|
201
201
|
|
202
|
-
~~Load data into outfile instead of `INSERT INTO... SELECT`. Avoid contention and
|
203
|
-
increase speed.~~ It showed that this wont result in major speed up.
|
204
|
-
|
205
202
|
Handle invalidation of 'update' entries in journal while replaying. Avoid
|
206
203
|
stale update replays.
|
207
204
|
|
@@ -214,10 +211,6 @@ problems with this, but the chunked copy could be sped up by factoring this
|
|
214
211
|
in. Currently a copy range may be completely empty, but there will still be
|
215
212
|
a `INSERT INTO... SELECT`.
|
216
213
|
|
217
|
-
Records inserted after the last insert id is retrieved and before the triggers
|
218
|
-
are created are currently lost. The table should be briefly locked while id is
|
219
|
-
read and triggers are applied.
|
220
|
-
|
221
214
|
## Contributing
|
222
215
|
|
223
216
|
We'll check out your contribution if you:
|
data/Rakefile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
@@ -14,11 +14,11 @@ Gem::Specification.new do |s|
|
|
14
14
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
15
15
|
s.require_paths = ["lib"]
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
end
|
17
|
+
s.add_dependency "activerecord", "~> 2.3.8"
|
18
|
+
s.add_dependency "activesupport", "~> 2.3.8"
|
20
19
|
|
21
|
-
|
22
|
-
|
23
|
-
|
20
|
+
# this should be a real dependency, but we're using a different gem in our code
|
21
|
+
s.add_development_dependency "mysql", "~> 2.8.1"
|
22
|
+
s.add_development_dependency "rspec", "=1.3.1"
|
23
|
+
s.add_development_dependency "rake"
|
24
24
|
end
|
@@ -100,14 +100,11 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
100
100
|
|
101
101
|
raise "chunk_size must be >= 1" unless chunk_size >= 1
|
102
102
|
|
103
|
-
started = Time.now
|
103
|
+
started = format_time(Time.now)
|
104
104
|
new_table = "lhmn_%s" % curr_table
|
105
105
|
old_table = "lhmo_%s_%s" % [started, curr_table]
|
106
106
|
journal_table = "lhmj_%s_%s" % [started, curr_table]
|
107
107
|
|
108
|
-
last_insert_id = last_insert_id(curr_table)
|
109
|
-
say "last inserted id in #{curr_table}: #{last_insert_id}"
|
110
|
-
|
111
108
|
begin
|
112
109
|
# clean tables. old tables are never deleted to guard against rollbacks.
|
113
110
|
execute %Q{drop table if exists %s} % new_table
|
@@ -120,6 +117,9 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
120
117
|
add_trigger_on_action(curr_table, journal_table, "update")
|
121
118
|
add_trigger_on_action(curr_table, journal_table, "delete")
|
122
119
|
|
120
|
+
last_insert_id = last_smaller_insert_id(curr_table, journal_table)
|
121
|
+
say "last inserted id in #{curr_table}: #{last_insert_id}"
|
122
|
+
|
123
123
|
# alter new table
|
124
124
|
default_values = {}
|
125
125
|
yield new_table, default_values
|
@@ -135,7 +135,7 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
135
135
|
curr_table,
|
136
136
|
wait
|
137
137
|
|
138
|
-
rename_tables curr_table
|
138
|
+
rename_tables [[ curr_table, old_table ], [ new_table, curr_table ]]
|
139
139
|
cleanup(curr_table)
|
140
140
|
|
141
141
|
# replay changes from the changes jornal
|
@@ -226,15 +226,26 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
226
226
|
end
|
227
227
|
end
|
228
228
|
|
229
|
+
# to satisfy spec "does not lose records during table clone step"
|
230
|
+
def self.last_smaller_insert_id(curr_table, journal_table)
|
231
|
+
with_master do
|
232
|
+
select_value(%{
|
233
|
+
SELECT max(#{curr_table}.id) as max_id from #{curr_table}
|
234
|
+
LEFT OUTER JOIN #{journal_table} ON #{journal_table}.id = #{curr_table}.id
|
235
|
+
WHERE #{journal_table}.id IS NULL
|
236
|
+
} % [ curr_table, journal_table ]).to_i
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
229
240
|
def self.last_insert_id(curr_table)
|
230
241
|
with_master do
|
231
|
-
|
242
|
+
select_value("SELECT max(id) FROM #{curr_table}").to_i
|
232
243
|
end
|
233
244
|
end
|
234
245
|
|
235
246
|
def self.table_column_names(table_name)
|
236
247
|
with_master do
|
237
|
-
|
248
|
+
select_values %Q{
|
238
249
|
select column_name
|
239
250
|
from information_schema.columns
|
240
251
|
where table_name = "%s"
|
@@ -382,4 +393,12 @@ class LargeHadronMigrator < ActiveRecord::Migration
|
|
382
393
|
def self.tick(col)
|
383
394
|
"`#{ col }`"
|
384
395
|
end
|
396
|
+
|
397
|
+
def self.format_time(time)
|
398
|
+
if RUBY_VERSION < "1.9"
|
399
|
+
time.strftime("%Y_%m_%d_%H_%M_%S_#{"%03d" % (time.usec / 1000)}")
|
400
|
+
else
|
401
|
+
time.strftime("%Y_%m_%d_%H_%M_%S_%3N")
|
402
|
+
end
|
403
|
+
end
|
385
404
|
end
|
@@ -6,13 +6,12 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
|
6
6
|
|
7
7
|
require "migrate/add_new_column"
|
8
8
|
|
9
|
-
describe "
|
9
|
+
describe "LargeHadronMigrator", "integration" do
|
10
10
|
include SpecHelper
|
11
11
|
|
12
12
|
before(:each) { recreate }
|
13
13
|
|
14
14
|
it "should add new column" do
|
15
|
-
|
16
15
|
table("addscolumn") do |t|
|
17
16
|
t.string :title
|
18
17
|
t.integer :rating
|
@@ -40,7 +39,7 @@ describe "LargeHadronMigration", "integration" do
|
|
40
39
|
t.timestamps
|
41
40
|
end
|
42
41
|
|
43
|
-
|
42
|
+
420.times do |i|
|
44
43
|
random_string = (0...rand(25)).map{65.+(rand(25)).chr}.join
|
45
44
|
sql "INSERT INTO `addscolumn` SET
|
46
45
|
`id` = #{i+1},
|
@@ -52,12 +51,57 @@ describe "LargeHadronMigration", "integration" do
|
|
52
51
|
|
53
52
|
ghost = AddNewColumn.up
|
54
53
|
|
55
|
-
truthiness_rows "addscolumn", ghost
|
54
|
+
truthiness_rows "addscolumn", ghost, 0, 420
|
56
55
|
end
|
57
|
-
end
|
58
56
|
|
57
|
+
context "trigger creation" do
|
58
|
+
before do
|
59
|
+
class LargeHadronMigrator
|
60
|
+
class << self
|
61
|
+
alias_method :orig_clone_table, :clone_table
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.clone_table(*args)
|
65
|
+
sleep 0.3
|
66
|
+
orig_clone_table(*args)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
after do
|
72
|
+
class LargeHadronMigrator
|
73
|
+
class << self
|
74
|
+
alias_method :clone_table, :orig_clone_table
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
it "does not lose records during trigger creation" do
|
80
|
+
table "addscolumn" do |t|
|
81
|
+
t.string :data
|
82
|
+
t.timestamps
|
83
|
+
end
|
84
|
+
|
85
|
+
parallel_insert = Thread.new do
|
86
|
+
20.times do |i|
|
87
|
+
sql %{
|
88
|
+
INSERT INTO addscolumn (data, created_at, updated_at)
|
89
|
+
VALUES ('#{ActiveSupport::SecureRandom.base64}', NOW(), NOW())
|
90
|
+
}
|
91
|
+
sleep 0.1
|
92
|
+
end
|
93
|
+
end
|
59
94
|
|
60
|
-
|
95
|
+
ghost = AddNewColumn.up
|
96
|
+
|
97
|
+
parallel_insert.join
|
98
|
+
|
99
|
+
select_value("SELECT count(*) FROM addscolumn").should == "20"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
describe "LargeHadronMigrator", "rename" do
|
61
105
|
include SpecHelper
|
62
106
|
|
63
107
|
before(:each) do
|
@@ -73,7 +117,7 @@ describe "LargeHadronMigration", "rename" do
|
|
73
117
|
t.integer :number
|
74
118
|
end
|
75
119
|
|
76
|
-
|
120
|
+
LargeHadronMigrator.rename_tables("renameme" => "renameme_new", "renamemetoo" => "renameme")
|
77
121
|
|
78
122
|
truthiness_column "renameme", "number", "int"
|
79
123
|
truthiness_column "renameme_new", "text", "varchar"
|
@@ -81,7 +125,7 @@ describe "LargeHadronMigration", "rename" do
|
|
81
125
|
|
82
126
|
end
|
83
127
|
|
84
|
-
describe "
|
128
|
+
describe "LargeHadronMigrator", "triggers" do
|
85
129
|
include SpecHelper
|
86
130
|
|
87
131
|
before(:each) do
|
@@ -93,7 +137,7 @@ describe "LargeHadronMigration", "triggers" do
|
|
93
137
|
t.timestamps
|
94
138
|
end
|
95
139
|
|
96
|
-
|
140
|
+
LargeHadronMigrator.clone_table_for_changes \
|
97
141
|
"triggerme",
|
98
142
|
"triggerme_changes"
|
99
143
|
end
|
@@ -104,18 +148,16 @@ describe "LargeHadronMigration", "triggers" do
|
|
104
148
|
end
|
105
149
|
|
106
150
|
it "should trigger on insert" do
|
107
|
-
|
151
|
+
LargeHadronMigrator.add_trigger_on_action \
|
108
152
|
"triggerme",
|
109
153
|
"triggerme_changes",
|
110
154
|
"insert"
|
111
155
|
|
112
156
|
# test
|
113
157
|
sql("insert into triggerme values (111, 'hallo', 5, NOW(), NOW())")
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
row['text'].should == 'hallo'
|
118
|
-
end
|
158
|
+
select_one("select * from triggerme_changes where id = 111").tap do |row|
|
159
|
+
row["hadron_action"].should == "insert"
|
160
|
+
row["text"].should == "hallo"
|
119
161
|
end
|
120
162
|
end
|
121
163
|
|
@@ -123,18 +165,16 @@ describe "LargeHadronMigration", "triggers" do
|
|
123
165
|
|
124
166
|
# setup
|
125
167
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
126
|
-
|
168
|
+
LargeHadronMigrator.add_trigger_on_action \
|
127
169
|
"triggerme",
|
128
170
|
"triggerme_changes",
|
129
171
|
"update"
|
130
172
|
|
131
173
|
# test
|
132
174
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
row['text'].should == 'goodbye'
|
137
|
-
end
|
175
|
+
select_one("select * from triggerme_changes where id = 111").tap do |row|
|
176
|
+
row["hadron_action"].should == "update"
|
177
|
+
row["text"].should == "goodbye"
|
138
178
|
end
|
139
179
|
end
|
140
180
|
|
@@ -142,28 +182,26 @@ describe "LargeHadronMigration", "triggers" do
|
|
142
182
|
|
143
183
|
# setup
|
144
184
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
145
|
-
|
185
|
+
LargeHadronMigrator.add_trigger_on_action \
|
146
186
|
"triggerme",
|
147
187
|
"triggerme_changes",
|
148
188
|
"delete"
|
149
189
|
|
150
190
|
# test
|
151
191
|
sql("delete from triggerme where id = '111'")
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
row['text'].should == 'hallo'
|
156
|
-
end
|
192
|
+
select_one("select * from triggerme_changes where id = 111").tap do |row|
|
193
|
+
row["hadron_action"].should == "delete"
|
194
|
+
row["text"].should == "hallo"
|
157
195
|
end
|
158
196
|
end
|
159
197
|
|
160
198
|
it "should trigger on create and update" do
|
161
|
-
|
199
|
+
LargeHadronMigrator.add_trigger_on_action \
|
162
200
|
"triggerme",
|
163
201
|
"triggerme_changes",
|
164
202
|
"insert"
|
165
203
|
|
166
|
-
|
204
|
+
LargeHadronMigrator.add_trigger_on_action \
|
167
205
|
"triggerme",
|
168
206
|
"triggerme_changes",
|
169
207
|
"update"
|
@@ -172,16 +210,12 @@ describe "LargeHadronMigration", "triggers" do
|
|
172
210
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
173
211
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
174
212
|
|
175
|
-
|
176
|
-
res.fetch_hash.tap do |row|
|
177
|
-
row['cnt'].should == '1'
|
178
|
-
end
|
179
|
-
end
|
213
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "1"
|
180
214
|
end
|
181
215
|
|
182
216
|
it "should trigger on multiple update" do
|
183
217
|
sql "insert into triggerme values (111, 'hallo', 5, NOW(), NOW())"
|
184
|
-
|
218
|
+
LargeHadronMigrator.add_trigger_on_action \
|
185
219
|
"triggerme",
|
186
220
|
"triggerme_changes",
|
187
221
|
"update"
|
@@ -190,25 +224,21 @@ describe "LargeHadronMigration", "triggers" do
|
|
190
224
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
191
225
|
sql("update triggerme set text = 'hallo again' where id = '111'")
|
192
226
|
|
193
|
-
|
194
|
-
res.fetch_hash.tap do |row|
|
195
|
-
row['cnt'].should == '1'
|
196
|
-
end
|
197
|
-
end
|
227
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "1"
|
198
228
|
end
|
199
229
|
|
200
230
|
it "should trigger on inser, update and delete" do
|
201
|
-
|
231
|
+
LargeHadronMigrator.add_trigger_on_action \
|
202
232
|
"triggerme",
|
203
233
|
"triggerme_changes",
|
204
234
|
"insert"
|
205
235
|
|
206
|
-
|
236
|
+
LargeHadronMigrator.add_trigger_on_action \
|
207
237
|
"triggerme",
|
208
238
|
"triggerme_changes",
|
209
239
|
"update"
|
210
240
|
|
211
|
-
|
241
|
+
LargeHadronMigrator.add_trigger_on_action \
|
212
242
|
"triggerme",
|
213
243
|
"triggerme_changes",
|
214
244
|
"delete"
|
@@ -218,38 +248,30 @@ describe "LargeHadronMigration", "triggers" do
|
|
218
248
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
219
249
|
sql("delete from triggerme where id = '111'")
|
220
250
|
|
221
|
-
|
222
|
-
res.fetch_hash.tap do |row|
|
223
|
-
row['cnt'].should == '1'
|
224
|
-
end
|
225
|
-
end
|
251
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "1"
|
226
252
|
end
|
227
253
|
|
228
254
|
it "should cleanup triggers" do
|
229
255
|
%w(insert update delete).each do |action|
|
230
|
-
|
256
|
+
LargeHadronMigrator.add_trigger_on_action \
|
231
257
|
"triggerme",
|
232
258
|
"triggerme_changes",
|
233
259
|
action
|
234
260
|
end
|
235
261
|
|
236
|
-
|
262
|
+
LargeHadronMigrator.cleanup "triggerme"
|
237
263
|
|
238
264
|
# test
|
239
265
|
sql("insert into triggerme values (111, 'hallo', 5, NOW(), NOW())")
|
240
266
|
sql("update triggerme set text = 'goodbye' where id = '111'")
|
241
267
|
sql("delete from triggerme where id = '111'")
|
242
268
|
|
243
|
-
|
244
|
-
res.fetch_hash.tap do |row|
|
245
|
-
row['cnt'].should == '0'
|
246
|
-
end
|
247
|
-
end
|
269
|
+
select_value("select count(*) from triggerme_changes where id = 111").should == "0"
|
248
270
|
end
|
249
271
|
|
250
272
|
end
|
251
273
|
|
252
|
-
describe "
|
274
|
+
describe "LargeHadronMigrator", "replaying changes" do
|
253
275
|
include SpecHelper
|
254
276
|
|
255
277
|
before(:each) do
|
@@ -285,22 +307,12 @@ describe "LargeHadronMigration", "replaying changes" do
|
|
285
307
|
values (3, 'goodbye', 5, NOW(), NOW(), 'delete')
|
286
308
|
}
|
287
309
|
|
288
|
-
|
310
|
+
LargeHadronMigrator.replay_insert_changes("source", "source_changes")
|
289
311
|
|
290
|
-
|
291
|
-
|
292
|
-
row['text'].should == 'goodbye'
|
293
|
-
end
|
294
|
-
end
|
295
|
-
|
296
|
-
sql("select count(*) as cnt from source where id = 3").tap do |res|
|
297
|
-
res.fetch_hash.tap do |row|
|
298
|
-
row['cnt'].should == '0'
|
299
|
-
end
|
300
|
-
end
|
312
|
+
select_value("select text from source where id = 2").should == "goodbye"
|
313
|
+
select_value("select count(*) from source where id = 3").should == "0"
|
301
314
|
end
|
302
315
|
|
303
|
-
|
304
316
|
it "should replay updates" do
|
305
317
|
sql %Q{
|
306
318
|
insert into source (id, text, number, created_at, updated_at)
|
@@ -312,13 +324,9 @@ describe "LargeHadronMigration", "replaying changes" do
|
|
312
324
|
values (1, 'goodbye', 5, NOW(), NOW(), 'update')
|
313
325
|
}
|
314
326
|
|
315
|
-
|
327
|
+
LargeHadronMigrator.replay_update_changes("source", "source_changes")
|
316
328
|
|
317
|
-
|
318
|
-
res.fetch_hash.tap do |row|
|
319
|
-
row['text'].should == 'goodbye'
|
320
|
-
end
|
321
|
-
end
|
329
|
+
select_value("select text from source where id = 1").should == "goodbye"
|
322
330
|
end
|
323
331
|
|
324
332
|
it "should replay deletes" do
|
@@ -333,23 +341,19 @@ describe "LargeHadronMigration", "replaying changes" do
|
|
333
341
|
values (1, 'goodbye', 5, NOW(), NOW(), 'delete')
|
334
342
|
}
|
335
343
|
|
336
|
-
|
344
|
+
LargeHadronMigrator.replay_delete_changes("source", "source_changes")
|
337
345
|
|
338
|
-
|
339
|
-
res.fetch_hash.tap do |row|
|
340
|
-
row['cnt'].should == '1'
|
341
|
-
end
|
342
|
-
end
|
346
|
+
select_value("select count(*) from source").should == "1"
|
343
347
|
end
|
344
348
|
|
345
349
|
it "doesn't replay delete if there are any" do
|
346
|
-
|
347
|
-
|
350
|
+
LargeHadronMigrator.should_receive(:execute).never
|
351
|
+
LargeHadronMigrator.replay_delete_changes("source", "source_changes")
|
348
352
|
end
|
349
353
|
|
350
354
|
end
|
351
355
|
|
352
|
-
describe "
|
356
|
+
describe "LargeHadronMigrator", "units" do
|
353
357
|
include SpecHelper
|
354
358
|
|
355
359
|
it "should return correct schema" do
|
@@ -366,11 +370,17 @@ describe "LargeHadronMigration", "units" do
|
|
366
370
|
(2, 'schmu', 5, NOW(), NOW())
|
367
371
|
}
|
368
372
|
|
369
|
-
schema =
|
373
|
+
schema = LargeHadronMigrator.schema_sql("source", "source_changes", 1000)
|
370
374
|
|
371
375
|
schema.should_not include('`source`')
|
372
376
|
schema.should include('`source_changes`')
|
373
377
|
schema.should include('1003')
|
374
378
|
end
|
375
|
-
end
|
376
379
|
|
380
|
+
describe ".format_time" do
|
381
|
+
it "formats a given Time object" do
|
382
|
+
time = Time.parse("2011-10-09 08:07:56")
|
383
|
+
LargeHadronMigrator.format_time(time).should == "2011_10_09_08_07_56_000"
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|
@@ -2,9 +2,9 @@
|
|
2
2
|
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek
|
3
3
|
#
|
4
4
|
|
5
|
-
class AddNewColumn <
|
5
|
+
class AddNewColumn < LargeHadronMigrator
|
6
6
|
def self.up
|
7
|
-
large_hadron_migrate "addscolumn", :chunk_size => 100 do |table_name|
|
7
|
+
large_hadron_migrate "addscolumn", :chunk_size => 100 do |table_name, _|
|
8
8
|
execute %Q{
|
9
9
|
alter table %s add column spam tinyint(1)
|
10
10
|
} % table_name
|
data/spec/spec_helper.rb
CHANGED
@@ -6,30 +6,32 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
6
6
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
7
7
|
|
8
8
|
require 'active_record'
|
9
|
-
require '
|
9
|
+
require 'large-hadron-migrator'
|
10
10
|
require 'spec'
|
11
11
|
require 'spec/autorun'
|
12
12
|
|
13
13
|
ActiveRecord::Base.establish_connection(
|
14
14
|
:adapter => 'mysql',
|
15
|
-
:database => '
|
16
|
-
:username => '
|
17
|
-
:password => '',
|
15
|
+
:database => 'large_hadron_migrator',
|
16
|
+
:username => '',
|
18
17
|
:host => 'localhost'
|
19
18
|
)
|
19
|
+
ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? true : false
|
20
20
|
|
21
21
|
module SpecHelper
|
22
|
+
delegate :select_one, :select_value, :to => :connection
|
23
|
+
|
22
24
|
def connection
|
23
25
|
ActiveRecord::Base.connection
|
24
26
|
end
|
25
27
|
|
26
|
-
def sql(args)
|
27
|
-
connection.execute(args)
|
28
|
+
def sql(*args)
|
29
|
+
connection.execute(*args)
|
28
30
|
end
|
29
31
|
|
30
32
|
def recreate
|
31
|
-
sql "drop database
|
32
|
-
sql "create database
|
33
|
+
sql "drop database large_hadron_migrator"
|
34
|
+
sql "create database large_hadron_migrator character set = 'UTF8'"
|
33
35
|
|
34
36
|
ActiveRecord::Base.connection.reconnect!
|
35
37
|
end
|
metadata
CHANGED
@@ -1,8 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: large-hadron-migrator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
4
5
|
prerelease:
|
5
|
-
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 0
|
10
|
+
version: 0.2.0
|
6
11
|
platform: ruby
|
7
12
|
authors:
|
8
13
|
- SoundCloud
|
@@ -12,7 +17,7 @@ autorequire:
|
|
12
17
|
bindir: bin
|
13
18
|
cert_chain: []
|
14
19
|
|
15
|
-
date: 2011-
|
20
|
+
date: 2011-11-26 00:00:00 Z
|
16
21
|
dependencies:
|
17
22
|
- !ruby/object:Gem::Dependency
|
18
23
|
name: activerecord
|
@@ -22,6 +27,11 @@ dependencies:
|
|
22
27
|
requirements:
|
23
28
|
- - ~>
|
24
29
|
- !ruby/object:Gem::Version
|
30
|
+
hash: 19
|
31
|
+
segments:
|
32
|
+
- 2
|
33
|
+
- 3
|
34
|
+
- 8
|
25
35
|
version: 2.3.8
|
26
36
|
type: :runtime
|
27
37
|
version_requirements: *id001
|
@@ -33,20 +43,60 @@ dependencies:
|
|
33
43
|
requirements:
|
34
44
|
- - ~>
|
35
45
|
- !ruby/object:Gem::Version
|
46
|
+
hash: 19
|
47
|
+
segments:
|
48
|
+
- 2
|
49
|
+
- 3
|
50
|
+
- 8
|
36
51
|
version: 2.3.8
|
37
52
|
type: :runtime
|
38
53
|
version_requirements: *id002
|
39
54
|
- !ruby/object:Gem::Dependency
|
40
|
-
name:
|
55
|
+
name: mysql
|
41
56
|
prerelease: false
|
42
57
|
requirement: &id003 !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ~>
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
hash: 45
|
63
|
+
segments:
|
64
|
+
- 2
|
65
|
+
- 8
|
66
|
+
- 1
|
67
|
+
version: 2.8.1
|
68
|
+
type: :development
|
69
|
+
version_requirements: *id003
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rspec
|
72
|
+
prerelease: false
|
73
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
43
74
|
none: false
|
44
75
|
requirements:
|
45
76
|
- - "="
|
46
77
|
- !ruby/object:Gem::Version
|
78
|
+
hash: 25
|
79
|
+
segments:
|
80
|
+
- 1
|
81
|
+
- 3
|
82
|
+
- 1
|
47
83
|
version: 1.3.1
|
48
84
|
type: :development
|
49
|
-
version_requirements: *
|
85
|
+
version_requirements: *id004
|
86
|
+
- !ruby/object:Gem::Dependency
|
87
|
+
name: rake
|
88
|
+
prerelease: false
|
89
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
hash: 3
|
95
|
+
segments:
|
96
|
+
- 0
|
97
|
+
version: "0"
|
98
|
+
type: :development
|
99
|
+
version_requirements: *id005
|
50
100
|
description: Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.
|
51
101
|
email: rany@soundcloud.com, tobi@soundcloud.com
|
52
102
|
executables: []
|
@@ -57,7 +107,8 @@ extra_rdoc_files: []
|
|
57
107
|
|
58
108
|
files:
|
59
109
|
- .gitignore
|
60
|
-
-
|
110
|
+
- .travis.yml
|
111
|
+
- CHANGELOG.md
|
61
112
|
- Gemfile
|
62
113
|
- Gemfile.lock
|
63
114
|
- LICENSE
|
@@ -66,7 +117,7 @@ files:
|
|
66
117
|
- VERSION
|
67
118
|
- large-hadron-migrator.gemspec
|
68
119
|
- lib/large-hadron-migrator.rb
|
69
|
-
- spec/
|
120
|
+
- spec/large_hadron_migrator_spec.rb
|
70
121
|
- spec/migrate/add_new_column.rb
|
71
122
|
- spec/spec_helper.rb
|
72
123
|
homepage: http://github.com/soundcloud/large-hadron-migrator
|
@@ -82,12 +133,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
82
133
|
requirements:
|
83
134
|
- - ">="
|
84
135
|
- !ruby/object:Gem::Version
|
136
|
+
hash: 3
|
137
|
+
segments:
|
138
|
+
- 0
|
85
139
|
version: "0"
|
86
140
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
141
|
none: false
|
88
142
|
requirements:
|
89
143
|
- - ">="
|
90
144
|
- !ruby/object:Gem::Version
|
145
|
+
hash: 3
|
146
|
+
segments:
|
147
|
+
- 0
|
91
148
|
version: "0"
|
92
149
|
requirements: []
|
93
150
|
|
@@ -97,6 +154,6 @@ signing_key:
|
|
97
154
|
specification_version: 3
|
98
155
|
summary: online schema changer for mysql
|
99
156
|
test_files:
|
100
|
-
- spec/
|
157
|
+
- spec/large_hadron_migrator_spec.rb
|
101
158
|
- spec/migrate/add_new_column.rb
|
102
159
|
- spec/spec_helper.rb
|
data/CHANGES.md
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
# 0.1.4
|
2
|
-
* Merged [Pullrequest #9](https://github.com/soundcloud/large-hadron-migrator/pull/9)
|
3
|
-
|
4
|
-
# 0.1.3
|
5
|
-
* code cleanup
|
6
|
-
* Merged [Pullrequest #8](https://github.com/soundcloud/large-hadron-migrator/pull/8)
|
7
|
-
* Merged [Pullrequest #7](https://github.com/soundcloud/large-hadron-migrator/pull/7)
|
8
|
-
* Merged [Pullrequest #4](https://github.com/soundcloud/large-hadron-migrator/pull/4)
|
9
|
-
* Merged [Pullrequest #1](https://github.com/soundcloud/large-hadron-migrator/pull/1)
|
10
|
-
|
11
|
-
# 0.1.2
|
12
|
-
* Initial Release
|