sbader-lhm 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. data/.gitignore +6 -0
  2. data/.travis.yml +10 -0
  3. data/CHANGELOG.md +99 -0
  4. data/LICENSE +27 -0
  5. data/README.md +146 -0
  6. data/Rakefile +20 -0
  7. data/bin/lhm-kill-queue +172 -0
  8. data/bin/lhm-spec-clobber.sh +36 -0
  9. data/bin/lhm-spec-grants.sh +25 -0
  10. data/bin/lhm-spec-setup-cluster.sh +67 -0
  11. data/bin/lhm-test-all.sh +10 -0
  12. data/gemfiles/ar-2.3_mysql.gemfile +5 -0
  13. data/gemfiles/ar-3.2_mysql.gemfile +5 -0
  14. data/gemfiles/ar-3.2_mysql2.gemfile +5 -0
  15. data/lhm.gemspec +27 -0
  16. data/lib/lhm.rb +45 -0
  17. data/lib/lhm/atomic_switcher.rb +49 -0
  18. data/lib/lhm/chunker.rb +114 -0
  19. data/lib/lhm/command.rb +46 -0
  20. data/lib/lhm/entangler.rb +98 -0
  21. data/lib/lhm/intersection.rb +63 -0
  22. data/lib/lhm/invoker.rb +49 -0
  23. data/lib/lhm/locked_switcher.rb +71 -0
  24. data/lib/lhm/migration.rb +30 -0
  25. data/lib/lhm/migrator.rb +219 -0
  26. data/lib/lhm/sql_helper.rb +85 -0
  27. data/lib/lhm/table.rb +97 -0
  28. data/lib/lhm/version.rb +6 -0
  29. data/spec/.lhm.example +4 -0
  30. data/spec/README.md +51 -0
  31. data/spec/bootstrap.rb +13 -0
  32. data/spec/fixtures/destination.ddl +6 -0
  33. data/spec/fixtures/origin.ddl +6 -0
  34. data/spec/fixtures/small_table.ddl +4 -0
  35. data/spec/fixtures/users.ddl +12 -0
  36. data/spec/integration/atomic_switcher_spec.rb +42 -0
  37. data/spec/integration/chunker_spec.rb +32 -0
  38. data/spec/integration/entangler_spec.rb +66 -0
  39. data/spec/integration/integration_helper.rb +140 -0
  40. data/spec/integration/lhm_spec.rb +204 -0
  41. data/spec/integration/locked_switcher_spec.rb +42 -0
  42. data/spec/integration/table_spec.rb +48 -0
  43. data/spec/unit/atomic_switcher_spec.rb +31 -0
  44. data/spec/unit/chunker_spec.rb +111 -0
  45. data/spec/unit/entangler_spec.rb +76 -0
  46. data/spec/unit/intersection_spec.rb +39 -0
  47. data/spec/unit/locked_switcher_spec.rb +51 -0
  48. data/spec/unit/migration_spec.rb +23 -0
  49. data/spec/unit/migrator_spec.rb +134 -0
  50. data/spec/unit/sql_helper_spec.rb +32 -0
  51. data/spec/unit/table_spec.rb +34 -0
  52. data/spec/unit/unit_helper.rb +14 -0
  53. metadata +173 -0
@@ -0,0 +1,204 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
5
+
6
+ require 'lhm'
7
+
8
+ describe Lhm do
9
+ include IntegrationHelper
10
+
11
+ before(:each) { connect_master! }
12
+
13
+ describe "changes" do
14
+ before(:each) do
15
+ table_create(:users)
16
+ end
17
+
18
+ it "should add a column" do
19
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
20
+ t.add_column(:logins, "INT(12) DEFAULT '0'")
21
+ end
22
+
23
+ slave do
24
+ table_read(:users).columns["logins"].must_equal({
25
+ :type => "int(12)",
26
+ :is_nullable => "YES",
27
+ :column_default => '0'
28
+ })
29
+ end
30
+ end
31
+
32
+ it "should copy all rows" do
33
+ 23.times { |n| execute("insert into users set reference = '#{ n }'") }
34
+
35
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
36
+ t.add_column(:logins, "INT(12) DEFAULT '0'")
37
+ end
38
+
39
+ slave do
40
+ count_all(:users).must_equal(23)
41
+ end
42
+ end
43
+
44
+ it "should remove a column" do
45
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
46
+ t.remove_column(:comment)
47
+ end
48
+
49
+ slave do
50
+ table_read(:users).columns["comment"].must_equal nil
51
+ end
52
+ end
53
+
54
+ it "should add an index" do
55
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
56
+ t.add_index([:comment, :created_at])
57
+ end
58
+
59
+ slave do
60
+ index_on_columns?(:users, [:comment, :created_at]).must_equal(true)
61
+ end
62
+ end
63
+
64
+ it "should add an index with a custom name" do
65
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
66
+ t.add_index([:comment, :created_at], :my_index_name)
67
+ end
68
+
69
+ slave do
70
+ index?(:users, :my_index_name).must_equal(true)
71
+ end
72
+ end
73
+
74
+ it "should add an index on a column with a reserved name" do
75
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
76
+ t.add_index(:group)
77
+ end
78
+
79
+ slave do
80
+ index_on_columns?(:users, :group).must_equal(true)
81
+ end
82
+ end
83
+
84
+ it "should add a unqiue index" do
85
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
86
+ t.add_unique_index(:comment)
87
+ end
88
+
89
+ slave do
90
+ index_on_columns?(:users, :comment, :unique).must_equal(true)
91
+ end
92
+ end
93
+
94
+ it "should remove an index" do
95
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
96
+ t.remove_index([:username, :created_at])
97
+ end
98
+
99
+ slave do
100
+ index_on_columns?(:users, [:username, :created_at]).must_equal(false)
101
+ end
102
+ end
103
+
104
+ it "should remove an index with a custom name" do
105
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
106
+ t.remove_index(:reference, :index_users_on_reference)
107
+ end
108
+
109
+ slave do
110
+ index?(:users, :index_users_on_reference).must_equal(false)
111
+ end
112
+ end
113
+
114
+ it "should apply a ddl statement" do
115
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
116
+ t.ddl("alter table %s add column flag tinyint(1)" % t.name)
117
+ end
118
+
119
+ slave do
120
+ table_read(:users).columns["flag"].must_equal({
121
+ :type => "tinyint(1)",
122
+ :is_nullable => "YES",
123
+ :column_default => nil
124
+ })
125
+ end
126
+ end
127
+
128
+ it "should change a column" do
129
+ Lhm.change_table(:users, :atomic_switch => false) do |t|
130
+ t.change_column(:comment, "varchar(20) DEFAULT 'none' NOT NULL")
131
+ end
132
+
133
+ slave do
134
+ table_read(:users).columns["comment"].must_equal({
135
+ :type => "varchar(20)",
136
+ :is_nullable => "NO",
137
+ :column_default => "none"
138
+ })
139
+ end
140
+ end
141
+
142
+ it "should change the last column in a table" do
143
+ table_create(:small_table)
144
+
145
+ Lhm.change_table(:small_table, :atomic_switch => false) do |t|
146
+ t.change_column(:id, "int(5)")
147
+ end
148
+
149
+ slave do
150
+ table_read(:small_table).columns["id"].must_equal({
151
+ :type => "int(5)",
152
+ :is_nullable => "NO",
153
+ :column_default => "0"
154
+ })
155
+ end
156
+ end
157
+
158
+ describe "parallel" do
159
+ it "should perserve inserts during migration" do
160
+ 50.times { |n| execute("insert into users set reference = '#{ n }'") }
161
+
162
+ insert = Thread.new do
163
+ 10.times do |n|
164
+ execute("insert into users set reference = '#{ 100 + n }'")
165
+ sleep(0.17)
166
+ end
167
+ end
168
+
169
+ options = { :stride => 10, :throttle => 97, :atomic_switch => false }
170
+ Lhm.change_table(:users, options) do |t|
171
+ t.add_column(:parallel, "INT(10) DEFAULT '0'")
172
+ end
173
+
174
+ insert.join
175
+
176
+ slave do
177
+ count_all(:users).must_equal(60)
178
+ end
179
+ end
180
+
181
+ it "should perserve deletes during migration" do
182
+ 50.times { |n| execute("insert into users set reference = '#{ n }'") }
183
+
184
+ delete = Thread.new do
185
+ 10.times do |n|
186
+ execute("delete from users where id = '#{ n + 1 }'")
187
+ sleep(0.17)
188
+ end
189
+ end
190
+
191
+ options = { :stride => 10, :throttle => 97, :atomic_switch => false }
192
+ Lhm.change_table(:users, options) do |t|
193
+ t.add_column(:parallel, "INT(10) DEFAULT '0'")
194
+ end
195
+
196
+ delete.join
197
+
198
+ slave do
199
+ count_all(:users).must_equal(40)
200
+ end
201
+ end
202
+ end
203
+ end
204
+ end
@@ -0,0 +1,42 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/locked_switcher'
9
+
10
+ describe Lhm::LockedSwitcher do
11
+ include IntegrationHelper
12
+
13
+ before(:each) { connect_master! }
14
+
15
+ describe "switching" do
16
+ before(:each) do
17
+ @origin = table_create("origin")
18
+ @destination = table_create("destination")
19
+ @migration = Lhm::Migration.new(@origin, @destination)
20
+ end
21
+
22
+ it "rename origin to archive" do
23
+ switcher = Lhm::LockedSwitcher.new(@migration, connection)
24
+ switcher.run
25
+
26
+ slave do
27
+ table_exists?(@origin).must_equal true
28
+ table_read(@migration.archive_name).columns.keys.must_include "origin"
29
+ end
30
+ end
31
+
32
+ it "rename destination to origin" do
33
+ switcher = Lhm::LockedSwitcher.new(@migration, connection)
34
+ switcher.run
35
+
36
+ slave do
37
+ table_exists?(@destination).must_equal false
38
+ table_read(@origin.name).columns.keys.must_include "destination"
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,48 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
5
+
6
+ require 'lhm'
7
+ require 'lhm/table'
8
+
9
+ describe Lhm::Table do
10
+ include IntegrationHelper
11
+
12
+ describe Lhm::Table::Parser do
13
+ describe "create table parsing" do
14
+ before(:each) do
15
+ connect_master!
16
+ @table = table_create(:users)
17
+ end
18
+
19
+ it "should parse table name in show create table" do
20
+ @table.name.must_equal("users")
21
+ end
22
+
23
+ it "should parse primary key" do
24
+ @table.pk.must_equal("id")
25
+ end
26
+
27
+ it "should parse column type in show create table" do
28
+ @table.columns["username"][:type].must_equal("varchar(255)")
29
+ end
30
+
31
+ it "should parse column metadata" do
32
+ @table.columns["username"][:column_default].must_equal nil
33
+ end
34
+
35
+ it "should parse indices" do
36
+ @table.
37
+ indices["index_users_on_username_and_created_at"].
38
+ must_equal(["username", "created_at"])
39
+ end
40
+
41
+ it "should parse index" do
42
+ @table.
43
+ indices["index_users_on_reference"].
44
+ must_equal(["reference"])
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,31 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/atomic_switcher'
9
+
10
+ describe Lhm::AtomicSwitcher do
11
+ include UnitHelper
12
+
13
+ before(:each) do
14
+ @start = Time.now
15
+ @origin = Lhm::Table.new("origin")
16
+ @destination = Lhm::Table.new("destination")
17
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
18
+ @switcher = Lhm::AtomicSwitcher.new(@migration, nil)
19
+ end
20
+
21
+ describe "atomic switch" do
22
+ it "should perform a single atomic rename" do
23
+ @switcher.
24
+ statements.
25
+ must_equal([
26
+ "rename table `origin` to `#{ @migration.archive_name }`, " +
27
+ "`destination` to `origin`"
28
+ ])
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,111 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/chunker'
9
+
10
+ describe Lhm::Chunker do
11
+ include UnitHelper
12
+
13
+ before(:each) do
14
+ @origin = Lhm::Table.new("origin")
15
+ @destination = Lhm::Table.new("destination")
16
+ @migration = Lhm::Migration.new(@origin, @destination)
17
+ @chunker = Lhm::Chunker.new(@migration, nil, { :start => 1, :limit => 10 })
18
+ end
19
+
20
+ describe "copy into" do
21
+ before(:each) do
22
+ @origin.columns["secret"] = { :metadata => "VARCHAR(255)"}
23
+ @destination.columns["secret"] = { :metadata => "VARCHAR(255)"}
24
+ end
25
+
26
+ it "should copy the correct range and column" do
27
+ @chunker.copy(from = 1, to = 100).must_equal(
28
+ "insert ignore into `destination` (`secret`) " +
29
+ "select `secret` from `origin` " +
30
+ "where `id` between 1 and 100"
31
+ )
32
+ end
33
+ end
34
+
35
+ describe "invalid" do
36
+ before do
37
+ @chunker = Lhm::Chunker.new(@migration, nil, { :start => 0, :limit => -1 })
38
+ end
39
+
40
+ it "should have zero chunks" do
41
+ @chunker.traversable_chunks_size.must_equal 0
42
+ end
43
+
44
+ it "should not iterate" do
45
+ @chunker.up_to do |bottom, top|
46
+ raise "should not iterate"
47
+ end
48
+ end
49
+ end
50
+
51
+ describe "one" do
52
+ before do
53
+ @chunker = Lhm::Chunker.new(@migration, nil, {
54
+ :stride => 100_000, :start => 1, :limit => 300_000
55
+ })
56
+ end
57
+
58
+ it "should have one chunk" do
59
+ @chunker.traversable_chunks_size.must_equal 3
60
+ end
61
+
62
+ it "should lower bound chunk on 1" do
63
+ @chunker.bottom(chunk = 1).must_equal 1
64
+ end
65
+
66
+ it "should upper bound chunk on 100" do
67
+ @chunker.top(chunk = 1).must_equal 100_000
68
+ end
69
+ end
70
+
71
+ describe "two" do
72
+ before do
73
+ @chunker = Lhm::Chunker.new(@migration, nil, {
74
+ :stride => 100_000, :start => 2, :limit => 150_000
75
+ })
76
+ end
77
+
78
+ it "should have two chunks" do
79
+ @chunker.traversable_chunks_size.must_equal 2
80
+ end
81
+
82
+ it "should lower bound second chunk on 100_000" do
83
+ @chunker.bottom(chunk = 2).must_equal 100_002
84
+ end
85
+
86
+ it "should upper bound second chunk on 150_000" do
87
+ @chunker.top(chunk = 2).must_equal 150_000
88
+ end
89
+ end
90
+
91
+ describe "iterating" do
92
+ before do
93
+ @chunker = Lhm::Chunker.new(@migration, nil, {
94
+ :stride => 100, :start => 53, :limit => 121
95
+ })
96
+ end
97
+
98
+ it "should iterate" do
99
+ @chunker.up_to do |bottom, top|
100
+ bottom.must_equal 53
101
+ top.must_equal 121
102
+ end
103
+ end
104
+ end
105
+
106
+ describe "throttling" do
107
+ it "should default to 100 milliseconds" do
108
+ @chunker.throttle_seconds.must_equal 0.1
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,76 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/entangler'
9
+
10
+ describe Lhm::Entangler do
11
+ include UnitHelper
12
+
13
+ before(:each) do
14
+ @origin = Lhm::Table.new("origin")
15
+ @destination = Lhm::Table.new("destination")
16
+ @migration = Lhm::Migration.new(@origin, @destination)
17
+ @entangler = Lhm::Entangler.new(@migration)
18
+ end
19
+
20
+ describe "activation" do
21
+ before(:each) do
22
+ @origin.columns["info"] = { :type => "varchar(255)" }
23
+ @origin.columns["tags"] = { :type => "varchar(255)" }
24
+
25
+ @destination.columns["info"] = { :type => "varchar(255)" }
26
+ @destination.columns["tags"] = { :type => "varchar(255)" }
27
+ end
28
+
29
+ it "should create insert trigger to destination table" do
30
+ ddl = %Q{
31
+ create trigger `lhmt_ins_origin`
32
+ after insert on `origin` for each row
33
+ replace into `destination` (`info`, `tags`) /* large hadron migration */
34
+ values (NEW.`info`, NEW.`tags`)
35
+ }
36
+
37
+ @entangler.entangle.must_include strip(ddl)
38
+ end
39
+
40
+ it "should create an update trigger to the destination table" do
41
+ ddl = %Q{
42
+ create trigger `lhmt_upd_origin`
43
+ after update on `origin` for each row
44
+ replace into `destination` (`info`, `tags`) /* large hadron migration */
45
+ values (NEW.`info`, NEW.`tags`)
46
+ }
47
+
48
+ @entangler.entangle.must_include strip(ddl)
49
+ end
50
+
51
+ it "should create a delete trigger to the destination table" do
52
+ ddl = %Q{
53
+ create trigger `lhmt_del_origin`
54
+ after delete on `origin` for each row
55
+ delete ignore from `destination` /* large hadron migration */
56
+ where `destination`.`id` = OLD.`id`
57
+ }
58
+
59
+ @entangler.entangle.must_include strip(ddl)
60
+ end
61
+ end
62
+
63
+ describe "removal" do
64
+ it "should remove insert trigger" do
65
+ @entangler.untangle.must_include("drop trigger if exists `lhmt_ins_origin`")
66
+ end
67
+
68
+ it "should remove update trigger" do
69
+ @entangler.untangle.must_include("drop trigger if exists `lhmt_upd_origin`")
70
+ end
71
+
72
+ it "should remove delete trigger" do
73
+ @entangler.untangle.must_include("drop trigger if exists `lhmt_del_origin`")
74
+ end
75
+ end
76
+ end