sbader-lhm 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. data/.gitignore +6 -0
  2. data/.travis.yml +10 -0
  3. data/CHANGELOG.md +99 -0
  4. data/LICENSE +27 -0
  5. data/README.md +146 -0
  6. data/Rakefile +20 -0
  7. data/bin/lhm-kill-queue +172 -0
  8. data/bin/lhm-spec-clobber.sh +36 -0
  9. data/bin/lhm-spec-grants.sh +25 -0
  10. data/bin/lhm-spec-setup-cluster.sh +67 -0
  11. data/bin/lhm-test-all.sh +10 -0
  12. data/gemfiles/ar-2.3_mysql.gemfile +5 -0
  13. data/gemfiles/ar-3.2_mysql.gemfile +5 -0
  14. data/gemfiles/ar-3.2_mysql2.gemfile +5 -0
  15. data/lhm.gemspec +27 -0
  16. data/lib/lhm.rb +45 -0
  17. data/lib/lhm/atomic_switcher.rb +49 -0
  18. data/lib/lhm/chunker.rb +114 -0
  19. data/lib/lhm/command.rb +46 -0
  20. data/lib/lhm/entangler.rb +98 -0
  21. data/lib/lhm/intersection.rb +63 -0
  22. data/lib/lhm/invoker.rb +49 -0
  23. data/lib/lhm/locked_switcher.rb +71 -0
  24. data/lib/lhm/migration.rb +30 -0
  25. data/lib/lhm/migrator.rb +219 -0
  26. data/lib/lhm/sql_helper.rb +85 -0
  27. data/lib/lhm/table.rb +97 -0
  28. data/lib/lhm/version.rb +6 -0
  29. data/spec/.lhm.example +4 -0
  30. data/spec/README.md +51 -0
  31. data/spec/bootstrap.rb +13 -0
  32. data/spec/fixtures/destination.ddl +6 -0
  33. data/spec/fixtures/origin.ddl +6 -0
  34. data/spec/fixtures/small_table.ddl +4 -0
  35. data/spec/fixtures/users.ddl +12 -0
  36. data/spec/integration/atomic_switcher_spec.rb +42 -0
  37. data/spec/integration/chunker_spec.rb +32 -0
  38. data/spec/integration/entangler_spec.rb +66 -0
  39. data/spec/integration/integration_helper.rb +140 -0
  40. data/spec/integration/lhm_spec.rb +204 -0
  41. data/spec/integration/locked_switcher_spec.rb +42 -0
  42. data/spec/integration/table_spec.rb +48 -0
  43. data/spec/unit/atomic_switcher_spec.rb +31 -0
  44. data/spec/unit/chunker_spec.rb +111 -0
  45. data/spec/unit/entangler_spec.rb +76 -0
  46. data/spec/unit/intersection_spec.rb +39 -0
  47. data/spec/unit/locked_switcher_spec.rb +51 -0
  48. data/spec/unit/migration_spec.rb +23 -0
  49. data/spec/unit/migrator_spec.rb +134 -0
  50. data/spec/unit/sql_helper_spec.rb +32 -0
  51. data/spec/unit/table_spec.rb +34 -0
  52. data/spec/unit/unit_helper.rb +14 -0
  53. metadata +173 -0
@@ -0,0 +1,39 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migrator'
8
+
9
+ describe Lhm::Intersection do
10
+ include UnitHelper
11
+
12
+ it "should not have dropped changes" do
13
+ origin = Lhm::Table.new("origin")
14
+ origin.columns["dropped"] = varchar
15
+ origin.columns["retained"] = varchar
16
+
17
+ destination = Lhm::Table.new("destination")
18
+ destination.columns["retained"] = varchar
19
+
20
+ intersection = Lhm::Intersection.new(origin, destination)
21
+ intersection.common.include?("dropped").must_equal(false)
22
+ end
23
+
24
+ it "should have unchanged columns" do
25
+ origin = Lhm::Table.new("origin")
26
+ origin.columns["dropped"] = varchar
27
+ origin.columns["retained"] = varchar
28
+
29
+ destination = Lhm::Table.new("destination")
30
+ destination.columns["retained"] = varchar
31
+
32
+ intersection = Lhm::Intersection.new(origin, destination)
33
+ intersection.common.must_equal(["retained"])
34
+ end
35
+
36
+ def varchar
37
+ { :metadata => "VARCHAR(255)"}
38
+ end
39
+ end
@@ -0,0 +1,51 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+ require 'lhm/locked_switcher'
9
+
10
+ describe Lhm::LockedSwitcher do
11
+ include UnitHelper
12
+
13
+ before(:each) do
14
+ @start = Time.now
15
+ @origin = Lhm::Table.new("origin")
16
+ @destination = Lhm::Table.new("destination")
17
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
18
+ @switcher = Lhm::LockedSwitcher.new(@migration, nil)
19
+ end
20
+
21
+ describe "uncommitted" do
22
+ it "should disable autocommit first" do
23
+ @switcher.
24
+ statements[0..1].
25
+ must_equal([
26
+ "set @lhm_auto_commit = @@session.autocommit",
27
+ "set session autocommit = 0"
28
+ ])
29
+ end
30
+
31
+ it "should reapply original autocommit settings at the end" do
32
+ @switcher.
33
+ statements[-1].
34
+ must_equal("set session autocommit = @lhm_auto_commit")
35
+ end
36
+ end
37
+
38
+ describe "switch" do
39
+ it "should lock origin and destination table, switch, commit and unlock" do
40
+ @switcher.
41
+ switch.
42
+ must_equal([
43
+ "lock table `origin` write, `destination` write",
44
+ "alter table `origin` rename `#{ @migration.archive_name }`",
45
+ "alter table `destination` rename `origin`",
46
+ "commit",
47
+ "unlock tables"
48
+ ])
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,23 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migration'
8
+
9
+ describe Lhm::Migration do
10
+ include UnitHelper
11
+
12
+ before(:each) do
13
+ @start = Time.now
14
+ @origin = Lhm::Table.new("origin")
15
+ @destination = Lhm::Table.new("destination")
16
+ @migration = Lhm::Migration.new(@origin, @destination, @start)
17
+ end
18
+
19
+ it "should name archive" do
20
+ stamp = "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
21
+ @migration.archive_name.must_equal "lhma_#{ @start.strftime(stamp) }_origin"
22
+ end
23
+ end
@@ -0,0 +1,134 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/migrator'
8
+
9
+ describe Lhm::Migrator do
10
+ include UnitHelper
11
+
12
+ before(:each) do
13
+ @table = Lhm::Table.new("alt")
14
+ @creator = Lhm::Migrator.new(@table)
15
+ end
16
+
17
+ describe "index changes" do
18
+ it "should add an index" do
19
+ @creator.add_index(:a)
20
+
21
+ @creator.statements.must_equal([
22
+ "create index `index_alt_on_a` on `lhmn_alt` (`a`)"
23
+ ])
24
+ end
25
+
26
+ it "should add a composite index" do
27
+ @creator.add_index([:a, :b])
28
+
29
+ @creator.statements.must_equal([
30
+ "create index `index_alt_on_a_and_b` on `lhmn_alt` (`a`, `b`)"
31
+ ])
32
+ end
33
+
34
+ it "should add an index with prefix length" do
35
+ @creator.add_index(["a(10)", "b"])
36
+
37
+ @creator.statements.must_equal([
38
+ "create index `index_alt_on_a_and_b` on `lhmn_alt` (`a`(10), `b`)"
39
+ ])
40
+ end
41
+
42
+ it "should add an index with a custom name" do
43
+ @creator.add_index([:a, :b], :custom_index_name)
44
+
45
+ @creator.statements.must_equal([
46
+ "create index `custom_index_name` on `lhmn_alt` (`a`, `b`)"
47
+ ])
48
+ end
49
+
50
+ it "should add a unique index" do
51
+ @creator.add_unique_index(["a(5)", :b])
52
+
53
+ @creator.statements.must_equal([
54
+ "create unique index `index_alt_on_a_and_b` on `lhmn_alt` (`a`(5), `b`)"
55
+ ])
56
+ end
57
+
58
+ it "should add a unique index with a custom name" do
59
+ @creator.add_unique_index([:a, :b], :custom_index_name)
60
+
61
+ @creator.statements.must_equal([
62
+ "create unique index `custom_index_name` on `lhmn_alt` (`a`, `b`)"
63
+ ])
64
+ end
65
+
66
+ it "should remove an index" do
67
+ @creator.remove_index(["b", "a"])
68
+
69
+ @creator.statements.must_equal([
70
+ "drop index `index_alt_on_b_and_a` on `lhmn_alt`"
71
+ ])
72
+ end
73
+
74
+ it "should remove an index with a custom name" do
75
+ @creator.remove_index([:a, :b], :custom_index_name)
76
+
77
+ @creator.statements.must_equal([
78
+ "drop index `custom_index_name` on `lhmn_alt`"
79
+ ])
80
+ end
81
+ end
82
+
83
+ describe "column changes" do
84
+ it "should add a column" do
85
+ @creator.add_column("logins", "INT(12)")
86
+
87
+ @creator.statements.must_equal([
88
+ "alter table `lhmn_alt` add column `logins` INT(12)"
89
+ ])
90
+ end
91
+
92
+ it "should remove a column" do
93
+ @creator.remove_column("logins")
94
+
95
+ @creator.statements.must_equal([
96
+ "alter table `lhmn_alt` drop `logins`"
97
+ ])
98
+ end
99
+
100
+ it "should change a column" do
101
+ @creator.change_column("logins", "INT(11)")
102
+
103
+ @creator.statements.must_equal([
104
+ "alter table `lhmn_alt` modify column `logins` INT(11)"
105
+ ])
106
+ end
107
+ end
108
+
109
+ describe "direct changes" do
110
+ it "should accept a ddl statement" do
111
+ ddl = @creator.ddl("alter table `%s` add column `f` tinyint(1)" % @creator.name)
112
+
113
+ @creator.statements.must_equal([
114
+ "alter table `lhmn_alt` add column `f` tinyint(1)"
115
+ ])
116
+ end
117
+ end
118
+
119
+ describe "multiple changes" do
120
+ it "should add two columns" do
121
+ @creator.add_column("first", "VARCHAR(64)")
122
+ @creator.add_column("last", "VARCHAR(64)")
123
+ @creator.statements.length.must_equal(2)
124
+
125
+ @creator.
126
+ statements[0].
127
+ must_equal("alter table `lhmn_alt` add column `first` VARCHAR(64)")
128
+
129
+ @creator.
130
+ statements[1].
131
+ must_equal("alter table `lhmn_alt` add column `last` VARCHAR(64)")
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,32 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/sql_helper'
7
+
8
+ describe Lhm::SqlHelper do
9
+ it "should name index with a single column" do
10
+ Lhm::SqlHelper.
11
+ idx_name(:users, :name).
12
+ must_equal("index_users_on_name")
13
+ end
14
+
15
+ it "should name index with multiple columns" do
16
+ Lhm::SqlHelper.
17
+ idx_name(:users, [:name, :firstname]).
18
+ must_equal("index_users_on_name_and_firstname")
19
+ end
20
+
21
+ it "should name index with prefixed column" do
22
+ Lhm::SqlHelper.
23
+ idx_name(:tracks, ["title(10)", "album"]).
24
+ must_equal("index_tracks_on_title_and_album")
25
+ end
26
+
27
+ it "should quote column names in index specification" do
28
+ Lhm::SqlHelper.
29
+ idx_spec(["title(10)", "album"]).
30
+ must_equal("`title`(10), `album`")
31
+ end
32
+ end
@@ -0,0 +1,34 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
5
+
6
+ require 'lhm/table'
7
+
8
+ describe Lhm::Table do
9
+ include UnitHelper
10
+
11
+ describe "names" do
12
+ it "should name destination" do
13
+ @table = Lhm::Table.new("users")
14
+ @table.destination_name.must_equal "lhmn_users"
15
+ end
16
+ end
17
+
18
+ describe "constraints" do
19
+ it "should be satisfied with a single column primary key called id" do
20
+ @table = Lhm::Table.new("table", "id")
21
+ @table.satisfies_primary_key?.must_equal true
22
+ end
23
+
24
+ it "should not be satisfied with a primary key unless called id" do
25
+ @table = Lhm::Table.new("table", "uuid")
26
+ @table.satisfies_primary_key?.must_equal false
27
+ end
28
+
29
+ it "should not be satisfied with multicolumn primary key" do
30
+ @table = Lhm::Table.new("table", ["id", "secondary"])
31
+ @table.satisfies_primary_key?.must_equal false
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,14 @@
1
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+
4
+ require File.expand_path(File.dirname(__FILE__)) + "/../bootstrap"
5
+
6
+ module UnitHelper
7
+ def fixture(name)
8
+ File.read $fixtures.join(name)
9
+ end
10
+
11
+ def strip(sql)
12
+ sql.strip.gsub(/\n */, "\n")
13
+ end
14
+ end
metadata ADDED
@@ -0,0 +1,173 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sbader-lhm
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - SoundCloud
9
+ - Rany Keddo
10
+ - Tobias Bielohlawek
11
+ - Tobias Schmidt
12
+ autorequire:
13
+ bindir: bin
14
+ cert_chain: []
15
+ date: 2012-06-10 00:00:00.000000000 Z
16
+ dependencies:
17
+ - !ruby/object:Gem::Dependency
18
+ name: minitest
19
+ requirement: !ruby/object:Gem::Requirement
20
+ none: false
21
+ requirements:
22
+ - - '='
23
+ - !ruby/object:Gem::Version
24
+ version: 2.10.0
25
+ type: :development
26
+ prerelease: false
27
+ version_requirements: !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - '='
31
+ - !ruby/object:Gem::Version
32
+ version: 2.10.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: rake
35
+ requirement: !ruby/object:Gem::Requirement
36
+ none: false
37
+ requirements:
38
+ - - ! '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ type: :development
42
+ prerelease: false
43
+ version_requirements: !ruby/object:Gem::Requirement
44
+ none: false
45
+ requirements:
46
+ - - ! '>='
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ - !ruby/object:Gem::Dependency
50
+ name: activerecord
51
+ requirement: !ruby/object:Gem::Requirement
52
+ none: false
53
+ requirements:
54
+ - - ! '>='
55
+ - !ruby/object:Gem::Version
56
+ version: '0'
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: !ruby/object:Gem::Requirement
60
+ none: false
61
+ requirements:
62
+ - - ! '>='
63
+ - !ruby/object:Gem::Version
64
+ version: '0'
65
+ description: Migrate large tables without downtime by copying to a temporary table
66
+ in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name
67
+ for verification.
68
+ email: rany@soundcloud.com, tobi@soundcloud.com, ts@soundcloud.com
69
+ executables:
70
+ - lhm-kill-queue
71
+ extensions: []
72
+ extra_rdoc_files: []
73
+ files:
74
+ - .gitignore
75
+ - .travis.yml
76
+ - CHANGELOG.md
77
+ - LICENSE
78
+ - README.md
79
+ - Rakefile
80
+ - bin/lhm-kill-queue
81
+ - bin/lhm-spec-clobber.sh
82
+ - bin/lhm-spec-grants.sh
83
+ - bin/lhm-spec-setup-cluster.sh
84
+ - bin/lhm-test-all.sh
85
+ - gemfiles/ar-2.3_mysql.gemfile
86
+ - gemfiles/ar-3.2_mysql.gemfile
87
+ - gemfiles/ar-3.2_mysql2.gemfile
88
+ - lhm.gemspec
89
+ - lib/lhm.rb
90
+ - lib/lhm/atomic_switcher.rb
91
+ - lib/lhm/chunker.rb
92
+ - lib/lhm/command.rb
93
+ - lib/lhm/entangler.rb
94
+ - lib/lhm/intersection.rb
95
+ - lib/lhm/invoker.rb
96
+ - lib/lhm/locked_switcher.rb
97
+ - lib/lhm/migration.rb
98
+ - lib/lhm/migrator.rb
99
+ - lib/lhm/sql_helper.rb
100
+ - lib/lhm/table.rb
101
+ - lib/lhm/version.rb
102
+ - spec/.lhm.example
103
+ - spec/README.md
104
+ - spec/bootstrap.rb
105
+ - spec/fixtures/destination.ddl
106
+ - spec/fixtures/origin.ddl
107
+ - spec/fixtures/small_table.ddl
108
+ - spec/fixtures/users.ddl
109
+ - spec/integration/atomic_switcher_spec.rb
110
+ - spec/integration/chunker_spec.rb
111
+ - spec/integration/entangler_spec.rb
112
+ - spec/integration/integration_helper.rb
113
+ - spec/integration/lhm_spec.rb
114
+ - spec/integration/locked_switcher_spec.rb
115
+ - spec/integration/table_spec.rb
116
+ - spec/unit/atomic_switcher_spec.rb
117
+ - spec/unit/chunker_spec.rb
118
+ - spec/unit/entangler_spec.rb
119
+ - spec/unit/intersection_spec.rb
120
+ - spec/unit/locked_switcher_spec.rb
121
+ - spec/unit/migration_spec.rb
122
+ - spec/unit/migrator_spec.rb
123
+ - spec/unit/sql_helper_spec.rb
124
+ - spec/unit/table_spec.rb
125
+ - spec/unit/unit_helper.rb
126
+ homepage: http://github.com/soundcloud/large-hadron-migrator
127
+ licenses: []
128
+ post_install_message:
129
+ rdoc_options: []
130
+ require_paths:
131
+ - lib
132
+ required_ruby_version: !ruby/object:Gem::Requirement
133
+ none: false
134
+ requirements:
135
+ - - ! '>='
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
138
+ required_rubygems_version: !ruby/object:Gem::Requirement
139
+ none: false
140
+ requirements:
141
+ - - ! '>='
142
+ - !ruby/object:Gem::Version
143
+ version: '0'
144
+ requirements: []
145
+ rubyforge_project:
146
+ rubygems_version: 1.8.23
147
+ signing_key:
148
+ specification_version: 3
149
+ summary: online schema changer for mysql
150
+ test_files:
151
+ - spec/README.md
152
+ - spec/bootstrap.rb
153
+ - spec/fixtures/destination.ddl
154
+ - spec/fixtures/origin.ddl
155
+ - spec/fixtures/small_table.ddl
156
+ - spec/fixtures/users.ddl
157
+ - spec/integration/atomic_switcher_spec.rb
158
+ - spec/integration/chunker_spec.rb
159
+ - spec/integration/entangler_spec.rb
160
+ - spec/integration/integration_helper.rb
161
+ - spec/integration/lhm_spec.rb
162
+ - spec/integration/locked_switcher_spec.rb
163
+ - spec/integration/table_spec.rb
164
+ - spec/unit/atomic_switcher_spec.rb
165
+ - spec/unit/chunker_spec.rb
166
+ - spec/unit/entangler_spec.rb
167
+ - spec/unit/intersection_spec.rb
168
+ - spec/unit/locked_switcher_spec.rb
169
+ - spec/unit/migration_spec.rb
170
+ - spec/unit/migrator_spec.rb
171
+ - spec/unit/sql_helper_spec.rb
172
+ - spec/unit/table_spec.rb
173
+ - spec/unit/unit_helper.rb