lhm 1.0.0.rc.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +5 -0
- data/.travis.yml +10 -0
- data/CHANGELOG.md +32 -0
- data/Gemfile +3 -0
- data/LICENSE +27 -0
- data/README.md +98 -0
- data/Rakefile +16 -0
- data/TODO +11 -0
- data/lhm.gemspec +29 -0
- data/lib/lhm.rb +20 -0
- data/lib/lhm/chunker.rb +73 -0
- data/lib/lhm/command.rb +70 -0
- data/lib/lhm/entangler.rb +105 -0
- data/lib/lhm/intersection.rb +42 -0
- data/lib/lhm/invoker.rb +37 -0
- data/lib/lhm/locked_switcher.rb +78 -0
- data/lib/lhm/migration.rb +34 -0
- data/lib/lhm/migrator.rb +125 -0
- data/lib/lhm/table.rb +87 -0
- data/spec/bootstrap.rb +16 -0
- data/spec/fixtures/destination.ddl +7 -0
- data/spec/fixtures/origin.ddl +7 -0
- data/spec/fixtures/users.ddl +11 -0
- data/spec/integration/chunker_spec.rb +31 -0
- data/spec/integration/entangler_spec.rb +60 -0
- data/spec/integration/integration_helper.rb +74 -0
- data/spec/integration/lhm_spec.rb +118 -0
- data/spec/integration/locked_switcher_spec.rb +41 -0
- data/spec/unit/chunker_spec.rb +79 -0
- data/spec/unit/entangler_spec.rb +79 -0
- data/spec/unit/intersection_spec.rb +42 -0
- data/spec/unit/locked_switcher_spec.rb +54 -0
- data/spec/unit/migration_spec.rb +26 -0
- data/spec/unit/migrator_spec.rb +81 -0
- data/spec/unit/table_spec.rb +88 -0
- data/spec/unit/unit_helper.rb +17 -0
- metadata +165 -0
@@ -0,0 +1,42 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
# Determine and format columns common to origin and destination.
|
6
|
+
#
|
7
|
+
|
8
|
+
module Lhm
|
9
|
+
class Intersection
|
10
|
+
def initialize(origin, destination)
|
11
|
+
@origin = origin
|
12
|
+
@destination = destination
|
13
|
+
end
|
14
|
+
|
15
|
+
def common
|
16
|
+
@origin.columns.keys & @destination.columns.keys
|
17
|
+
end
|
18
|
+
|
19
|
+
def escaped
|
20
|
+
common.map { |name| tick(name) }
|
21
|
+
end
|
22
|
+
|
23
|
+
def joined
|
24
|
+
escaped.join(", ")
|
25
|
+
end
|
26
|
+
|
27
|
+
def typed(type)
|
28
|
+
common.map { |name| qualified(name, type) }.join(", ")
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def qualified(name, type)
|
34
|
+
"#{ type }.`#{ name }`"
|
35
|
+
end
|
36
|
+
|
37
|
+
def tick(name)
|
38
|
+
"`#{ name }`"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
data/lib/lhm/invoker.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
# Copies an origin table to an altered destination table. Live activity is
|
6
|
+
# synchronized into the destination table using triggers.
|
7
|
+
#
|
8
|
+
# Once the origin and destination tables have converged, origin is archived
|
9
|
+
# and replaced by destination.
|
10
|
+
#
|
11
|
+
|
12
|
+
require 'lhm/chunker'
|
13
|
+
require 'lhm/entangler'
|
14
|
+
require 'lhm/locked_switcher'
|
15
|
+
require 'lhm/migration'
|
16
|
+
require 'lhm/migrator'
|
17
|
+
|
18
|
+
module Lhm
|
19
|
+
class Invoker
|
20
|
+
attr_reader :migrator
|
21
|
+
|
22
|
+
def initialize(origin, connection)
|
23
|
+
@connection = connection
|
24
|
+
@migrator = Migrator.new(origin, connection)
|
25
|
+
end
|
26
|
+
|
27
|
+
def run(chunk_options = {})
|
28
|
+
migration = @migrator.run
|
29
|
+
|
30
|
+
Entangler.new(migration, @connection).run do |tangler|
|
31
|
+
Chunker.new(migration, tangler.epoch, @connection, chunk_options).run
|
32
|
+
LockedSwitcher.new(migration, @connection).run
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
@@ -0,0 +1,78 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
# Switches origin with destination table with a write lock. Use this as a safe
|
6
|
+
# alternative to rename, which can cause slave inconsistencies:
|
7
|
+
#
|
8
|
+
# http://bugs.mysql.com/bug.php?id=39675
|
9
|
+
#
|
10
|
+
# LockedSwitcher adopts the Facebook strategy, with the following caveat:
|
11
|
+
#
|
12
|
+
# "Since alter table causes an implicit commit in innodb, innodb locks get
|
13
|
+
# released after the first alter table. So any transaction that sneaks in
|
14
|
+
# after the first alter table and before the second alter table gets
|
15
|
+
# a 'table not found' error. The second alter table is expected to be very
|
16
|
+
# fast though because copytable is not visible to other transactions and so
|
17
|
+
# there is no need to wait."
|
18
|
+
#
|
19
|
+
|
20
|
+
require 'lhm/command'
|
21
|
+
require 'lhm/migration'
|
22
|
+
|
23
|
+
module Lhm
|
24
|
+
class LockedSwitcher
|
25
|
+
include Command
|
26
|
+
|
27
|
+
def initialize(migration, connection = nil)
|
28
|
+
@migration = migration
|
29
|
+
@connection = connection
|
30
|
+
@origin = migration.origin
|
31
|
+
@destination = migration.destination
|
32
|
+
end
|
33
|
+
|
34
|
+
def statements
|
35
|
+
uncommitted { switch }
|
36
|
+
end
|
37
|
+
|
38
|
+
def switch
|
39
|
+
[
|
40
|
+
"lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
|
41
|
+
"alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
|
42
|
+
"alter table `#{ @destination.name }` rename `#{ @origin.name }`",
|
43
|
+
"commit",
|
44
|
+
"unlock tables"
|
45
|
+
]
|
46
|
+
end
|
47
|
+
|
48
|
+
def uncommitted(&block)
|
49
|
+
[
|
50
|
+
"set @lhm_auto_commit = @@session.autocommit",
|
51
|
+
"set session autocommit = 0",
|
52
|
+
*yield,
|
53
|
+
"set session autocommit = @lhm_auto_commit"
|
54
|
+
]
|
55
|
+
end
|
56
|
+
|
57
|
+
#
|
58
|
+
# Command interface
|
59
|
+
#
|
60
|
+
|
61
|
+
def validate
|
62
|
+
unless table?(@origin.name) && table?(@destination.name)
|
63
|
+
error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def revert
|
68
|
+
sql "unlock tables"
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def execute
|
74
|
+
sql statements
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
|
6
|
+
require 'lhm/table'
|
7
|
+
require 'lhm/intersection'
|
8
|
+
|
9
|
+
module Lhm
|
10
|
+
class Migration
|
11
|
+
attr_reader :origin, :destination
|
12
|
+
|
13
|
+
def initialize(origin, destination, time = Time.now)
|
14
|
+
@origin = origin
|
15
|
+
@destination = destination
|
16
|
+
@start = time
|
17
|
+
end
|
18
|
+
|
19
|
+
def archive_name
|
20
|
+
"lhma_#{ startstamp }_#{ @origin.name }"
|
21
|
+
end
|
22
|
+
|
23
|
+
def intersection
|
24
|
+
Intersection.new(@origin, @destination)
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def startstamp
|
30
|
+
@start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
data/lib/lhm/migrator.rb
ADDED
@@ -0,0 +1,125 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
# Copies existing schema and applies changes using alter on the empty table.
|
6
|
+
# `run` returns a Migration which can be used for the remaining process.
|
7
|
+
#
|
8
|
+
|
9
|
+
require 'lhm/command'
|
10
|
+
require 'lhm/migration'
|
11
|
+
|
12
|
+
module Lhm
|
13
|
+
class Migrator
|
14
|
+
include Command
|
15
|
+
|
16
|
+
attr_reader :name, :statements
|
17
|
+
|
18
|
+
def initialize(table, connection = nil)
|
19
|
+
@connection = connection
|
20
|
+
@origin = table
|
21
|
+
@name = table.destination_name
|
22
|
+
@statements = []
|
23
|
+
end
|
24
|
+
|
25
|
+
def ddl(statement)
|
26
|
+
statements << statement
|
27
|
+
end
|
28
|
+
|
29
|
+
#
|
30
|
+
# Add a column to a table:
|
31
|
+
#
|
32
|
+
# hadron_change_table("users") do |t|
|
33
|
+
# t.add_column(:logins, "INT(12) DEFAULT '0'")
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
|
37
|
+
def add_column(name, definition = "")
|
38
|
+
ddl = "alter table `%s` add column `%s` %s" % [@name, name, definition]
|
39
|
+
statements << ddl.strip
|
40
|
+
end
|
41
|
+
|
42
|
+
#
|
43
|
+
# Remove a column from a table:
|
44
|
+
#
|
45
|
+
# hadron_change_table("users") do |t|
|
46
|
+
# t.remove_column(:comment)
|
47
|
+
# end
|
48
|
+
#
|
49
|
+
|
50
|
+
def remove_column(name)
|
51
|
+
ddl = "alter table `%s` drop `%s`" % [@name, name]
|
52
|
+
statements << ddl.strip
|
53
|
+
end
|
54
|
+
|
55
|
+
#
|
56
|
+
# Add an index to a table:
|
57
|
+
#
|
58
|
+
# hadron_change_table("users") do |t|
|
59
|
+
# t.add_index([:comment, :created_at])
|
60
|
+
# end
|
61
|
+
#
|
62
|
+
|
63
|
+
def add_index(cols)
|
64
|
+
ddl = "create index `%s` on %s" % [@origin.idx_name(cols), idx_spec(cols)]
|
65
|
+
statements << ddl.strip
|
66
|
+
end
|
67
|
+
|
68
|
+
#
|
69
|
+
# Remove an index from a table
|
70
|
+
#
|
71
|
+
# hadron_change_table("users") do |t|
|
72
|
+
# t.remove_index(:username, :created_at)
|
73
|
+
# end
|
74
|
+
#
|
75
|
+
|
76
|
+
def remove_index(*cols)
|
77
|
+
ddl = "drop index `%s` on `%s`" % [@origin.idx_name(cols), @name]
|
78
|
+
statements << ddl.strip
|
79
|
+
end
|
80
|
+
|
81
|
+
#
|
82
|
+
# Command implementation
|
83
|
+
#
|
84
|
+
|
85
|
+
def validate
|
86
|
+
unless table?(@origin.name)
|
87
|
+
error("could not find origin table #{ @origin.name }")
|
88
|
+
end
|
89
|
+
|
90
|
+
unless @origin.satisfies_primary_key?
|
91
|
+
error("origin does not satisfy primary key requirements")
|
92
|
+
end
|
93
|
+
|
94
|
+
dest = @origin.destination_name
|
95
|
+
|
96
|
+
if table?(dest)
|
97
|
+
error("#{ dest } should not exist; not cleaned up from previous run?")
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
|
103
|
+
def execute
|
104
|
+
destination_create
|
105
|
+
sql(@statements)
|
106
|
+
Migration.new(@origin, destination_read)
|
107
|
+
end
|
108
|
+
|
109
|
+
def destination_create
|
110
|
+
original = "CREATE TABLE `#{ @origin.name }`"
|
111
|
+
replacement = "CREATE TABLE `#{ @origin.destination_name }`"
|
112
|
+
|
113
|
+
sql(@origin.ddl.gsub(original, replacement))
|
114
|
+
end
|
115
|
+
|
116
|
+
def destination_read
|
117
|
+
Table.parse(@origin.destination_name, connection)
|
118
|
+
end
|
119
|
+
|
120
|
+
def idx_spec(cols)
|
121
|
+
"#{ @name }(#{ [*cols].map(&:to_s).join(', ') })"
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
data/lib/lhm/table.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
|
6
|
+
module Lhm
|
7
|
+
class Table
|
8
|
+
attr_reader :name, :columns, :indices, :pk, :ddl
|
9
|
+
|
10
|
+
def initialize(name, pk = "id", ddl = nil)
|
11
|
+
@name = name
|
12
|
+
@columns = {}
|
13
|
+
@indices = {}
|
14
|
+
@pk = pk
|
15
|
+
@ddl = ddl
|
16
|
+
end
|
17
|
+
|
18
|
+
def satisfies_primary_key?
|
19
|
+
@pk == "id"
|
20
|
+
end
|
21
|
+
|
22
|
+
def destination_name
|
23
|
+
"lhmn_#{ @name }"
|
24
|
+
end
|
25
|
+
|
26
|
+
def idx_name(cols)
|
27
|
+
"index_#{ @name }_on_" + [*cols].join("_and_")
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.parse(table_name, connection)
|
31
|
+
sql = "show create table `#{ table_name }`"
|
32
|
+
ddl = connection.execute(sql).fetch_row.last
|
33
|
+
|
34
|
+
Parser.new(ddl).parse
|
35
|
+
end
|
36
|
+
|
37
|
+
class Parser
|
38
|
+
def initialize(ddl)
|
39
|
+
@ddl = ddl
|
40
|
+
end
|
41
|
+
|
42
|
+
def lines
|
43
|
+
@ddl.lines.to_a.map(&:strip).reject(&:empty?)
|
44
|
+
end
|
45
|
+
|
46
|
+
def create_definitions
|
47
|
+
lines[1..-2]
|
48
|
+
end
|
49
|
+
|
50
|
+
def parse
|
51
|
+
_, name = *lines.first.match("`([^ ]*)`")
|
52
|
+
pk_line = create_definitions.grep(primary).first
|
53
|
+
|
54
|
+
if pk_line
|
55
|
+
_, pk = *pk_line.match(primary)
|
56
|
+
table = Table.new(name, pk, @ddl)
|
57
|
+
|
58
|
+
create_definitions.each do |definition|
|
59
|
+
case definition
|
60
|
+
when index
|
61
|
+
table.indices[$1] = { :metadata => $2 }
|
62
|
+
when column
|
63
|
+
table.columns[$1] = { :type => $2, :metadata => $3 }
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
table
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def primary
|
74
|
+
/^PRIMARY KEY (?:USING (?:HASH|[BR]TREE) )?\(`([^ ]*)`\),?$/
|
75
|
+
end
|
76
|
+
|
77
|
+
def index
|
78
|
+
/^(?:UNIQUE )?(?:INDEX|KEY) `([^ ]*)` (.*?),?$/
|
79
|
+
end
|
80
|
+
|
81
|
+
def column
|
82
|
+
/^`([^ ]*)` ([^ ]*) (.*?),?$/
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
data/spec/bootstrap.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
3
|
+
# Schmidt
|
4
|
+
#
|
5
|
+
|
6
|
+
require 'minitest/spec'
|
7
|
+
require 'minitest/autorun'
|
8
|
+
require 'minitest/mock'
|
9
|
+
require "pathname"
|
10
|
+
|
11
|
+
$project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
|
12
|
+
$spec = $project.join("spec")
|
13
|
+
$fixtures = $spec.join("fixtures")
|
14
|
+
|
15
|
+
$: << $project.join("lib").to_s
|
16
|
+
|
@@ -0,0 +1,11 @@
|
|
1
|
+
CREATE TABLE `users` (
|
2
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
3
|
+
`reference` int(11) DEFAULT NULL,
|
4
|
+
`username` varchar(255) DEFAULT NULL,
|
5
|
+
`created_at` datetime DEFAULT NULL,
|
6
|
+
`comment` varchar(20) DEFAULT NULL,
|
7
|
+
PRIMARY KEY (`id`),
|
8
|
+
UNIQUE KEY `index_users_on_reference` (`reference`),
|
9
|
+
KEY `index_users_on_username_and_created_at` (`username`,`created_at`)
|
10
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8
|
11
|
+
|