lhm 1.0.0.rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,42 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+ # Determine and format columns common to origin and destination.
6
+ #
7
+
8
+ module Lhm
9
+ class Intersection
10
+ def initialize(origin, destination)
11
+ @origin = origin
12
+ @destination = destination
13
+ end
14
+
15
+ def common
16
+ @origin.columns.keys & @destination.columns.keys
17
+ end
18
+
19
+ def escaped
20
+ common.map { |name| tick(name) }
21
+ end
22
+
23
+ def joined
24
+ escaped.join(", ")
25
+ end
26
+
27
+ def typed(type)
28
+ common.map { |name| qualified(name, type) }.join(", ")
29
+ end
30
+
31
+ private
32
+
33
+ def qualified(name, type)
34
+ "#{ type }.`#{ name }`"
35
+ end
36
+
37
+ def tick(name)
38
+ "`#{ name }`"
39
+ end
40
+ end
41
+ end
42
+
@@ -0,0 +1,37 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+ # Copies an origin table to an altered destination table. Live activity is
6
+ # synchronized into the destination table using triggers.
7
+ #
8
+ # Once the origin and destination tables have converged, origin is archived
9
+ # and replaced by destination.
10
+ #
11
+
12
+ require 'lhm/chunker'
13
+ require 'lhm/entangler'
14
+ require 'lhm/locked_switcher'
15
+ require 'lhm/migration'
16
+ require 'lhm/migrator'
17
+
18
+ module Lhm
19
+ class Invoker
20
+ attr_reader :migrator
21
+
22
+ def initialize(origin, connection)
23
+ @connection = connection
24
+ @migrator = Migrator.new(origin, connection)
25
+ end
26
+
27
+ def run(chunk_options = {})
28
+ migration = @migrator.run
29
+
30
+ Entangler.new(migration, @connection).run do |tangler|
31
+ Chunker.new(migration, tangler.epoch, @connection, chunk_options).run
32
+ LockedSwitcher.new(migration, @connection).run
33
+ end
34
+ end
35
+ end
36
+ end
37
+
@@ -0,0 +1,78 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+ # Switches origin with destination table with a write lock. Use this as a safe
6
+ # alternative to rename, which can cause slave inconsistencies:
7
+ #
8
+ # http://bugs.mysql.com/bug.php?id=39675
9
+ #
10
+ # LockedSwitcher adopts the Facebook strategy, with the following caveat:
11
+ #
12
+ # "Since alter table causes an implicit commit in innodb, innodb locks get
13
+ # released after the first alter table. So any transaction that sneaks in
14
+ # after the first alter table and before the second alter table gets
15
+ # a 'table not found' error. The second alter table is expected to be very
16
+ # fast though because copytable is not visible to other transactions and so
17
+ # there is no need to wait."
18
+ #
19
+
20
+ require 'lhm/command'
21
+ require 'lhm/migration'
22
+
23
+ module Lhm
24
+ class LockedSwitcher
25
+ include Command
26
+
27
+ def initialize(migration, connection = nil)
28
+ @migration = migration
29
+ @connection = connection
30
+ @origin = migration.origin
31
+ @destination = migration.destination
32
+ end
33
+
34
+ def statements
35
+ uncommitted { switch }
36
+ end
37
+
38
+ def switch
39
+ [
40
+ "lock table `#{ @origin.name }` write, `#{ @destination.name }` write",
41
+ "alter table `#{ @origin.name }` rename `#{ @migration.archive_name }`",
42
+ "alter table `#{ @destination.name }` rename `#{ @origin.name }`",
43
+ "commit",
44
+ "unlock tables"
45
+ ]
46
+ end
47
+
48
+ def uncommitted(&block)
49
+ [
50
+ "set @lhm_auto_commit = @@session.autocommit",
51
+ "set session autocommit = 0",
52
+ *yield,
53
+ "set session autocommit = @lhm_auto_commit"
54
+ ]
55
+ end
56
+
57
+ #
58
+ # Command interface
59
+ #
60
+
61
+ def validate
62
+ unless table?(@origin.name) && table?(@destination.name)
63
+ error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
64
+ end
65
+ end
66
+
67
+ def revert
68
+ sql "unlock tables"
69
+ end
70
+
71
+ private
72
+
73
+ def execute
74
+ sql statements
75
+ end
76
+ end
77
+ end
78
+
@@ -0,0 +1,34 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+
6
+ require 'lhm/table'
7
+ require 'lhm/intersection'
8
+
9
+ module Lhm
10
+ class Migration
11
+ attr_reader :origin, :destination
12
+
13
+ def initialize(origin, destination, time = Time.now)
14
+ @origin = origin
15
+ @destination = destination
16
+ @start = time
17
+ end
18
+
19
+ def archive_name
20
+ "lhma_#{ startstamp }_#{ @origin.name }"
21
+ end
22
+
23
+ def intersection
24
+ Intersection.new(@origin, @destination)
25
+ end
26
+
27
+ private
28
+
29
+ def startstamp
30
+ @start.strftime "%Y_%m_%d_%H_%M_%S_#{ "%03d" % (@start.usec / 1000) }"
31
+ end
32
+ end
33
+ end
34
+
@@ -0,0 +1,125 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+ # Copies existing schema and applies changes using alter on the empty table.
6
+ # `run` returns a Migration which can be used for the remaining process.
7
+ #
8
+
9
+ require 'lhm/command'
10
+ require 'lhm/migration'
11
+
12
+ module Lhm
13
+ class Migrator
14
+ include Command
15
+
16
+ attr_reader :name, :statements
17
+
18
+ def initialize(table, connection = nil)
19
+ @connection = connection
20
+ @origin = table
21
+ @name = table.destination_name
22
+ @statements = []
23
+ end
24
+
25
+ def ddl(statement)
26
+ statements << statement
27
+ end
28
+
29
+ #
30
+ # Add a column to a table:
31
+ #
32
+ # hadron_change_table("users") do |t|
33
+ # t.add_column(:logins, "INT(12) DEFAULT '0'")
34
+ # end
35
+ #
36
+
37
+ def add_column(name, definition = "")
38
+ ddl = "alter table `%s` add column `%s` %s" % [@name, name, definition]
39
+ statements << ddl.strip
40
+ end
41
+
42
+ #
43
+ # Remove a column from a table:
44
+ #
45
+ # hadron_change_table("users") do |t|
46
+ # t.remove_column(:comment)
47
+ # end
48
+ #
49
+
50
+ def remove_column(name)
51
+ ddl = "alter table `%s` drop `%s`" % [@name, name]
52
+ statements << ddl.strip
53
+ end
54
+
55
+ #
56
+ # Add an index to a table:
57
+ #
58
+ # hadron_change_table("users") do |t|
59
+ # t.add_index([:comment, :created_at])
60
+ # end
61
+ #
62
+
63
+ def add_index(cols)
64
+ ddl = "create index `%s` on %s" % [@origin.idx_name(cols), idx_spec(cols)]
65
+ statements << ddl.strip
66
+ end
67
+
68
+ #
69
+ # Remove an index from a table
70
+ #
71
+ # hadron_change_table("users") do |t|
72
+ # t.remove_index(:username, :created_at)
73
+ # end
74
+ #
75
+
76
+ def remove_index(*cols)
77
+ ddl = "drop index `%s` on `%s`" % [@origin.idx_name(cols), @name]
78
+ statements << ddl.strip
79
+ end
80
+
81
+ #
82
+ # Command implementation
83
+ #
84
+
85
+ def validate
86
+ unless table?(@origin.name)
87
+ error("could not find origin table #{ @origin.name }")
88
+ end
89
+
90
+ unless @origin.satisfies_primary_key?
91
+ error("origin does not satisfy primary key requirements")
92
+ end
93
+
94
+ dest = @origin.destination_name
95
+
96
+ if table?(dest)
97
+ error("#{ dest } should not exist; not cleaned up from previous run?")
98
+ end
99
+ end
100
+
101
+ private
102
+
103
+ def execute
104
+ destination_create
105
+ sql(@statements)
106
+ Migration.new(@origin, destination_read)
107
+ end
108
+
109
+ def destination_create
110
+ original = "CREATE TABLE `#{ @origin.name }`"
111
+ replacement = "CREATE TABLE `#{ @origin.destination_name }`"
112
+
113
+ sql(@origin.ddl.gsub(original, replacement))
114
+ end
115
+
116
+ def destination_read
117
+ Table.parse(@origin.destination_name, connection)
118
+ end
119
+
120
+ def idx_spec(cols)
121
+ "#{ @name }(#{ [*cols].map(&:to_s).join(', ') })"
122
+ end
123
+ end
124
+ end
125
+
@@ -0,0 +1,87 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+
6
+ module Lhm
7
+ class Table
8
+ attr_reader :name, :columns, :indices, :pk, :ddl
9
+
10
+ def initialize(name, pk = "id", ddl = nil)
11
+ @name = name
12
+ @columns = {}
13
+ @indices = {}
14
+ @pk = pk
15
+ @ddl = ddl
16
+ end
17
+
18
+ def satisfies_primary_key?
19
+ @pk == "id"
20
+ end
21
+
22
+ def destination_name
23
+ "lhmn_#{ @name }"
24
+ end
25
+
26
+ def idx_name(cols)
27
+ "index_#{ @name }_on_" + [*cols].join("_and_")
28
+ end
29
+
30
+ def self.parse(table_name, connection)
31
+ sql = "show create table `#{ table_name }`"
32
+ ddl = connection.execute(sql).fetch_row.last
33
+
34
+ Parser.new(ddl).parse
35
+ end
36
+
37
+ class Parser
38
+ def initialize(ddl)
39
+ @ddl = ddl
40
+ end
41
+
42
+ def lines
43
+ @ddl.lines.to_a.map(&:strip).reject(&:empty?)
44
+ end
45
+
46
+ def create_definitions
47
+ lines[1..-2]
48
+ end
49
+
50
+ def parse
51
+ _, name = *lines.first.match("`([^ ]*)`")
52
+ pk_line = create_definitions.grep(primary).first
53
+
54
+ if pk_line
55
+ _, pk = *pk_line.match(primary)
56
+ table = Table.new(name, pk, @ddl)
57
+
58
+ create_definitions.each do |definition|
59
+ case definition
60
+ when index
61
+ table.indices[$1] = { :metadata => $2 }
62
+ when column
63
+ table.columns[$1] = { :type => $2, :metadata => $3 }
64
+ end
65
+ end
66
+
67
+ table
68
+ end
69
+ end
70
+
71
+ private
72
+
73
+ def primary
74
+ /^PRIMARY KEY (?:USING (?:HASH|[BR]TREE) )?\(`([^ ]*)`\),?$/
75
+ end
76
+
77
+ def index
78
+ /^(?:UNIQUE )?(?:INDEX|KEY) `([^ ]*)` (.*?),?$/
79
+ end
80
+
81
+ def column
82
+ /^`([^ ]*)` ([^ ]*) (.*?),?$/
83
+ end
84
+ end
85
+ end
86
+ end
87
+
@@ -0,0 +1,16 @@
1
+ #
2
+ # Copyright (c) 2011, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
3
+ # Schmidt
4
+ #
5
+
6
+ require 'minitest/spec'
7
+ require 'minitest/autorun'
8
+ require 'minitest/mock'
9
+ require "pathname"
10
+
11
+ $project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
12
+ $spec = $project.join("spec")
13
+ $fixtures = $spec.join("fixtures")
14
+
15
+ $: << $project.join("lib").to_s
16
+
@@ -0,0 +1,7 @@
1
+ CREATE TABLE `destination` (
2
+ `id` int(11) NOT NULL AUTO_INCREMENT,
3
+ `destination` int(11) DEFAULT NULL,
4
+ `common` varchar(255) DEFAULT NULL,
5
+ PRIMARY KEY (`id`)
6
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8
7
+
@@ -0,0 +1,7 @@
1
+ CREATE TABLE `origin` (
2
+ `id` int(11) NOT NULL AUTO_INCREMENT,
3
+ `origin` int(11) DEFAULT NULL,
4
+ `common` varchar(255) DEFAULT NULL,
5
+ PRIMARY KEY (`id`)
6
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8
7
+
@@ -0,0 +1,11 @@
1
+ CREATE TABLE `users` (
2
+ `id` int(11) NOT NULL AUTO_INCREMENT,
3
+ `reference` int(11) DEFAULT NULL,
4
+ `username` varchar(255) DEFAULT NULL,
5
+ `created_at` datetime DEFAULT NULL,
6
+ `comment` varchar(20) DEFAULT NULL,
7
+ PRIMARY KEY (`id`),
8
+ UNIQUE KEY `index_users_on_reference` (`reference`),
9
+ KEY `index_users_on_username_and_created_at` (`username`,`created_at`)
10
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8
11
+