lhm 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +5 -0
- data/.travis.yml +3 -1
- data/README.md +14 -6
- data/Rakefile +4 -2
- data/bin/lhm-spec-clobber.sh +4 -3
- data/bin/lhm-spec-setup-cluster.sh +1 -2
- data/lhm.gemspec +1 -2
- data/lib/lhm.rb +39 -10
- data/lib/lhm/chunker.rb +42 -38
- data/lib/lhm/command.rb +3 -1
- data/lib/lhm/invoker.rb +30 -9
- data/lib/lhm/printer.rb +56 -0
- data/lib/lhm/throttler.rb +32 -0
- data/lib/lhm/throttler/time.rb +29 -0
- data/lib/lhm/version.rb +1 -1
- data/spec/.lhm.example +1 -1
- data/spec/README.md +20 -13
- data/spec/fixtures/lines.ddl +7 -0
- data/spec/integration/chunker_spec.rb +9 -3
- data/spec/integration/cleanup_spec.rb +19 -5
- data/spec/integration/integration_helper.rb +26 -18
- data/spec/integration/lhm_spec.rb +22 -2
- data/spec/integration/table_spec.rb +0 -2
- data/spec/{bootstrap.rb → test_helper.rb} +17 -2
- data/spec/unit/chunker_spec.rb +86 -77
- data/spec/unit/datamapper_connection_spec.rb +1 -0
- data/spec/unit/lhm_spec.rb +29 -0
- data/spec/unit/printer_spec.rb +79 -0
- data/spec/unit/throttler_spec.rb +73 -0
- data/spec/unit/unit_helper.rb +1 -13
- metadata +22 -17
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZWY5MTRiZmFmYTBkNzg0NDhhNjM1MjBlOGM5OWQ1YTljMTkzOTYxOQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MjE3ODI5NDk3MjUxNmRlNzFkN2ZmYTVmZjZmNWM1OTk3MzA1N2NiNA==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MjBlNGMyNjllMDU5Y2ZmY2E0MjQwOWY0YzIzODk5YjQ0ZDAxOWVlMjU2OTA0
|
10
|
+
ZWUxYmJmMzFiYzg1MTMzNmMzYzI5OWU3Y2M2MjBmOTQ3MzllZjA5MmViMGNh
|
11
|
+
MzQ5NmM0YzVmZGMxZWQ3NzdiNmIxMzI1MGE2ZDM0MTU4NGU1NjQ=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZmFlNTRkMDkyOWJlOTkzZjA2ZTdiNjA1MGNjZWVjODNiMGY3OWE1MWY2YmVk
|
14
|
+
ZWFmZGFhMGJhODNjNGVjZGE1NmQyZWRjMTg5NzBkZDY5NWMxNDJmNWE2NTU3
|
15
|
+
ODc4MmYxMWRmODM5NDdiNTRkNmVlMGQwZTEzNmE5MzM4YzlhMzY=
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -182,23 +182,31 @@ during the run will happen on the new table as well.
|
|
182
182
|
## Cleaning up after an interrupted Lhm run
|
183
183
|
|
184
184
|
If an Lhm migration is interrupted, it may leave behind the temporary tables
|
185
|
-
used in the migration. If the migration is re-started, the
|
186
|
-
of these tables will cause an error.
|
187
|
-
to drop any orphaned Lhm temporary tables.
|
185
|
+
and/or triggers used in the migration. If the migration is re-started, the
|
186
|
+
unexpected presence of these tables will cause an error.
|
188
187
|
|
189
|
-
|
188
|
+
In this case, `Lhm.cleanup` can be used to drop any orphaned Lhm temporary tables or triggers.
|
189
|
+
|
190
|
+
To see what Lhm tables/triggers are found:
|
190
191
|
|
191
192
|
```ruby
|
192
193
|
Lhm.cleanup
|
193
194
|
```
|
194
195
|
|
195
|
-
To remove any Lhm tables found:
|
196
|
+
To remove any Lhm tables/triggers found:
|
196
197
|
```ruby
|
197
198
|
Lhm.cleanup(true)
|
198
199
|
```
|
199
200
|
|
200
201
|
## Contributing
|
201
202
|
|
203
|
+
First, get set up for local development:
|
204
|
+
|
205
|
+
git clone git://github.com/soundcloud/lhm.git
|
206
|
+
cd lhm
|
207
|
+
|
208
|
+
To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md).
|
209
|
+
|
202
210
|
We'll check out your contribution if you:
|
203
211
|
|
204
212
|
* Provide a comprehensive suite of tests for your fork.
|
@@ -223,4 +231,4 @@ The license is included as LICENSE in this directory.
|
|
223
231
|
[2]: https://github.com/freels/table_migrator
|
224
232
|
[3]: http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html
|
225
233
|
[4]: https://travis-ci.org/soundcloud/lhm
|
226
|
-
[5]: https://travis-ci.org/soundcloud/lhm.
|
234
|
+
[5]: https://travis-ci.org/soundcloud/lhm.svg?branch=master
|
data/Rakefile
CHANGED
@@ -4,13 +4,15 @@ require 'bundler'
|
|
4
4
|
Bundler::GemHelper.install_tasks
|
5
5
|
|
6
6
|
Rake::TestTask.new("unit") do |t|
|
7
|
-
t.libs
|
7
|
+
t.libs << 'lib'
|
8
|
+
t.libs << 'spec'
|
8
9
|
t.test_files = FileList['spec/unit/*_spec.rb']
|
9
10
|
t.verbose = true
|
10
11
|
end
|
11
12
|
|
12
13
|
Rake::TestTask.new("integration") do |t|
|
13
|
-
t.libs
|
14
|
+
t.libs << 'lib'
|
15
|
+
t.libs << 'spec'
|
14
16
|
t.test_files = FileList['spec/integration/*_spec.rb']
|
15
17
|
t.verbose = true
|
16
18
|
end
|
data/bin/lhm-spec-clobber.sh
CHANGED
@@ -8,12 +8,13 @@ source ~/.lhm
|
|
8
8
|
lhmkill() {
|
9
9
|
echo killing lhm-cluster
|
10
10
|
ps -ef | sed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
|
11
|
+
echo running homebrew mysql instance
|
12
|
+
launchctl load -w ~/Library/LaunchAgents/homebrew.mxcl.mysql.plist
|
11
13
|
sleep 2
|
12
14
|
}
|
13
15
|
|
14
|
-
echo stopping
|
15
|
-
|
16
|
-
"$mysqldir"/bin/mysqladmin shutdown || { echo mysqladmin did not shut down anything; }
|
16
|
+
echo stopping homebrew running mysql instance
|
17
|
+
ls -lrt -d -1 ~/Library/LaunchAgents/* | grep 'mysql.plist' | xargs launchctl unload -w
|
17
18
|
|
18
19
|
lhmkill
|
19
20
|
|
@@ -13,7 +13,6 @@ source ~/.lhm
|
|
13
13
|
# Main
|
14
14
|
#
|
15
15
|
|
16
|
-
install_bin="$(echo ./*/mysql_install_db)"
|
17
16
|
|
18
17
|
mkdir -p "$basedir/master/data" "$basedir/slave/data"
|
19
18
|
|
@@ -61,7 +60,7 @@ CNF
|
|
61
60
|
|
62
61
|
(
|
63
62
|
cd "$mysqldir"
|
63
|
+
install_bin="$(echo ./*/mysql_install_db | tr " " "\\n" | head -1)"
|
64
64
|
$install_bin --datadir="$basedir/master/data"
|
65
65
|
$install_bin --datadir="$basedir/slave/data"
|
66
|
-
|
67
66
|
)
|
data/lhm.gemspec
CHANGED
data/lib/lhm.rb
CHANGED
@@ -4,7 +4,9 @@
|
|
4
4
|
require 'lhm/table'
|
5
5
|
require 'lhm/invoker'
|
6
6
|
require 'lhm/connection'
|
7
|
+
require 'lhm/throttler'
|
7
8
|
require 'lhm/version'
|
9
|
+
require 'logger'
|
8
10
|
|
9
11
|
# Large hadron migrator - online schema change tool
|
10
12
|
#
|
@@ -17,6 +19,10 @@ require 'lhm/version'
|
|
17
19
|
# end
|
18
20
|
#
|
19
21
|
module Lhm
|
22
|
+
extend Throttler
|
23
|
+
extend self
|
24
|
+
|
25
|
+
DEFAULT_LOGGER_OPTIONS = { level: Logger::INFO, file: STDOUT }
|
20
26
|
|
21
27
|
# Alters a table with the changes described in the block
|
22
28
|
#
|
@@ -37,7 +43,7 @@ module Lhm
|
|
37
43
|
# @yield [Migrator] Yielded Migrator object records the changes
|
38
44
|
# @return [Boolean] Returns true if the migration finishes
|
39
45
|
# @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
|
40
|
-
def
|
46
|
+
def change_table(table_name, options = {}, &block)
|
41
47
|
origin = Table.parse(table_name, connection)
|
42
48
|
invoker = Invoker.new(origin, connection)
|
43
49
|
block.call(invoker.migrator)
|
@@ -45,28 +51,36 @@ module Lhm
|
|
45
51
|
true
|
46
52
|
end
|
47
53
|
|
48
|
-
def
|
49
|
-
lhm_tables = connection.select_values("show tables").select
|
50
|
-
|
51
|
-
|
52
|
-
|
54
|
+
def cleanup(run = false)
|
55
|
+
lhm_tables = connection.select_values("show tables").select { |name| name =~ /^lhm(a|n)_/ }
|
56
|
+
lhm_triggers = connection.select_values("show triggers").collect do |trigger|
|
57
|
+
trigger.respond_to?(:trigger) ? trigger.trigger : trigger
|
58
|
+
end.select { |name| name =~ /^lhmt/ }
|
59
|
+
|
53
60
|
if run
|
61
|
+
lhm_triggers.each do |trigger|
|
62
|
+
connection.execute("drop trigger if exists #{trigger}")
|
63
|
+
end
|
54
64
|
lhm_tables.each do |table|
|
55
|
-
connection.execute("drop table #{table}")
|
65
|
+
connection.execute("drop table if exists #{table}")
|
56
66
|
end
|
57
67
|
true
|
68
|
+
elsif lhm_tables.empty? && lhm_triggers.empty?
|
69
|
+
puts "Everything is clean. Nothing to do."
|
70
|
+
true
|
58
71
|
else
|
59
72
|
puts "Existing LHM backup tables: #{lhm_tables.join(", ")}."
|
73
|
+
puts "Existing LHM triggers: #{lhm_triggers.join(", ")}."
|
60
74
|
puts "Run Lhm.cleanup(true) to drop them all."
|
61
75
|
false
|
62
76
|
end
|
63
77
|
end
|
64
78
|
|
65
|
-
def
|
79
|
+
def setup(adapter)
|
66
80
|
@@adapter = adapter
|
67
81
|
end
|
68
82
|
|
69
|
-
def
|
83
|
+
def adapter
|
70
84
|
@@adapter ||=
|
71
85
|
begin
|
72
86
|
raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
|
@@ -74,9 +88,24 @@ module Lhm
|
|
74
88
|
end
|
75
89
|
end
|
76
90
|
|
91
|
+
def self.logger=(new_logger)
|
92
|
+
@@logger = new_logger
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.logger
|
96
|
+
@@logger ||=
|
97
|
+
begin
|
98
|
+
logger = Logger.new(DEFAULT_LOGGER_OPTIONS[:file])
|
99
|
+
logger.level = DEFAULT_LOGGER_OPTIONS[:level]
|
100
|
+
logger.formatter = nil
|
101
|
+
logger
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
77
105
|
protected
|
78
106
|
|
79
|
-
def
|
107
|
+
def connection
|
80
108
|
Connection.new(adapter)
|
81
109
|
end
|
110
|
+
|
82
111
|
end
|
data/lib/lhm/chunker.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
2
2
|
# Schmidt
|
3
|
-
|
4
3
|
require 'lhm/command'
|
5
4
|
require 'lhm/sql_helper'
|
5
|
+
require 'lhm/printer'
|
6
6
|
|
7
7
|
module Lhm
|
8
8
|
class Chunker
|
@@ -16,55 +16,72 @@ module Lhm
|
|
16
16
|
def initialize(migration, connection = nil, options = {})
|
17
17
|
@migration = migration
|
18
18
|
@connection = connection
|
19
|
-
@
|
20
|
-
@throttle = options[:throttle] || 100
|
19
|
+
@throttler = options[:throttler]
|
21
20
|
@start = options[:start] || select_start
|
22
21
|
@limit = options[:limit] || select_limit
|
22
|
+
@printer = options[:printer] || Printer::Percentage.new
|
23
23
|
end
|
24
24
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
25
|
+
def execute
|
26
|
+
return unless @start && @limit
|
27
|
+
@next_to_insert = @start
|
28
|
+
until @next_to_insert >= @limit
|
29
|
+
stride = @throttler.stride
|
30
|
+
affected_rows = @connection.update(copy(bottom, top(stride)))
|
31
|
+
|
32
|
+
if @throttler && affected_rows > 0
|
33
|
+
@throttler.run
|
34
|
+
end
|
35
|
+
|
36
|
+
@printer.notify(bottom, @limit)
|
37
|
+
@next_to_insert = top(stride) + 1
|
29
38
|
end
|
39
|
+
@printer.end
|
30
40
|
end
|
31
41
|
|
32
|
-
|
33
|
-
@limit && @start ? ((@limit - @start + 1) / @stride.to_f).ceil : 0
|
34
|
-
end
|
42
|
+
private
|
35
43
|
|
36
|
-
def bottom
|
37
|
-
|
44
|
+
def bottom
|
45
|
+
@next_to_insert
|
38
46
|
end
|
39
47
|
|
40
|
-
def top(
|
41
|
-
[
|
48
|
+
def top(stride)
|
49
|
+
[(@next_to_insert + stride - 1), @limit].min
|
42
50
|
end
|
43
51
|
|
44
52
|
def copy(lowest, highest)
|
45
53
|
"insert ignore into `#{ destination_name }` (#{ columns }) " +
|
46
54
|
"select #{ select_columns } from `#{ origin_name }` " +
|
47
|
-
"#{ conditions }
|
55
|
+
"#{ conditions } `#{ origin_name }`.`id` between #{ lowest } and #{ highest }"
|
48
56
|
end
|
49
57
|
|
50
58
|
def select_start
|
51
|
-
start = connection.select_value("select min(id) from
|
59
|
+
start = connection.select_value("select min(id) from `#{ origin_name }`")
|
52
60
|
start ? start.to_i : nil
|
53
61
|
end
|
54
62
|
|
55
63
|
def select_limit
|
56
|
-
limit = connection.select_value("select max(id) from
|
64
|
+
limit = connection.select_value("select max(id) from `#{ origin_name }`")
|
57
65
|
limit ? limit.to_i : nil
|
58
66
|
end
|
59
67
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
68
|
+
#XXX this is extremely brittle and doesn't work when filter contains more
|
69
|
+
#than one SQL clause, e.g. "where ... group by foo". Before making any
|
70
|
+
#more changes here, please consider either:
|
71
|
+
#
|
72
|
+
#1. Letting users only specify part of defined clauses (i.e. don't allow
|
73
|
+
#`filter` on Migrator to accept both WHERE and INNER JOIN
|
74
|
+
#2. Changing query building so that it uses structured data rather than
|
75
|
+
#strings until the last possible moment.
|
66
76
|
def conditions
|
67
|
-
|
77
|
+
if @migration.conditions
|
78
|
+
@migration.conditions.
|
79
|
+
sub(/\)\Z/, "").
|
80
|
+
#put any where conditions in parens
|
81
|
+
sub(/where\s(\w.*)\Z/, "where (\\1)") + " and"
|
82
|
+
else
|
83
|
+
"where"
|
84
|
+
end
|
68
85
|
end
|
69
86
|
|
70
87
|
def destination_name
|
@@ -80,7 +97,7 @@ module Lhm
|
|
80
97
|
end
|
81
98
|
|
82
99
|
def select_columns
|
83
|
-
@select_columns ||= @migration.intersection.typed(origin_name)
|
100
|
+
@select_columns ||= @migration.intersection.typed("`#{origin_name}`")
|
84
101
|
end
|
85
102
|
|
86
103
|
def validate
|
@@ -88,18 +105,5 @@ module Lhm
|
|
88
105
|
error("impossible chunk options (limit must be greater than start)")
|
89
106
|
end
|
90
107
|
end
|
91
|
-
|
92
|
-
def execute
|
93
|
-
up_to do |lowest, highest|
|
94
|
-
affected_rows = @connection.update(copy(lowest, highest))
|
95
|
-
|
96
|
-
if affected_rows > 0
|
97
|
-
sleep(throttle_seconds)
|
98
|
-
end
|
99
|
-
|
100
|
-
print "."
|
101
|
-
end
|
102
|
-
print "\n"
|
103
|
-
end
|
104
108
|
end
|
105
109
|
end
|
data/lib/lhm/command.rb
CHANGED
@@ -7,6 +7,7 @@ module Lhm
|
|
7
7
|
|
8
8
|
module Command
|
9
9
|
def run(&block)
|
10
|
+
Lhm.logger.info "Starting run of class=#{self.class}"
|
10
11
|
validate
|
11
12
|
|
12
13
|
if(block_given?)
|
@@ -16,7 +17,8 @@ module Lhm
|
|
16
17
|
else
|
17
18
|
execute
|
18
19
|
end
|
19
|
-
rescue
|
20
|
+
rescue => e
|
21
|
+
Lhm.logger.error "Error in class=#{self.class}, reverting. exception=#{e.class} message=#{e.message}"
|
20
22
|
revert
|
21
23
|
raise
|
22
24
|
end
|
data/lib/lhm/invoker.rb
CHANGED
@@ -24,6 +24,24 @@ module Lhm
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def run(options = {})
|
27
|
+
normalize_options(options)
|
28
|
+
migration = @migrator.run
|
29
|
+
|
30
|
+
Entangler.new(migration, @connection).run do
|
31
|
+
Chunker.new(migration, @connection, options).run
|
32
|
+
if options[:atomic_switch]
|
33
|
+
AtomicSwitcher.new(migration, @connection).run
|
34
|
+
else
|
35
|
+
LockedSwitcher.new(migration, @connection).run
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def normalize_options(options)
|
43
|
+
Lhm.logger.info "Starting LHM run on table=#{@migrator.name}"
|
44
|
+
|
27
45
|
if !options.include?(:atomic_switch)
|
28
46
|
if supports_atomic_switch?
|
29
47
|
options[:atomic_switch] = true
|
@@ -34,16 +52,19 @@ module Lhm
|
|
34
52
|
end
|
35
53
|
end
|
36
54
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
end
|
55
|
+
if options[:throttler]
|
56
|
+
options[:throttler] = Throttler::Factory.create_throttler(*options[:throttler])
|
57
|
+
elsif options[:throttle] || options[:stride]
|
58
|
+
# we still support the throttle and stride as a Fixnum input
|
59
|
+
warn "throttle option will no longer accept a Fixnum in the next versions."
|
60
|
+
options[:throttler] = Throttler::LegacyTime.new(options[:throttle], options[:stride])
|
61
|
+
else
|
62
|
+
options[:throttler] = Lhm.throttler
|
46
63
|
end
|
64
|
+
|
65
|
+
rescue => e
|
66
|
+
Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
|
67
|
+
raise
|
47
68
|
end
|
48
69
|
end
|
49
70
|
end
|