lhm 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ ZWY5MTRiZmFmYTBkNzg0NDhhNjM1MjBlOGM5OWQ1YTljMTkzOTYxOQ==
5
+ data.tar.gz: !binary |-
6
+ MjE3ODI5NDk3MjUxNmRlNzFkN2ZmYTVmZjZmNWM1OTk3MzA1N2NiNA==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ MjBlNGMyNjllMDU5Y2ZmY2E0MjQwOWY0YzIzODk5YjQ0ZDAxOWVlMjU2OTA0
10
+ ZWUxYmJmMzFiYzg1MTMzNmMzYzI5OWU3Y2M2MjBmOTQ3MzllZjA5MmViMGNh
11
+ MzQ5NmM0YzVmZGMxZWQ3NzdiNmIxMzI1MGE2ZDM0MTU4NGU1NjQ=
12
+ data.tar.gz: !binary |-
13
+ ZmFlNTRkMDkyOWJlOTkzZjA2ZTdiNjA1MGNjZWVjODNiMGY3OWE1MWY2YmVk
14
+ ZWFmZGFhMGJhODNjNGVjZGE1NmQyZWRjMTg5NzBkZDY5NWMxNDJmNWE2NTU3
15
+ ODc4MmYxMWRmODM5NDdiNTRkNmVlMGQwZTEzNmE5MzM4YzlhMzY=
data/.gitignore CHANGED
@@ -6,3 +6,8 @@ pkg/*
6
6
  .rvmrc
7
7
  .ruby-version
8
8
  .ruby-gemset
9
+ bin/rake
10
+ spec/integration/database.yml
11
+ Gemfile
12
+ gemfiles/vendor
13
+ omg.ponies
@@ -4,9 +4,11 @@ before_script:
4
4
  rvm:
5
5
  - 1.9.3
6
6
  - 2.0.0
7
+ - 2.1.1
7
8
  matrix:
8
9
  allow_failures:
9
- - rvm: 2.0.0
10
+ - gemfile: gemfiles/ar-2.3_mysql.gemfile
11
+ fast_finish: true
10
12
  gemfile:
11
13
  - gemfiles/ar-2.3_mysql.gemfile
12
14
  - gemfiles/ar-3.2_mysql.gemfile
data/README.md CHANGED
@@ -182,23 +182,31 @@ during the run will happen on the new table as well.
182
182
  ## Cleaning up after an interrupted Lhm run
183
183
 
184
184
  If an Lhm migration is interrupted, it may leave behind the temporary tables
185
- used in the migration. If the migration is re-started, the unexpected presence
186
- of these tables will cause an error. In this case, `Lhm.cleanup` can be used
187
- to drop any orphaned Lhm temporary tables.
185
+ and/or triggers used in the migration. If the migration is re-started, the
186
+ unexpected presence of these tables will cause an error.
188
187
 
189
- To see what Lhm tables are found:
188
+ In this case, `Lhm.cleanup` can be used to drop any orphaned Lhm temporary tables or triggers.
189
+
190
+ To see what Lhm tables/triggers are found:
190
191
 
191
192
  ```ruby
192
193
  Lhm.cleanup
193
194
  ```
194
195
 
195
- To remove any Lhm tables found:
196
+ To remove any Lhm tables/triggers found:
196
197
  ```ruby
197
198
  Lhm.cleanup(true)
198
199
  ```
199
200
 
200
201
  ## Contributing
201
202
 
203
+ First, get set up for local development:
204
+
205
+ git clone git://github.com/soundcloud/lhm.git
206
+ cd lhm
207
+
208
+ To run the tests, follow the instructions on [spec/README](https://github.com/soundcloud/lhm/blob/master/spec/README.md).
209
+
202
210
  We'll check out your contribution if you:
203
211
 
204
212
  * Provide a comprehensive suite of tests for your fork.
@@ -223,4 +231,4 @@ The license is included as LICENSE in this directory.
223
231
  [2]: https://github.com/freels/table_migrator
224
232
  [3]: http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html
225
233
  [4]: https://travis-ci.org/soundcloud/lhm
226
- [5]: https://travis-ci.org/soundcloud/lhm.png?branch=master
234
+ [5]: https://travis-ci.org/soundcloud/lhm.svg?branch=master
data/Rakefile CHANGED
@@ -4,13 +4,15 @@ require 'bundler'
4
4
  Bundler::GemHelper.install_tasks
5
5
 
6
6
  Rake::TestTask.new("unit") do |t|
7
- t.libs.push "lib"
7
+ t.libs << 'lib'
8
+ t.libs << 'spec'
8
9
  t.test_files = FileList['spec/unit/*_spec.rb']
9
10
  t.verbose = true
10
11
  end
11
12
 
12
13
  Rake::TestTask.new("integration") do |t|
13
- t.libs.push "lib"
14
+ t.libs << 'lib'
15
+ t.libs << 'spec'
14
16
  t.test_files = FileList['spec/integration/*_spec.rb']
15
17
  t.verbose = true
16
18
  end
@@ -8,12 +8,13 @@ source ~/.lhm
8
8
  lhmkill() {
9
9
  echo killing lhm-cluster
10
10
  ps -ef | sed -n "/[m]ysqld.*lhm-cluster/p" | awk '{ print $2 }' | xargs kill
11
+ echo running homebrew mysql instance
12
+ launchctl load -w ~/Library/LaunchAgents/homebrew.mxcl.mysql.plist
11
13
  sleep 2
12
14
  }
13
15
 
14
- echo stopping other running mysql instance
15
- launchctl remove com.mysql.mysqld || { echo launchctl did not remove mysqld; }
16
- "$mysqldir"/bin/mysqladmin shutdown || { echo mysqladmin did not shut down anything; }
16
+ echo stopping homebrew running mysql instance
17
+ ls -lrt -d -1 ~/Library/LaunchAgents/* | grep 'mysql.plist' | xargs launchctl unload -w
17
18
 
18
19
  lhmkill
19
20
 
@@ -13,7 +13,6 @@ source ~/.lhm
13
13
  # Main
14
14
  #
15
15
 
16
- install_bin="$(echo ./*/mysql_install_db)"
17
16
 
18
17
  mkdir -p "$basedir/master/data" "$basedir/slave/data"
19
18
 
@@ -61,7 +60,7 @@ CNF
61
60
 
62
61
  (
63
62
  cd "$mysqldir"
63
+ install_bin="$(echo ./*/mysql_install_db | tr " " "\\n" | head -1)"
64
64
  $install_bin --datadir="$basedir/master/data"
65
65
  $install_bin --datadir="$basedir/slave/data"
66
-
67
66
  )
@@ -19,7 +19,6 @@ Gem::Specification.new do |s|
19
19
  s.require_paths = ["lib"]
20
20
  s.executables = ["lhm-kill-queue"]
21
21
 
22
- s.add_development_dependency "minitest", "= 2.10.0"
22
+ s.add_development_dependency "minitest", "~> 5.0.8"
23
23
  s.add_development_dependency "rake"
24
24
  end
25
-
data/lib/lhm.rb CHANGED
@@ -4,7 +4,9 @@
4
4
  require 'lhm/table'
5
5
  require 'lhm/invoker'
6
6
  require 'lhm/connection'
7
+ require 'lhm/throttler'
7
8
  require 'lhm/version'
9
+ require 'logger'
8
10
 
9
11
  # Large hadron migrator - online schema change tool
10
12
  #
@@ -17,6 +19,10 @@ require 'lhm/version'
17
19
  # end
18
20
  #
19
21
  module Lhm
22
+ extend Throttler
23
+ extend self
24
+
25
+ DEFAULT_LOGGER_OPTIONS = { level: Logger::INFO, file: STDOUT }
20
26
 
21
27
  # Alters a table with the changes described in the block
22
28
  #
@@ -37,7 +43,7 @@ module Lhm
37
43
  # @yield [Migrator] Yielded Migrator object records the changes
38
44
  # @return [Boolean] Returns true if the migration finishes
39
45
  # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
40
- def self.change_table(table_name, options = {}, &block)
46
+ def change_table(table_name, options = {}, &block)
41
47
  origin = Table.parse(table_name, connection)
42
48
  invoker = Invoker.new(origin, connection)
43
49
  block.call(invoker.migrator)
@@ -45,28 +51,36 @@ module Lhm
45
51
  true
46
52
  end
47
53
 
48
- def self.cleanup(run = false)
49
- lhm_tables = connection.select_values("show tables").select do |name|
50
- name =~ /^lhm(a|n)_/
51
- end
52
- return true if lhm_tables.empty?
54
+ def cleanup(run = false)
55
+ lhm_tables = connection.select_values("show tables").select { |name| name =~ /^lhm(a|n)_/ }
56
+ lhm_triggers = connection.select_values("show triggers").collect do |trigger|
57
+ trigger.respond_to?(:trigger) ? trigger.trigger : trigger
58
+ end.select { |name| name =~ /^lhmt/ }
59
+
53
60
  if run
61
+ lhm_triggers.each do |trigger|
62
+ connection.execute("drop trigger if exists #{trigger}")
63
+ end
54
64
  lhm_tables.each do |table|
55
- connection.execute("drop table #{table}")
65
+ connection.execute("drop table if exists #{table}")
56
66
  end
57
67
  true
68
+ elsif lhm_tables.empty? && lhm_triggers.empty?
69
+ puts "Everything is clean. Nothing to do."
70
+ true
58
71
  else
59
72
  puts "Existing LHM backup tables: #{lhm_tables.join(", ")}."
73
+ puts "Existing LHM triggers: #{lhm_triggers.join(", ")}."
60
74
  puts "Run Lhm.cleanup(true) to drop them all."
61
75
  false
62
76
  end
63
77
  end
64
78
 
65
- def self.setup(adapter)
79
+ def setup(adapter)
66
80
  @@adapter = adapter
67
81
  end
68
82
 
69
- def self.adapter
83
+ def adapter
70
84
  @@adapter ||=
71
85
  begin
72
86
  raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
@@ -74,9 +88,24 @@ module Lhm
74
88
  end
75
89
  end
76
90
 
91
+ def self.logger=(new_logger)
92
+ @@logger = new_logger
93
+ end
94
+
95
+ def self.logger
96
+ @@logger ||=
97
+ begin
98
+ logger = Logger.new(DEFAULT_LOGGER_OPTIONS[:file])
99
+ logger.level = DEFAULT_LOGGER_OPTIONS[:level]
100
+ logger.formatter = nil
101
+ logger
102
+ end
103
+ end
104
+
77
105
  protected
78
106
 
79
- def self.connection
107
+ def connection
80
108
  Connection.new(adapter)
81
109
  end
110
+
82
111
  end
@@ -1,8 +1,8 @@
1
1
  # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
2
  # Schmidt
3
-
4
3
  require 'lhm/command'
5
4
  require 'lhm/sql_helper'
5
+ require 'lhm/printer'
6
6
 
7
7
  module Lhm
8
8
  class Chunker
@@ -16,55 +16,72 @@ module Lhm
16
16
  def initialize(migration, connection = nil, options = {})
17
17
  @migration = migration
18
18
  @connection = connection
19
- @stride = options[:stride] || 40_000
20
- @throttle = options[:throttle] || 100
19
+ @throttler = options[:throttler]
21
20
  @start = options[:start] || select_start
22
21
  @limit = options[:limit] || select_limit
22
+ @printer = options[:printer] || Printer::Percentage.new
23
23
  end
24
24
 
25
- # Copies chunks of size `stride`, starting from `start` up to id `limit`.
26
- def up_to(&block)
27
- 1.upto(traversable_chunks_size) do |n|
28
- yield(bottom(n), top(n))
25
+ def execute
26
+ return unless @start && @limit
27
+ @next_to_insert = @start
28
+ until @next_to_insert >= @limit
29
+ stride = @throttler.stride
30
+ affected_rows = @connection.update(copy(bottom, top(stride)))
31
+
32
+ if @throttler && affected_rows > 0
33
+ @throttler.run
34
+ end
35
+
36
+ @printer.notify(bottom, @limit)
37
+ @next_to_insert = top(stride) + 1
29
38
  end
39
+ @printer.end
30
40
  end
31
41
 
32
- def traversable_chunks_size
33
- @limit && @start ? ((@limit - @start + 1) / @stride.to_f).ceil : 0
34
- end
42
+ private
35
43
 
36
- def bottom(chunk)
37
- (chunk - 1) * @stride + @start
44
+ def bottom
45
+ @next_to_insert
38
46
  end
39
47
 
40
- def top(chunk)
41
- [chunk * @stride + @start - 1, @limit].min
48
+ def top(stride)
49
+ [(@next_to_insert + stride - 1), @limit].min
42
50
  end
43
51
 
44
52
  def copy(lowest, highest)
45
53
  "insert ignore into `#{ destination_name }` (#{ columns }) " +
46
54
  "select #{ select_columns } from `#{ origin_name }` " +
47
- "#{ conditions } #{ origin_name }.`id` between #{ lowest } and #{ highest }"
55
+ "#{ conditions } `#{ origin_name }`.`id` between #{ lowest } and #{ highest }"
48
56
  end
49
57
 
50
58
  def select_start
51
- start = connection.select_value("select min(id) from #{ origin_name }")
59
+ start = connection.select_value("select min(id) from `#{ origin_name }`")
52
60
  start ? start.to_i : nil
53
61
  end
54
62
 
55
63
  def select_limit
56
- limit = connection.select_value("select max(id) from #{ origin_name }")
64
+ limit = connection.select_value("select max(id) from `#{ origin_name }`")
57
65
  limit ? limit.to_i : nil
58
66
  end
59
67
 
60
- def throttle_seconds
61
- @throttle / 1000.0
62
- end
63
-
64
- private
65
-
68
+ #XXX this is extremely brittle and doesn't work when filter contains more
69
+ #than one SQL clause, e.g. "where ... group by foo". Before making any
70
+ #more changes here, please consider either:
71
+ #
72
+ #1. Letting users only specify part of defined clauses (i.e. don't allow
73
+ #`filter` on Migrator to accept both WHERE and INNER JOIN
74
+ #2. Changing query building so that it uses structured data rather than
75
+ #strings until the last possible moment.
66
76
  def conditions
67
- @migration.conditions ? "#{@migration.conditions} and" : "where"
77
+ if @migration.conditions
78
+ @migration.conditions.
79
+ sub(/\)\Z/, "").
80
+ #put any where conditions in parens
81
+ sub(/where\s(\w.*)\Z/, "where (\\1)") + " and"
82
+ else
83
+ "where"
84
+ end
68
85
  end
69
86
 
70
87
  def destination_name
@@ -80,7 +97,7 @@ module Lhm
80
97
  end
81
98
 
82
99
  def select_columns
83
- @select_columns ||= @migration.intersection.typed(origin_name)
100
+ @select_columns ||= @migration.intersection.typed("`#{origin_name}`")
84
101
  end
85
102
 
86
103
  def validate
@@ -88,18 +105,5 @@ module Lhm
88
105
  error("impossible chunk options (limit must be greater than start)")
89
106
  end
90
107
  end
91
-
92
- def execute
93
- up_to do |lowest, highest|
94
- affected_rows = @connection.update(copy(lowest, highest))
95
-
96
- if affected_rows > 0
97
- sleep(throttle_seconds)
98
- end
99
-
100
- print "."
101
- end
102
- print "\n"
103
- end
104
108
  end
105
109
  end
@@ -7,6 +7,7 @@ module Lhm
7
7
 
8
8
  module Command
9
9
  def run(&block)
10
+ Lhm.logger.info "Starting run of class=#{self.class}"
10
11
  validate
11
12
 
12
13
  if(block_given?)
@@ -16,7 +17,8 @@ module Lhm
16
17
  else
17
18
  execute
18
19
  end
19
- rescue
20
+ rescue => e
21
+ Lhm.logger.error "Error in class=#{self.class}, reverting. exception=#{e.class} message=#{e.message}"
20
22
  revert
21
23
  raise
22
24
  end
@@ -24,6 +24,24 @@ module Lhm
24
24
  end
25
25
 
26
26
  def run(options = {})
27
+ normalize_options(options)
28
+ migration = @migrator.run
29
+
30
+ Entangler.new(migration, @connection).run do
31
+ Chunker.new(migration, @connection, options).run
32
+ if options[:atomic_switch]
33
+ AtomicSwitcher.new(migration, @connection).run
34
+ else
35
+ LockedSwitcher.new(migration, @connection).run
36
+ end
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def normalize_options(options)
43
+ Lhm.logger.info "Starting LHM run on table=#{@migrator.name}"
44
+
27
45
  if !options.include?(:atomic_switch)
28
46
  if supports_atomic_switch?
29
47
  options[:atomic_switch] = true
@@ -34,16 +52,19 @@ module Lhm
34
52
  end
35
53
  end
36
54
 
37
- migration = @migrator.run
38
-
39
- Entangler.new(migration, @connection).run do
40
- Chunker.new(migration, @connection, options).run
41
- if options[:atomic_switch]
42
- AtomicSwitcher.new(migration, @connection).run
43
- else
44
- LockedSwitcher.new(migration, @connection).run
45
- end
55
+ if options[:throttler]
56
+ options[:throttler] = Throttler::Factory.create_throttler(*options[:throttler])
57
+ elsif options[:throttle] || options[:stride]
58
+ # we still support the throttle and stride as a Fixnum input
59
+ warn "throttle option will no longer accept a Fixnum in the next versions."
60
+ options[:throttler] = Throttler::LegacyTime.new(options[:throttle], options[:stride])
61
+ else
62
+ options[:throttler] = Lhm.throttler
46
63
  end
64
+
65
+ rescue => e
66
+ Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
67
+ raise
47
68
  end
48
69
  end
49
70
  end