mysql_big_table_migration 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2cf0a701c21368899da7dd1cb5ca1004a403513b
4
+ data.tar.gz: f2d290bdf85b48ba4140e360e730c7de86a50b1f
5
+ SHA512:
6
+ metadata.gz: caf05416fd733107981ddee51522b5c893c47d612da106525c5658503ee52ce1887a84d56b50b2bdc961ae714e479fea5a3c9bad9b78acb9c6e6631e440ad429
7
+ data.tar.gz: f19ea5701c1c20fd916368b5eb8b7e9fa84634340c2a03cb55de79e6df1a6b954a25490d487c6203a3a4f5597668e17fb787e59018aa3abdb9f7d8e87214f885
@@ -0,0 +1,5 @@
1
+ *.*~
2
+ *.swp
3
+ test/debug.log
4
+ Gemfile.lock
5
+ pkg/
@@ -0,0 +1,23 @@
1
+ language: ruby
2
+ env:
3
+ - DB=mysql
4
+ rvm:
5
+ - 1.9
6
+ - 2.1
7
+ - 2.2
8
+ - rbx-2.5.2
9
+ bundler_args: --without production
10
+ before_script:
11
+ - mysql -e 'create database mysql_big_table_migration_test'
12
+ before_install:
13
+ - gem update --system
14
+ services:
15
+ - mysql
16
+ script:
17
+ - bundle install
18
+ - bundle exec rake test
19
+ cache:
20
+ - bundler
21
+ os:
22
+ - linux
23
+ - osx
@@ -0,0 +1,17 @@
1
+ TODO:
2
+
3
+ * Support change_table :foo do |t| style migrations (with option to activate temp table)
4
+ * Support ActiveRecord::Migration::CommandRecorder (rake db:rollback)
5
+ * Ensure index names based on original table
6
+
7
+ 0.1.2:
8
+
9
+ * Moved repo from thickpaddy/mysql_big_table_migration to MakeYourLaws/mysql_big_table_migration
10
+ * Added dependencies on ActiveRecord & Rails, removed unused rdoc, updated to minitest, moved from Rails::VERSION to ActiveRecord::VERSION
11
+ * Fixed tests - mysql can't convert 'foo0' to an integer
12
+ * Moved dependencies into gemspec, updated Gemfile.lock
13
+ * Pushed to RubyGems
14
+
15
+ <= 0.1.1:
16
+
17
+ See https://github.com/thickpaddy/mysql_big_table_migration/commits/master
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2010 [name of plugin creator]
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,58 @@
1
+ [![Build Status](https://travis-ci.org/MakeYourLaws/mysql_big_table_migration.svg)](https://travis-ci.org/MakeYourLaws/mysql_big_table_migration)
2
+
3
+ MysqlBigTableMigration
4
+ ======================
5
+
6
+ A Rails plugin that adds methods to ActiveRecord::Migration to allow columns
7
+ and indexes to be added to and removed from large tables with millions of
8
+ rows in MySQL, without leaving processes seemingly stalled in state "copy
9
+ to tmp table".
10
+
11
+ For each of the standard transformations that operate on columns or indexes,
12
+ this plugin adds a "using_tmp_table" version. These methods create a
13
+ temporary table with the same structure as the table to be altered, applies
14
+ the transformation to the temp table, copies data from the source table to
15
+ the temp table and then replaces the source table with the temporary one.
16
+
17
+ While it does try to ensure that data is consistent at the end of the entire
18
+ process by locking tables and looking for rows created or modified during
19
+ copying, this is NOT TRANSACTION SAFE as it (a) relies on timestamp columns
20
+ and (b) doesn't handle rows that have been deleted from the source table
21
+ after being copied to the temporary table.
22
+
23
+ Installation
24
+ ============
25
+
26
+ source 'https://rubygems.org' do
27
+ ...
28
+ gem 'mysql_big_table_migration'
29
+ ...
30
+ end
31
+
32
+
33
+ Example
34
+ =======
35
+
36
+
37
+ class AddIndexOnSomeColumnToSomeTable < ActiveRecord::Migration
38
+ def self.up
39
+ add_index_using_tmp_table :some_table, :some_column
40
+ end
41
+
42
+ def self.down
43
+ remove_index_using_tmp_table :some_table, :some_column
44
+ end
45
+ end
46
+
47
+
48
+ Copyright (c) 2010 Mark Woods, released under the MIT license
49
+
50
+ Testing
51
+ ========
52
+
53
+ You will need to bundle install dependencies for the project, as well as create a test database. To install dependencies run `bundle install`. To create the database run the following in a MySQL prompt as an admin user:
54
+
55
+ CREATE DATABASE mysql_big_table_migration_test;
56
+ GRANT ALL PRIVILEGES ON *.* TO 'dev'@'localhost' IDENTIFIED BY 'password';
57
+
58
+ After bundling dependencies, and setting up the test database, run the tests with `rake test`
@@ -0,0 +1,15 @@
1
+ require 'rake'
2
+ require 'rake/testtask'
3
+
4
+ desc 'Default: run unit tests.'
5
+ task :default => :test
6
+
7
+ desc 'Test the mysql_big_table_migration plugin.'
8
+ Rake::TestTask.new(:test) do |t|
9
+ t.libs << 'lib'
10
+ t.pattern = 'test/*_test.rb'
11
+ t.verbose = true
12
+ end
13
+
14
+ Bundler::GemHelper.install_tasks
15
+ Dir.glob('tasks/*.rake').each { |r| import r }
data/init.rb ADDED
@@ -0,0 +1 @@
1
+ require 'mysql_big_table_migration'
@@ -0,0 +1 @@
1
+ # Install hook code here
@@ -0,0 +1,195 @@
1
+ module MySQLBigTableMigration
2
+
3
+ SUPPORTED_ADAPTERS = [
4
+ "ActiveRecord::ConnectionAdapters::MysqlAdapter",
5
+ "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
6
+ ]
7
+
8
+ def rename_column(table, old_name, new_name)
9
+ push_state unless @state_stack.try(:any?)
10
+ current_state[:renames] ||= {}
11
+ current_state[:renames][old_name] = new_name
12
+ super
13
+ end
14
+
15
+ def add_column_using_tmp_table(table_name, *args)
16
+ with_tmp_table(table_name) do |tmp_table_name|
17
+ begin
18
+ add_column tmp_table_name, *args
19
+ rescue ActiveRecord::StatementInvalid => e
20
+ raise unless e.message.include?("Duplicate column name")
21
+ end
22
+ end
23
+ end
24
+
25
+ def remove_column_using_tmp_table(table_name, column_name)
26
+ with_tmp_table(table_name) do |tmp_table_name|
27
+ begin
28
+ remove_column tmp_table_name, column_name
29
+ rescue ActiveRecord::StatementInvalid => e
30
+ raise unless e.message.include?("check that column/key exists")
31
+ end
32
+ end
33
+ end
34
+
35
+ def rename_column_using_tmp_table(table_name, column_name, new_column_name)
36
+ with_tmp_table(table_name) { |tmp_table_name| rename_column(tmp_table_name, column_name, new_column_name) }
37
+ end
38
+
39
+ def change_column_using_tmp_table(table_name, column_name, type, options={})
40
+ with_tmp_table(table_name) { |tmp_table_name| change_column(tmp_table_name, column_name, type, options) }
41
+ end
42
+
43
+ def add_index_using_tmp_table(table_name, column_name, options={})
44
+ # generate the index name using the original table name if no name provided
45
+ options[:name] = index_name(table_name, :column => Array(column_name)) if options[:name].nil?
46
+ with_tmp_table(table_name) { |tmp_table_name| add_index(tmp_table_name, column_name, options) }
47
+ end
48
+
49
+ def remove_index_using_tmp_table(table_name, options={})
50
+ with_tmp_table(table_name) { |tmp_table_name| remove_index(tmp_table_name, :name => index_name(table_name, options)) }
51
+ end
52
+
53
+ def with_tmp_table(table_name)
54
+ raise ArgumentError, "block expected" unless block_given?
55
+
56
+ unless SUPPORTED_ADAPTERS.include? connection.class.name
57
+ puts "Warning: Unsupported connection adapter '#{connection.class.name}' for MySQL Big Table Migration Plugin"
58
+ puts " Migration methods will still be executed, but without using a temp table."
59
+ yield table_name
60
+ return
61
+ end
62
+
63
+ table_name = table_name.to_s
64
+ new_table_name = "tmp_new_" + table_name
65
+ old_table_name = "tmp_old_" + table_name
66
+
67
+ begin
68
+ say "Creating temporary table #{new_table_name} like #{table_name}..."
69
+ connection.execute("CREATE TABLE #{new_table_name} LIKE #{table_name}")
70
+
71
+ # yield the temporary table name to the block, which should alter the table using standard migration methods
72
+ push_state
73
+ yield new_table_name
74
+ state = pop_state
75
+
76
+ rename_columns ||= state[:renames] || {}
77
+
78
+ # get column names to copy *after* yielding to block - could drop a column from new table
79
+ # note: do not get column names using the column_names method, we need to make sure we avoid obtaining a cached array of column names
80
+ old_column_names = []
81
+ each_result_hash(connection.execute("DESCRIBE #{table_name}")) { |row| old_column_names << row['Field'] } # see ruby mysql docs for more info
82
+ new_column_names = []
83
+ each_result_hash(connection.execute("DESCRIBE #{new_table_name}")) { |row| new_column_names << row['Field'] }
84
+
85
+ shared_columns = old_column_names & new_column_names
86
+
87
+ old_column_list = column_list(shared_columns + rename_columns.keys)
88
+ new_column_list = column_list(shared_columns + rename_columns.values)
89
+
90
+ timestamp_before_migration = fetch_result_row(connection.execute("SELECT CURRENT_TIMESTAMP"))[0] # note: string, not time object
91
+ max_id_before_migration = fetch_result_row(connection.execute("SELECT MAX(id) FROM #{table_name}"))[0].to_i
92
+
93
+ if max_id_before_migration == 0
94
+ say "Source table is empty, no rows to copy into temporary table"
95
+ else
96
+ batch_size = mysql_big_table_migration_bach_size
97
+ start = fetch_result_row(connection.execute("SELECT MIN(id) FROM #{table_name}"))[0].to_i
98
+ counter = start
99
+ say "Inserting into temporary table in batches of #{batch_size}..."
100
+ say "Approximately #{max_id_before_migration-start+1} rows to process, first row has id #{start}", true
101
+ while counter <= ( max = fetch_result_row(connection.execute("SELECT MAX(id) FROM #{table_name}"))[0].to_i )
102
+ percentage_complete = mysql_big_table_migration_completion start, counter, max
103
+ say "Processing rows with ids between #{counter} and #{(counter+batch_size)-1} (#{percentage_complete}% complete)", true
104
+ connection.execute("INSERT INTO #{new_table_name} (#{new_column_list}) SELECT #{old_column_list} FROM #{table_name} WHERE id >= #{counter} AND id < #{counter + batch_size}")
105
+ counter = counter + batch_size
106
+ end
107
+ say "Finished inserting into temporary table"
108
+ end
109
+
110
+ rescue Exception => e
111
+ drop_table new_table_name
112
+ raise
113
+ end
114
+
115
+ say "Replacing source table with temporary table..."
116
+ rename_table table_name, old_table_name
117
+ rename_table new_table_name, table_name
118
+
119
+ say "Cleaning up, checking for rows created/updated during migration, dropping old table..."
120
+ begin
121
+ connection.execute("LOCK TABLES #{table_name} WRITE, #{old_table_name} READ")
122
+ recently_created_or_updated_conditions = "id > #{max_id_before_migration}"
123
+ recently_created_or_updated_conditions << " OR updated_at > '#{timestamp_before_migration}'" if old_column_names.include?("updated_at")
124
+ connection.execute("REPLACE INTO #{table_name} (#{new_column_list}) SELECT #{old_column_list} FROM #{old_table_name} WHERE #{recently_created_or_updated_conditions}")
125
+ rescue Exception => e
126
+ puts "Failed to lock tables and do final cleanup. This may not be anything to worry about, especially on an infrequently used table."
127
+ puts "ERROR MESSAGE: " + e.message
128
+ ensure
129
+ connection.execute("UNLOCK TABLES")
130
+ end
131
+ drop_table old_table_name
132
+ end
133
+
134
+ private
135
+
136
+ def mysql_big_table_migration_completion(start, counter, max)
137
+ number_done = counter - start + 1
138
+ number_to_do = max - start + 1
139
+ (number_done * 100 / number_to_do.to_f).to_i
140
+ end
141
+
142
+ def mysql_big_table_migration_bach_size
143
+ 10000
144
+ end
145
+
146
+ def connection
147
+ ActiveRecord::Base.connection
148
+ end
149
+
150
+ def each_result_hash(result, &block)
151
+ case connection.class.name
152
+ when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
153
+ result.each_hash(&block)
154
+ when "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
155
+ result.each(:as => :hash, &block)
156
+ end
157
+ end
158
+
159
+ def fetch_result_row(result)
160
+ case connection.class.name
161
+ when "ActiveRecord::ConnectionAdapters::MysqlAdapter"
162
+ result.fetch_row
163
+ when "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
164
+ result.first
165
+ end
166
+ end
167
+
168
+ def column_list(column_names)
169
+ "`" + column_names.join("`, `") + "`"
170
+ end
171
+
172
+ def push_state
173
+ @state_stack ||= []
174
+ @state_stack.push({})
175
+ end
176
+
177
+ def pop_state
178
+ @state_stack.pop
179
+ end
180
+
181
+ def current_state
182
+ @state_stack.last
183
+ end
184
+
185
+ end
186
+
187
+ if Object.const_defined?("ActiveRecord")
188
+ class ActiveRecord::Migration
189
+ if ActiveRecord::VERSION::STRING < "3.0"
190
+ extend MySQLBigTableMigration
191
+ else
192
+ include MySQLBigTableMigration
193
+ end
194
+ end
195
+ end
@@ -0,0 +1,3 @@
1
+ module MySQLBigTableMigration
2
+ VERSION = "0.1.2" unless defined? MySQLBigTableMigration::VERSION
3
+ end
@@ -0,0 +1,36 @@
1
+ $LOAD_PATH.unshift 'lib'
2
+ require "mysql_big_table_migration/version"
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = "mysql_big_table_migration"
6
+ s.version = MySQLBigTableMigration::VERSION
7
+ s.platform = Gem::Platform::RUBY
8
+ s.date = Time.now.strftime('%Y-%m-%d')
9
+ s.summary = "allow columns and indexes to be added to and removed from large tables"
10
+ s.homepage = "http://github.com/MakeYourLaws/mysql_big_table_migration"
11
+ s.email = "sai@makeyourlaws.org"
12
+ s.authors = [ "Mark Woods" ]
13
+ s.has_rdoc = false
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test}/*`.split("\n")
17
+ # s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.add_dependency "activerecord"
21
+ s.add_dependency "activesupport"
22
+ s.add_development_dependency "bundler"
23
+ s.add_development_dependency "fileutils"
24
+ s.add_development_dependency "logger"
25
+ s.add_development_dependency "minitest"
26
+ s.add_development_dependency "mysql"
27
+ s.add_development_dependency "mysql2"
28
+ s.add_development_dependency "rake"
29
+
30
+ s.description = <<desc
31
+ A Rails plugin that adds methods to ActiveRecord::Migration to allow columns
32
+ and indexes to be added to and removed from large tables with millions of
33
+ rows in MySQL, without leaving processes seemingly stalled in state "copy
34
+ to tmp table".
35
+ desc
36
+ end
@@ -0,0 +1,2 @@
1
+ require 'mysql_big_table_migration'
2
+
@@ -0,0 +1,4 @@
1
+ # desc "Explaining what the task does"
2
+ # task :mysql_big_table_migration do
3
+ # # Task goes here
4
+ # end
@@ -0,0 +1,17 @@
1
+ basics: &basics
2
+ username: travis
3
+ database: mysql_big_table_migration_test
4
+ socket: <%= [
5
+ '/var/lib/mysql/mysql.sock',
6
+ '/var/run/mysqld/mysqld.sock',
7
+ '/tmp/mysqld.sock',
8
+ '/tmp/mysql.sock',
9
+ ].detect { |socket| File.exist?(socket) } %>
10
+
11
+ mysql:
12
+ <<: *basics
13
+ adapter: mysql
14
+
15
+ mysql2:
16
+ <<: *basics
17
+ adapter: mysql2
@@ -0,0 +1,262 @@
1
+ require_relative 'test_helper'
2
+
3
+ class MysqlBigTableMigrationTest < Minitest::Test
4
+ extend DatabaseTest
5
+
6
+ test_against_all_configs :methods_are_added_to_migration do
7
+ if ActiveRecord::VERSION::STRING < "3.0"
8
+ method_target = ActiveRecord::Migration
9
+ else
10
+ method_target = ActiveRecord::Migration.new
11
+ end
12
+
13
+ MySQLBigTableMigration.instance_methods(false).each do |method|
14
+ assert_respond_to method_target, method
15
+ end
16
+ end
17
+
18
+ test_against_all_configs :with_tmp_table_creates_tmp_table do
19
+ silence_stream($stdout) do
20
+ ActiveRecord::Migration.send(:with_tmp_table, :test_table) {}
21
+ end
22
+ assert_match "CREATE TABLE tmp_new_test_table LIKE test_table", read_log_file
23
+ end
24
+
25
+ class SmallBatchMigration < ActiveRecord::Migration
26
+ def mysql_big_table_migration_bach_size
27
+ 4
28
+ end
29
+ end
30
+
31
+ test_against_all_configs :with_tmp_table_copies_all_rows do
32
+ silence_stream($stdout) do
33
+ SmallBatchMigration.new.send(:with_tmp_table, :test_table) {}
34
+ end
35
+
36
+ assert_equal 5, test_table_rows.length
37
+ end
38
+
39
+
40
+ test_against_all_configs :with_exactly_one_row, fixture_row_count: 1 do
41
+ silence_stream($stdout) do
42
+ ActiveRecord::Migration.send(:with_tmp_table, :test_table) {}
43
+ end
44
+
45
+ assert_equal 1, test_table_rows.length
46
+ end
47
+
48
+ test_against_all_configs :add_column_using_tmp_table do
49
+ silence_stream($stdout) do
50
+ ActiveRecord::Migration.add_column_using_tmp_table(:test_table, :baz, :string)
51
+ end
52
+
53
+ fields = test_table_fields
54
+ assert_equal 4, fields.length
55
+ assert_equal "baz", fields[3]["Field"]
56
+ assert_equal "varchar(255)", fields[3]["Type"]
57
+
58
+ results = test_table_rows
59
+ assert_equal 5, results.length
60
+ assert_equal "foo2", results[2]["foo"]
61
+ assert_equal "bar3", results[3]["bar"]
62
+ assert_equal nil, results[4]["baz"]
63
+ end
64
+
65
+ test_against_all_configs :remove_column_using_tmp_table do
66
+ silence_stream($stdout) do
67
+ ActiveRecord::Migration.remove_column_using_tmp_table(:test_table, :bar)
68
+ end
69
+
70
+ fields = test_table_fields
71
+ assert_equal 2, fields.length
72
+ assert_equal "id", fields[0]["Field"]
73
+ assert_equal "foo", fields[1]["Field"]
74
+
75
+ results = test_table_rows
76
+ assert_equal 5, results.length
77
+ assert_equal "foo2", results[2]["foo"]
78
+ assert !results[3].has_key?("bar")
79
+ end
80
+
81
+ test_against_all_configs :rename_column_using_tmp_table do
82
+ silence_stream($stdout) do
83
+ ActiveRecord::Migration.rename_column_using_tmp_table(:test_table, :foo, :baz)
84
+ end
85
+
86
+ fields = test_table_fields
87
+ assert_equal 3, fields.length
88
+ assert_equal "id", fields[0]["Field"]
89
+ assert_equal "int(11)", fields[0]["Type"]
90
+ assert_equal "baz", fields[1]["Field"]
91
+ assert_equal "varchar(255)", fields[1]["Type"]
92
+ assert_equal "bar", fields[2]["Field"]
93
+ assert_equal "varchar(255)", fields[2]["Type"]
94
+
95
+ results = test_table_rows
96
+ assert_equal 5, results.length
97
+ 5.times do |i|
98
+ assert_equal "foo#{i}", results[i]["baz"]
99
+ assert_equal "bar#{i}", results[i]["bar"]
100
+ end
101
+ end
102
+
103
+ test_against_all_configs :change_column_using_tmp_table do
104
+ silence_stream($stdout) do
105
+ ActiveRecord::Migration.change_column_using_tmp_table(:test_table, :bar, :text)
106
+ end
107
+
108
+ fields = test_table_fields
109
+ assert_equal 3, fields.length
110
+ assert_equal "id", fields[0]["Field"]
111
+ assert_equal "foo", fields[1]["Field"]
112
+ assert_equal "bar", fields[2]["Field"]
113
+ assert_equal "text", fields[2]["Type"]
114
+
115
+ results = test_table_rows
116
+ assert_equal 5, results.length
117
+ assert_equal "foo2", results[2]["foo"]
118
+ assert_equal "bar3", results[3]["bar"]
119
+ end
120
+
121
+ test_against_all_configs :add_index_using_tmp_table do
122
+ silence_stream($stdout) do
123
+ ActiveRecord::Migration.add_index_using_tmp_table(:test_table, :bar)
124
+ end
125
+
126
+ indexes = result_hashes("SHOW INDEX FROM test_table")
127
+ assert_equal 3, indexes.length
128
+ assert_equal "id", indexes[0]["Column_name"]
129
+ assert_equal "foo", indexes[1]["Column_name"]
130
+ assert_equal "bar", indexes[2]["Column_name"]
131
+ end
132
+
133
+ test_against_all_configs :remove_index_using_tmp_table do
134
+ silence_stream($stdout) do
135
+ ActiveRecord::Migration.remove_index_using_tmp_table(:test_table, :foo)
136
+ end
137
+
138
+ indexes = result_hashes("SHOW INDEX FROM test_table")
139
+ assert_equal 1, indexes.length
140
+ assert_equal "id", indexes[0]["Column_name"]
141
+ end
142
+
143
+ test_against_all_configs :rename_with_remove do
144
+ silence_stream($stdout) do
145
+ ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
146
+ ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
147
+ ActiveRecord::Migration.remove_column tmp_table_name, :foo
148
+ end
149
+ end
150
+
151
+ fields = test_table_fields
152
+ assert_equal 2, fields.length
153
+ assert_equal "id", fields[0]["Field"]
154
+ assert_equal "baz", fields[1]["Field"]
155
+
156
+ results = test_table_rows
157
+ assert_equal 5, results.length
158
+ 5.times do |i|
159
+ assert_equal "bar#{i}", results[i]["baz"]
160
+ end
161
+ end
162
+
163
+ test_against_all_configs :rename_with_add do
164
+ silence_stream($stdout) do
165
+ ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
166
+ ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
167
+ ActiveRecord::Migration.add_column tmp_table_name, :dummy, :integer
168
+ end
169
+ end
170
+
171
+ fields = test_table_fields
172
+ assert_equal 4, fields.length
173
+ assert_equal "id", fields[0]["Field"]
174
+ assert_equal "foo", fields[1]["Field"]
175
+ assert_equal "baz", fields[2]["Field"]
176
+ assert_equal "dummy", fields[3]["Field"]
177
+
178
+ results = test_table_rows
179
+ assert_equal 5, results.length
180
+ 5.times do |i|
181
+ assert_equal "foo#{i}", results[i]["foo"]
182
+ assert_equal "bar#{i}", results[i]["baz"]
183
+ assert_equal nil, results[i]["dummy"]
184
+ end
185
+ end
186
+
187
+ test_against_all_configs :rename_with_change do
188
+ silence_stream($stdout) do
189
+ ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
190
+ ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
191
+ # MySQL can't properly change string data to integer ("Incorrect integer value: 'foo0'")
192
+ # Also, limit: 3 will fail ("Data too long") w/ sql_mode = 'STRICT_ALL_TABLES'
193
+ ActiveRecord::Migration.change_column tmp_table_name, :foo, :string, limit: 10
194
+ end
195
+ end
196
+
197
+ fields = test_table_fields
198
+ assert_equal 3, fields.length
199
+ assert_equal "id", fields[0]["Field"]
200
+ assert_equal "foo", fields[1]["Field"]
201
+ assert_equal "varchar(10)", fields[1]["Type"]
202
+ assert_equal "baz", fields[2]["Field"]
203
+
204
+ results = test_table_rows
205
+ assert_equal 5, results.length
206
+ 5.times do |i|
207
+ assert_equal "foo#{i}", results[i]["foo"]
208
+ assert_equal "bar#{i}", results[i]["baz"]
209
+ end
210
+ end
211
+
212
+ test_against_all_configs :rename_with_rename do
213
+ silence_stream($stdout) do
214
+ ActiveRecord::Migration.with_tmp_table(:test_table) do |tmp_table_name|
215
+ ActiveRecord::Migration.rename_column tmp_table_name, :bar, :baz
216
+ ActiveRecord::Migration.rename_column tmp_table_name, :foo, :dummy
217
+ end
218
+ end
219
+
220
+ fields = test_table_fields
221
+ assert_equal 3, fields.length
222
+ assert_equal "id", fields[0]["Field"]
223
+ assert_equal "dummy", fields[1]["Field"]
224
+ assert_equal "baz", fields[2]["Field"]
225
+
226
+ results = test_table_rows
227
+ assert_equal 5, results.length
228
+ 5.times do |i|
229
+ assert_equal "foo#{i}", results[i]["dummy"]
230
+ assert_equal "bar#{i}", results[i]["baz"]
231
+ end
232
+ end
233
+
234
+ test_against_all_configs :rename_column do
235
+ silence_stream($stdout) do
236
+ ActiveRecord::Migration.rename_column :test_table, :bar, :baz
237
+ end
238
+
239
+ fields = test_table_fields
240
+ assert_equal 3, fields.length
241
+ assert_equal "id", fields[0]["Field"]
242
+ assert_equal "foo", fields[1]["Field"]
243
+ assert_equal "baz", fields[2]["Field"]
244
+
245
+ results = test_table_rows
246
+ assert_equal 5, results.length
247
+ 5.times do |i|
248
+ assert_equal "foo#{i}", results[i]["foo"]
249
+ assert_equal "bar#{i}", results[i]["baz"]
250
+ end
251
+ end
252
+
253
+ private
254
+
255
+ def test_table_fields
256
+ result_hashes("DESCRIBE test_table")
257
+ end
258
+
259
+ def test_table_rows
260
+ result_hashes("SELECT * FROM test_table")
261
+ end
262
+ end
@@ -0,0 +1,8 @@
1
+ ActiveRecord::Schema.define(:version => 0) do
2
+ create_table :test_table, :force => true do |t|
3
+ t.string :foo
4
+ t.string :bar
5
+ end
6
+
7
+ add_index :test_table, [:foo]
8
+ end
@@ -0,0 +1,111 @@
1
+ require 'rubygems'
2
+ require 'fileutils'
3
+ require 'minitest/autorun'
4
+ require 'active_record'
5
+ require 'active_record/connection_adapters/mysql_adapter'
6
+ require 'active_support'
7
+ require 'active_support/core_ext'
8
+ require 'mysql'
9
+ require 'mysql2'
10
+ require 'logger'
11
+ require 'yaml'
12
+ require File.dirname(__FILE__) + "/../lib/mysql_big_table_migration"
13
+
14
+ TEST_CONFIGS = ["mysql", "mysql2"]
15
+
16
+ Mysql::Result.class_eval do
17
+ unless respond_to?(:all_hashes)
18
+ def all_hashes
19
+ rows = []
20
+ each_hash do |row|
21
+ rows << row
22
+ end
23
+ rows
24
+ end
25
+ end
26
+ end
27
+
28
+ def read_log_file
29
+ @log.string
30
+ end
31
+
32
+ def load_schema(adapter)
33
+ config = YAML.load(ERB.new(File.read(File.join(File.dirname(__FILE__), 'database.yml'))).result)
34
+ @log = StringIO.new
35
+ ActiveRecord::Base.logger = Logger.new(@log)
36
+ ActiveRecord::Base.establish_connection(config[adapter.to_s])
37
+ load(File.join(File.dirname(__FILE__), "schema.rb"))
38
+ end
39
+
40
+ def load_fixtures(options = {})
41
+ connection = ActiveRecord::Base.connection
42
+
43
+ connection.execute("DELETE FROM test_table;")
44
+ (options[:fixture_row_count] || 5).times do |i|
45
+ connection.execute("INSERT INTO test_table (foo, bar) VALUES ('foo#{i}', 'bar#{i}');")
46
+ end
47
+ end
48
+
49
+ def assert_valid_database_setup(options = {})
50
+ fields = result_hashes("DESCRIBE test_table")
51
+ assert_equal 3, fields.length
52
+ assert_equal "id", fields[0]["Field"]
53
+ assert_equal "int(11)", fields[0]["Type"]
54
+ assert_equal "foo", fields[1]["Field"]
55
+ assert_equal "varchar(255)", fields[1]["Type"]
56
+ assert_equal "bar", fields[2]["Field"]
57
+ assert_equal "varchar(255)", fields[2]["Type"]
58
+
59
+ indexes = result_hashes("SHOW INDEX FROM test_table")
60
+ assert_equal 2, indexes.length
61
+ assert_equal "id", indexes[0]["Column_name"]
62
+ assert_equal "foo", indexes[1]["Column_name"]
63
+
64
+ results = result_hashes("SELECT * FROM test_table")
65
+ assert_equal options[:fixture_row_count] || 5, results.length
66
+ end
67
+
68
+ def result_hashes(query)
69
+ result = connection.execute(query)
70
+ case connection
71
+ when ActiveRecord::ConnectionAdapters::MysqlAdapter
72
+ result.all_hashes
73
+ when ActiveRecord::ConnectionAdapters::Mysql2Adapter
74
+ result.each(:as => :hash)
75
+ else
76
+ raise "Unknown adapter"
77
+ end
78
+ end
79
+
80
+ # Proc#bind removed from ActiveSupport 4+
81
+ class Proc
82
+ def bind(object)
83
+ block, time = self, Time.now
84
+ object.class_eval do
85
+ method_name = "__bind_#{time.to_i}_#{time.usec}"
86
+ define_method(method_name, &block)
87
+ method = instance_method(method_name)
88
+ remove_method(method_name)
89
+ method
90
+ end.bind(object)
91
+ end
92
+ end
93
+
94
+ module DatabaseTest
95
+ def test_against_all_configs(name, options = {}, &block)
96
+ TEST_CONFIGS.each do |config|
97
+ self.send(:define_method, :"test_#{name.to_s}_with_#{config}") do
98
+ silence_stream($stdout) do
99
+ load_schema(config)
100
+ load_fixtures options
101
+ end
102
+ assert_valid_database_setup options
103
+ block.bind(self).call
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ def connection
110
+ ActiveRecord::Base.connection
111
+ end
@@ -0,0 +1 @@
1
+ # Uninstall hook code here
metadata ADDED
@@ -0,0 +1,191 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mysql_big_table_migration
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.2
5
+ platform: ruby
6
+ authors:
7
+ - Mark Woods
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-04-12 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ type: :runtime
15
+ name: activerecord
16
+ prerelease: false
17
+ requirement: !ruby/object:Gem::Requirement
18
+ requirements:
19
+ - - ">="
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ type: :runtime
29
+ name: activesupport
30
+ prerelease: false
31
+ requirement: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - ">="
34
+ - !ruby/object:Gem::Version
35
+ version: '0'
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ type: :development
43
+ name: bundler
44
+ prerelease: false
45
+ requirement: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ">="
48
+ - !ruby/object:Gem::Version
49
+ version: '0'
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ type: :development
57
+ name: fileutils
58
+ prerelease: false
59
+ requirement: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ">="
62
+ - !ruby/object:Gem::Version
63
+ version: '0'
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ type: :development
71
+ name: logger
72
+ prerelease: false
73
+ requirement: !ruby/object:Gem::Requirement
74
+ requirements:
75
+ - - ">="
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ type: :development
85
+ name: minitest
86
+ prerelease: false
87
+ requirement: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ type: :development
99
+ name: mysql
100
+ prerelease: false
101
+ requirement: !ruby/object:Gem::Requirement
102
+ requirements:
103
+ - - ">="
104
+ - !ruby/object:Gem::Version
105
+ version: '0'
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ type: :development
113
+ name: mysql2
114
+ prerelease: false
115
+ requirement: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - ">="
118
+ - !ruby/object:Gem::Version
119
+ version: '0'
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ type: :development
127
+ name: rake
128
+ prerelease: false
129
+ requirement: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ description: |2
140
+ A Rails plugin that adds methods to ActiveRecord::Migration to allow columns
141
+ and indexes to be added to and removed from large tables with millions of
142
+ rows in MySQL, without leaving processes seemingly stalled in state "copy
143
+ to tmp table".
144
+ email: sai@makeyourlaws.org
145
+ executables: []
146
+ extensions: []
147
+ extra_rdoc_files: []
148
+ files:
149
+ - ".gitignore"
150
+ - ".travis.yml"
151
+ - CHANGELOG
152
+ - Gemfile
153
+ - MIT-LICENSE
154
+ - README.md
155
+ - Rakefile
156
+ - init.rb
157
+ - install.rb
158
+ - lib/mysql_big_table_migration.rb
159
+ - lib/mysql_big_table_migration/version.rb
160
+ - mysql_big_table_migration.gemspec
161
+ - rails/init.rb
162
+ - tasks/mysql_big_table_migration_tasks.rake
163
+ - test/database.yml
164
+ - test/mysql_big_table_migration_test.rb
165
+ - test/schema.rb
166
+ - test/test_helper.rb
167
+ - uninstall.rb
168
+ homepage: http://github.com/MakeYourLaws/mysql_big_table_migration
169
+ licenses: []
170
+ metadata: {}
171
+ post_install_message:
172
+ rdoc_options: []
173
+ require_paths:
174
+ - lib
175
+ required_ruby_version: !ruby/object:Gem::Requirement
176
+ requirements:
177
+ - - ">="
178
+ - !ruby/object:Gem::Version
179
+ version: '0'
180
+ required_rubygems_version: !ruby/object:Gem::Requirement
181
+ requirements:
182
+ - - ">="
183
+ - !ruby/object:Gem::Version
184
+ version: '0'
185
+ requirements: []
186
+ rubyforge_project:
187
+ rubygems_version: 2.4.6
188
+ signing_key:
189
+ specification_version: 4
190
+ summary: allow columns and indexes to be added to and removed from large tables
191
+ test_files: []