large-hadron-migrator 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES.md +3 -0
- data/README.md +6 -4
- data/VERSION +1 -1
- data/large-hadron-migrator.gemspec +1 -1
- data/lib/{large_hadron_migration.rb → large-hadron-migrator.rb} +2 -2
- metadata +5 -16
data/CHANGES.md
CHANGED
data/README.md
CHANGED
@@ -50,10 +50,10 @@ twitter solution [1], it does not require the presence of an indexed
|
|
50
50
|
|
51
51
|
## Usage
|
52
52
|
|
53
|
-
Large Hadron
|
53
|
+
Large Hadron Migrator is currently implemented as a Rails ActiveRecord
|
54
54
|
Migration.
|
55
55
|
|
56
|
-
class AddIndexToEmails <
|
56
|
+
class AddIndexToEmails < LargeHadronMigrator
|
57
57
|
def self.up
|
58
58
|
large_hadron_migrate :emails, :wait => 0.2 do |table_name|
|
59
59
|
execute %Q{
|
@@ -199,12 +199,14 @@ while migrating in a master slave setup.
|
|
199
199
|
|
200
200
|
## Todos
|
201
201
|
|
202
|
-
Load data into outfile instead of `INSERT INTO... SELECT`. Avoid contention and
|
203
|
-
increase speed.
|
202
|
+
~~Load data into outfile instead of `INSERT INTO... SELECT`. Avoid contention and
|
203
|
+
increase speed.~~ It showed that this wont result in major speed up.
|
204
204
|
|
205
205
|
Handle invalidation of 'update' entries in journal while replaying. Avoid
|
206
206
|
stale update replays.
|
207
207
|
|
208
|
+
Support for other Databases than MySQL
|
209
|
+
|
208
210
|
Some other optimizations:
|
209
211
|
|
210
212
|
Deletions create gaps in the primary key id integer column. LHM has no
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.4
|
@@ -14,7 +14,7 @@ Gem::Specification.new do |s|
|
|
14
14
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
15
15
|
s.require_paths = ["lib"]
|
16
16
|
|
17
|
-
['activerecord ~>2.3.8', 'activesupport ~>2.3.8'
|
17
|
+
['activerecord ~>2.3.8', 'activesupport ~>2.3.8'].each do |gem|
|
18
18
|
s.add_dependency *gem.split(' ')
|
19
19
|
end
|
20
20
|
|
@@ -14,7 +14,7 @@ require 'benchmark'
|
|
14
14
|
#
|
15
15
|
# USAGE:
|
16
16
|
#
|
17
|
-
# class AddIndexToEmails <
|
17
|
+
# class AddIndexToEmails < LargeHadronMigrator
|
18
18
|
# def self.up
|
19
19
|
# large_hadron_migrate :emails, :wait => 0.2 do |table_name|
|
20
20
|
# execute %Q{
|
@@ -80,7 +80,7 @@ require 'benchmark'
|
|
80
80
|
#
|
81
81
|
# Deploying the new version will hurt your head. Don't do it.
|
82
82
|
#
|
83
|
-
class
|
83
|
+
class LargeHadronMigrator < ActiveRecord::Migration
|
84
84
|
|
85
85
|
# id_window must be larger than the number of inserts
|
86
86
|
# added to the journal table. if this is not the case,
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: large-hadron-migrator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.1.
|
5
|
+
version: 0.1.4
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- SoundCloud
|
@@ -12,7 +12,7 @@ autorequire:
|
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
14
|
|
15
|
-
date: 2011-09-
|
15
|
+
date: 2011-09-29 00:00:00 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: activerecord
|
@@ -36,28 +36,17 @@ dependencies:
|
|
36
36
|
version: 2.3.8
|
37
37
|
type: :runtime
|
38
38
|
version_requirements: *id002
|
39
|
-
- !ruby/object:Gem::Dependency
|
40
|
-
name: mysql
|
41
|
-
prerelease: false
|
42
|
-
requirement: &id003 !ruby/object:Gem::Requirement
|
43
|
-
none: false
|
44
|
-
requirements:
|
45
|
-
- - "="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: 2.8.1
|
48
|
-
type: :runtime
|
49
|
-
version_requirements: *id003
|
50
39
|
- !ruby/object:Gem::Dependency
|
51
40
|
name: rspec
|
52
41
|
prerelease: false
|
53
|
-
requirement: &
|
42
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
54
43
|
none: false
|
55
44
|
requirements:
|
56
45
|
- - "="
|
57
46
|
- !ruby/object:Gem::Version
|
58
47
|
version: 1.3.1
|
59
48
|
type: :development
|
60
|
-
version_requirements: *
|
49
|
+
version_requirements: *id003
|
61
50
|
description: Migrate large tables without downtime by copying to a temporary table in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name for verification.
|
62
51
|
email: rany@soundcloud.com, tobi@soundcloud.com
|
63
52
|
executables: []
|
@@ -76,7 +65,7 @@ files:
|
|
76
65
|
- Rakefile
|
77
66
|
- VERSION
|
78
67
|
- large-hadron-migrator.gemspec
|
79
|
-
- lib/
|
68
|
+
- lib/large-hadron-migrator.rb
|
80
69
|
- spec/large_hadron_migration_spec.rb
|
81
70
|
- spec/migrate/add_new_column.rb
|
82
71
|
- spec/spec_helper.rb
|