active_record_shards 2.3.1 → 2.4.1
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/active_record_shards/connection_switcher.rb +1 -1
- data/lib/active_record_shards/migration.rb +71 -0
- data/lib/active_record_shards.rb +1 -0
- data/test/connection_switching_test.rb +6 -1
- data/test/database.yml +9 -18
- data/test/failure_migration/20110824010215_failure_migration.rb +14 -0
- data/test/helper.rb +6 -7
- data/test/migrations/20110824010216_shard_migration.rb +12 -0
- data/test/migrations/20110829215912_account_migration.rb +13 -0
- data/test/migrator_test.rb +88 -0
- data/test/schema.rb +20 -20
- metadata +15 -6
- data/test/test.log +0 -4229
@@ -0,0 +1,71 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
class Migrator
|
3
|
+
class << self
|
4
|
+
[:up, :down, :run].each do |m|
|
5
|
+
define_method("#{m}_with_sharding") do |*args|
|
6
|
+
ActiveRecord::Base.on_shard(nil) do
|
7
|
+
self.send("#{m}_without_sharding", *args)
|
8
|
+
end
|
9
|
+
ActiveRecord::Base.on_all_shards do
|
10
|
+
self.send("#{m}_without_sharding", *args)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
alias_method_chain m.to_sym, :sharding
|
14
|
+
end
|
15
|
+
|
16
|
+
def bootstrap_migrations_from_nil_shard(migrations_path, this_migration=nil)
|
17
|
+
migrations = nil
|
18
|
+
ActiveRecord::Base.on_shard(nil) do
|
19
|
+
migrations = ActiveRecord::Migrator.new(:up, migrations_path).migrated
|
20
|
+
end
|
21
|
+
|
22
|
+
puts "inserting #{migrations.size} migrations on all shards..."
|
23
|
+
ActiveRecord::Base.on_all_shards do
|
24
|
+
migrator = ActiveRecord::Migrator.new(:up, migrations_path)
|
25
|
+
migrations.each do |m|
|
26
|
+
migrator.__send__(:record_version_state_after_migrating, m)
|
27
|
+
end
|
28
|
+
if this_migration
|
29
|
+
migrator.__send__(:record_version_state_after_migrating, this_migration)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
module ActiveRecordShards
|
38
|
+
module MigrationExtension
|
39
|
+
attr_accessor :migration_shard
|
40
|
+
|
41
|
+
def shard(arg = nil)
|
42
|
+
self.migration_shard = arg
|
43
|
+
end
|
44
|
+
|
45
|
+
def migrate_with_forced_shard(direction)
|
46
|
+
if migration_shard.blank?
|
47
|
+
raise "Can't run migrations with a shard spec: this may be :all, :none, or a specific shard (for data-fixups) please call shard(arg) in your migration"
|
48
|
+
end
|
49
|
+
|
50
|
+
shard = ActiveRecord::Base.current_shard_selection.shard
|
51
|
+
|
52
|
+
if shard.nil?
|
53
|
+
return if migration_shard != :none
|
54
|
+
else
|
55
|
+
return if migration_shard == :none
|
56
|
+
return if migration_shard != :all && migration_shard.to_s != shard.to_s
|
57
|
+
end
|
58
|
+
|
59
|
+
migrate_without_forced_shard(direction)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
ActiveRecord::Migration.class_eval do
|
65
|
+
extend ActiveRecordShards::MigrationExtension
|
66
|
+
class << self
|
67
|
+
alias_method_chain :migrate, :forced_shard
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
ActiveRecord::MigrationProxy.delegate :migration_shard, :to => :migration
|
data/lib/active_record_shards.rb
CHANGED
@@ -6,6 +6,7 @@ require 'active_record_shards/shard_selection'
|
|
6
6
|
require 'active_record_shards/connection_switcher'
|
7
7
|
require 'active_record_shards/association_collection_connection_selection'
|
8
8
|
require 'active_record_shards/connection_pool'
|
9
|
+
require 'active_record_shards/migration'
|
9
10
|
|
10
11
|
ActiveRecord::Base.extend(ActiveRecordShards::ConfigurationParser)
|
11
12
|
ActiveRecord::Base.extend(ActiveRecordShards::Model)
|
@@ -48,8 +48,10 @@ class ConnectionSwitchenTest < ActiveSupport::TestCase
|
|
48
48
|
assert_not_equal(@shard_0_master.select_value("SELECT DATABASE()"), @shard_1_master.select_value("SELECT DATABASE()"))
|
49
49
|
|
50
50
|
@database_names = []
|
51
|
-
|
51
|
+
@database_shards = []
|
52
|
+
ActiveRecord::Base.on_all_shards do |shard|
|
52
53
|
@database_names << ActiveRecord::Base.connection.select_value("SELECT DATABASE()")
|
54
|
+
@database_shards << shard
|
53
55
|
end
|
54
56
|
end
|
55
57
|
|
@@ -57,6 +59,9 @@ class ConnectionSwitchenTest < ActiveSupport::TestCase
|
|
57
59
|
assert_equal(2, @database_names.size)
|
58
60
|
assert_contains(@database_names, @shard_0_master.select_value("SELECT DATABASE()"))
|
59
61
|
assert_contains(@database_names, @shard_1_master.select_value("SELECT DATABASE()"))
|
62
|
+
assert_equal(2, @database_shards.size)
|
63
|
+
assert_contains(@database_shards, "0")
|
64
|
+
assert_contains(@database_shards, "1")
|
60
65
|
end
|
61
66
|
end
|
62
67
|
end
|
data/test/database.yml
CHANGED
@@ -4,8 +4,7 @@ test:
|
|
4
4
|
database: ars_test
|
5
5
|
username: root
|
6
6
|
password:
|
7
|
-
|
8
|
-
host: localhost
|
7
|
+
host: 127.0.0.1
|
9
8
|
shard_names: ['0', '1']
|
10
9
|
|
11
10
|
test_slave:
|
@@ -14,8 +13,7 @@ test_slave:
|
|
14
13
|
database: ars_test_slave
|
15
14
|
username: root
|
16
15
|
password:
|
17
|
-
|
18
|
-
host: localhost
|
16
|
+
host: 127.0.0.1
|
19
17
|
|
20
18
|
test_shard_0:
|
21
19
|
adapter: mysql
|
@@ -23,8 +21,7 @@ test_shard_0:
|
|
23
21
|
database: ars_test_shard0
|
24
22
|
username: root
|
25
23
|
password:
|
26
|
-
|
27
|
-
host: localhost
|
24
|
+
host: 127.0.0.1
|
28
25
|
|
29
26
|
test_shard_0_slave:
|
30
27
|
adapter: mysql
|
@@ -32,8 +29,7 @@ test_shard_0_slave:
|
|
32
29
|
database: ars_test_shard0_slave
|
33
30
|
username: root
|
34
31
|
password:
|
35
|
-
|
36
|
-
host: localhost
|
32
|
+
host: 127.0.0.1
|
37
33
|
|
38
34
|
test_shard_1:
|
39
35
|
adapter: mysql
|
@@ -41,8 +37,7 @@ test_shard_1:
|
|
41
37
|
database: ars_test_shard1
|
42
38
|
username: root
|
43
39
|
password:
|
44
|
-
|
45
|
-
host: localhost
|
40
|
+
host: 127.0.0.1
|
46
41
|
|
47
42
|
test_shard_1_slave:
|
48
43
|
adapter: mysql
|
@@ -50,8 +45,7 @@ test_shard_1_slave:
|
|
50
45
|
database: ars_test_shard1_slave
|
51
46
|
username: root
|
52
47
|
password:
|
53
|
-
|
54
|
-
host: localhost
|
48
|
+
host: 127.0.0.1
|
55
49
|
|
56
50
|
test2:
|
57
51
|
adapter: mysql
|
@@ -59,8 +53,7 @@ test2:
|
|
59
53
|
database: ars_test2
|
60
54
|
username: root
|
61
55
|
password:
|
62
|
-
|
63
|
-
host: localhost
|
56
|
+
host: 127.0.0.1
|
64
57
|
|
65
58
|
test2_slave:
|
66
59
|
adapter: mysql
|
@@ -68,8 +61,7 @@ test2_slave:
|
|
68
61
|
database: ars_test2_slave
|
69
62
|
username: root
|
70
63
|
password:
|
71
|
-
|
72
|
-
host: localhost
|
64
|
+
host: 127.0.0.1
|
73
65
|
|
74
66
|
alternative:
|
75
67
|
adapter: mysql
|
@@ -77,6 +69,5 @@ alternative:
|
|
77
69
|
database: ars_test_alternative
|
78
70
|
username: root
|
79
71
|
password:
|
80
|
-
|
81
|
-
host: localhost
|
72
|
+
host: 127.0.0.1
|
82
73
|
shard_names: ['0', '1']
|
@@ -0,0 +1,14 @@
|
|
1
|
+
class FailureMigration < ActiveRecord::Migration
|
2
|
+
shard :all
|
3
|
+
|
4
|
+
def self.up
|
5
|
+
$fail_at_two ||= 0
|
6
|
+
$fail_at_two += 1
|
7
|
+
raise "FAIL FAIL FAIL" if $fail_at_two == 2
|
8
|
+
add_column :tickets, :sharded_column, :integer
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.down
|
12
|
+
remove_column :tickets, :sharded_column
|
13
|
+
end
|
14
|
+
end
|
data/test/helper.rb
CHANGED
@@ -18,17 +18,16 @@ RAILS_ENV = "test"
|
|
18
18
|
ActiveRecord::Base.logger = Logger.new(File.dirname(__FILE__) + "/test.log")
|
19
19
|
ActiveRecord::Base.configurations = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
ActiveRecord::Base.connection
|
26
|
-
rescue Mysql::Error => e
|
21
|
+
def init_schema
|
22
|
+
ActiveRecord::Base.configurations.each do |name, conf|
|
23
|
+
puts "Setting up the #{conf['database']} db"
|
24
|
+
`echo "drop DATABASE if exists #{conf['database']}" | mysql --user=#{conf['username']}`
|
27
25
|
`echo "create DATABASE #{conf['database']}" | mysql --user=#{conf['username']}`
|
28
26
|
ActiveRecord::Base.establish_connection(name)
|
27
|
+
load(File.dirname(__FILE__) + "/schema.rb")
|
29
28
|
end
|
30
|
-
load(File.dirname(__FILE__) + "/schema.rb")
|
31
29
|
end
|
30
|
+
init_schema
|
32
31
|
|
33
32
|
require 'models'
|
34
33
|
|
@@ -0,0 +1,12 @@
|
|
1
|
+
#This is to recreate the deleted onetime payment for DealFish
|
2
|
+
class ShardMigration < ActiveRecord::Migration
|
3
|
+
shard :all
|
4
|
+
|
5
|
+
def self.up
|
6
|
+
add_column :emails, :sharded_column, :integer
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.down
|
10
|
+
remove_column :emails, :sharded_column
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# See ticket 130214
|
2
|
+
# Setting up an odd initial three year term for box.net to be paid by credit card by the normal means
|
3
|
+
class AccountMigration < ActiveRecord::Migration
|
4
|
+
shard :none
|
5
|
+
|
6
|
+
def self.up
|
7
|
+
add_column :accounts, :non_sharded_column, :integer
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.down
|
11
|
+
remove_column :accounts, :non_sharded_column
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require File.expand_path('helper', File.dirname(__FILE__))
|
2
|
+
|
3
|
+
class CowardlyMigration < ActiveRecord::Migration
|
4
|
+
def self.up
|
5
|
+
"not gonna happen"
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.down
|
9
|
+
"uh uh"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class MigratorTest < ActiveSupport::TestCase
|
14
|
+
def setup
|
15
|
+
init_schema
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_migrator
|
19
|
+
migration_path = File.join(File.dirname(__FILE__), "/migrations")
|
20
|
+
ActiveRecord::Migrator.migrate(migration_path)
|
21
|
+
ActiveRecord::Base.on_all_shards do
|
22
|
+
assert ActiveRecord::Base.connection.table_exists?(:schema_migrations), "Schema Migrations doesn't exist"
|
23
|
+
assert ActiveRecord::Base.connection.table_exists?(:accounts)
|
24
|
+
assert ActiveRecord::Base.connection.select_value("select version from schema_migrations where version = '20110824010216'")
|
25
|
+
assert ActiveRecord::Base.connection.select_value("select version from schema_migrations where version = '20110829215912'")
|
26
|
+
end
|
27
|
+
|
28
|
+
ActiveRecord::Base.on_all_shards do
|
29
|
+
assert table_has_column?("emails", "sharded_column")
|
30
|
+
assert !table_has_column?("accounts", "non_sharded_column")
|
31
|
+
end
|
32
|
+
|
33
|
+
ActiveRecord::Base.on_shard(nil) do
|
34
|
+
assert !table_has_column?("emails", "sharded_column")
|
35
|
+
assert table_has_column?("accounts", "non_sharded_column")
|
36
|
+
end
|
37
|
+
|
38
|
+
# now test down/ up
|
39
|
+
ActiveRecord::Migrator.run(:down, migration_path, 20110824010216)
|
40
|
+
ActiveRecord::Base.on_all_shards do
|
41
|
+
assert !table_has_column?("emails", "sharded_column")
|
42
|
+
end
|
43
|
+
|
44
|
+
ActiveRecord::Migrator.run(:down, migration_path, 20110829215912)
|
45
|
+
ActiveRecord::Base.on_shard(nil) do
|
46
|
+
assert !table_has_column?("accounts", "non_sharded_column")
|
47
|
+
end
|
48
|
+
|
49
|
+
ActiveRecord::Migrator.run(:up, migration_path, 20110824010216)
|
50
|
+
ActiveRecord::Base.on_all_shards do
|
51
|
+
assert table_has_column?("emails", "sharded_column")
|
52
|
+
end
|
53
|
+
|
54
|
+
ActiveRecord::Migrator.run(:up, migration_path, 20110829215912)
|
55
|
+
ActiveRecord::Base.on_shard(nil) do
|
56
|
+
assert table_has_column?("accounts", "non_sharded_column")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_bad_migration
|
61
|
+
assert_raises(RuntimeError) do
|
62
|
+
CowardlyMigration.up
|
63
|
+
end
|
64
|
+
|
65
|
+
assert_raises(RuntimeError) do
|
66
|
+
CowardlyMigration.down
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_failing_migration
|
71
|
+
# like, if you have to break a migration in the middle somewhere.
|
72
|
+
begin
|
73
|
+
migration_path = File.join(File.dirname(__FILE__), "/failure_migration")
|
74
|
+
ActiveRecord::Migrator.migrate(migration_path)
|
75
|
+
rescue
|
76
|
+
retry
|
77
|
+
end
|
78
|
+
|
79
|
+
ActiveRecord::Base.on_all_shards do
|
80
|
+
assert table_has_column?("tickets", "sharded_column")
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
def table_has_column?(table, column)
|
86
|
+
ActiveRecord::Base.connection.select_value("show create table #{table}") =~ /#{column}/
|
87
|
+
end
|
88
|
+
end
|
data/test/schema.rb
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
ActiveRecord::Schema.define(:version => 1) do
|
2
|
+
suppress_messages do
|
3
|
+
create_table "accounts", :force => true do |t|
|
4
|
+
t.string "name"
|
5
|
+
t.datetime "created_at"
|
6
|
+
t.datetime "updated_at"
|
7
|
+
end
|
2
8
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
t.string "from"
|
11
|
-
t.string "to"
|
12
|
-
t.text "mail"
|
13
|
-
t.datetime "created_at"
|
14
|
-
t.datetime "updated_at"
|
15
|
-
end
|
9
|
+
create_table "emails", :force => true do |t|
|
10
|
+
t.string "from"
|
11
|
+
t.string "to"
|
12
|
+
t.text "mail"
|
13
|
+
t.datetime "created_at"
|
14
|
+
t.datetime "updated_at"
|
15
|
+
end
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
17
|
+
create_table "tickets", :force => true do |t|
|
18
|
+
t.string "title"
|
19
|
+
t.integer "account_id"
|
20
|
+
t.datetime "created_at"
|
21
|
+
t.datetime "updated_at"
|
22
|
+
end
|
22
23
|
end
|
23
|
-
|
24
|
-
end
|
24
|
+
end
|
metadata
CHANGED
@@ -1,22 +1,23 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 29
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 2
|
8
|
-
-
|
8
|
+
- 4
|
9
9
|
- 1
|
10
|
-
version: 2.
|
10
|
+
version: 2.4.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Mick Staugaard
|
14
14
|
- Eric Chapweske
|
15
|
+
- Ben Osheroff
|
15
16
|
autorequire:
|
16
17
|
bindir: bin
|
17
18
|
cert_chain: []
|
18
19
|
|
19
|
-
date: 2011-
|
20
|
+
date: 2011-08-31 00:00:00 -07:00
|
20
21
|
default_executable:
|
21
22
|
dependencies:
|
22
23
|
- !ruby/object:Gem::Dependency
|
@@ -109,6 +110,7 @@ description: Easily run queries on shard and slave databases.
|
|
109
110
|
email:
|
110
111
|
- mick@staugaard.com
|
111
112
|
- eac@zendesk.com
|
113
|
+
- ben@gimbo.net
|
112
114
|
executables: []
|
113
115
|
|
114
116
|
extensions: []
|
@@ -120,6 +122,7 @@ files:
|
|
120
122
|
- lib/active_record_shards/configuration_parser.rb
|
121
123
|
- lib/active_record_shards/connection_pool.rb
|
122
124
|
- lib/active_record_shards/connection_switcher.rb
|
125
|
+
- lib/active_record_shards/migration.rb
|
123
126
|
- lib/active_record_shards/model.rb
|
124
127
|
- lib/active_record_shards/shard_selection.rb
|
125
128
|
- lib/active_record_shards/tasks.rb
|
@@ -129,10 +132,13 @@ files:
|
|
129
132
|
- test/connection_switching_test.rb
|
130
133
|
- test/database.yml
|
131
134
|
- test/database_parse_test.yml
|
135
|
+
- test/failure_migration/20110824010215_failure_migration.rb
|
132
136
|
- test/helper.rb
|
137
|
+
- test/migrations/20110824010216_shard_migration.rb
|
138
|
+
- test/migrations/20110829215912_account_migration.rb
|
139
|
+
- test/migrator_test.rb
|
133
140
|
- test/models.rb
|
134
141
|
- test/schema.rb
|
135
|
-
- test/test.log
|
136
142
|
has_rdoc: true
|
137
143
|
homepage: http://github.com/staugaard/active_record_shards
|
138
144
|
licenses: []
|
@@ -172,7 +178,10 @@ test_files:
|
|
172
178
|
- test/connection_switching_test.rb
|
173
179
|
- test/database.yml
|
174
180
|
- test/database_parse_test.yml
|
181
|
+
- test/failure_migration/20110824010215_failure_migration.rb
|
175
182
|
- test/helper.rb
|
183
|
+
- test/migrations/20110824010216_shard_migration.rb
|
184
|
+
- test/migrations/20110829215912_account_migration.rb
|
185
|
+
- test/migrator_test.rb
|
176
186
|
- test/models.rb
|
177
187
|
- test/schema.rb
|
178
|
-
- test/test.log
|