active_record_shards 2.3.1 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/active_record_shards/connection_switcher.rb +1 -1
- data/lib/active_record_shards/migration.rb +71 -0
- data/lib/active_record_shards.rb +1 -0
- data/test/connection_switching_test.rb +6 -1
- data/test/database.yml +9 -18
- data/test/failure_migration/20110824010215_failure_migration.rb +14 -0
- data/test/helper.rb +6 -7
- data/test/migrations/20110824010216_shard_migration.rb +12 -0
- data/test/migrations/20110829215912_account_migration.rb +13 -0
- data/test/migrator_test.rb +88 -0
- data/test/schema.rb +20 -20
- metadata +15 -6
- data/test/test.log +0 -4229
@@ -0,0 +1,71 @@
|
|
1
|
+
module ActiveRecord
|
2
|
+
class Migrator
|
3
|
+
class << self
|
4
|
+
[:up, :down, :run].each do |m|
|
5
|
+
define_method("#{m}_with_sharding") do |*args|
|
6
|
+
ActiveRecord::Base.on_shard(nil) do
|
7
|
+
self.send("#{m}_without_sharding", *args)
|
8
|
+
end
|
9
|
+
ActiveRecord::Base.on_all_shards do
|
10
|
+
self.send("#{m}_without_sharding", *args)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
alias_method_chain m.to_sym, :sharding
|
14
|
+
end
|
15
|
+
|
16
|
+
def bootstrap_migrations_from_nil_shard(migrations_path, this_migration=nil)
|
17
|
+
migrations = nil
|
18
|
+
ActiveRecord::Base.on_shard(nil) do
|
19
|
+
migrations = ActiveRecord::Migrator.new(:up, migrations_path).migrated
|
20
|
+
end
|
21
|
+
|
22
|
+
puts "inserting #{migrations.size} migrations on all shards..."
|
23
|
+
ActiveRecord::Base.on_all_shards do
|
24
|
+
migrator = ActiveRecord::Migrator.new(:up, migrations_path)
|
25
|
+
migrations.each do |m|
|
26
|
+
migrator.__send__(:record_version_state_after_migrating, m)
|
27
|
+
end
|
28
|
+
if this_migration
|
29
|
+
migrator.__send__(:record_version_state_after_migrating, this_migration)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
module ActiveRecordShards
|
38
|
+
module MigrationExtension
|
39
|
+
attr_accessor :migration_shard
|
40
|
+
|
41
|
+
def shard(arg = nil)
|
42
|
+
self.migration_shard = arg
|
43
|
+
end
|
44
|
+
|
45
|
+
def migrate_with_forced_shard(direction)
|
46
|
+
if migration_shard.blank?
|
47
|
+
raise "Can't run migrations with a shard spec: this may be :all, :none, or a specific shard (for data-fixups) please call shard(arg) in your migration"
|
48
|
+
end
|
49
|
+
|
50
|
+
shard = ActiveRecord::Base.current_shard_selection.shard
|
51
|
+
|
52
|
+
if shard.nil?
|
53
|
+
return if migration_shard != :none
|
54
|
+
else
|
55
|
+
return if migration_shard == :none
|
56
|
+
return if migration_shard != :all && migration_shard.to_s != shard.to_s
|
57
|
+
end
|
58
|
+
|
59
|
+
migrate_without_forced_shard(direction)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
ActiveRecord::Migration.class_eval do
|
65
|
+
extend ActiveRecordShards::MigrationExtension
|
66
|
+
class << self
|
67
|
+
alias_method_chain :migrate, :forced_shard
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
ActiveRecord::MigrationProxy.delegate :migration_shard, :to => :migration
|
data/lib/active_record_shards.rb
CHANGED
@@ -6,6 +6,7 @@ require 'active_record_shards/shard_selection'
|
|
6
6
|
require 'active_record_shards/connection_switcher'
|
7
7
|
require 'active_record_shards/association_collection_connection_selection'
|
8
8
|
require 'active_record_shards/connection_pool'
|
9
|
+
require 'active_record_shards/migration'
|
9
10
|
|
10
11
|
ActiveRecord::Base.extend(ActiveRecordShards::ConfigurationParser)
|
11
12
|
ActiveRecord::Base.extend(ActiveRecordShards::Model)
|
@@ -48,8 +48,10 @@ class ConnectionSwitchenTest < ActiveSupport::TestCase
|
|
48
48
|
assert_not_equal(@shard_0_master.select_value("SELECT DATABASE()"), @shard_1_master.select_value("SELECT DATABASE()"))
|
49
49
|
|
50
50
|
@database_names = []
|
51
|
-
|
51
|
+
@database_shards = []
|
52
|
+
ActiveRecord::Base.on_all_shards do |shard|
|
52
53
|
@database_names << ActiveRecord::Base.connection.select_value("SELECT DATABASE()")
|
54
|
+
@database_shards << shard
|
53
55
|
end
|
54
56
|
end
|
55
57
|
|
@@ -57,6 +59,9 @@ class ConnectionSwitchenTest < ActiveSupport::TestCase
|
|
57
59
|
assert_equal(2, @database_names.size)
|
58
60
|
assert_contains(@database_names, @shard_0_master.select_value("SELECT DATABASE()"))
|
59
61
|
assert_contains(@database_names, @shard_1_master.select_value("SELECT DATABASE()"))
|
62
|
+
assert_equal(2, @database_shards.size)
|
63
|
+
assert_contains(@database_shards, "0")
|
64
|
+
assert_contains(@database_shards, "1")
|
60
65
|
end
|
61
66
|
end
|
62
67
|
end
|
data/test/database.yml
CHANGED
@@ -4,8 +4,7 @@ test:
|
|
4
4
|
database: ars_test
|
5
5
|
username: root
|
6
6
|
password:
|
7
|
-
|
8
|
-
host: localhost
|
7
|
+
host: 127.0.0.1
|
9
8
|
shard_names: ['0', '1']
|
10
9
|
|
11
10
|
test_slave:
|
@@ -14,8 +13,7 @@ test_slave:
|
|
14
13
|
database: ars_test_slave
|
15
14
|
username: root
|
16
15
|
password:
|
17
|
-
|
18
|
-
host: localhost
|
16
|
+
host: 127.0.0.1
|
19
17
|
|
20
18
|
test_shard_0:
|
21
19
|
adapter: mysql
|
@@ -23,8 +21,7 @@ test_shard_0:
|
|
23
21
|
database: ars_test_shard0
|
24
22
|
username: root
|
25
23
|
password:
|
26
|
-
|
27
|
-
host: localhost
|
24
|
+
host: 127.0.0.1
|
28
25
|
|
29
26
|
test_shard_0_slave:
|
30
27
|
adapter: mysql
|
@@ -32,8 +29,7 @@ test_shard_0_slave:
|
|
32
29
|
database: ars_test_shard0_slave
|
33
30
|
username: root
|
34
31
|
password:
|
35
|
-
|
36
|
-
host: localhost
|
32
|
+
host: 127.0.0.1
|
37
33
|
|
38
34
|
test_shard_1:
|
39
35
|
adapter: mysql
|
@@ -41,8 +37,7 @@ test_shard_1:
|
|
41
37
|
database: ars_test_shard1
|
42
38
|
username: root
|
43
39
|
password:
|
44
|
-
|
45
|
-
host: localhost
|
40
|
+
host: 127.0.0.1
|
46
41
|
|
47
42
|
test_shard_1_slave:
|
48
43
|
adapter: mysql
|
@@ -50,8 +45,7 @@ test_shard_1_slave:
|
|
50
45
|
database: ars_test_shard1_slave
|
51
46
|
username: root
|
52
47
|
password:
|
53
|
-
|
54
|
-
host: localhost
|
48
|
+
host: 127.0.0.1
|
55
49
|
|
56
50
|
test2:
|
57
51
|
adapter: mysql
|
@@ -59,8 +53,7 @@ test2:
|
|
59
53
|
database: ars_test2
|
60
54
|
username: root
|
61
55
|
password:
|
62
|
-
|
63
|
-
host: localhost
|
56
|
+
host: 127.0.0.1
|
64
57
|
|
65
58
|
test2_slave:
|
66
59
|
adapter: mysql
|
@@ -68,8 +61,7 @@ test2_slave:
|
|
68
61
|
database: ars_test2_slave
|
69
62
|
username: root
|
70
63
|
password:
|
71
|
-
|
72
|
-
host: localhost
|
64
|
+
host: 127.0.0.1
|
73
65
|
|
74
66
|
alternative:
|
75
67
|
adapter: mysql
|
@@ -77,6 +69,5 @@ alternative:
|
|
77
69
|
database: ars_test_alternative
|
78
70
|
username: root
|
79
71
|
password:
|
80
|
-
|
81
|
-
host: localhost
|
72
|
+
host: 127.0.0.1
|
82
73
|
shard_names: ['0', '1']
|
@@ -0,0 +1,14 @@
|
|
1
|
+
class FailureMigration < ActiveRecord::Migration
|
2
|
+
shard :all
|
3
|
+
|
4
|
+
def self.up
|
5
|
+
$fail_at_two ||= 0
|
6
|
+
$fail_at_two += 1
|
7
|
+
raise "FAIL FAIL FAIL" if $fail_at_two == 2
|
8
|
+
add_column :tickets, :sharded_column, :integer
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.down
|
12
|
+
remove_column :tickets, :sharded_column
|
13
|
+
end
|
14
|
+
end
|
data/test/helper.rb
CHANGED
@@ -18,17 +18,16 @@ RAILS_ENV = "test"
|
|
18
18
|
ActiveRecord::Base.logger = Logger.new(File.dirname(__FILE__) + "/test.log")
|
19
19
|
ActiveRecord::Base.configurations = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
ActiveRecord::Base.connection
|
26
|
-
rescue Mysql::Error => e
|
21
|
+
def init_schema
|
22
|
+
ActiveRecord::Base.configurations.each do |name, conf|
|
23
|
+
puts "Setting up the #{conf['database']} db"
|
24
|
+
`echo "drop DATABASE if exists #{conf['database']}" | mysql --user=#{conf['username']}`
|
27
25
|
`echo "create DATABASE #{conf['database']}" | mysql --user=#{conf['username']}`
|
28
26
|
ActiveRecord::Base.establish_connection(name)
|
27
|
+
load(File.dirname(__FILE__) + "/schema.rb")
|
29
28
|
end
|
30
|
-
load(File.dirname(__FILE__) + "/schema.rb")
|
31
29
|
end
|
30
|
+
init_schema
|
32
31
|
|
33
32
|
require 'models'
|
34
33
|
|
@@ -0,0 +1,12 @@
|
|
1
|
+
#This is to recreate the deleted onetime payment for DealFish
|
2
|
+
class ShardMigration < ActiveRecord::Migration
|
3
|
+
shard :all
|
4
|
+
|
5
|
+
def self.up
|
6
|
+
add_column :emails, :sharded_column, :integer
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.down
|
10
|
+
remove_column :emails, :sharded_column
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# See ticket 130214
|
2
|
+
# Setting up an odd initial three year term for box.net to be paid by credit card by the normal means
|
3
|
+
class AccountMigration < ActiveRecord::Migration
|
4
|
+
shard :none
|
5
|
+
|
6
|
+
def self.up
|
7
|
+
add_column :accounts, :non_sharded_column, :integer
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.down
|
11
|
+
remove_column :accounts, :non_sharded_column
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require File.expand_path('helper', File.dirname(__FILE__))
|
2
|
+
|
3
|
+
class CowardlyMigration < ActiveRecord::Migration
|
4
|
+
def self.up
|
5
|
+
"not gonna happen"
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.down
|
9
|
+
"uh uh"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class MigratorTest < ActiveSupport::TestCase
|
14
|
+
def setup
|
15
|
+
init_schema
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_migrator
|
19
|
+
migration_path = File.join(File.dirname(__FILE__), "/migrations")
|
20
|
+
ActiveRecord::Migrator.migrate(migration_path)
|
21
|
+
ActiveRecord::Base.on_all_shards do
|
22
|
+
assert ActiveRecord::Base.connection.table_exists?(:schema_migrations), "Schema Migrations doesn't exist"
|
23
|
+
assert ActiveRecord::Base.connection.table_exists?(:accounts)
|
24
|
+
assert ActiveRecord::Base.connection.select_value("select version from schema_migrations where version = '20110824010216'")
|
25
|
+
assert ActiveRecord::Base.connection.select_value("select version from schema_migrations where version = '20110829215912'")
|
26
|
+
end
|
27
|
+
|
28
|
+
ActiveRecord::Base.on_all_shards do
|
29
|
+
assert table_has_column?("emails", "sharded_column")
|
30
|
+
assert !table_has_column?("accounts", "non_sharded_column")
|
31
|
+
end
|
32
|
+
|
33
|
+
ActiveRecord::Base.on_shard(nil) do
|
34
|
+
assert !table_has_column?("emails", "sharded_column")
|
35
|
+
assert table_has_column?("accounts", "non_sharded_column")
|
36
|
+
end
|
37
|
+
|
38
|
+
# now test down/ up
|
39
|
+
ActiveRecord::Migrator.run(:down, migration_path, 20110824010216)
|
40
|
+
ActiveRecord::Base.on_all_shards do
|
41
|
+
assert !table_has_column?("emails", "sharded_column")
|
42
|
+
end
|
43
|
+
|
44
|
+
ActiveRecord::Migrator.run(:down, migration_path, 20110829215912)
|
45
|
+
ActiveRecord::Base.on_shard(nil) do
|
46
|
+
assert !table_has_column?("accounts", "non_sharded_column")
|
47
|
+
end
|
48
|
+
|
49
|
+
ActiveRecord::Migrator.run(:up, migration_path, 20110824010216)
|
50
|
+
ActiveRecord::Base.on_all_shards do
|
51
|
+
assert table_has_column?("emails", "sharded_column")
|
52
|
+
end
|
53
|
+
|
54
|
+
ActiveRecord::Migrator.run(:up, migration_path, 20110829215912)
|
55
|
+
ActiveRecord::Base.on_shard(nil) do
|
56
|
+
assert table_has_column?("accounts", "non_sharded_column")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_bad_migration
|
61
|
+
assert_raises(RuntimeError) do
|
62
|
+
CowardlyMigration.up
|
63
|
+
end
|
64
|
+
|
65
|
+
assert_raises(RuntimeError) do
|
66
|
+
CowardlyMigration.down
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_failing_migration
|
71
|
+
# like, if you have to break a migration in the middle somewhere.
|
72
|
+
begin
|
73
|
+
migration_path = File.join(File.dirname(__FILE__), "/failure_migration")
|
74
|
+
ActiveRecord::Migrator.migrate(migration_path)
|
75
|
+
rescue
|
76
|
+
retry
|
77
|
+
end
|
78
|
+
|
79
|
+
ActiveRecord::Base.on_all_shards do
|
80
|
+
assert table_has_column?("tickets", "sharded_column")
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
def table_has_column?(table, column)
|
86
|
+
ActiveRecord::Base.connection.select_value("show create table #{table}") =~ /#{column}/
|
87
|
+
end
|
88
|
+
end
|
data/test/schema.rb
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
ActiveRecord::Schema.define(:version => 1) do
|
2
|
+
suppress_messages do
|
3
|
+
create_table "accounts", :force => true do |t|
|
4
|
+
t.string "name"
|
5
|
+
t.datetime "created_at"
|
6
|
+
t.datetime "updated_at"
|
7
|
+
end
|
2
8
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
t.string "from"
|
11
|
-
t.string "to"
|
12
|
-
t.text "mail"
|
13
|
-
t.datetime "created_at"
|
14
|
-
t.datetime "updated_at"
|
15
|
-
end
|
9
|
+
create_table "emails", :force => true do |t|
|
10
|
+
t.string "from"
|
11
|
+
t.string "to"
|
12
|
+
t.text "mail"
|
13
|
+
t.datetime "created_at"
|
14
|
+
t.datetime "updated_at"
|
15
|
+
end
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
17
|
+
create_table "tickets", :force => true do |t|
|
18
|
+
t.string "title"
|
19
|
+
t.integer "account_id"
|
20
|
+
t.datetime "created_at"
|
21
|
+
t.datetime "updated_at"
|
22
|
+
end
|
22
23
|
end
|
23
|
-
|
24
|
-
end
|
24
|
+
end
|
metadata
CHANGED
@@ -1,22 +1,23 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 29
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 2
|
8
|
-
-
|
8
|
+
- 4
|
9
9
|
- 1
|
10
|
-
version: 2.
|
10
|
+
version: 2.4.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Mick Staugaard
|
14
14
|
- Eric Chapweske
|
15
|
+
- Ben Osheroff
|
15
16
|
autorequire:
|
16
17
|
bindir: bin
|
17
18
|
cert_chain: []
|
18
19
|
|
19
|
-
date: 2011-
|
20
|
+
date: 2011-08-31 00:00:00 -07:00
|
20
21
|
default_executable:
|
21
22
|
dependencies:
|
22
23
|
- !ruby/object:Gem::Dependency
|
@@ -109,6 +110,7 @@ description: Easily run queries on shard and slave databases.
|
|
109
110
|
email:
|
110
111
|
- mick@staugaard.com
|
111
112
|
- eac@zendesk.com
|
113
|
+
- ben@gimbo.net
|
112
114
|
executables: []
|
113
115
|
|
114
116
|
extensions: []
|
@@ -120,6 +122,7 @@ files:
|
|
120
122
|
- lib/active_record_shards/configuration_parser.rb
|
121
123
|
- lib/active_record_shards/connection_pool.rb
|
122
124
|
- lib/active_record_shards/connection_switcher.rb
|
125
|
+
- lib/active_record_shards/migration.rb
|
123
126
|
- lib/active_record_shards/model.rb
|
124
127
|
- lib/active_record_shards/shard_selection.rb
|
125
128
|
- lib/active_record_shards/tasks.rb
|
@@ -129,10 +132,13 @@ files:
|
|
129
132
|
- test/connection_switching_test.rb
|
130
133
|
- test/database.yml
|
131
134
|
- test/database_parse_test.yml
|
135
|
+
- test/failure_migration/20110824010215_failure_migration.rb
|
132
136
|
- test/helper.rb
|
137
|
+
- test/migrations/20110824010216_shard_migration.rb
|
138
|
+
- test/migrations/20110829215912_account_migration.rb
|
139
|
+
- test/migrator_test.rb
|
133
140
|
- test/models.rb
|
134
141
|
- test/schema.rb
|
135
|
-
- test/test.log
|
136
142
|
has_rdoc: true
|
137
143
|
homepage: http://github.com/staugaard/active_record_shards
|
138
144
|
licenses: []
|
@@ -172,7 +178,10 @@ test_files:
|
|
172
178
|
- test/connection_switching_test.rb
|
173
179
|
- test/database.yml
|
174
180
|
- test/database_parse_test.yml
|
181
|
+
- test/failure_migration/20110824010215_failure_migration.rb
|
175
182
|
- test/helper.rb
|
183
|
+
- test/migrations/20110824010216_shard_migration.rb
|
184
|
+
- test/migrations/20110829215912_account_migration.rb
|
185
|
+
- test/migrator_test.rb
|
176
186
|
- test/models.rb
|
177
187
|
- test/schema.rb
|
178
|
-
- test/test.log
|