active_record_shards 3.15.2 → 3.19.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +15 -15
- data/lib/active_record_shards.rb +109 -6
- data/lib/active_record_shards/association_collection_connection_selection.rb +23 -14
- data/lib/active_record_shards/configuration_parser.rb +14 -5
- data/lib/active_record_shards/connection_handler.rb +1 -0
- data/lib/active_record_shards/connection_pool.rb +1 -0
- data/lib/active_record_shards/connection_specification.rb +1 -0
- data/lib/active_record_shards/{connection_switcher-4-0.rb → connection_switcher-4-2.rb} +2 -2
- data/lib/active_record_shards/connection_switcher.rb +63 -42
- data/lib/active_record_shards/default_replica_patches.rb +278 -0
- data/lib/active_record_shards/default_slave_patches.rb +3 -132
- data/lib/active_record_shards/deprecation.rb +12 -0
- data/lib/active_record_shards/migration.rb +5 -2
- data/lib/active_record_shards/model.rb +16 -11
- data/lib/active_record_shards/patches-4-2.rb +1 -0
- data/lib/active_record_shards/schema_dumper_extension.rb +6 -5
- data/lib/active_record_shards/shard_selection.rb +17 -14
- data/lib/active_record_shards/shard_support.rb +1 -0
- data/lib/active_record_shards/sql_comments.rb +7 -4
- data/lib/active_record_shards/tasks.rb +19 -13
- metadata +58 -42
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module ActiveRecordShards
|
3
4
|
module SchemaDumperExtension
|
4
5
|
def dump(stream)
|
@@ -23,15 +24,15 @@ module ActiveRecordShards
|
|
23
24
|
def shard_header(stream)
|
24
25
|
define_params = @version ? "version: #{@version}" : ""
|
25
26
|
|
26
|
-
stream.puts
|
27
|
+
stream.puts <<~HEADER
|
27
28
|
|
28
29
|
|
29
|
-
# This section generated by active_record_shards
|
30
|
+
# This section generated by active_record_shards
|
30
31
|
|
31
|
-
ActiveRecord::Base.on_all_shards do
|
32
|
-
ActiveRecord::Schema.define(#{define_params}) do
|
32
|
+
ActiveRecord::Base.on_all_shards do
|
33
|
+
ActiveRecord::Schema.define(#{define_params}) do
|
33
34
|
|
34
|
-
HEADER
|
35
|
+
HEADER
|
35
36
|
end
|
36
37
|
|
37
38
|
def shard_trailer(stream)
|
@@ -1,11 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module ActiveRecordShards
|
3
4
|
class ShardSelection
|
4
5
|
NO_SHARD = :_no_shard
|
5
6
|
cattr_accessor :default_shard
|
6
7
|
|
7
8
|
def initialize
|
8
|
-
@
|
9
|
+
@on_replica = false
|
9
10
|
@shard = nil
|
10
11
|
end
|
11
12
|
|
@@ -21,17 +22,17 @@ module ActiveRecordShards
|
|
21
22
|
end
|
22
23
|
end
|
23
24
|
|
24
|
-
def shard_name(klass = nil,
|
25
|
+
def shard_name(klass = nil, try_replica = true)
|
25
26
|
the_shard = shard(klass)
|
26
27
|
|
27
28
|
@shard_names ||= {}
|
28
29
|
@shard_names[ActiveRecordShards.rails_env] ||= {}
|
29
30
|
@shard_names[ActiveRecordShards.rails_env][the_shard] ||= {}
|
30
|
-
@shard_names[ActiveRecordShards.rails_env][the_shard][
|
31
|
-
@shard_names[ActiveRecordShards.rails_env][the_shard][
|
31
|
+
@shard_names[ActiveRecordShards.rails_env][the_shard][try_replica] ||= {}
|
32
|
+
@shard_names[ActiveRecordShards.rails_env][the_shard][try_replica][@on_replica] ||= begin
|
32
33
|
s = ActiveRecordShards.rails_env.dup
|
33
34
|
s << "_shard_#{the_shard}" if the_shard
|
34
|
-
s << "
|
35
|
+
s << "_replica" if @on_replica && try_replica
|
35
36
|
s
|
36
37
|
end
|
37
38
|
end
|
@@ -46,7 +47,7 @@ module ActiveRecordShards
|
|
46
47
|
end
|
47
48
|
end
|
48
49
|
|
49
|
-
PRIMARY = "primary"
|
50
|
+
PRIMARY = "primary"
|
50
51
|
def resolve_connection_name(sharded:, configurations:)
|
51
52
|
resolved_shard = sharded ? shard : nil
|
52
53
|
env = ActiveRecordShards.rails_env
|
@@ -54,11 +55,11 @@ module ActiveRecordShards
|
|
54
55
|
@connection_names ||= {}
|
55
56
|
@connection_names[env] ||= {}
|
56
57
|
@connection_names[env][resolved_shard] ||= {}
|
57
|
-
@connection_names[env][resolved_shard][@
|
58
|
+
@connection_names[env][resolved_shard][@on_replica] ||= begin
|
58
59
|
name = env.dup
|
59
60
|
name << "_shard_#{resolved_shard}" if resolved_shard
|
60
|
-
if @
|
61
|
-
"#{name}
|
61
|
+
if @on_replica && configurations["#{name}_replica"]
|
62
|
+
"#{name}_replica"
|
62
63
|
else
|
63
64
|
# ActiveRecord always names its default connection pool 'primary'
|
64
65
|
# while everything else is named by the configuration name
|
@@ -73,16 +74,18 @@ module ActiveRecordShards
|
|
73
74
|
@shard = (new_shard || NO_SHARD)
|
74
75
|
end
|
75
76
|
|
76
|
-
def
|
77
|
-
@
|
77
|
+
def on_replica?
|
78
|
+
@on_replica
|
78
79
|
end
|
80
|
+
alias_method :on_slave?, :on_replica?
|
79
81
|
|
80
|
-
def
|
81
|
-
@
|
82
|
+
def on_replica=(new_replica)
|
83
|
+
@on_replica = (new_replica == true)
|
82
84
|
end
|
85
|
+
alias_method :on_slave=, :on_replica=
|
83
86
|
|
84
87
|
def options
|
85
|
-
{ shard: @shard,
|
88
|
+
{ shard: @shard, replica: @on_replica }
|
86
89
|
end
|
87
90
|
end
|
88
91
|
end
|
@@ -1,16 +1,19 @@
|
|
1
|
-
# show which connection was picked to debug
|
1
|
+
# show which connection was picked to debug primary/replica slowness when both servers are the same
|
2
2
|
module ActiveRecordShards
|
3
3
|
module SqlComments
|
4
4
|
module Methods
|
5
5
|
def execute(query, name = nil)
|
6
|
-
|
7
|
-
|
6
|
+
shard = ActiveRecord::Base.current_shard_selection.shard
|
7
|
+
shard_text = shard ? "shard #{shard}" : 'unsharded'
|
8
|
+
replica = ActiveRecord::Base.current_shard_selection.on_replica?
|
9
|
+
replica_text = replica ? 'replica' : 'primary'
|
10
|
+
query = "/* #{shard_text} #{replica_text} */ " + query
|
8
11
|
super(query, name)
|
9
12
|
end
|
10
13
|
end
|
11
14
|
|
12
15
|
def self.enable
|
13
|
-
ActiveRecord::Base.
|
16
|
+
ActiveRecord::Base.on_replica do
|
14
17
|
ActiveRecord::Base.on_shard(nil) do
|
15
18
|
ActiveRecord::Base.connection.class.prepend(Methods)
|
16
19
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'active_record_shards'
|
3
4
|
|
4
5
|
%w[db:drop db:create db:abort_if_pending_migrations db:reset db:test:purge].each do |name|
|
@@ -8,15 +9,16 @@ end
|
|
8
9
|
namespace :db do
|
9
10
|
desc 'Drops the database for the current RAILS_ENV including shards'
|
10
11
|
task drop: :load_config do
|
11
|
-
ActiveRecord::Base.configurations.each do |key, conf|
|
12
|
-
next if !key.
|
12
|
+
ActiveRecord::Base.configurations.to_h.each do |key, conf|
|
13
|
+
next if !key.start_with?(ActiveRecordShards.rails_env) || key.end_with?("_replica", "_slave")
|
14
|
+
|
13
15
|
begin
|
14
16
|
ActiveRecordShards::Tasks.root_connection(conf).drop_database(conf['database'])
|
15
17
|
# rescue ActiveRecord::NoDatabaseError # TODO: exists in AR but never is raised here ...
|
16
18
|
# $stderr.puts "Database '#{conf['database']}' does not exist"
|
17
|
-
rescue StandardError =>
|
18
|
-
|
19
|
-
|
19
|
+
rescue StandardError => e
|
20
|
+
warn e, *e.backtrace
|
21
|
+
warn "Couldn't drop #{conf['database']}"
|
20
22
|
end
|
21
23
|
end
|
22
24
|
end
|
@@ -28,8 +30,9 @@ namespace :db do
|
|
28
30
|
|
29
31
|
desc "Create the database defined in config/database.yml for the current RAILS_ENV including shards"
|
30
32
|
task create: :load_config do
|
31
|
-
ActiveRecord::Base.configurations.each do |key, conf|
|
32
|
-
next if !key.
|
33
|
+
ActiveRecord::Base.configurations.to_h.each do |key, conf|
|
34
|
+
next if !key.start_with?(ActiveRecordShards.rails_env) || key.end_with?("_replica", "_slave")
|
35
|
+
|
33
36
|
begin
|
34
37
|
# MysqlAdapter takes charset instead of encoding in Rails 4.2 or greater
|
35
38
|
# https://github.com/rails/rails/blob/4-2-stable/activerecord/lib/active_record/tasks/mysql_database_tasks.rb#L85-L96
|
@@ -37,11 +40,11 @@ namespace :db do
|
|
37
40
|
symbolized_configuration[:charset] = symbolized_configuration[:encoding]
|
38
41
|
|
39
42
|
ActiveRecordShards::Tasks.root_connection(conf).create_database(conf['database'], symbolized_configuration)
|
40
|
-
rescue ActiveRecord::StatementInvalid =>
|
41
|
-
if
|
43
|
+
rescue ActiveRecord::StatementInvalid => e
|
44
|
+
if e.message.include?('database exists')
|
42
45
|
puts "#{conf['database']} already exists"
|
43
46
|
else
|
44
|
-
raise
|
47
|
+
raise e
|
45
48
|
end
|
46
49
|
end
|
47
50
|
end
|
@@ -52,7 +55,10 @@ namespace :db do
|
|
52
55
|
task abort_if_pending_migrations: :environment do
|
53
56
|
if defined? ActiveRecord
|
54
57
|
pending_migrations =
|
55
|
-
if ActiveRecord::VERSION::
|
58
|
+
if ActiveRecord::VERSION::MAJOR >= 6
|
59
|
+
migrations = ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths, ActiveRecord::SchemaMigration).migrations
|
60
|
+
ActiveRecord::Migrator.new(:up, migrations, ActiveRecord::SchemaMigration).pending_migrations
|
61
|
+
elsif ActiveRecord::VERSION::STRING >= "5.2.0"
|
56
62
|
migrations = ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths).migrations
|
57
63
|
ActiveRecord::Migrator.new(:up, migrations).pending_migrations
|
58
64
|
else
|
@@ -60,9 +66,9 @@ namespace :db do
|
|
60
66
|
end
|
61
67
|
|
62
68
|
if pending_migrations.any?
|
63
|
-
|
69
|
+
warn "You have #{pending_migrations.size} pending migrations:"
|
64
70
|
pending_migrations.each do |pending_migration|
|
65
|
-
|
71
|
+
warn ' %4d %s' % [pending_migration.version, pending_migration.name]
|
66
72
|
end
|
67
73
|
abort %(Run "rake db:migrate" to update your database then try again.)
|
68
74
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.19.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Quorning
|
@@ -10,10 +10,10 @@ authors:
|
|
10
10
|
- Mick Staugaard
|
11
11
|
- Eric Chapweske
|
12
12
|
- Ben Osheroff
|
13
|
-
autorequire:
|
13
|
+
autorequire:
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
|
-
date:
|
16
|
+
date: 2021-07-09 00:00:00.000000000 Z
|
17
17
|
dependencies:
|
18
18
|
- !ruby/object:Gem::Dependency
|
19
19
|
name: activerecord
|
@@ -24,7 +24,7 @@ dependencies:
|
|
24
24
|
version: '4.2'
|
25
25
|
- - "<"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '6.
|
27
|
+
version: '6.1'
|
28
28
|
type: :runtime
|
29
29
|
prerelease: false
|
30
30
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
version: '4.2'
|
35
35
|
- - "<"
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: '6.
|
37
|
+
version: '6.1'
|
38
38
|
- !ruby/object:Gem::Dependency
|
39
39
|
name: activesupport
|
40
40
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,7 +44,7 @@ dependencies:
|
|
44
44
|
version: '4.2'
|
45
45
|
- - "<"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '6.
|
47
|
+
version: '6.1'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -54,37 +54,37 @@ dependencies:
|
|
54
54
|
version: '4.2'
|
55
55
|
- - "<"
|
56
56
|
- !ruby/object:Gem::Version
|
57
|
-
version: '6.
|
57
|
+
version: '6.1'
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
|
-
name:
|
59
|
+
name: bump
|
60
60
|
requirement: !ruby/object:Gem::Requirement
|
61
61
|
requirements:
|
62
|
-
- - "
|
62
|
+
- - ">="
|
63
63
|
- !ruby/object:Gem::Version
|
64
|
-
version: '
|
64
|
+
version: '0'
|
65
65
|
type: :development
|
66
66
|
prerelease: false
|
67
67
|
version_requirements: !ruby/object:Gem::Requirement
|
68
68
|
requirements:
|
69
|
-
- - "
|
69
|
+
- - ">="
|
70
70
|
- !ruby/object:Gem::Version
|
71
|
-
version: '
|
71
|
+
version: '0'
|
72
72
|
- !ruby/object:Gem::Dependency
|
73
|
-
name:
|
73
|
+
name: minitest
|
74
74
|
requirement: !ruby/object:Gem::Requirement
|
75
75
|
requirements:
|
76
76
|
- - ">="
|
77
77
|
- !ruby/object:Gem::Version
|
78
|
-
version:
|
78
|
+
version: 5.10.0
|
79
79
|
type: :development
|
80
80
|
prerelease: false
|
81
81
|
version_requirements: !ruby/object:Gem::Requirement
|
82
82
|
requirements:
|
83
83
|
- - ">="
|
84
84
|
- !ruby/object:Gem::Version
|
85
|
-
version:
|
85
|
+
version: 5.10.0
|
86
86
|
- !ruby/object:Gem::Dependency
|
87
|
-
name:
|
87
|
+
name: minitest-rg
|
88
88
|
requirement: !ruby/object:Gem::Requirement
|
89
89
|
requirements:
|
90
90
|
- - ">="
|
@@ -98,21 +98,21 @@ dependencies:
|
|
98
98
|
- !ruby/object:Gem::Version
|
99
99
|
version: '0'
|
100
100
|
- !ruby/object:Gem::Dependency
|
101
|
-
name:
|
101
|
+
name: mocha
|
102
102
|
requirement: !ruby/object:Gem::Requirement
|
103
103
|
requirements:
|
104
|
-
- -
|
104
|
+
- - ">="
|
105
105
|
- !ruby/object:Gem::Version
|
106
|
-
version:
|
106
|
+
version: 1.4.0
|
107
107
|
type: :development
|
108
108
|
prerelease: false
|
109
109
|
version_requirements: !ruby/object:Gem::Requirement
|
110
110
|
requirements:
|
111
|
-
- -
|
111
|
+
- - ">="
|
112
112
|
- !ruby/object:Gem::Version
|
113
|
-
version:
|
113
|
+
version: 1.4.0
|
114
114
|
- !ruby/object:Gem::Dependency
|
115
|
-
name:
|
115
|
+
name: mysql2
|
116
116
|
requirement: !ruby/object:Gem::Requirement
|
117
117
|
requirements:
|
118
118
|
- - ">="
|
@@ -126,48 +126,62 @@ dependencies:
|
|
126
126
|
- !ruby/object:Gem::Version
|
127
127
|
version: '0'
|
128
128
|
- !ruby/object:Gem::Dependency
|
129
|
-
name:
|
129
|
+
name: rake
|
130
130
|
requirement: !ruby/object:Gem::Requirement
|
131
131
|
requirements:
|
132
|
-
- - "
|
132
|
+
- - "~>"
|
133
133
|
- !ruby/object:Gem::Version
|
134
|
-
version: '0'
|
134
|
+
version: '12.0'
|
135
135
|
type: :development
|
136
136
|
prerelease: false
|
137
137
|
version_requirements: !ruby/object:Gem::Requirement
|
138
138
|
requirements:
|
139
|
-
- - "
|
139
|
+
- - "~>"
|
140
140
|
- !ruby/object:Gem::Version
|
141
|
-
version: '0'
|
141
|
+
version: '12.0'
|
142
142
|
- !ruby/object:Gem::Dependency
|
143
|
-
name:
|
143
|
+
name: rubocop
|
144
144
|
requirement: !ruby/object:Gem::Requirement
|
145
145
|
requirements:
|
146
|
-
- - "
|
146
|
+
- - "~>"
|
147
147
|
- !ruby/object:Gem::Version
|
148
|
-
version:
|
148
|
+
version: 0.77.0
|
149
149
|
type: :development
|
150
150
|
prerelease: false
|
151
151
|
version_requirements: !ruby/object:Gem::Requirement
|
152
152
|
requirements:
|
153
|
-
- - "
|
153
|
+
- - "~>"
|
154
154
|
- !ruby/object:Gem::Version
|
155
|
-
version:
|
155
|
+
version: 0.77.0
|
156
156
|
- !ruby/object:Gem::Dependency
|
157
|
-
name:
|
157
|
+
name: rubocop-minitest
|
158
158
|
requirement: !ruby/object:Gem::Requirement
|
159
159
|
requirements:
|
160
|
-
- - "
|
160
|
+
- - "~>"
|
161
161
|
- !ruby/object:Gem::Version
|
162
|
-
version: 0.
|
162
|
+
version: 0.5.0
|
163
163
|
type: :development
|
164
164
|
prerelease: false
|
165
165
|
version_requirements: !ruby/object:Gem::Requirement
|
166
166
|
requirements:
|
167
|
-
- - "
|
167
|
+
- - "~>"
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
version: 0.5.0
|
170
|
+
- !ruby/object:Gem::Dependency
|
171
|
+
name: rubocop-performance
|
172
|
+
requirement: !ruby/object:Gem::Requirement
|
173
|
+
requirements:
|
174
|
+
- - "~>"
|
175
|
+
- !ruby/object:Gem::Version
|
176
|
+
version: 1.5.1
|
177
|
+
type: :development
|
178
|
+
prerelease: false
|
179
|
+
version_requirements: !ruby/object:Gem::Requirement
|
180
|
+
requirements:
|
181
|
+
- - "~>"
|
168
182
|
- !ruby/object:Gem::Version
|
169
|
-
version:
|
170
|
-
description: Easily run queries on shard and
|
183
|
+
version: 1.5.1
|
184
|
+
description: Easily run queries on shard and replica databases.
|
171
185
|
email:
|
172
186
|
- bquorning@zendesk.com
|
173
187
|
- gabe@zendesk.com
|
@@ -184,11 +198,13 @@ files:
|
|
184
198
|
- lib/active_record_shards/connection_handler.rb
|
185
199
|
- lib/active_record_shards/connection_pool.rb
|
186
200
|
- lib/active_record_shards/connection_specification.rb
|
187
|
-
- lib/active_record_shards/connection_switcher-4-
|
201
|
+
- lib/active_record_shards/connection_switcher-4-2.rb
|
188
202
|
- lib/active_record_shards/connection_switcher-5-0.rb
|
189
203
|
- lib/active_record_shards/connection_switcher-5-1.rb
|
190
204
|
- lib/active_record_shards/connection_switcher.rb
|
205
|
+
- lib/active_record_shards/default_replica_patches.rb
|
191
206
|
- lib/active_record_shards/default_slave_patches.rb
|
207
|
+
- lib/active_record_shards/deprecation.rb
|
192
208
|
- lib/active_record_shards/migration.rb
|
193
209
|
- lib/active_record_shards/model.rb
|
194
210
|
- lib/active_record_shards/patches-4-2.rb
|
@@ -201,7 +217,7 @@ homepage: https://github.com/zendesk/active_record_shards
|
|
201
217
|
licenses:
|
202
218
|
- MIT
|
203
219
|
metadata: {}
|
204
|
-
post_install_message:
|
220
|
+
post_install_message:
|
205
221
|
rdoc_options: []
|
206
222
|
require_paths:
|
207
223
|
- lib
|
@@ -216,8 +232,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
216
232
|
- !ruby/object:Gem::Version
|
217
233
|
version: '0'
|
218
234
|
requirements: []
|
219
|
-
rubygems_version: 3.
|
220
|
-
signing_key:
|
235
|
+
rubygems_version: 3.2.16
|
236
|
+
signing_key:
|
221
237
|
specification_version: 4
|
222
238
|
summary: Simple database switching for ActiveRecord.
|
223
239
|
test_files: []
|