switchman 3.4.2 → 3.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +15 -14
- data/db/migrate/20180828183945_add_default_shard_index.rb +1 -1
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +10 -4
- data/lib/switchman/active_record/associations.rb +16 -5
- data/lib/switchman/active_record/attribute_methods.rb +67 -22
- data/lib/switchman/active_record/base.rb +32 -12
- data/lib/switchman/active_record/calculations.rb +37 -33
- data/lib/switchman/active_record/connection_pool.rb +4 -2
- data/lib/switchman/active_record/database_configurations.rb +12 -7
- data/lib/switchman/active_record/finder_methods.rb +1 -1
- data/lib/switchman/active_record/log_subscriber.rb +2 -2
- data/lib/switchman/active_record/migration.rb +4 -2
- data/lib/switchman/active_record/postgresql_adapter.rb +11 -10
- data/lib/switchman/active_record/query_cache.rb +1 -1
- data/lib/switchman/active_record/query_methods.rb +72 -26
- data/lib/switchman/active_record/relation.rb +13 -7
- data/lib/switchman/active_record/spawn_methods.rb +2 -2
- data/lib/switchman/active_record/statement_cache.rb +2 -2
- data/lib/switchman/active_record/tasks/database_tasks.rb +1 -1
- data/lib/switchman/active_record/test_fixtures.rb +19 -16
- data/lib/switchman/active_support/cache.rb +4 -1
- data/lib/switchman/arel.rb +6 -6
- data/lib/switchman/call_super.rb +1 -1
- data/lib/switchman/database_server.rb +20 -16
- data/lib/switchman/default_shard.rb +3 -3
- data/lib/switchman/engine.rb +33 -18
- data/lib/switchman/environment.rb +2 -2
- data/lib/switchman/errors.rb +4 -1
- data/lib/switchman/guard_rail/relation.rb +1 -1
- data/lib/switchman/parallel.rb +1 -1
- data/lib/switchman/r_spec_helper.rb +10 -10
- data/lib/switchman/shard.rb +30 -23
- data/lib/switchman/sharded_instrumenter.rb +5 -1
- data/lib/switchman/test_helper.rb +1 -1
- data/lib/switchman/version.rb +1 -1
- data/lib/switchman.rb +10 -4
- data/lib/tasks/switchman.rake +40 -37
- metadata +9 -9
data/lib/tasks/switchman.rake
CHANGED
@@ -2,8 +2,8 @@
|
|
2
2
|
|
3
3
|
# In rails 7.0+ if you have only 1 db in the env it doesn't try to do explicit activation
|
4
4
|
# (and for rails purposes we only have one db per env because each database server is a separate env)
|
5
|
-
if Rails.version <
|
6
|
-
task_prefix = Rake::Task.task_defined?(
|
5
|
+
if Rails.version < "7.0"
|
6
|
+
task_prefix = Rake::Task.task_defined?("app:db:migrate") ? "app:db" : "db"
|
7
7
|
Rake::Task["#{task_prefix}:migrate"].clear_actions.enhance do
|
8
8
|
ActiveRecord::Tasks::DatabaseTasks.migrate
|
9
9
|
# Ensure this doesn't blow up when running inside the dummy app
|
@@ -13,26 +13,27 @@ end
|
|
13
13
|
|
14
14
|
module Switchman
|
15
15
|
module Rake
|
16
|
-
def self.filter_database_servers
|
17
|
-
|
18
|
-
|
16
|
+
def self.filter_database_servers
|
17
|
+
# use a local variable so that the current chain is closed over in the following lambda
|
18
|
+
chain = filter_database_servers_chain
|
19
|
+
@filter_database_servers_chain = ->(servers) { yield(servers, chain) }
|
19
20
|
end
|
20
21
|
|
21
22
|
def self.scope(base_scope = Shard,
|
22
|
-
database_server: ENV.fetch(
|
23
|
-
shard: ENV.fetch(
|
23
|
+
database_server: ENV.fetch("DATABASE_SERVER", nil),
|
24
|
+
shard: ENV.fetch("SHARD", nil))
|
24
25
|
servers = DatabaseServer.all
|
25
26
|
|
26
27
|
if database_server
|
27
28
|
servers = database_server
|
28
|
-
if servers.first ==
|
29
|
+
if servers.first == "-"
|
29
30
|
negative = true
|
30
31
|
servers = servers[1..]
|
31
32
|
end
|
32
|
-
servers = servers.split(
|
33
|
-
open = servers.delete(
|
33
|
+
servers = servers.split(",")
|
34
|
+
open = servers.delete("open")
|
34
35
|
|
35
|
-
servers = servers.
|
36
|
+
servers = servers.filter_map { |server| DatabaseServer.find(server) }
|
36
37
|
if open
|
37
38
|
open_servers = DatabaseServer.all.select { |server| server.config[:open] }
|
38
39
|
servers.concat(open_servers)
|
@@ -44,7 +45,7 @@ module Switchman
|
|
44
45
|
|
45
46
|
servers = filter_database_servers_chain.call(servers)
|
46
47
|
|
47
|
-
scope = base_scope.order(::Arel.sql(
|
48
|
+
scope = base_scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
|
48
49
|
if servers != DatabaseServer.all
|
49
50
|
database_server_ids = servers.map(&:id)
|
50
51
|
database_server_ids << nil if servers.include?(Shard.default.database_server)
|
@@ -57,7 +58,7 @@ module Switchman
|
|
57
58
|
end
|
58
59
|
|
59
60
|
def self.options
|
60
|
-
{ parallel: ENV[
|
61
|
+
{ parallel: ENV["PARALLEL"].to_i }
|
61
62
|
end
|
62
63
|
|
63
64
|
# classes - an array or proc, to activate as the current shard during the
|
@@ -69,7 +70,7 @@ module Switchman
|
|
69
70
|
|
70
71
|
old_task.enhance do |*task_args|
|
71
72
|
if ::Rails.env.test?
|
72
|
-
require
|
73
|
+
require "switchman/test_helper"
|
73
74
|
TestHelper.recreate_persistent_test_shards(dont_create: true)
|
74
75
|
end
|
75
76
|
|
@@ -86,7 +87,9 @@ module Switchman
|
|
86
87
|
nil
|
87
88
|
end
|
88
89
|
rescue => e
|
89
|
-
|
90
|
+
if options[:parallel] != 0
|
91
|
+
warn "Exception from #{e.current_shard.id}: #{e.current_shard.description}:\n#{e.full_message}"
|
92
|
+
end
|
90
93
|
raise
|
91
94
|
end
|
92
95
|
end
|
@@ -98,7 +101,7 @@ module Switchman
|
|
98
101
|
end
|
99
102
|
|
100
103
|
def self.shard_scope(scope, raw_shard_ids)
|
101
|
-
raw_shard_ids = raw_shard_ids.split(
|
104
|
+
raw_shard_ids = raw_shard_ids.split(",")
|
102
105
|
|
103
106
|
shard_ids = []
|
104
107
|
negative_shard_ids = []
|
@@ -108,13 +111,13 @@ module Switchman
|
|
108
111
|
|
109
112
|
raw_shard_ids.each do |id|
|
110
113
|
case id
|
111
|
-
when
|
114
|
+
when "default"
|
112
115
|
shard_ids << Shard.default.id
|
113
|
-
when
|
116
|
+
when "-default"
|
114
117
|
negative_shard_ids << Shard.default.id
|
115
|
-
when
|
118
|
+
when "primary"
|
116
119
|
shard_ids.concat(Shard.primary.pluck(:id))
|
117
|
-
when
|
120
|
+
when "-primary"
|
118
121
|
negative_shard_ids.concat(Shard.primary.pluck(:id))
|
119
122
|
when /^(-?)(\d+)?\.\.(\.)?(\d+)?$/
|
120
123
|
negative, start, open, finish = $1.present?, $2, $3.present?, $4
|
@@ -122,8 +125,8 @@ module Switchman
|
|
122
125
|
|
123
126
|
range = []
|
124
127
|
range << "id>=#{start}" if start
|
125
|
-
range << "id<#{
|
126
|
-
(negative ? negative_ranges : ranges) << "(#{range.join(
|
128
|
+
range << "id<#{"=" unless open}#{finish}" if finish
|
129
|
+
(negative ? negative_ranges : ranges) << "(#{range.join(" AND ")})"
|
127
130
|
when /^-(\d+)$/
|
128
131
|
negative_shard_ids << $1.to_i
|
129
132
|
when /^\d+$/
|
@@ -151,21 +154,21 @@ module Switchman
|
|
151
154
|
select = []
|
152
155
|
if index != 1
|
153
156
|
subscope = subscope.offset(per_chunk * (index - 1))
|
154
|
-
select <<
|
157
|
+
select << "MIN(id) AS min_id"
|
155
158
|
end
|
156
159
|
if index != denominator
|
157
160
|
subscope = subscope.limit(per_chunk)
|
158
|
-
select <<
|
161
|
+
select << "MAX(id) AS max_id"
|
159
162
|
end
|
160
163
|
|
161
|
-
result = Shard.from(subscope).select(select.join(
|
164
|
+
result = Shard.from(subscope).select(select.join(", ")).to_a.first
|
162
165
|
range = case index
|
163
166
|
when 1
|
164
|
-
"id<=#{result[
|
167
|
+
"id<=#{result["max_id"]}"
|
165
168
|
when denominator
|
166
|
-
"id>=#{result[
|
169
|
+
"id>=#{result["min_id"]}"
|
167
170
|
else
|
168
|
-
"(id>=#{result[
|
171
|
+
"(id>=#{result["min_id"]} AND id<=#{result["max_id"]})"
|
169
172
|
end
|
170
173
|
|
171
174
|
(numerator.negative? ? negative_ranges : ranges) << range
|
@@ -186,16 +189,16 @@ module Switchman
|
|
186
189
|
|
187
190
|
conditions = []
|
188
191
|
positive_queries = []
|
189
|
-
positive_queries << ranges.join(
|
192
|
+
positive_queries << ranges.join(" OR ") unless ranges.empty?
|
190
193
|
unless shard_ids.empty?
|
191
|
-
positive_queries <<
|
194
|
+
positive_queries << "id IN (?)"
|
192
195
|
conditions << shard_ids
|
193
196
|
end
|
194
|
-
positive_query = positive_queries.join(
|
197
|
+
positive_query = positive_queries.join(" OR ")
|
195
198
|
scope = scope.where(positive_query, *conditions) unless positive_queries.empty?
|
196
199
|
|
197
|
-
scope = scope.where("NOT (#{negative_ranges.join(
|
198
|
-
scope = scope.where(
|
200
|
+
scope = scope.where("NOT (#{negative_ranges.join(" OR")})") unless negative_ranges.empty?
|
201
|
+
scope = scope.where("id NOT IN (?)", negative_shard_ids) unless negative_shard_ids.empty?
|
199
202
|
scope
|
200
203
|
end
|
201
204
|
|
@@ -208,19 +211,19 @@ module Switchman
|
|
208
211
|
module PostgreSQLDatabaseTasks
|
209
212
|
def structure_dump(filename, extra_flags = nil)
|
210
213
|
set_psql_env
|
211
|
-
args = [
|
214
|
+
args = ["--schema-only", "--no-privileges", "--no-owner", "--file", filename]
|
212
215
|
args.concat(Array(extra_flags)) if extra_flags
|
213
216
|
shard = Shard.current.name
|
214
217
|
serialized_search_path = shard
|
215
218
|
args << "--schema=#{Shellwords.escape(shard)}"
|
216
219
|
|
217
220
|
ignore_tables = ::ActiveRecord::SchemaDumper.ignore_tables
|
218
|
-
args += ignore_tables.flat_map { |table| [
|
221
|
+
args += ignore_tables.flat_map { |table| ["-T", table] } if ignore_tables.any?
|
219
222
|
|
220
223
|
args << db_config.database
|
221
|
-
run_cmd(
|
224
|
+
run_cmd("pg_dump", args, "dumping")
|
222
225
|
remove_sql_header_comments(filename)
|
223
|
-
File.open(filename,
|
226
|
+
File.open(filename, "a") { |f| f << "SET search_path TO #{serialized_search_path};\n\n" }
|
224
227
|
end
|
225
228
|
end
|
226
229
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: switchman
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cody Cutrer
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2023-
|
13
|
+
date: 2023-05-02 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
@@ -86,14 +86,14 @@ dependencies:
|
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: 2.3
|
89
|
+
version: '2.3'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: 2.3
|
96
|
+
version: '2.3'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: byebug
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -170,14 +170,14 @@ dependencies:
|
|
170
170
|
requirements:
|
171
171
|
- - "~>"
|
172
172
|
- !ruby/object:Gem::Version
|
173
|
-
version: '
|
173
|
+
version: '6.0'
|
174
174
|
type: :development
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
|
-
version: '
|
180
|
+
version: '6.0'
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
182
|
name: rubocop
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -193,19 +193,19 @@ dependencies:
|
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: '1.10'
|
195
195
|
- !ruby/object:Gem::Dependency
|
196
|
-
name: rubocop-
|
196
|
+
name: rubocop-inst
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
198
198
|
requirements:
|
199
199
|
- - "~>"
|
200
200
|
- !ruby/object:Gem::Version
|
201
|
-
version: '1
|
201
|
+
version: '1'
|
202
202
|
type: :development
|
203
203
|
prerelease: false
|
204
204
|
version_requirements: !ruby/object:Gem::Requirement
|
205
205
|
requirements:
|
206
206
|
- - "~>"
|
207
207
|
- !ruby/object:Gem::Version
|
208
|
-
version: '1
|
208
|
+
version: '1'
|
209
209
|
- !ruby/object:Gem::Dependency
|
210
210
|
name: rubocop-rake
|
211
211
|
requirement: !ruby/object:Gem::Requirement
|