switchman 2.0.13 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +10 -2
- data/app/models/switchman/shard.rb +234 -271
- data/app/models/switchman/unsharded_record.rb +7 -0
- data/db/migrate/20130328212039_create_switchman_shards.rb +1 -1
- data/db/migrate/20130328224244_create_default_shard.rb +5 -5
- data/db/migrate/20161206323434_add_back_default_string_limits_switchman.rb +1 -0
- data/db/migrate/20180828183945_add_default_shard_index.rb +2 -2
- data/db/migrate/20180828192111_add_timestamps_to_shards.rb +7 -5
- data/db/migrate/20190114212900_add_unique_name_indexes.rb +5 -3
- data/lib/switchman.rb +3 -5
- data/lib/switchman/action_controller/caching.rb +2 -2
- data/lib/switchman/active_record/abstract_adapter.rb +1 -0
- data/lib/switchman/active_record/association.rb +78 -89
- data/lib/switchman/active_record/attribute_methods.rb +58 -52
- data/lib/switchman/active_record/base.rb +58 -59
- data/lib/switchman/active_record/calculations.rb +74 -67
- data/lib/switchman/active_record/connection_pool.rb +14 -41
- data/lib/switchman/active_record/database_configurations.rb +34 -0
- data/lib/switchman/active_record/database_configurations/database_config.rb +13 -0
- data/lib/switchman/active_record/finder_methods.rb +11 -16
- data/lib/switchman/active_record/log_subscriber.rb +4 -8
- data/lib/switchman/active_record/migration.rb +6 -47
- data/lib/switchman/active_record/model_schema.rb +1 -1
- data/lib/switchman/active_record/persistence.rb +4 -6
- data/lib/switchman/active_record/postgresql_adapter.rb +124 -168
- data/lib/switchman/active_record/predicate_builder.rb +2 -2
- data/lib/switchman/active_record/query_cache.rb +18 -19
- data/lib/switchman/active_record/query_methods.rb +172 -197
- data/lib/switchman/active_record/reflection.rb +6 -10
- data/lib/switchman/active_record/relation.rb +30 -78
- data/lib/switchman/active_record/spawn_methods.rb +27 -29
- data/lib/switchman/active_record/statement_cache.rb +18 -35
- data/lib/switchman/active_record/tasks/database_tasks.rb +16 -0
- data/lib/switchman/active_support/cache.rb +3 -5
- data/lib/switchman/arel.rb +13 -8
- data/lib/switchman/database_server.rb +121 -142
- data/lib/switchman/default_shard.rb +52 -16
- data/lib/switchman/engine.rb +61 -58
- data/lib/switchman/environment.rb +4 -8
- data/lib/switchman/errors.rb +1 -0
- data/lib/switchman/guard_rail.rb +6 -19
- data/lib/switchman/guard_rail/relation.rb +5 -7
- data/lib/switchman/r_spec_helper.rb +29 -37
- data/lib/switchman/rails.rb +14 -12
- data/lib/switchman/schema_cache.rb +1 -9
- data/lib/switchman/sharded_instrumenter.rb +1 -1
- data/lib/switchman/standard_error.rb +15 -3
- data/lib/switchman/test_helper.rb +7 -11
- data/lib/switchman/version.rb +1 -1
- data/lib/tasks/switchman.rake +54 -69
- metadata +87 -45
- data/lib/switchman/active_record/batches.rb +0 -11
- data/lib/switchman/active_record/connection_handler.rb +0 -172
- data/lib/switchman/active_record/where_clause_factory.rb +0 -36
- data/lib/switchman/connection_pool_proxy.rb +0 -173
data/lib/switchman/arel.rb
CHANGED
@@ -3,29 +3,34 @@
|
|
3
3
|
module Switchman
|
4
4
|
module Arel
|
5
5
|
module Table
|
6
|
-
def
|
7
|
-
|
6
|
+
def klass
|
7
|
+
@klass || ::ActiveRecord::Base
|
8
8
|
end
|
9
9
|
end
|
10
|
+
|
10
11
|
module Visitors
|
11
12
|
module ToSql
|
12
|
-
|
13
|
+
# rubocop:disable Naming/MethodName
|
14
|
+
|
15
|
+
def visit_Arel_Nodes_TableAlias(*args)
|
13
16
|
o, collector = args
|
14
17
|
collector = visit o.relation, collector
|
15
|
-
collector <<
|
18
|
+
collector << ' '
|
16
19
|
collector << quote_local_table_name(o.name)
|
17
20
|
end
|
18
21
|
|
19
|
-
def visit_Arel_Attributes_Attribute
|
22
|
+
def visit_Arel_Attributes_Attribute(*args)
|
20
23
|
o = args.first
|
21
24
|
join_name = o.relation.table_alias || o.relation.name
|
22
25
|
result = "#{quote_local_table_name join_name}.#{quote_column_name o.name}"
|
23
|
-
|
24
|
-
result
|
26
|
+
args.last << result
|
25
27
|
end
|
26
28
|
|
27
|
-
|
29
|
+
# rubocop:enable Naming/MethodName
|
30
|
+
|
31
|
+
def quote_local_table_name(name)
|
28
32
|
return name if ::Arel::Nodes::SqlLiteral === name
|
33
|
+
|
29
34
|
@connection.quote_local_table_name(name)
|
30
35
|
end
|
31
36
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'securerandom'
|
4
4
|
|
5
5
|
module Switchman
|
6
6
|
class DatabaseServer
|
@@ -13,16 +13,22 @@ module Switchman
|
|
13
13
|
database_servers.values
|
14
14
|
end
|
15
15
|
|
16
|
+
def all_roles
|
17
|
+
@all_roles ||= all.map(&:roles).flatten.uniq
|
18
|
+
end
|
19
|
+
|
16
20
|
def find(id_or_all)
|
17
|
-
return
|
18
|
-
return id_or_all.map { |id|
|
21
|
+
return all if id_or_all == :all
|
22
|
+
return id_or_all.map { |id| database_servers[id || ::Rails.env] }.compact.uniq if id_or_all.is_a?(Array)
|
23
|
+
|
19
24
|
database_servers[id_or_all || ::Rails.env]
|
20
25
|
end
|
21
26
|
|
22
27
|
def create(settings = {})
|
23
|
-
raise
|
28
|
+
raise 'database servers should be set up in database.yml' unless ::Rails.env.test?
|
29
|
+
|
24
30
|
id = settings[:id]
|
25
|
-
|
31
|
+
unless id
|
26
32
|
@id ||= 0
|
27
33
|
@id += 1
|
28
34
|
id = @id
|
@@ -30,25 +36,42 @@ module Switchman
|
|
30
36
|
server = DatabaseServer.new(id.to_s, settings)
|
31
37
|
server.instance_variable_set(:@fake, true)
|
32
38
|
database_servers[server.id] = server
|
39
|
+
::ActiveRecord::Base.configurations.configurations <<
|
40
|
+
::ActiveRecord::DatabaseConfigurations::HashConfig.new(::Rails.env, "#{server.id}/primary", settings)
|
41
|
+
Shard.send(:initialize_sharding)
|
42
|
+
server
|
33
43
|
end
|
34
44
|
|
35
45
|
def server_for_new_shard
|
36
46
|
servers = all.select { |s| s.config[:open] }
|
37
47
|
return find(nil) if servers.empty?
|
48
|
+
|
38
49
|
servers[rand(servers.length)]
|
39
50
|
end
|
40
51
|
|
41
52
|
private
|
53
|
+
|
54
|
+
def reference_role(role)
|
55
|
+
return if all_roles.include?(role)
|
56
|
+
|
57
|
+
@all_roles << role
|
58
|
+
Shard.send(:initialize_sharding)
|
59
|
+
end
|
60
|
+
|
42
61
|
def database_servers
|
43
62
|
unless @database_servers
|
44
63
|
@database_servers = {}.with_indifferent_access
|
45
|
-
|
46
|
-
|
47
|
-
|
64
|
+
::ActiveRecord::Base.configurations.configurations.each do |config|
|
65
|
+
if config.name.include?('/')
|
66
|
+
name, role = config.name.split('/')
|
67
|
+
else
|
68
|
+
name, role = config.env_name, config.name
|
48
69
|
end
|
49
|
-
|
50
|
-
|
51
|
-
@database_servers[
|
70
|
+
|
71
|
+
if role == 'primary'
|
72
|
+
@database_servers[name] = DatabaseServer.new(config.env_name, config.configuration_hash)
|
73
|
+
else
|
74
|
+
@database_servers[name].roles << role
|
52
75
|
end
|
53
76
|
end
|
54
77
|
end
|
@@ -56,15 +79,33 @@ module Switchman
|
|
56
79
|
end
|
57
80
|
end
|
58
81
|
|
82
|
+
attr_reader :roles
|
83
|
+
|
59
84
|
def initialize(id = nil, config = {})
|
60
85
|
@id = id
|
61
86
|
@config = config.deep_symbolize_keys
|
62
87
|
@configs = {}
|
88
|
+
@roles = [:primary]
|
89
|
+
end
|
90
|
+
|
91
|
+
def connects_to_hash
|
92
|
+
self.class.all_roles.map do |role|
|
93
|
+
config_role = role
|
94
|
+
config_role = :primary unless roles.include?(role)
|
95
|
+
config_name = :"#{id}/#{config_role}"
|
96
|
+
config_name = :primary if id == ::Rails.env && config_role == :primary
|
97
|
+
[role.to_sym, config_name]
|
98
|
+
end.to_h
|
63
99
|
end
|
64
100
|
|
65
101
|
def destroy
|
66
|
-
|
67
|
-
|
102
|
+
self.class.send(:database_servers).delete(id) if id
|
103
|
+
Shard.sharded_models.each do |klass|
|
104
|
+
self.class.all_roles.each do |role|
|
105
|
+
klass.connection_handler.remove_connection_pool(klass.connection_specification_name, role: role,
|
106
|
+
shard: id.to_sym)
|
107
|
+
end
|
108
|
+
end
|
68
109
|
end
|
69
110
|
|
70
111
|
def fake?
|
@@ -73,14 +114,15 @@ module Switchman
|
|
73
114
|
|
74
115
|
def config(environment = :primary)
|
75
116
|
@configs[environment] ||= begin
|
76
|
-
|
117
|
+
case @config[environment]
|
118
|
+
when Array
|
77
119
|
@config[environment].map do |config|
|
78
120
|
config = @config.merge((config || {}).symbolize_keys)
|
79
121
|
# make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
|
80
122
|
config.delete(environment)
|
81
123
|
config
|
82
124
|
end
|
83
|
-
|
125
|
+
when Hash
|
84
126
|
@config.merge(@config[environment])
|
85
127
|
else
|
86
128
|
@config
|
@@ -111,172 +153,109 @@ module Switchman
|
|
111
153
|
guard!(old_env)
|
112
154
|
end
|
113
155
|
|
114
|
-
def shareable?
|
115
|
-
@shareable_environment_key ||= []
|
116
|
-
environment = guard_rail_environment
|
117
|
-
explicit_user = ::GuardRail.global_config[:username]
|
118
|
-
return @shareable if @shareable_environment_key == [environment, explicit_user]
|
119
|
-
@shareable_environment_key = [environment, explicit_user]
|
120
|
-
if explicit_user
|
121
|
-
username = explicit_user
|
122
|
-
else
|
123
|
-
config = self.config(environment)
|
124
|
-
config = config.first if config.is_a?(Array)
|
125
|
-
username = config[:username]
|
126
|
-
end
|
127
|
-
@shareable = username !~ /%?\{[a-zA-Z0-9_]+\}/
|
128
|
-
end
|
129
|
-
|
130
156
|
def shards
|
131
|
-
if
|
132
|
-
Shard.where(
|
157
|
+
if id == ::Rails.env
|
158
|
+
Shard.where('database_server_id IS NULL OR database_server_id=?', id)
|
133
159
|
else
|
134
|
-
Shard.where(:
|
160
|
+
Shard.where(database_server_id: id)
|
135
161
|
end
|
136
162
|
end
|
137
163
|
|
138
|
-
def
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
def create_new_shard(options = {})
|
143
|
-
raise NotImplementedError.new("Cannot create new shards when sharding isn't initialized") unless Shard.default.is_a?(Shard)
|
144
|
-
|
145
|
-
name = options[:name]
|
146
|
-
create_schema = options[:schema]
|
147
|
-
# look for another shard associated with this db
|
148
|
-
other_shard = self.shards.where("name<>':memory:' OR name IS NULL").order(:id).first
|
149
|
-
|
150
|
-
case config[:adapter]
|
151
|
-
when 'postgresql'
|
152
|
-
create_statement = lambda { "CREATE SCHEMA #{name}" }
|
153
|
-
password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
|
154
|
-
else
|
155
|
-
create_statement = lambda { "CREATE DATABASE #{name}" }
|
164
|
+
def create_new_shard(id: nil, name: nil, schema: true)
|
165
|
+
unless Shard.default.is_a?(Shard)
|
166
|
+
raise NotImplementedError,
|
167
|
+
"Cannot create new shards when sharding isn't initialized"
|
156
168
|
end
|
169
|
+
|
170
|
+
create_statement = -> { "CREATE SCHEMA #{name}" }
|
171
|
+
password = " PASSWORD #{::ActiveRecord::Base.connection.quote(config[:password])}" if config[:password]
|
157
172
|
sharding_config = Switchman.config
|
158
173
|
config_create_statement = sharding_config[config[:adapter]]&.[](:create_statement)
|
159
174
|
config_create_statement ||= sharding_config[:create_statement]
|
160
175
|
if config_create_statement
|
161
176
|
create_commands = Array(config_create_statement).dup
|
162
177
|
create_statement = lambda {
|
163
|
-
create_commands.map { |statement| statement
|
178
|
+
create_commands.map { |statement| format(statement, name: name, password: password) }
|
164
179
|
}
|
165
180
|
end
|
166
181
|
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
173
|
-
next_id.to_i
|
174
|
-
else
|
175
|
-
nil
|
176
|
-
end
|
177
|
-
end
|
182
|
+
id ||= begin
|
183
|
+
id_seq = Shard.connection.quote(Shard.connection.quote_table_name('switchman_shards_id_seq'))
|
184
|
+
next_id = Shard.connection.select_value("SELECT nextval(#{id_seq})")
|
185
|
+
next_id.to_i
|
186
|
+
end
|
178
187
|
|
179
|
-
|
180
|
-
base_name = self.config[:database].to_s % self.config
|
181
|
-
base_name = nil if base_name == ':memory:'
|
182
|
-
base_name << '_' if base_name
|
183
|
-
base_id = shard_id || SecureRandom.uuid
|
184
|
-
name = "#{base_name}shard_#{base_id}"
|
185
|
-
end
|
188
|
+
name ||= "#{config[:database]}_shard_#{id}"
|
186
189
|
|
187
|
-
|
188
|
-
|
189
|
-
:
|
190
|
+
Shard.connection.transaction do
|
191
|
+
shard = Shard.create!(id: id,
|
192
|
+
name: name,
|
193
|
+
database_server_id: self.id)
|
190
194
|
schema_already_existed = false
|
191
195
|
|
192
196
|
begin
|
193
197
|
self.class.creating_new_shard = true
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
migrate = ::Rails.version >= '5.2' ?
|
220
|
-
-> { ::ActiveRecord::Base.connection.migration_context.migrate } :
|
221
|
-
-> { ::ActiveRecord::Migrator.migrate(::ActiveRecord::Migrator.migrations_paths) }
|
222
|
-
if ::ActiveRecord::Base.connection.supports_ddl_transactions?
|
223
|
-
::ActiveRecord::Base.connection.transaction(requires_new: true, &migrate)
|
224
|
-
else
|
225
|
-
migrate.call
|
226
|
-
end
|
227
|
-
reset_column_information
|
228
|
-
::ActiveRecord::Base.descendants.reject { |m| m == Shard || !m.table_exists? }.each(&:define_attribute_methods)
|
198
|
+
DatabaseServer.send(:reference_role, :deploy)
|
199
|
+
::ActiveRecord::Base.connected_to(shard: self.id.to_sym, role: :deploy) do
|
200
|
+
if create_statement
|
201
|
+
if ::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}")
|
202
|
+
schema_already_existed = true
|
203
|
+
raise 'This schema already exists; cannot overwrite'
|
204
|
+
end
|
205
|
+
Array(create_statement.call).each do |stmt|
|
206
|
+
::ActiveRecord::Base.connection.execute(stmt)
|
207
|
+
end
|
208
|
+
end
|
209
|
+
if config[:adapter] == 'postgresql'
|
210
|
+
old_proc = ::ActiveRecord::Base.connection.raw_connection.set_notice_processor do
|
211
|
+
end
|
212
|
+
end
|
213
|
+
old_verbose = ::ActiveRecord::Migration.verbose
|
214
|
+
::ActiveRecord::Migration.verbose = false
|
215
|
+
|
216
|
+
unless schema == false
|
217
|
+
shard.activate do
|
218
|
+
reset_column_information
|
219
|
+
|
220
|
+
::ActiveRecord::Base.connection.transaction(requires_new: true) do
|
221
|
+
::ActiveRecord::Base.connection.migration_context.migrate
|
229
222
|
end
|
230
|
-
|
231
|
-
::ActiveRecord::
|
232
|
-
|
223
|
+
reset_column_information
|
224
|
+
::ActiveRecord::Base.descendants.reject do |m|
|
225
|
+
m <= UnshardedRecord || !m.table_exists?
|
226
|
+
end.each(&:define_attribute_methods)
|
233
227
|
end
|
234
228
|
end
|
229
|
+
ensure
|
230
|
+
::ActiveRecord::Migration.verbose = old_verbose
|
231
|
+
::ActiveRecord::Base.connection.raw_connection.set_notice_processor(&old_proc) if old_proc
|
235
232
|
end
|
236
233
|
shard
|
237
234
|
rescue
|
238
235
|
shard.destroy
|
239
|
-
unless schema_already_existed
|
240
|
-
|
241
|
-
end
|
242
|
-
reset_column_information unless create_schema == false rescue nil
|
236
|
+
shard.drop_database rescue nil unless schema_already_existed
|
237
|
+
reset_column_information unless schema == false rescue nil
|
243
238
|
raise
|
244
239
|
ensure
|
245
240
|
self.class.creating_new_shard = false
|
246
241
|
end
|
247
242
|
end
|
248
|
-
|
249
|
-
if Shard.connection.supports_ddl_transactions? && self.shareable? && other_shard
|
250
|
-
Shard.transaction do
|
251
|
-
other_shard.activate do
|
252
|
-
::ActiveRecord::Base.connection.transaction(&create_shard)
|
253
|
-
end
|
254
|
-
end
|
255
|
-
else
|
256
|
-
create_shard.call
|
257
|
-
end
|
258
243
|
end
|
259
244
|
|
260
245
|
def cache_store
|
261
|
-
|
262
|
-
@cache_store = Switchman.config[:cache_map][self.id] || Switchman.config[:cache_map][::Rails.env]
|
263
|
-
end
|
246
|
+
@cache_store ||= Switchman.config[:cache_map][id] || Switchman.config[:cache_map][::Rails.env]
|
264
247
|
@cache_store
|
265
248
|
end
|
266
249
|
|
267
250
|
def shard_name(shard)
|
268
|
-
if config[:shard_name]
|
269
|
-
|
270
|
-
|
271
|
-
if
|
272
|
-
|
273
|
-
|
274
|
-
::ActiveRecord::Base.connection.current_schemas.first rescue nil
|
275
|
-
else
|
276
|
-
shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
|
277
|
-
end
|
251
|
+
return config[:shard_name] if config[:shard_name]
|
252
|
+
|
253
|
+
if shard == :bootstrap
|
254
|
+
# rescue nil because the database may not exist yet; if it doesn't,
|
255
|
+
# it will shortly, and this will be re-invoked
|
256
|
+
::ActiveRecord::Base.connection.current_schemas.first rescue nil
|
278
257
|
else
|
279
|
-
|
258
|
+
shard.activate { ::ActiveRecord::Base.connection_pool.default_schema }
|
280
259
|
end
|
281
260
|
end
|
282
261
|
|
@@ -290,9 +269,9 @@ module Switchman
|
|
290
269
|
end
|
291
270
|
|
292
271
|
private
|
272
|
+
|
293
273
|
def reset_column_information
|
294
|
-
::ActiveRecord::Base.descendants.reject { |m| m
|
295
|
-
::ActiveRecord::Base.connection_handler.switchman_connection_pool_proxies.each { |pool| pool.schema_cache.clear! }
|
274
|
+
::ActiveRecord::Base.descendants.reject { |m| m <= UnshardedRecord }.each(&:reset_column_information)
|
296
275
|
end
|
297
276
|
end
|
298
277
|
end
|
@@ -4,17 +4,44 @@ require 'switchman/database_server'
|
|
4
4
|
|
5
5
|
module Switchman
|
6
6
|
class DefaultShard
|
7
|
-
def id
|
7
|
+
def id
|
8
|
+
'default'
|
9
|
+
end
|
8
10
|
alias cache_key id
|
9
|
-
def activate(*
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
def
|
14
|
-
|
15
|
-
def
|
16
|
-
|
17
|
-
|
11
|
+
def activate(*_classes)
|
12
|
+
yield
|
13
|
+
end
|
14
|
+
|
15
|
+
def activate!(*classes); end
|
16
|
+
|
17
|
+
def default?
|
18
|
+
true
|
19
|
+
end
|
20
|
+
|
21
|
+
def primary?
|
22
|
+
true
|
23
|
+
end
|
24
|
+
|
25
|
+
def relative_id_for(local_id, _target = nil)
|
26
|
+
local_id
|
27
|
+
end
|
28
|
+
|
29
|
+
def global_id_for(local_id)
|
30
|
+
local_id
|
31
|
+
end
|
32
|
+
|
33
|
+
def database_server_id
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
|
37
|
+
def database_server
|
38
|
+
DatabaseServer.find(nil)
|
39
|
+
end
|
40
|
+
|
41
|
+
def new_record?
|
42
|
+
false
|
43
|
+
end
|
44
|
+
|
18
45
|
def name
|
19
46
|
unless instance_variable_defined?(:@name)
|
20
47
|
@name = nil # prevent taking this branch on recursion
|
@@ -22,18 +49,27 @@ module Switchman
|
|
22
49
|
end
|
23
50
|
@name
|
24
51
|
end
|
25
|
-
|
52
|
+
|
53
|
+
def description
|
54
|
+
::Rails.env
|
55
|
+
end
|
56
|
+
|
26
57
|
# The default's shard is always the default shard
|
27
|
-
def shard
|
28
|
-
|
58
|
+
def shard
|
59
|
+
self
|
60
|
+
end
|
61
|
+
|
62
|
+
def _dump(_depth)
|
29
63
|
''
|
30
64
|
end
|
31
|
-
|
65
|
+
|
66
|
+
def self._load(_str)
|
32
67
|
Shard.default
|
33
68
|
end
|
34
69
|
|
35
|
-
def ==(
|
36
|
-
return true if
|
70
|
+
def ==(other)
|
71
|
+
return true if other.is_a?(DefaultShard) || (other.is_a?(Shard) && other[:default])
|
72
|
+
|
37
73
|
super
|
38
74
|
end
|
39
75
|
|