pgtk 0.32.4 → 0.32.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +17 -16
- data/lib/pgtk/impatient/too_slow.rb +18 -0
- data/lib/pgtk/impatient.rb +23 -14
- data/lib/pgtk/liquibase_task.rb +3 -2
- data/lib/pgtk/liquicheck_task/must_error.rb +17 -0
- data/lib/pgtk/liquicheck_task.rb +2 -4
- data/lib/pgtk/pgsql_task.rb +41 -28
- data/lib/pgtk/pool/busy.rb +17 -0
- data/lib/pgtk/pool/iterable_queue.rb +83 -0
- data/lib/pgtk/pool/txn.rb +62 -0
- data/lib/pgtk/pool.rb +43 -152
- data/lib/pgtk/retry/exhausted.rb +17 -0
- data/lib/pgtk/retry.rb +2 -5
- data/lib/pgtk/spy.rb +4 -4
- data/lib/pgtk/stash.rb +68 -53
- data/lib/pgtk/version.rb +1 -1
- data/lib/pgtk/wire/direct.rb +41 -0
- data/lib/pgtk/wire/env.rb +48 -0
- data/lib/pgtk/wire/yaml.rb +47 -0
- data/lib/pgtk/wire.rb +4 -103
- data/pgtk.gemspec +1 -1
- data/resources/pom.xml +1 -1
- metadata +13 -4
data/lib/pgtk/pool.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'loog'
|
|
4
3
|
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
5
4
|
# SPDX-License-Identifier: MIT
|
|
6
5
|
|
|
7
6
|
require 'ellipsized'
|
|
7
|
+
require 'loog'
|
|
8
8
|
require 'pg'
|
|
9
9
|
require 'tago'
|
|
10
10
|
require_relative '../pgtk'
|
|
@@ -50,11 +50,6 @@ require_relative 'wire'
|
|
|
50
50
|
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
51
51
|
# License:: MIT
|
|
52
52
|
class Pgtk::Pool
|
|
53
|
-
# Raised when no connection becomes available from the pool within
|
|
54
|
-
# the configured timeout. Indicates that all connections are currently
|
|
55
|
-
# taken by other threads and none was returned in time.
|
|
56
|
-
class Busy < StandardError; end
|
|
57
|
-
|
|
58
53
|
# Constructor.
|
|
59
54
|
#
|
|
60
55
|
# The +idle+ option guards against the cold-slot SSL desync that bites
|
|
@@ -78,6 +73,7 @@ class Pgtk::Pool
|
|
|
78
73
|
@idle = idle
|
|
79
74
|
@log = log
|
|
80
75
|
@pool = IterableQueue.new(max, timeout)
|
|
76
|
+
@lock = Mutex.new
|
|
81
77
|
@started = false
|
|
82
78
|
end
|
|
83
79
|
|
|
@@ -85,7 +81,12 @@ class Pgtk::Pool
|
|
|
85
81
|
#
|
|
86
82
|
# @return [String] Version of PostgreSQL server
|
|
87
83
|
def version
|
|
88
|
-
@version ||=
|
|
84
|
+
@version ||=
|
|
85
|
+
begin
|
|
86
|
+
conn = @pool.pop
|
|
87
|
+
@pool.push(conn)
|
|
88
|
+
conn.parameter_status('server_version').split[0]
|
|
89
|
+
end
|
|
89
90
|
end
|
|
90
91
|
|
|
91
92
|
# Get as much details about it as possible.
|
|
@@ -108,20 +109,22 @@ class Pgtk::Pool
|
|
|
108
109
|
# open at the same time. For example, Heroku free PostgreSQL database
|
|
109
110
|
# allows only one connection open.
|
|
110
111
|
def start!
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
@
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
112
|
+
@lock.synchronize do
|
|
113
|
+
return if @started
|
|
114
|
+
@max.times do
|
|
115
|
+
@pool.push(@wire.connection)
|
|
116
|
+
end
|
|
117
|
+
(2 * @max).times do
|
|
118
|
+
connect { |c| c.exec('SELECT 1') }
|
|
119
|
+
rescue StandardError => e
|
|
120
|
+
@log.warn("Pool warm-up query failed, slot will be retried: #{e.message.strip}")
|
|
121
|
+
end
|
|
122
|
+
@max.times do
|
|
123
|
+
connect { |c| c.exec('SELECT 1') }
|
|
124
|
+
end
|
|
125
|
+
@started = true
|
|
126
|
+
@log.debug("PostgreSQL pool started with #{@max} connections")
|
|
122
127
|
end
|
|
123
|
-
@started = true
|
|
124
|
-
@log.debug("PostgreSQL pool started with #{@max} connections")
|
|
125
128
|
end
|
|
126
129
|
|
|
127
130
|
# Make a query and return the result as an array of hashes. For example,
|
|
@@ -198,9 +201,7 @@ class Pgtk::Pool
|
|
|
198
201
|
t = Txn.new(c, @log)
|
|
199
202
|
t.exec('START TRANSACTION')
|
|
200
203
|
begin
|
|
201
|
-
|
|
202
|
-
t.exec('COMMIT')
|
|
203
|
-
r
|
|
204
|
+
yield(t).tap { t.exec('COMMIT') }
|
|
204
205
|
ensure
|
|
205
206
|
if c.transaction_status != PG::Constants::PQTRANS_IDLE
|
|
206
207
|
begin
|
|
@@ -213,124 +214,6 @@ class Pgtk::Pool
|
|
|
213
214
|
end
|
|
214
215
|
end
|
|
215
216
|
|
|
216
|
-
# Thread-safe queue implementation that supports iteration.
|
|
217
|
-
# Unlike Ruby's Queue class, this implementation allows safe iteration
|
|
218
|
-
# over all elements while maintaining thread safety for concurrent access.
|
|
219
|
-
#
|
|
220
|
-
# This class is used internally by Pool to store database connections
|
|
221
|
-
# and provide the ability to iterate over them for inspection purposes.
|
|
222
|
-
#
|
|
223
|
-
# The queue is bounded by size. When an item is taken out, it remains in
|
|
224
|
-
# the internal array but is marked as "taken". When returned, it's placed
|
|
225
|
-
# back in its original slot and marked as available.
|
|
226
|
-
class IterableQueue
|
|
227
|
-
def initialize(size, timeout)
|
|
228
|
-
@size = size
|
|
229
|
-
@timeout = timeout
|
|
230
|
-
@items = []
|
|
231
|
-
@taken = []
|
|
232
|
-
@free = []
|
|
233
|
-
@mutex = Mutex.new
|
|
234
|
-
@condition = ConditionVariable.new
|
|
235
|
-
end
|
|
236
|
-
|
|
237
|
-
def push(item)
|
|
238
|
-
@mutex.synchronize do
|
|
239
|
-
if @items.size < @size
|
|
240
|
-
@items << item
|
|
241
|
-
@taken << false
|
|
242
|
-
@free << (@items.size - 1)
|
|
243
|
-
else
|
|
244
|
-
index = @items.index(item)
|
|
245
|
-
if index.nil?
|
|
246
|
-
index = @taken.index(true)
|
|
247
|
-
raise(StandardError, 'No taken slot found') if index.nil?
|
|
248
|
-
@items[index] = item
|
|
249
|
-
end
|
|
250
|
-
@taken[index] = false
|
|
251
|
-
@free << index
|
|
252
|
-
end
|
|
253
|
-
@condition.signal
|
|
254
|
-
end
|
|
255
|
-
end
|
|
256
|
-
|
|
257
|
-
def pop
|
|
258
|
-
@mutex.synchronize do
|
|
259
|
-
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
|
|
260
|
-
while @free.empty?
|
|
261
|
-
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
262
|
-
raise(Busy, "No free connection appeared in the pool after #{@timeout}s of waiting") if remaining <= 0
|
|
263
|
-
@condition.wait(@mutex, remaining)
|
|
264
|
-
end
|
|
265
|
-
index = @free.shift
|
|
266
|
-
@taken[index] = true
|
|
267
|
-
@items[index]
|
|
268
|
-
end
|
|
269
|
-
end
|
|
270
|
-
|
|
271
|
-
def size
|
|
272
|
-
@mutex.synchronize do
|
|
273
|
-
@items.size
|
|
274
|
-
end
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
def map(&)
|
|
278
|
-
@mutex.synchronize do
|
|
279
|
-
@items.map(&)
|
|
280
|
-
end
|
|
281
|
-
end
|
|
282
|
-
end
|
|
283
|
-
|
|
284
|
-
# A temporary class to execute a single SQL request.
|
|
285
|
-
class Txn
|
|
286
|
-
def initialize(conn, log)
|
|
287
|
-
@conn = conn
|
|
288
|
-
@log = log
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
# Exec a single parameterized command.
|
|
292
|
-
# @param [String] query The SQL query with params inside (possibly)
|
|
293
|
-
# @param [Array] args List of arguments
|
|
294
|
-
# @param [Integer] result Should be 0 for text results, 1 for binary
|
|
295
|
-
# @yield [Hash] Rows
|
|
296
|
-
def exec(query, args = [], result = 0)
|
|
297
|
-
start = Time.now
|
|
298
|
-
sql = query.is_a?(Array) ? query.join(' ') : query
|
|
299
|
-
@conn.instance_variable_set(:@pgtk_last_query, sql)
|
|
300
|
-
@conn.instance_variable_set(:@pgtk_started_at, start)
|
|
301
|
-
begin
|
|
302
|
-
out =
|
|
303
|
-
if args.empty?
|
|
304
|
-
@conn.exec(sql) do |res|
|
|
305
|
-
if block_given?
|
|
306
|
-
yield(res)
|
|
307
|
-
else
|
|
308
|
-
res.each.to_a
|
|
309
|
-
end
|
|
310
|
-
end
|
|
311
|
-
else
|
|
312
|
-
@conn.exec_params(sql, args, result) do |res|
|
|
313
|
-
if block_given?
|
|
314
|
-
yield(res)
|
|
315
|
-
else
|
|
316
|
-
res.each.to_a
|
|
317
|
-
end
|
|
318
|
-
end
|
|
319
|
-
end
|
|
320
|
-
rescue StandardError => e
|
|
321
|
-
@log.error("#{sql} -> #{e.message}")
|
|
322
|
-
raise(e)
|
|
323
|
-
end
|
|
324
|
-
lag = Time.now - start
|
|
325
|
-
if lag < 1
|
|
326
|
-
@log.debug("#{sql} >> #{start.ago} / #{@conn.object_id}")
|
|
327
|
-
else
|
|
328
|
-
@log.info("#{sql} >> #{start.ago}")
|
|
329
|
-
end
|
|
330
|
-
out
|
|
331
|
-
end
|
|
332
|
-
end
|
|
333
|
-
|
|
334
217
|
private
|
|
335
218
|
|
|
336
219
|
def connect
|
|
@@ -371,9 +254,9 @@ class Pgtk::Pool
|
|
|
371
254
|
end
|
|
372
255
|
|
|
373
256
|
def stale(conn)
|
|
374
|
-
return
|
|
257
|
+
return if @idle.nil?
|
|
375
258
|
last = conn.instance_variable_get(:@pgtk_last_used)
|
|
376
|
-
return
|
|
259
|
+
return if last.nil? || Time.now - last < @idle
|
|
377
260
|
begin
|
|
378
261
|
conn.exec('SELECT 1')
|
|
379
262
|
nil
|
|
@@ -383,19 +266,23 @@ class Pgtk::Pool
|
|
|
383
266
|
end
|
|
384
267
|
|
|
385
268
|
def info(conn)
|
|
386
|
-
pipelines = { PG::Constants::PQ_PIPELINE_ON => 'ON', PG::Constants::PQ_PIPELINE_OFF => 'OFF',
|
|
387
|
-
PG::Constants::PQ_PIPELINE_ABORTED => 'ABORTED' }
|
|
388
|
-
statuses = { PG::Constants::CONNECTION_OK => 'OK', PG::Constants::CONNECTION_BAD => 'BAD' }
|
|
389
|
-
transactions = { PG::Constants::PQTRANS_IDLE => 'IDLE', PG::Constants::PQTRANS_ACTIVE => 'ACTIVE',
|
|
390
|
-
PG::Constants::PQTRANS_INTRANS => 'INTRANS', PG::Constants::PQTRANS_INERROR => 'INERROR',
|
|
391
|
-
PG::Constants::PQTRANS_UNKNOWN => 'UNKNOWN' }
|
|
392
269
|
conn.instance_variable_set(:@pgtk_pid, conn.backend_pid)
|
|
393
270
|
parts = [
|
|
394
271
|
' ',
|
|
395
272
|
"##{conn.backend_pid}",
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
273
|
+
{
|
|
274
|
+
PG::Constants::PQ_PIPELINE_ON => 'ON', PG::Constants::PQ_PIPELINE_OFF => 'OFF',
|
|
275
|
+
PG::Constants::PQ_PIPELINE_ABORTED => 'ABORTED'
|
|
276
|
+
}.fetch(conn.pipeline_status, "pipeline_status=#{conn.pipeline_status}"),
|
|
277
|
+
{ PG::Constants::CONNECTION_OK => 'OK', PG::Constants::CONNECTION_BAD => 'BAD' }.fetch(
|
|
278
|
+
conn.status,
|
|
279
|
+
"status=#{conn.status}"
|
|
280
|
+
),
|
|
281
|
+
{
|
|
282
|
+
PG::Constants::PQTRANS_IDLE => 'IDLE', PG::Constants::PQTRANS_ACTIVE => 'ACTIVE',
|
|
283
|
+
PG::Constants::PQTRANS_INTRANS => 'INTRANS', PG::Constants::PQTRANS_INERROR => 'INERROR',
|
|
284
|
+
PG::Constants::PQTRANS_UNKNOWN => 'UNKNOWN'
|
|
285
|
+
}.fetch(conn.transaction_status, "transaction_status=#{conn.transaction_status}")
|
|
399
286
|
]
|
|
400
287
|
if conn.transaction_status != PG::Constants::PQTRANS_IDLE
|
|
401
288
|
started = conn.instance_variable_get(:@pgtk_started_at)
|
|
@@ -434,3 +321,7 @@ class Pgtk::Pool
|
|
|
434
321
|
@wire.connection
|
|
435
322
|
end
|
|
436
323
|
end
|
|
324
|
+
|
|
325
|
+
require_relative 'pool/busy'
|
|
326
|
+
require_relative 'pool/iterable_queue'
|
|
327
|
+
require_relative 'pool/txn'
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
4
|
+
# SPDX-License-Identifier: MIT
|
|
5
|
+
|
|
6
|
+
require_relative '../../pgtk'
|
|
7
|
+
|
|
8
|
+
class Pgtk::Retry; end
|
|
9
|
+
|
|
10
|
+
# Raised when all retry attempts have been exhausted. The original
|
|
11
|
+
# exception that caused the last failure is available via #cause,
|
|
12
|
+
# so its message and stack trace are preserved for debugging.
|
|
13
|
+
#
|
|
14
|
+
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
|
15
|
+
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
16
|
+
# License:: MIT
|
|
17
|
+
class Pgtk::Retry::Exhausted < StandardError; end
|
data/lib/pgtk/retry.rb
CHANGED
|
@@ -49,11 +49,6 @@ require_relative 'impatient'
|
|
|
49
49
|
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
50
50
|
# License:: MIT
|
|
51
51
|
class Pgtk::Retry
|
|
52
|
-
# Raised when all retry attempts have been exhausted. The original
|
|
53
|
-
# exception that caused the last failure is available via #cause,
|
|
54
|
-
# so its message and stack trace are preserved for debugging.
|
|
55
|
-
class Exhausted < StandardError; end
|
|
56
|
-
|
|
57
52
|
BACKOFFS = [0.05, 0.2, 1.0].freeze
|
|
58
53
|
|
|
59
54
|
# Constructor.
|
|
@@ -122,3 +117,5 @@ class Pgtk::Retry
|
|
|
122
117
|
@pool.transaction(&)
|
|
123
118
|
end
|
|
124
119
|
end
|
|
120
|
+
|
|
121
|
+
require_relative 'retry/exhausted'
|
data/lib/pgtk/spy.rb
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'loog'
|
|
4
3
|
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
5
4
|
# SPDX-License-Identifier: MIT
|
|
6
5
|
|
|
6
|
+
require 'loog'
|
|
7
7
|
require 'pg'
|
|
8
8
|
require_relative '../pgtk'
|
|
9
9
|
require_relative 'wire'
|
|
@@ -83,9 +83,9 @@ class Pgtk::Spy
|
|
|
83
83
|
# @return [Array] Result rows
|
|
84
84
|
def exec(sql, *)
|
|
85
85
|
start = Time.now
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
@pool.exec(sql, *).tap do
|
|
87
|
+
@block&.call(sql.is_a?(Array) ? sql.join(' ') : sql, Time.now - start)
|
|
88
|
+
end
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
# Run a transaction with spying on each SQL query.
|
data/lib/pgtk/stash.rb
CHANGED
|
@@ -29,8 +29,11 @@ require_relative '../pgtk'
|
|
|
29
29
|
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
30
30
|
# License:: MIT
|
|
31
31
|
class Pgtk::Stash
|
|
32
|
-
MODS = %w[
|
|
33
|
-
|
|
32
|
+
MODS = %w[
|
|
33
|
+
INSERT DELETE UPDATE LOCK VACUUM TRANSACTION COMMIT ROLLBACK
|
|
34
|
+
REINDEX TRUNCATE CREATE ALTER DROP SET START
|
|
35
|
+
].freeze
|
|
36
|
+
MODS_RE = Regexp.new("\\A(#{MODS.join('|')})(\\s|$)")
|
|
34
37
|
|
|
35
38
|
IDENT = '[a-z_][a-z0-9_]*'
|
|
36
39
|
|
|
@@ -39,9 +42,13 @@ class Pgtk::Stash
|
|
|
39
42
|
|
|
40
43
|
READS_RE = Regexp.new("(?<=^|\\s)(?:FROM|JOIN)\\s(#{IDENT})(?=\\s|;|$)")
|
|
41
44
|
|
|
42
|
-
|
|
45
|
+
NONDETERMINISTIC = /
|
|
46
|
+
\b(?:NOW|CURRENT_TIMESTAMP|CURRENT_DATE|CURRENT_TIME|
|
|
47
|
+
LOCALTIMESTAMP|LOCALTIME|RANDOM|GEN_RANDOM_UUID|
|
|
48
|
+
CLOCK_TIMESTAMP)(?:\s*\(|(?=[^a-z0-9_]|$))
|
|
49
|
+
/ix
|
|
43
50
|
|
|
44
|
-
private_constant :MODS, :ALTS, :IDENT, :MODS_RE, :ALTS_RE, :READS_RE, :
|
|
51
|
+
private_constant :MODS, :ALTS, :IDENT, :MODS_RE, :ALTS_RE, :READS_RE, :NONDETERMINISTIC
|
|
45
52
|
|
|
46
53
|
# Initialize a new Stash with query caching.
|
|
47
54
|
#
|
|
@@ -76,7 +83,9 @@ class Pgtk::Stash
|
|
|
76
83
|
@pool = pool
|
|
77
84
|
@stash = stash
|
|
78
85
|
@stash[:table_mod] ||= {}
|
|
79
|
-
@stash[:table_inflight]
|
|
86
|
+
unless @stash[:table_inflight].default_proc
|
|
87
|
+
@stash[:table_inflight] = Hash.new { |h, k| h[k] = Concurrent::AtomicFixnum.new(0) }
|
|
88
|
+
end
|
|
80
89
|
@loog = loog
|
|
81
90
|
@entrance = entrance
|
|
82
91
|
@refill = refill
|
|
@@ -109,8 +118,7 @@ class Pgtk::Stash
|
|
|
109
118
|
# @return [String] Multi-line text representation of the current cache state
|
|
110
119
|
def dump
|
|
111
120
|
@entrance.with_read_lock do
|
|
112
|
-
|
|
113
|
-
body(qq)
|
|
121
|
+
body(queries)
|
|
114
122
|
end
|
|
115
123
|
end
|
|
116
124
|
|
|
@@ -122,8 +130,10 @@ class Pgtk::Stash
|
|
|
122
130
|
# @return [PG::Result] Query result object
|
|
123
131
|
def exec(query, params = [], result = 0)
|
|
124
132
|
pure = (query.is_a?(Array) ? query.join(' ') : query).gsub(/\s+/, ' ').strip
|
|
125
|
-
if MODS_RE.match?(pure)
|
|
133
|
+
if MODS_RE.match?(pure)
|
|
126
134
|
modify(pure, params, result)
|
|
135
|
+
elsif /(^|\s)pg_[a-z_]+\(/.match?(pure)
|
|
136
|
+
@pool.exec(pure, params, result)
|
|
127
137
|
else
|
|
128
138
|
select(pure, params, result)
|
|
129
139
|
end
|
|
@@ -146,7 +156,7 @@ class Pgtk::Stash
|
|
|
146
156
|
{
|
|
147
157
|
q: q.dup,
|
|
148
158
|
c: kk.values.count,
|
|
149
|
-
p: kk.values.sum { |vv| vv[:popularity] },
|
|
159
|
+
p: kk.values.sum { |vv| vv[:popularity]&.value || 0 },
|
|
150
160
|
s: kk.values.count { |vv| vv[:stale] },
|
|
151
161
|
u: kk.values.map { |vv| vv[:used] }.max || Time.now
|
|
152
162
|
}
|
|
@@ -223,49 +233,42 @@ class Pgtk::Stash
|
|
|
223
233
|
tables = pure.scan(ALTS_RE).flatten
|
|
224
234
|
tables.uniq!
|
|
225
235
|
affected = (tables + tables.flat_map { |t| @cascades&.fetch(t, []) || [] }).uniq
|
|
226
|
-
@
|
|
227
|
-
affected.each { |t| @stash[:table_inflight][t] = (@stash[:table_inflight][t] || 0) + 1 }
|
|
228
|
-
end
|
|
236
|
+
affected.each { |t| @stash[:table_inflight][t].increment }
|
|
229
237
|
begin
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
@stash[:queries][q][key][:stale] = stamp
|
|
238
|
+
@pool.exec(pure, params, result).tap do
|
|
239
|
+
now = Time.now
|
|
240
|
+
@entrance.with_write_lock do
|
|
241
|
+
affected.each do |t|
|
|
242
|
+
@stash[:table_inflight][t].decrement
|
|
243
|
+
@stash[:table_mod][t] = (@stash[:table_mod][t] || 0) + 1
|
|
244
|
+
@stash[:tables][t]&.each do |q|
|
|
245
|
+
@stash[:queries][q]&.each_key do |key|
|
|
246
|
+
@stash[:queries][q][key][:stale] = now
|
|
247
|
+
end
|
|
241
248
|
end
|
|
242
249
|
end
|
|
243
250
|
end
|
|
244
251
|
end
|
|
245
|
-
ret
|
|
246
252
|
rescue StandardError
|
|
247
|
-
@
|
|
248
|
-
affected.each { |t| @stash[:table_inflight][t] -= 1 }
|
|
249
|
-
end
|
|
253
|
+
affected.each { |t| @stash[:table_inflight][t].decrement }
|
|
250
254
|
raise
|
|
251
255
|
end
|
|
252
256
|
end
|
|
253
257
|
|
|
254
258
|
def select(pure, params, result)
|
|
255
|
-
|
|
256
|
-
ret
|
|
257
|
-
if ret.nil? || @stash.dig(:queries, pure, key, :stale)
|
|
259
|
+
ret = @stash.dig(:queries, pure, params, :ret)
|
|
260
|
+
if ret.nil? || @stash.dig(:queries, pure, params, :stale)
|
|
258
261
|
tables = pure.scan(READS_RE).flatten
|
|
259
262
|
tables.uniq!
|
|
260
263
|
marks = tables.to_h { |t| [t, @stash[:table_mod][t]] }
|
|
261
264
|
ret = @pool.exec(pure, params, result)
|
|
262
|
-
cache(pure,
|
|
265
|
+
cache(pure, params, result, ret, tables, marks) unless pure.match?(NONDETERMINISTIC)
|
|
263
266
|
end
|
|
264
|
-
bump(pure,
|
|
267
|
+
bump(pure, params) if @stash.dig(:queries, pure, params)
|
|
265
268
|
ret
|
|
266
269
|
end
|
|
267
270
|
|
|
268
|
-
def cache(pure,
|
|
271
|
+
def cache(pure, params, result, ret, tables, marks)
|
|
269
272
|
raise(ArgumentError, "No tables at #{pure.inspect}") if tables.empty?
|
|
270
273
|
@entrance.with_write_lock do
|
|
271
274
|
tables.each do |t|
|
|
@@ -273,7 +276,7 @@ class Pgtk::Stash
|
|
|
273
276
|
@stash[:tables][t].append(pure).uniq!
|
|
274
277
|
end
|
|
275
278
|
@stash[:queries][pure] ||= {}
|
|
276
|
-
existing = @stash[:queries][pure][
|
|
279
|
+
existing = @stash[:queries][pure][params]
|
|
277
280
|
stillborn = tables.any? { |t| (cur = @stash[:table_mod][t]) && cur != marks[t] }
|
|
278
281
|
entry = { ret:, params:, result:, used: Time.now }
|
|
279
282
|
entry[:stale] =
|
|
@@ -283,31 +286,38 @@ class Pgtk::Stash
|
|
|
283
286
|
Time.now
|
|
284
287
|
end
|
|
285
288
|
entry.delete(:stale) if entry[:stale].nil?
|
|
286
|
-
|
|
289
|
+
entry[:popularity] = (existing && existing[:popularity]) || Concurrent::AtomicFixnum.new
|
|
290
|
+
@stash[:queries][pure][params] = entry
|
|
287
291
|
end
|
|
288
292
|
end
|
|
289
293
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
294
|
+
# Bump popularity and last-used time of a cached entry.
|
|
295
|
+
#
|
|
296
|
+
# This runs without the exclusive write lock: popularity is a
|
|
297
|
+
# +Concurrent::AtomicFixnum+ and +used+ is a plain assignment of an
|
|
298
|
+
# immutable value, neither of which mutates the structure of the
|
|
299
|
+
# +@stash[:queries]+ tree. Keeping it off the writer lock means a
|
|
300
|
+
# read-only workload never serializes through the writer (see #414).
|
|
301
|
+
#
|
|
302
|
+
# @return [void]
|
|
303
|
+
def bump(pure, params)
|
|
304
|
+
entry = @stash.dig(:queries, pure, params)
|
|
305
|
+
return unless entry
|
|
306
|
+
(entry[:popularity] ||= Concurrent::AtomicFixnum.new).increment
|
|
307
|
+
entry[:used] = Time.now
|
|
296
308
|
end
|
|
297
309
|
|
|
298
310
|
# Calculate total number of cached query results.
|
|
299
311
|
#
|
|
300
312
|
# @return [Integer] Total count of cached query results
|
|
301
313
|
def cached
|
|
302
|
-
@entrance.
|
|
314
|
+
@entrance.with_read_lock do
|
|
303
315
|
@stash[:queries].values.sum { |kk| kk.values.size }
|
|
304
316
|
end
|
|
305
317
|
end
|
|
306
318
|
|
|
307
319
|
# Discover ON DELETE CASCADE / ON UPDATE CASCADE foreign keys so that a
|
|
308
320
|
# modify on the parent table also invalidates cached queries on children.
|
|
309
|
-
#
|
|
310
|
-
# @return [nil]
|
|
311
321
|
def cascade!
|
|
312
322
|
direct = Hash.new { |h, k| h[k] = [] }
|
|
313
323
|
@pool.exec(<<~SQL).each { |r| direct[r['parent']] << r['child'] }
|
|
@@ -324,7 +334,6 @@ class Pgtk::Stash
|
|
|
324
334
|
AND tc.table_schema NOT IN ('pg_catalog', 'information_schema')
|
|
325
335
|
SQL
|
|
326
336
|
@cascades = direct.keys.to_h { |p| [p, transitive(p, direct, []).uniq] }
|
|
327
|
-
nil
|
|
328
337
|
end
|
|
329
338
|
|
|
330
339
|
def transitive(parent, direct, seen)
|
|
@@ -356,6 +365,8 @@ class Pgtk::Stash
|
|
|
356
365
|
end
|
|
357
366
|
end
|
|
358
367
|
end
|
|
368
|
+
rescue StandardError => e
|
|
369
|
+
@loog.warn("Stash capper crashed: #{e.class}: #{e.message}")
|
|
359
370
|
end
|
|
360
371
|
end
|
|
361
372
|
|
|
@@ -367,26 +378,30 @@ class Pgtk::Stash
|
|
|
367
378
|
evict(q) if @stash[:queries][q].empty?
|
|
368
379
|
end
|
|
369
380
|
end
|
|
381
|
+
rescue StandardError => e
|
|
382
|
+
@loog.warn("Stash retiree crashed: #{e.class}: #{e.message}")
|
|
370
383
|
end
|
|
371
384
|
end
|
|
372
385
|
|
|
373
|
-
def evict(query)
|
|
374
|
-
@stash[:queries].delete(query)
|
|
375
|
-
@stash[:tables].each_value { |list| list.delete(query) }
|
|
376
|
-
@stash[:tables].delete_if { |_, list| list.empty? }
|
|
377
|
-
end
|
|
378
|
-
|
|
379
386
|
def refiller!
|
|
380
387
|
Concurrent::TimerTask.execute(execution_interval: @refill, executor: @tpool) do
|
|
381
388
|
ranked.each { |q| replenish(q) }
|
|
389
|
+
rescue StandardError => e
|
|
390
|
+
@loog.warn("Stash refiller crashed: #{e.class}: #{e.message}")
|
|
382
391
|
end
|
|
383
392
|
end
|
|
384
393
|
|
|
394
|
+
def evict(query)
|
|
395
|
+
@stash[:queries].delete(query)
|
|
396
|
+
@stash[:tables].each_value { |list| list.delete(query) }
|
|
397
|
+
@stash[:tables].delete_if { |_, list| list.empty? }
|
|
398
|
+
end
|
|
399
|
+
|
|
385
400
|
def ranked
|
|
386
401
|
qq =
|
|
387
402
|
@entrance.with_write_lock do
|
|
388
403
|
@stash[:queries]
|
|
389
|
-
.map { |k, v| [k, v.values.sum { |vv| vv[:popularity] }, v.values.any? { |vv| vv[:stale] }] }
|
|
404
|
+
.map { |k, v| [k, v.values.sum { |vv| vv[:popularity]&.value || 0 }, v.values.any? { |vv| vv[:stale] }] }
|
|
390
405
|
end
|
|
391
406
|
qq.select { _1[2] }.sort_by { -_1[1] }.map { _1[0] }
|
|
392
407
|
end
|
|
@@ -414,7 +429,7 @@ class Pgtk::Stash
|
|
|
414
429
|
next unless h
|
|
415
430
|
next unless h[:stale] == mark
|
|
416
431
|
next if pinned.any? { |t, m| @stash[:table_mod][t] != m }
|
|
417
|
-
next if tables.any? { |t|
|
|
432
|
+
next if tables.any? { |t| @stash[:table_inflight][t].value.positive? }
|
|
418
433
|
h[:ret] = ret
|
|
419
434
|
h.delete(:stale)
|
|
420
435
|
end
|
data/lib/pgtk/version.rb
CHANGED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
4
|
+
# SPDX-License-Identifier: MIT
|
|
5
|
+
|
|
6
|
+
require 'pg'
|
|
7
|
+
require_relative '../../pgtk'
|
|
8
|
+
|
|
9
|
+
module Pgtk::Wire; end
|
|
10
|
+
|
|
11
|
+
# Simple wire with details.
|
|
12
|
+
#
|
|
13
|
+
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
|
14
|
+
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
15
|
+
# License:: MIT
|
|
16
|
+
class Pgtk::Wire::Direct
|
|
17
|
+
# Constructor.
|
|
18
|
+
#
|
|
19
|
+
# @param [String] host Host name of the PostgreSQL server
|
|
20
|
+
# @param [Integer] port Port number of the PostgreSQL server
|
|
21
|
+
# @param [String] dbname Database name
|
|
22
|
+
# @param [String] user Username
|
|
23
|
+
# @param [String] password Password
|
|
24
|
+
# @param [Hash] opts Extra options forwarded to +PG.connect+ (e.g. +sslmode+,
|
|
25
|
+
# +connect_timeout+, +keepalives+, +keepalives_idle+, +application_name+)
|
|
26
|
+
def initialize(host:, port:, dbname:, user:, password:, **opts)
|
|
27
|
+
raise(ArgumentError, "The host can't be nil") if host.nil?
|
|
28
|
+
@host = host
|
|
29
|
+
raise(ArgumentError, "The port can't be nil") if port.nil?
|
|
30
|
+
@port = port
|
|
31
|
+
@dbname = dbname
|
|
32
|
+
@user = user
|
|
33
|
+
@password = password
|
|
34
|
+
@opts = opts
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Create a new connection to PostgreSQL server.
|
|
38
|
+
def connection
|
|
39
|
+
PG.connect(dbname: @dbname, host: @host, port: @port, user: @user, password: @password, **@opts)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
4
|
+
# SPDX-License-Identifier: MIT
|
|
5
|
+
|
|
6
|
+
require 'cgi'
|
|
7
|
+
require 'uri'
|
|
8
|
+
require_relative '../../pgtk'
|
|
9
|
+
require_relative 'direct'
|
|
10
|
+
|
|
11
|
+
module Pgtk::Wire; end
|
|
12
|
+
|
|
13
|
+
# Using ENV variable.
|
|
14
|
+
#
|
|
15
|
+
# The value of the variable should be in this format:
|
|
16
|
+
#
|
|
17
|
+
# postgres://user:password@host:port/dbname
|
|
18
|
+
#
|
|
19
|
+
# Author:: Yegor Bugayenko (yegor256@gmail.com)
|
|
20
|
+
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
21
|
+
# License:: MIT
|
|
22
|
+
class Pgtk::Wire::Env
|
|
23
|
+
# Constructor.
|
|
24
|
+
#
|
|
25
|
+
# @param [String] var The name of the environment variable with the connection URL
|
|
26
|
+
# @param [Hash] opts Extra options forwarded to +PG.connect+ (e.g. +sslmode+,
|
|
27
|
+
# +connect_timeout+, +keepalives+, +keepalives_idle+, +application_name+).
|
|
28
|
+
# Explicit kwargs win over options carried in the URL query string on conflict.
|
|
29
|
+
def initialize(var = 'DATABASE_URL', **opts)
|
|
30
|
+
raise(ArgumentError, "The name of the environment variable can't be nil") if var.nil?
|
|
31
|
+
@value = ENV.fetch(var, nil)
|
|
32
|
+
raise(ArgumentError, "The environment variable #{var.inspect} is not set") if @value.nil?
|
|
33
|
+
@opts = opts
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Create a new connection to PostgreSQL server.
|
|
37
|
+
def connection
|
|
38
|
+
uri = URI(@value)
|
|
39
|
+
Pgtk::Wire::Direct.new(
|
|
40
|
+
host: CGI.unescape(uri.host),
|
|
41
|
+
port: uri.port || 5432,
|
|
42
|
+
dbname: CGI.unescape(uri.path[1..]),
|
|
43
|
+
user: CGI.unescape(uri.userinfo.split(':')[0]),
|
|
44
|
+
password: CGI.unescape(uri.userinfo.split(':')[1]),
|
|
45
|
+
**(uri.query ? URI.decode_www_form(uri.query).to_h.transform_keys(&:to_sym) : {}).merge(@opts)
|
|
46
|
+
).connection
|
|
47
|
+
end
|
|
48
|
+
end
|