pgtk 0.32.5 → 0.32.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +15 -14
- data/README.md +12 -0
- data/lib/pgtk/impatient.rb +27 -3
- data/lib/pgtk/pgsql_task.rb +18 -1
- data/lib/pgtk/pool.rb +46 -15
- data/lib/pgtk/spy.rb +16 -3
- data/lib/pgtk/stash.rb +68 -37
- data/lib/pgtk/version.rb +1 -1
- data/lib/pgtk/wire/env.rb +1 -1
- data/lib/pgtk/wire/yaml.rb +7 -2
- data/pgtk.gemspec +1 -1
- data/resources/pom.xml +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 54a200f1f191d63b73c0137b1a1ace4aa34667f041d141365b47d9c90d96f191
|
|
4
|
+
data.tar.gz: 1518e5031b71a69e3955cabe77397d05917398fb87a12b94ee5c3aa57ab107eb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 28bbce0d3b2cfcbe2347f376738938e0e714f22b89fd2696052c56ecf7be83c242e9f0a87c76c235f109ec2008a45a1471ea157fc78f3ac9a8b8b71641ac2556
|
|
7
|
+
data.tar.gz: e94795ee7e4de524c77b5fddefb7a0d2ea9e85ec05beb09ebf81000919afec9a7b54d8a0fb480d9e24c16990fed7b7b142769caefddaefdd32d504f8086f5f46
|
data/Gemfile.lock
CHANGED
|
@@ -2,7 +2,7 @@ PATH
|
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
4
|
pgtk (0.0.0)
|
|
5
|
-
|
|
5
|
+
cgi (~> 0.0)
|
|
6
6
|
concurrent-ruby (~> 1.3)
|
|
7
7
|
donce (~> 0.0)
|
|
8
8
|
ellipsized (~> 0.3)
|
|
@@ -22,7 +22,8 @@ GEM
|
|
|
22
22
|
ast (2.4.3)
|
|
23
23
|
backtrace (0.4.1)
|
|
24
24
|
builder (3.3.0)
|
|
25
|
-
|
|
25
|
+
cgi (0.5.1)
|
|
26
|
+
concurrent-ruby (1.3.7)
|
|
26
27
|
differ (0.1.2)
|
|
27
28
|
docile (1.4.1)
|
|
28
29
|
donce (0.7.0)
|
|
@@ -35,7 +36,7 @@ GEM
|
|
|
35
36
|
tago (~> 0.1)
|
|
36
37
|
ellipsized (0.3.0)
|
|
37
38
|
joined (0.4.0)
|
|
38
|
-
json (2.19.
|
|
39
|
+
json (2.19.9)
|
|
39
40
|
language_server-protocol (3.17.0.5)
|
|
40
41
|
lint_roller (1.1.0)
|
|
41
42
|
logger (1.7.0)
|
|
@@ -52,21 +53,21 @@ GEM
|
|
|
52
53
|
minitest (>= 5.0, < 7)
|
|
53
54
|
ruby-progressbar
|
|
54
55
|
minitest-stub-const (0.6)
|
|
55
|
-
nokogiri (1.19.
|
|
56
|
+
nokogiri (1.19.4-aarch64-linux-gnu)
|
|
56
57
|
racc (~> 1.4)
|
|
57
|
-
nokogiri (1.19.
|
|
58
|
+
nokogiri (1.19.4-aarch64-linux-musl)
|
|
58
59
|
racc (~> 1.4)
|
|
59
|
-
nokogiri (1.19.
|
|
60
|
+
nokogiri (1.19.4-arm-linux-gnu)
|
|
60
61
|
racc (~> 1.4)
|
|
61
|
-
nokogiri (1.19.
|
|
62
|
+
nokogiri (1.19.4-arm-linux-musl)
|
|
62
63
|
racc (~> 1.4)
|
|
63
|
-
nokogiri (1.19.
|
|
64
|
+
nokogiri (1.19.4-arm64-darwin)
|
|
64
65
|
racc (~> 1.4)
|
|
65
|
-
nokogiri (1.19.
|
|
66
|
+
nokogiri (1.19.4-x86_64-darwin)
|
|
66
67
|
racc (~> 1.4)
|
|
67
|
-
nokogiri (1.19.
|
|
68
|
+
nokogiri (1.19.4-x86_64-linux-gnu)
|
|
68
69
|
racc (~> 1.4)
|
|
69
|
-
nokogiri (1.19.
|
|
70
|
+
nokogiri (1.19.4-x86_64-linux-musl)
|
|
70
71
|
racc (~> 1.4)
|
|
71
72
|
os (1.1.4)
|
|
72
73
|
parallel (2.1.0)
|
|
@@ -94,7 +95,7 @@ GEM
|
|
|
94
95
|
tago (~> 0.0)
|
|
95
96
|
regexp_parser (2.12.0)
|
|
96
97
|
rexml (3.4.4)
|
|
97
|
-
rubocop (1.
|
|
98
|
+
rubocop (1.87.0)
|
|
98
99
|
json (~> 2.3)
|
|
99
100
|
language_server-protocol (~> 3.17.0.2)
|
|
100
101
|
lint_roller (~> 1.1.0)
|
|
@@ -108,7 +109,7 @@ GEM
|
|
|
108
109
|
rubocop-ast (1.49.1)
|
|
109
110
|
parser (>= 3.3.7.2)
|
|
110
111
|
prism (~> 1.7)
|
|
111
|
-
rubocop-elegant (0.
|
|
112
|
+
rubocop-elegant (0.6.0)
|
|
112
113
|
lint_roller (~> 1.1)
|
|
113
114
|
rubocop (~> 1.75)
|
|
114
115
|
rubocop-minitest (0.39.1)
|
|
@@ -147,7 +148,7 @@ GEM
|
|
|
147
148
|
nokogiri (~> 1.8)
|
|
148
149
|
rainbow (~> 3.0)
|
|
149
150
|
slop (~> 4.4)
|
|
150
|
-
yard (0.9.
|
|
151
|
+
yard (0.9.44)
|
|
151
152
|
|
|
152
153
|
PLATFORMS
|
|
153
154
|
aarch64-linux
|
data/README.md
CHANGED
|
@@ -243,6 +243,18 @@ You can exclude specific queries from timeout enforcement using regex patterns:
|
|
|
243
243
|
impatient = Pgtk::Impatient.new(pool, 2, /^SELECT/, /^VACUUM/)
|
|
244
244
|
```
|
|
245
245
|
|
|
246
|
+
Excluded queries are never wrapped in a transaction, because statements such as
|
|
247
|
+
`VACUUM`, `REINDEX`, or `CREATE INDEX CONCURRENTLY` cannot run inside a
|
|
248
|
+
transaction block. Instead, a longer fallback timeout (300 seconds by default,
|
|
249
|
+
configurable via the `default:` argument) is applied at the session level with
|
|
250
|
+
`SET statement_timeout` before the query runs, and reset afterwards. Pass
|
|
251
|
+
`default: 0` to run excluded queries with no timeout at all:
|
|
252
|
+
|
|
253
|
+
```ruby
|
|
254
|
+
# Give excluded queries a 60-second fallback timeout
|
|
255
|
+
impatient = Pgtk::Impatient.new(pool, 2, /^VACUUM/, default: 60)
|
|
256
|
+
```
|
|
257
|
+
|
|
246
258
|
The timeout is enforced on the server side: each query is wrapped in a tiny
|
|
247
259
|
transaction that issues `SET LOCAL statement_timeout`, and PostgreSQL itself
|
|
248
260
|
terminates the query at the deadline. This guarantees the server-side
|
data/lib/pgtk/impatient.rb
CHANGED
|
@@ -20,6 +20,14 @@ require_relative '../pgtk'
|
|
|
20
20
|
# transaction-pool PgBouncer that does not forward client disconnects to in-flight
|
|
21
21
|
# server queries). On timeout, +TooSlow+ is raised.
|
|
22
22
|
#
|
|
23
|
+
# Queries that match one of the +off+ regular expressions are excluded from
|
|
24
|
+
# this checking. They are never wrapped in a transaction, because some
|
|
25
|
+
# statements (such as +VACUUM+, +REINDEX+, or +CREATE INDEX CONCURRENTLY+)
|
|
26
|
+
# cannot run inside a transaction block. Instead, a session-level
|
|
27
|
+
# +SET statement_timeout+ is applied on the same connection before the query
|
|
28
|
+
# runs (and reset afterwards), using the +default+ fallback timeout. Pass
|
|
29
|
+
# +default: 0+ to run excluded queries with no timeout at all.
|
|
30
|
+
#
|
|
23
31
|
# Basic usage:
|
|
24
32
|
#
|
|
25
33
|
# # Create and configure a regular pool
|
|
@@ -63,10 +71,12 @@ class Pgtk::Impatient
|
|
|
63
71
|
# @param [Pgtk::Pool] pool The pool to decorate
|
|
64
72
|
# @param [Integer] timeout Timeout in seconds for each SQL query
|
|
65
73
|
# @param [Array<Regex>] off List of regex to exclude queries from checking
|
|
66
|
-
|
|
74
|
+
# @param [Integer] default Fallback timeout in seconds for excluded queries (0 = no timeout)
|
|
75
|
+
def initialize(pool, timeout, *off, default: 300)
|
|
67
76
|
@pool = pool
|
|
68
77
|
@timeout = timeout
|
|
69
78
|
@off = off
|
|
79
|
+
@default = default
|
|
70
80
|
end
|
|
71
81
|
|
|
72
82
|
# Start a new connection pool with the given arguments.
|
|
@@ -86,7 +96,7 @@ class Pgtk::Impatient
|
|
|
86
96
|
[
|
|
87
97
|
@pool.dump,
|
|
88
98
|
'',
|
|
89
|
-
"Pgtk::Impatient (timeout=#{@timeout}s):",
|
|
99
|
+
"Pgtk::Impatient (timeout=#{@timeout}s, default=#{@default}s):",
|
|
90
100
|
@off.map { |re| " #{re}" }
|
|
91
101
|
].join("\n")
|
|
92
102
|
end
|
|
@@ -100,13 +110,27 @@ class Pgtk::Impatient
|
|
|
100
110
|
# behind a transaction-pool PgBouncer). When the deadline fires, the
|
|
101
111
|
# underlying +PG::QueryCanceled+ is translated to +TooSlow+.
|
|
102
112
|
#
|
|
113
|
+
# Queries matching one of the +off+ regular expressions bypass this
|
|
114
|
+
# transaction. They run on a single connection without a transaction block,
|
|
115
|
+
# guarded by a session-level +SET statement_timeout+ (the +default+ fallback,
|
|
116
|
+
# reset afterwards) or by no timeout at all when +default+ is zero. This keeps
|
|
117
|
+
# statements that cannot run inside a transaction, such as +VACUUM+ or
|
|
118
|
+
# +REINDEX+, working as expected.
|
|
119
|
+
#
|
|
103
120
|
# @param [String, Array] query The SQL query with params inside (possibly)
|
|
104
121
|
# @param [Array] args List of arguments
|
|
105
122
|
# @return [Array] Result rows
|
|
106
123
|
# @raise [TooSlow] If the query takes too long
|
|
107
124
|
def exec(query, *args)
|
|
108
125
|
sql = query.is_a?(Array) ? query.join(' ') : query
|
|
109
|
-
|
|
126
|
+
if @off.any? { |re| re.match?(sql) }
|
|
127
|
+
ms = Integer(@default * 1000)
|
|
128
|
+
return @pool.exec(sql, *args) if ms.zero?
|
|
129
|
+
return @pool.session do |t|
|
|
130
|
+
t.exec("SET statement_timeout = #{ms}")
|
|
131
|
+
t.exec(sql, *args).tap { t.exec('RESET statement_timeout') }
|
|
132
|
+
end
|
|
133
|
+
end
|
|
110
134
|
start = Time.now
|
|
111
135
|
ms = [Integer(@timeout * 1000), 1].max
|
|
112
136
|
begin
|
data/lib/pgtk/pgsql_task.rb
CHANGED
|
@@ -129,6 +129,10 @@ class Pgtk::PgsqlTask < Rake::TaskLib
|
|
|
129
129
|
"-e POSTGRES_DB=#{Shellwords.escape(@dbname)}",
|
|
130
130
|
'--detach',
|
|
131
131
|
'--rm',
|
|
132
|
+
'--health-cmd', 'pg_isready',
|
|
133
|
+
'--health-interval', '1s',
|
|
134
|
+
'--health-timeout', '2s',
|
|
135
|
+
'--health-retries', '10',
|
|
132
136
|
'postgres:18.1',
|
|
133
137
|
@config.map { |k, v| "-c #{Shellwords.escape("#{k}=#{v}")}" },
|
|
134
138
|
stdout:
|
|
@@ -148,6 +152,15 @@ class Pgtk::PgsqlTask < Rake::TaskLib
|
|
|
148
152
|
rescue WaitUtil::TimeoutError => e
|
|
149
153
|
raise(IOError, "Failed to start PostgreSQL Docker container #{container.inspect}: #{e.message}")
|
|
150
154
|
end
|
|
155
|
+
(30 * 2).times do
|
|
156
|
+
status = qbash(
|
|
157
|
+
"docker inspect --format='{{.State.Health.Status}}' #{Shellwords.escape(container)}",
|
|
158
|
+
accept: nil, both: true
|
|
159
|
+
)[0]&.strip
|
|
160
|
+
break if status == 'healthy'
|
|
161
|
+
raise(IOError, "PostgreSQL container #{container.inspect} is #{status}") if status == 'unhealthy'
|
|
162
|
+
sleep(0.5)
|
|
163
|
+
end
|
|
151
164
|
container
|
|
152
165
|
end
|
|
153
166
|
|
|
@@ -181,13 +194,17 @@ class Pgtk::PgsqlTask < Rake::TaskLib
|
|
|
181
194
|
puts("PostgreSQL killed in PID #{pid}") unless @quiet
|
|
182
195
|
end
|
|
183
196
|
begin
|
|
184
|
-
WaitUtil.wait_for_service('PG in local', 'localhost', port, timeout_sec: 10, delay_sec: 0.
|
|
197
|
+
WaitUtil.wait_for_service('PG in local', 'localhost', port, timeout_sec: 10, delay_sec: 0.2)
|
|
185
198
|
rescue WaitUtil::TimeoutError => e
|
|
186
199
|
puts("+ #{cmd}")
|
|
187
200
|
puts("stdout:\n#{File.read(File.join(home, 'stdout.txt'))}")
|
|
188
201
|
puts("stderr:\n#{File.read(File.join(home, 'stderr.txt'))}")
|
|
189
202
|
raise(IOError, "Failed to start PostgreSQL database server on port #{port}: #{e.message}")
|
|
190
203
|
end
|
|
204
|
+
(10 * 2).times do
|
|
205
|
+
break if qbash("pg_isready -h localhost -p #{port}", accept: nil, both: true)[1].zero?
|
|
206
|
+
sleep(0.5)
|
|
207
|
+
end
|
|
191
208
|
qbash(
|
|
192
209
|
'createdb',
|
|
193
210
|
'--host', 'localhost',
|
data/lib/pgtk/pool.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'loog'
|
|
4
3
|
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
5
4
|
# SPDX-License-Identifier: MIT
|
|
6
5
|
|
|
7
6
|
require 'ellipsized'
|
|
7
|
+
require 'loog'
|
|
8
8
|
require 'pg'
|
|
9
9
|
require 'tago'
|
|
10
10
|
require_relative '../pgtk'
|
|
@@ -73,6 +73,7 @@ class Pgtk::Pool
|
|
|
73
73
|
@idle = idle
|
|
74
74
|
@log = log
|
|
75
75
|
@pool = IterableQueue.new(max, timeout)
|
|
76
|
+
@lock = Mutex.new
|
|
76
77
|
@started = false
|
|
77
78
|
end
|
|
78
79
|
|
|
@@ -80,7 +81,12 @@ class Pgtk::Pool
|
|
|
80
81
|
#
|
|
81
82
|
# @return [String] Version of PostgreSQL server
|
|
82
83
|
def version
|
|
83
|
-
@version ||=
|
|
84
|
+
@version ||=
|
|
85
|
+
begin
|
|
86
|
+
conn = @pool.pop
|
|
87
|
+
@pool.push(conn)
|
|
88
|
+
conn.parameter_status('server_version').split[0]
|
|
89
|
+
end
|
|
84
90
|
end
|
|
85
91
|
|
|
86
92
|
# Get as much details about it as possible.
|
|
@@ -103,20 +109,22 @@ class Pgtk::Pool
|
|
|
103
109
|
# open at the same time. For example, Heroku free PostgreSQL database
|
|
104
110
|
# allows only one connection open.
|
|
105
111
|
def start!
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
@
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
112
|
+
@lock.synchronize do
|
|
113
|
+
return if @started
|
|
114
|
+
@max.times do
|
|
115
|
+
@pool.push(@wire.connection)
|
|
116
|
+
end
|
|
117
|
+
(2 * @max).times do
|
|
118
|
+
connect { |c| c.exec('SELECT 1') }
|
|
119
|
+
rescue StandardError => e
|
|
120
|
+
@log.warn("Pool warm-up query failed, slot will be retried: #{e.message.strip}")
|
|
121
|
+
end
|
|
122
|
+
@max.times do
|
|
123
|
+
connect { |c| c.exec('SELECT 1') }
|
|
124
|
+
end
|
|
125
|
+
@started = true
|
|
126
|
+
@log.debug("PostgreSQL pool started with #{@max} connections")
|
|
117
127
|
end
|
|
118
|
-
@started = true
|
|
119
|
-
@log.debug("PostgreSQL pool started with #{@max} connections")
|
|
120
128
|
end
|
|
121
129
|
|
|
122
130
|
# Make a query and return the result as an array of hashes. For example,
|
|
@@ -206,6 +214,29 @@ class Pgtk::Pool
|
|
|
206
214
|
end
|
|
207
215
|
end
|
|
208
216
|
|
|
217
|
+
# Grab a single connection from the pool and yield an executor bound to it,
|
|
218
|
+
# WITHOUT starting a transaction. Unlike +transaction+, no +START TRANSACTION+
|
|
219
|
+
# is issued, which makes this suitable for statements that PostgreSQL refuses
|
|
220
|
+
# to run inside a transaction block, such as +VACUUM+, +REINDEX+, or
|
|
221
|
+
# +CREATE INDEX CONCURRENTLY+. All statements executed through the yielded
|
|
222
|
+
# object run on the same connection, so a session-level setting (for example
|
|
223
|
+
# +SET statement_timeout+) made earlier in the block stays in effect for the
|
|
224
|
+
# statements that follow:
|
|
225
|
+
#
|
|
226
|
+
# pgsql.session do |s|
|
|
227
|
+
# s.exec('SET statement_timeout = 5000')
|
|
228
|
+
# s.exec('VACUUM book')
|
|
229
|
+
# s.exec('RESET statement_timeout')
|
|
230
|
+
# end
|
|
231
|
+
#
|
|
232
|
+
# @yield [Object] Yields an executor that responds to +exec+
|
|
233
|
+
# @return [Object] Result of the block
|
|
234
|
+
def session
|
|
235
|
+
connect do |c|
|
|
236
|
+
yield(Txn.new(c, @log))
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
209
240
|
private
|
|
210
241
|
|
|
211
242
|
def connect
|
data/lib/pgtk/spy.rb
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'loog'
|
|
4
3
|
# SPDX-FileCopyrightText: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
5
4
|
# SPDX-License-Identifier: MIT
|
|
6
5
|
|
|
6
|
+
require 'loog'
|
|
7
7
|
require 'pg'
|
|
8
8
|
require_relative '../pgtk'
|
|
9
9
|
require_relative 'wire'
|
|
@@ -82,8 +82,10 @@ class Pgtk::Spy
|
|
|
82
82
|
# @param [String] sql The SQL query with params inside (possibly)
|
|
83
83
|
# @return [Array] Result rows
|
|
84
84
|
def exec(sql, *)
|
|
85
|
-
|
|
86
|
-
@pool.exec(sql, *)
|
|
85
|
+
start = Time.now
|
|
86
|
+
@pool.exec(sql, *).tap do
|
|
87
|
+
@block&.call(sql.is_a?(Array) ? sql.join(' ') : sql, Time.now - start)
|
|
88
|
+
end
|
|
87
89
|
end
|
|
88
90
|
|
|
89
91
|
# Run a transaction with spying on each SQL query.
|
|
@@ -95,4 +97,15 @@ class Pgtk::Spy
|
|
|
95
97
|
yield(Pgtk::Spy.new(t, &@block))
|
|
96
98
|
end
|
|
97
99
|
end
|
|
100
|
+
|
|
101
|
+
# Run statements on a single connection (no transaction) with spying on
|
|
102
|
+
# each SQL query.
|
|
103
|
+
#
|
|
104
|
+
# @yield [Pgtk::Spy] Yields a spy bound to one connection
|
|
105
|
+
# @return [Object] Result of the block
|
|
106
|
+
def session
|
|
107
|
+
@pool.session do |t|
|
|
108
|
+
yield(Pgtk::Spy.new(t, &@block))
|
|
109
|
+
end
|
|
110
|
+
end
|
|
98
111
|
end
|
data/lib/pgtk/stash.rb
CHANGED
|
@@ -29,8 +29,11 @@ require_relative '../pgtk'
|
|
|
29
29
|
# Copyright:: Copyright (c) 2019-2026 Yegor Bugayenko
|
|
30
30
|
# License:: MIT
|
|
31
31
|
class Pgtk::Stash
|
|
32
|
-
MODS = %w[
|
|
33
|
-
|
|
32
|
+
MODS = %w[
|
|
33
|
+
INSERT DELETE UPDATE LOCK VACUUM TRANSACTION COMMIT ROLLBACK
|
|
34
|
+
REINDEX TRUNCATE CREATE ALTER DROP SET START
|
|
35
|
+
].freeze
|
|
36
|
+
MODS_RE = Regexp.new("\\A(#{MODS.join('|')})(\\s|$)")
|
|
34
37
|
|
|
35
38
|
IDENT = '[a-z_][a-z0-9_]*'
|
|
36
39
|
|
|
@@ -39,9 +42,13 @@ class Pgtk::Stash
|
|
|
39
42
|
|
|
40
43
|
READS_RE = Regexp.new("(?<=^|\\s)(?:FROM|JOIN)\\s(#{IDENT})(?=\\s|;|$)")
|
|
41
44
|
|
|
42
|
-
|
|
45
|
+
NONDETERMINISTIC = /
|
|
46
|
+
\b(?:NOW|CURRENT_TIMESTAMP|CURRENT_DATE|CURRENT_TIME|
|
|
47
|
+
LOCALTIMESTAMP|LOCALTIME|RANDOM|GEN_RANDOM_UUID|
|
|
48
|
+
CLOCK_TIMESTAMP)(?:\s*\(|(?=[^a-z0-9_]|$))
|
|
49
|
+
/ix
|
|
43
50
|
|
|
44
|
-
private_constant :MODS, :ALTS, :IDENT, :MODS_RE, :ALTS_RE, :READS_RE, :
|
|
51
|
+
private_constant :MODS, :ALTS, :IDENT, :MODS_RE, :ALTS_RE, :READS_RE, :NONDETERMINISTIC
|
|
45
52
|
|
|
46
53
|
# Initialize a new Stash with query caching.
|
|
47
54
|
#
|
|
@@ -76,7 +83,9 @@ class Pgtk::Stash
|
|
|
76
83
|
@pool = pool
|
|
77
84
|
@stash = stash
|
|
78
85
|
@stash[:table_mod] ||= {}
|
|
79
|
-
@stash[:table_inflight]
|
|
86
|
+
unless @stash[:table_inflight].default_proc
|
|
87
|
+
@stash[:table_inflight] = Hash.new { |h, k| h[k] = Concurrent::AtomicFixnum.new(0) }
|
|
88
|
+
end
|
|
80
89
|
@loog = loog
|
|
81
90
|
@entrance = entrance
|
|
82
91
|
@refill = refill
|
|
@@ -121,8 +130,10 @@ class Pgtk::Stash
|
|
|
121
130
|
# @return [PG::Result] Query result object
|
|
122
131
|
def exec(query, params = [], result = 0)
|
|
123
132
|
pure = (query.is_a?(Array) ? query.join(' ') : query).gsub(/\s+/, ' ').strip
|
|
124
|
-
if MODS_RE.match?(pure)
|
|
133
|
+
if MODS_RE.match?(pure)
|
|
125
134
|
modify(pure, params, result)
|
|
135
|
+
elsif /(^|\s)pg_[a-z_]+\(/.match?(pure)
|
|
136
|
+
@pool.exec(pure, params, result)
|
|
126
137
|
else
|
|
127
138
|
select(pure, params, result)
|
|
128
139
|
end
|
|
@@ -138,6 +149,16 @@ class Pgtk::Stash
|
|
|
138
149
|
end
|
|
139
150
|
end
|
|
140
151
|
|
|
152
|
+
# Run statements on a single connection, without a transaction.
|
|
153
|
+
#
|
|
154
|
+
# @yield [Pgtk::Stash] A stash bound to the connection
|
|
155
|
+
# @return [Object] The result of the block
|
|
156
|
+
def session
|
|
157
|
+
@pool.session do |t|
|
|
158
|
+
yield(Pgtk::Stash.new(t, stash: @stash, loog: @loog, entrance: @entrance))
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
141
162
|
private
|
|
142
163
|
|
|
143
164
|
def queries
|
|
@@ -145,7 +166,7 @@ class Pgtk::Stash
|
|
|
145
166
|
{
|
|
146
167
|
q: q.dup,
|
|
147
168
|
c: kk.values.count,
|
|
148
|
-
p: kk.values.sum { |vv| vv[:popularity] },
|
|
169
|
+
p: kk.values.sum { |vv| vv[:popularity]&.value || 0 },
|
|
149
170
|
s: kk.values.count { |vv| vv[:stale] },
|
|
150
171
|
u: kk.values.map { |vv| vv[:used] }.max || Time.now
|
|
151
172
|
}
|
|
@@ -222,15 +243,13 @@ class Pgtk::Stash
|
|
|
222
243
|
tables = pure.scan(ALTS_RE).flatten
|
|
223
244
|
tables.uniq!
|
|
224
245
|
affected = (tables + tables.flat_map { |t| @cascades&.fetch(t, []) || [] }).uniq
|
|
225
|
-
@
|
|
226
|
-
affected.each { |t| @stash[:table_inflight][t] = (@stash[:table_inflight][t] || 0) + 1 }
|
|
227
|
-
end
|
|
246
|
+
affected.each { |t| @stash[:table_inflight][t].increment }
|
|
228
247
|
begin
|
|
229
248
|
@pool.exec(pure, params, result).tap do
|
|
230
249
|
now = Time.now
|
|
231
250
|
@entrance.with_write_lock do
|
|
232
251
|
affected.each do |t|
|
|
233
|
-
@stash[:table_inflight][t]
|
|
252
|
+
@stash[:table_inflight][t].decrement
|
|
234
253
|
@stash[:table_mod][t] = (@stash[:table_mod][t] || 0) + 1
|
|
235
254
|
@stash[:tables][t]&.each do |q|
|
|
236
255
|
@stash[:queries][q]&.each_key do |key|
|
|
@@ -241,28 +260,25 @@ class Pgtk::Stash
|
|
|
241
260
|
end
|
|
242
261
|
end
|
|
243
262
|
rescue StandardError
|
|
244
|
-
@
|
|
245
|
-
affected.each { |t| @stash[:table_inflight][t] -= 1 }
|
|
246
|
-
end
|
|
263
|
+
affected.each { |t| @stash[:table_inflight][t].decrement }
|
|
247
264
|
raise
|
|
248
265
|
end
|
|
249
266
|
end
|
|
250
267
|
|
|
251
268
|
def select(pure, params, result)
|
|
252
|
-
|
|
253
|
-
ret
|
|
254
|
-
if ret.nil? || @stash.dig(:queries, pure, key, :stale)
|
|
269
|
+
ret = @stash.dig(:queries, pure, params, :ret)
|
|
270
|
+
if ret.nil? || @stash.dig(:queries, pure, params, :stale)
|
|
255
271
|
tables = pure.scan(READS_RE).flatten
|
|
256
272
|
tables.uniq!
|
|
257
273
|
marks = tables.to_h { |t| [t, @stash[:table_mod][t]] }
|
|
258
274
|
ret = @pool.exec(pure, params, result)
|
|
259
|
-
cache(pure,
|
|
275
|
+
cache(pure, params, result, ret, tables, marks) unless pure.match?(NONDETERMINISTIC)
|
|
260
276
|
end
|
|
261
|
-
bump(pure,
|
|
277
|
+
bump(pure, params) if @stash.dig(:queries, pure, params)
|
|
262
278
|
ret
|
|
263
279
|
end
|
|
264
280
|
|
|
265
|
-
def cache(pure,
|
|
281
|
+
def cache(pure, params, result, ret, tables, marks)
|
|
266
282
|
raise(ArgumentError, "No tables at #{pure.inspect}") if tables.empty?
|
|
267
283
|
@entrance.with_write_lock do
|
|
268
284
|
tables.each do |t|
|
|
@@ -270,7 +286,7 @@ class Pgtk::Stash
|
|
|
270
286
|
@stash[:tables][t].append(pure).uniq!
|
|
271
287
|
end
|
|
272
288
|
@stash[:queries][pure] ||= {}
|
|
273
|
-
existing = @stash[:queries][pure][
|
|
289
|
+
existing = @stash[:queries][pure][params]
|
|
274
290
|
stillborn = tables.any? { |t| (cur = @stash[:table_mod][t]) && cur != marks[t] }
|
|
275
291
|
entry = { ret:, params:, result:, used: Time.now }
|
|
276
292
|
entry[:stale] =
|
|
@@ -280,23 +296,32 @@ class Pgtk::Stash
|
|
|
280
296
|
Time.now
|
|
281
297
|
end
|
|
282
298
|
entry.delete(:stale) if entry[:stale].nil?
|
|
283
|
-
|
|
299
|
+
entry[:popularity] = (existing && existing[:popularity]) || Concurrent::AtomicFixnum.new
|
|
300
|
+
@stash[:queries][pure][params] = entry
|
|
284
301
|
end
|
|
285
302
|
end
|
|
286
303
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
304
|
+
# Bump popularity and last-used time of a cached entry.
|
|
305
|
+
#
|
|
306
|
+
# This runs without the exclusive write lock: popularity is a
|
|
307
|
+
# +Concurrent::AtomicFixnum+ and +used+ is a plain assignment of an
|
|
308
|
+
# immutable value, neither of which mutates the structure of the
|
|
309
|
+
# +@stash[:queries]+ tree. Keeping it off the writer lock means a
|
|
310
|
+
# read-only workload never serializes through the writer (see #414).
|
|
311
|
+
#
|
|
312
|
+
# @return [void]
|
|
313
|
+
def bump(pure, params)
|
|
314
|
+
entry = @stash.dig(:queries, pure, params)
|
|
315
|
+
return unless entry
|
|
316
|
+
(entry[:popularity] ||= Concurrent::AtomicFixnum.new).increment
|
|
317
|
+
entry[:used] = Time.now
|
|
293
318
|
end
|
|
294
319
|
|
|
295
320
|
# Calculate total number of cached query results.
|
|
296
321
|
#
|
|
297
322
|
# @return [Integer] Total count of cached query results
|
|
298
323
|
def cached
|
|
299
|
-
@entrance.
|
|
324
|
+
@entrance.with_read_lock do
|
|
300
325
|
@stash[:queries].values.sum { |kk| kk.values.size }
|
|
301
326
|
end
|
|
302
327
|
end
|
|
@@ -350,6 +375,8 @@ class Pgtk::Stash
|
|
|
350
375
|
end
|
|
351
376
|
end
|
|
352
377
|
end
|
|
378
|
+
rescue StandardError => e
|
|
379
|
+
@loog.warn("Stash capper crashed: #{e.class}: #{e.message}")
|
|
353
380
|
end
|
|
354
381
|
end
|
|
355
382
|
|
|
@@ -361,26 +388,30 @@ class Pgtk::Stash
|
|
|
361
388
|
evict(q) if @stash[:queries][q].empty?
|
|
362
389
|
end
|
|
363
390
|
end
|
|
391
|
+
rescue StandardError => e
|
|
392
|
+
@loog.warn("Stash retiree crashed: #{e.class}: #{e.message}")
|
|
364
393
|
end
|
|
365
394
|
end
|
|
366
395
|
|
|
367
|
-
def evict(query)
|
|
368
|
-
@stash[:queries].delete(query)
|
|
369
|
-
@stash[:tables].each_value { |list| list.delete(query) }
|
|
370
|
-
@stash[:tables].delete_if { |_, list| list.empty? }
|
|
371
|
-
end
|
|
372
|
-
|
|
373
396
|
def refiller!
|
|
374
397
|
Concurrent::TimerTask.execute(execution_interval: @refill, executor: @tpool) do
|
|
375
398
|
ranked.each { |q| replenish(q) }
|
|
399
|
+
rescue StandardError => e
|
|
400
|
+
@loog.warn("Stash refiller crashed: #{e.class}: #{e.message}")
|
|
376
401
|
end
|
|
377
402
|
end
|
|
378
403
|
|
|
404
|
+
def evict(query)
|
|
405
|
+
@stash[:queries].delete(query)
|
|
406
|
+
@stash[:tables].each_value { |list| list.delete(query) }
|
|
407
|
+
@stash[:tables].delete_if { |_, list| list.empty? }
|
|
408
|
+
end
|
|
409
|
+
|
|
379
410
|
def ranked
|
|
380
411
|
qq =
|
|
381
412
|
@entrance.with_write_lock do
|
|
382
413
|
@stash[:queries]
|
|
383
|
-
.map { |k, v| [k, v.values.sum { |vv| vv[:popularity] }, v.values.any? { |vv| vv[:stale] }] }
|
|
414
|
+
.map { |k, v| [k, v.values.sum { |vv| vv[:popularity]&.value || 0 }, v.values.any? { |vv| vv[:stale] }] }
|
|
384
415
|
end
|
|
385
416
|
qq.select { _1[2] }.sort_by { -_1[1] }.map { _1[0] }
|
|
386
417
|
end
|
|
@@ -408,7 +439,7 @@ class Pgtk::Stash
|
|
|
408
439
|
next unless h
|
|
409
440
|
next unless h[:stale] == mark
|
|
410
441
|
next if pinned.any? { |t, m| @stash[:table_mod][t] != m }
|
|
411
|
-
next if tables.any? { |t|
|
|
442
|
+
next if tables.any? { |t| @stash[:table_inflight][t].value.positive? }
|
|
412
443
|
h[:ret] = ret
|
|
413
444
|
h.delete(:stale)
|
|
414
445
|
end
|
data/lib/pgtk/version.rb
CHANGED
data/lib/pgtk/wire/env.rb
CHANGED
|
@@ -29,7 +29,7 @@ class Pgtk::Wire::Env
|
|
|
29
29
|
def initialize(var = 'DATABASE_URL', **opts)
|
|
30
30
|
raise(ArgumentError, "The name of the environment variable can't be nil") if var.nil?
|
|
31
31
|
@value = ENV.fetch(var, nil)
|
|
32
|
-
raise(ArgumentError, "The environment variable #{
|
|
32
|
+
raise(ArgumentError, "The environment variable #{var.inspect} is not set") if @value.nil?
|
|
33
33
|
@opts = opts
|
|
34
34
|
end
|
|
35
35
|
|
data/lib/pgtk/wire/yaml.rb
CHANGED
|
@@ -19,11 +19,14 @@ class Pgtk::Wire::Yaml
|
|
|
19
19
|
#
|
|
20
20
|
# @param [String] file Path to the YAML configuration file
|
|
21
21
|
# @param [String] node The root node name in the YAML file containing PostgreSQL configuration
|
|
22
|
-
|
|
22
|
+
# @param [Hash] opts Extra options forwarded to +PG.connect+ (e.g. +sslmode+,
|
|
23
|
+
# +connect_timeout+, +keepalives+). These override values from the YAML file.
|
|
24
|
+
def initialize(file, node = 'pgsql', **opts)
|
|
23
25
|
raise(ArgumentError, "The name of the file can't be nil") if file.nil?
|
|
24
26
|
@file = file
|
|
25
27
|
raise(ArgumentError, "The name of the node in the YAML file can't be nil") if node.nil?
|
|
26
28
|
@node = node
|
|
29
|
+
@opts = opts
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
# Create a new connection to PostgreSQL server.
|
|
@@ -36,7 +39,9 @@ class Pgtk::Wire::Yaml
|
|
|
36
39
|
port: cfg[@node]['port'],
|
|
37
40
|
dbname: cfg[@node]['dbname'],
|
|
38
41
|
user: cfg[@node]['user'],
|
|
39
|
-
password: cfg[@node]['password']
|
|
42
|
+
password: cfg[@node]['password'],
|
|
43
|
+
**cfg[@node].except(*%w[host port dbname user password url]).transform_keys(&:to_sym),
|
|
44
|
+
**@opts
|
|
40
45
|
).connection
|
|
41
46
|
end
|
|
42
47
|
end
|
data/pgtk.gemspec
CHANGED
|
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
|
|
|
26
26
|
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
27
27
|
s.rdoc_options = ['--charset=UTF-8']
|
|
28
28
|
s.extra_rdoc_files = ['README.md', 'LICENSE.txt']
|
|
29
|
-
s.add_dependency('
|
|
29
|
+
s.add_dependency('cgi', '~>0.0')
|
|
30
30
|
s.add_dependency('concurrent-ruby', '~>1.3')
|
|
31
31
|
s.add_dependency('donce', '~>0.0')
|
|
32
32
|
s.add_dependency('ellipsized', '~>0.3')
|
data/resources/pom.xml
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
<version>0.0.0</version>
|
|
11
11
|
<packaging>pom</packaging>
|
|
12
12
|
<properties>
|
|
13
|
-
<postgresql.version>42.7.
|
|
13
|
+
<postgresql.version>42.7.12</postgresql.version>
|
|
14
14
|
<liquibase.version>5.0.3</liquibase.version>
|
|
15
15
|
</properties>
|
|
16
16
|
<dependencies>
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgtk
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.32.
|
|
4
|
+
version: 0.32.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yegor Bugayenko
|
|
@@ -10,19 +10,19 @@ cert_chain: []
|
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
|
-
name:
|
|
13
|
+
name: cgi
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '0.
|
|
18
|
+
version: '0.0'
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - "~>"
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: '0.
|
|
25
|
+
version: '0.0'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
27
|
name: concurrent-ruby
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|