@mistriwork/engine 1.0.0-rc.68 → 1.0.0-rc.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/cache/cacheStore.d.ts +1 -12
- package/dist/adapters/cache/cacheStore.js +23 -1
- package/dist/adapters/cache/resp.d.ts +3 -1
- package/dist/adapters/cache/resp.js +48 -8
- package/dist/adapters/connect.js +5 -2
- package/dist/adapters/http/egress.js +1 -0
- package/dist/adapters/net/networkPolicy.js +22 -10
- package/dist/adapters/postgresPool.d.ts +12 -3
- package/dist/adapters/postgresPool.js +50 -9
- package/dist/adapters/postgresSql.js +6 -0
- package/dist/adapters/reservedSql.d.ts +2 -1
- package/dist/adapters/reservedSql.js +39 -15
- package/dist/adapters/sqliteStatementProcess.js +99 -49
- package/dist/server/clientHttp.js +8 -4
- package/dist/server/columnGrants.js +4 -0
- package/dist/server/http/context.d.ts +3 -1
- package/dist/server/http/dataRoutes.js +1 -0
- package/dist/server/http/exportRoute.d.ts +3 -0
- package/dist/server/http/exportRoute.js +3 -1
- package/dist/server/http/rateLimit.js +4 -2
- package/dist/server/http/trust.d.ts +0 -14
- package/dist/server/http/trust.js +24 -8
- package/dist/shared/frameworkWords.d.ts +66 -2
- package/dist/shared/frameworkWords.js +66 -2
- package/dist/shared/frameworkWordsEs.d.ts +9 -0
- package/dist/shared/frameworkWordsEs.js +109 -0
- package/package.json +2 -1
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Somewhere to keep a value for a while: in this process by default, in a
|
|
3
|
-
* Valkey cache when the config names one, so every instance of a deployment
|
|
4
|
-
* shares it.
|
|
5
|
-
*
|
|
6
|
-
* Never a source of truth. Everything here is a copy or a count that is
|
|
7
|
-
* allowed to vanish — a cached API answer, a rate-limit window — so every
|
|
8
|
-
* write carries a time to live (there is no "forever"), a read that cannot
|
|
9
|
-
* reach the cache is a miss, and a write that cannot is dropped. The one
|
|
10
|
-
* caller that must know the difference, the rate limiter, is told through
|
|
11
|
-
* `incr`, which throws `CacheUnavailable` rather than inventing a count.
|
|
12
|
-
*/
|
|
13
1
|
import { type RespClient } from './resp.js';
|
|
14
2
|
export { CacheUnavailable } from './resp.js';
|
|
15
3
|
export interface CacheStore {
|
|
@@ -61,6 +49,7 @@ export declare class ValkeyCacheStore implements CacheStore {
|
|
|
61
49
|
private readonly client;
|
|
62
50
|
private readonly onUnavailable;
|
|
63
51
|
readonly kind: 'valkey';
|
|
52
|
+
private scriptLoaded;
|
|
64
53
|
constructor(client: RespClient, onUnavailable?: (message: string) => void);
|
|
65
54
|
get(key: string): Promise<string | undefined>;
|
|
66
55
|
set(key: string, value: string, ttlMs: number): Promise<void>;
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* caller that must know the difference, the rate limiter, is told through
|
|
11
11
|
* `incr`, which throws `CacheUnavailable` rather than inventing a count.
|
|
12
12
|
*/
|
|
13
|
+
import { createHash } from 'node:crypto';
|
|
13
14
|
import { CacheUnavailable, RespReplyError } from './resp.js';
|
|
14
15
|
export { CacheUnavailable } from './resp.js';
|
|
15
16
|
/* Thirty days: a copy kept longer than that is a copy nobody meant to keep. */
|
|
@@ -92,6 +93,7 @@ if n == tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[1]) end
|
|
|
92
93
|
local t = redis.call('PTTL', KEYS[1])
|
|
93
94
|
if t < 0 then redis.call('PEXPIRE', KEYS[1], ARGV[1]) t = tonumber(ARGV[1]) end
|
|
94
95
|
return {n, t}`;
|
|
96
|
+
const COUNT_IN_WINDOW_SHA = createHash('sha1').update(COUNT_IN_WINDOW).digest('hex');
|
|
95
97
|
/**
|
|
96
98
|
* A Valkey cache shared by every instance. `onUnavailable` hears each
|
|
97
99
|
* outage's reason — the server logs it once — while reads carry on as misses.
|
|
@@ -100,6 +102,7 @@ export class ValkeyCacheStore {
|
|
|
100
102
|
client;
|
|
101
103
|
onUnavailable;
|
|
102
104
|
kind = 'valkey';
|
|
105
|
+
scriptLoaded = false;
|
|
103
106
|
constructor(client, onUnavailable = () => { }) {
|
|
104
107
|
this.client = client;
|
|
105
108
|
this.onUnavailable = onUnavailable;
|
|
@@ -136,7 +139,26 @@ export class ValkeyCacheStore {
|
|
|
136
139
|
assertBy(by);
|
|
137
140
|
let reply;
|
|
138
141
|
try {
|
|
139
|
-
|
|
142
|
+
if (this.scriptLoaded) {
|
|
143
|
+
try {
|
|
144
|
+
reply = await this.client.command('EVALSHA', COUNT_IN_WINDOW_SHA, 1, key, ttlMs, by);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
/* Valkey restarts clear its script cache; fall back to EVAL once to reload it. */
|
|
148
|
+
if (error instanceof RespReplyError && /NOSCRIPT/i.test(error.message)) {
|
|
149
|
+
this.scriptLoaded = false;
|
|
150
|
+
reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs, by);
|
|
151
|
+
this.scriptLoaded = true;
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs, by);
|
|
160
|
+
this.scriptLoaded = true;
|
|
161
|
+
}
|
|
140
162
|
}
|
|
141
163
|
catch (error) {
|
|
142
164
|
this.unavailable(error);
|
|
@@ -32,8 +32,10 @@ export declare function decodeReply(buffer: Buffer, start?: number): {
|
|
|
32
32
|
export declare class RespClient {
|
|
33
33
|
private readonly options;
|
|
34
34
|
private socket?;
|
|
35
|
+
private connectingSocket?;
|
|
35
36
|
private opening?;
|
|
36
|
-
private
|
|
37
|
+
private chunks;
|
|
38
|
+
private buffered;
|
|
37
39
|
private readonly waiting;
|
|
38
40
|
private quietUntil;
|
|
39
41
|
private lastFailure;
|
|
@@ -86,11 +86,14 @@ export function decodeReply(buffer, start = 0) {
|
|
|
86
86
|
throw new CacheUnavailable(`the cache answered something that is not RESP ("${type}").`);
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
|
+
const MAX_WAITING = 1024;
|
|
89
90
|
export class RespClient {
|
|
90
91
|
options;
|
|
91
92
|
socket;
|
|
93
|
+
connectingSocket;
|
|
92
94
|
opening;
|
|
93
|
-
|
|
95
|
+
chunks = [];
|
|
96
|
+
buffered = 0;
|
|
94
97
|
waiting = [];
|
|
95
98
|
quietUntil = 0;
|
|
96
99
|
lastFailure = '';
|
|
@@ -107,11 +110,16 @@ export class RespClient {
|
|
|
107
110
|
throw new CacheUnavailable('the cache connection is closed.');
|
|
108
111
|
if (Date.now() < this.quietUntil)
|
|
109
112
|
throw new CacheUnavailable(this.lastFailure);
|
|
113
|
+
if (this.waiting.length >= MAX_WAITING)
|
|
114
|
+
throw new CacheUnavailable(`${this.where}: too many commands waiting for a reply.`);
|
|
110
115
|
const socket = await this.connected();
|
|
111
116
|
return this.write(socket, args);
|
|
112
117
|
}
|
|
113
118
|
async close() {
|
|
114
119
|
this.closed = true;
|
|
120
|
+
const connecting = this.connectingSocket;
|
|
121
|
+
this.connectingSocket = undefined;
|
|
122
|
+
connecting?.destroy();
|
|
115
123
|
const socket = this.socket;
|
|
116
124
|
this.socket = undefined;
|
|
117
125
|
if (socket) {
|
|
@@ -125,6 +133,9 @@ export class RespClient {
|
|
|
125
133
|
}
|
|
126
134
|
}
|
|
127
135
|
write(socket, args) {
|
|
136
|
+
if (this.waiting.length >= MAX_WAITING) {
|
|
137
|
+
return Promise.reject(new CacheUnavailable(`${this.where}: too many commands waiting for a reply.`));
|
|
138
|
+
}
|
|
128
139
|
return new Promise((resolve, reject) => {
|
|
129
140
|
const timer = setTimeout(() => this.fail(`${this.where} did not answer within ${(this.options.commandTimeoutMs ?? 1_000) / 1000} s.`), this.options.commandTimeoutMs ?? 1_000);
|
|
130
141
|
this.waiting.push({
|
|
@@ -146,9 +157,14 @@ export class RespClient {
|
|
|
146
157
|
this.lastFailure = message;
|
|
147
158
|
this.quietUntil = Date.now() + (this.options.retryAfterMs ?? 2_000);
|
|
148
159
|
const socket = this.socket;
|
|
160
|
+
const connecting = this.connectingSocket;
|
|
149
161
|
this.socket = undefined;
|
|
150
|
-
this.
|
|
162
|
+
this.connectingSocket = undefined;
|
|
163
|
+
this.chunks = [];
|
|
164
|
+
this.buffered = 0;
|
|
151
165
|
socket?.destroy();
|
|
166
|
+
if (connecting && connecting !== socket)
|
|
167
|
+
connecting.destroy();
|
|
152
168
|
for (const waiting of this.waiting.splice(0))
|
|
153
169
|
waiting.reject(new CacheUnavailable(message));
|
|
154
170
|
}
|
|
@@ -188,14 +204,20 @@ export class RespClient {
|
|
|
188
204
|
this.quietUntil = Date.now() + (this.options.retryAfterMs ?? 2_000);
|
|
189
205
|
throw error;
|
|
190
206
|
});
|
|
207
|
+
if (this.closed) {
|
|
208
|
+
socket.destroy();
|
|
209
|
+
throw new CacheUnavailable('the cache connection is closed.');
|
|
210
|
+
}
|
|
191
211
|
socket.setNoDelay(true);
|
|
192
212
|
socket.on('data', (chunk) => this.read(chunk));
|
|
193
213
|
socket.on('error', (error) => this.fail(`${this.where}: ${error.code ?? 'the connection failed'}.`));
|
|
194
214
|
socket.on('close', () => {
|
|
195
|
-
if (this.socket === socket)
|
|
215
|
+
if (this.socket === socket || this.connectingSocket === socket)
|
|
196
216
|
this.fail(`${this.where} closed the connection.`);
|
|
197
217
|
});
|
|
198
|
-
|
|
218
|
+
/* Hold the socket privately until AUTH and SELECT finish so concurrent callers
|
|
219
|
+
await `this.opening` and can never pipeline commands before authentication. */
|
|
220
|
+
this.connectingSocket = socket;
|
|
199
221
|
try {
|
|
200
222
|
if (this.options.password !== undefined) {
|
|
201
223
|
await this.write(socket, this.options.username ? ['AUTH', this.options.username, this.options.password] : ['AUTH', this.options.password]);
|
|
@@ -210,21 +232,39 @@ export class RespClient {
|
|
|
210
232
|
this.fail(message);
|
|
211
233
|
throw new CacheUnavailable(message);
|
|
212
234
|
}
|
|
235
|
+
this.connectingSocket = undefined;
|
|
236
|
+
if (this.closed) {
|
|
237
|
+
socket.destroy();
|
|
238
|
+
throw new CacheUnavailable('the cache connection is closed.');
|
|
239
|
+
}
|
|
240
|
+
this.socket = socket;
|
|
213
241
|
return socket;
|
|
214
242
|
}
|
|
215
243
|
read(chunk) {
|
|
216
|
-
this.
|
|
244
|
+
this.chunks.push(chunk);
|
|
245
|
+
this.buffered += chunk.length;
|
|
246
|
+
const buffer = this.chunks.length === 1 ? this.chunks[0] : Buffer.concat(this.chunks, this.buffered);
|
|
247
|
+
let offset = 0;
|
|
217
248
|
for (;;) {
|
|
218
249
|
let reply;
|
|
219
250
|
try {
|
|
220
|
-
reply = decodeReply(
|
|
251
|
+
reply = decodeReply(buffer, offset);
|
|
221
252
|
}
|
|
222
253
|
catch (error) {
|
|
223
254
|
return this.fail(error.message);
|
|
224
255
|
}
|
|
225
|
-
if (!reply)
|
|
256
|
+
if (!reply) {
|
|
257
|
+
if (offset > 0) {
|
|
258
|
+
const rest = buffer.subarray(offset);
|
|
259
|
+
this.chunks = rest.length ? [rest] : [];
|
|
260
|
+
this.buffered = rest.length;
|
|
261
|
+
}
|
|
262
|
+
else if (this.chunks.length > 1) {
|
|
263
|
+
this.chunks = [buffer];
|
|
264
|
+
}
|
|
226
265
|
return;
|
|
227
|
-
|
|
266
|
+
}
|
|
267
|
+
offset = reply.end;
|
|
228
268
|
const waiting = this.waiting.shift();
|
|
229
269
|
if (!waiting)
|
|
230
270
|
continue;
|
package/dist/adapters/connect.js
CHANGED
|
@@ -10,13 +10,16 @@ export function connect(target, options = {}) {
|
|
|
10
10
|
const pool = replicas || options.network ? { ...options.postgres, ...(replicas ? { replicas } : {}), network: options.network } : options.postgres;
|
|
11
11
|
return new PostgresAdapter(target, options.label, options.schemas ?? [], undefined, pool);
|
|
12
12
|
}
|
|
13
|
+
if (options.network) {
|
|
14
|
+
throw new Error('Untrusted connections may only use PostgreSQL (postgres://…).');
|
|
15
|
+
}
|
|
13
16
|
/* Said by name rather than as an unknown scheme: MySQL and MariaDB were never in 1.0, and a config that
|
|
14
17
|
still names one should hear that, not "don't know how to open". */
|
|
15
18
|
if (/^(mysql|mariadb):\/\//i.test(target))
|
|
16
19
|
throw new Error(MYSQL_REFUSAL);
|
|
17
20
|
/* Said before the file is opened: a SQLite file is read where it lies,
|
|
18
21
|
and a copy of it elsewhere would be another database, not a replica. */
|
|
19
|
-
if (replicas && (/^sqlite:/i.test(target) ||
|
|
22
|
+
if (replicas && (/^sqlite:/i.test(target) || !target.includes('://')))
|
|
20
23
|
throw new Error('SQLite has no replicas: a file is read where it is. Remove "replicas".');
|
|
21
24
|
if (/^sqlite:/i.test(target)) {
|
|
22
25
|
const file = target.replace(/^sqlite:(\/\/)?/i, '');
|
|
@@ -24,7 +27,7 @@ export function connect(target, options = {}) {
|
|
|
24
27
|
}
|
|
25
28
|
// A bare path is a SQLite file. This is the common case for the demo and
|
|
26
29
|
// for anyone poking at an app's local database, so it needs no scheme.
|
|
27
|
-
if (
|
|
30
|
+
if (!target.includes('://')) {
|
|
28
31
|
return new SqliteAdapter(target, options.label ?? target);
|
|
29
32
|
}
|
|
30
33
|
/* The scheme, not the target.
|
|
@@ -136,6 +136,7 @@ function requestUntraced(url, asked, policy) {
|
|
|
136
136
|
method: asked.method,
|
|
137
137
|
headers,
|
|
138
138
|
lookup: checkedLookup(policy, refused),
|
|
139
|
+
...(policy.resolver ? { agent: false } : {}),
|
|
139
140
|
timeout: policy.connectTimeoutMs,
|
|
140
141
|
}, (response) => {
|
|
141
142
|
const status = response.statusCode ?? 0;
|
|
@@ -30,6 +30,7 @@ const NEVER = ranges([
|
|
|
30
30
|
]);
|
|
31
31
|
const LINK_LOCAL = ranges([
|
|
32
32
|
['169.254.0.0', 16],
|
|
33
|
+
['100.100.100.200', 32],
|
|
33
34
|
['fe80::', 10],
|
|
34
35
|
['fd00:ec2::254', 128],
|
|
35
36
|
]);
|
|
@@ -62,21 +63,27 @@ function sixteen(address) {
|
|
|
62
63
|
return undefined;
|
|
63
64
|
return all.flatMap((group) => [parseInt(group, 16) >> 8, parseInt(group, 16) & 255]);
|
|
64
65
|
}
|
|
65
|
-
/* The IPv4
|
|
66
|
-
NAT64 (64:ff9b::/96), 6to4 (2002::/16)
|
|
67
|
-
IPv4-mapped spelling (`::ffff:a.b.c.d`)
|
|
66
|
+
/* The IPv4 addresses an IPv6 spelling carries inside it, which is the address a gateway or a tunnel then connects to:
|
|
67
|
+
NAT64 (64:ff9b::/96), 6to4 (2002::/16), SIIT (::ffff:0:0/96), Teredo (2001:0000::/32, server + XOR-inverted client),
|
|
68
|
+
and the long-retired IPv4-compatible form (::/96). BlockList already reads the IPv4-mapped spelling (`::ffff:a.b.c.d`)
|
|
69
|
+
by its IPv4 rules. */
|
|
68
70
|
function embedded(address) {
|
|
69
71
|
const bytes = sixteen(address);
|
|
70
72
|
if (!bytes)
|
|
71
|
-
return
|
|
73
|
+
return [];
|
|
72
74
|
const v4 = (at) => bytes.slice(at, at + 4).join('.');
|
|
73
75
|
if (bytes[0] === 0x20 && bytes[1] === 0x02)
|
|
74
|
-
return v4(2);
|
|
76
|
+
return [v4(2)];
|
|
77
|
+
if (bytes[0] === 0x20 && bytes[1] === 0x01 && bytes[2] === 0x00 && bytes[3] === 0x00) {
|
|
78
|
+
return [v4(4), [bytes[12] ^ 0xff, bytes[13] ^ 0xff, bytes[14] ^ 0xff, bytes[15] ^ 0xff].join('.')];
|
|
79
|
+
}
|
|
75
80
|
if (bytes.slice(0, 12).every((byte, index) => byte === [0, 0x64, 0xff, 0x9b, 0, 0, 0, 0, 0, 0, 0, 0][index]))
|
|
76
|
-
return v4(12);
|
|
81
|
+
return [v4(12)];
|
|
82
|
+
if (bytes.slice(0, 12).every((byte, index) => byte === [0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0][index]))
|
|
83
|
+
return [v4(12)];
|
|
77
84
|
if (bytes.slice(0, 12).every((byte) => byte === 0))
|
|
78
|
-
return v4(12);
|
|
79
|
-
return
|
|
85
|
+
return [v4(12)];
|
|
86
|
+
return [];
|
|
80
87
|
}
|
|
81
88
|
function reachable(address, policy) {
|
|
82
89
|
const type = isIP(address) === 6 ? 'ipv6' : 'ipv4';
|
|
@@ -87,8 +94,13 @@ function reachable(address, policy) {
|
|
|
87
94
|
if (!policy.allowPrivate && PRIVATE.check(address, type))
|
|
88
95
|
return false;
|
|
89
96
|
// An address that wraps another is as reachable as the one inside it.
|
|
90
|
-
|
|
91
|
-
|
|
97
|
+
if (type === 'ipv6') {
|
|
98
|
+
for (const inside of embedded(address)) {
|
|
99
|
+
if (!reachable(inside, policy))
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return true;
|
|
92
104
|
}
|
|
93
105
|
/**
|
|
94
106
|
* A host written as an address is never looked up, so it is checked before
|
|
@@ -148,6 +148,10 @@ export declare class PgConnectionManager {
|
|
|
148
148
|
private readonly limitSaid;
|
|
149
149
|
/** Gives back what this manager told the connection budget it may hold (connectionBudget.ts). */
|
|
150
150
|
private readonly releaseBudget;
|
|
151
|
+
/** Bounded concurrency for best-effort pg_cancel_backend helper connections. */
|
|
152
|
+
private cancelsInFlight;
|
|
153
|
+
private readonly cancelQueue;
|
|
154
|
+
private readonly pendingCancels;
|
|
151
155
|
constructor(connectionString: string, onLoad?: (pg: unknown) => void, load?: PgPoolLoader, options?: PgConnectionOptions);
|
|
152
156
|
/**
|
|
153
157
|
* A handle for single statements on the reader pool. Each statement leases
|
|
@@ -192,7 +196,9 @@ export declare class PgConnectionManager {
|
|
|
192
196
|
/**
|
|
193
197
|
* A connection, or PoolBusy once what is left of the queue-wait budget, `waitMs`, is spent. The driver
|
|
194
198
|
* cannot withdraw a request it has queued, so one that arrives late is given
|
|
195
|
-
* straight back to the pool rather than leaked
|
|
199
|
+
* straight back to the pool rather than leaked — and its admission slot stays
|
|
200
|
+
* held until that late connect settles so concurrent timeouts cannot pile
|
|
201
|
+
* phantom waiters onto pg-pool.
|
|
196
202
|
*/
|
|
197
203
|
private connectWithin;
|
|
198
204
|
/**
|
|
@@ -213,10 +219,13 @@ export declare class PgConnectionManager {
|
|
|
213
219
|
/**
|
|
214
220
|
* Stop what backend `pid` is running, with pg_cancel_backend from a
|
|
215
221
|
* connection of its own: the pool's may all be busy, one of them with the
|
|
216
|
-
* statement being stopped. Best effort —
|
|
217
|
-
*
|
|
222
|
+
* statement being stopped. Best effort — bounded to at most two concurrent
|
|
223
|
+
* helper connections so a cancel burst cannot exhaust the database's
|
|
224
|
+
* connection slots; anything beyond the cap leaves the statement to its
|
|
225
|
+
* statement_timeout.
|
|
218
226
|
*/
|
|
219
227
|
private cancel;
|
|
228
|
+
private runCancel;
|
|
220
229
|
private endPools;
|
|
221
230
|
}
|
|
222
231
|
/** The pool shape a holder of a shared manager names: the sizes, and whether its readers pipeline. */
|
|
@@ -160,6 +160,10 @@ export class PgConnectionManager {
|
|
|
160
160
|
limitSaid = {};
|
|
161
161
|
/** Gives back what this manager told the connection budget it may hold (connectionBudget.ts). */
|
|
162
162
|
releaseBudget;
|
|
163
|
+
/** Bounded concurrency for best-effort pg_cancel_backend helper connections. */
|
|
164
|
+
cancelsInFlight = 0;
|
|
165
|
+
cancelQueue = [];
|
|
166
|
+
pendingCancels = new Set();
|
|
163
167
|
constructor(connectionString, onLoad = () => { }, load = loadPg, options = {}) {
|
|
164
168
|
this.connectionString = connectionString;
|
|
165
169
|
this.onLoad = onLoad;
|
|
@@ -440,7 +444,11 @@ export class PgConnectionManager {
|
|
|
440
444
|
if (bounded) {
|
|
441
445
|
const deadline = performance.now() + this.queueWaitMs;
|
|
442
446
|
place = await this.admissions[role].enter(this.queueWaitMs);
|
|
443
|
-
client = await this.connectWithin(pool, role, Math.max(1, deadline - performance.now()))
|
|
447
|
+
client = await this.connectWithin(pool, role, Math.max(1, deadline - performance.now()), () => {
|
|
448
|
+
const held = place;
|
|
449
|
+
place = undefined;
|
|
450
|
+
return held;
|
|
451
|
+
});
|
|
444
452
|
}
|
|
445
453
|
else
|
|
446
454
|
client = await this.connectHeld(pool, role);
|
|
@@ -627,25 +635,37 @@ export class PgConnectionManager {
|
|
|
627
635
|
/**
|
|
628
636
|
* A connection, or PoolBusy once what is left of the queue-wait budget, `waitMs`, is spent. The driver
|
|
629
637
|
* cannot withdraw a request it has queued, so one that arrives late is given
|
|
630
|
-
* straight back to the pool rather than leaked
|
|
638
|
+
* straight back to the pool rather than leaked — and its admission slot stays
|
|
639
|
+
* held until that late connect settles so concurrent timeouts cannot pile
|
|
640
|
+
* phantom waiters onto pg-pool.
|
|
631
641
|
*/
|
|
632
|
-
connectWithin(pool, role, waitMs) {
|
|
642
|
+
connectWithin(pool, role, waitMs, claimPlace) {
|
|
633
643
|
const connecting = this.connectHeld(pool, role);
|
|
634
644
|
return new Promise((resolve, reject) => {
|
|
635
645
|
let over = false;
|
|
646
|
+
let latePlace;
|
|
636
647
|
const timer = setTimeout(() => {
|
|
637
648
|
over = true;
|
|
649
|
+
latePlace = claimPlace?.();
|
|
638
650
|
reject(new PoolBusy(role, 'queue_timeout'));
|
|
639
651
|
}, waitMs);
|
|
640
652
|
connecting.then((client) => {
|
|
641
653
|
clearTimeout(timer);
|
|
642
|
-
if (over)
|
|
643
|
-
|
|
654
|
+
if (over) {
|
|
655
|
+
try {
|
|
656
|
+
client.release();
|
|
657
|
+
}
|
|
658
|
+
finally {
|
|
659
|
+
latePlace?.();
|
|
660
|
+
}
|
|
661
|
+
}
|
|
644
662
|
else
|
|
645
663
|
resolve(client);
|
|
646
664
|
}, (error) => {
|
|
647
665
|
clearTimeout(timer);
|
|
648
|
-
if (
|
|
666
|
+
if (over)
|
|
667
|
+
latePlace?.();
|
|
668
|
+
else
|
|
649
669
|
reject(error);
|
|
650
670
|
});
|
|
651
671
|
});
|
|
@@ -711,12 +731,26 @@ export class PgConnectionManager {
|
|
|
711
731
|
/**
|
|
712
732
|
* Stop what backend `pid` is running, with pg_cancel_backend from a
|
|
713
733
|
* connection of its own: the pool's may all be busy, one of them with the
|
|
714
|
-
* statement being stopped. Best effort —
|
|
715
|
-
*
|
|
734
|
+
* statement being stopped. Best effort — bounded to at most two concurrent
|
|
735
|
+
* helper connections so a cancel burst cannot exhaust the database's
|
|
736
|
+
* connection slots; anything beyond the cap leaves the statement to its
|
|
737
|
+
* statement_timeout.
|
|
716
738
|
*/
|
|
717
739
|
async cancel(pid) {
|
|
718
|
-
if (!pid)
|
|
740
|
+
if (!pid || this.closed || this.pendingCancels.has(pid))
|
|
741
|
+
return;
|
|
742
|
+
if (this.cancelsInFlight >= 2) {
|
|
743
|
+
if (this.cancelQueue.length >= 8)
|
|
744
|
+
return;
|
|
745
|
+
this.pendingCancels.add(pid);
|
|
746
|
+
this.cancelQueue.push(pid);
|
|
719
747
|
return;
|
|
748
|
+
}
|
|
749
|
+
this.pendingCancels.add(pid);
|
|
750
|
+
await this.runCancel(pid);
|
|
751
|
+
}
|
|
752
|
+
async runCancel(pid) {
|
|
753
|
+
this.cancelsInFlight += 1;
|
|
720
754
|
try {
|
|
721
755
|
const pg = (await this.load());
|
|
722
756
|
const Client = pg.Client ?? pg.default?.Client;
|
|
@@ -738,6 +772,13 @@ export class PgConnectionManager {
|
|
|
738
772
|
catch {
|
|
739
773
|
/* The statement's cap still stops it. */
|
|
740
774
|
}
|
|
775
|
+
finally {
|
|
776
|
+
this.pendingCancels.delete(pid);
|
|
777
|
+
this.cancelsInFlight -= 1;
|
|
778
|
+
const next = this.cancelQueue.shift();
|
|
779
|
+
if (next !== undefined && !this.closed)
|
|
780
|
+
void this.runCancel(next);
|
|
781
|
+
}
|
|
741
782
|
}
|
|
742
783
|
async endPools() {
|
|
743
784
|
const opened = await Promise.allSettled([this.reader, this.writer, this.background]);
|
|
@@ -64,6 +64,12 @@ export async function runPostgresSql(connections, writes, request) {
|
|
|
64
64
|
transaction open on a connection that outlives the request. */
|
|
65
65
|
return writes.run('all', () => connections.withWriteClient(async (write) => {
|
|
66
66
|
const started = performance.now();
|
|
67
|
+
/* When the reserved-table guard is active, check the planner's relation
|
|
68
|
+
list for explainable DML before executing, so views or search_path
|
|
69
|
+
aliases pointing at runtime tables are caught before any write lands. */
|
|
70
|
+
if (request.refuseReserved && /^\s*(?:with|insert|update|delete|merge|select|values|table)\b/i.test(text)) {
|
|
71
|
+
await refuseReservedPlan(write, request.text, []);
|
|
72
|
+
}
|
|
67
73
|
// Extended protocol: one statement, as on the read path below.
|
|
68
74
|
const result = await write.query({ text, values: [], rowMode: 'array', queryMode: 'extended' });
|
|
69
75
|
return {
|
|
@@ -18,7 +18,8 @@ export declare function refuseReservedText(sql: string, dialect: Dialect): void;
|
|
|
18
18
|
* The Postgres check before a statement is given a connection: its text,
|
|
19
19
|
* and for a read, whether the planner can list what it reads — the engine
|
|
20
20
|
* half below needs a plan, so a statement the planner cannot explain is
|
|
21
|
-
* refused rather than run unchecked.
|
|
21
|
+
* refused rather than run unchecked. For a write, anonymous `DO` blocks are
|
|
22
|
+
* also refused because they can run dynamic SQL the planner never sees.
|
|
22
23
|
*/
|
|
23
24
|
export declare function refusePostgresReserved(sql: string, write?: boolean): void;
|
|
24
25
|
/**
|
|
@@ -27,9 +27,13 @@ import { looksWrappable, Refusal, trimStatement } from './adapter.js';
|
|
|
27
27
|
schema does. The name is the engine's, not an import of a store: the engine
|
|
28
28
|
hides it in a database the browser opens with no App framework installed. */
|
|
29
29
|
export const RESERVED_SCHEMAS = [RUNTIME_SCHEMA];
|
|
30
|
+
/* Postgres planner statistics catalogs can expose most-common-value samples and
|
|
31
|
+
histograms from reserved runtime/sign-in tables even when those tables are blocked. */
|
|
32
|
+
const STATS_RELATIONS = new Set(['pg_stats', 'pg_statistic', 'pg_statistic_ext_data', 'pg_stats_ext', 'pg_stats_ext_exprs']);
|
|
30
33
|
/** Whether a relation, as the engine names it, is tablewalk's own. */
|
|
31
34
|
export function isReservedRelation(schema, name) {
|
|
32
|
-
|
|
35
|
+
const lower = name.toLowerCase();
|
|
36
|
+
return isRuntimeTable(name) || STATS_RELATIONS.has(lower) || (schema !== undefined && RESERVED_SCHEMAS.includes(schema.toLowerCase()));
|
|
33
37
|
}
|
|
34
38
|
/** The refusal for a statement found to touch `name`, by whichever half found it. */
|
|
35
39
|
export function reservedRefusal(name) {
|
|
@@ -50,15 +54,19 @@ function tokenize(sql, dialect) {
|
|
|
50
54
|
const n = sql.length;
|
|
51
55
|
let i = 0;
|
|
52
56
|
/* A quoted run with its closing character doubled as the escape. A
|
|
53
|
-
backslash escape (Postgres's E'…') can
|
|
54
|
-
|
|
57
|
+
backslash escape (Postgres's E'…') can encode characters in octal/hex/unicode
|
|
58
|
+
which this check does not decode; those fail closed. */
|
|
55
59
|
const quoted = (close, backslash) => {
|
|
56
60
|
let value = '';
|
|
61
|
+
let undecodedEscape = false;
|
|
57
62
|
i += 1;
|
|
58
63
|
while (i < n) {
|
|
59
64
|
const ch = sql[i];
|
|
60
65
|
if (backslash && ch === '\\' && i + 1 < n) {
|
|
61
|
-
|
|
66
|
+
const nextCh = sql[i + 1];
|
|
67
|
+
if (/[0-7xuU]/.test(nextCh))
|
|
68
|
+
undecodedEscape = true;
|
|
69
|
+
value += nextCh;
|
|
62
70
|
i += 2;
|
|
63
71
|
continue;
|
|
64
72
|
}
|
|
@@ -74,7 +82,7 @@ function tokenize(sql, dialect) {
|
|
|
74
82
|
value += ch;
|
|
75
83
|
i += 1;
|
|
76
84
|
}
|
|
77
|
-
return value;
|
|
85
|
+
return { value, undecodedEscape };
|
|
78
86
|
};
|
|
79
87
|
while (i < n) {
|
|
80
88
|
const ch = sql[i];
|
|
@@ -125,7 +133,7 @@ function tokenize(sql, dialect) {
|
|
|
125
133
|
this check does not decode; one without escapes reads as it is. */
|
|
126
134
|
i += 2;
|
|
127
135
|
const close = sql[i];
|
|
128
|
-
const value = quoted(close, false);
|
|
136
|
+
const { value } = quoted(close, false);
|
|
129
137
|
if (value.includes('\\') || /^\s*uescape\b/i.test(sql.slice(i))) {
|
|
130
138
|
return ('This statement spells a name with Unicode escapes (U&"…"), which the reserved-table check does not decode, ' +
|
|
131
139
|
'so it is refused rather than run. Spell the name plainly.');
|
|
@@ -136,22 +144,27 @@ function tokenize(sql, dialect) {
|
|
|
136
144
|
if (ch === "'") {
|
|
137
145
|
/* E'…' is Postgres's backslash-escaped string. */
|
|
138
146
|
const escaped = dialect === 'postgres' && /[eE]$/.test(sql.slice(Math.max(0, i - 1), i));
|
|
139
|
-
|
|
147
|
+
const { value, undecodedEscape } = quoted("'", escaped);
|
|
148
|
+
if (undecodedEscape) {
|
|
149
|
+
return ("This statement uses octal, hex or Unicode escapes in E'…', which the reserved-table check does not decode, " +
|
|
150
|
+
'so it is refused rather than run. Spell the text plainly.');
|
|
151
|
+
}
|
|
152
|
+
tokens.push({ kind: 'text', value });
|
|
140
153
|
continue;
|
|
141
154
|
}
|
|
142
155
|
if (ch === '"') {
|
|
143
|
-
tokens.push({ kind: 'quoted', value: quoted('"', false) });
|
|
156
|
+
tokens.push({ kind: 'quoted', value: quoted('"', false).value });
|
|
144
157
|
continue;
|
|
145
158
|
}
|
|
146
159
|
if (ch === '`' && dialect !== 'postgres') {
|
|
147
|
-
tokens.push({ kind: 'quoted', value: quoted('`', false) });
|
|
160
|
+
tokens.push({ kind: 'quoted', value: quoted('`', false).value });
|
|
148
161
|
continue;
|
|
149
162
|
}
|
|
150
163
|
if (ch === '[' && dialect === 'sqlite') {
|
|
151
|
-
tokens.push({ kind: 'quoted', value: quoted(']', false) });
|
|
164
|
+
tokens.push({ kind: 'quoted', value: quoted(']', false).value });
|
|
152
165
|
continue;
|
|
153
166
|
}
|
|
154
|
-
const word = /^[A-Za-z_
|
|
167
|
+
const word = /^[A-Za-z_\u0080-\uffff][\w$\u0080-\uffff]*/.exec(sql.slice(i))?.[0];
|
|
155
168
|
if (word) {
|
|
156
169
|
tokens.push({ kind: 'word', value: word });
|
|
157
170
|
i += word.length;
|
|
@@ -164,6 +177,7 @@ function tokenize(sql, dialect) {
|
|
|
164
177
|
// A match in text (a comment, a string)
|
|
165
178
|
// takes the rest of the name, so the refusal names the table, not the prefix.
|
|
166
179
|
const RESERVED_TEXT = new RegExp(`(?:${RUNTIME_TABLE_PREFIXES.join('|')})[\\w$]*|\\b(?:${RESERVED_SCHEMAS.join('|')})\\b`, 'i');
|
|
180
|
+
const STATS_TEXT = /\b(?:pg_stats|pg_statistic|pg_statistic_ext_data|pg_stats_ext|pg_stats_ext_exprs)\b/i;
|
|
167
181
|
/**
|
|
168
182
|
* Why this statement must not run under the guard, or null when nothing in
|
|
169
183
|
* its text reaches a reserved relation.
|
|
@@ -174,13 +188,14 @@ export function reservedInText(sql, dialect) {
|
|
|
174
188
|
return tokens;
|
|
175
189
|
for (const token of tokens) {
|
|
176
190
|
if (token.kind === 'text') {
|
|
177
|
-
const found = RESERVED_TEXT.exec(token.value);
|
|
191
|
+
const found = RESERVED_TEXT.exec(token.value) ?? (dialect === 'postgres' ? STATS_TEXT.exec(token.value) : null);
|
|
178
192
|
if (found)
|
|
179
193
|
return reservedRefusal(found[0]).message;
|
|
180
194
|
continue;
|
|
181
195
|
}
|
|
182
196
|
const name = token.value;
|
|
183
|
-
|
|
197
|
+
const lower = name.toLowerCase();
|
|
198
|
+
if (isRuntimeTable(name) || (dialect === 'postgres' && (RESERVED_SCHEMAS.includes(lower) || STATS_RELATIONS.has(lower)))) {
|
|
184
199
|
return reservedRefusal(name).message;
|
|
185
200
|
}
|
|
186
201
|
if (dialect === 'postgres' && COMPUTED_RELATION.test(name)) {
|
|
@@ -201,11 +216,20 @@ export function refuseReservedText(sql, dialect) {
|
|
|
201
216
|
* The Postgres check before a statement is given a connection: its text,
|
|
202
217
|
* and for a read, whether the planner can list what it reads — the engine
|
|
203
218
|
* half below needs a plan, so a statement the planner cannot explain is
|
|
204
|
-
* refused rather than run unchecked.
|
|
219
|
+
* refused rather than run unchecked. For a write, anonymous `DO` blocks are
|
|
220
|
+
* also refused because they can run dynamic SQL the planner never sees.
|
|
205
221
|
*/
|
|
206
222
|
export function refusePostgresReserved(sql, write = false) {
|
|
207
223
|
refuseReservedText(sql, 'postgres');
|
|
208
|
-
if (
|
|
224
|
+
if (write) {
|
|
225
|
+
const tokens = tokenize(sql, 'postgres');
|
|
226
|
+
const first = Array.isArray(tokens) ? tokens.find((token) => token.kind !== 'text') : undefined;
|
|
227
|
+
if (first?.kind === 'word' && /^do$/i.test(first.value)) {
|
|
228
|
+
throw new Refusal('The reserved-table check refuses DO blocks because procedural code can execute dynamic SQL that cannot be checked.');
|
|
229
|
+
}
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (!looksWrappable(sql)) {
|
|
209
233
|
throw new Refusal("The reserved-table check reads the planner's list of what a statement scans, so it " +
|
|
210
234
|
'runs the statements the planner can explain: SELECT, WITH, TABLE and VALUES.');
|
|
211
235
|
}
|