@mistriwork/engine 1.0.0-rc.69 → 1.0.0-rc.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/cache/cacheStore.d.ts +1 -12
- package/dist/adapters/cache/cacheStore.js +23 -1
- package/dist/adapters/cache/resp.d.ts +3 -1
- package/dist/adapters/cache/resp.js +48 -8
- package/dist/adapters/connect.js +5 -2
- package/dist/adapters/http/egress.js +1 -0
- package/dist/adapters/net/networkPolicy.js +22 -10
- package/dist/adapters/postgresPool.d.ts +12 -3
- package/dist/adapters/postgresPool.js +50 -9
- package/dist/adapters/postgresSql.js +6 -0
- package/dist/adapters/reservedSql.d.ts +2 -1
- package/dist/adapters/reservedSql.js +39 -15
- package/dist/adapters/sqliteStatementProcess.js +99 -49
- package/dist/server/clientHttp.js +8 -4
- package/dist/server/columnGrants.js +4 -0
- package/dist/server/http/context.d.ts +3 -1
- package/dist/server/http/dataRoutes.js +1 -0
- package/dist/server/http/exportRoute.d.ts +3 -0
- package/dist/server/http/exportRoute.js +3 -1
- package/dist/server/http/rateLimit.js +4 -2
- package/dist/server/http/trust.d.ts +0 -14
- package/dist/server/http/trust.js +24 -8
- package/dist/shared/frameworkWords.d.ts +1 -0
- package/dist/shared/frameworkWords.js +1 -0
- package/dist/shared/frameworkWordsEs.js +1 -0
- package/package.json +1 -1
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Somewhere to keep a value for a while: in this process by default, in a
|
|
3
|
-
* Valkey cache when the config names one, so every instance of a deployment
|
|
4
|
-
* shares it.
|
|
5
|
-
*
|
|
6
|
-
* Never a source of truth. Everything here is a copy or a count that is
|
|
7
|
-
* allowed to vanish — a cached API answer, a rate-limit window — so every
|
|
8
|
-
* write carries a time to live (there is no "forever"), a read that cannot
|
|
9
|
-
* reach the cache is a miss, and a write that cannot is dropped. The one
|
|
10
|
-
* caller that must know the difference, the rate limiter, is told through
|
|
11
|
-
* `incr`, which throws `CacheUnavailable` rather than inventing a count.
|
|
12
|
-
*/
|
|
13
1
|
import { type RespClient } from './resp.js';
|
|
14
2
|
export { CacheUnavailable } from './resp.js';
|
|
15
3
|
export interface CacheStore {
|
|
@@ -61,6 +49,7 @@ export declare class ValkeyCacheStore implements CacheStore {
|
|
|
61
49
|
private readonly client;
|
|
62
50
|
private readonly onUnavailable;
|
|
63
51
|
readonly kind: 'valkey';
|
|
52
|
+
private scriptLoaded;
|
|
64
53
|
constructor(client: RespClient, onUnavailable?: (message: string) => void);
|
|
65
54
|
get(key: string): Promise<string | undefined>;
|
|
66
55
|
set(key: string, value: string, ttlMs: number): Promise<void>;
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* caller that must know the difference, the rate limiter, is told through
|
|
11
11
|
* `incr`, which throws `CacheUnavailable` rather than inventing a count.
|
|
12
12
|
*/
|
|
13
|
+
import { createHash } from 'node:crypto';
|
|
13
14
|
import { CacheUnavailable, RespReplyError } from './resp.js';
|
|
14
15
|
export { CacheUnavailable } from './resp.js';
|
|
15
16
|
/* Thirty days: a copy kept longer than that is a copy nobody meant to keep. */
|
|
@@ -92,6 +93,7 @@ if n == tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[1]) end
|
|
|
92
93
|
local t = redis.call('PTTL', KEYS[1])
|
|
93
94
|
if t < 0 then redis.call('PEXPIRE', KEYS[1], ARGV[1]) t = tonumber(ARGV[1]) end
|
|
94
95
|
return {n, t}`;
|
|
96
|
+
const COUNT_IN_WINDOW_SHA = createHash('sha1').update(COUNT_IN_WINDOW).digest('hex');
|
|
95
97
|
/**
|
|
96
98
|
* A Valkey cache shared by every instance. `onUnavailable` hears each
|
|
97
99
|
* outage's reason — the server logs it once — while reads carry on as misses.
|
|
@@ -100,6 +102,7 @@ export class ValkeyCacheStore {
|
|
|
100
102
|
client;
|
|
101
103
|
onUnavailable;
|
|
102
104
|
kind = 'valkey';
|
|
105
|
+
scriptLoaded = false;
|
|
103
106
|
constructor(client, onUnavailable = () => { }) {
|
|
104
107
|
this.client = client;
|
|
105
108
|
this.onUnavailable = onUnavailable;
|
|
@@ -136,7 +139,26 @@ export class ValkeyCacheStore {
|
|
|
136
139
|
assertBy(by);
|
|
137
140
|
let reply;
|
|
138
141
|
try {
|
|
139
|
-
|
|
142
|
+
if (this.scriptLoaded) {
|
|
143
|
+
try {
|
|
144
|
+
reply = await this.client.command('EVALSHA', COUNT_IN_WINDOW_SHA, 1, key, ttlMs, by);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
/* Valkey restarts clear its script cache; fall back to EVAL once to reload it. */
|
|
148
|
+
if (error instanceof RespReplyError && /NOSCRIPT/i.test(error.message)) {
|
|
149
|
+
this.scriptLoaded = false;
|
|
150
|
+
reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs, by);
|
|
151
|
+
this.scriptLoaded = true;
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs, by);
|
|
160
|
+
this.scriptLoaded = true;
|
|
161
|
+
}
|
|
140
162
|
}
|
|
141
163
|
catch (error) {
|
|
142
164
|
this.unavailable(error);
|
|
@@ -32,8 +32,10 @@ export declare function decodeReply(buffer: Buffer, start?: number): {
|
|
|
32
32
|
export declare class RespClient {
|
|
33
33
|
private readonly options;
|
|
34
34
|
private socket?;
|
|
35
|
+
private connectingSocket?;
|
|
35
36
|
private opening?;
|
|
36
|
-
private
|
|
37
|
+
private chunks;
|
|
38
|
+
private buffered;
|
|
37
39
|
private readonly waiting;
|
|
38
40
|
private quietUntil;
|
|
39
41
|
private lastFailure;
|
|
@@ -86,11 +86,14 @@ export function decodeReply(buffer, start = 0) {
|
|
|
86
86
|
throw new CacheUnavailable(`the cache answered something that is not RESP ("${type}").`);
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
|
+
const MAX_WAITING = 1024;
|
|
89
90
|
export class RespClient {
|
|
90
91
|
options;
|
|
91
92
|
socket;
|
|
93
|
+
connectingSocket;
|
|
92
94
|
opening;
|
|
93
|
-
|
|
95
|
+
chunks = [];
|
|
96
|
+
buffered = 0;
|
|
94
97
|
waiting = [];
|
|
95
98
|
quietUntil = 0;
|
|
96
99
|
lastFailure = '';
|
|
@@ -107,11 +110,16 @@ export class RespClient {
|
|
|
107
110
|
throw new CacheUnavailable('the cache connection is closed.');
|
|
108
111
|
if (Date.now() < this.quietUntil)
|
|
109
112
|
throw new CacheUnavailable(this.lastFailure);
|
|
113
|
+
if (this.waiting.length >= MAX_WAITING)
|
|
114
|
+
throw new CacheUnavailable(`${this.where}: too many commands waiting for a reply.`);
|
|
110
115
|
const socket = await this.connected();
|
|
111
116
|
return this.write(socket, args);
|
|
112
117
|
}
|
|
113
118
|
async close() {
|
|
114
119
|
this.closed = true;
|
|
120
|
+
const connecting = this.connectingSocket;
|
|
121
|
+
this.connectingSocket = undefined;
|
|
122
|
+
connecting?.destroy();
|
|
115
123
|
const socket = this.socket;
|
|
116
124
|
this.socket = undefined;
|
|
117
125
|
if (socket) {
|
|
@@ -125,6 +133,9 @@ export class RespClient {
|
|
|
125
133
|
}
|
|
126
134
|
}
|
|
127
135
|
write(socket, args) {
|
|
136
|
+
if (this.waiting.length >= MAX_WAITING) {
|
|
137
|
+
return Promise.reject(new CacheUnavailable(`${this.where}: too many commands waiting for a reply.`));
|
|
138
|
+
}
|
|
128
139
|
return new Promise((resolve, reject) => {
|
|
129
140
|
const timer = setTimeout(() => this.fail(`${this.where} did not answer within ${(this.options.commandTimeoutMs ?? 1_000) / 1000} s.`), this.options.commandTimeoutMs ?? 1_000);
|
|
130
141
|
this.waiting.push({
|
|
@@ -146,9 +157,14 @@ export class RespClient {
|
|
|
146
157
|
this.lastFailure = message;
|
|
147
158
|
this.quietUntil = Date.now() + (this.options.retryAfterMs ?? 2_000);
|
|
148
159
|
const socket = this.socket;
|
|
160
|
+
const connecting = this.connectingSocket;
|
|
149
161
|
this.socket = undefined;
|
|
150
|
-
this.
|
|
162
|
+
this.connectingSocket = undefined;
|
|
163
|
+
this.chunks = [];
|
|
164
|
+
this.buffered = 0;
|
|
151
165
|
socket?.destroy();
|
|
166
|
+
if (connecting && connecting !== socket)
|
|
167
|
+
connecting.destroy();
|
|
152
168
|
for (const waiting of this.waiting.splice(0))
|
|
153
169
|
waiting.reject(new CacheUnavailable(message));
|
|
154
170
|
}
|
|
@@ -188,14 +204,20 @@ export class RespClient {
|
|
|
188
204
|
this.quietUntil = Date.now() + (this.options.retryAfterMs ?? 2_000);
|
|
189
205
|
throw error;
|
|
190
206
|
});
|
|
207
|
+
if (this.closed) {
|
|
208
|
+
socket.destroy();
|
|
209
|
+
throw new CacheUnavailable('the cache connection is closed.');
|
|
210
|
+
}
|
|
191
211
|
socket.setNoDelay(true);
|
|
192
212
|
socket.on('data', (chunk) => this.read(chunk));
|
|
193
213
|
socket.on('error', (error) => this.fail(`${this.where}: ${error.code ?? 'the connection failed'}.`));
|
|
194
214
|
socket.on('close', () => {
|
|
195
|
-
if (this.socket === socket)
|
|
215
|
+
if (this.socket === socket || this.connectingSocket === socket)
|
|
196
216
|
this.fail(`${this.where} closed the connection.`);
|
|
197
217
|
});
|
|
198
|
-
|
|
218
|
+
/* Hold the socket privately until AUTH and SELECT finish so concurrent callers
|
|
219
|
+
await `this.opening` and can never pipeline commands before authentication. */
|
|
220
|
+
this.connectingSocket = socket;
|
|
199
221
|
try {
|
|
200
222
|
if (this.options.password !== undefined) {
|
|
201
223
|
await this.write(socket, this.options.username ? ['AUTH', this.options.username, this.options.password] : ['AUTH', this.options.password]);
|
|
@@ -210,21 +232,39 @@ export class RespClient {
|
|
|
210
232
|
this.fail(message);
|
|
211
233
|
throw new CacheUnavailable(message);
|
|
212
234
|
}
|
|
235
|
+
this.connectingSocket = undefined;
|
|
236
|
+
if (this.closed) {
|
|
237
|
+
socket.destroy();
|
|
238
|
+
throw new CacheUnavailable('the cache connection is closed.');
|
|
239
|
+
}
|
|
240
|
+
this.socket = socket;
|
|
213
241
|
return socket;
|
|
214
242
|
}
|
|
215
243
|
read(chunk) {
|
|
216
|
-
this.
|
|
244
|
+
this.chunks.push(chunk);
|
|
245
|
+
this.buffered += chunk.length;
|
|
246
|
+
const buffer = this.chunks.length === 1 ? this.chunks[0] : Buffer.concat(this.chunks, this.buffered);
|
|
247
|
+
let offset = 0;
|
|
217
248
|
for (;;) {
|
|
218
249
|
let reply;
|
|
219
250
|
try {
|
|
220
|
-
reply = decodeReply(
|
|
251
|
+
reply = decodeReply(buffer, offset);
|
|
221
252
|
}
|
|
222
253
|
catch (error) {
|
|
223
254
|
return this.fail(error.message);
|
|
224
255
|
}
|
|
225
|
-
if (!reply)
|
|
256
|
+
if (!reply) {
|
|
257
|
+
if (offset > 0) {
|
|
258
|
+
const rest = buffer.subarray(offset);
|
|
259
|
+
this.chunks = rest.length ? [rest] : [];
|
|
260
|
+
this.buffered = rest.length;
|
|
261
|
+
}
|
|
262
|
+
else if (this.chunks.length > 1) {
|
|
263
|
+
this.chunks = [buffer];
|
|
264
|
+
}
|
|
226
265
|
return;
|
|
227
|
-
|
|
266
|
+
}
|
|
267
|
+
offset = reply.end;
|
|
228
268
|
const waiting = this.waiting.shift();
|
|
229
269
|
if (!waiting)
|
|
230
270
|
continue;
|
package/dist/adapters/connect.js
CHANGED
|
@@ -10,13 +10,16 @@ export function connect(target, options = {}) {
|
|
|
10
10
|
const pool = replicas || options.network ? { ...options.postgres, ...(replicas ? { replicas } : {}), network: options.network } : options.postgres;
|
|
11
11
|
return new PostgresAdapter(target, options.label, options.schemas ?? [], undefined, pool);
|
|
12
12
|
}
|
|
13
|
+
if (options.network) {
|
|
14
|
+
throw new Error('Untrusted connections may only use PostgreSQL (postgres://…).');
|
|
15
|
+
}
|
|
13
16
|
/* Said by name rather than as an unknown scheme: MySQL and MariaDB were never in 1.0, and a config that
|
|
14
17
|
still names one should hear that, not "don't know how to open". */
|
|
15
18
|
if (/^(mysql|mariadb):\/\//i.test(target))
|
|
16
19
|
throw new Error(MYSQL_REFUSAL);
|
|
17
20
|
/* Said before the file is opened: a SQLite file is read where it lies,
|
|
18
21
|
and a copy of it elsewhere would be another database, not a replica. */
|
|
19
|
-
if (replicas && (/^sqlite:/i.test(target) ||
|
|
22
|
+
if (replicas && (/^sqlite:/i.test(target) || !target.includes('://')))
|
|
20
23
|
throw new Error('SQLite has no replicas: a file is read where it is. Remove "replicas".');
|
|
21
24
|
if (/^sqlite:/i.test(target)) {
|
|
22
25
|
const file = target.replace(/^sqlite:(\/\/)?/i, '');
|
|
@@ -24,7 +27,7 @@ export function connect(target, options = {}) {
|
|
|
24
27
|
}
|
|
25
28
|
// A bare path is a SQLite file. This is the common case for the demo and
|
|
26
29
|
// for anyone poking at an app's local database, so it needs no scheme.
|
|
27
|
-
if (
|
|
30
|
+
if (!target.includes('://')) {
|
|
28
31
|
return new SqliteAdapter(target, options.label ?? target);
|
|
29
32
|
}
|
|
30
33
|
/* The scheme, not the target.
|
|
@@ -136,6 +136,7 @@ function requestUntraced(url, asked, policy) {
|
|
|
136
136
|
method: asked.method,
|
|
137
137
|
headers,
|
|
138
138
|
lookup: checkedLookup(policy, refused),
|
|
139
|
+
...(policy.resolver ? { agent: false } : {}),
|
|
139
140
|
timeout: policy.connectTimeoutMs,
|
|
140
141
|
}, (response) => {
|
|
141
142
|
const status = response.statusCode ?? 0;
|
|
@@ -30,6 +30,7 @@ const NEVER = ranges([
|
|
|
30
30
|
]);
|
|
31
31
|
const LINK_LOCAL = ranges([
|
|
32
32
|
['169.254.0.0', 16],
|
|
33
|
+
['100.100.100.200', 32],
|
|
33
34
|
['fe80::', 10],
|
|
34
35
|
['fd00:ec2::254', 128],
|
|
35
36
|
]);
|
|
@@ -62,21 +63,27 @@ function sixteen(address) {
|
|
|
62
63
|
return undefined;
|
|
63
64
|
return all.flatMap((group) => [parseInt(group, 16) >> 8, parseInt(group, 16) & 255]);
|
|
64
65
|
}
|
|
65
|
-
/* The IPv4
|
|
66
|
-
NAT64 (64:ff9b::/96), 6to4 (2002::/16)
|
|
67
|
-
IPv4-mapped spelling (`::ffff:a.b.c.d`)
|
|
66
|
+
/* The IPv4 addresses an IPv6 spelling carries inside it, which is the address a gateway or a tunnel then connects to:
|
|
67
|
+
NAT64 (64:ff9b::/96), 6to4 (2002::/16), SIIT (::ffff:0:0/96), Teredo (2001:0000::/32, server + XOR-inverted client),
|
|
68
|
+
and the long-retired IPv4-compatible form (::/96). BlockList already reads the IPv4-mapped spelling (`::ffff:a.b.c.d`)
|
|
69
|
+
by its IPv4 rules. */
|
|
68
70
|
function embedded(address) {
|
|
69
71
|
const bytes = sixteen(address);
|
|
70
72
|
if (!bytes)
|
|
71
|
-
return
|
|
73
|
+
return [];
|
|
72
74
|
const v4 = (at) => bytes.slice(at, at + 4).join('.');
|
|
73
75
|
if (bytes[0] === 0x20 && bytes[1] === 0x02)
|
|
74
|
-
return v4(2);
|
|
76
|
+
return [v4(2)];
|
|
77
|
+
if (bytes[0] === 0x20 && bytes[1] === 0x01 && bytes[2] === 0x00 && bytes[3] === 0x00) {
|
|
78
|
+
return [v4(4), [bytes[12] ^ 0xff, bytes[13] ^ 0xff, bytes[14] ^ 0xff, bytes[15] ^ 0xff].join('.')];
|
|
79
|
+
}
|
|
75
80
|
if (bytes.slice(0, 12).every((byte, index) => byte === [0, 0x64, 0xff, 0x9b, 0, 0, 0, 0, 0, 0, 0, 0][index]))
|
|
76
|
-
return v4(12);
|
|
81
|
+
return [v4(12)];
|
|
82
|
+
if (bytes.slice(0, 12).every((byte, index) => byte === [0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0][index]))
|
|
83
|
+
return [v4(12)];
|
|
77
84
|
if (bytes.slice(0, 12).every((byte) => byte === 0))
|
|
78
|
-
return v4(12);
|
|
79
|
-
return
|
|
85
|
+
return [v4(12)];
|
|
86
|
+
return [];
|
|
80
87
|
}
|
|
81
88
|
function reachable(address, policy) {
|
|
82
89
|
const type = isIP(address) === 6 ? 'ipv6' : 'ipv4';
|
|
@@ -87,8 +94,13 @@ function reachable(address, policy) {
|
|
|
87
94
|
if (!policy.allowPrivate && PRIVATE.check(address, type))
|
|
88
95
|
return false;
|
|
89
96
|
// An address that wraps another is as reachable as the one inside it.
|
|
90
|
-
|
|
91
|
-
|
|
97
|
+
if (type === 'ipv6') {
|
|
98
|
+
for (const inside of embedded(address)) {
|
|
99
|
+
if (!reachable(inside, policy))
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return true;
|
|
92
104
|
}
|
|
93
105
|
/**
|
|
94
106
|
* A host written as an address is never looked up, so it is checked before
|
|
@@ -148,6 +148,10 @@ export declare class PgConnectionManager {
|
|
|
148
148
|
private readonly limitSaid;
|
|
149
149
|
/** Gives back what this manager told the connection budget it may hold (connectionBudget.ts). */
|
|
150
150
|
private readonly releaseBudget;
|
|
151
|
+
/** Bounded concurrency for best-effort pg_cancel_backend helper connections. */
|
|
152
|
+
private cancelsInFlight;
|
|
153
|
+
private readonly cancelQueue;
|
|
154
|
+
private readonly pendingCancels;
|
|
151
155
|
constructor(connectionString: string, onLoad?: (pg: unknown) => void, load?: PgPoolLoader, options?: PgConnectionOptions);
|
|
152
156
|
/**
|
|
153
157
|
* A handle for single statements on the reader pool. Each statement leases
|
|
@@ -192,7 +196,9 @@ export declare class PgConnectionManager {
|
|
|
192
196
|
/**
|
|
193
197
|
* A connection, or PoolBusy once what is left of the queue-wait budget, `waitMs`, is spent. The driver
|
|
194
198
|
* cannot withdraw a request it has queued, so one that arrives late is given
|
|
195
|
-
* straight back to the pool rather than leaked
|
|
199
|
+
* straight back to the pool rather than leaked — and its admission slot stays
|
|
200
|
+
* held until that late connect settles so concurrent timeouts cannot pile
|
|
201
|
+
* phantom waiters onto pg-pool.
|
|
196
202
|
*/
|
|
197
203
|
private connectWithin;
|
|
198
204
|
/**
|
|
@@ -213,10 +219,13 @@ export declare class PgConnectionManager {
|
|
|
213
219
|
/**
|
|
214
220
|
* Stop what backend `pid` is running, with pg_cancel_backend from a
|
|
215
221
|
* connection of its own: the pool's may all be busy, one of them with the
|
|
216
|
-
* statement being stopped. Best effort —
|
|
217
|
-
*
|
|
222
|
+
* statement being stopped. Best effort — bounded to at most two concurrent
|
|
223
|
+
* helper connections so a cancel burst cannot exhaust the database's
|
|
224
|
+
* connection slots; anything beyond the cap leaves the statement to its
|
|
225
|
+
* statement_timeout.
|
|
218
226
|
*/
|
|
219
227
|
private cancel;
|
|
228
|
+
private runCancel;
|
|
220
229
|
private endPools;
|
|
221
230
|
}
|
|
222
231
|
/** The pool shape a holder of a shared manager names: the sizes, and whether its readers pipeline. */
|
|
@@ -160,6 +160,10 @@ export class PgConnectionManager {
|
|
|
160
160
|
limitSaid = {};
|
|
161
161
|
/** Gives back what this manager told the connection budget it may hold (connectionBudget.ts). */
|
|
162
162
|
releaseBudget;
|
|
163
|
+
/** Bounded concurrency for best-effort pg_cancel_backend helper connections. */
|
|
164
|
+
cancelsInFlight = 0;
|
|
165
|
+
cancelQueue = [];
|
|
166
|
+
pendingCancels = new Set();
|
|
163
167
|
constructor(connectionString, onLoad = () => { }, load = loadPg, options = {}) {
|
|
164
168
|
this.connectionString = connectionString;
|
|
165
169
|
this.onLoad = onLoad;
|
|
@@ -440,7 +444,11 @@ export class PgConnectionManager {
|
|
|
440
444
|
if (bounded) {
|
|
441
445
|
const deadline = performance.now() + this.queueWaitMs;
|
|
442
446
|
place = await this.admissions[role].enter(this.queueWaitMs);
|
|
443
|
-
client = await this.connectWithin(pool, role, Math.max(1, deadline - performance.now()))
|
|
447
|
+
client = await this.connectWithin(pool, role, Math.max(1, deadline - performance.now()), () => {
|
|
448
|
+
const held = place;
|
|
449
|
+
place = undefined;
|
|
450
|
+
return held;
|
|
451
|
+
});
|
|
444
452
|
}
|
|
445
453
|
else
|
|
446
454
|
client = await this.connectHeld(pool, role);
|
|
@@ -627,25 +635,37 @@ export class PgConnectionManager {
|
|
|
627
635
|
/**
|
|
628
636
|
* A connection, or PoolBusy once what is left of the queue-wait budget, `waitMs`, is spent. The driver
|
|
629
637
|
* cannot withdraw a request it has queued, so one that arrives late is given
|
|
630
|
-
* straight back to the pool rather than leaked
|
|
638
|
+
* straight back to the pool rather than leaked — and its admission slot stays
|
|
639
|
+
* held until that late connect settles so concurrent timeouts cannot pile
|
|
640
|
+
* phantom waiters onto pg-pool.
|
|
631
641
|
*/
|
|
632
|
-
connectWithin(pool, role, waitMs) {
|
|
642
|
+
connectWithin(pool, role, waitMs, claimPlace) {
|
|
633
643
|
const connecting = this.connectHeld(pool, role);
|
|
634
644
|
return new Promise((resolve, reject) => {
|
|
635
645
|
let over = false;
|
|
646
|
+
let latePlace;
|
|
636
647
|
const timer = setTimeout(() => {
|
|
637
648
|
over = true;
|
|
649
|
+
latePlace = claimPlace?.();
|
|
638
650
|
reject(new PoolBusy(role, 'queue_timeout'));
|
|
639
651
|
}, waitMs);
|
|
640
652
|
connecting.then((client) => {
|
|
641
653
|
clearTimeout(timer);
|
|
642
|
-
if (over)
|
|
643
|
-
|
|
654
|
+
if (over) {
|
|
655
|
+
try {
|
|
656
|
+
client.release();
|
|
657
|
+
}
|
|
658
|
+
finally {
|
|
659
|
+
latePlace?.();
|
|
660
|
+
}
|
|
661
|
+
}
|
|
644
662
|
else
|
|
645
663
|
resolve(client);
|
|
646
664
|
}, (error) => {
|
|
647
665
|
clearTimeout(timer);
|
|
648
|
-
if (
|
|
666
|
+
if (over)
|
|
667
|
+
latePlace?.();
|
|
668
|
+
else
|
|
649
669
|
reject(error);
|
|
650
670
|
});
|
|
651
671
|
});
|
|
@@ -711,12 +731,26 @@ export class PgConnectionManager {
|
|
|
711
731
|
/**
|
|
712
732
|
* Stop what backend `pid` is running, with pg_cancel_backend from a
|
|
713
733
|
* connection of its own: the pool's may all be busy, one of them with the
|
|
714
|
-
* statement being stopped. Best effort —
|
|
715
|
-
*
|
|
734
|
+
* statement being stopped. Best effort — bounded to at most two concurrent
|
|
735
|
+
* helper connections so a cancel burst cannot exhaust the database's
|
|
736
|
+
* connection slots; anything beyond the cap leaves the statement to its
|
|
737
|
+
* statement_timeout.
|
|
716
738
|
*/
|
|
717
739
|
async cancel(pid) {
|
|
718
|
-
if (!pid)
|
|
740
|
+
if (!pid || this.closed || this.pendingCancels.has(pid))
|
|
741
|
+
return;
|
|
742
|
+
if (this.cancelsInFlight >= 2) {
|
|
743
|
+
if (this.cancelQueue.length >= 8)
|
|
744
|
+
return;
|
|
745
|
+
this.pendingCancels.add(pid);
|
|
746
|
+
this.cancelQueue.push(pid);
|
|
719
747
|
return;
|
|
748
|
+
}
|
|
749
|
+
this.pendingCancels.add(pid);
|
|
750
|
+
await this.runCancel(pid);
|
|
751
|
+
}
|
|
752
|
+
async runCancel(pid) {
|
|
753
|
+
this.cancelsInFlight += 1;
|
|
720
754
|
try {
|
|
721
755
|
const pg = (await this.load());
|
|
722
756
|
const Client = pg.Client ?? pg.default?.Client;
|
|
@@ -738,6 +772,13 @@ export class PgConnectionManager {
|
|
|
738
772
|
catch {
|
|
739
773
|
/* The statement's cap still stops it. */
|
|
740
774
|
}
|
|
775
|
+
finally {
|
|
776
|
+
this.pendingCancels.delete(pid);
|
|
777
|
+
this.cancelsInFlight -= 1;
|
|
778
|
+
const next = this.cancelQueue.shift();
|
|
779
|
+
if (next !== undefined && !this.closed)
|
|
780
|
+
void this.runCancel(next);
|
|
781
|
+
}
|
|
741
782
|
}
|
|
742
783
|
async endPools() {
|
|
743
784
|
const opened = await Promise.allSettled([this.reader, this.writer, this.background]);
|
|
@@ -64,6 +64,12 @@ export async function runPostgresSql(connections, writes, request) {
|
|
|
64
64
|
transaction open on a connection that outlives the request. */
|
|
65
65
|
return writes.run('all', () => connections.withWriteClient(async (write) => {
|
|
66
66
|
const started = performance.now();
|
|
67
|
+
/* When the reserved-table guard is active, check the planner's relation
|
|
68
|
+
list for explainable DML before executing, so views or search_path
|
|
69
|
+
aliases pointing at runtime tables are caught before any write lands. */
|
|
70
|
+
if (request.refuseReserved && /^\s*(?:with|insert|update|delete|merge|select|values|table)\b/i.test(text)) {
|
|
71
|
+
await refuseReservedPlan(write, request.text, []);
|
|
72
|
+
}
|
|
67
73
|
// Extended protocol: one statement, as on the read path below.
|
|
68
74
|
const result = await write.query({ text, values: [], rowMode: 'array', queryMode: 'extended' });
|
|
69
75
|
return {
|
|
@@ -18,7 +18,8 @@ export declare function refuseReservedText(sql: string, dialect: Dialect): void;
|
|
|
18
18
|
* The Postgres check before a statement is given a connection: its text,
|
|
19
19
|
* and for a read, whether the planner can list what it reads — the engine
|
|
20
20
|
* half below needs a plan, so a statement the planner cannot explain is
|
|
21
|
-
* refused rather than run unchecked.
|
|
21
|
+
* refused rather than run unchecked. For a write, anonymous `DO` blocks are
|
|
22
|
+
* also refused because they can run dynamic SQL the planner never sees.
|
|
22
23
|
*/
|
|
23
24
|
export declare function refusePostgresReserved(sql: string, write?: boolean): void;
|
|
24
25
|
/**
|
|
@@ -27,9 +27,13 @@ import { looksWrappable, Refusal, trimStatement } from './adapter.js';
|
|
|
27
27
|
schema does. The name is the engine's, not an import of a store: the engine
|
|
28
28
|
hides it in a database the browser opens with no App framework installed. */
|
|
29
29
|
export const RESERVED_SCHEMAS = [RUNTIME_SCHEMA];
|
|
30
|
+
/* Postgres planner statistics catalogs can expose most-common-value samples and
|
|
31
|
+
histograms from reserved runtime/sign-in tables even when those tables are blocked. */
|
|
32
|
+
const STATS_RELATIONS = new Set(['pg_stats', 'pg_statistic', 'pg_statistic_ext_data', 'pg_stats_ext', 'pg_stats_ext_exprs']);
|
|
30
33
|
/** Whether a relation, as the engine names it, is tablewalk's own. */
|
|
31
34
|
export function isReservedRelation(schema, name) {
|
|
32
|
-
|
|
35
|
+
const lower = name.toLowerCase();
|
|
36
|
+
return isRuntimeTable(name) || STATS_RELATIONS.has(lower) || (schema !== undefined && RESERVED_SCHEMAS.includes(schema.toLowerCase()));
|
|
33
37
|
}
|
|
34
38
|
/** The refusal for a statement found to touch `name`, by whichever half found it. */
|
|
35
39
|
export function reservedRefusal(name) {
|
|
@@ -50,15 +54,19 @@ function tokenize(sql, dialect) {
|
|
|
50
54
|
const n = sql.length;
|
|
51
55
|
let i = 0;
|
|
52
56
|
/* A quoted run with its closing character doubled as the escape. A
|
|
53
|
-
backslash escape (Postgres's E'…') can
|
|
54
|
-
|
|
57
|
+
backslash escape (Postgres's E'…') can encode characters in octal/hex/unicode
|
|
58
|
+
which this check does not decode; those fail closed. */
|
|
55
59
|
const quoted = (close, backslash) => {
|
|
56
60
|
let value = '';
|
|
61
|
+
let undecodedEscape = false;
|
|
57
62
|
i += 1;
|
|
58
63
|
while (i < n) {
|
|
59
64
|
const ch = sql[i];
|
|
60
65
|
if (backslash && ch === '\\' && i + 1 < n) {
|
|
61
|
-
|
|
66
|
+
const nextCh = sql[i + 1];
|
|
67
|
+
if (/[0-7xuU]/.test(nextCh))
|
|
68
|
+
undecodedEscape = true;
|
|
69
|
+
value += nextCh;
|
|
62
70
|
i += 2;
|
|
63
71
|
continue;
|
|
64
72
|
}
|
|
@@ -74,7 +82,7 @@ function tokenize(sql, dialect) {
|
|
|
74
82
|
value += ch;
|
|
75
83
|
i += 1;
|
|
76
84
|
}
|
|
77
|
-
return value;
|
|
85
|
+
return { value, undecodedEscape };
|
|
78
86
|
};
|
|
79
87
|
while (i < n) {
|
|
80
88
|
const ch = sql[i];
|
|
@@ -125,7 +133,7 @@ function tokenize(sql, dialect) {
|
|
|
125
133
|
this check does not decode; one without escapes reads as it is. */
|
|
126
134
|
i += 2;
|
|
127
135
|
const close = sql[i];
|
|
128
|
-
const value = quoted(close, false);
|
|
136
|
+
const { value } = quoted(close, false);
|
|
129
137
|
if (value.includes('\\') || /^\s*uescape\b/i.test(sql.slice(i))) {
|
|
130
138
|
return ('This statement spells a name with Unicode escapes (U&"…"), which the reserved-table check does not decode, ' +
|
|
131
139
|
'so it is refused rather than run. Spell the name plainly.');
|
|
@@ -136,22 +144,27 @@ function tokenize(sql, dialect) {
|
|
|
136
144
|
if (ch === "'") {
|
|
137
145
|
/* E'…' is Postgres's backslash-escaped string. */
|
|
138
146
|
const escaped = dialect === 'postgres' && /[eE]$/.test(sql.slice(Math.max(0, i - 1), i));
|
|
139
|
-
|
|
147
|
+
const { value, undecodedEscape } = quoted("'", escaped);
|
|
148
|
+
if (undecodedEscape) {
|
|
149
|
+
return ("This statement uses octal, hex or Unicode escapes in E'…', which the reserved-table check does not decode, " +
|
|
150
|
+
'so it is refused rather than run. Spell the text plainly.');
|
|
151
|
+
}
|
|
152
|
+
tokens.push({ kind: 'text', value });
|
|
140
153
|
continue;
|
|
141
154
|
}
|
|
142
155
|
if (ch === '"') {
|
|
143
|
-
tokens.push({ kind: 'quoted', value: quoted('"', false) });
|
|
156
|
+
tokens.push({ kind: 'quoted', value: quoted('"', false).value });
|
|
144
157
|
continue;
|
|
145
158
|
}
|
|
146
159
|
if (ch === '`' && dialect !== 'postgres') {
|
|
147
|
-
tokens.push({ kind: 'quoted', value: quoted('`', false) });
|
|
160
|
+
tokens.push({ kind: 'quoted', value: quoted('`', false).value });
|
|
148
161
|
continue;
|
|
149
162
|
}
|
|
150
163
|
if (ch === '[' && dialect === 'sqlite') {
|
|
151
|
-
tokens.push({ kind: 'quoted', value: quoted(']', false) });
|
|
164
|
+
tokens.push({ kind: 'quoted', value: quoted(']', false).value });
|
|
152
165
|
continue;
|
|
153
166
|
}
|
|
154
|
-
const word = /^[A-Za-z_
|
|
167
|
+
const word = /^[A-Za-z_\u0080-\uffff][\w$\u0080-\uffff]*/.exec(sql.slice(i))?.[0];
|
|
155
168
|
if (word) {
|
|
156
169
|
tokens.push({ kind: 'word', value: word });
|
|
157
170
|
i += word.length;
|
|
@@ -164,6 +177,7 @@ function tokenize(sql, dialect) {
|
|
|
164
177
|
// A match in text (a comment, a string)
|
|
165
178
|
// takes the rest of the name, so the refusal names the table, not the prefix.
|
|
166
179
|
const RESERVED_TEXT = new RegExp(`(?:${RUNTIME_TABLE_PREFIXES.join('|')})[\\w$]*|\\b(?:${RESERVED_SCHEMAS.join('|')})\\b`, 'i');
|
|
180
|
+
const STATS_TEXT = /\b(?:pg_stats|pg_statistic|pg_statistic_ext_data|pg_stats_ext|pg_stats_ext_exprs)\b/i;
|
|
167
181
|
/**
|
|
168
182
|
* Why this statement must not run under the guard, or null when nothing in
|
|
169
183
|
* its text reaches a reserved relation.
|
|
@@ -174,13 +188,14 @@ export function reservedInText(sql, dialect) {
|
|
|
174
188
|
return tokens;
|
|
175
189
|
for (const token of tokens) {
|
|
176
190
|
if (token.kind === 'text') {
|
|
177
|
-
const found = RESERVED_TEXT.exec(token.value);
|
|
191
|
+
const found = RESERVED_TEXT.exec(token.value) ?? (dialect === 'postgres' ? STATS_TEXT.exec(token.value) : null);
|
|
178
192
|
if (found)
|
|
179
193
|
return reservedRefusal(found[0]).message;
|
|
180
194
|
continue;
|
|
181
195
|
}
|
|
182
196
|
const name = token.value;
|
|
183
|
-
|
|
197
|
+
const lower = name.toLowerCase();
|
|
198
|
+
if (isRuntimeTable(name) || (dialect === 'postgres' && (RESERVED_SCHEMAS.includes(lower) || STATS_RELATIONS.has(lower)))) {
|
|
184
199
|
return reservedRefusal(name).message;
|
|
185
200
|
}
|
|
186
201
|
if (dialect === 'postgres' && COMPUTED_RELATION.test(name)) {
|
|
@@ -201,11 +216,20 @@ export function refuseReservedText(sql, dialect) {
|
|
|
201
216
|
* The Postgres check before a statement is given a connection: its text,
|
|
202
217
|
* and for a read, whether the planner can list what it reads — the engine
|
|
203
218
|
* half below needs a plan, so a statement the planner cannot explain is
|
|
204
|
-
* refused rather than run unchecked.
|
|
219
|
+
* refused rather than run unchecked. For a write, anonymous `DO` blocks are
|
|
220
|
+
* also refused because they can run dynamic SQL the planner never sees.
|
|
205
221
|
*/
|
|
206
222
|
export function refusePostgresReserved(sql, write = false) {
|
|
207
223
|
refuseReservedText(sql, 'postgres');
|
|
208
|
-
if (
|
|
224
|
+
if (write) {
|
|
225
|
+
const tokens = tokenize(sql, 'postgres');
|
|
226
|
+
const first = Array.isArray(tokens) ? tokens.find((token) => token.kind !== 'text') : undefined;
|
|
227
|
+
if (first?.kind === 'word' && /^do$/i.test(first.value)) {
|
|
228
|
+
throw new Refusal('The reserved-table check refuses DO blocks because procedural code can execute dynamic SQL that cannot be checked.');
|
|
229
|
+
}
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (!looksWrappable(sql)) {
|
|
209
233
|
throw new Refusal("The reserved-table check reads the planner's list of what a statement scans, so it " +
|
|
210
234
|
'runs the statements the planner can explain: SELECT, WITH, TABLE and VALUES.');
|
|
211
235
|
}
|
|
@@ -21,6 +21,50 @@ import { RUNTIME_TABLE_PREFIXES } from '../shared/runtimeTables.js';
|
|
|
21
21
|
import { onCancel, StatementStopped } from './statementControl.js';
|
|
22
22
|
/* How long the child may take to start before the statement's own cap. */
|
|
23
23
|
const STARTUP_MS = 10_000;
|
|
24
|
+
/* Bound concurrent child Node processes so a burst of slow SQLite reads cannot
|
|
25
|
+
fork-bomb the host; extra requests wait briefly in a bounded queue. */
|
|
26
|
+
const MAX_CONCURRENT_PROCESSES = 4;
|
|
27
|
+
const MAX_QUEUED_PROCESSES = 32;
|
|
28
|
+
let activeProcesses = 0;
|
|
29
|
+
const processQueue = [];
|
|
30
|
+
function acquireProcessSlot(signal, capMs) {
|
|
31
|
+
if (signal.aborted)
|
|
32
|
+
return Promise.reject(new StatementStopped('cancelled'));
|
|
33
|
+
const release = () => {
|
|
34
|
+
activeProcesses -= 1;
|
|
35
|
+
const next = processQueue.shift();
|
|
36
|
+
if (next)
|
|
37
|
+
next();
|
|
38
|
+
};
|
|
39
|
+
if (activeProcesses < MAX_CONCURRENT_PROCESSES) {
|
|
40
|
+
activeProcesses += 1;
|
|
41
|
+
return Promise.resolve(release);
|
|
42
|
+
}
|
|
43
|
+
if (processQueue.length >= MAX_QUEUED_PROCESSES) {
|
|
44
|
+
return Promise.reject(new StatementStopped('timeout', capMs));
|
|
45
|
+
}
|
|
46
|
+
return new Promise((resolve, reject) => {
|
|
47
|
+
let settled = false;
|
|
48
|
+
const wake = () => {
|
|
49
|
+
if (settled)
|
|
50
|
+
return;
|
|
51
|
+
settled = true;
|
|
52
|
+
disarm();
|
|
53
|
+
activeProcesses += 1;
|
|
54
|
+
resolve(release);
|
|
55
|
+
};
|
|
56
|
+
const disarm = onCancel(signal, () => {
|
|
57
|
+
if (settled)
|
|
58
|
+
return;
|
|
59
|
+
settled = true;
|
|
60
|
+
const idx = processQueue.indexOf(wake);
|
|
61
|
+
if (idx !== -1)
|
|
62
|
+
processQueue.splice(idx, 1);
|
|
63
|
+
reject(new StatementStopped('cancelled'));
|
|
64
|
+
});
|
|
65
|
+
processQueue.push(wake);
|
|
66
|
+
});
|
|
67
|
+
}
|
|
24
68
|
/* The child. SQLITE_OK and SQLITE_READ are the C API's authorizer codes. The
|
|
25
69
|
rows stop one past the limit, so an unwrapped statement is not read whole
|
|
26
70
|
to learn that there was more. */
|
|
@@ -75,57 +119,63 @@ process.send({ ready: true });
|
|
|
75
119
|
* `signal` aborts. The engine's own errors come back as errors with the
|
|
76
120
|
* engine's words, as they would in-process.
|
|
77
121
|
*/
|
|
78
|
-
export function runInStatementProcess(job, capMs, signal) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const child = spawn(process.execPath, ['-e', CHILD], {
|
|
85
|
-
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
|
86
|
-
serialization: 'advanced',
|
|
87
|
-
/* Electron's binary is Node only when told so; plain Node ignores it. */
|
|
88
|
-
env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' },
|
|
89
|
-
windowsHide: true,
|
|
90
|
-
});
|
|
91
|
-
let stderr = '';
|
|
92
|
-
child.stderr?.on('data', (chunk) => {
|
|
93
|
-
stderr = (stderr + chunk.toString()).slice(-2_000);
|
|
94
|
-
});
|
|
95
|
-
let settled = false;
|
|
96
|
-
let timer;
|
|
97
|
-
let disarm = () => { };
|
|
98
|
-
const settle = (outcome) => {
|
|
99
|
-
if (settled)
|
|
100
|
-
return;
|
|
101
|
-
settled = true;
|
|
102
|
-
clearTimeout(timer);
|
|
103
|
-
disarm();
|
|
104
|
-
outcome();
|
|
105
|
-
};
|
|
106
|
-
const stop = (reason) => settle(() => {
|
|
107
|
-
child.kill('SIGKILL');
|
|
108
|
-
reject(new StatementStopped(reason, capMs));
|
|
109
|
-
});
|
|
110
|
-
timer = setTimeout(() => stop('timeout'), STARTUP_MS + capMs);
|
|
111
|
-
disarm = onCancel(signal, () => stop('cancelled'));
|
|
112
|
-
child.on('message', (message) => {
|
|
113
|
-
if (message.ready) {
|
|
114
|
-
/* The cap is the statement's, not the process start's. */
|
|
115
|
-
clearTimeout(timer);
|
|
116
|
-
timer = setTimeout(() => stop('timeout'), capMs);
|
|
117
|
-
child.send({ ...job, prefixes: RUNTIME_TABLE_PREFIXES });
|
|
122
|
+
export async function runInStatementProcess(job, capMs, signal) {
|
|
123
|
+
const releaseSlot = await acquireProcessSlot(signal, capMs);
|
|
124
|
+
try {
|
|
125
|
+
return await new Promise((resolve, reject) => {
|
|
126
|
+
if (signal.aborted) {
|
|
127
|
+
reject(new StatementStopped('cancelled'));
|
|
118
128
|
return;
|
|
119
129
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
130
|
+
const child = spawn(process.execPath, ['-e', CHILD], {
|
|
131
|
+
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
|
132
|
+
serialization: 'advanced',
|
|
133
|
+
/* Electron's binary is Node only when told so; plain Node ignores it. */
|
|
134
|
+
env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' },
|
|
135
|
+
windowsHide: true,
|
|
136
|
+
});
|
|
137
|
+
let stderr = '';
|
|
138
|
+
child.stderr?.on('data', (chunk) => {
|
|
139
|
+
stderr = (stderr + chunk.toString()).slice(-2_000);
|
|
140
|
+
});
|
|
141
|
+
let settled = false;
|
|
142
|
+
let timer;
|
|
143
|
+
let disarm = () => { };
|
|
144
|
+
const settle = (outcome) => {
|
|
145
|
+
if (settled)
|
|
146
|
+
return;
|
|
147
|
+
settled = true;
|
|
148
|
+
clearTimeout(timer);
|
|
149
|
+
disarm();
|
|
150
|
+
outcome();
|
|
151
|
+
};
|
|
152
|
+
const stop = (reason) => settle(() => {
|
|
153
|
+
child.kill('SIGKILL');
|
|
154
|
+
reject(new StatementStopped(reason, capMs));
|
|
125
155
|
});
|
|
156
|
+
timer = setTimeout(() => stop('timeout'), STARTUP_MS + capMs);
|
|
157
|
+
disarm = onCancel(signal, () => stop('cancelled'));
|
|
158
|
+
child.on('message', (message) => {
|
|
159
|
+
if (message.ready) {
|
|
160
|
+
/* The cap is the statement's, not the process start's. */
|
|
161
|
+
clearTimeout(timer);
|
|
162
|
+
timer = setTimeout(() => stop('timeout'), capMs);
|
|
163
|
+
child.send({ ...job, prefixes: RUNTIME_TABLE_PREFIXES });
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
settle(() => {
|
|
167
|
+
if (message.error !== undefined)
|
|
168
|
+
reject(new Error(message.error));
|
|
169
|
+
else
|
|
170
|
+
resolve(message);
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
child.once('error', (error) => settle(() => reject(error)));
|
|
174
|
+
child.once('exit', (code, killedBy) => settle(() => reject(new Error(`The SQLite statement process ended without an answer (${killedBy ?? `exit ${code}`}).` +
|
|
175
|
+
(stderr.trim() ? ` ${stderr.trim()}` : '')))));
|
|
126
176
|
});
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
}
|
|
177
|
+
}
|
|
178
|
+
finally {
|
|
179
|
+
releaseSlot();
|
|
180
|
+
}
|
|
131
181
|
}
|
|
@@ -62,6 +62,7 @@ export function shellPolicy(html, { fromSource = false, socket } = {}) {
|
|
|
62
62
|
origin, typed by the page, never a URL a column names. */
|
|
63
63
|
"frame-src 'self' blob:",
|
|
64
64
|
"base-uri 'none'",
|
|
65
|
+
"form-action 'self'",
|
|
65
66
|
"frame-ancestors 'self'",
|
|
66
67
|
/* From source only: Vite's client, when its hot-reload socket drops,
|
|
67
68
|
waits for the server in a shared worker it makes from a blob. The
|
|
@@ -488,8 +489,11 @@ export async function serveClient(files, req, res, shell) {
|
|
|
488
489
|
const raw = await readFile(join(files.source, 'index.html'), 'utf8');
|
|
489
490
|
const data = await pageData(shell, head);
|
|
490
491
|
const pane = head ? undefined : await drawnPane(shell, data);
|
|
492
|
+
/* Hash only the static template's inline scripts before dynamic
|
|
493
|
+
head, pane or data blocks are spliced in. */
|
|
494
|
+
const template = await vite.transformIndexHtml(sourceURL, raw);
|
|
491
495
|
/* No built files to preload from source; an App theme's own faces are the App's, and are said as in a build. */
|
|
492
|
-
const html = withData(withPane(withHead(withLooks(
|
|
496
|
+
const html = withData(withPane(withHead(withLooks(template, shell, []), shell), pane), data);
|
|
493
497
|
if (res.destroyed || res.writableEnded)
|
|
494
498
|
return;
|
|
495
499
|
res.writeHead(200, {
|
|
@@ -497,7 +501,7 @@ export async function serveClient(files, req, res, shell) {
|
|
|
497
501
|
'cache-control': pageCache(data, shell),
|
|
498
502
|
...(data || shell?.branding?.private ? { vary: 'Cookie' } : {}),
|
|
499
503
|
'x-content-type-options': 'nosniff',
|
|
500
|
-
'content-security-policy': pagePolicy(shellPolicy(
|
|
504
|
+
'content-security-policy': pagePolicy(shellPolicy(template, { fromSource: true, socket: vite.socket }), shell),
|
|
501
505
|
...pagePermissions(shell),
|
|
502
506
|
});
|
|
503
507
|
res.end(head ? undefined : html);
|
|
@@ -671,7 +675,7 @@ export function builtClient(dir) {
|
|
|
671
675
|
vary: asset.variants.length > 1 ? 'Accept-Encoding, Cookie' : 'Cookie',
|
|
672
676
|
...(asset.policy
|
|
673
677
|
? {
|
|
674
|
-
'content-security-policy': pagePolicy(
|
|
678
|
+
'content-security-policy': pagePolicy(asset.policy, shell),
|
|
675
679
|
...pagePermissions(shell),
|
|
676
680
|
}
|
|
677
681
|
: {}),
|
|
@@ -700,7 +704,7 @@ export function builtClient(dir) {
|
|
|
700
704
|
'x-content-type-options': 'nosniff',
|
|
701
705
|
...(asset.policy
|
|
702
706
|
? {
|
|
703
|
-
'content-security-policy': pagePolicy(
|
|
707
|
+
'content-security-policy': pagePolicy(asset.policy, shell),
|
|
704
708
|
...pagePermissions(shell),
|
|
705
709
|
}
|
|
706
710
|
: {}),
|
|
@@ -429,6 +429,8 @@ export function resourceChecks(full, access) {
|
|
|
429
429
|
assertFields(request.table, columns, 'read', server);
|
|
430
430
|
for (const order of request.orderBy ?? [])
|
|
431
431
|
assertFields(request.table, [order.column], 'read', server);
|
|
432
|
+
if (request.lanes)
|
|
433
|
+
assertFields(request.table, [request.lanes.by], 'read', server);
|
|
432
434
|
return { ...request, columns, filter: filterFor(request.table, request.filter, server) };
|
|
433
435
|
};
|
|
434
436
|
/* An aggregate reads the columns it keys and measures by, and its where's, as a query reads its own. */
|
|
@@ -542,6 +544,8 @@ export function resourceChecks(full, access) {
|
|
|
542
544
|
assertExpressions(view.base, view.expressions, view.computed);
|
|
543
545
|
assertTags(view.tagged);
|
|
544
546
|
assertAnyRows(view.filter);
|
|
547
|
+
if (view.lanes)
|
|
548
|
+
assertFields(view.base, [view.lanes.by]);
|
|
545
549
|
// Validate against the pruned graph before the underlying full catalog can
|
|
546
550
|
// resolve a forbidden forward path, filter, sort or reverse aggregate.
|
|
547
551
|
const compiled = compileView(schema, view, ANSI_STYLE);
|
|
@@ -17,7 +17,7 @@ import type { DatabaseSchema } from '../../shared/schema.js';
|
|
|
17
17
|
import type { QueryPerson } from '../../shared/query.js';
|
|
18
18
|
import type { ColumnGrant } from '../columnGrants.js';
|
|
19
19
|
import type { RequestDiagnostics } from '../operations/diagnostics.js';
|
|
20
|
-
import type { Bound } from './trust.js';
|
|
20
|
+
import type { Bound, ProxyRange } from './trust.js';
|
|
21
21
|
/**
|
|
22
22
|
* The App framework's types for what a request carries on its behalf: the
|
|
23
23
|
* Apps it serves, an App's resources and its reader's grants, its commands,
|
|
@@ -190,5 +190,7 @@ export interface PostContext {
|
|
|
190
190
|
readonly appHost?: HostedApp;
|
|
191
191
|
/** Who `me` is in an App's conditions, read through this request's reader (server/app/appMe.ts). */
|
|
192
192
|
readonly me?: () => Promise<QueryPerson>;
|
|
193
|
+
/** `--trust-proxy` ranges, for keying anonymous exports by forwarded client address. */
|
|
194
|
+
readonly trustedProxies?: readonly ProxyRange[];
|
|
193
195
|
}
|
|
194
196
|
export {};
|
|
@@ -75,6 +75,7 @@ export async function postData(ctx, hooks) {
|
|
|
75
75
|
rowPolicy: Boolean(ctx.policyContext),
|
|
76
76
|
appConfigured,
|
|
77
77
|
appNow: clockNow,
|
|
78
|
+
...(ctx.bound.trustedProxies ? { trustedProxies: ctx.bound.trustedProxies } : {}),
|
|
78
79
|
...hooks.extras?.(ctx, source, reads),
|
|
79
80
|
}, hooks);
|
|
80
81
|
};
|
|
@@ -51,6 +51,7 @@ import type { Adapter } from '../../adapters/adapter.js';
|
|
|
51
51
|
import { parseQuery } from '../../shared/query.js';
|
|
52
52
|
import { type DatabaseSchema } from '../../shared/schema.js';
|
|
53
53
|
import type { PostContext } from './context.js';
|
|
54
|
+
import { type ProxyRange } from './trust.js';
|
|
54
55
|
/** Rows read per statement, and so the most the server holds of a file at once:
|
|
55
56
|
* large enough that a million-row file is a thousand reads, small enough that
|
|
56
57
|
* one batch is never a memory event. */
|
|
@@ -90,6 +91,8 @@ interface ExportAsked {
|
|
|
90
91
|
answer?: (status: number, body: unknown) => void;
|
|
91
92
|
/** Who is exporting, counted against `EXPORTS_IN_FLIGHT`; the client's address where no one is named. */
|
|
92
93
|
who?: string;
|
|
94
|
+
/** `--trust-proxy` ranges, for resolving the client address behind a reverse proxy when `who` is unset. */
|
|
95
|
+
trustedProxies?: readonly ProxyRange[];
|
|
93
96
|
}
|
|
94
97
|
export declare function exportRoute(post: PostContext, hooks: ExportHooks): Promise<boolean>;
|
|
95
98
|
/**
|
|
@@ -8,6 +8,7 @@ import { CRLF, csvField, csvRecord, jsonRecord } from '../../shared/csv.js';
|
|
|
8
8
|
import { readRecords, readView } from '../resourceReadOperations.js';
|
|
9
9
|
import { recordsPaging, settlePage, viewPaging } from '../keysetReadHttp.js';
|
|
10
10
|
import { acceptedEncoding, httpRead, policyResponses, send } from './reply.js';
|
|
11
|
+
import { forwardedClient } from './trust.js';
|
|
11
12
|
import { rollupReads } from '../../shared/rollups.js';
|
|
12
13
|
const EXPORT_PATH = '/api/export';
|
|
13
14
|
/** Rows read per statement, and so the most the server holds of a file at once:
|
|
@@ -38,6 +39,7 @@ export async function exportRoute(post, hooks) {
|
|
|
38
39
|
now: post.appNow,
|
|
39
40
|
may: hooks.may(post),
|
|
40
41
|
...(post.who ? { who: post.who } : {}),
|
|
42
|
+
...(post.trustedProxies ? { trustedProxies: post.trustedProxies } : {}),
|
|
41
43
|
...(person ? { person: (text) => person(post, text) } : {}),
|
|
42
44
|
...(amounts ? { amounts: (tableId) => amounts(post, tableId) } : {}),
|
|
43
45
|
});
|
|
@@ -66,7 +68,7 @@ export async function exportFile(asked) {
|
|
|
66
68
|
return undefined;
|
|
67
69
|
}
|
|
68
70
|
/* Counted until the response closes, whichever way it ends: a refusal, the file, or a client that left. */
|
|
69
|
-
const who = asked.who ?? `address:${res.req
|
|
71
|
+
const who = asked.who ?? `address:${res.req ? forwardedClient(res.req, asked.trustedProxies) : ''}`;
|
|
70
72
|
const running = inFlight.get(who) ?? 0;
|
|
71
73
|
if (running >= EXPORTS_IN_FLIGHT) {
|
|
72
74
|
answer(429, { code: 'too_many_exports', error: `At most ${EXPORTS_IN_FLIGHT} exports run at once. Wait for one to finish.` });
|
|
@@ -232,10 +232,12 @@ export class SharedWindowLimiter {
|
|
|
232
232
|
this.fallback.reset();
|
|
233
233
|
}
|
|
234
234
|
async count(windows, key, weight) {
|
|
235
|
+
const replies = await Promise.all(windows.map((window, i) => this.cache.incr(`${key}:${i}`, window.ms, weight)));
|
|
235
236
|
let wait = 0;
|
|
236
237
|
let remaining = Infinity;
|
|
237
|
-
for (
|
|
238
|
-
const { count, ttlMs } =
|
|
238
|
+
for (let i = 0; i < windows.length; i++) {
|
|
239
|
+
const { count, ttlMs } = replies[i];
|
|
240
|
+
const window = windows[i];
|
|
239
241
|
if (count > window.limit)
|
|
240
242
|
wait = Math.max(wait, Math.max(1, Math.ceil(ttlMs / 1000)));
|
|
241
243
|
else
|
|
@@ -127,20 +127,6 @@ export declare function forwardedClient(req: {
|
|
|
127
127
|
remoteAddress?: string;
|
|
128
128
|
};
|
|
129
129
|
}, trusted?: readonly ProxyRange[]): string;
|
|
130
|
-
/**
|
|
131
|
-
* Who is asking, when the operator has said which header carries it.
|
|
132
|
-
*
|
|
133
|
-
* `--auth-proxy-header X-Forwarded-User` means a proxy in front signs
|
|
134
|
-
* people in and writes the result into that header on the way through.
|
|
135
|
-
* A request without it did not come through the proxy — or came through
|
|
136
|
-
* one that is misconfigured — and is refused with the header named. So is
|
|
137
|
-
* a request that carries it from a peer that is not the proxy: the header
|
|
138
|
-
* is only a name anyone can type, so a value from an address neither on
|
|
139
|
-
* loopback nor in `--auth-proxy-from` is a forgery, not a sign-in. Loopback is
|
|
140
|
-
* exempt: the container's own health check, a curl on the box, the
|
|
141
|
-
* operator's terminal. Nothing here is authentication; it is the proxy's
|
|
142
|
-
* decision, carried to the journal.
|
|
143
|
-
*/
|
|
144
130
|
export declare function identityFor(req: {
|
|
145
131
|
headers: Record<string, string | string[] | undefined>;
|
|
146
132
|
socket?: {
|
|
@@ -279,11 +279,18 @@ function forwardedHops(raw) {
|
|
|
279
279
|
* one that is misconfigured — and is refused with the header named. So is
|
|
280
280
|
* a request that carries it from a peer that is not the proxy: the header
|
|
281
281
|
* is only a name anyone can type, so a value from an address neither on
|
|
282
|
-
* loopback nor in `--auth-proxy-from` is a forgery, not a sign-in.
|
|
283
|
-
*
|
|
284
|
-
*
|
|
285
|
-
*
|
|
282
|
+
* loopback nor in `--auth-proxy-from` is a forgery, not a sign-in.
|
|
283
|
+
*
|
|
284
|
+
* A request from a loopback peer with no identity is the local operator
|
|
285
|
+
* (`who: 'local'`: the container's health check, a curl on the box, the
|
|
286
|
+
* operator's terminal) only when nothing says it was proxied. It is
|
|
287
|
+
* refused like any other when the listener has `--trust-proxy` addresses,
|
|
288
|
+
* the request carries a Forwarded or X-Forwarded-* header, or its Host is
|
|
289
|
+
* not a loopback name: a reverse proxy on the same host that dropped the
|
|
290
|
+
* identity header must not be mistaken for the operator. Nothing here is
|
|
291
|
+
* authentication; it is the proxy's decision, carried to the journal.
|
|
286
292
|
*/
|
|
293
|
+
const PROXIED_HEADERS = ['forwarded', 'x-forwarded-for', 'x-forwarded-host', 'x-forwarded-proto'];
|
|
287
294
|
export function identityFor(req, bound) {
|
|
288
295
|
if (!bound.identityHeader)
|
|
289
296
|
return {};
|
|
@@ -292,7 +299,16 @@ export function identityFor(req, bound) {
|
|
|
292
299
|
const peer = req.socket?.remoteAddress;
|
|
293
300
|
if (who && trustedProxyPeer(peer, bound.identityProxies))
|
|
294
301
|
return { who };
|
|
295
|
-
|
|
302
|
+
/* A request arriving over loopback from a reverse proxy (nginx/Caddy on the
|
|
303
|
+
same host) carries forwarded headers or a non-loopback Host, or arrives on
|
|
304
|
+
a listener configured with --trust-proxy. Never silently treat a proxied
|
|
305
|
+
request that dropped the identity header as the local operator. */
|
|
306
|
+
const rawHost = req.headers.host;
|
|
307
|
+
const host = (Array.isArray(rawHost) ? rawHost[0] : rawHost)?.trim();
|
|
308
|
+
const proxied = Boolean(bound.trustedProxies?.length) ||
|
|
309
|
+
PROXIED_HEADERS.some((header) => req.headers[header] !== undefined) ||
|
|
310
|
+
Boolean(host && !LOOPBACK.test(host));
|
|
311
|
+
if (loopbackPeer(peer) && !proxied)
|
|
296
312
|
return { who: 'local' };
|
|
297
313
|
return {
|
|
298
314
|
refusal: who
|
|
@@ -334,9 +350,9 @@ export function untrustedTargetRefusal(target) {
|
|
|
334
350
|
return ('This server is reachable beyond this machine, so a connection added from the browser cannot ' +
|
|
335
351
|
'reference environment variables. Put it in the config file on the server, or type the password.');
|
|
336
352
|
}
|
|
337
|
-
if (!/^
|
|
338
|
-
return ('This server is reachable beyond this machine, so the browser
|
|
339
|
-
'
|
|
353
|
+
if (!/^postgres(ql)?:\/\//i.test(target)) {
|
|
354
|
+
return ('This server is reachable beyond this machine, so the browser can only add a PostgreSQL connection ' +
|
|
355
|
+
'(postgres://…), never a SQLite file or other scheme. Name a SQLite file in the config file or on the command line.');
|
|
340
356
|
}
|
|
341
357
|
return untrustedDatabaseRefusal(target);
|
|
342
358
|
}
|
|
@@ -13,6 +13,7 @@ export declare const FRAMEWORK_WORDS: {
|
|
|
13
13
|
readonly 'mistri.form.step': 'Step {step} of {count}';
|
|
14
14
|
readonly 'mistri.form.stepNamed': 'Step {step} of {count}: {label}';
|
|
15
15
|
readonly 'mistri.form.back': 'Back';
|
|
16
|
+
readonly 'mistri.form.continue': 'Continue';
|
|
16
17
|
readonly 'mistri.form.optional': '(optional)';
|
|
17
18
|
readonly 'mistri.form.required': '(required)';
|
|
18
19
|
readonly 'mistri.form.reference': 'Your reference';
|
|
@@ -13,6 +13,7 @@ export const FRAMEWORK_WORDS = {
|
|
|
13
13
|
'mistri.form.step': 'Step {step} of {count}',
|
|
14
14
|
'mistri.form.stepNamed': 'Step {step} of {count}: {label}',
|
|
15
15
|
'mistri.form.back': 'Back',
|
|
16
|
+
'mistri.form.continue': 'Continue',
|
|
16
17
|
'mistri.form.optional': '(optional)',
|
|
17
18
|
'mistri.form.required': '(required)',
|
|
18
19
|
'mistri.form.reference': 'Your reference',
|
|
@@ -2,6 +2,7 @@ export const FRAMEWORK_WORDS_ES = Object.freeze({
|
|
|
2
2
|
'mistri.form.step': 'Paso {step} de {count}',
|
|
3
3
|
'mistri.form.stepNamed': 'Paso {step} de {count}: {label}',
|
|
4
4
|
'mistri.form.back': 'Atrás',
|
|
5
|
+
'mistri.form.continue': 'Continuar',
|
|
5
6
|
'mistri.form.optional': '(opcional)',
|
|
6
7
|
'mistri.form.required': '(obligatorio)',
|
|
7
8
|
'mistri.form.reference': 'Tu referencia',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mistriwork/engine",
|
|
3
|
-
"version": "1.0.0-rc.
|
|
3
|
+
"version": "1.0.0-rc.71",
|
|
4
4
|
"description": "The engine both products stand on: the adapters (SQLite, Postgres), the catalog, the query language, keyset pages, the HTTP kit and the shared rules. No React.",
|
|
5
5
|
"homepage": "https://mistriwork.dev",
|
|
6
6
|
"bugs": {
|