@mnmal-ai/substrate 1.0.0-alpha.0 → 1.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -9
- package/dist/boot-store-BVqfFxh1.js +4060 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-Cp4PaIYO.d.ts} +2 -87
- package/dist/cli-hSCBywhE.js +3126 -0
- package/dist/client.d.ts +130 -8
- package/dist/client.js +6 -5
- package/dist/{plugin-t2g2wX0b.d.ts → dataset-CejF-Fr_.d.ts} +1640 -1320
- package/dist/{agent-token-BIboWOU0.js → env-utils-CGNazUNt.js} +31 -2
- package/dist/{errors-D4vKuwz8.d.ts → errors-7mycvNGq.d.ts} +1 -1
- package/dist/{index-D7mSOFlM.d.ts → index-DgSfwx7M.d.ts} +1 -1
- package/dist/node-version-C5FSmwJX.js +26 -0
- package/dist/pack.d.ts +2 -2
- package/dist/pack.js +1 -1
- package/dist/pg-edges-store-Lxsl5ZnK.js +381 -0
- package/dist/{plugin-j1jt1l3n.js → plugin-CtwOINCM.js} +171 -18
- package/dist/proposals-bootstrap.js +6 -4
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +15 -4
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -2
- package/dist/recall-bootstrap.js +13 -4
- package/dist/recall.config.schema.json +12 -0
- package/dist/recall.d.ts +54 -9
- package/dist/recall.js +2 -2
- package/dist/{repl-C52wyfMs.js → repl-DqfHelK7.js} +62 -29
- package/dist/{row-ByLFE2Wx.d.ts → row-BKKCXIG6.d.ts} +5 -3
- package/dist/run-B5dTbZ3g.js +1997 -0
- package/dist/{schema-g4FQLr0V.d.ts → schema-DoSc-4ig.d.ts} +44 -7
- package/dist/script-C6DSa_Na.js +134 -0
- package/dist/sdk.d.ts +90 -12
- package/dist/sdk.js +5 -110
- package/dist/{src-D91yij6C.js → src-5SJ5GEIM.js} +1557 -327
- package/dist/{src-DuopV8u7.js → src-7CUSMK66.js} +148 -19
- package/dist/{src-CrEgiMJQ.js → src-Bi849yhn.js} +39 -15
- package/dist/{src-D88LvCX8.js → src-CqT-QWZ_.js} +697 -57
- package/dist/src-CyvV7dCK.js +4333 -0
- package/dist/{row-DK7lPZVe.js → src-Da0t6si5.js} +11 -11
- package/dist/{src-C4DzaIQZ.js → src-hZb8wfvg.js} +12 -1
- package/dist/substrate-admin.js +17 -781
- package/dist/substrate.js +4 -5471
- package/dist/testing.d.ts +557 -8
- package/dist/testing.js +39 -3
- package/dist/{ws-C72iVS5T.d.ts → ws-ByZyHk7m.d.ts} +3 -3
- package/package.json +21 -20
- package/dist/dataset-BXkus75W.js +0 -1527
- package/dist/env-utils-B7SGmoeJ.js +0 -21
- package/dist/src-CzxvmhFn.js +0 -2327
|
@@ -0,0 +1,4333 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { $ as assertJsonPathPredicate, Bt as ReservedField, Pt as normalizeIsoTimestamp, Q as preResolveExtensionOperatorsDetailed, Ut as UnsupportedFilterError, Yt as assertNoRepeatedIds, at as decodeCursor, ct as applyExtensionResultMeta, et as isJsonPathPredicate, jt as createKeyedMutex, lt as fieldRequested, ot as encodeCursor, st as fullBuiltinOperators, zt as NamespaceMismatch } from "./src-5SJ5GEIM.js";
|
|
3
|
+
import { x as naturalOrderOf } from "./src-hZb8wfvg.js";
|
|
4
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
5
|
+
|
|
6
|
+
//#region ../../packages/substrate-dataset-pg/src/advisory-lock.ts
|
|
7
|
+
/**
|
|
8
|
+
* Session advisory lock on one reserved connection, for
|
|
9
|
+
* `PgDataset.withAdvisoryLock`. Keys are scoped to the session's current
|
|
10
|
+
* schema, so stores in different schemas of one database never contend.
|
|
11
|
+
*/
|
|
12
|
+
const LOCAL_LOCKS = /* @__PURE__ */ new WeakMap();
|
|
13
|
+
/** Selects the current session as `pid` and `started`, for {@link retireSession}. */
|
|
14
|
+
const SESSION_COLUMNS = `pg_backend_pid() AS pid,
|
|
15
|
+
(SELECT backend_start::text FROM pg_stat_activity WHERE pid = pg_backend_pid()) AS started`;
|
|
16
|
+
/**
|
|
17
|
+
* Retires a session that may still hold an advisory lock after its unlock
|
|
18
|
+
* threw. True when its connection may go back to the pool. The session is
|
|
19
|
+
* terminated from another pooled connection, which drops every lock it holds
|
|
20
|
+
* and retires the connection; a reserved handle has no `end()`, and a backend
|
|
21
|
+
* terminating itself mid-query leaves postgres.js a stale error that surfaces
|
|
22
|
+
* uncaught on the connection's next use. It is matched on `backend_start` as
|
|
23
|
+
* well as `pid`, because once it has ended its pid can name another session.
|
|
24
|
+
*/
|
|
25
|
+
async function retireSession(sql, session) {
|
|
26
|
+
if (sql.options.max <= 1) return true;
|
|
27
|
+
try {
|
|
28
|
+
return (await sql`
|
|
29
|
+
SELECT pg_terminate_backend(pid) FROM pg_stat_activity
|
|
30
|
+
WHERE pid = ${session.pid} AND backend_start = ${session.started}::timestamptz`).length === 0;
|
|
31
|
+
} catch {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** Unlock `key` on `conn`. True when `conn` may go back to the pool. */
|
|
36
|
+
async function unlock(sql, conn, key, session) {
|
|
37
|
+
try {
|
|
38
|
+
await conn`SELECT pg_advisory_unlock(hashtext(current_schema() || ':' || ${key}))`;
|
|
39
|
+
return true;
|
|
40
|
+
} catch {
|
|
41
|
+
return retireSession(sql, session);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Run `fn` while holding the advisory lock of `key` in the current schema, waiting until it
|
|
46
|
+
* is free. Lock and unlock share one reserved connection, because an unlock
|
|
47
|
+
* through the pool can land on another session and leave the lock held.
|
|
48
|
+
*
|
|
49
|
+
* Callers in this process queue on a local mutex first, so waiters for one key
|
|
50
|
+
* occupy at most one pooled connection per process; otherwise enough blocked
|
|
51
|
+
* waiters could take every connection and starve the holder's own `fn`.
|
|
52
|
+
*
|
|
53
|
+
* @returns whatever `fn` resolves to
|
|
54
|
+
* @throws whatever `fn` throws, after the lock is released
|
|
55
|
+
*/
|
|
56
|
+
function withPgAdvisoryLock(sql, key, fn) {
|
|
57
|
+
let local = LOCAL_LOCKS.get(sql);
|
|
58
|
+
if (local === void 0) {
|
|
59
|
+
local = createKeyedMutex();
|
|
60
|
+
LOCAL_LOCKS.set(sql, local);
|
|
61
|
+
}
|
|
62
|
+
return local(key, async () => {
|
|
63
|
+
const conn = await sql.reserve();
|
|
64
|
+
let reusable = true;
|
|
65
|
+
try {
|
|
66
|
+
const [row] = await conn`
|
|
67
|
+
SELECT pg_advisory_lock(hashtext(current_schema() || ':' || ${key})), pg_backend_pid() AS pid,
|
|
68
|
+
(SELECT backend_start::text FROM pg_stat_activity WHERE pid = pg_backend_pid()) AS started`;
|
|
69
|
+
try {
|
|
70
|
+
return await fn();
|
|
71
|
+
} finally {
|
|
72
|
+
reusable = await unlock(sql, conn, key, row ?? {
|
|
73
|
+
pid: 0,
|
|
74
|
+
started: "-infinity"
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
} finally {
|
|
78
|
+
if (reusable) conn.release();
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Run `fn` only if the advisory lock of `key` in the current schema is free right now,
|
|
84
|
+
* on one reserved connection like `withPgAdvisoryLock`. A held lock resolves
|
|
85
|
+
* `{ acquired: false }` without running `fn` and without waiting.
|
|
86
|
+
*
|
|
87
|
+
* @throws whatever `fn` throws, after the lock is released
|
|
88
|
+
*/
|
|
89
|
+
async function tryWithPgAdvisoryLock(sql, key, fn) {
|
|
90
|
+
const conn = await sql.reserve();
|
|
91
|
+
let reusable = true;
|
|
92
|
+
try {
|
|
93
|
+
const [row] = await conn`
|
|
94
|
+
SELECT pg_try_advisory_lock(hashtext(current_schema() || ':' || ${key})) AS ok, pg_backend_pid() AS pid,
|
|
95
|
+
(SELECT backend_start::text FROM pg_stat_activity WHERE pid = pg_backend_pid()) AS started`;
|
|
96
|
+
if (row?.ok !== true) return { acquired: false };
|
|
97
|
+
try {
|
|
98
|
+
return {
|
|
99
|
+
acquired: true,
|
|
100
|
+
value: await fn()
|
|
101
|
+
};
|
|
102
|
+
} finally {
|
|
103
|
+
reusable = await unlock(sql, conn, key, row);
|
|
104
|
+
}
|
|
105
|
+
} finally {
|
|
106
|
+
if (reusable) conn.release();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
//#endregion
|
|
111
|
+
//#region ../../packages/substrate-dataset-pg/src/layout.ts
|
|
112
|
+
/**
|
|
113
|
+
* How `rows` is stored. `partitioned` is LIST(namespace) then LIST(type);
|
|
114
|
+
* `legacy` is one plain table. A database keeps the layout it was created with.
|
|
115
|
+
*/
|
|
116
|
+
/** The layout of `rows` in the current schema, or `'absent'` when there is no `rows`. */
|
|
117
|
+
async function detectLayout(sql) {
|
|
118
|
+
const [row] = await sql`
|
|
119
|
+
SELECT c.relkind::text AS relkind FROM pg_class c
|
|
120
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
121
|
+
WHERE n.nspname = current_schema() AND c.relname = 'rows'
|
|
122
|
+
`;
|
|
123
|
+
if (row === void 0) return "absent";
|
|
124
|
+
return row.relkind === "p" ? "partitioned" : "legacy";
|
|
125
|
+
}
|
|
126
|
+
/** pg's identifier limit, in bytes. */
|
|
127
|
+
const MAX_IDENTIFIER = 63;
|
|
128
|
+
/**
|
|
129
|
+
* `rows__<ns>` or `rows__<ns>__<type>`, folded to `[a-z0-9_]`. When folding
|
|
130
|
+
* changed a name, or the result is too long for pg, an 8-hex hash of the raw
|
|
131
|
+
* names is appended so distinct names never share a partition.
|
|
132
|
+
*/
|
|
133
|
+
function partitionName(namespace, type) {
|
|
134
|
+
const raw = type === void 0 ? [namespace] : [namespace, type];
|
|
135
|
+
const readable = `rows__${raw.join("__")}`;
|
|
136
|
+
const folded = readable.toLowerCase().replace(/[^a-z0-9_]/g, "_");
|
|
137
|
+
if (folded === readable && folded.length <= 63) return folded;
|
|
138
|
+
return hashedName(folded, raw);
|
|
139
|
+
}
|
|
140
|
+
/** `base` cut to fit `max` bytes with an 8-hex hash of `seed` appended. `base` must be ASCII. */
|
|
141
|
+
function hashedName(base, seed, max = 63) {
|
|
142
|
+
const hash = createHash("sha256").update(JSON.stringify(seed)).digest("hex").slice(0, 8);
|
|
143
|
+
return `${base.slice(0, max - hash.length - 1)}_${hash}`;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
//#endregion
|
|
147
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0001_baseline.ts
|
|
148
|
+
/**
|
|
149
|
+
* 0001_baseline — the engine's schema: rows, edges, the idempotency cache and
|
|
150
|
+
* the short-id sidecar.
|
|
151
|
+
*
|
|
152
|
+
* Plugins own their tables and any columns they promote out of `rows.body`
|
|
153
|
+
* (see `PgDatasetOptions.promotedColumns`), and add them in their own kernel
|
|
154
|
+
* migrations. Idempotent, so re-running it changes nothing. `TX_STATEMENTS`
|
|
155
|
+
* run in one transaction; `CONCURRENT_STATEMENTS` run one by one after it
|
|
156
|
+
* commits, because Postgres refuses `CREATE INDEX CONCURRENTLY` in a
|
|
157
|
+
* transaction.
|
|
158
|
+
*/
|
|
159
|
+
const TX_STATEMENTS$4 = [
|
|
160
|
+
`CREATE TABLE IF NOT EXISTS rows (
|
|
161
|
+
namespace text NOT NULL,
|
|
162
|
+
type text NOT NULL,
|
|
163
|
+
id text NOT NULL,
|
|
164
|
+
body jsonb NOT NULL,
|
|
165
|
+
deleted_at timestamptz NULL,
|
|
166
|
+
created_at timestamptz NULL,
|
|
167
|
+
updated_at timestamptz NULL,
|
|
168
|
+
created_by_kind text NULL,
|
|
169
|
+
created_by_id text NULL,
|
|
170
|
+
updated_by_kind text NULL,
|
|
171
|
+
updated_by_id text NULL,
|
|
172
|
+
PRIMARY KEY (namespace, type, id),
|
|
173
|
+
CONSTRAINT rows_created_by_kind_check
|
|
174
|
+
CHECK (created_by_kind IS NULL OR created_by_kind IN ('agent','user','system')),
|
|
175
|
+
CONSTRAINT rows_updated_by_kind_check
|
|
176
|
+
CHECK (updated_by_kind IS NULL OR updated_by_kind IN ('agent','user','system'))
|
|
177
|
+
)`,
|
|
178
|
+
`CREATE TABLE IF NOT EXISTS edges (
|
|
179
|
+
id uuid PRIMARY KEY,
|
|
180
|
+
from_ns text NOT NULL,
|
|
181
|
+
from_type text NOT NULL,
|
|
182
|
+
from_id text NOT NULL,
|
|
183
|
+
from_bridged boolean NOT NULL DEFAULT false,
|
|
184
|
+
to_ns text NOT NULL,
|
|
185
|
+
to_type text NOT NULL,
|
|
186
|
+
to_id text NOT NULL,
|
|
187
|
+
to_bridged boolean NOT NULL DEFAULT false,
|
|
188
|
+
field_name text NOT NULL,
|
|
189
|
+
policy text NOT NULL,
|
|
190
|
+
created_at timestamptz NOT NULL,
|
|
191
|
+
deleted_at timestamptz NULL,
|
|
192
|
+
last_swept_at timestamptz NULL,
|
|
193
|
+
blocked boolean NOT NULL DEFAULT false,
|
|
194
|
+
CONSTRAINT edges_policy_check
|
|
195
|
+
CHECK (policy IN ('set_null','cascade','restrict','orphan'))
|
|
196
|
+
)`,
|
|
197
|
+
`CREATE TABLE IF NOT EXISTS idempotency (
|
|
198
|
+
key text PRIMARY KEY,
|
|
199
|
+
envelope jsonb NOT NULL,
|
|
200
|
+
expires_at timestamptz NOT NULL,
|
|
201
|
+
created_at timestamptz NOT NULL DEFAULT now()
|
|
202
|
+
)`,
|
|
203
|
+
`CREATE TABLE IF NOT EXISTS short_ids (
|
|
204
|
+
namespace text NOT NULL,
|
|
205
|
+
type text NOT NULL,
|
|
206
|
+
short_id bigint NOT NULL,
|
|
207
|
+
row_id text NOT NULL,
|
|
208
|
+
PRIMARY KEY (namespace, type, short_id),
|
|
209
|
+
CONSTRAINT short_ids_row_id_fkey FOREIGN KEY (namespace, type, row_id)
|
|
210
|
+
REFERENCES rows (namespace, type, id) ON DELETE CASCADE
|
|
211
|
+
)`,
|
|
212
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS substrate_key_live_kid_uniq
|
|
213
|
+
ON rows ((body ->> 'kid'))
|
|
214
|
+
WHERE namespace = '_system'
|
|
215
|
+
AND type = 'Key'
|
|
216
|
+
AND deleted_at IS NULL
|
|
217
|
+
AND (body ->> 'revokedAt') IS NULL`
|
|
218
|
+
];
|
|
219
|
+
const CONCURRENT_STATEMENTS$3 = [
|
|
220
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS rows_ns_type_created_at_idx
|
|
221
|
+
ON rows (namespace, type, created_at DESC NULLS LAST) WHERE deleted_at IS NULL`,
|
|
222
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS rows_ns_type_updated_at_idx
|
|
223
|
+
ON rows (namespace, type, updated_at DESC NULLS LAST) WHERE deleted_at IS NULL`,
|
|
224
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS rows_tombstoned_idx
|
|
225
|
+
ON rows (deleted_at) WHERE deleted_at IS NOT NULL`,
|
|
226
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS edges_to_live_idx
|
|
227
|
+
ON edges (to_ns, to_type, to_id) WHERE deleted_at IS NULL`,
|
|
228
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS edges_from_live_idx
|
|
229
|
+
ON edges (from_ns, from_type, from_id) WHERE deleted_at IS NULL`,
|
|
230
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS edges_cursor_idx
|
|
231
|
+
ON edges (id) WHERE deleted_at IS NULL`,
|
|
232
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS edges_tombstoned_idx
|
|
233
|
+
ON edges (deleted_at) WHERE deleted_at IS NOT NULL`,
|
|
234
|
+
`CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS edges_logical_identity_unique_idx
|
|
235
|
+
ON edges (from_ns, from_type, from_id, field_name, to_ns, to_type, to_id)
|
|
236
|
+
WHERE deleted_at IS NULL`,
|
|
237
|
+
`CREATE INDEX CONCURRENTLY IF NOT EXISTS idempotency_expires_at_idx
|
|
238
|
+
ON idempotency (expires_at)`,
|
|
239
|
+
`CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS short_ids_row_uniq_idx
|
|
240
|
+
ON short_ids (namespace, type, row_id)`
|
|
241
|
+
];
|
|
242
|
+
|
|
243
|
+
//#endregion
|
|
244
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0001p_partitioned_baseline.ts
|
|
245
|
+
/**
|
|
246
|
+
* 0001p_partitioned_baseline — the engine's schema for a fresh database, with
|
|
247
|
+
* `rows` partitioned LIST(namespace) then LIST(type). Same columns, keys and
|
|
248
|
+
* side tables as `0001_baseline`.
|
|
249
|
+
*
|
|
250
|
+
* Partition names come from `partitionName`; a namespace partition carries a
|
|
251
|
+
* DEFAULT sub-partition named `partitionName(ns) + '__default'`. `_system` is
|
|
252
|
+
* built that way here, with its `Key` partition carrying the live-kid index.
|
|
253
|
+
* `rows` gets no parent-level indexes beyond its primary key.
|
|
254
|
+
*/
|
|
255
|
+
/** `partitionName('_system')`; a test pins it to the function. */
|
|
256
|
+
const SYSTEM_PARTITION = "rows___system";
|
|
257
|
+
/** `partitionName('_system', 'Key')`; a test pins it to the function. */
|
|
258
|
+
const SYSTEM_KEY_PARTITION = "rows___system__key_c3038cb3";
|
|
259
|
+
const TX_STATEMENTS$3 = [
|
|
260
|
+
`CREATE TABLE IF NOT EXISTS rows (
|
|
261
|
+
namespace text NOT NULL,
|
|
262
|
+
type text NOT NULL,
|
|
263
|
+
id text NOT NULL,
|
|
264
|
+
body jsonb NOT NULL,
|
|
265
|
+
deleted_at timestamptz NULL,
|
|
266
|
+
created_at timestamptz NULL,
|
|
267
|
+
updated_at timestamptz NULL,
|
|
268
|
+
created_by_kind text NULL,
|
|
269
|
+
created_by_id text NULL,
|
|
270
|
+
updated_by_kind text NULL,
|
|
271
|
+
updated_by_id text NULL,
|
|
272
|
+
PRIMARY KEY (namespace, type, id),
|
|
273
|
+
CONSTRAINT rows_created_by_kind_check
|
|
274
|
+
CHECK (created_by_kind IS NULL OR created_by_kind IN ('agent','user','system')),
|
|
275
|
+
CONSTRAINT rows_updated_by_kind_check
|
|
276
|
+
CHECK (updated_by_kind IS NULL OR updated_by_kind IN ('agent','user','system'))
|
|
277
|
+
) PARTITION BY LIST (namespace)`,
|
|
278
|
+
`CREATE TABLE IF NOT EXISTS rows__default PARTITION OF rows DEFAULT`,
|
|
279
|
+
`CREATE TABLE IF NOT EXISTS ${SYSTEM_PARTITION} PARTITION OF rows
|
|
280
|
+
FOR VALUES IN ('_system') PARTITION BY LIST (type)`,
|
|
281
|
+
`CREATE TABLE IF NOT EXISTS ${SYSTEM_PARTITION}__default PARTITION OF ${SYSTEM_PARTITION} DEFAULT`,
|
|
282
|
+
`CREATE TABLE IF NOT EXISTS ${SYSTEM_KEY_PARTITION} PARTITION OF ${SYSTEM_PARTITION}
|
|
283
|
+
FOR VALUES IN ('Key')`,
|
|
284
|
+
`CREATE TABLE IF NOT EXISTS edges (
|
|
285
|
+
id uuid PRIMARY KEY,
|
|
286
|
+
from_ns text NOT NULL,
|
|
287
|
+
from_type text NOT NULL,
|
|
288
|
+
from_id text NOT NULL,
|
|
289
|
+
from_bridged boolean NOT NULL DEFAULT false,
|
|
290
|
+
to_ns text NOT NULL,
|
|
291
|
+
to_type text NOT NULL,
|
|
292
|
+
to_id text NOT NULL,
|
|
293
|
+
to_bridged boolean NOT NULL DEFAULT false,
|
|
294
|
+
field_name text NOT NULL,
|
|
295
|
+
policy text NOT NULL,
|
|
296
|
+
created_at timestamptz NOT NULL,
|
|
297
|
+
deleted_at timestamptz NULL,
|
|
298
|
+
last_swept_at timestamptz NULL,
|
|
299
|
+
blocked boolean NOT NULL DEFAULT false,
|
|
300
|
+
CONSTRAINT edges_policy_check
|
|
301
|
+
CHECK (policy IN ('set_null','cascade','restrict','orphan'))
|
|
302
|
+
)`,
|
|
303
|
+
`CREATE TABLE IF NOT EXISTS idempotency (
|
|
304
|
+
key text PRIMARY KEY,
|
|
305
|
+
envelope jsonb NOT NULL,
|
|
306
|
+
expires_at timestamptz NOT NULL,
|
|
307
|
+
created_at timestamptz NOT NULL DEFAULT now()
|
|
308
|
+
)`,
|
|
309
|
+
`CREATE TABLE IF NOT EXISTS short_ids (
|
|
310
|
+
namespace text NOT NULL,
|
|
311
|
+
type text NOT NULL,
|
|
312
|
+
short_id bigint NOT NULL,
|
|
313
|
+
row_id text NOT NULL,
|
|
314
|
+
PRIMARY KEY (namespace, type, short_id),
|
|
315
|
+
CONSTRAINT short_ids_row_id_fkey FOREIGN KEY (namespace, type, row_id)
|
|
316
|
+
REFERENCES rows (namespace, type, id) ON DELETE CASCADE
|
|
317
|
+
)`,
|
|
318
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS substrate_key_live_kid_uniq
|
|
319
|
+
ON ${SYSTEM_KEY_PARTITION} ((body ->> 'kid'))
|
|
320
|
+
WHERE deleted_at IS NULL
|
|
321
|
+
AND (body ->> 'revokedAt') IS NULL`
|
|
322
|
+
];
|
|
323
|
+
/** 0001's concurrent indexes, less those on `rows`. */
|
|
324
|
+
const CONCURRENT_STATEMENTS$2 = CONCURRENT_STATEMENTS$3.filter((s) => !/\sON rows\s/.test(s));
|
|
325
|
+
|
|
326
|
+
//#endregion
|
|
327
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0002_uuid_format_check.ts
|
|
328
|
+
/**
|
|
329
|
+
* 0002_uuid_format_check — CHECK constraints enforcing UUID format on
|
|
330
|
+
* `rows.id`, `short_ids.row_id`, `edges.from_id` and `edges.to_id`.
|
|
331
|
+
* (`edges.id` is a native uuid.)
|
|
332
|
+
*
|
|
333
|
+
* OPT-IN via `SUBSTRATE_ENFORCE_UUID_CHECK=true`: the test corpus uses synthetic
|
|
334
|
+
* non-UUID ids. Production enables it.
|
|
335
|
+
*
|
|
336
|
+
* The CHECK is the primary enforcement: a direct `dataset.write` bypasses
|
|
337
|
+
* autoCRUD's `randomUUID()` and must supply a UUID-shape id itself.
|
|
338
|
+
*/
|
|
339
|
+
/** The id shape the CHECKs enforce; migrate-store's preflight scans sources against it. */
|
|
340
|
+
const UUID_REGEX = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
|
|
341
|
+
const TX_STATEMENTS$2 = [
|
|
342
|
+
`DO $$ BEGIN
|
|
343
|
+
ALTER TABLE rows ADD CONSTRAINT rows_id_uuid_format
|
|
344
|
+
CHECK (id ~ '${UUID_REGEX}');
|
|
345
|
+
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
346
|
+
END $$`,
|
|
347
|
+
`DO $$ BEGIN
|
|
348
|
+
ALTER TABLE short_ids ADD CONSTRAINT short_ids_row_id_uuid_format
|
|
349
|
+
CHECK (row_id ~ '${UUID_REGEX}');
|
|
350
|
+
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
351
|
+
END $$`,
|
|
352
|
+
`DO $$ BEGIN
|
|
353
|
+
ALTER TABLE edges ADD CONSTRAINT edges_from_id_uuid_format
|
|
354
|
+
CHECK (from_id ~ '${UUID_REGEX}');
|
|
355
|
+
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
356
|
+
END $$`,
|
|
357
|
+
`DO $$ BEGIN
|
|
358
|
+
ALTER TABLE edges ADD CONSTRAINT edges_to_id_uuid_format
|
|
359
|
+
CHECK (to_id ~ '${UUID_REGEX}');
|
|
360
|
+
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
361
|
+
END $$`
|
|
362
|
+
];
|
|
363
|
+
|
|
364
|
+
//#endregion
|
|
365
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0003_safe_casts.ts
|
|
366
|
+
/**
|
|
367
|
+
* 0003_safe_casts — IMMUTABLE jsonb → numeric / timestamptz casts that return
|
|
368
|
+
* NULL on a wrong-shaped value instead of raising, so they can back index
|
|
369
|
+
* expressions and filters over untyped body fields.
|
|
370
|
+
*
|
|
371
|
+
* substrate_ts reads the ISO grammar of substrate-types' `normalizeIsoTimestamp`
|
|
372
|
+
* (same groups: date, HH:MM, seconds, first 6 fraction digits, offset). It is
|
|
373
|
+
* IMMUTABLE because the text is rebuilt with an explicit offset before the
|
|
374
|
+
* cast (TimeZone cannot matter) and is ISO (DateStyle cannot matter).
|
|
375
|
+
*/
|
|
376
|
+
const TS_SHAPE = String.raw`^([0-9]{4}-[0-9]{2}-[0-9]{2})(?:[T ]((?:[01][0-9]|2[0-3]):[0-5][0-9])(?::([0-5][0-9])(?:\.([0-9]{1,6})[0-9]{0,3})?)?)?(Z|[+-](?:0[0-9]|1[0-5])(?::?[0-5][0-9])?)?$`;
|
|
377
|
+
const TX_STATEMENTS$1 = [`CREATE OR REPLACE FUNCTION substrate_num(v jsonb) RETURNS pg_catalog.numeric
|
|
378
|
+
LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
|
379
|
+
AS $$ SELECT CASE
|
|
380
|
+
WHEN pg_catalog.jsonb_typeof(v) OPERATOR(pg_catalog.=) 'number'
|
|
381
|
+
THEN (v OPERATOR(pg_catalog.#>>) '{}')::pg_catalog.numeric
|
|
382
|
+
END $$`, `CREATE OR REPLACE FUNCTION substrate_ts(v jsonb) RETURNS pg_catalog.timestamptz
|
|
383
|
+
LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
|
384
|
+
AS $$ SELECT CASE
|
|
385
|
+
WHEN pg_catalog.pg_input_is_valid(t, 'pg_catalog.timestamptz') THEN t::pg_catalog.timestamptz
|
|
386
|
+
END
|
|
387
|
+
FROM (
|
|
388
|
+
-- No offset is UTC; a date alone is its midnight UTC; fraction digits past 6 are dropped.
|
|
389
|
+
SELECT m[1] OPERATOR(pg_catalog.||) 'T'
|
|
390
|
+
OPERATOR(pg_catalog.||) COALESCE(m[2], '00:00') OPERATOR(pg_catalog.||) ':'
|
|
391
|
+
OPERATOR(pg_catalog.||) COALESCE(m[3], '00')
|
|
392
|
+
OPERATOR(pg_catalog.||) COALESCE('.' OPERATOR(pg_catalog.||) m[4], '')
|
|
393
|
+
OPERATOR(pg_catalog.||) COALESCE(m[5], 'Z') AS t
|
|
394
|
+
FROM pg_catalog.regexp_match(
|
|
395
|
+
CASE WHEN pg_catalog.jsonb_typeof(v) OPERATOR(pg_catalog.=) 'string'
|
|
396
|
+
THEN v OPERATOR(pg_catalog.#>>) '{}' END,
|
|
397
|
+
'${TS_SHAPE}') AS m
|
|
398
|
+
) AS n $$`];
|
|
399
|
+
const CONCURRENT_STATEMENTS$1 = [];
|
|
400
|
+
|
|
401
|
+
//#endregion
|
|
402
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0004_visible_seq.ts
|
|
403
|
+
/**
|
|
404
|
+
* 0004_visible_seq — commit-ordered sequence for visible-order types.
|
|
405
|
+
*
|
|
406
|
+
* `rows.seq` holds a row's place in its type's commit order; it stays NULL on
|
|
407
|
+
* every type not declared `ordering: 'visible'`. On the partitioned layout the
|
|
408
|
+
* column propagates from the parent to every partition. Adding a nullable
|
|
409
|
+
* column with no default rewrites nothing.
|
|
410
|
+
*
|
|
411
|
+
* `visible_seqs` holds each visible-order type's last assigned seq. It only
|
|
412
|
+
* ever increases, so purging the newest rows never lets a later insert reuse
|
|
413
|
+
* their seq. Writers read and advance it under the type's advisory lock.
|
|
414
|
+
*/
|
|
415
|
+
const TX_STATEMENTS = ["ALTER TABLE rows ADD COLUMN IF NOT EXISTS seq bigint NULL", `CREATE TABLE IF NOT EXISTS visible_seqs (
|
|
416
|
+
namespace text NOT NULL,
|
|
417
|
+
type text NOT NULL,
|
|
418
|
+
last_seq bigint NOT NULL,
|
|
419
|
+
PRIMARY KEY (namespace, type)
|
|
420
|
+
)`];
|
|
421
|
+
const CONCURRENT_STATEMENTS = [];
|
|
422
|
+
|
|
423
|
+
//#endregion
|
|
424
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/primitives.ts
|
|
425
|
+
const idToText = (id) => typeof id === "number" ? String(id) : id;
|
|
426
|
+
/**
|
|
427
|
+
* Row-metadata field → native column. Predicates on these fields use the
|
|
428
|
+
* indexed columns rather than `body ->> field`. The `r.` alias matches
|
|
429
|
+
* `read()`'s SELECT.
|
|
430
|
+
*/
|
|
431
|
+
const METADATA_COLUMNS$1 = {
|
|
432
|
+
createdAt: "r.created_at",
|
|
433
|
+
updatedAt: "r.updated_at",
|
|
434
|
+
createdById: "r.created_by_id",
|
|
435
|
+
createdByKind: "r.created_by_kind",
|
|
436
|
+
updatedById: "r.updated_by_id",
|
|
437
|
+
updatedByKind: "r.updated_by_kind",
|
|
438
|
+
seq: "r.seq"
|
|
439
|
+
};
|
|
440
|
+
function isMetadataField(field) {
|
|
441
|
+
return Object.hasOwn(METADATA_COLUMNS$1, field);
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Legal bare column identifier: lower-snake, no quoting, no qualifier. This
|
|
445
|
+
* allow-list is what stands between a caller-supplied map and `sql.unsafe`.
|
|
446
|
+
*/
|
|
447
|
+
const COLUMN_NAME_PATTERN = /^[a-z_][a-z0-9_]*$/;
|
|
448
|
+
/**
|
|
449
|
+
* Validates a caller-supplied map and returns the lookup the SQL builders use.
|
|
450
|
+
* Throws if any column is not a bare identifier.
|
|
451
|
+
*
|
|
452
|
+
* The validation is load-bearing: the builders interpolate the column through
|
|
453
|
+
* `sql.unsafe`, and the map may come from config.
|
|
454
|
+
*/
|
|
455
|
+
function resolvePromotedColumns(map) {
|
|
456
|
+
const resolved = /* @__PURE__ */ new Map();
|
|
457
|
+
for (const [field, def] of Object.entries(map)) {
|
|
458
|
+
if (!COLUMN_NAME_PATTERN.test(def.column)) throw new Error(`promotedColumns['${field}']: column '${def.column}' is not a bare identifier (expected ${COLUMN_NAME_PATTERN.source}). The table qualifier is added internally; pass 'severity', not 'r.severity'.`);
|
|
459
|
+
resolved.set(field, def);
|
|
460
|
+
}
|
|
461
|
+
return (field) => resolved.get(field);
|
|
462
|
+
}
|
|
463
|
+
/** The qualified expression source for a promoted column — `r.<column>`. */
|
|
464
|
+
function promotedColumnSql(def) {
|
|
465
|
+
return `r.${def.column}`;
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Row-shape keys that DO NOT live in `r.body` — they're synthesised from
|
|
469
|
+
* dedicated columns (`_metadata`, `shortId`, `_deletedAt`) or extracted by
|
|
470
|
+
* `materialize()` (`id` lives both in the column and in body, but is force-
|
|
471
|
+
* included by the projection builder regardless of whether the caller asked).
|
|
472
|
+
*
|
|
473
|
+
* Used by `buildBodyProjection` to filter caller-supplied `fields` keys when
|
|
474
|
+
* deciding which body keys to materialise from JSONB.
|
|
475
|
+
*/
|
|
476
|
+
const NON_BODY_FIELD_KEYS = /* @__PURE__ */ new Set([
|
|
477
|
+
"_metadata",
|
|
478
|
+
"_deletedAt",
|
|
479
|
+
"shortId"
|
|
480
|
+
]);
|
|
481
|
+
/** An operand's text: the shared ISO grammar in its explicit-offset form (no offset is UTC), as `substrate_ts` reads a stored value. */
|
|
482
|
+
const isoOf = (v) => v instanceof Date ? v.toISOString() : normalizeIsoTimestamp(String(v)) ?? String(v);
|
|
483
|
+
/**
|
|
484
|
+
* A timestamptz parameter, bound as TEXT and cast in SQL. When pg infers a
|
|
485
|
+
* parameter's type as timestamptz (`$1::timestamptz`, or a compare against a
|
|
486
|
+
* timestamptz column), postgres.js serialises the value through a JS Date,
|
|
487
|
+
* which keeps milliseconds only: `.123456` arrives as `.123`, so equality
|
|
488
|
+
* misses and every range compare is off by the dropped microseconds.
|
|
489
|
+
*/
|
|
490
|
+
function tsParam(sql, v) {
|
|
491
|
+
return sql`${isoOf(v)}::text::timestamptz`;
|
|
492
|
+
}
|
|
493
|
+
/** {@link tsParam} for an array operand (`= ANY(...)`). */
|
|
494
|
+
function tsArrayParam(sql, vs) {
|
|
495
|
+
return sql`${vs.map(isoOf)}::text[]::timestamptz[]`;
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* A complete SQL string literal for `s`: `'o''neil'`. A value holding a
|
|
499
|
+
* backslash takes the `E'…'` form with the backslash doubled, because in a
|
|
500
|
+
* plain literal `standard_conforming_strings = off` would read `\'` as an
|
|
501
|
+
* escaped quote and let the value break out. Without a backslash the plain
|
|
502
|
+
* form reads the same under either setting, and keeps the text of existing
|
|
503
|
+
* index DDL unchanged. pg literals cannot carry NUL, so that is refused.
|
|
504
|
+
*/
|
|
505
|
+
function sqlStringLiteral(s) {
|
|
506
|
+
if (s.includes("\0")) throw new UnsupportedFilterError(s, "<name>", "refused: the field name contains NUL");
|
|
507
|
+
const quoted = s.replace(/'/g, "''");
|
|
508
|
+
return s.includes("\\") ? `E'${quoted.replace(/\\/g, "\\\\")}'` : `'${quoted}'`;
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* Body-field expressions as SQL TEXT, with every name inlined as a literal.
|
|
512
|
+
* The compiler and the index DDL both build from these, because an
|
|
513
|
+
* expression index serves only a filter whose expression matches it.
|
|
514
|
+
*/
|
|
515
|
+
const jsonSql = (path) => `body${path.map((p) => ` -> ${sqlStringLiteral(p)}`).join("")}`;
|
|
516
|
+
const rawTextSql = (path) => `(${jsonSql(path.slice(0, -1))} ->> ${sqlStringLiteral(path[path.length - 1])})`;
|
|
517
|
+
const textSql = (path) => `(${rawTextSql(path)} COLLATE "C")`;
|
|
518
|
+
const numSql = (path) => `substrate_num(${jsonSql(path)})`;
|
|
519
|
+
const tsSql = (path) => `substrate_ts(${jsonSql(path)})`;
|
|
520
|
+
/** `((body -> 'a' ->> 'b') COLLATE "C")`: the one text expression for compares and sorts. */
|
|
521
|
+
function bodyText(sql, path) {
|
|
522
|
+
return sql.unsafe(textSql(path));
|
|
523
|
+
}
|
|
524
|
+
/** `(body -> 'a' ->> 'b')` uncollated, for ILIKE, which a "C" collation would not case-fold. */
|
|
525
|
+
function bodyRawText(sql, path) {
|
|
526
|
+
return sql.unsafe(rawTextSql(path));
|
|
527
|
+
}
|
|
528
|
+
/** `(body -> 'a' -> 'b')`: the leaf stays jsonb. */
|
|
529
|
+
function bodyJson(sql, path) {
|
|
530
|
+
return sql.unsafe(`(${jsonSql(path)})`);
|
|
531
|
+
}
|
|
532
|
+
/** `substrate_num(body -> …)`: the number, or NULL for any other jsonb. */
|
|
533
|
+
function bodyNum(sql, path) {
|
|
534
|
+
return sql.unsafe(numSql(path));
|
|
535
|
+
}
|
|
536
|
+
/** `substrate_ts(body -> …)`: the instant, or NULL for anything but an ISO date or date-time string. */
|
|
537
|
+
function bodyTs(sql, path) {
|
|
538
|
+
return sql.unsafe(tsSql(path));
|
|
539
|
+
}
|
|
540
|
+
/** The DDL text of the expression the compiler emits for a top-level field. */
|
|
541
|
+
function indexExpressionSql(kind, field) {
|
|
542
|
+
return kind === "text" ? textSql([field]) : kind === "number" ? numSql([field]) : tsSql([field]);
|
|
543
|
+
}
|
|
544
|
+
/**
|
|
545
|
+
* `cond` where `expr` is not NULL, else FALSE, never NULL, so a caller's NOT
|
|
546
|
+
* stays sound. The bare `cond` conjunct keeps it index-servable, where a
|
|
547
|
+
* COALESCE would hide it from the planner.
|
|
548
|
+
*/
|
|
549
|
+
function notNullAnd(sql, expr, cond) {
|
|
550
|
+
return sql`(${expr} IS NOT NULL AND ${cond})`;
|
|
551
|
+
}
|
|
552
|
+
const asJsonValue = (v) => v;
|
|
553
|
+
function scalarParam(value) {
|
|
554
|
+
if (value instanceof Date) return value.toISOString();
|
|
555
|
+
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
556
|
+
return String(value);
|
|
557
|
+
}
|
|
558
|
+
function escapeLike(s) {
|
|
559
|
+
return s.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
|
|
560
|
+
}
|
|
561
|
+
/** Operand-shape guard for string operands. See UnsupportedFilterError. */
|
|
562
|
+
function metaStr(field, op, v) {
|
|
563
|
+
if (typeof v !== "string") throw new UnsupportedFilterError(field, op, "requires a string value");
|
|
564
|
+
return v;
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* Join a list of boolean fragments with AND. Returns a degenerate `TRUE`
|
|
568
|
+
* fragment when the list is empty so callers can always drop it into a
|
|
569
|
+
* `WHERE ${whereSql}` slot without a branch.
|
|
570
|
+
*
|
|
571
|
+
* Composition is flat: postgres.js joins an interpolated Fragment[] with
|
|
572
|
+
* spaces, so interleaving `AND` fragments yields one template with no nested
|
|
573
|
+
* PendingQuery per predicate.
|
|
574
|
+
*/
|
|
575
|
+
function combineAnd(sql, fragments) {
|
|
576
|
+
if (fragments.length === 0) return sql`TRUE`;
|
|
577
|
+
if (fragments.length === 1) return sql`(${fragments[0]})`;
|
|
578
|
+
return sql`(${interleaveOperator(sql, fragments, " AND ")})`;
|
|
579
|
+
}
|
|
580
|
+
function combineOr(sql, fragments) {
|
|
581
|
+
if (fragments.length === 0) return sql`FALSE`;
|
|
582
|
+
if (fragments.length === 1) return sql`(${fragments[0]})`;
|
|
583
|
+
return sql`(${interleaveOperator(sql, fragments, " OR ")})`;
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Build `[frag0, sql`<op>`, frag1, sql`<op>`, ...]` so postgres.js's
|
|
587
|
+
* Fragment[] interpolation produces `frag0 <op> frag1 <op> ...` in a single
|
|
588
|
+
* flat template. Callers handle the empty and single-fragment cases.
|
|
589
|
+
*/
|
|
590
|
+
function interleaveOperator(sql, fragments, op) {
|
|
591
|
+
const opFragment = op === " AND " ? sql`AND` : sql`OR`;
|
|
592
|
+
const out = [fragments[0]];
|
|
593
|
+
for (let i = 1; i < fragments.length; i++) out.push(opFragment, fragments[i]);
|
|
594
|
+
return out;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
//#endregion
|
|
598
|
+
//#region ../../packages/substrate-dataset-pg/src/server-version.ts
|
|
599
|
+
/** PostgreSQL 18.0, as `server_version_num` spells it. */
|
|
600
|
+
const MIN_SERVER_VERSION_NUM = 18e4;
|
|
601
|
+
/** The connected server is older than {@link MIN_SERVER_VERSION_NUM}. */
|
|
602
|
+
var UnsupportedServerVersionError = class extends Error {
|
|
603
|
+
found;
|
|
604
|
+
constructor(found) {
|
|
605
|
+
super(`substrate requires PostgreSQL 18 or newer; the server is PostgreSQL ${Math.floor(found / 1e4)}.${found % 1e4} (server_version_num ${found})`);
|
|
606
|
+
this.found = found;
|
|
607
|
+
this.name = "UnsupportedServerVersionError";
|
|
608
|
+
}
|
|
609
|
+
};
|
|
610
|
+
/** Throws {@link UnsupportedServerVersionError} when `num` is below the minimum. */
|
|
611
|
+
function checkServerVersionNum(num) {
|
|
612
|
+
if (num < 18e4) throw new UnsupportedServerVersionError(num);
|
|
613
|
+
}
|
|
614
|
+
/** Refuses a server older than PostgreSQL 18. */
|
|
615
|
+
async function assertServerVersion(sql) {
|
|
616
|
+
const [row] = await sql`
|
|
617
|
+
SELECT current_setting('server_version_num')::int AS num
|
|
618
|
+
`;
|
|
619
|
+
checkServerVersionNum(row?.num ?? 0);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
//#endregion
|
|
623
|
+
//#region ../../packages/substrate-dataset-pg/src/migrate.ts
|
|
624
|
+
/**
|
|
625
|
+
* Schema migrations for the pg Dataset + EdgesStore. Spec: §4.5, §4.6,
|
|
626
|
+
* §5.13.
|
|
627
|
+
*
|
|
628
|
+
* Each migration is a numbered module under `migrations/` exporting two
|
|
629
|
+
* statement buckets:
|
|
630
|
+
*
|
|
631
|
+
* - `txStatements` (or legacy `statements`) — DDL/DML safe inside a single
|
|
632
|
+
* transaction. The runner wraps the whole bucket in `BEGIN ... COMMIT`,
|
|
633
|
+
* so a partial-apply rolls back cleanly. Most migrations use this bucket
|
|
634
|
+
* exclusively.
|
|
635
|
+
* - `concurrentStatements` — `CREATE INDEX CONCURRENTLY`,
|
|
636
|
+
* `DROP INDEX CONCURRENTLY`, `REINDEX CONCURRENTLY`, etc. Postgres rejects
|
|
637
|
+
* these inside a transaction, so the runner executes them one-by-one
|
|
638
|
+
* AFTER the tx commits. A failure in this phase leaves the tx-phase work
|
|
639
|
+
* durable; re-running the migration is safe as long as each statement is
|
|
640
|
+
* `IF NOT EXISTS` / `IF EXISTS` guarded (idempotent), which is the
|
|
641
|
+
* convention for the concurrent bucket.
|
|
642
|
+
*
|
|
643
|
+
* The runner records applied versions in `migrations(version,
|
|
644
|
+
* applied_at, checksum)` and skips already-applied migrations on re-run.
|
|
645
|
+
* Crucially, the version is only marked applied AFTER both the tx phase
|
|
646
|
+
* commits AND the concurrent phase finishes successfully — a concurrent
|
|
647
|
+
* failure throws and the migration is left "in progress" so a re-run picks
|
|
648
|
+
* up where it left off (idempotent statements absorb the partial state).
|
|
649
|
+
*
|
|
650
|
+
* New migrations go in `migrations/` and are registered in MIGRATIONS below.
|
|
651
|
+
* Shipped migration files are IMMUTABLE — change schema only via a new file.
|
|
652
|
+
*/
|
|
653
|
+
const LEGACY_BASELINE = {
|
|
654
|
+
version: "0001_baseline",
|
|
655
|
+
txStatements: TX_STATEMENTS$4,
|
|
656
|
+
concurrentStatements: CONCURRENT_STATEMENTS$3
|
|
657
|
+
};
|
|
658
|
+
const PARTITIONED_BASELINE = {
|
|
659
|
+
version: "0001p_partitioned_baseline",
|
|
660
|
+
txStatements: TX_STATEMENTS$3,
|
|
661
|
+
concurrentStatements: CONCURRENT_STATEMENTS$2
|
|
662
|
+
};
|
|
663
|
+
/** Migrations after the baseline, run in both layouts. */
|
|
664
|
+
const SHARED_INPUTS = [{
|
|
665
|
+
version: "0003_safe_casts",
|
|
666
|
+
txStatements: TX_STATEMENTS$1,
|
|
667
|
+
concurrentStatements: CONCURRENT_STATEMENTS$1
|
|
668
|
+
}, {
|
|
669
|
+
version: "0004_visible_seq",
|
|
670
|
+
txStatements: TX_STATEMENTS,
|
|
671
|
+
concurrentStatements: CONCURRENT_STATEMENTS
|
|
672
|
+
}];
|
|
673
|
+
function normalise(input) {
|
|
674
|
+
if ("statements" in input && input.statements !== void 0) return {
|
|
675
|
+
version: input.version,
|
|
676
|
+
txStatements: input.statements,
|
|
677
|
+
concurrentStatements: []
|
|
678
|
+
};
|
|
679
|
+
return {
|
|
680
|
+
version: input.version,
|
|
681
|
+
txStatements: input.txStatements ?? [],
|
|
682
|
+
concurrentStatements: input.concurrentStatements ?? []
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
/**
|
|
686
|
+
* Opt-in migration: UUID-format CHECKs on row, short-id and edge ids.
|
|
687
|
+
* Registered only when `SUBSTRATE_ENFORCE_UUID_CHECK=true`, because the test
|
|
688
|
+
* corpus uses synthetic non-UUID ids.
|
|
689
|
+
*/
|
|
690
|
+
/** The ledger version of the opt-in UUID-format CHECKs. */
|
|
691
|
+
const UUID_CHECK_MIGRATION = "0002_uuid_format_check";
|
|
692
|
+
const UUID_CHECK_INPUT = {
|
|
693
|
+
version: UUID_CHECK_MIGRATION,
|
|
694
|
+
txStatements: TX_STATEMENTS$2
|
|
695
|
+
};
|
|
696
|
+
/**
|
|
697
|
+
* The baseline for `layout`, then the gated 0002 when
|
|
698
|
+
* `SUBSTRATE_ENFORCE_UUID_CHECK=true` (or the explicit override is true), then
|
|
699
|
+
* the migrations both layouts share. Used by `runMigrations` and exported for
|
|
700
|
+
* unit tests.
|
|
701
|
+
*/
|
|
702
|
+
function getMigrationList(enforceUuidCheck = process.env.SUBSTRATE_ENFORCE_UUID_CHECK === "true", layout = "legacy") {
|
|
703
|
+
return [
|
|
704
|
+
layout === "partitioned" ? PARTITIONED_BASELINE : LEGACY_BASELINE,
|
|
705
|
+
...enforceUuidCheck === true ? [UUID_CHECK_INPUT] : [],
|
|
706
|
+
...SHARED_INPUTS
|
|
707
|
+
].map(normalise);
|
|
708
|
+
}
|
|
709
|
+
/** @internal — the legacy layout's always-on list, exported for unit tests. */
|
|
710
|
+
const MIGRATIONS = getMigrationList(false);
|
|
711
|
+
/**
|
|
712
|
+
* Raised when the ledger records one layout's baseline but `rows` has the
|
|
713
|
+
* other layout, which means the schema was changed outside the runner.
|
|
714
|
+
*/
|
|
715
|
+
var StorageLayoutMismatchError = class extends Error {
|
|
716
|
+
name = "StorageLayoutMismatchError";
|
|
717
|
+
constructor(layout, recordedBaseline) {
|
|
718
|
+
super(`rows is ${layout} but the migrations ledger records ${recordedBaseline}. A database keeps the layout it was created with; restore the matching schema or migrate into a fresh one.`);
|
|
719
|
+
}
|
|
720
|
+
};
|
|
721
|
+
/** The table `substrate-admin migrate-store` keeps its progress in, on the target. */
|
|
722
|
+
const MIGRATE_STORE_MARKER_TABLE = "substrate_migrate_store_progress";
|
|
723
|
+
var StoreMigrationInProgressError = class extends Error {
|
|
724
|
+
name = "StoreMigrationInProgressError";
|
|
725
|
+
constructor() {
|
|
726
|
+
super(`${MIGRATE_STORE_MARKER_TABLE} exists: substrate-admin migrate-store has not finished populating this store. Rerun substrate-admin migrate-store to finish or restart it, then boot.`);
|
|
727
|
+
}
|
|
728
|
+
};
|
|
729
|
+
/**
|
|
730
|
+
* Throws `StoreMigrationInProgressError` when the current schema carries the
|
|
731
|
+
* migrate-store marker. Compares `nspname` rather than casting, so a schema
|
|
732
|
+
* name that needs quoting still matches.
|
|
733
|
+
*/
|
|
734
|
+
async function assertNotMigrating(sql) {
|
|
735
|
+
const [row] = await sql`
|
|
736
|
+
SELECT EXISTS (
|
|
737
|
+
SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
738
|
+
WHERE c.relname = ${MIGRATE_STORE_MARKER_TABLE} AND n.nspname = current_schema()
|
|
739
|
+
) AS present
|
|
740
|
+
`;
|
|
741
|
+
if (row?.present === true) throw new StoreMigrationInProgressError();
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* The layout migrations target: the layout `rows` has; else the one the
|
|
745
|
+
* ledger's baseline records; else `fallback`, then `SUBSTRATE_STORAGE_LAYOUT`,
|
|
746
|
+
* then `partitioned`. Throws when `rows` and the ledger disagree.
|
|
747
|
+
*/
|
|
748
|
+
async function resolveLayout(sql, applied, fallback) {
|
|
749
|
+
const recorded = applied.has(LEGACY_BASELINE.version) ? "legacy" : applied.has(PARTITIONED_BASELINE.version) ? "partitioned" : void 0;
|
|
750
|
+
const actual = await detectLayout(sql);
|
|
751
|
+
if (actual === "absent") return recorded ?? fallback ?? envLayout() ?? "partitioned";
|
|
752
|
+
if (recorded !== void 0 && recorded !== actual) throw new StorageLayoutMismatchError(actual, recorded === "legacy" ? LEGACY_BASELINE.version : PARTITIONED_BASELINE.version);
|
|
753
|
+
return actual;
|
|
754
|
+
}
|
|
755
|
+
function envLayout() {
|
|
756
|
+
const v = process.env.SUBSTRATE_STORAGE_LAYOUT;
|
|
757
|
+
if (v === void 0 || v === "") return void 0;
|
|
758
|
+
if (v === "legacy" || v === "partitioned") return v;
|
|
759
|
+
throw new Error(`SUBSTRATE_STORAGE_LAYOUT must be 'legacy' or 'partitioned', got '${v}'`);
|
|
760
|
+
}
|
|
761
|
+
/**
|
|
762
|
+
* sha256 across both buckets in fixed order (tx then concurrent).
|
|
763
|
+
*/
|
|
764
|
+
/**
|
|
765
|
+
* @internal — exported for unit tests. Not re-exported from the package index.
|
|
766
|
+
*/
|
|
767
|
+
function checksum(migration) {
|
|
768
|
+
const hasher = createHash("sha256");
|
|
769
|
+
for (const s of migration.txStatements) {
|
|
770
|
+
hasher.update(s);
|
|
771
|
+
hasher.update(" ");
|
|
772
|
+
}
|
|
773
|
+
for (const s of migration.concurrentStatements) {
|
|
774
|
+
hasher.update(s);
|
|
775
|
+
hasher.update(" ");
|
|
776
|
+
}
|
|
777
|
+
return hasher.digest("hex");
|
|
778
|
+
}
|
|
779
|
+
var NoCurrentSchemaError = class extends Error {
|
|
780
|
+
name = "NoCurrentSchemaError";
|
|
781
|
+
constructor() {
|
|
782
|
+
super("the search_path resolves to no existing schema, so there is nowhere to migrate. Create the schema, or correct search_path in the connection options.");
|
|
783
|
+
}
|
|
784
|
+
};
|
|
785
|
+
/** The schema names substrate supports: plain lowercase identifiers that never need quoting. */
|
|
786
|
+
const PLAIN_SCHEMA_NAME = /^[a-z_][a-z0-9_]*$/;
|
|
787
|
+
var UnsupportedSchemaNameError = class extends Error {
|
|
788
|
+
schema;
|
|
789
|
+
name = "UnsupportedSchemaNameError";
|
|
790
|
+
constructor(schema) {
|
|
791
|
+
super(`schema ${JSON.stringify(schema)} is not supported: substrate needs a plain lowercase schema name (lowercase letters, digits and underscores, not starting with a digit).`);
|
|
792
|
+
this.schema = schema;
|
|
793
|
+
}
|
|
794
|
+
};
|
|
795
|
+
async function runMigrations(sql, opts = {}) {
|
|
796
|
+
await assertServerVersion(sql);
|
|
797
|
+
await withMigrationLock(sql, (conn) => runMigrationsLocked(sql, conn, opts));
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* Versions of the core migration list not yet applied to `sql`'s database,
|
|
801
|
+
* read without applying or creating anything. All of them when the
|
|
802
|
+
* `migrations` table does not exist.
|
|
803
|
+
*/
|
|
804
|
+
async function pendingMigrations(sql) {
|
|
805
|
+
const [ledger] = await sql`
|
|
806
|
+
SELECT to_regclass('migrations') IS NOT NULL AS present
|
|
807
|
+
`;
|
|
808
|
+
const applied = ledger?.present === true ? new Set((await sql`SELECT version FROM migrations`).map((r) => r.version)) : /* @__PURE__ */ new Set();
|
|
809
|
+
return getMigrationList(void 0, await resolveLayout(sql, applied, void 0)).map((m) => m.version).filter((v) => !applied.has(v));
|
|
810
|
+
}
|
|
811
|
+
const MIGRATION_LOCK_KEY = `hashtext('substrate-migrations:' || current_schema())`;
|
|
812
|
+
/** Refuses a search_path that resolves to no schema, or a schema name that is not plain lowercase. */
|
|
813
|
+
async function assertStoreSchema(sql) {
|
|
814
|
+
const [current] = await sql`SELECT current_schema() AS schema`;
|
|
815
|
+
if (current?.schema == null) throw new NoCurrentSchemaError();
|
|
816
|
+
if (!PLAIN_SCHEMA_NAME.test(current.schema)) throw new UnsupportedSchemaNameError(current.schema);
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Runs `fn` while holding the migration advisory lock of the current schema on
|
|
820
|
+
* one reserved connection.
|
|
821
|
+
*
|
|
822
|
+
* The lock and unlock must share a session: through the pool, the unlock can
|
|
823
|
+
* land on another connection and leave the lock held. `fn` is handed that same
|
|
824
|
+
* connection, so a pool of one cannot deadlock waiting for a second. It has no
|
|
825
|
+
* `begin()`; open transactions with `BEGIN`/`COMMIT` statements.
|
|
826
|
+
*/
|
|
827
|
+
async function withMigrationLock(sql, fn) {
|
|
828
|
+
await assertStoreSchema(sql);
|
|
829
|
+
const lockConn = await sql.reserve();
|
|
830
|
+
try {
|
|
831
|
+
return await withPolledLock(sql, lockConn, "migration", MIGRATION_LOCK_KEY, [], void 0, () => fn(lockConn));
|
|
832
|
+
} finally {
|
|
833
|
+
lockConn.release();
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
/** Per schema and table; `$1` is the table name. */
|
|
837
|
+
const INDEX_LOCK_KEY = `hashtext('substrate-index-build:' || current_schema() || '.' || $1::text)`;
|
|
838
|
+
/** `withTableIndexLock` gave up waiting because its signal aborted. */
|
|
839
|
+
var IndexLockAborted = class extends Error {
|
|
840
|
+
constructor() {
|
|
841
|
+
super("index build lock wait aborted");
|
|
842
|
+
this.name = "IndexLockAborted";
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
/**
|
|
846
|
+
* Runs `fn` while `conn` holds the index-build advisory lock of `table` (in the
|
|
847
|
+
* current schema). Every engine CREATE or DROP INDEX CONCURRENTLY, swap and
|
|
848
|
+
* statistics step on a table runs under it: two concurrent builds on one table
|
|
849
|
+
* deadlock in Postgres (measured on pg17 and pg18: one is aborted with 40P01), while
|
|
850
|
+
* builds on different tables do not. It is taken after the migration lock,
|
|
851
|
+
* never before it, so a holder of this lock never waits for the migration lock.
|
|
852
|
+
*
|
|
853
|
+
* `conn` must be a connection reserved from `sql` with no open transaction,
|
|
854
|
+
* and so with no snapshot a holder's concurrent build would wait for. The lock
|
|
855
|
+
* is polled, as the migration lock is, and the poll gives up with
|
|
856
|
+
* {@link IndexLockAborted} once `signal` aborts. If the lock cannot be
|
|
857
|
+
* released, `conn`'s session is retired from `sql`, and the caller's
|
|
858
|
+
* `release()` of `conn` becomes a no-op, so it never returns to the pool.
|
|
859
|
+
*/
|
|
860
|
+
function withTableIndexLock(sql, conn, table, signal, fn) {
|
|
861
|
+
return withPolledLock(sql, conn, `index-build ${table}`, INDEX_LOCK_KEY, [table], signal, fn);
|
|
862
|
+
}
|
|
863
|
+
/**
|
|
864
|
+
* Runs `fn` while `conn` holds the session advisory lock `key` (SQL over
|
|
865
|
+
* `params`; `name` says which in logs), polled every 250 ms with pg_try_advisory_lock until it is free or
|
|
866
|
+
* `signal` aborts ({@link IndexLockAborted}). If the lock cannot be released,
|
|
867
|
+
* `conn`'s session is retired from `sql`, since it may still hold the lock,
|
|
868
|
+
* and `conn.release()` becomes a no-op so it never returns to the pool. When
|
|
869
|
+
* `fn` threw, its error is the one rethrown.
|
|
870
|
+
*/
|
|
871
|
+
async function withPolledLock(sql, conn, name, key, params, signal, fn) {
|
|
872
|
+
let session;
|
|
873
|
+
for (;;) {
|
|
874
|
+
if (signal?.aborted === true) throw new IndexLockAborted();
|
|
875
|
+
const [row] = await conn.unsafe(`SELECT pg_try_advisory_lock(${key}) AS acquired, ${SESSION_COLUMNS}`, params);
|
|
876
|
+
if (row?.acquired === true) {
|
|
877
|
+
session = row;
|
|
878
|
+
break;
|
|
879
|
+
}
|
|
880
|
+
await abortableSleep(250, signal);
|
|
881
|
+
}
|
|
882
|
+
const release = async () => {
|
|
883
|
+
const unlock = () => conn.unsafe(`SELECT pg_advisory_unlock(${key}) AS released`, params);
|
|
884
|
+
try {
|
|
885
|
+
const [row] = await unlock().catch(unlock);
|
|
886
|
+
if (row?.released === false) process.stderr.write(`substrate: advisory unlock of the ${name} lock reported it was not held by this session\n`);
|
|
887
|
+
} catch (err) {
|
|
888
|
+
if (!await retireSession(sql, session)) conn.release = () => {};
|
|
889
|
+
throw err;
|
|
890
|
+
}
|
|
891
|
+
};
|
|
892
|
+
let result;
|
|
893
|
+
try {
|
|
894
|
+
result = await fn();
|
|
895
|
+
} catch (err) {
|
|
896
|
+
await release().catch(() => {});
|
|
897
|
+
throw err;
|
|
898
|
+
}
|
|
899
|
+
await release();
|
|
900
|
+
return result;
|
|
901
|
+
}
|
|
902
|
+
/** Resolves after `ms`, or as soon as `signal` aborts. */
|
|
903
|
+
function abortableSleep(ms, signal) {
|
|
904
|
+
return new Promise((resolve) => {
|
|
905
|
+
const done = () => {
|
|
906
|
+
clearTimeout(timer);
|
|
907
|
+
signal?.removeEventListener("abort", done);
|
|
908
|
+
resolve();
|
|
909
|
+
};
|
|
910
|
+
const timer = setTimeout(done, ms);
|
|
911
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
/** `sql` is the pool; `conn` holds the migration lock and runs every statement. */
|
|
915
|
+
async function runMigrationsLocked(pool, sql, opts) {
|
|
916
|
+
if (opts.force === true) await sql`DROP TABLE IF EXISTS migrations`;
|
|
917
|
+
await sql`
|
|
918
|
+
CREATE TABLE IF NOT EXISTS migrations (
|
|
919
|
+
version text PRIMARY KEY,
|
|
920
|
+
applied_at timestamptz NOT NULL DEFAULT now(),
|
|
921
|
+
checksum text NOT NULL
|
|
922
|
+
)
|
|
923
|
+
`;
|
|
924
|
+
await sql`
|
|
925
|
+
CREATE TABLE IF NOT EXISTS migrations_audit (
|
|
926
|
+
id bigserial PRIMARY KEY,
|
|
927
|
+
version text NOT NULL,
|
|
928
|
+
previous_checksum text NOT NULL,
|
|
929
|
+
new_checksum text NOT NULL,
|
|
930
|
+
observed_at timestamptz NOT NULL DEFAULT now(),
|
|
931
|
+
reason text NOT NULL DEFAULT 'rehash',
|
|
932
|
+
build_sha text
|
|
933
|
+
)
|
|
934
|
+
`;
|
|
935
|
+
const applied = await sql`
|
|
936
|
+
SELECT version, checksum FROM migrations
|
|
937
|
+
`;
|
|
938
|
+
const appliedByVersion = new Map(applied.map((r) => [r.version, r.checksum]));
|
|
939
|
+
const REHASH_LIST = [...getMigrationList(true), normalise(PARTITIONED_BASELINE)];
|
|
940
|
+
const buildSha = process.env.SUBSTRATE_BUILD_SHA ?? "unknown";
|
|
941
|
+
for (const migration of REHASH_LIST) {
|
|
942
|
+
const recorded = appliedByVersion.get(migration.version);
|
|
943
|
+
if (recorded === void 0) continue;
|
|
944
|
+
const current = checksum(migration);
|
|
945
|
+
if (recorded === current) continue;
|
|
946
|
+
process.stderr.write(`⚠️ migrations rehash: ${migration.version} (was=${recorded.slice(0, 12)} now=${current.slice(0, 12)} build=${buildSha}) — framework hash-format change masquerading as drift; recurring WARNs for the same version across deploys signal a true file edit.\n`);
|
|
947
|
+
await sql`
|
|
948
|
+
INSERT INTO migrations_audit (version, previous_checksum, new_checksum, build_sha)
|
|
949
|
+
VALUES (${migration.version}, ${recorded}, ${current}, ${buildSha})
|
|
950
|
+
`;
|
|
951
|
+
await sql`
|
|
952
|
+
UPDATE migrations SET checksum = ${current} WHERE version = ${migration.version}
|
|
953
|
+
`;
|
|
954
|
+
appliedByVersion.set(migration.version, current);
|
|
955
|
+
}
|
|
956
|
+
const layout = await resolveLayout(sql, new Set(appliedByVersion.keys()), opts.layout);
|
|
957
|
+
for (const migration of getMigrationList(opts.enforceUuidCheck, layout)) {
|
|
958
|
+
if (appliedByVersion.get(migration.version) !== void 0) continue;
|
|
959
|
+
await applyOneMigration(pool, sql, migration);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
/**
|
|
963
|
+
* Apply one migration through the tracked machinery: tx-safe statements in a
|
|
964
|
+
* single BEGIN/COMMIT bucket, then concurrent statements one-by-one OUTSIDE any
|
|
965
|
+
* tx (Postgres rejects `CREATE INDEX CONCURRENTLY` in a transaction), then the
|
|
966
|
+
* `migrations` row. The version lands only after the concurrent phase
|
|
967
|
+
* succeeds, so a partial concurrent failure leaves the migration "in progress"
|
|
968
|
+
* and a re-run (statements are IF [NOT] EXISTS guarded) resumes it.
|
|
969
|
+
*/
|
|
970
|
+
async function applyOneMigration(pool, sql, migration) {
|
|
971
|
+
if (migration.txStatements.length > 0) {
|
|
972
|
+
await sql.unsafe("BEGIN");
|
|
973
|
+
try {
|
|
974
|
+
for (const stmt of migration.txStatements) await sql.unsafe(stmt);
|
|
975
|
+
await sql.unsafe("COMMIT");
|
|
976
|
+
} catch (err) {
|
|
977
|
+
await sql.unsafe("ROLLBACK");
|
|
978
|
+
throw err;
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
const concurrentPhase = async () => {
|
|
982
|
+
for (const stmt of migration.concurrentStatements) try {
|
|
983
|
+
await dropInvalidIndexFor(sql, stmt);
|
|
984
|
+
await sql.unsafe(stmt);
|
|
985
|
+
} catch (err) {
|
|
986
|
+
process.stderr.write(`migration ${migration.version}: concurrent-phase failure on statement; tx-phase changes are durable, re-run to resume concurrent phase: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
987
|
+
throw err;
|
|
988
|
+
}
|
|
989
|
+
};
|
|
990
|
+
if (migration.concurrentStatements.length > 0) await withTableIndexLock(pool, sql, "rows", void 0, concurrentPhase);
|
|
991
|
+
await sql`
|
|
992
|
+
INSERT INTO migrations (version, checksum)
|
|
993
|
+
VALUES (${migration.version}, ${checksum(migration)})
|
|
994
|
+
`;
|
|
995
|
+
}
|
|
996
|
+
const CONCURRENT_INDEX_RE = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+IF\s+NOT\s+EXISTS\s+([a-z_][a-z0-9_]*)\s/i;
|
|
997
|
+
/**
|
|
998
|
+
* A failed `CREATE INDEX CONCURRENTLY` leaves an INVALID index behind, which
|
|
999
|
+
* `IF NOT EXISTS` would accept forever. Drops the statement's index first when
|
|
1000
|
+
* it exists and is invalid, so the re-run builds it again.
|
|
1001
|
+
*/
|
|
1002
|
+
async function dropInvalidIndexFor(sql, stmt) {
|
|
1003
|
+
const name = CONCURRENT_INDEX_RE.exec(stmt)?.[1];
|
|
1004
|
+
if (name !== void 0) await dropIndexIfInvalid(sql, name);
|
|
1005
|
+
}
|
|
1006
|
+
/**
|
|
1007
|
+
* Drops index `name` (a bare lower-snake identifier) when it exists and is
|
|
1008
|
+
* INVALID, as a failed concurrent build leaves it. True when it dropped one.
|
|
1009
|
+
*/
|
|
1010
|
+
async function dropIndexIfInvalid(sql, name) {
|
|
1011
|
+
if (!/^[a-z_][a-z0-9_]*$/.test(name)) throw new Error(`not a bare index name: '${name}'`);
|
|
1012
|
+
const [row] = await sql`
|
|
1013
|
+
SELECT NOT indisvalid AS invalid FROM pg_index WHERE indexrelid = to_regclass(${name})
|
|
1014
|
+
`;
|
|
1015
|
+
if (row?.invalid !== true) return false;
|
|
1016
|
+
await sql.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`);
|
|
1017
|
+
return true;
|
|
1018
|
+
}
|
|
1019
|
+
/**
|
|
1020
|
+
* `rows_<ns>_<type>_<field>_<hash>_<suffix>`, within pg's 63-byte identifier
|
|
1021
|
+
* limit. The readable part is folded to `[a-z0-9_]` and capped, so it can
|
|
1022
|
+
* collide (`my-tags` and `my_tags`, or a long shared prefix); the hash of the
|
|
1023
|
+
* raw names keeps such fields from sharing one index or constraint.
|
|
1024
|
+
*/
|
|
1025
|
+
function generatedIdentifier(namespace, type, field, suffix) {
|
|
1026
|
+
const fold = (s) => s.toLowerCase().replace(/[^a-z0-9_]/g, "_");
|
|
1027
|
+
const tail = `_${createHash("sha256").update(JSON.stringify([
|
|
1028
|
+
namespace,
|
|
1029
|
+
type,
|
|
1030
|
+
field
|
|
1031
|
+
])).digest("hex").slice(0, 8)}_${suffix}`;
|
|
1032
|
+
return `rows_${fold(namespace)}_${fold(type)}_${fold(field)}`.slice(0, 63 - tail.length) + tail;
|
|
1033
|
+
}
|
|
1034
|
+
const LIST_GIN_PREFIX = "list-gin:";
|
|
1035
|
+
/** True for a ledger version written by `runDerivedMigrations`, which a store re-derives at boot. */
|
|
1036
|
+
function isDerivedMigration(version) {
|
|
1037
|
+
return version.startsWith(LIST_GIN_PREFIX);
|
|
1038
|
+
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Derives the tracked, idempotent CONCURRENTLY GIN migration for one indexed
|
|
1041
|
+
* field. The version key uses the raw ns/type/field.
|
|
1042
|
+
*/
|
|
1043
|
+
function deriveListIndexMigration(f) {
|
|
1044
|
+
const name = generatedIdentifier(f.namespace, f.type, f.field, "gin");
|
|
1045
|
+
const ns = sqlStringLiteral(f.namespace);
|
|
1046
|
+
const type = sqlStringLiteral(f.type);
|
|
1047
|
+
const field = sqlStringLiteral(f.field);
|
|
1048
|
+
return {
|
|
1049
|
+
version: `${LIST_GIN_PREFIX}${f.namespace}:${f.type}:${f.field}`,
|
|
1050
|
+
txStatements: [],
|
|
1051
|
+
concurrentStatements: [`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${name}\n ON rows USING GIN ((body -> ${field}) jsonb_path_ops)\n WHERE namespace = ${ns} AND type = ${type} AND deleted_at IS NULL`]
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Applies migrations derived from field definitions through the same tracked,
|
|
1056
|
+
* advisory-locked machinery as the static list. Called after plugins load,
|
|
1057
|
+
* since `runMigrations` has no field-def knowledge. Idempotent: applied
|
|
1058
|
+
* versions are skipped.
|
|
1059
|
+
*
|
|
1060
|
+
* On the partitioned layout the list/json GIN migrations are skipped: those
|
|
1061
|
+
* indexes live on each type's partition, built by `reconcileTypeIndexes`, and
|
|
1062
|
+
* Postgres cannot build one CONCURRENTLY on the partitioned parent.
|
|
1063
|
+
*/
|
|
1064
|
+
async function runDerivedMigrations(sql, all) {
|
|
1065
|
+
const derived = all.length > 0 && await detectLayout(sql) === "partitioned" ? all.filter((m) => !m.version.startsWith(LIST_GIN_PREFIX)) : all;
|
|
1066
|
+
if (derived.length === 0) return;
|
|
1067
|
+
await withMigrationLock(sql, async (conn) => {
|
|
1068
|
+
await conn`
|
|
1069
|
+
CREATE TABLE IF NOT EXISTS migrations (
|
|
1070
|
+
version text PRIMARY KEY,
|
|
1071
|
+
applied_at timestamptz NOT NULL DEFAULT now(),
|
|
1072
|
+
checksum text NOT NULL
|
|
1073
|
+
)
|
|
1074
|
+
`;
|
|
1075
|
+
const applied = await conn`SELECT version FROM migrations`;
|
|
1076
|
+
const appliedVersions = new Set(applied.map((r) => r.version));
|
|
1077
|
+
const pending = derived.map(normalise).filter((m) => !appliedVersions.has(m.version));
|
|
1078
|
+
for (const migration of pending) await applyOneMigration(sql, conn, migration);
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
//#endregion
|
|
1083
|
+
//#region ../../packages/substrate-dataset-pg/src/type-storage.ts
|
|
1084
|
+
/** lock_not_available (NOWAIT or lock_timeout), duplicate_table (a racing creator took the name), deadlock. */
|
|
1085
|
+
const RETRYABLE = /* @__PURE__ */ new Set([
|
|
1086
|
+
"55P03",
|
|
1087
|
+
"42P07",
|
|
1088
|
+
"40P01"
|
|
1089
|
+
]);
|
|
1090
|
+
/** Longest a single lock wait may queue other sessions behind the move. */
|
|
1091
|
+
const LOCK_TIMEOUT = "500ms";
|
|
1092
|
+
const GIVE_UP_AFTER_MS = 1e4;
|
|
1093
|
+
/** The move kept losing its lock waits and gave up; nothing changed and no row moved. */
|
|
1094
|
+
var TypeStorageLockTimeoutError = class extends Error {
|
|
1095
|
+
namespace;
|
|
1096
|
+
type;
|
|
1097
|
+
constructor(namespace, type, cause) {
|
|
1098
|
+
super(`partition for ${namespace}/${type} was not created: locks stayed busy for ${GIVE_UP_AFTER_MS}ms. Its rows stay readable in the DEFAULT partition; call ensureTypeStorage again later.`, { cause });
|
|
1099
|
+
this.namespace = namespace;
|
|
1100
|
+
this.type = type;
|
|
1101
|
+
this.name = "TypeStorageLockTimeoutError";
|
|
1102
|
+
}
|
|
1103
|
+
};
|
|
1104
|
+
/** `name` as a quoted SQL identifier. */
|
|
1105
|
+
function ident(name) {
|
|
1106
|
+
return `"${name.replace(/"/g, "\"\"")}"`;
|
|
1107
|
+
}
|
|
1108
|
+
/** The bound text `pg_get_expr` prints for a one-value LIST partition, or DEFAULT. */
|
|
1109
|
+
function boundText(value) {
|
|
1110
|
+
return value === null ? "DEFAULT" : `FOR VALUES IN ('${value.replace(/'/g, "''")}')`;
|
|
1111
|
+
}
|
|
1112
|
+
/** The partition of `parent` (current schema) whose bound is `value`, or DEFAULT for null. */
|
|
1113
|
+
async function findPartition(sql, parent, value) {
|
|
1114
|
+
const [row] = await sql`
|
|
1115
|
+
SELECT c.relname FROM pg_inherits i
|
|
1116
|
+
JOIN pg_class c ON c.oid = i.inhrelid
|
|
1117
|
+
JOIN pg_class p ON p.oid = i.inhparent
|
|
1118
|
+
WHERE p.relname = ${parent} AND p.relnamespace = current_schema()::regnamespace
|
|
1119
|
+
AND pg_get_expr(c.relpartbound, c.oid) = ${boundText(value)}`;
|
|
1120
|
+
return row?.relname;
|
|
1121
|
+
}
|
|
1122
|
+
/** The partition of `namespace`/`type` (current schema), found by both bounds in one query. */
|
|
1123
|
+
async function typePartition(sql, namespace, type) {
|
|
1124
|
+
return (await typePartitions(sql, namespace, type))[0];
|
|
1125
|
+
}
|
|
1126
|
+
/** The type partitions of `namespace` (current schema), or only `type`'s when given. */
|
|
1127
|
+
async function typePartitions(sql, namespace, type) {
|
|
1128
|
+
return (await sql`
|
|
1129
|
+
SELECT c.relname FROM pg_partition_tree('rows') t
|
|
1130
|
+
JOIN pg_class c ON c.oid = t.relid
|
|
1131
|
+
JOIN pg_class p ON p.oid = t.parentrelid
|
|
1132
|
+
WHERE t.level = 2
|
|
1133
|
+
AND pg_get_expr(p.relpartbound, p.oid) = ${boundText(namespace)}
|
|
1134
|
+
${type === void 0 ? sql`` : sql`AND pg_get_expr(c.relpartbound, c.oid) = ${boundText(type)}`}`).map((r) => r.relname);
|
|
1135
|
+
}
|
|
1136
|
+
async function freeName(sql, preferred, seed) {
|
|
1137
|
+
for (const name of [preferred, hashedName(preferred, seed)]) {
|
|
1138
|
+
if (name.length > 63) continue;
|
|
1139
|
+
if ((await sql`
|
|
1140
|
+
SELECT 1 FROM pg_class
|
|
1141
|
+
WHERE relname = ${name} AND relnamespace = current_schema()::regnamespace`).length === 0) return name;
|
|
1142
|
+
}
|
|
1143
|
+
throw new Error(`no free partition name for '${preferred}'`);
|
|
1144
|
+
}
|
|
1145
|
+
/**
|
|
1146
|
+
* Creates the partition of `level.parent` for `level.value` unless one exists,
|
|
1147
|
+
* moving its rows out of the parent's DEFAULT. Returns the partition's name.
|
|
1148
|
+
*/
|
|
1149
|
+
async function attachPartition(tx, level) {
|
|
1150
|
+
const { parent, column, value } = level;
|
|
1151
|
+
await tx`SELECT pg_advisory_xact_lock(hashtext(current_schema() || ':' || ${level.lockKey}))`;
|
|
1152
|
+
await tx.unsafe(`SET LOCAL lock_timeout = '${LOCK_TIMEOUT}'`);
|
|
1153
|
+
if (parent !== "rows") await tx.unsafe("LOCK TABLE ONLY rows IN ACCESS SHARE MODE");
|
|
1154
|
+
await tx.unsafe(`LOCK TABLE ONLY ${ident(parent)} IN ACCESS EXCLUSIVE MODE`);
|
|
1155
|
+
await tx.unsafe("LOCK TABLE short_ids IN SHARE ROW EXCLUSIVE MODE NOWAIT");
|
|
1156
|
+
const existing = await findPartition(tx, parent, value);
|
|
1157
|
+
if (existing !== void 0) return existing;
|
|
1158
|
+
const seed = column === "namespace" ? [value] : [level.namespace, value];
|
|
1159
|
+
const name = await freeName(tx, level.preferredName, seed);
|
|
1160
|
+
await tx.unsafe(`CREATE TABLE ${ident(name)} (LIKE rows INCLUDING DEFAULTS)` + (level.subPartitioned ? " PARTITION BY LIST (type)" : ""));
|
|
1161
|
+
if (level.subPartitioned) {
|
|
1162
|
+
const dflt = await freeName(tx, `${name}__default`, [level.namespace, null]);
|
|
1163
|
+
await tx.unsafe(`CREATE TABLE ${ident(dflt)} PARTITION OF ${ident(name)} DEFAULT`);
|
|
1164
|
+
}
|
|
1165
|
+
const source = await findPartition(tx, parent, null);
|
|
1166
|
+
if (source !== void 0) {
|
|
1167
|
+
const [cols] = await tx`
|
|
1168
|
+
SELECT string_agg(quote_ident(attname), ', ' ORDER BY attnum) AS list
|
|
1169
|
+
FROM pg_attribute
|
|
1170
|
+
WHERE attrelid = 'rows'::regclass AND attnum > 0 AND NOT attisdropped`;
|
|
1171
|
+
const list = cols?.list;
|
|
1172
|
+
if (list == null) throw new Error("rows has no columns in the current schema");
|
|
1173
|
+
const src = ident(source);
|
|
1174
|
+
await tx.unsafe(`CREATE TEMP TABLE substrate_moved_short_ids ON COMMIT DROP AS
|
|
1175
|
+
SELECT s.namespace, s.type, s.short_id, s.row_id FROM short_ids s
|
|
1176
|
+
JOIN ${src} r ON r.namespace = s.namespace AND r.type = s.type AND r.id = s.row_id
|
|
1177
|
+
WHERE r.${column} = $1`, [value]);
|
|
1178
|
+
await tx.unsafe(`DELETE FROM short_ids s USING substrate_moved_short_ids m
|
|
1179
|
+
WHERE s.namespace = m.namespace AND s.type = m.type AND s.short_id = m.short_id`);
|
|
1180
|
+
const copied = await tx.unsafe(`INSERT INTO ${ident(name)} (${list}) SELECT ${list} FROM ${src} WHERE ${column} = $1`, [value]);
|
|
1181
|
+
const deleted = await tx.unsafe(`DELETE FROM ${src} WHERE ${column} = $1`, [value]);
|
|
1182
|
+
if (copied.count !== deleted.count) throw new Error(`partition move of ${column} '${value}' copied ${copied.count} rows but deleted ${deleted.count}`);
|
|
1183
|
+
}
|
|
1184
|
+
const checks = await tx`
|
|
1185
|
+
SELECT conname AS name, pg_get_constraintdef(oid) AS def FROM pg_constraint
|
|
1186
|
+
WHERE contype = 'c' AND conrelid = (
|
|
1187
|
+
SELECT oid FROM pg_class
|
|
1188
|
+
WHERE relname = ${parent} AND relnamespace = current_schema()::regnamespace)
|
|
1189
|
+
ORDER BY conname`;
|
|
1190
|
+
for (const { name: check, def } of checks) await tx.unsafe(`ALTER TABLE ${ident(name)} ADD CONSTRAINT ${ident(check)} ${def}`);
|
|
1191
|
+
await tx.unsafe(`ALTER TABLE ${ident(parent)} ATTACH PARTITION ${ident(name)} ${boundText(value)}`);
|
|
1192
|
+
if (source !== void 0) {
|
|
1193
|
+
await tx.unsafe(`INSERT INTO short_ids (namespace, type, short_id, row_id)
|
|
1194
|
+
SELECT namespace, type, short_id, row_id FROM substrate_moved_short_ids`);
|
|
1195
|
+
await tx.unsafe("DROP TABLE substrate_moved_short_ids");
|
|
1196
|
+
}
|
|
1197
|
+
return name;
|
|
1198
|
+
}
|
|
1199
|
+
async function withRetry(namespace, type, fn) {
|
|
1200
|
+
const deadline = Date.now() + GIVE_UP_AFTER_MS;
|
|
1201
|
+
for (;;) try {
|
|
1202
|
+
return await fn();
|
|
1203
|
+
} catch (err) {
|
|
1204
|
+
const code = err.code;
|
|
1205
|
+
if (typeof code !== "string" || !RETRYABLE.has(code)) throw err;
|
|
1206
|
+
if (Date.now() >= deadline) throw new TypeStorageLockTimeoutError(namespace, type, err);
|
|
1207
|
+
await new Promise((resolve) => setTimeout(resolve, 20 + Math.random() * 80));
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
/**
|
|
1211
|
+
* Makes sure `namespace/type` has a partition of its own, creating the
|
|
1212
|
+
* namespace partition first if needed, and drains their rows out of the
|
|
1213
|
+
* DEFAULT partitions. Idempotent; without partitions to create it only reads
|
|
1214
|
+
* the catalog. Partitioned layout only.
|
|
1215
|
+
*/
|
|
1216
|
+
async function ensureTypeStorage(sql, namespace, type) {
|
|
1217
|
+
if (await typePartition(sql, namespace, type) !== void 0) return;
|
|
1218
|
+
let nsPart = await findPartition(sql, "rows", namespace);
|
|
1219
|
+
if (nsPart === void 0) nsPart = await withRetry(namespace, type, () => sql.begin((tx) => attachPartition(tx, {
|
|
1220
|
+
parent: "rows",
|
|
1221
|
+
column: "namespace",
|
|
1222
|
+
value: namespace,
|
|
1223
|
+
namespace,
|
|
1224
|
+
lockKey: `substrate-type-storage:${namespace}`,
|
|
1225
|
+
preferredName: partitionName(namespace),
|
|
1226
|
+
subPartitioned: true
|
|
1227
|
+
})));
|
|
1228
|
+
const parent = nsPart;
|
|
1229
|
+
await withRetry(namespace, type, () => sql.begin((tx) => attachPartition(tx, {
|
|
1230
|
+
parent,
|
|
1231
|
+
column: "type",
|
|
1232
|
+
value: type,
|
|
1233
|
+
namespace,
|
|
1234
|
+
lockKey: `substrate-type-storage:${namespace}/${type}`,
|
|
1235
|
+
preferredName: partitionName(namespace, type),
|
|
1236
|
+
subPartitioned: false
|
|
1237
|
+
})));
|
|
1238
|
+
}
|
|
1239
|
+
/** One line per (namespace, type) with rows in a DEFAULT partition. Two queries however many partitions. */
|
|
1240
|
+
async function storageWarnings(sql) {
|
|
1241
|
+
const defaults = await sql`
|
|
1242
|
+
SELECT c.relname FROM pg_partition_tree('rows') t
|
|
1243
|
+
JOIN pg_class c ON c.oid = t.relid
|
|
1244
|
+
WHERE pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
|
1245
|
+
ORDER BY c.relname`;
|
|
1246
|
+
if (defaults.length === 0) return [];
|
|
1247
|
+
return (await sql.unsafe(`${defaults.map(({ relname }) => `SELECT '${relname.replace(/'/g, "''")}' AS part, namespace, type, count(*)::int AS n
|
|
1248
|
+
FROM ${ident(relname)} GROUP BY namespace, type`).join(" UNION ALL ")} ORDER BY part, namespace, type`)).map((h) => `${h.part} holds ${h.n} rows of ${h.namespace}/${h.type}`);
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
//#endregion
|
|
1252
|
+
//#region ../../packages/substrate-dataset-pg/src/index-reconciler.ts
|
|
1253
|
+
/**
|
|
1254
|
+
* Builds a type's indexes and keeps them matching their declaration.
|
|
1255
|
+
*
|
|
1256
|
+
* The desired set is, in both layouts, a derived btree per crossRef (unless
|
|
1257
|
+
* `indexed: false`) and per `indexed: true` scalar, keyed (field, natural-order
|
|
1258
|
+
* key, id), and the declared `indexes`. The partitioned layout adds GIN
|
|
1259
|
+
* `jsonb_path_ops` for `indexed: true` list and json fields and the engine's
|
|
1260
|
+
* timestamp and tombstone indexes. Partitioned indexes are on the type's
|
|
1261
|
+
* partition; legacy ones are partial indexes on `rows` scoped to the type.
|
|
1262
|
+
*
|
|
1263
|
+
* Each desired index is compared, by name, with what the catalog holds:
|
|
1264
|
+
* identical is kept; same table, uniqueness and key fields but a different
|
|
1265
|
+
* expression or predicate is swapped (built under a temporary name, the old one
|
|
1266
|
+
* dropped only once the new one is valid and ready, then renamed, so a unique
|
|
1267
|
+
* index is enforced throughout); anything else is refused. Definitions are
|
|
1268
|
+
* compared as pg deparses them: the desired ones are deparsed by building them
|
|
1269
|
+
* on an empty temp table first.
|
|
1270
|
+
*
|
|
1271
|
+
* Every index is partial, and the planner ignores a partial index's
|
|
1272
|
+
* expression statistics, so each body-keyed key expression also gets an
|
|
1273
|
+
* expression statistics object on the same table (`_stat`), marked as this
|
|
1274
|
+
* type's. One whose expression is no longer indexed is dropped.
|
|
1275
|
+
*
|
|
1276
|
+
* Builds run CONCURRENTLY, so never inside a transaction, and each under its
|
|
1277
|
+
* table's index-build lock, since two concurrent builds on one table deadlock.
|
|
1278
|
+
* Unique ones also hold the migration lock, which serialises them with
|
|
1279
|
+
* migrations and other boots; non-unique ones do not, so a long build never
|
|
1280
|
+
* holds up a booting process. What still meets a non-unique build is DDL on
|
|
1281
|
+
* its table outside these locks: a kernel migration's ALTER or non-concurrent
|
|
1282
|
+
* CREATE INDEX on `rows` waits for the build, and can deadlock with it, in
|
|
1283
|
+
* which case Postgres aborts one with 40P01 (a failed build is retried by the
|
|
1284
|
+
* next reconcile; a failed migration fails that boot).
|
|
1285
|
+
*/
|
|
1286
|
+
/** A unique index was declared over values that are already duplicated among live rows. */
|
|
1287
|
+
var UniqueIndexDuplicatesError = class extends Error {
|
|
1288
|
+
namespace;
|
|
1289
|
+
type;
|
|
1290
|
+
fields;
|
|
1291
|
+
count;
|
|
1292
|
+
code = "unique_index_duplicates";
|
|
1293
|
+
constructor(namespace, type, fields, count) {
|
|
1294
|
+
super(`unique index on ${namespace}/${type} (${fields.join(", ")}) was not built: ${count} value(s) are held by more than one live row. Resolve them, then restart.`);
|
|
1295
|
+
this.namespace = namespace;
|
|
1296
|
+
this.type = type;
|
|
1297
|
+
this.fields = fields;
|
|
1298
|
+
this.count = count;
|
|
1299
|
+
this.name = "UniqueIndexDuplicatesError";
|
|
1300
|
+
}
|
|
1301
|
+
};
|
|
1302
|
+
/** An index of the declared name exists with a definition the reconciler may not replace. */
|
|
1303
|
+
var IndexDefinitionConflictError = class extends Error {
|
|
1304
|
+
indexName;
|
|
1305
|
+
existing;
|
|
1306
|
+
declared;
|
|
1307
|
+
code = "index_definition_conflict";
|
|
1308
|
+
constructor(indexName, existing, declared) {
|
|
1309
|
+
super(`index ${indexName} exists with a different definition.\n existing: ${existing}\n declared: ${declared}`);
|
|
1310
|
+
this.indexName = indexName;
|
|
1311
|
+
this.existing = existing;
|
|
1312
|
+
this.declared = declared;
|
|
1313
|
+
this.name = "IndexDefinitionConflictError";
|
|
1314
|
+
}
|
|
1315
|
+
};
|
|
1316
|
+
/** `PgDataset.dropEngineIndex` refused, and dropped nothing. */
|
|
1317
|
+
var IndexDropRefusedError = class extends Error {
|
|
1318
|
+
code;
|
|
1319
|
+
constructor(code, message) {
|
|
1320
|
+
super(message);
|
|
1321
|
+
this.code = code;
|
|
1322
|
+
this.name = "IndexDropRefusedError";
|
|
1323
|
+
}
|
|
1324
|
+
};
|
|
1325
|
+
/**
|
|
1326
|
+
* The last failure logged per index, so a retry pass that fails the same way
|
|
1327
|
+
* stays quiet. shortcut: keyed by index name across schemas, process-wide.
|
|
1328
|
+
*/
|
|
1329
|
+
const lastLogged = /* @__PURE__ */ new Map();
|
|
1330
|
+
/** Writes `text` unless `key` is what was last logged for index `name`. */
|
|
1331
|
+
function logChanged(name, key, text) {
|
|
1332
|
+
if (lastLogged.get(name) === key) return;
|
|
1333
|
+
lastLogged.set(name, key);
|
|
1334
|
+
process.stderr.write(text);
|
|
1335
|
+
}
|
|
1336
|
+
/** Pause after a non-unique build releases its table lock, so a polling waiter gets it. */
|
|
1337
|
+
const YIELD_MS = 300;
|
|
1338
|
+
/** Marks an index this reconciler built or adopted. */
|
|
1339
|
+
const MARK = "substrate: reconciled";
|
|
1340
|
+
const BARE_NAME = /^[a-z_][a-z0-9_]*$/;
|
|
1341
|
+
/**
|
|
1342
|
+
* Reconciles the type's indexes and reports each one. On the partitioned layout
|
|
1343
|
+
* the type's partition must exist (`ensureTypeStorage`). Must not be called
|
|
1344
|
+
* while this session's pool already holds the migration lock on another
|
|
1345
|
+
* connection, since it waits for that lock.
|
|
1346
|
+
*
|
|
1347
|
+
* Unique indexes reconcile under the migration lock, which serialises them
|
|
1348
|
+
* with migrations and other boots; a type with none takes no migration lock. Non-unique indexes and statistics take
|
|
1349
|
+
* only the table's index-build lock (`withTableIndexLock`), one index at a
|
|
1350
|
+
* time, so a long build never holds up another process's migrations, nor its
|
|
1351
|
+
* unique builds on other tables. An index the catalog already holds as
|
|
1352
|
+
* declared takes no lock at all.
|
|
1353
|
+
*/
|
|
1354
|
+
async function reconcileTypeIndexes(sql, args, opts = {}) {
|
|
1355
|
+
const only = opts.only ?? "all";
|
|
1356
|
+
const all = desiredSpecs(args);
|
|
1357
|
+
const out = [];
|
|
1358
|
+
const unique = all.filter((s) => s.unique);
|
|
1359
|
+
const first = only === "unique" ? all : only === "all" ? unique : [];
|
|
1360
|
+
if (first.length > 0) {
|
|
1361
|
+
const target = await targetTable(sql, args);
|
|
1362
|
+
const pass = async (conn) => {
|
|
1363
|
+
const probed = [];
|
|
1364
|
+
for (const spec of first) if (spec.unique || await describeIndex(conn, spec.name, "current") !== void 0) probed.push(spec);
|
|
1365
|
+
const desired = await deparse(conn, probed);
|
|
1366
|
+
const states = [];
|
|
1367
|
+
for (const spec of first) {
|
|
1368
|
+
const i = probed.indexOf(spec);
|
|
1369
|
+
if (i < 0) {
|
|
1370
|
+
states.push(indexState(spec, "pending", "unchanged"));
|
|
1371
|
+
continue;
|
|
1372
|
+
}
|
|
1373
|
+
const d = desired[i];
|
|
1374
|
+
const held = await heldAsDeclared(conn, target, spec, d);
|
|
1375
|
+
states.push(held !== void 0 ? held : spec.unique ? await withTableIndexLock(sql, conn, target, void 0, () => reconcileOne(conn, args, target, spec, d, {})) : indexState(spec, "pending", "unchanged"));
|
|
1376
|
+
}
|
|
1377
|
+
return states;
|
|
1378
|
+
};
|
|
1379
|
+
out.push(...unique.length > 0 ? await withMigrationLock(sql, pass) : await reserved(sql, pass));
|
|
1380
|
+
}
|
|
1381
|
+
if (only !== "unique") out.push(...await reconcileNonUnique(sql, args, all, opts));
|
|
1382
|
+
return out;
|
|
1383
|
+
}
|
|
1384
|
+
/** Runs `fn` on a connection reserved from `sql`, released after. */
|
|
1385
|
+
async function reserved(sql, fn) {
|
|
1386
|
+
const conn = await sql.reserve();
|
|
1387
|
+
try {
|
|
1388
|
+
return await fn(conn);
|
|
1389
|
+
} finally {
|
|
1390
|
+
conn.release();
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
/** Non-unique indexes, then statistics, each under the table's index-build lock. */
|
|
1394
|
+
async function reconcileNonUnique(sql, args, all, opts) {
|
|
1395
|
+
const specs = all.filter((s) => !s.unique);
|
|
1396
|
+
if (specs.length === 0 && (await heldStatistics(sql, args).catch(() => [])).length === 0) return [];
|
|
1397
|
+
const target = await targetTable(sql, args);
|
|
1398
|
+
const conn = await sql.reserve();
|
|
1399
|
+
const out = [];
|
|
1400
|
+
try {
|
|
1401
|
+
const desired = await deparse(conn, specs);
|
|
1402
|
+
for (const [i, spec] of specs.entries()) {
|
|
1403
|
+
const d = desired[i];
|
|
1404
|
+
let state = await heldAsDeclared(conn, target, spec, d);
|
|
1405
|
+
if (state === void 0) {
|
|
1406
|
+
try {
|
|
1407
|
+
state = await withTableIndexLock(sql, conn, target, opts.signal, () => reconcileOne(conn, args, target, spec, d, opts));
|
|
1408
|
+
} catch (err) {
|
|
1409
|
+
if (err instanceof IndexLockAborted) return out;
|
|
1410
|
+
if (!(err instanceof IndexDefinitionConflictError)) throw err;
|
|
1411
|
+
logChanged(spec.name, err.message, `${err.message}\n`);
|
|
1412
|
+
state = indexState(spec, "failed", "unchanged", conflictReason(err));
|
|
1413
|
+
}
|
|
1414
|
+
await abortableSleep(YIELD_MS, opts.signal);
|
|
1415
|
+
}
|
|
1416
|
+
if (state.state === "valid") lastLogged.delete(spec.name);
|
|
1417
|
+
opts.onState?.(state);
|
|
1418
|
+
out.push(state);
|
|
1419
|
+
if (opts.signal?.aborted === true) return out;
|
|
1420
|
+
}
|
|
1421
|
+
try {
|
|
1422
|
+
await withTableIndexLock(sql, conn, target, opts.signal, () => reconcileStatistics(conn, args, target, all, opts.signal));
|
|
1423
|
+
} catch (err) {
|
|
1424
|
+
if (!(err instanceof IndexLockAborted)) throw err;
|
|
1425
|
+
}
|
|
1426
|
+
return out;
|
|
1427
|
+
} finally {
|
|
1428
|
+
conn.release();
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
function indexState(spec, state, outcome, error) {
|
|
1432
|
+
return {
|
|
1433
|
+
name: spec.name,
|
|
1434
|
+
fields: spec.fields,
|
|
1435
|
+
unique: spec.unique,
|
|
1436
|
+
state,
|
|
1437
|
+
outcome,
|
|
1438
|
+
...error !== void 0 ? { error } : {}
|
|
1439
|
+
};
|
|
1440
|
+
}
|
|
1441
|
+
const conflictReason = (err) => `${err.code}: ${err.indexName} exists with a different definition; drop or rename it`;
|
|
1442
|
+
/**
|
|
1443
|
+
* `valid`, unchanged, when the catalog holds `spec` exactly as declared,
|
|
1444
|
+
* valid, marked, and with no swap leftover: nothing to do, so no lock.
|
|
1445
|
+
* Otherwise undefined.
|
|
1446
|
+
*/
|
|
1447
|
+
async function heldAsDeclared(conn, target, spec, desired) {
|
|
1448
|
+
const named = await describeIndex(conn, spec.name, "current");
|
|
1449
|
+
if (named === void 0 || named.table !== target || named.def !== desired.def || !named.valid || !named.ready || !named.marked) return;
|
|
1450
|
+
if (await describeIndex(conn, swapNameFor(spec.name), "current") !== void 0) return void 0;
|
|
1451
|
+
return indexState(spec, "valid", "unchanged");
|
|
1452
|
+
}
|
|
1453
|
+
/** Marks a statistics object as this type's, so another type's never matches. */
|
|
1454
|
+
const statMark = (args) => `substrate: statistics of ${JSON.stringify([args.namespace, args.type])}`;
|
|
1455
|
+
const STAT_NAME = /_[0-9a-f]{8}_stat$/;
|
|
1456
|
+
/** One per body-keyed key expression; a promoted column has column statistics already. */
|
|
1457
|
+
function desiredStatistics(args, specs) {
|
|
1458
|
+
const promoted = args.promotedColumns ?? {};
|
|
1459
|
+
const byField = /* @__PURE__ */ new Map();
|
|
1460
|
+
for (const spec of specs) for (const [i, expr] of spec.keys.entries()) {
|
|
1461
|
+
const field = spec.fields[i];
|
|
1462
|
+
if (metadataColumn(field) !== void 0 || Object.hasOwn(promoted, field) || byField.has(field)) continue;
|
|
1463
|
+
const hash = createHash("sha256").update(expr).digest("hex").slice(0, 6);
|
|
1464
|
+
byField.set(field, {
|
|
1465
|
+
name: generatedIdentifier(args.namespace, args.type, `${field}_${hash}`, "stat"),
|
|
1466
|
+
expr
|
|
1467
|
+
});
|
|
1468
|
+
}
|
|
1469
|
+
return [...byField.values()];
|
|
1470
|
+
}
|
|
1471
|
+
/** The statistics objects marked as this type's, on any table, with the table each is on. */
|
|
1472
|
+
async function heldStatistics(sql, args) {
|
|
1473
|
+
return (await sql`
|
|
1474
|
+
SELECT s.stxname AS name, t.relname AS table FROM pg_statistic_ext s
|
|
1475
|
+
JOIN pg_class t ON t.oid = s.stxrelid
|
|
1476
|
+
WHERE s.stxnamespace = current_schema()::regnamespace
|
|
1477
|
+
AND obj_description(s.oid, 'pg_statistic_ext') = ${statMark(args)}`).filter((r) => STAT_NAME.test(r.name));
|
|
1478
|
+
}
|
|
1479
|
+
async function reconcileStatistics(conn, args, target, specs, signal) {
|
|
1480
|
+
const out = {
|
|
1481
|
+
created: [],
|
|
1482
|
+
dropped: [],
|
|
1483
|
+
analyzed: false
|
|
1484
|
+
};
|
|
1485
|
+
const attempt = async (what, run) => {
|
|
1486
|
+
try {
|
|
1487
|
+
await run();
|
|
1488
|
+
return true;
|
|
1489
|
+
} catch (err) {
|
|
1490
|
+
process.stderr.write(`statistics ${what} on ${target} failed and will be retried: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
1491
|
+
return false;
|
|
1492
|
+
}
|
|
1493
|
+
};
|
|
1494
|
+
let marked = [];
|
|
1495
|
+
const leftovers = [];
|
|
1496
|
+
if (!await attempt("lookup", async () => {
|
|
1497
|
+
marked = await heldStatistics(conn, args);
|
|
1498
|
+
const held = new Set((await heldIndexes(conn, [target])).filter((i) => i.marked && i.valid && i.ready).map((i) => i.name));
|
|
1499
|
+
for (const [field, def] of Object.entries(args.fields)) if ((def.kind === "crossRef" || def.kind === "scalar" && def.type !== "json") && held.has(generatedIdentifier(args.namespace, args.type, field, "idx"))) leftovers.push({
|
|
1500
|
+
fields: [field],
|
|
1501
|
+
keys: [keyExpr(field, def, args.promotedColumns ?? {})]
|
|
1502
|
+
});
|
|
1503
|
+
})) return out;
|
|
1504
|
+
const desired = desiredStatistics(args, [...specs, ...leftovers]);
|
|
1505
|
+
const wanted = new Set(desired.map((d) => d.name));
|
|
1506
|
+
const held = [];
|
|
1507
|
+
for (const { name, table } of marked) {
|
|
1508
|
+
if (wanted.has(name) && table === target) {
|
|
1509
|
+
held.push(name);
|
|
1510
|
+
continue;
|
|
1511
|
+
}
|
|
1512
|
+
if (!BARE_NAME.test(name)) {
|
|
1513
|
+
process.stderr.write(`statistics ${name} on ${table} is not a generated name; left in place\n`);
|
|
1514
|
+
continue;
|
|
1515
|
+
}
|
|
1516
|
+
if (await attempt(`${name} drop`, () => conn.unsafe(`DROP STATISTICS IF EXISTS ${ident(name)}`))) out.dropped.push(name);
|
|
1517
|
+
}
|
|
1518
|
+
for (const { name, expr } of desired) {
|
|
1519
|
+
if (held.includes(name)) continue;
|
|
1520
|
+
if (await attempt(`${name} create`, async () => {
|
|
1521
|
+
await conn.unsafe("BEGIN");
|
|
1522
|
+
try {
|
|
1523
|
+
await conn.unsafe(`CREATE STATISTICS ${ident(name)} ON (${expr}) FROM ${ident(target)}`);
|
|
1524
|
+
await conn.unsafe(`COMMENT ON STATISTICS ${ident(name)} IS ${sqlStringLiteral(statMark(args))}`);
|
|
1525
|
+
await conn.unsafe("COMMIT");
|
|
1526
|
+
} catch (err) {
|
|
1527
|
+
await conn.unsafe("ROLLBACK");
|
|
1528
|
+
throw err;
|
|
1529
|
+
}
|
|
1530
|
+
})) out.created.push(name);
|
|
1531
|
+
}
|
|
1532
|
+
await attempt("analyze", async () => {
|
|
1533
|
+
if (!(out.created.length > 0 || held.length > 0 && (await conn`
|
|
1534
|
+
SELECT 1 FROM pg_stats_ext_exprs
|
|
1535
|
+
WHERE statistics_schemaname = current_schema()
|
|
1536
|
+
AND statistics_name = ANY(${held}) AND null_frac IS NULL
|
|
1537
|
+
LIMIT 1`).length > 0)) return;
|
|
1538
|
+
await cancellable(conn, `ANALYZE ${ident(target)}`, signal);
|
|
1539
|
+
out.analyzed = true;
|
|
1540
|
+
});
|
|
1541
|
+
return out;
|
|
1542
|
+
}
|
|
1543
|
+
/**
|
|
1544
|
+
* Each index the type should have, as the catalog rows `held` (the indexes on
|
|
1545
|
+
* its table) show it. Builds nothing and queries nothing. An index `held`
|
|
1546
|
+
* lacks is `failed`, as a build that has not succeeded is.
|
|
1547
|
+
*/
|
|
1548
|
+
function describeTypeIndexes(args, held) {
|
|
1549
|
+
const byName = new Map(held.map((i) => [i.name, i]));
|
|
1550
|
+
return desiredSpecs(args).map((spec) => indexState(spec, stateFrom(byName.get(spec.name)), "unchanged"));
|
|
1551
|
+
}
|
|
1552
|
+
/** A catalog index's state; one that is not there is `failed`, as a build that has not succeeded is. */
|
|
1553
|
+
const stateFrom = (d) => d === void 0 ? "failed" : d.valid && d.ready ? "valid" : d.ready ? "building" : "invalid";
|
|
1554
|
+
/**
|
|
1555
|
+
* Engine-owned indexes of the type (`held`: its indexes, by `belongsToType`)
|
|
1556
|
+
* that its declaration no longer asks for. They are reported, never dropped:
|
|
1557
|
+
* removing an index is the operator's call. Both layouts.
|
|
1558
|
+
*/
|
|
1559
|
+
function undeclaredIndexes(args, held) {
|
|
1560
|
+
const desired = new Set(desiredSpecs(args).map((s) => s.name));
|
|
1561
|
+
return held.filter((i) => isEngineOwned(i) && !desired.has(i.name)).map((i) => i.name).sort();
|
|
1562
|
+
}
|
|
1563
|
+
/**
|
|
1564
|
+
* The legacy migrations' shared per-type order indexes on `rows`
|
|
1565
|
+
* (`0001_baseline`): `(namespace, type, created_at|updated_at DESC)`, live.
|
|
1566
|
+
* Not the reconciler's, so never `remove` candidates, but they serve reads.
|
|
1567
|
+
*/
|
|
1568
|
+
const LEGACY_SHARED_KEYS = [{
|
|
1569
|
+
name: "rows_ns_type_created_at_idx",
|
|
1570
|
+
keys: [["createdAt", "desc"]]
|
|
1571
|
+
}, {
|
|
1572
|
+
name: "rows_ns_type_updated_at_idx",
|
|
1573
|
+
keys: [["updatedAt", "desc"]]
|
|
1574
|
+
}];
|
|
1575
|
+
/**
|
|
1576
|
+
* Each index that serves the type's reads, with the keys a read can use, in
|
|
1577
|
+
* order: the reconciler's, and on legacy the migrations' shared order indexes.
|
|
1578
|
+
*/
|
|
1579
|
+
function desiredIndexKeys(args) {
|
|
1580
|
+
const own = desiredSpecs(args).map((s) => ({
|
|
1581
|
+
name: s.name,
|
|
1582
|
+
keys: s.order,
|
|
1583
|
+
unique: s.unique,
|
|
1584
|
+
...s.only !== void 0 ? { only: s.only } : {}
|
|
1585
|
+
}));
|
|
1586
|
+
if (args.layout !== "legacy") return own;
|
|
1587
|
+
return [...own, ...LEGACY_SHARED_KEYS.map((k) => ({
|
|
1588
|
+
...k,
|
|
1589
|
+
unique: false
|
|
1590
|
+
}))];
|
|
1591
|
+
}
|
|
1592
|
+
/** Every index on `tables` (current schema), with its scan count and size. */
|
|
1593
|
+
async function heldIndexes(sql, tables) {
|
|
1594
|
+
return sql`
|
|
1595
|
+
SELECT t.relname AS "table", c.relname AS name, i.indisunique AS "unique",
|
|
1596
|
+
obj_description(i.indexrelid, 'pg_class') IS NOT DISTINCT FROM ${MARK} AS marked,
|
|
1597
|
+
i.indisvalid AS valid, i.indisready AS ready,
|
|
1598
|
+
pg_get_expr(i.indpred, i.indrelid) AS predicate,
|
|
1599
|
+
c.reltuples::float8 AS reltuples,
|
|
1600
|
+
COALESCE(s.idx_scan, 0)::float8 AS scans,
|
|
1601
|
+
to_char(s.last_idx_scan AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS "lastScanAt",
|
|
1602
|
+
pg_relation_size(i.indexrelid)::float8 AS "sizeBytes"
|
|
1603
|
+
FROM pg_index i
|
|
1604
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
1605
|
+
JOIN pg_class t ON t.oid = i.indrelid
|
|
1606
|
+
LEFT JOIN pg_stat_user_indexes s ON s.indexrelid = i.indexrelid
|
|
1607
|
+
WHERE t.relnamespace = current_schema()::regnamespace AND t.relname = ANY(${tables})
|
|
1608
|
+
ORDER BY c.relname`;
|
|
1609
|
+
}
|
|
1610
|
+
/** The name tail `generatedIdentifier` gives every engine index, whatever truncated its head. */
|
|
1611
|
+
const ENGINE_SUFFIX = /_[0-9a-f]{8}_(idx|uniq|gin|base)$/;
|
|
1612
|
+
/**
|
|
1613
|
+
* Whether `index` belongs to `namespace`/`type`: on partitioned, it is on the
|
|
1614
|
+
* type's own partition (`table`, resolved by bound); on legacy, it is on `rows`
|
|
1615
|
+
* and its predicate opens with the type's exact namespace and type literals,
|
|
1616
|
+
* as pg deparses the predicate the reconciler emits. Never by name prefix:
|
|
1617
|
+
* names fold, so two types can share one, and a long one is truncated.
|
|
1618
|
+
*/
|
|
1619
|
+
function belongsToType(index, namespace, type, layout, table) {
|
|
1620
|
+
if (index.table !== table) return false;
|
|
1621
|
+
if (layout === "partitioned") return true;
|
|
1622
|
+
const lit = (v) => `'${v.replace(/'/g, "''")}'::text`;
|
|
1623
|
+
const scope = `((namespace = ${lit(namespace)}) AND (type = ${lit(type)})`;
|
|
1624
|
+
return index.predicate !== null && (index.predicate === `${scope})` || index.predicate.startsWith(`${scope} AND `));
|
|
1625
|
+
}
|
|
1626
|
+
const LITERAL = "'((?:[^']|'')*)'::text";
|
|
1627
|
+
const LIVE_SCOPE = new RegExp(`^\\(\\(namespace = ${LITERAL}\\) AND \\(type = ${LITERAL}\\) AND \\(deleted_at IS NULL\\)\\)$`, "s");
|
|
1628
|
+
const SCOPE = new RegExp(`^\\(\\(namespace = ${LITERAL}\\) AND \\(type = ${LITERAL}\\)(?: AND |\\)$)`, "s");
|
|
1629
|
+
/** The namespace and type a legacy index's predicate is scoped to, as `belongsToType` reads it. Else undefined. */
|
|
1630
|
+
function legacyScope(predicate) {
|
|
1631
|
+
const m = predicate === null ? null : SCOPE.exec(predicate);
|
|
1632
|
+
if (m === null || m[1] === void 0 || m[2] === void 0) return void 0;
|
|
1633
|
+
return {
|
|
1634
|
+
namespace: m[1].replace(/''/g, "'"),
|
|
1635
|
+
type: m[2].replace(/''/g, "'")
|
|
1636
|
+
};
|
|
1637
|
+
}
|
|
1638
|
+
/**
|
|
1639
|
+
* The namespace and type a legacy index's predicate is exactly the live scope
|
|
1640
|
+
* of (`namespace = … AND type = … AND deleted_at IS NULL`, as pg deparses the
|
|
1641
|
+
* reconciler's), so its row count is the type's live rows. Else undefined.
|
|
1642
|
+
*/
|
|
1643
|
+
function legacyLiveScope(predicate) {
|
|
1644
|
+
const m = predicate === null ? null : LIVE_SCOPE.exec(predicate);
|
|
1645
|
+
if (m === null || m[1] === void 0 || m[2] === void 0) return void 0;
|
|
1646
|
+
return {
|
|
1647
|
+
namespace: m[1].replace(/''/g, "'"),
|
|
1648
|
+
type: m[2].replace(/''/g, "'")
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
1651
|
+
/** Built or adopted by the reconciler: its comment marker and an engine-generated name. */
|
|
1652
|
+
const isEngineOwned = (index) => index.marked && ENGINE_SUFFIX.test(index.name);
|
|
1653
|
+
/**
|
|
1654
|
+
* Every index of `namespace`/`type` (see `belongsToType`), each flagged
|
|
1655
|
+
* whether the engine owns it. `[]` when the type has no partition yet.
|
|
1656
|
+
*/
|
|
1657
|
+
async function typeIndexesOf(sql, namespace, type, layout) {
|
|
1658
|
+
const table = layout === "partitioned" ? await typePartition(sql, namespace, type) : "rows";
|
|
1659
|
+
if (table === void 0) return [];
|
|
1660
|
+
return (await heldIndexes(sql, [table])).filter((i) => belongsToType(i, namespace, type, layout, table)).map((i) => ({
|
|
1661
|
+
...i,
|
|
1662
|
+
engineOwned: isEngineOwned(i)
|
|
1663
|
+
}));
|
|
1664
|
+
}
|
|
1665
|
+
async function targetTable(sql, args) {
|
|
1666
|
+
if (args.layout === "legacy") return "rows";
|
|
1667
|
+
const part = await typePartition(sql, args.namespace, args.type);
|
|
1668
|
+
if (part === void 0) throw new Error(`${args.namespace}/${args.type} has no partition; call ensureTypeStorage before reconciling its indexes`);
|
|
1669
|
+
return part;
|
|
1670
|
+
}
|
|
1671
|
+
/** Metadata columns an index may key on, by the name reads use: plain columns, as natural order keys them. */
|
|
1672
|
+
const METADATA_COLUMNS = {
|
|
1673
|
+
createdAt: "created_at",
|
|
1674
|
+
updatedAt: "updated_at",
|
|
1675
|
+
seq: "seq"
|
|
1676
|
+
};
|
|
1677
|
+
const metadataColumn = (field) => Object.hasOwn(METADATA_COLUMNS, field) ? METADATA_COLUMNS[field] : void 0;
|
|
1678
|
+
function keyExpr(field, def, promoted) {
|
|
1679
|
+
const column = metadataColumn(field);
|
|
1680
|
+
if (column !== void 0) return column;
|
|
1681
|
+
const meta = Object.hasOwn(promoted, field) ? promoted[field] : void 0;
|
|
1682
|
+
if (meta !== void 0) return meta.kind === "text" ? `${meta.column} COLLATE "C"` : meta.column;
|
|
1683
|
+
if (def?.kind === "crossRef") return indexExpressionSql("text", field);
|
|
1684
|
+
if (def?.kind === "scalar" && def.type !== "json") return indexExpressionSql(def.type === "number" ? "number" : def.type === "date" ? "date" : "text", field);
|
|
1685
|
+
throw new Error(`field '${field}' is not a declared scalar or crossRef field and cannot be indexed`);
|
|
1686
|
+
}
|
|
1687
|
+
function desiredSpecs(args) {
|
|
1688
|
+
const { namespace, type, layout, fields } = args;
|
|
1689
|
+
const promoted = args.promotedColumns ?? {};
|
|
1690
|
+
resolvePromotedColumns(promoted);
|
|
1691
|
+
const live = layout === "legacy" ? `namespace = ${sqlStringLiteral(namespace)} AND type = ${sqlStringLiteral(type)} AND deleted_at IS NULL` : "deleted_at IS NULL";
|
|
1692
|
+
const specs = [];
|
|
1693
|
+
const btree = (decl, name, withoutId = false, natural, naturalKey) => {
|
|
1694
|
+
const cols = decl.fields.map((f) => typeof f === "string" ? {
|
|
1695
|
+
field: f,
|
|
1696
|
+
dir: "asc"
|
|
1697
|
+
} : f);
|
|
1698
|
+
const keys = cols.map((c) => keyExpr(c.field, fields[c.field], promoted));
|
|
1699
|
+
let predicate = live;
|
|
1700
|
+
if (decl.where !== void 0 && decl.where !== "live") for (const [f, v] of Object.entries(decl.where)) {
|
|
1701
|
+
const meta = Object.hasOwn(promoted, f) ? promoted[f] : void 0;
|
|
1702
|
+
const column = meta?.column;
|
|
1703
|
+
if (v !== null && typeof v !== "boolean") throw new Error(`index predicate on '${f}' must be a boolean or null`);
|
|
1704
|
+
const def = fields[f];
|
|
1705
|
+
const isBoolean = meta !== void 0 ? meta.kind === "boolean" : def?.kind === "scalar" && def.type === "boolean";
|
|
1706
|
+
if (v !== null && !isBoolean) throw new Error(`index predicate '${f}: ${v}' compares a boolean, but '${f}' is not a boolean field`);
|
|
1707
|
+
const lhs = column ?? indexExpressionSql("text", f);
|
|
1708
|
+
predicate += v === null ? ` AND ${lhs} IS NULL` : column !== void 0 ? ` AND ${column} = ${v ? "TRUE" : "FALSE"}` : ` AND ${lhs} = ${v ? "'true'" : "'false'"}`;
|
|
1709
|
+
}
|
|
1710
|
+
const colSql = keys.map((k, i) => `(${k})${cols[i]?.dir === "desc" ? " DESC NULLS LAST" : ""}`);
|
|
1711
|
+
if (natural !== void 0) colSql.push(natural);
|
|
1712
|
+
if (decl.unique !== true && !withoutId) colSql.push("id");
|
|
1713
|
+
return {
|
|
1714
|
+
name,
|
|
1715
|
+
fields: cols.map((c) => c.field),
|
|
1716
|
+
unique: decl.unique === true,
|
|
1717
|
+
tail: `USING btree (${colSql.join(", ")}) WHERE ${predicate}`,
|
|
1718
|
+
keys,
|
|
1719
|
+
predicate,
|
|
1720
|
+
...decl.where !== void 0 && decl.where !== "live" ? { only: { ...decl.where } } : {},
|
|
1721
|
+
order: [...cols.map((c) => [c.field, c.dir === "desc" ? "desc" : "asc"]), ...naturalKey !== void 0 ? [naturalKey] : []]
|
|
1722
|
+
};
|
|
1723
|
+
};
|
|
1724
|
+
const [natField, natDir] = Object.entries(args.orderedBy)[0] ?? ["createdAt", "desc"];
|
|
1725
|
+
const natural = `${metadataColumn(natField) ?? `(${keyExpr(natField, fields[natField], promoted)})`}${natDir === "desc" ? " DESC NULLS LAST" : ""}`;
|
|
1726
|
+
for (const [field, def] of Object.entries(fields)) {
|
|
1727
|
+
if (!(def.kind === "crossRef" ? def.indexed !== false : def.kind === "scalar" && def.type !== "json" && def.indexed === true)) continue;
|
|
1728
|
+
const own = field === natField;
|
|
1729
|
+
specs.push({
|
|
1730
|
+
...btree({ fields: [{
|
|
1731
|
+
field,
|
|
1732
|
+
dir: own ? natDir : "asc"
|
|
1733
|
+
}] }, generatedIdentifier(namespace, type, field, "idx"), false, own ? void 0 : natural, own ? void 0 : [natField, natDir]),
|
|
1734
|
+
derived: true
|
|
1735
|
+
});
|
|
1736
|
+
}
|
|
1737
|
+
if (layout === "partitioned") {
|
|
1738
|
+
for (const [field, def] of Object.entries(fields)) {
|
|
1739
|
+
if (def.kind === "struct" || def.indexed !== true) continue;
|
|
1740
|
+
if (def.kind === "list" || def.kind === "scalar" && def.type === "json") specs.push({
|
|
1741
|
+
name: generatedIdentifier(namespace, type, field, "gin"),
|
|
1742
|
+
fields: [field],
|
|
1743
|
+
unique: false,
|
|
1744
|
+
tail: `USING gin ((body -> ${sqlStringLiteral(field)}) jsonb_path_ops) WHERE ${live}`,
|
|
1745
|
+
keys: [],
|
|
1746
|
+
predicate: live,
|
|
1747
|
+
order: []
|
|
1748
|
+
});
|
|
1749
|
+
}
|
|
1750
|
+
const base = (column, tail, order = []) => ({
|
|
1751
|
+
name: generatedIdentifier(namespace, type, column, "base"),
|
|
1752
|
+
fields: [],
|
|
1753
|
+
unique: false,
|
|
1754
|
+
tail,
|
|
1755
|
+
keys: [],
|
|
1756
|
+
predicate: live,
|
|
1757
|
+
order
|
|
1758
|
+
});
|
|
1759
|
+
specs.push(base("created_at", `USING btree (created_at DESC NULLS LAST, id) WHERE ${live}`, [["createdAt", "desc"]]), base("updated_at", `USING btree (updated_at DESC NULLS LAST, id) WHERE ${live}`, [["updatedAt", "desc"]]), base("deleted_at", "USING btree (deleted_at) WHERE deleted_at IS NOT NULL"));
|
|
1760
|
+
}
|
|
1761
|
+
if (args.ordering === "visible") specs.push({
|
|
1762
|
+
name: generatedIdentifier(namespace, type, "seq", "base"),
|
|
1763
|
+
fields: [],
|
|
1764
|
+
unique: false,
|
|
1765
|
+
tail: `USING btree (seq, id) WHERE ${live}`,
|
|
1766
|
+
keys: [],
|
|
1767
|
+
predicate: live,
|
|
1768
|
+
order: [["seq", "asc"]]
|
|
1769
|
+
});
|
|
1770
|
+
for (const decl of args.indexes) {
|
|
1771
|
+
const only = decl.fields.length === 1 ? decl.fields[0] : void 0;
|
|
1772
|
+
const onlyField = typeof only === "string" ? only : only?.dir === "asc" ? only.field : void 0;
|
|
1773
|
+
if (decl.name === void 0 && decl.unique !== true && (decl.where === void 0 || decl.where === "live") && onlyField !== void 0 && fields[onlyField]?.kind === "crossRef" && specs.some((s) => s.derived === true && s.fields[0] === onlyField)) continue;
|
|
1774
|
+
const keys = decl.fields.map((f) => typeof f === "string" ? f : f.field).join("_");
|
|
1775
|
+
const suffix = decl.unique === true ? "uniq" : "idx";
|
|
1776
|
+
let name = decl.name ?? generatedIdentifier(namespace, type, keys, suffix);
|
|
1777
|
+
if (decl.name === void 0) {
|
|
1778
|
+
const spec = btree(decl, name);
|
|
1779
|
+
const implicit = specs.find((s) => s.name === name);
|
|
1780
|
+
if (implicit !== void 0 && createSql(implicit, "rows") !== createSql(spec, "rows")) {
|
|
1781
|
+
const hash = createHash("sha256").update(`${spec.unique}|${btree(decl, name, true).tail}`).digest("hex").slice(0, 6);
|
|
1782
|
+
name = generatedIdentifier(namespace, type, `${keys}_${hash}`, suffix);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
if (!BARE_NAME.test(name) || name.length > 63) throw new Error(`index name '${name}' must be a lower-snake identifier of at most 63 bytes`);
|
|
1786
|
+
specs.push(btree(decl, name));
|
|
1787
|
+
}
|
|
1788
|
+
const byName = /* @__PURE__ */ new Map();
|
|
1789
|
+
for (const spec of specs) {
|
|
1790
|
+
const seen = byName.get(spec.name);
|
|
1791
|
+
if (seen === void 0) byName.set(spec.name, spec);
|
|
1792
|
+
else if (createSql(seen, "rows") !== createSql(spec, "rows")) throw new IndexDefinitionConflictError(spec.name, createSql(seen, "rows"), createSql(spec, "rows"));
|
|
1793
|
+
}
|
|
1794
|
+
return [...byName.values()];
|
|
1795
|
+
}
|
|
1796
|
+
function createSql(spec, table, name = spec.name, concurrently = false) {
|
|
1797
|
+
return `CREATE ${spec.unique ? "UNIQUE " : ""}INDEX ${concurrently ? "CONCURRENTLY " : ""}${name} ON ${ident(table)} ${spec.tail}`;
|
|
1798
|
+
}
|
|
1799
|
+
function described(row) {
|
|
1800
|
+
return {
|
|
1801
|
+
def: row.def.replace(/^(CREATE (?:UNIQUE )?INDEX) \S+ ON \S+ /, "$1 "),
|
|
1802
|
+
table: row.table,
|
|
1803
|
+
unique: row.unique,
|
|
1804
|
+
valid: row.valid,
|
|
1805
|
+
ready: row.ready,
|
|
1806
|
+
predicate: row.predicate,
|
|
1807
|
+
keyFields: row.cols.map((c) => {
|
|
1808
|
+
const lits = [...c.matchAll(/->>? '((?:[^']|'')*)'::text/g)].map((m) => m[1].replace(/''/g, "'"));
|
|
1809
|
+
return lits.length > 0 ? lits.join(" ") : c;
|
|
1810
|
+
}),
|
|
1811
|
+
marked: row.comment === MARK
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
1814
|
+
async function describeIndex(sql, name, schema) {
|
|
1815
|
+
const [row] = await sql`
|
|
1816
|
+
SELECT pg_get_indexdef(i.indexrelid) AS def, t.relname AS table,
|
|
1817
|
+
i.indisunique AS unique, i.indisvalid AS valid, i.indisready AS ready,
|
|
1818
|
+
pg_get_expr(i.indpred, i.indrelid) AS predicate,
|
|
1819
|
+
ARRAY(SELECT pg_get_indexdef(i.indexrelid, k, true)
|
|
1820
|
+
FROM generate_series(1, i.indnkeyatts) k ORDER BY k) AS cols,
|
|
1821
|
+
obj_description(i.indexrelid, 'pg_class') AS comment
|
|
1822
|
+
FROM pg_index i
|
|
1823
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
1824
|
+
JOIN pg_class t ON t.oid = i.indrelid
|
|
1825
|
+
WHERE c.relname = ${name}
|
|
1826
|
+
AND c.relnamespace = ${schema === "temp" ? sql`pg_my_temp_schema()` : sql`current_schema()::regnamespace`}`;
|
|
1827
|
+
return row === void 0 ? void 0 : described(row);
|
|
1828
|
+
}
|
|
1829
|
+
/** Each spec as pg deparses it, built on an empty temp copy of `rows` and rolled back. */
|
|
1830
|
+
async function deparse(conn, specs) {
|
|
1831
|
+
await conn.unsafe("BEGIN");
|
|
1832
|
+
try {
|
|
1833
|
+
await conn.unsafe("CREATE TEMP TABLE substrate_index_probe (LIKE rows)");
|
|
1834
|
+
const out = [];
|
|
1835
|
+
for (const [i, spec] of specs.entries()) {
|
|
1836
|
+
const name = `substrate_index_probe_${i}`;
|
|
1837
|
+
await conn.unsafe(createSql(spec, "substrate_index_probe", name));
|
|
1838
|
+
out.push(await describeIndex(conn, name, "temp"));
|
|
1839
|
+
}
|
|
1840
|
+
return out;
|
|
1841
|
+
} finally {
|
|
1842
|
+
await conn.unsafe("ROLLBACK");
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
/** `<name>__swap`, hashed to fit 63 bytes when needed. */
|
|
1846
|
+
function swapNameFor(name) {
|
|
1847
|
+
const plain = `${name}__swap`;
|
|
1848
|
+
if (plain.length <= 63) return plain;
|
|
1849
|
+
return `${hashedName(name, name, 63 - 6)}__swap`;
|
|
1850
|
+
}
|
|
1851
|
+
async function stateOf(conn, name) {
|
|
1852
|
+
return stateFrom(await describeIndex(conn, name, "current"));
|
|
1853
|
+
}
|
|
1854
|
+
async function reconcileOne(conn, args, target, spec, desired, opts) {
|
|
1855
|
+
const report = (state, outcome) => indexState(spec, state, outcome);
|
|
1856
|
+
const signal = spec.unique ? void 0 : opts.signal;
|
|
1857
|
+
const swapName = swapNameFor(spec.name);
|
|
1858
|
+
await dropIndexIfInvalid(conn, spec.name);
|
|
1859
|
+
await dropIndexIfInvalid(conn, swapName);
|
|
1860
|
+
const named = await describeIndex(conn, spec.name, "current");
|
|
1861
|
+
const leftover = await describeIndex(conn, swapName, "current");
|
|
1862
|
+
const identical = (d) => d !== void 0 && d.table === target && d.def === desired.def;
|
|
1863
|
+
if (identical(named)) {
|
|
1864
|
+
if (leftover !== void 0) await dropConcurrently(conn, swapName, signal);
|
|
1865
|
+
if (named?.marked === true) return report("valid", "unchanged");
|
|
1866
|
+
await mark(conn, spec.name);
|
|
1867
|
+
return report("valid", "adopted");
|
|
1868
|
+
}
|
|
1869
|
+
const existing = named ?? leftover;
|
|
1870
|
+
if (existing === void 0) {
|
|
1871
|
+
const built = await build(conn, args, target, spec, spec.name, opts);
|
|
1872
|
+
if (built !== void 0) return built;
|
|
1873
|
+
await mark(conn, spec.name);
|
|
1874
|
+
return report(await stateOf(conn, spec.name), "created");
|
|
1875
|
+
}
|
|
1876
|
+
const fieldOf = new Map(Object.entries(args.promotedColumns ?? {}).map(([field, meta]) => [meta.column, field]));
|
|
1877
|
+
const keyIdentity = (d) => {
|
|
1878
|
+
const keys = d.keyFields.map((k) => fieldOf.get(k) ?? k);
|
|
1879
|
+
if (!d.unique && keys.length > 1 && keys.at(-1) === "id") keys.pop();
|
|
1880
|
+
return keys;
|
|
1881
|
+
};
|
|
1882
|
+
const have = keyIdentity(existing);
|
|
1883
|
+
const want = keyIdentity(desired);
|
|
1884
|
+
const n = spec.fields.length;
|
|
1885
|
+
const sameKeys = spec.derived === true ? have.length <= n + 1 && have.slice(0, n).join("\n") === want.slice(0, n).join("\n") : have.join("\n") === want.join("\n");
|
|
1886
|
+
if (!(existing.table === target && existing.unique === desired.unique && sameKeys)) throw new IndexDefinitionConflictError(spec.name, existing.def.replace(/^(CREATE (?:UNIQUE )?INDEX) /, `$1 ${named === void 0 ? swapName : spec.name} ON ${existing.table} `), createSql(spec, target));
|
|
1887
|
+
if (named === void 0) {
|
|
1888
|
+
if (identical(leftover)) await conn.unsafe(`ALTER INDEX ${swapName} RENAME TO ${spec.name}`);
|
|
1889
|
+
else {
|
|
1890
|
+
const built = await build(conn, args, target, spec, spec.name, opts);
|
|
1891
|
+
if (built !== void 0) return built;
|
|
1892
|
+
await dropConcurrently(conn, swapName, signal);
|
|
1893
|
+
}
|
|
1894
|
+
await mark(conn, spec.name);
|
|
1895
|
+
return report(await stateOf(conn, spec.name), "swapped");
|
|
1896
|
+
}
|
|
1897
|
+
if (!identical(leftover)) {
|
|
1898
|
+
if (leftover !== void 0) await dropConcurrently(conn, swapName, signal);
|
|
1899
|
+
const built = await build(conn, args, target, spec, swapName, opts);
|
|
1900
|
+
if (built !== void 0) return built;
|
|
1901
|
+
}
|
|
1902
|
+
await dropConcurrently(conn, spec.name, signal);
|
|
1903
|
+
await conn.unsafe(`ALTER INDEX ${swapName} RENAME TO ${spec.name}`);
|
|
1904
|
+
await mark(conn, spec.name);
|
|
1905
|
+
return report(await stateOf(conn, spec.name), "swapped");
|
|
1906
|
+
}
|
|
1907
|
+
/**
|
|
1908
|
+
* Builds `spec` as `name` CONCURRENTLY, refusing a unique one over duplicate
|
|
1909
|
+
* live values. Returns `failed` when a non-unique build fails or is cancelled
|
|
1910
|
+
* through `opts.signal`, which costs only a scan until the next reconcile
|
|
1911
|
+
* drops the INVALID leftover and retries it, and `pending` when the disk
|
|
1912
|
+
* guard refuses it; a unique one throws.
|
|
1913
|
+
*/
|
|
1914
|
+
async function build(conn, args, target, spec, name, opts) {
|
|
1915
|
+
if (spec.unique) await assertNoDuplicates(conn, args, target, spec);
|
|
1916
|
+
if (!spec.unique && opts.diskGuard !== void 0) {
|
|
1917
|
+
const need = await buildEstimate(conn, target, spec.name);
|
|
1918
|
+
const verdict = await opts.diskGuard.check(need);
|
|
1919
|
+
if (!verdict.ok) {
|
|
1920
|
+
logChanged(spec.name, "insufficient_disk", `index ${name} on ${target} was not built and will be retried: ${verdict.freeBytes} bytes free, ${verdict.needBytes} needed (estimate plus margin)\n`);
|
|
1921
|
+
return {
|
|
1922
|
+
...indexState(spec, "pending", "unchanged"),
|
|
1923
|
+
reason: "insufficient_disk"
|
|
1924
|
+
};
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
opts.onBuild?.(spec.name);
|
|
1928
|
+
try {
|
|
1929
|
+
await cancellable(conn, createSql(spec, target, name, true), spec.unique ? void 0 : opts.signal);
|
|
1930
|
+
return;
|
|
1931
|
+
} catch (err) {
|
|
1932
|
+
if (spec.unique) throw err;
|
|
1933
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1934
|
+
logChanged(spec.name, `build_failed: ${message}`, `index ${name} on ${target} failed to build and will be retried: ${message}\n`);
|
|
1935
|
+
return indexState(spec, "failed", "unchanged", `build_failed: ${message}`);
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
/**
|
|
1939
|
+
* The type's non-unique indexes a reconcile would build or swap (not held
|
|
1940
|
+
* exactly as declared), each with the disk guard's estimate for its build.
|
|
1941
|
+
* Changes nothing; the type's storage must exist.
|
|
1942
|
+
*/
|
|
1943
|
+
async function pendingIndexBuilds(sql, args) {
|
|
1944
|
+
const specs = desiredSpecs(args).filter((s) => !s.unique);
|
|
1945
|
+
if (specs.length === 0) return [];
|
|
1946
|
+
const target = await targetTable(sql, args);
|
|
1947
|
+
const conn = await sql.reserve();
|
|
1948
|
+
try {
|
|
1949
|
+
const desired = await deparse(conn, specs);
|
|
1950
|
+
const out = [];
|
|
1951
|
+
for (const [i, spec] of specs.entries()) {
|
|
1952
|
+
if (await heldAsDeclared(conn, target, spec, desired[i]) !== void 0) continue;
|
|
1953
|
+
out.push({
|
|
1954
|
+
name: spec.name,
|
|
1955
|
+
estimateBytes: await buildEstimate(conn, target, spec.name)
|
|
1956
|
+
});
|
|
1957
|
+
}
|
|
1958
|
+
return out;
|
|
1959
|
+
} finally {
|
|
1960
|
+
conn.release();
|
|
1961
|
+
}
|
|
1962
|
+
}
|
|
1963
|
+
/**
|
|
1964
|
+
* Bytes a build of index `name` on `target` may take: the index it replaces (under
|
|
1965
|
+
* its name or as a `__swap` leftover), else 30% of the table.
|
|
1966
|
+
* shortcut: on legacy the table is the shared `rows`, so a new index is
|
|
1967
|
+
* estimated against every type's rows, pessimistic but safe; estimate from the
|
|
1968
|
+
* type's own rows if that refuses builds a full disk could hold.
|
|
1969
|
+
*/
|
|
1970
|
+
async function buildEstimate(conn, target, name) {
|
|
1971
|
+
const [row] = await conn`
|
|
1972
|
+
SELECT COALESCE(
|
|
1973
|
+
pg_relation_size(to_regclass(${name})),
|
|
1974
|
+
pg_relation_size(to_regclass(${swapNameFor(name)})),
|
|
1975
|
+
(pg_table_size(to_regclass(${ident(target)})) * 0.3)::bigint,
|
|
1976
|
+
0)::text AS n`;
|
|
1977
|
+
return Number(row?.n ?? 0);
|
|
1978
|
+
}
|
|
1979
|
+
/** Runs `text` on `conn`, cancelling it in flight when `signal` aborts. */
|
|
1980
|
+
async function cancellable(conn, text, signal) {
|
|
1981
|
+
const query = conn.unsafe(text);
|
|
1982
|
+
const cancel = () => {
|
|
1983
|
+
Promise.resolve(query.cancel()).catch(() => void 0);
|
|
1984
|
+
};
|
|
1985
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
1986
|
+
try {
|
|
1987
|
+
if (signal?.aborted === true) cancel();
|
|
1988
|
+
await query;
|
|
1989
|
+
} finally {
|
|
1990
|
+
signal?.removeEventListener("abort", cancel);
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
async function dropConcurrently(conn, name, signal) {
|
|
1994
|
+
await cancellable(conn, `DROP INDEX CONCURRENTLY IF EXISTS ${name}`, signal);
|
|
1995
|
+
}
|
|
1996
|
+
async function mark(conn, name) {
|
|
1997
|
+
await conn.unsafe(`COMMENT ON INDEX ${name} IS '${MARK}'`);
|
|
1998
|
+
}
|
|
1999
|
+
async function assertNoDuplicates(conn, args, target, spec) {
|
|
2000
|
+
const present = spec.keys.map((k) => ` AND (${k}) IS NOT NULL`).join("");
|
|
2001
|
+
const [row] = await conn.unsafe(`SELECT count(*)::int AS n FROM (
|
|
2002
|
+
SELECT 1 FROM ${ident(target)} WHERE ${spec.predicate}${present}
|
|
2003
|
+
GROUP BY ${spec.keys.map((k) => `(${k})`).join(", ")} HAVING count(*) > 1) d`);
|
|
2004
|
+
const count = row?.n ?? 0;
|
|
2005
|
+
if (count > 0) throw new UniqueIndexDuplicatesError(args.namespace, args.type, spec.fields, count);
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
//#endregion
|
|
2009
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-short-id-store.ts
|
|
2010
|
+
/**
|
|
2011
|
+
* Derives a sequence name from (ns, type): lowercased, non-alphanumerics
|
|
2012
|
+
* replaced with `_`, joined with a `__` separator. Safe for double-quoted
|
|
2013
|
+
* identifier use.
|
|
2014
|
+
*
|
|
2015
|
+
* Distinct pairs can still share a sequence (`Foo` and `foo`, `a-` and `a_`,
|
|
2016
|
+
* ('foo_', 'bar') and ('foo', '_bar') through the separator, names past pg's
|
|
2017
|
+
* 63-byte limit). That only leaves gaps: numbers stay unique per (ns, type),
|
|
2018
|
+
* the `short_ids` primary key. The name is not changed to avoid it, because
|
|
2019
|
+
* renaming would restart existing sequences at 1. Underscore-led namespaces
|
|
2020
|
+
* such as `_system_proposals` are therefore fine.
|
|
2021
|
+
*
|
|
2022
|
+
* Exported for migrate-store, which advances every sequence after a copy.
|
|
2023
|
+
*/
|
|
2024
|
+
function seqName(ns, type) {
|
|
2025
|
+
const sanitize = (s) => s.toLowerCase().replace(/[^a-z0-9]/g, "_");
|
|
2026
|
+
return `${sanitize(ns)}__${sanitize(type)}_short_id_seq`;
|
|
2027
|
+
}
|
|
2028
|
+
/**
|
|
2029
|
+
* Parses a `bigint` column value (postgres.js returns a string) into a JS
|
|
2030
|
+
* number. Throws if it exceeds `Number.MAX_SAFE_INTEGER`, where parsing
|
|
2031
|
+
* would silently round into collisions. `undefined` (no row) returns `0`.
|
|
2032
|
+
*
|
|
2033
|
+
* @internal Exported for unit testing of the safe-integer assertion. Not
|
|
2034
|
+
* part of the public package API.
|
|
2035
|
+
*/
|
|
2036
|
+
function parseShortId(raw) {
|
|
2037
|
+
if (raw === void 0) return 0;
|
|
2038
|
+
const n = Number.parseInt(raw, 10);
|
|
2039
|
+
if (!Number.isSafeInteger(n)) throw new Error(`PgShortIdStore: short_id ${raw} exceeds JS safe integer range`);
|
|
2040
|
+
return n;
|
|
2041
|
+
}
|
|
2042
|
+
var PgShortIdStore = class {
|
|
2043
|
+
sql;
|
|
2044
|
+
#ensuredSequences = /* @__PURE__ */ new Set();
|
|
2045
|
+
/** Wraps a single PG handle (Sql or TransactionSql); does not own the connection. */
|
|
2046
|
+
constructor(sql) {
|
|
2047
|
+
this.sql = sql;
|
|
2048
|
+
}
|
|
2049
|
+
async ensureSequence(ns, type) {
|
|
2050
|
+
const name = seqName(ns, type);
|
|
2051
|
+
const cacheKey = `${ns}\0${type}`;
|
|
2052
|
+
if (this.#ensuredSequences.has(cacheKey)) return name;
|
|
2053
|
+
await this.sql.unsafe(`DO $$ BEGIN PERFORM pg_advisory_xact_lock(hashtext(current_schema() || '.' || '${name}')); CREATE SEQUENCE IF NOT EXISTS "${name}"; END $$`);
|
|
2054
|
+
this.#ensuredSequences.add(cacheKey);
|
|
2055
|
+
return name;
|
|
2056
|
+
}
|
|
2057
|
+
/**
|
|
2058
|
+
* Allocate a new short-id without binding it to a row. The caller is
|
|
2059
|
+
* responsible for calling `bind` before the row is visible to consumers.
|
|
2060
|
+
* Used by the migrate backfill's preload path.
|
|
2061
|
+
*/
|
|
2062
|
+
async allocateUnbound(ns, type) {
|
|
2063
|
+
const name = await this.ensureSequence(ns, type);
|
|
2064
|
+
return parseShortId((await this.sql.unsafe(`SELECT nextval('"${name}"') AS nextval`))[0]?.nextval);
|
|
2065
|
+
}
|
|
2066
|
+
/**
|
|
2067
|
+
* Bind a previously-allocated short-id to a row. `ON CONFLICT DO NOTHING`
|
|
2068
|
+
* covers both the PK `(namespace, type, short_id)` and the UNIQUE index
|
|
2069
|
+
* `(namespace, type, row_id)` — idempotent for backfill re-runs.
|
|
2070
|
+
*/
|
|
2071
|
+
async bind(ns, type, shortId, rowId) {
|
|
2072
|
+
await this.sql`
|
|
2073
|
+
INSERT INTO short_ids (namespace, type, short_id, row_id)
|
|
2074
|
+
VALUES (${ns}, ${type}, ${shortId}, ${rowId})
|
|
2075
|
+
ON CONFLICT DO NOTHING
|
|
2076
|
+
`;
|
|
2077
|
+
}
|
|
2078
|
+
/**
|
|
2079
|
+
* Resolve a numeric short-id to its row UUID. Returns `null` when no
|
|
2080
|
+
* binding exists for the given `(ns, type, shortId)` triple.
|
|
2081
|
+
*/
|
|
2082
|
+
async resolveShortId(ns, type, shortId) {
|
|
2083
|
+
return (await this.sql`
|
|
2084
|
+
SELECT row_id FROM short_ids
|
|
2085
|
+
WHERE namespace = ${ns} AND type = ${type} AND short_id = ${shortId}
|
|
2086
|
+
`)[0]?.row_id ?? null;
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Look up the short-id assigned to a row UUID. Returns `null` when the row
|
|
2090
|
+
* has no short-id binding yet.
|
|
2091
|
+
*/
|
|
2092
|
+
async getShortIdForRow(ns, type, rowId) {
|
|
2093
|
+
const rows = await this.sql`
|
|
2094
|
+
SELECT short_id FROM short_ids
|
|
2095
|
+
WHERE namespace = ${ns} AND type = ${type} AND row_id = ${rowId}
|
|
2096
|
+
`;
|
|
2097
|
+
if (!rows[0]) return null;
|
|
2098
|
+
return parseShortId(rows[0].short_id);
|
|
2099
|
+
}
|
|
2100
|
+
/**
|
|
2101
|
+
* Hard-delete all short-id rows for a specific (ns, type). Returns the count
|
|
2102
|
+
* of rows deleted. The sequence for that type is left in place — sequences
|
|
2103
|
+
* are schema objects and the caller can recreate one if needed.
|
|
2104
|
+
*/
|
|
2105
|
+
async dropType(ns, type) {
|
|
2106
|
+
return (await this.sql`
|
|
2107
|
+
DELETE FROM short_ids
|
|
2108
|
+
WHERE namespace = ${ns} AND type = ${type}
|
|
2109
|
+
RETURNING short_id
|
|
2110
|
+
`).length;
|
|
2111
|
+
}
|
|
2112
|
+
/**
|
|
2113
|
+
* Hard-delete all short-id rows for every type in a namespace. Returns the
|
|
2114
|
+
* total count of rows deleted across all types.
|
|
2115
|
+
*/
|
|
2116
|
+
async dropNamespace(ns) {
|
|
2117
|
+
return (await this.sql`
|
|
2118
|
+
DELETE FROM short_ids
|
|
2119
|
+
WHERE namespace = ${ns}
|
|
2120
|
+
RETURNING short_id
|
|
2121
|
+
`).length;
|
|
2122
|
+
}
|
|
2123
|
+
/**
|
|
2124
|
+
* Bulk-insert pre-existing mappings and advance the sequence past the max
|
|
2125
|
+
* shortId seen. Used by substrate-migrate backfill. Idempotent — re-runs are
|
|
2126
|
+
* no-ops via ON CONFLICT DO NOTHING + GREATEST on setval.
|
|
2127
|
+
*/
|
|
2128
|
+
async preload(ns, type, entries) {
|
|
2129
|
+
if (entries.length === 0) return;
|
|
2130
|
+
await this.sql`
|
|
2131
|
+
INSERT INTO short_ids ${this.sql(entries.map((e) => ({
|
|
2132
|
+
namespace: ns,
|
|
2133
|
+
type,
|
|
2134
|
+
short_id: e.shortId,
|
|
2135
|
+
row_id: e.rowId
|
|
2136
|
+
})), "namespace", "type", "short_id", "row_id")}
|
|
2137
|
+
ON CONFLICT (namespace, type, short_id) DO NOTHING
|
|
2138
|
+
`;
|
|
2139
|
+
const max = entries.reduce((m, e) => Math.max(m, e.shortId), 0);
|
|
2140
|
+
const name = await this.ensureSequence(ns, type);
|
|
2141
|
+
await this.sql.unsafe(`SELECT setval('"${name}"', GREATEST((SELECT last_value FROM "${name}"), ${max}))`);
|
|
2142
|
+
}
|
|
2143
|
+
};
|
|
2144
|
+
|
|
2145
|
+
//#endregion
|
|
2146
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/projection.ts
|
|
2147
|
+
/**
|
|
2148
|
+
* Returns the body SELECT expression for a read.
|
|
2149
|
+
*
|
|
2150
|
+
* - `fields === undefined` → the full body.
|
|
2151
|
+
* - `fields` supplied → `jsonb_build_object(...)` over `id` plus the
|
|
2152
|
+
* requested top-level body keys. `id` is always included because
|
|
2153
|
+
* `materialize()` reads `body.id` to preserve numeric ids.
|
|
2154
|
+
*
|
|
2155
|
+
* Keys are bound as parameters, never concatenated.
|
|
2156
|
+
*/
|
|
2157
|
+
function buildBodyProjection(sql, fields) {
|
|
2158
|
+
if (fields === void 0) return sql`r.body`;
|
|
2159
|
+
const bodyKeys = /* @__PURE__ */ new Set(["id"]);
|
|
2160
|
+
for (const key of Object.keys(fields)) {
|
|
2161
|
+
if (NON_BODY_FIELD_KEYS.has(key)) continue;
|
|
2162
|
+
bodyKeys.add(key);
|
|
2163
|
+
}
|
|
2164
|
+
const keyArr = Array.from(bodyKeys);
|
|
2165
|
+
if (keyArr.length === 0) return sql`'{}'::jsonb`;
|
|
2166
|
+
const pairs = keyArr.map((k) => sql`${k}::text, r.body->${k}`);
|
|
2167
|
+
const objects = [];
|
|
2168
|
+
for (let i = 0; i < pairs.length; i += MAX_KEYS_PER_CALL) objects.push(sql`jsonb_build_object(${join(pairs.slice(i, i + MAX_KEYS_PER_CALL), sql`,`)})`);
|
|
2169
|
+
return sql`${join(objects, sql` || `)}`;
|
|
2170
|
+
}
|
|
2171
|
+
const MAX_KEYS_PER_CALL = 50;
|
|
2172
|
+
function join(parts, sep) {
|
|
2173
|
+
const out = [];
|
|
2174
|
+
for (const [i, p] of parts.entries()) {
|
|
2175
|
+
if (i > 0) out.push(sep);
|
|
2176
|
+
out.push(p);
|
|
2177
|
+
}
|
|
2178
|
+
return out;
|
|
2179
|
+
}
|
|
2180
|
+
function assembleRowMetadata(record) {
|
|
2181
|
+
const seq = record.seq == null ? void 0 : Number(record.seq);
|
|
2182
|
+
if (seq === void 0 && record.created_at === null && record.updated_at === null && record.created_by_kind === null && record.created_by_id === null && record.updated_by_kind === null && record.updated_by_id === null) return;
|
|
2183
|
+
const createdBy = record.created_by_kind !== null && record.created_by_id !== null ? {
|
|
2184
|
+
kind: record.created_by_kind,
|
|
2185
|
+
id: record.created_by_id
|
|
2186
|
+
} : null;
|
|
2187
|
+
const updatedBy = record.updated_by_kind !== null && record.updated_by_id !== null ? {
|
|
2188
|
+
kind: record.updated_by_kind,
|
|
2189
|
+
id: record.updated_by_id
|
|
2190
|
+
} : null;
|
|
2191
|
+
return {
|
|
2192
|
+
createdAt: record.created_at !== null ? record.created_at.toISOString() : null,
|
|
2193
|
+
updatedAt: record.updated_at !== null ? record.updated_at.toISOString() : null,
|
|
2194
|
+
createdBy,
|
|
2195
|
+
updatedBy,
|
|
2196
|
+
...seq !== void 0 ? { seq } : {}
|
|
2197
|
+
};
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
//#endregion
|
|
2201
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/sorting.ts
|
|
2202
|
+
function normalizeOrderBy(orderBy) {
|
|
2203
|
+
const out = [];
|
|
2204
|
+
if (orderBy !== void 0) for (const clause of orderBy) for (const [field, dir] of Object.entries(clause)) out.push([field, dir]);
|
|
2205
|
+
if (!out.some(([f]) => f === "id")) out.push(["id", "asc"]);
|
|
2206
|
+
return out;
|
|
2207
|
+
}
|
|
2208
|
+
/**
|
|
2209
|
+
* Maps a FieldDef to the pg cast applied to its `body ->> field` text
|
|
2210
|
+
* extraction. Returns `null` for kinds where text compare is correct (string),
|
|
2211
|
+
* opaque (json), structural (struct/list/crossRef), or undeclared.
|
|
2212
|
+
*
|
|
2213
|
+
* Type → cast table:
|
|
2214
|
+
* field.number() / 'number' → 'numeric'
|
|
2215
|
+
* field.date() / 'date' → 'timestamptz'
|
|
2216
|
+
* field.boolean() / 'boolean' → 'boolean'
|
|
2217
|
+
* field.string() / 'string' → null (text compare is correct)
|
|
2218
|
+
* field.json() / 'json' → null (opaque body — orderBy is rejected
|
|
2219
|
+
* by the validator anyway)
|
|
2220
|
+
* crossRef / struct / list → null (orderBy on these isn't a scalar
|
|
2221
|
+
* comparison; let text-compare
|
|
2222
|
+
* surface the bug at validation)
|
|
2223
|
+
*/
|
|
2224
|
+
function resolveBodyOrderByCast(def) {
|
|
2225
|
+
if (def === void 0) return null;
|
|
2226
|
+
if (def.kind !== "scalar") return null;
|
|
2227
|
+
switch (def.type) {
|
|
2228
|
+
case "number": return "numeric";
|
|
2229
|
+
case "date": return "timestamptz";
|
|
2230
|
+
case "boolean": return "boolean";
|
|
2231
|
+
default: return null;
|
|
2232
|
+
}
|
|
2233
|
+
}
|
|
2234
|
+
function buildOrderBy(sql, orderBy, castFor, promotedFor) {
|
|
2235
|
+
if (orderBy.length === 0) return sql``;
|
|
2236
|
+
const parts = [];
|
|
2237
|
+
for (const [field, dir] of orderBy) parts.push(orderByFragment(sql, field, dir, castFor(field), promotedFor));
|
|
2238
|
+
if (parts.length === 1) return sql`ORDER BY ${parts[0]}`;
|
|
2239
|
+
const commaFragment = sql`,`;
|
|
2240
|
+
const interleaved = [parts[0]];
|
|
2241
|
+
for (let i = 1; i < parts.length; i++) interleaved.push(commaFragment, parts[i]);
|
|
2242
|
+
return sql`ORDER BY ${interleaved}`;
|
|
2243
|
+
}
|
|
2244
|
+
/**
|
|
2245
|
+
* The SQL expression a field SORTS BY — the single source for ORDER BY and for
|
|
2246
|
+
* both halves of the keyset cursor comparison.
|
|
2247
|
+
*
|
|
2248
|
+
* Do not case out field kinds anywhere else: if sorting and paging use
|
|
2249
|
+
* different expressions, rows silently vanish from later pages.
|
|
2250
|
+
*/
|
|
2251
|
+
function sortExpr(sql, field, bodyCast, promotedFor) {
|
|
2252
|
+
if (field === "id") return sql`id`;
|
|
2253
|
+
if (field === "shortId") return sql`s.short_id`;
|
|
2254
|
+
if (isMetadataField(field)) {
|
|
2255
|
+
const col = METADATA_COLUMNS$1[field];
|
|
2256
|
+
return field === "createdAt" || field === "updatedAt" || field === "seq" ? sql.unsafe(col) : sql.unsafe(`${col} COLLATE "C"`);
|
|
2257
|
+
}
|
|
2258
|
+
const promoted = promotedFor(field);
|
|
2259
|
+
if (promoted !== void 0) {
|
|
2260
|
+
const col = promotedColumnSql(promoted);
|
|
2261
|
+
return sql.unsafe(promoted.kind === "text" ? `${col} COLLATE "C"` : col);
|
|
2262
|
+
}
|
|
2263
|
+
return bodyOrderByExpr(sql, field, bodyCast);
|
|
2264
|
+
}
|
|
2265
|
+
/**
|
|
2266
|
+
* Projects each non-`id` ORDER BY field's sort expression alongside the row,
|
|
2267
|
+
* so the keyset cursor's value comes from the expression that sorted, not the
|
|
2268
|
+
* materialized row (which holds only the projected fields). Returns the SELECT
|
|
2269
|
+
* fragment and the field → alias map.
|
|
2270
|
+
*
|
|
2271
|
+
* Aliases are POSITIONAL (`__sort_0`, ...), not field-derived: pg truncates
|
|
2272
|
+
* identifiers at 63 bytes, so long field names could collide.
|
|
2273
|
+
*/
|
|
2274
|
+
function buildSortKeyProjection(sql, orderBy, castFor, promotedFor) {
|
|
2275
|
+
const aliases = /* @__PURE__ */ new Map();
|
|
2276
|
+
const fragments = [];
|
|
2277
|
+
for (const [field] of orderBy) {
|
|
2278
|
+
if (field === "id" || aliases.has(field)) continue;
|
|
2279
|
+
const alias = `__sort_${aliases.size}`;
|
|
2280
|
+
aliases.set(field, alias);
|
|
2281
|
+
const cast = castFor(field);
|
|
2282
|
+
const expr = sortExpr(sql, field, cast, promotedFor);
|
|
2283
|
+
fragments.push(sortKeyIsTimestamp(field, cast, promotedFor) ? sql`, (${expr})::text AS ${sql(alias)}` : sql`, ${expr} AS ${sql(alias)}`);
|
|
2284
|
+
}
|
|
2285
|
+
if (fragments.length === 0) return {
|
|
2286
|
+
fragment: sql``,
|
|
2287
|
+
aliases
|
|
2288
|
+
};
|
|
2289
|
+
return {
|
|
2290
|
+
fragment: fragments,
|
|
2291
|
+
aliases
|
|
2292
|
+
};
|
|
2293
|
+
}
|
|
2294
|
+
/** Whether `buildSortKeyProjection` projects the field's key as timestamptz text. */
|
|
2295
|
+
function sortKeyIsTimestamp(field, cast, promotedFor) {
|
|
2296
|
+
return field === "createdAt" || field === "updatedAt" || cast === "timestamptz" && !isMetadataField(field) && promotedFor(field) === void 0;
|
|
2297
|
+
}
|
|
2298
|
+
/**
|
|
2299
|
+
* `buildOrderBy` re-expressed over a subquery `rel` that projected the sort
|
|
2300
|
+
* keys under `aliases` (from `buildSortKeyProjection`) and `id`. Same
|
|
2301
|
+
* directions and NULLS LAST; a timestamp key, projected as text, casts back
|
|
2302
|
+
* so it orders as a timestamp.
|
|
2303
|
+
*/
|
|
2304
|
+
function buildProjectedOrderBy(sql, orderBy, aliases, castFor, promotedFor, rel) {
|
|
2305
|
+
const parts = [];
|
|
2306
|
+
for (const [field, dir] of orderBy) {
|
|
2307
|
+
let expr;
|
|
2308
|
+
if (field === "id") expr = `${rel}.id`;
|
|
2309
|
+
else {
|
|
2310
|
+
const alias = aliases.get(field);
|
|
2311
|
+
if (alias === void 0) throw new Error(`orderBy key '${field}' has no projected alias`);
|
|
2312
|
+
expr = sortKeyIsTimestamp(field, castFor(field), promotedFor) ? `${rel}.${alias}::timestamptz` : `${rel}.${alias}`;
|
|
2313
|
+
}
|
|
2314
|
+
parts.push(`${expr} ${dir === "asc" ? "ASC" : "DESC"} NULLS LAST`);
|
|
2315
|
+
}
|
|
2316
|
+
return parts.length === 0 ? sql`` : sql.unsafe(`ORDER BY ${parts.join(", ")}`);
|
|
2317
|
+
}
|
|
2318
|
+
/**
|
|
2319
|
+
* The parameter side of a comparison against `sortExpr(field)`, cast to match
|
|
2320
|
+
* it. Keep its cases in step with `sortExpr`.
|
|
2321
|
+
*/
|
|
2322
|
+
function sortParam(sql, field, value, bodyCast, promotedFor) {
|
|
2323
|
+
if (field === "id") return sql`${idToText(value)}`;
|
|
2324
|
+
if (field === "shortId") return sql`${value}`;
|
|
2325
|
+
if (field === "seq") return sql`${String(value)}::bigint`;
|
|
2326
|
+
if (isMetadataField(field)) return field === "createdAt" || field === "updatedAt" ? tsParam(sql, value) : sql`${value}`;
|
|
2327
|
+
const promoted = promotedFor(field);
|
|
2328
|
+
if (promoted !== void 0) return promoted.kind === "boolean" ? sql`${Boolean(value)}::boolean` : sql`${String(value)}::text`;
|
|
2329
|
+
return castParam(sql, value, bodyCast);
|
|
2330
|
+
}
|
|
2331
|
+
function orderByFragment(sql, field, dir, bodyCast, promotedFor) {
|
|
2332
|
+
const expr = sortExpr(sql, field, bodyCast, promotedFor);
|
|
2333
|
+
return dir === "asc" ? sql`${expr} ASC NULLS LAST` : sql`${expr} DESC NULLS LAST`;
|
|
2334
|
+
}
|
|
2335
|
+
/**
|
|
2336
|
+
* The body-field expression for a sort: the filter compiler's expression for
|
|
2337
|
+
* the declared kind, so an expression index serves both. A wrong-shaped number
|
|
2338
|
+
* or date reads as NULL and sorts last.
|
|
2339
|
+
*/
|
|
2340
|
+
function bodyOrderByExpr(sql, field, cast) {
|
|
2341
|
+
switch (cast) {
|
|
2342
|
+
case null: return bodyText(sql, [field]);
|
|
2343
|
+
case "numeric": return bodyNum(sql, [field]);
|
|
2344
|
+
case "timestamptz": return bodyTs(sql, [field]);
|
|
2345
|
+
case "boolean": return sql`${bodyRawText(sql, [field])}::boolean`;
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
/**
|
|
2349
|
+
* Cursor tuple compare. For `ORDER BY a ASC, b DESC, id ASC`, the "next page"
|
|
2350
|
+
* predicate is the OR of equality-prefixes:
|
|
2351
|
+
* (a > cursor.a)
|
|
2352
|
+
* OR (a = cursor.a AND b < cursor.b)
|
|
2353
|
+
* OR (a = cursor.a AND b = cursor.b AND id > cursor.id)
|
|
2354
|
+
*
|
|
2355
|
+
* A top-level OR cannot be an index condition, so a nullable leading key `a`
|
|
2356
|
+
* is bounded instead, and only its non-null rows are matched; the null tail
|
|
2357
|
+
* after a non-null cursor is `buildCursorNullTail`'s second query:
|
|
2358
|
+
* cursor.a NOT NULL — a >= cursor.a AND (a > cursor.a OR (a = cursor.a AND <rest>))
|
|
2359
|
+
* (`<=` and `<` for DESC)
|
|
2360
|
+
* cursor.a IS NULL — a IS NULL AND <rest>
|
|
2361
|
+
* where `<rest>` is the disjunction over the later keys. A leading `id` is
|
|
2362
|
+
* NOT NULL and keeps the plain disjunction, whose first arm is sargable.
|
|
2363
|
+
*
|
|
2364
|
+
* We build the disjunction explicitly — postgres's ROW(...) tuple compare
|
|
2365
|
+
* would be simpler but doesn't honour per-column direction mixing.
|
|
2366
|
+
*/
|
|
2367
|
+
function buildCursorCompare(sql, orderBy, values, cursorId, castFor, promotedFor) {
|
|
2368
|
+
const resolved = resolveCursorKeys(orderBy, values, cursorId);
|
|
2369
|
+
const [lead, ...rest] = resolved;
|
|
2370
|
+
if (lead === void 0) return null;
|
|
2371
|
+
if (lead.field === "id") return cursorDisjunction(sql, resolved, castFor, promotedFor);
|
|
2372
|
+
const restCmp = cursorDisjunction(sql, rest, castFor, promotedFor);
|
|
2373
|
+
const cast = castFor(lead.field);
|
|
2374
|
+
const eq = cursorEqFragment(sql, lead.field, lead.value, cast, promotedFor);
|
|
2375
|
+
if (lead.value === null || lead.value === void 0) return combineAnd(sql, [eq, restCmp]);
|
|
2376
|
+
const cmp = (op) => sql`${sortExpr(sql, lead.field, cast, promotedFor)} ${sql.unsafe(op)} ${sortParam(sql, lead.field, lead.value, cast, promotedFor)}`;
|
|
2377
|
+
const asc = lead.dir === "asc";
|
|
2378
|
+
return combineAnd(sql, [cmp(asc ? ">=" : "<="), combineOr(sql, [cmp(asc ? ">" : "<"), combineAnd(sql, [eq, restCmp])])]);
|
|
2379
|
+
}
|
|
2380
|
+
/**
|
|
2381
|
+
* The null tail that follows a cursor under NULLS LAST: `lead IS NULL` when
|
|
2382
|
+
* the leading key is nullable (not `id`) and the cursor's leading value is
|
|
2383
|
+
* not null, else null. Every row it matches sorts after the cursor, so the
|
|
2384
|
+
* tail query needs no other cursor condition. `createdAt` and `updatedAt` count
|
|
2385
|
+
* as nullable: their columns are nullable in both baselines.
|
|
2386
|
+
*/
|
|
2387
|
+
function buildCursorNullTail(sql, orderBy, values, castFor, promotedFor) {
|
|
2388
|
+
const lead = orderBy[0];
|
|
2389
|
+
if (lead === void 0 || lead[0] === "id") return null;
|
|
2390
|
+
if (values[0] === null || values[0] === void 0) return null;
|
|
2391
|
+
return sql`${sortExpr(sql, lead[0], castFor(lead[0]), promotedFor)} IS NULL`;
|
|
2392
|
+
}
|
|
2393
|
+
/** Pairs each orderBy key with its cursor value: `id` from `cursorId`, the rest from `values` in order. */
|
|
2394
|
+
function resolveCursorKeys(orderBy, values, cursorId) {
|
|
2395
|
+
const resolved = [];
|
|
2396
|
+
let nonIdIdx = 0;
|
|
2397
|
+
for (const [field, dir] of orderBy) if (field === "id") resolved.push({
|
|
2398
|
+
field,
|
|
2399
|
+
dir,
|
|
2400
|
+
value: cursorId
|
|
2401
|
+
});
|
|
2402
|
+
else {
|
|
2403
|
+
resolved.push({
|
|
2404
|
+
field,
|
|
2405
|
+
dir,
|
|
2406
|
+
value: values[nonIdIdx]
|
|
2407
|
+
});
|
|
2408
|
+
nonIdIdx++;
|
|
2409
|
+
}
|
|
2410
|
+
return resolved;
|
|
2411
|
+
}
|
|
2412
|
+
/** The OR over i of (eq₁ AND ... AND eqᵢ₋₁ AND cmpᵢ). */
|
|
2413
|
+
function cursorDisjunction(sql, resolved, castFor, promotedFor) {
|
|
2414
|
+
const orParts = [];
|
|
2415
|
+
for (let i = 0; i < resolved.length; i++) {
|
|
2416
|
+
const parts = [];
|
|
2417
|
+
for (let j = 0; j < i; j++) {
|
|
2418
|
+
const prior = resolved[j];
|
|
2419
|
+
parts.push(cursorEqFragment(sql, prior.field, prior.value, castFor(prior.field), promotedFor));
|
|
2420
|
+
}
|
|
2421
|
+
const cur = resolved[i];
|
|
2422
|
+
parts.push(cursorCmpFragment(sql, cur.field, cur.dir, cur.value, castFor(cur.field), promotedFor));
|
|
2423
|
+
orParts.push(combineAnd(sql, parts));
|
|
2424
|
+
}
|
|
2425
|
+
return combineOr(sql, orParts);
|
|
2426
|
+
}
|
|
2427
|
+
/**
|
|
2428
|
+
* Equality against the sort key, for the `eq₁ AND ... AND cmpᵢ` prefix of the
|
|
2429
|
+
* keyset disjunction. A NULL sort key compares with `IS NULL`, never `= NULL`.
|
|
2430
|
+
*/
|
|
2431
|
+
function cursorEqFragment(sql, field, value, bodyCast, promotedFor) {
|
|
2432
|
+
const expr = sortExpr(sql, field, bodyCast, promotedFor);
|
|
2433
|
+
if (value === null || value === void 0) return sql`${expr} IS NULL`;
|
|
2434
|
+
return sql`${expr} = ${sortParam(sql, field, value, bodyCast, promotedFor)}`;
|
|
2435
|
+
}
|
|
2436
|
+
/**
|
|
2437
|
+
* "Strictly after the cursor" for one sort key, under `NULLS LAST`. On a
|
|
2438
|
+
* nullable key a bare `expr > $1` would exclude null rows, since `NULL > x` is NULL.
|
|
2439
|
+
* Used for the keys after a nullable leading key, which `buildCursorCompare`
|
|
2440
|
+
* bounds instead. The two cases:
|
|
2441
|
+
*
|
|
2442
|
+
* cursor value NOT NULL — the rows after it are the ones further along in
|
|
2443
|
+
* direction, PLUS the entire null tail, hence the `OR ... IS NULL`.
|
|
2444
|
+
* cursor value IS NULL — nothing sorts after a null, so no row qualifies on
|
|
2445
|
+
* THIS key and the disjunction must fall through to the next one. `false`
|
|
2446
|
+
* is the correct answer, not `IS NULL`: the outer construction pairs this
|
|
2447
|
+
* with `eq` on the preceding keys, and returning `IS NULL` here would
|
|
2448
|
+
* re-emit every null row that has already been served.
|
|
2449
|
+
*/
|
|
2450
|
+
function cursorCmpFragment(sql, field, dir, value, bodyCast, promotedFor) {
|
|
2451
|
+
const expr = sortExpr(sql, field, bodyCast, promotedFor);
|
|
2452
|
+
if (value === null || value === void 0) return sql`false`;
|
|
2453
|
+
const param = sortParam(sql, field, value, bodyCast, promotedFor);
|
|
2454
|
+
if (field === "id") return dir === "asc" ? sql`${expr} > ${param}` : sql`${expr} < ${param}`;
|
|
2455
|
+
return dir === "asc" ? sql`(${expr} > ${param} OR ${expr} IS NULL)` : sql`(${expr} < ${param} OR ${expr} IS NULL)`;
|
|
2456
|
+
}
|
|
2457
|
+
/**
|
|
2458
|
+
* The parameter side of a body-field cursor compare, with the cast matching
|
|
2459
|
+
* the orderBy expression: `castParam(sql, 5, 'numeric')` → `$N::numeric`.
|
|
2460
|
+
* Returns the bare parameter when `cast === null`.
|
|
2461
|
+
*/
|
|
2462
|
+
function castParam(sql, value, cast) {
|
|
2463
|
+
if (cast === "timestamptz") return tsParam(sql, value);
|
|
2464
|
+
const param = scalarParam(value);
|
|
2465
|
+
if (cast === null) return sql`${param}`;
|
|
2466
|
+
return sql`${param}${sql.unsafe(`::${cast}`)}`;
|
|
2467
|
+
}
|
|
2468
|
+
|
|
2469
|
+
//#endregion
|
|
2470
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/predicates-json-path.ts
|
|
2471
|
+
/**
|
|
2472
|
+
* Compiles `{ field: { path: [...], <op> } }` on a json field.
|
|
2473
|
+
*
|
|
2474
|
+
* Must return exactly the rows `@mnmal-ai/substrate-filter`'s json-path.ts
|
|
2475
|
+
* evaluator does. Three compilation choices carry that:
|
|
2476
|
+
*
|
|
2477
|
+
* - `eq` / `in` / `has` are containment, `(body -> field) @> doc`. Type-strict
|
|
2478
|
+
* by construction (1995 does not contain "1995"). Written against the
|
|
2479
|
+
* FIELD, not the body, so it matches the scoped GIN that
|
|
2480
|
+
* `field.json({ indexed: true })` derives.
|
|
2481
|
+
* - Everything typed is `COALESCE(CASE WHEN jsonb_typeof(x) = … THEN … END,
|
|
2482
|
+
* FALSE)`. CASE because SQL does not promise AND short-circuits, so a
|
|
2483
|
+
* guarded `::numeric` could still meet 'unknown' and fail the whole query;
|
|
2484
|
+
* COALESCE because a bare CASE yields NULL on a mismatch and `NOT NULL` is
|
|
2485
|
+
* NULL, which would silently drop rows from a caller's `NOT`.
|
|
2486
|
+
* - The walk is the `->` chain, never `#>`: `#>` treats "0" as an array
|
|
2487
|
+
* index, and the evaluator walks objects only. `->` with a text key on an
|
|
2488
|
+
* array is NULL, which agrees.
|
|
2489
|
+
*/
|
|
2490
|
+
/** `{ p0: { … pN: leaf } }` — the document `(body -> field) @>` tests for. */
|
|
2491
|
+
function nestPath(path, leaf) {
|
|
2492
|
+
let doc = leaf;
|
|
2493
|
+
for (let i = path.length - 1; i >= 0; i--) doc = { [path[i]]: doc };
|
|
2494
|
+
return doc;
|
|
2495
|
+
}
|
|
2496
|
+
function buildJsonPathPredicate(sql, field, predicate) {
|
|
2497
|
+
assertJsonPathPredicate(field, predicate);
|
|
2498
|
+
const path = predicate.path;
|
|
2499
|
+
const x = bodyJson(sql, [field, ...path]);
|
|
2500
|
+
const text = sql`(${x} #>> '{}')`;
|
|
2501
|
+
const fieldExpr = bodyJson(sql, [field]);
|
|
2502
|
+
const containsDoc = (leaf) => sql`${fieldExpr} @> ${sql.json(asJsonValue(nestPath(path, leaf)))}`;
|
|
2503
|
+
const contains = (leaf) => sql`(${fieldExpr} IS NOT NULL AND ${containsDoc(leaf)})`;
|
|
2504
|
+
const isNull = sql`(${x} IS NULL OR jsonb_typeof(${x}) = 'null')`;
|
|
2505
|
+
const eq = (v) => v === null ? isNull : contains(v);
|
|
2506
|
+
const inList = (vs) => {
|
|
2507
|
+
const members = vs.filter((v) => v !== null);
|
|
2508
|
+
const parts = [];
|
|
2509
|
+
if (members.length > 0) parts.push(sql`(${fieldExpr} IS NOT NULL AND ${combineOr(sql, members.map(containsDoc))})`);
|
|
2510
|
+
if (members.length < vs.length) parts.push(isNull);
|
|
2511
|
+
return parts.length === 0 ? sql`FALSE` : combineOr(sql, parts);
|
|
2512
|
+
};
|
|
2513
|
+
const guarded = (jsonType, cond) => sql`COALESCE(CASE WHEN jsonb_typeof(${x}) = ${jsonType} THEN ${cond} END, FALSE)`;
|
|
2514
|
+
const cmp = (op, operand) => {
|
|
2515
|
+
if (typeof operand === "number") {
|
|
2516
|
+
const n = sql`${text}::numeric`;
|
|
2517
|
+
switch (op) {
|
|
2518
|
+
case ">": return guarded("number", sql`${n} > ${operand}`);
|
|
2519
|
+
case ">=": return guarded("number", sql`${n} >= ${operand}`);
|
|
2520
|
+
case "<": return guarded("number", sql`${n} < ${operand}`);
|
|
2521
|
+
case "<=": return guarded("number", sql`${n} <= ${operand}`);
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
const s = sql`${text} COLLATE "C"`;
|
|
2525
|
+
switch (op) {
|
|
2526
|
+
case ">": return guarded("string", sql`${s} > ${operand}`);
|
|
2527
|
+
case ">=": return guarded("string", sql`${s} >= ${operand}`);
|
|
2528
|
+
case "<": return guarded("string", sql`${s} < ${operand}`);
|
|
2529
|
+
case "<=": return guarded("string", sql`${s} <= ${operand}`);
|
|
2530
|
+
}
|
|
2531
|
+
};
|
|
2532
|
+
const fragments = [];
|
|
2533
|
+
for (const [op, operand] of Object.entries(predicate)) switch (op) {
|
|
2534
|
+
case "path": break;
|
|
2535
|
+
case "eq":
|
|
2536
|
+
fragments.push(eq(operand));
|
|
2537
|
+
break;
|
|
2538
|
+
case "neq":
|
|
2539
|
+
fragments.push(sql`NOT (${eq(operand)})`);
|
|
2540
|
+
break;
|
|
2541
|
+
case "in":
|
|
2542
|
+
fragments.push(inList(operand));
|
|
2543
|
+
break;
|
|
2544
|
+
case "nin":
|
|
2545
|
+
fragments.push(sql`NOT (${inList(operand)})`);
|
|
2546
|
+
break;
|
|
2547
|
+
case "gt":
|
|
2548
|
+
fragments.push(cmp(">", operand));
|
|
2549
|
+
break;
|
|
2550
|
+
case "gte":
|
|
2551
|
+
fragments.push(cmp(">=", operand));
|
|
2552
|
+
break;
|
|
2553
|
+
case "lt":
|
|
2554
|
+
fragments.push(cmp("<", operand));
|
|
2555
|
+
break;
|
|
2556
|
+
case "lte":
|
|
2557
|
+
fragments.push(cmp("<=", operand));
|
|
2558
|
+
break;
|
|
2559
|
+
case "between": {
|
|
2560
|
+
const [lo, hi] = operand;
|
|
2561
|
+
fragments.push(typeof lo === "number" ? guarded("number", sql`${text}::numeric BETWEEN ${lo} AND ${hi}`) : guarded("string", sql`${text} COLLATE "C" BETWEEN ${lo} AND ${hi}`));
|
|
2562
|
+
break;
|
|
2563
|
+
}
|
|
2564
|
+
case "contains":
|
|
2565
|
+
fragments.push(guarded("string", sql`${text} ILIKE ${`%${escapeLike(operand)}%`}`));
|
|
2566
|
+
break;
|
|
2567
|
+
case "startsWith":
|
|
2568
|
+
fragments.push(guarded("string", sql`${text} ILIKE ${`${escapeLike(operand)}%`}`));
|
|
2569
|
+
break;
|
|
2570
|
+
case "endsWith":
|
|
2571
|
+
fragments.push(guarded("string", sql`${text} ILIKE ${`%${escapeLike(operand)}`}`));
|
|
2572
|
+
break;
|
|
2573
|
+
default: fragments.push(contains([operand]));
|
|
2574
|
+
}
|
|
2575
|
+
return combineAnd(sql, fragments);
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
//#endregion
|
|
2579
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/predicates-metadata.ts
|
|
2580
|
+
/**
|
|
2581
|
+
* WHERE predicates against the native row-metadata columns on `rows`,
|
|
2582
|
+
* rather than `body ->> field` extraction.
|
|
2583
|
+
*/
|
|
2584
|
+
/**
|
|
2585
|
+
* Builds WHERE predicates against a metadata column (e.g. `r.created_at`).
|
|
2586
|
+
* Returns null when no operator produced a predicate; throws
|
|
2587
|
+
* `UnsupportedFilterError` on an unknown operator.
|
|
2588
|
+
*
|
|
2589
|
+
* Timestamp columns take equality and range operators; the `*ByKind` /
|
|
2590
|
+
* `*ById` text columns take equality, range and pattern operators. The column
|
|
2591
|
+
* comes from the constant `METADATA_COLUMNS`, so `sql.unsafe` is safe here.
|
|
2592
|
+
*/
|
|
2593
|
+
function buildMetadataColumnPredicate(sql, field, predicate) {
|
|
2594
|
+
const colName = METADATA_COLUMNS$1[field];
|
|
2595
|
+
const col = sql.unsafe(colName);
|
|
2596
|
+
if (field === "seq") {
|
|
2597
|
+
const n = (v) => sql`${String(v)}::bigint`;
|
|
2598
|
+
return orderedColumnPredicate(sql, field, col, predicate, {
|
|
2599
|
+
one: n,
|
|
2600
|
+
bound: n,
|
|
2601
|
+
many: (vs) => sql`${vs.map(String)}::bigint[]`,
|
|
2602
|
+
strictBetween: true
|
|
2603
|
+
});
|
|
2604
|
+
}
|
|
2605
|
+
if (field === "createdAt" || field === "updatedAt") return orderedColumnPredicate(sql, field, col, predicate, {
|
|
2606
|
+
one: (v) => typeof v === "string" || v instanceof Date ? tsParam(sql, v) : null,
|
|
2607
|
+
bound: (v) => tsParam(sql, v),
|
|
2608
|
+
many: (vs) => tsArrayParam(sql, vs),
|
|
2609
|
+
strictBetween: false
|
|
2610
|
+
});
|
|
2611
|
+
const fragments = [];
|
|
2612
|
+
for (const [op, operand] of Object.entries(predicate)) {
|
|
2613
|
+
let frag = null;
|
|
2614
|
+
switch (op) {
|
|
2615
|
+
case "eq":
|
|
2616
|
+
if (operand === null) frag = sql`${col} IS NULL`;
|
|
2617
|
+
else if (typeof operand === "string") frag = sql`${col} = ${operand}`;
|
|
2618
|
+
break;
|
|
2619
|
+
case "neq":
|
|
2620
|
+
if (operand === null) frag = sql`${col} IS NOT NULL`;
|
|
2621
|
+
else if (typeof operand === "string") frag = sql`${col} IS DISTINCT FROM ${operand}`;
|
|
2622
|
+
break;
|
|
2623
|
+
case "in":
|
|
2624
|
+
case "nin":
|
|
2625
|
+
if (Array.isArray(operand)) {
|
|
2626
|
+
const member = memberOf(sql, col, sql`${operand.filter((v) => typeof v === "string")}::text[]`, operand);
|
|
2627
|
+
frag = op === "in" ? member : sql`NOT ${member}`;
|
|
2628
|
+
}
|
|
2629
|
+
break;
|
|
2630
|
+
case "contains":
|
|
2631
|
+
if (typeof operand === "string") frag = sql`${col} ILIKE ${`%${escapeLike(operand)}%`}`;
|
|
2632
|
+
break;
|
|
2633
|
+
case "startsWith":
|
|
2634
|
+
if (typeof operand === "string") frag = sql`${col} ILIKE ${`${escapeLike(operand)}%`}`;
|
|
2635
|
+
break;
|
|
2636
|
+
case "endsWith":
|
|
2637
|
+
if (typeof operand === "string") frag = sql`${col} ILIKE ${`%${escapeLike(operand)}`}`;
|
|
2638
|
+
break;
|
|
2639
|
+
case "gt":
|
|
2640
|
+
frag = sql`${col} COLLATE "C" > ${metaStr(field, op, operand)}`;
|
|
2641
|
+
break;
|
|
2642
|
+
case "gte":
|
|
2643
|
+
frag = sql`${col} COLLATE "C" >= ${metaStr(field, op, operand)}`;
|
|
2644
|
+
break;
|
|
2645
|
+
case "lt":
|
|
2646
|
+
frag = sql`${col} COLLATE "C" < ${metaStr(field, op, operand)}`;
|
|
2647
|
+
break;
|
|
2648
|
+
case "lte":
|
|
2649
|
+
frag = sql`${col} COLLATE "C" <= ${metaStr(field, op, operand)}`;
|
|
2650
|
+
break;
|
|
2651
|
+
case "between":
|
|
2652
|
+
if (!Array.isArray(operand) || operand.length !== 2) throw new UnsupportedFilterError(field, op, "requires a two-element [low, high] array");
|
|
2653
|
+
frag = sql`${col} COLLATE "C" BETWEEN ${metaStr(field, op, operand[0])} AND ${metaStr(field, op, operand[1])}`;
|
|
2654
|
+
break;
|
|
2655
|
+
default: throw new UnsupportedFilterError(field, op, "is not a recognised filter operator");
|
|
2656
|
+
}
|
|
2657
|
+
if (frag !== null) fragments.push(frag);
|
|
2658
|
+
}
|
|
2659
|
+
if (fragments.length === 0) return null;
|
|
2660
|
+
return combineAnd(sql, fragments);
|
|
2661
|
+
}
|
|
2662
|
+
/** Equality, range and membership predicates on one ordered column (`seq`, a timestamp). */
|
|
2663
|
+
function orderedColumnPredicate(sql, field, col, predicate, bind) {
|
|
2664
|
+
const fragments = [];
|
|
2665
|
+
for (const [op, operand] of Object.entries(predicate)) {
|
|
2666
|
+
let frag = null;
|
|
2667
|
+
const v = bind.one(operand);
|
|
2668
|
+
switch (op) {
|
|
2669
|
+
case "eq":
|
|
2670
|
+
if (operand === null) frag = sql`${col} IS NULL`;
|
|
2671
|
+
else if (v !== null) frag = sql`${col} = ${v}`;
|
|
2672
|
+
break;
|
|
2673
|
+
case "neq":
|
|
2674
|
+
if (operand === null) frag = sql`${col} IS NOT NULL`;
|
|
2675
|
+
else if (v !== null) frag = sql`${col} IS DISTINCT FROM ${v}`;
|
|
2676
|
+
break;
|
|
2677
|
+
case "gt":
|
|
2678
|
+
if (v !== null) frag = sql`${col} > ${v}`;
|
|
2679
|
+
break;
|
|
2680
|
+
case "gte":
|
|
2681
|
+
if (v !== null) frag = sql`${col} >= ${v}`;
|
|
2682
|
+
break;
|
|
2683
|
+
case "lt":
|
|
2684
|
+
if (v !== null) frag = sql`${col} < ${v}`;
|
|
2685
|
+
break;
|
|
2686
|
+
case "lte":
|
|
2687
|
+
if (v !== null) frag = sql`${col} <= ${v}`;
|
|
2688
|
+
break;
|
|
2689
|
+
case "between":
|
|
2690
|
+
if (Array.isArray(operand) && operand.length === 2) frag = sql`${col} BETWEEN ${bind.bound(operand[0])} AND ${bind.bound(operand[1])}`;
|
|
2691
|
+
else if (bind.strictBetween) throw new UnsupportedFilterError(field, op, "requires a two-element [low, high] array");
|
|
2692
|
+
break;
|
|
2693
|
+
case "in":
|
|
2694
|
+
case "nin":
|
|
2695
|
+
if (Array.isArray(operand)) {
|
|
2696
|
+
const member = memberOf(sql, col, bind.many(operand.filter((e) => e !== null)), operand);
|
|
2697
|
+
frag = op === "in" ? member : sql`NOT ${member}`;
|
|
2698
|
+
}
|
|
2699
|
+
break;
|
|
2700
|
+
default: throw new UnsupportedFilterError(field, op, "is not a recognised filter operator");
|
|
2701
|
+
}
|
|
2702
|
+
if (frag !== null) fragments.push(frag);
|
|
2703
|
+
}
|
|
2704
|
+
if (fragments.length === 0) return null;
|
|
2705
|
+
return combineAnd(sql, fragments);
|
|
2706
|
+
}
|
|
2707
|
+
/**
|
|
2708
|
+
* Membership of `col` in `operand`, never NULL: an empty list matches nothing,
|
|
2709
|
+
* and a null element matches a null column, as the memory evaluator does.
|
|
2710
|
+
*/
|
|
2711
|
+
function memberOf(sql, col, values, operand) {
|
|
2712
|
+
const parts = [sql`COALESCE(${col} = ANY(${values}), FALSE)`];
|
|
2713
|
+
if (operand.includes(null)) parts.push(sql`${col} IS NULL`);
|
|
2714
|
+
return combineOr(sql, parts);
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2717
|
+
//#endregion
|
|
2718
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/predicates-promoted.ts
|
|
2719
|
+
/**
|
|
2720
|
+
* WHERE predicates against body-promoted columns. These read the column
|
|
2721
|
+
* ONLY, with no body fallback — see `PgDatasetOptions.promotedColumns`.
|
|
2722
|
+
*/
|
|
2723
|
+
/**
|
|
2724
|
+
* Builds the WHERE predicate for a promoted column. Returns null for an empty
|
|
2725
|
+
* predicate; throws `UnsupportedFilterError` on an unknown operator or a
|
|
2726
|
+
* wrong-typed operand.
|
|
2727
|
+
*
|
|
2728
|
+
* Per value V: `col = V::T` (text under `COLLATE "C"`), or `col IS NULL` when V is null. `in` / `nin`
|
|
2729
|
+
* OR/AND the per-value predicates.
|
|
2730
|
+
*/
|
|
2731
|
+
function buildPromotedBodyColumnPredicate(sql, field, meta, predicate) {
|
|
2732
|
+
const col = sql.unsafe(promotedColumnSql(meta));
|
|
2733
|
+
const isBool = meta.kind === "boolean";
|
|
2734
|
+
const fragments = [];
|
|
2735
|
+
/** Operand-shape guard. A wrong-typed operand is a caller error, not a non-match. */
|
|
2736
|
+
const str = (op, v) => {
|
|
2737
|
+
if (typeof v !== "string") throw new UnsupportedFilterError(field, op, `requires a string value on a ${meta.kind} column`);
|
|
2738
|
+
return v;
|
|
2739
|
+
};
|
|
2740
|
+
const singleEq = (value) => {
|
|
2741
|
+
if (value === null) return sql`${col} IS NULL`;
|
|
2742
|
+
if (isBool) {
|
|
2743
|
+
if (typeof value !== "boolean") throw new UnsupportedFilterError(field, "eq", "requires a boolean value");
|
|
2744
|
+
return sql`${col} = ${sql.unsafe(value ? "TRUE" : "FALSE")}`;
|
|
2745
|
+
}
|
|
2746
|
+
return sql`${col} COLLATE "C" = ${str("eq", value)}::text`;
|
|
2747
|
+
};
|
|
2748
|
+
/**
|
|
2749
|
+
* `neq` ADMITS NULLs, matching the body path and the memory substrate, so
|
|
2750
|
+
* `eq` and `neq` partition the rows. `neq: null` is the is-not-null filter.
|
|
2751
|
+
*/
|
|
2752
|
+
const singleNeq = (value) => {
|
|
2753
|
+
if (value === null) return sql`${col} IS NOT NULL`;
|
|
2754
|
+
if (isBool) {
|
|
2755
|
+
if (typeof value !== "boolean") throw new UnsupportedFilterError(field, "neq", "requires a boolean value");
|
|
2756
|
+
return sql`${col} IS DISTINCT FROM ${sql.unsafe(value ? "TRUE" : "FALSE")}`;
|
|
2757
|
+
}
|
|
2758
|
+
return sql`${col} COLLATE "C" IS DISTINCT FROM ${str("neq", value)}::text`;
|
|
2759
|
+
};
|
|
2760
|
+
/**
|
|
2761
|
+
* Range / pattern operators are text-column only; throws on a boolean column.
|
|
2762
|
+
*/
|
|
2763
|
+
const textOnly = (op) => {
|
|
2764
|
+
if (isBool) throw new UnsupportedFilterError(field, op, `is not applicable to a boolean column`);
|
|
2765
|
+
};
|
|
2766
|
+
for (const [op, operand] of Object.entries(predicate)) {
|
|
2767
|
+
let frag;
|
|
2768
|
+
switch (op) {
|
|
2769
|
+
case "eq":
|
|
2770
|
+
frag = singleEq(operand);
|
|
2771
|
+
break;
|
|
2772
|
+
case "neq":
|
|
2773
|
+
frag = singleNeq(operand);
|
|
2774
|
+
break;
|
|
2775
|
+
case "in":
|
|
2776
|
+
if (!Array.isArray(operand)) throw new UnsupportedFilterError(field, op, "requires an array value");
|
|
2777
|
+
if (operand.length === 0) {
|
|
2778
|
+
frag = sql`FALSE`;
|
|
2779
|
+
break;
|
|
2780
|
+
}
|
|
2781
|
+
frag = combineOr(sql, operand.map(singleEq));
|
|
2782
|
+
break;
|
|
2783
|
+
case "nin":
|
|
2784
|
+
if (!Array.isArray(operand)) throw new UnsupportedFilterError(field, op, "requires an array value");
|
|
2785
|
+
if (operand.length === 0) {
|
|
2786
|
+
frag = sql`TRUE`;
|
|
2787
|
+
break;
|
|
2788
|
+
}
|
|
2789
|
+
frag = combineAnd(sql, operand.map(singleNeq));
|
|
2790
|
+
break;
|
|
2791
|
+
case "gt":
|
|
2792
|
+
textOnly(op);
|
|
2793
|
+
frag = sql`${col} COLLATE "C" > ${str(op, operand)}::text`;
|
|
2794
|
+
break;
|
|
2795
|
+
case "gte":
|
|
2796
|
+
textOnly(op);
|
|
2797
|
+
frag = sql`${col} COLLATE "C" >= ${str(op, operand)}::text`;
|
|
2798
|
+
break;
|
|
2799
|
+
case "lt":
|
|
2800
|
+
textOnly(op);
|
|
2801
|
+
frag = sql`${col} COLLATE "C" < ${str(op, operand)}::text`;
|
|
2802
|
+
break;
|
|
2803
|
+
case "lte":
|
|
2804
|
+
textOnly(op);
|
|
2805
|
+
frag = sql`${col} COLLATE "C" <= ${str(op, operand)}::text`;
|
|
2806
|
+
break;
|
|
2807
|
+
case "between":
|
|
2808
|
+
textOnly(op);
|
|
2809
|
+
if (!Array.isArray(operand) || operand.length !== 2) throw new UnsupportedFilterError(field, op, "requires a two-element [low, high] array");
|
|
2810
|
+
frag = sql`${col} COLLATE "C" BETWEEN ${str(op, operand[0])}::text AND ${str(op, operand[1])}::text`;
|
|
2811
|
+
break;
|
|
2812
|
+
case "contains":
|
|
2813
|
+
textOnly(op);
|
|
2814
|
+
frag = sql`${col} ILIKE ${`%${escapeLike(str(op, operand))}%`}`;
|
|
2815
|
+
break;
|
|
2816
|
+
case "startsWith":
|
|
2817
|
+
textOnly(op);
|
|
2818
|
+
frag = sql`${col} ILIKE ${`${escapeLike(str(op, operand))}%`}`;
|
|
2819
|
+
break;
|
|
2820
|
+
case "endsWith":
|
|
2821
|
+
textOnly(op);
|
|
2822
|
+
frag = sql`${col} ILIKE ${`%${escapeLike(str(op, operand))}`}`;
|
|
2823
|
+
break;
|
|
2824
|
+
default: throw new UnsupportedFilterError(field, op, "is not a recognised filter operator");
|
|
2825
|
+
}
|
|
2826
|
+
fragments.push(frag);
|
|
2827
|
+
}
|
|
2828
|
+
if (fragments.length === 0) return null;
|
|
2829
|
+
return combineAnd(sql, fragments);
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
//#endregion
|
|
2833
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/where.ts
|
|
2834
|
+
/**
|
|
2835
|
+
* WHERE-clause → SQL translation: logical combinators, per-field predicate
|
|
2836
|
+
* dispatch (id / shortId / metadata / promoted-column / body-path), the
|
|
2837
|
+
* operator-fragment compiler, and the `conditionalUpdate` match fragment.
|
|
2838
|
+
*/
|
|
2839
|
+
function buildWhere(sql, where, castFor, promotedFor) {
|
|
2840
|
+
if (where === null || typeof where !== "object" || Array.isArray(where)) return null;
|
|
2841
|
+
const keys = Object.keys(where);
|
|
2842
|
+
if (keys.length === 0) return null;
|
|
2843
|
+
const fragments = [];
|
|
2844
|
+
for (const k of keys) {
|
|
2845
|
+
const child = where[k];
|
|
2846
|
+
if (k === "AND" || k === "OR") {
|
|
2847
|
+
if (!Array.isArray(child)) continue;
|
|
2848
|
+
const subs = [];
|
|
2849
|
+
for (const sub of child) {
|
|
2850
|
+
const frag = buildWhere(sql, sub, castFor, promotedFor);
|
|
2851
|
+
if (frag !== null) subs.push(frag);
|
|
2852
|
+
}
|
|
2853
|
+
if (subs.length > 0) fragments.push(k === "AND" ? combineAnd(sql, subs) : combineOr(sql, subs));
|
|
2854
|
+
} else if (k === "NOT") {
|
|
2855
|
+
const inner = buildWhere(sql, child, castFor, promotedFor);
|
|
2856
|
+
if (inner !== null) fragments.push(sql`NOT (${inner})`);
|
|
2857
|
+
} else {
|
|
2858
|
+
const frag = buildFieldPredicate(sql, k, child, castFor, promotedFor);
|
|
2859
|
+
if (frag !== null) fragments.push(frag);
|
|
2860
|
+
}
|
|
2861
|
+
}
|
|
2862
|
+
if (fragments.length === 0) return null;
|
|
2863
|
+
return combineAnd(sql, fragments);
|
|
2864
|
+
}
|
|
2865
|
+
/** True when `field` is a predicate key anywhere in the tree, under AND/OR/NOT included. */
|
|
2866
|
+
function whereReferencesField(where, field) {
|
|
2867
|
+
if (where === null || typeof where !== "object" || Array.isArray(where)) return false;
|
|
2868
|
+
for (const [k, child] of Object.entries(where)) {
|
|
2869
|
+
if (k === field) return true;
|
|
2870
|
+
if ((k === "AND" || k === "OR") && Array.isArray(child)) {
|
|
2871
|
+
if (child.some((sub) => whereReferencesField(sub, field))) return true;
|
|
2872
|
+
} else if (k === "NOT" && whereReferencesField(child, field)) return true;
|
|
2873
|
+
}
|
|
2874
|
+
return false;
|
|
2875
|
+
}
|
|
2876
|
+
function buildFieldPredicate(sql, field, predicate, castFor, promotedFor) {
|
|
2877
|
+
if (predicate === null || typeof predicate !== "object" || Array.isArray(predicate)) throw new UnsupportedFilterError(field, "<none>", `expected a predicate object such as { eq: ... }, got ${Array.isArray(predicate) ? "an array" : typeof predicate}`);
|
|
2878
|
+
if (field === "id") return buildIdPredicate(sql, predicate);
|
|
2879
|
+
if (field === "shortId") return buildShortIdPredicate(sql, predicate);
|
|
2880
|
+
if (isMetadataField(field)) return buildMetadataColumnPredicate(sql, field, predicate);
|
|
2881
|
+
const promotedField = promotedFor(field);
|
|
2882
|
+
if (promotedField !== void 0) return buildPromotedBodyColumnPredicate(sql, field, promotedField, predicate);
|
|
2883
|
+
if (isJsonPathPredicate(predicate)) return buildJsonPathPredicate(sql, field, predicate);
|
|
2884
|
+
return buildFieldPredicateForPath(sql, [field], predicate, castFor);
|
|
2885
|
+
}
|
|
2886
|
+
function buildFieldPredicateForPath(sql, path, predicate, castFor) {
|
|
2887
|
+
const field = path.join(".");
|
|
2888
|
+
const entries = Object.entries(predicate);
|
|
2889
|
+
if (entries.length > 0 && !entries.some(([k]) => KNOWN_BODY_OPS.has(k)) && entries.every(([, v]) => v !== null && typeof v === "object" && !Array.isArray(v))) {
|
|
2890
|
+
const subs = [];
|
|
2891
|
+
for (const [subField, subPredicate] of entries) {
|
|
2892
|
+
const frag = buildBodyPathPredicate(sql, [...path, subField], subPredicate, castFor);
|
|
2893
|
+
if (frag !== null) subs.push(frag);
|
|
2894
|
+
}
|
|
2895
|
+
if (subs.length === 0) return null;
|
|
2896
|
+
return combineAnd(sql, subs);
|
|
2897
|
+
}
|
|
2898
|
+
const declaredCast = path.length === 1 ? castFor(field) : null;
|
|
2899
|
+
const fragments = [];
|
|
2900
|
+
for (const [op, operand] of entries) {
|
|
2901
|
+
const frag = buildOpFragment(sql, path, op, operand, declaredCast);
|
|
2902
|
+
if (frag !== null) fragments.push(frag);
|
|
2903
|
+
}
|
|
2904
|
+
if (fragments.length === 0) return null;
|
|
2905
|
+
return combineAnd(sql, fragments);
|
|
2906
|
+
}
|
|
2907
|
+
/**
|
|
2908
|
+
* Operators the generic body compiler recognises. Used only to tell an operator
|
|
2909
|
+
* predicate from a nested-struct sub-clause; the authoritative per-operator
|
|
2910
|
+
* handling is `buildOpFragment`'s switch.
|
|
2911
|
+
*/
|
|
2912
|
+
const KNOWN_BODY_OPS = /* @__PURE__ */ new Set([
|
|
2913
|
+
"eq",
|
|
2914
|
+
"neq",
|
|
2915
|
+
"gt",
|
|
2916
|
+
"gte",
|
|
2917
|
+
"lt",
|
|
2918
|
+
"lte",
|
|
2919
|
+
"between",
|
|
2920
|
+
"in",
|
|
2921
|
+
"nin",
|
|
2922
|
+
"contains",
|
|
2923
|
+
"startsWith",
|
|
2924
|
+
"endsWith",
|
|
2925
|
+
"has"
|
|
2926
|
+
]);
|
|
2927
|
+
/** Recurse into a struct sub-clause, carrying the accumulated body path. */
|
|
2928
|
+
function buildBodyPathPredicate(sql, path, predicate, castFor) {
|
|
2929
|
+
if (predicate === null || typeof predicate !== "object" || Array.isArray(predicate)) throw new UnsupportedFilterError(path.join("."), "<none>", `expected a predicate object such as { eq: ... }, got ${Array.isArray(predicate) ? "an array" : typeof predicate}`);
|
|
2930
|
+
return buildFieldPredicateForPath(sql, path, predicate, castFor);
|
|
2931
|
+
}
|
|
2932
|
+
/**
|
|
2933
|
+
* Build WHERE predicates against the native `id` column (the row PK).
|
|
2934
|
+
*
|
|
2935
|
+
* Range operators compare the text ids lexicographically. Numeric ids
|
|
2936
|
+
* stringify cleanly.
|
|
2937
|
+
*/
|
|
2938
|
+
function buildIdPredicate(sql, predicate) {
|
|
2939
|
+
const fragments = [];
|
|
2940
|
+
for (const [op, operand] of Object.entries(predicate)) {
|
|
2941
|
+
let frag = null;
|
|
2942
|
+
switch (op) {
|
|
2943
|
+
case "eq":
|
|
2944
|
+
if (operand === null || operand === void 0) frag = sql`id IS NULL`;
|
|
2945
|
+
else frag = sql`id = ${idToText(operand)}`;
|
|
2946
|
+
break;
|
|
2947
|
+
case "neq":
|
|
2948
|
+
if (operand === null || operand === void 0) frag = sql`id IS NOT NULL`;
|
|
2949
|
+
else frag = sql`id IS DISTINCT FROM ${idToText(operand)}`;
|
|
2950
|
+
break;
|
|
2951
|
+
case "in":
|
|
2952
|
+
if (Array.isArray(operand) && operand.length > 0) frag = sql`id IN ${sql(operand.map((v) => idToText(v)))}`;
|
|
2953
|
+
else if (Array.isArray(operand) && operand.length === 0) frag = sql`FALSE`;
|
|
2954
|
+
break;
|
|
2955
|
+
case "nin":
|
|
2956
|
+
if (Array.isArray(operand) && operand.length > 0) frag = sql`id NOT IN ${sql(operand.map((v) => idToText(v)))}`;
|
|
2957
|
+
break;
|
|
2958
|
+
case "contains":
|
|
2959
|
+
if (typeof operand === "string") frag = sql`id ILIKE ${`%${escapeLike(operand)}%`}`;
|
|
2960
|
+
break;
|
|
2961
|
+
case "startsWith":
|
|
2962
|
+
if (typeof operand === "string") frag = sql`id ILIKE ${`${escapeLike(operand)}%`}`;
|
|
2963
|
+
break;
|
|
2964
|
+
case "endsWith":
|
|
2965
|
+
if (typeof operand === "string") frag = sql`id ILIKE ${`%${escapeLike(operand)}`}`;
|
|
2966
|
+
break;
|
|
2967
|
+
case "gt":
|
|
2968
|
+
frag = sql`id > ${metaStr("id", op, operand)}`;
|
|
2969
|
+
break;
|
|
2970
|
+
case "gte":
|
|
2971
|
+
frag = sql`id >= ${metaStr("id", op, operand)}`;
|
|
2972
|
+
break;
|
|
2973
|
+
case "lt":
|
|
2974
|
+
frag = sql`id < ${metaStr("id", op, operand)}`;
|
|
2975
|
+
break;
|
|
2976
|
+
case "lte":
|
|
2977
|
+
frag = sql`id <= ${metaStr("id", op, operand)}`;
|
|
2978
|
+
break;
|
|
2979
|
+
case "between":
|
|
2980
|
+
if (!Array.isArray(operand) || operand.length !== 2) throw new UnsupportedFilterError("id", op, "requires a two-element [low, high] array");
|
|
2981
|
+
frag = sql`id BETWEEN ${metaStr("id", op, operand[0])} AND ${metaStr("id", op, operand[1])}`;
|
|
2982
|
+
break;
|
|
2983
|
+
default: throw new UnsupportedFilterError("id", op, "is not a recognised filter operator");
|
|
2984
|
+
}
|
|
2985
|
+
if (frag !== null) fragments.push(frag);
|
|
2986
|
+
}
|
|
2987
|
+
if (fragments.length === 0) return null;
|
|
2988
|
+
return combineAnd(sql, fragments);
|
|
2989
|
+
}
|
|
2990
|
+
/** Operand-shape guards for the numeric shortId column: anything else is refused, never dropped. */
|
|
2991
|
+
function num(field, op, v) {
|
|
2992
|
+
if (typeof v !== "number") throw new UnsupportedFilterError(field, op, "requires a number");
|
|
2993
|
+
return v;
|
|
2994
|
+
}
|
|
2995
|
+
function numArray(field, op, v) {
|
|
2996
|
+
if (!Array.isArray(v) || v.some((e) => typeof e !== "number")) throw new UnsupportedFilterError(field, op, "requires an array of numbers");
|
|
2997
|
+
return v;
|
|
2998
|
+
}
|
|
2999
|
+
/** Build WHERE predicates against `s.short_id` (the LEFT-joined sidecar column). */
|
|
3000
|
+
function buildShortIdPredicate(sql, predicate) {
|
|
3001
|
+
const col = sql`s.short_id`;
|
|
3002
|
+
const fragments = [];
|
|
3003
|
+
for (const [op, operand] of Object.entries(predicate)) {
|
|
3004
|
+
let frag = null;
|
|
3005
|
+
switch (op) {
|
|
3006
|
+
case "eq":
|
|
3007
|
+
frag = sql`${col} = ${num("shortId", op, operand)}`;
|
|
3008
|
+
break;
|
|
3009
|
+
case "neq":
|
|
3010
|
+
frag = sql`${col} IS DISTINCT FROM ${num("shortId", op, operand)}`;
|
|
3011
|
+
break;
|
|
3012
|
+
case "gt":
|
|
3013
|
+
frag = sql`${col} > ${num("shortId", op, operand)}`;
|
|
3014
|
+
break;
|
|
3015
|
+
case "gte":
|
|
3016
|
+
frag = sql`${col} >= ${num("shortId", op, operand)}`;
|
|
3017
|
+
break;
|
|
3018
|
+
case "lt":
|
|
3019
|
+
frag = sql`${col} < ${num("shortId", op, operand)}`;
|
|
3020
|
+
break;
|
|
3021
|
+
case "lte":
|
|
3022
|
+
frag = sql`${col} <= ${num("shortId", op, operand)}`;
|
|
3023
|
+
break;
|
|
3024
|
+
case "between": {
|
|
3025
|
+
const range = numArray("shortId", op, operand);
|
|
3026
|
+
if (range.length !== 2) throw new UnsupportedFilterError("shortId", op, "requires a two-element [low, high] array");
|
|
3027
|
+
frag = sql`${col} BETWEEN ${range[0]} AND ${range[1]}`;
|
|
3028
|
+
break;
|
|
3029
|
+
}
|
|
3030
|
+
case "in":
|
|
3031
|
+
frag = numArray("shortId", op, operand).length === 0 ? sql`FALSE` : sql`${col} = ANY(${numArray("shortId", op, operand)})`;
|
|
3032
|
+
break;
|
|
3033
|
+
case "nin":
|
|
3034
|
+
frag = numArray("shortId", op, operand).length === 0 ? sql`TRUE` : sql`${col} <> ALL(${numArray("shortId", op, operand)})`;
|
|
3035
|
+
break;
|
|
3036
|
+
default: throw new UnsupportedFilterError("shortId", op, "is not a recognised filter operator");
|
|
3037
|
+
}
|
|
3038
|
+
if (frag !== null) fragments.push(frag);
|
|
3039
|
+
}
|
|
3040
|
+
if (fragments.length === 0) return null;
|
|
3041
|
+
return combineAnd(sql, fragments);
|
|
3042
|
+
}
|
|
3043
|
+
/**
|
|
3044
|
+
* Build a single SQL comparison for (field, op, operand). Operands come from
|
|
3045
|
+
* callers (attacker-controlled in principle), so every one goes through a
|
|
3046
|
+
* parameterised slot. Field names are inlined as escaped literals, so that
|
|
3047
|
+
* expression indexes match the compiled text.
|
|
3048
|
+
*
|
|
3049
|
+
* Case-sensitivity note (design call): `contains` / `startsWith` / `endsWith`
|
|
3050
|
+
* use ILIKE for case-INSENSITIVE matching. Memory dataset mirrors this.
|
|
3051
|
+
*/
|
|
3052
|
+
function buildOpFragment(sql, path, op, operand, declaredCast) {
|
|
3053
|
+
const field = path.join(".");
|
|
3054
|
+
const textExpr = bodyText(sql, path);
|
|
3055
|
+
const num = bodyNum(sql, path);
|
|
3056
|
+
const ts = bodyTs(sql, path);
|
|
3057
|
+
const isTs = (v) => v instanceof Date || declaredCast === "timestamptz" && typeof v === "string";
|
|
3058
|
+
const anyOf = (values) => {
|
|
3059
|
+
const nums = values.filter((v) => typeof v === "number");
|
|
3060
|
+
const rest = values.filter((v) => typeof v !== "number");
|
|
3061
|
+
const parts = [];
|
|
3062
|
+
if (nums.length > 0) parts.push(notNullAnd(sql, num, sql`${num} = ANY(${nums})`));
|
|
3063
|
+
if (rest.length > 0) parts.push(isTs(rest[0]) ? notNullAnd(sql, ts, sql`${ts} = ANY(${tsArrayParam(sql, rest)})`) : notNullAnd(sql, textExpr, sql`${textExpr} = ANY(${rest.map(scalarParam)})`));
|
|
3064
|
+
return combineOr(sql, parts);
|
|
3065
|
+
};
|
|
3066
|
+
switch (op) {
|
|
3067
|
+
case "eq":
|
|
3068
|
+
if (operand === null) return sql`${textExpr} IS NULL`;
|
|
3069
|
+
if (typeof operand === "boolean") return sql`${textExpr} = ${sql.unsafe(operand ? "'true'" : "'false'")}`;
|
|
3070
|
+
if (typeof operand === "number") return notNullAnd(sql, num, sql`${num} = ${operand}`);
|
|
3071
|
+
if (isTs(operand)) return notNullAnd(sql, ts, sql`${ts} = ${tsParam(sql, operand)}`);
|
|
3072
|
+
return sql`${textExpr} = ${scalarParam(operand)}`;
|
|
3073
|
+
case "neq":
|
|
3074
|
+
if (operand === null) return sql`${textExpr} IS NOT NULL`;
|
|
3075
|
+
if (typeof operand === "boolean") return sql`${textExpr} IS DISTINCT FROM ${sql.unsafe(operand ? "'true'" : "'false'")}`;
|
|
3076
|
+
if (typeof operand === "number") return sql`NOT ${notNullAnd(sql, num, sql`${num} = ${operand}`)}`;
|
|
3077
|
+
if (isTs(operand)) return sql`${ts} IS DISTINCT FROM ${tsParam(sql, operand)}`;
|
|
3078
|
+
return sql`${textExpr} IS DISTINCT FROM ${scalarParam(operand)}`;
|
|
3079
|
+
case "gt": return rangeCmp(sql, path, operand, ">", declaredCast);
|
|
3080
|
+
case "gte": return rangeCmp(sql, path, operand, ">=", declaredCast);
|
|
3081
|
+
case "lt": return rangeCmp(sql, path, operand, "<", declaredCast);
|
|
3082
|
+
case "lte": return rangeCmp(sql, path, operand, "<=", declaredCast);
|
|
3083
|
+
case "between": {
|
|
3084
|
+
if (!Array.isArray(operand) || operand.length !== 2) throw new UnsupportedFilterError(field, op, "requires a two-element [low, high] array");
|
|
3085
|
+
const low = operand[0];
|
|
3086
|
+
const high = operand[1];
|
|
3087
|
+
if (typeof low === "number" && typeof high === "number") return notNullAnd(sql, num, sql`${num} BETWEEN ${low} AND ${high}`);
|
|
3088
|
+
if (isTs(low) || isTs(high)) return notNullAnd(sql, ts, sql`${ts} BETWEEN ${tsParam(sql, low)} AND ${tsParam(sql, high)}`);
|
|
3089
|
+
return sql`${textExpr} BETWEEN ${scalarParam(low)} AND ${scalarParam(high)}`;
|
|
3090
|
+
}
|
|
3091
|
+
case "in": {
|
|
3092
|
+
if (!Array.isArray(operand)) throw new UnsupportedFilterError(field, op, "requires an array value");
|
|
3093
|
+
const values = operand.filter((v) => v !== null && v !== void 0);
|
|
3094
|
+
const wantsNull = values.length < operand.length;
|
|
3095
|
+
const any = values.length === 0 ? sql`FALSE` : anyOf(values);
|
|
3096
|
+
return wantsNull ? sql`(${any} OR ${textExpr} IS NULL)` : any;
|
|
3097
|
+
}
|
|
3098
|
+
case "nin": {
|
|
3099
|
+
if (!Array.isArray(operand)) throw new UnsupportedFilterError(field, op, "requires an array value");
|
|
3100
|
+
if (operand.length === 0) return sql`TRUE`;
|
|
3101
|
+
const values = operand.filter((v) => v !== null && v !== void 0);
|
|
3102
|
+
if (values.length < operand.length) {
|
|
3103
|
+
if (values.length === 0) return sql`${textExpr} IS NOT NULL`;
|
|
3104
|
+
return sql`(${textExpr} IS NOT NULL AND NOT ${anyOf(values)})`;
|
|
3105
|
+
}
|
|
3106
|
+
return sql`(${textExpr} IS NULL OR NOT ${anyOf(values)})`;
|
|
3107
|
+
}
|
|
3108
|
+
case "contains":
|
|
3109
|
+
case "startsWith":
|
|
3110
|
+
case "endsWith": {
|
|
3111
|
+
if (typeof operand !== "string") throw new UnsupportedFilterError(field, op, "requires a string value");
|
|
3112
|
+
const e = escapeLike(operand);
|
|
3113
|
+
const pattern = op === "contains" ? `%${e}%` : op === "startsWith" ? `${e}%` : `%${e}`;
|
|
3114
|
+
return sql`${bodyRawText(sql, path)} ILIKE ${pattern}`;
|
|
3115
|
+
}
|
|
3116
|
+
case "has": {
|
|
3117
|
+
const elem = operand instanceof Date ? operand.toISOString() : operand;
|
|
3118
|
+
return sql`${bodyJson(sql, path)} @> ${sql.json(asJsonValue([elem]))}`;
|
|
3119
|
+
}
|
|
3120
|
+
default: throw new UnsupportedFilterError(field, op, "is not a recognised filter operator");
|
|
3121
|
+
}
|
|
3122
|
+
}
|
|
3123
|
+
/**
|
|
3124
|
+
* An ordered compare, typed by the operand: numbers through `substrate_num`,
|
|
3125
|
+
* Dates (or ISO-string operands on a `date`-declared field) through
|
|
3126
|
+
* `substrate_ts`, plain strings as collated text. A stored value of another
|
|
3127
|
+
* shape reads as NULL and never matches.
|
|
3128
|
+
*/
|
|
3129
|
+
function rangeCmp(sql, path, operand, cmp, declaredCast) {
|
|
3130
|
+
const compare = (expr, param) => {
|
|
3131
|
+
switch (cmp) {
|
|
3132
|
+
case ">": return sql`${expr} > ${param}`;
|
|
3133
|
+
case ">=": return sql`${expr} >= ${param}`;
|
|
3134
|
+
case "<": return sql`${expr} < ${param}`;
|
|
3135
|
+
case "<=": return sql`${expr} <= ${param}`;
|
|
3136
|
+
}
|
|
3137
|
+
};
|
|
3138
|
+
if (typeof operand === "number") {
|
|
3139
|
+
const num = bodyNum(sql, path);
|
|
3140
|
+
return notNullAnd(sql, num, compare(num, operand));
|
|
3141
|
+
}
|
|
3142
|
+
if (operand instanceof Date || declaredCast === "timestamptz" && typeof operand === "string") {
|
|
3143
|
+
const ts = bodyTs(sql, path);
|
|
3144
|
+
return notNullAnd(sql, ts, compare(ts, tsParam(sql, operand)));
|
|
3145
|
+
}
|
|
3146
|
+
if (typeof operand === "string") return compare(bodyText(sql, path), operand);
|
|
3147
|
+
throw new UnsupportedFilterError(path.join("."), cmp, `cannot compare against a ${operand === null ? "null" : typeof operand} value`);
|
|
3148
|
+
}
|
|
3149
|
+
/**
|
|
3150
|
+
* Predicate fragment used by `conditionalUpdate`. Mirrors the equality rules
|
|
3151
|
+
* of `read()`'s `eq` operator so a `conditionalUpdate({ consumedAt: null })`
|
|
3152
|
+
* matches the same rows a subsequent `read({ where: { consumedAt: { eq: null } } })`
|
|
3153
|
+
* would. `null` maps to `IS NULL`; a `Date` compares through `substrate_ts`;
|
|
3154
|
+
* other scalars compare via the textual representation `->>` returns.
|
|
3155
|
+
*/
|
|
3156
|
+
function conditionalMatchFragment(sql, field, expected) {
|
|
3157
|
+
const textExpr = bodyText(sql, [field]);
|
|
3158
|
+
if (expected === null || expected === void 0) return sql`${textExpr} IS NULL`;
|
|
3159
|
+
if (typeof expected === "boolean") return sql`${textExpr} = ${expected ? "true" : "false"}`;
|
|
3160
|
+
if (typeof expected === "number") {
|
|
3161
|
+
const num = bodyNum(sql, [field]);
|
|
3162
|
+
return notNullAnd(sql, num, sql`${num} = ${expected}`);
|
|
3163
|
+
}
|
|
3164
|
+
if (expected instanceof Date) return sql`${bodyTs(sql, [field])} = ${tsParam(sql, expected)}`;
|
|
3165
|
+
return sql`${textExpr} = ${String(expected)}`;
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
//#endregion
|
|
3169
|
+
//#region ../../packages/substrate-dataset-pg/src/tx-scope.ts
|
|
3170
|
+
/**
|
|
3171
|
+
* Substrate identity for pg-backed stores: stores built on the same
|
|
3172
|
+
* `postgres.Sql` pool reference share a `txScopeId`, which lets the engine
|
|
3173
|
+
* run their mutations in one transaction.
|
|
3174
|
+
*
|
|
3175
|
+
* Keyed by `Sql` reference, not connection options: two pools on the same
|
|
3176
|
+
* database cannot share a transaction, so they are distinct substrates.
|
|
3177
|
+
* `TransactionSql` handles inherit their parent's id via constructor options.
|
|
3178
|
+
*/
|
|
3179
|
+
const SUBSTRATE_IDS = /* @__PURE__ */ new WeakMap();
|
|
3180
|
+
function txScopeIdFor(sql) {
|
|
3181
|
+
const existing = SUBSTRATE_IDS.get(sql);
|
|
3182
|
+
if (existing !== void 0) return existing;
|
|
3183
|
+
const id = `pg:${randomUUID()}`;
|
|
3184
|
+
SUBSTRATE_IDS.set(sql, id);
|
|
3185
|
+
return id;
|
|
3186
|
+
}
|
|
3187
|
+
|
|
3188
|
+
//#endregion
|
|
3189
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-dataset.ts
|
|
3190
|
+
/**
|
|
3191
|
+
* PgDataset — PostgreSQL-backed reference implementation of Dataset.
|
|
3192
|
+
* Spec: §4.5, §5.2.
|
|
3193
|
+
*
|
|
3194
|
+
* One instance per namespace; rows live in a single `rows` table keyed
|
|
3195
|
+
* by (namespace, type, id). The caller owns the `postgres` Sql connection —
|
|
3196
|
+
* this class never opens or closes pools.
|
|
3197
|
+
*
|
|
3198
|
+
* Reserved field policy (mirrors MemoryDataset):
|
|
3199
|
+
* - `_deletedAt` is managed exclusively by tombstone() / purge().
|
|
3200
|
+
* - write() strips `_deletedAt` (including explicit null) from inputs before
|
|
3201
|
+
* persisting; the column is the only source of truth on read.
|
|
3202
|
+
* - updateField() with fieldName === '_deletedAt' throws ReservedField.
|
|
3203
|
+
*
|
|
3204
|
+
* Id-type preservation: `Id` is `string | number`. The PK column is `text`
|
|
3205
|
+
* (so we can coerce either shape through one column), but the *original*
|
|
3206
|
+
* runtime type is mirrored inside `body.id` during write() and read back out
|
|
3207
|
+
* of the body on read() — this keeps numeric ids round-tripping as numbers.
|
|
3208
|
+
*
|
|
3209
|
+
* Filter contract (§5.2): where / orderBy / cursor are translated to SQL via
|
|
3210
|
+
* postgres.js tagged templates — every caller-supplied value goes through
|
|
3211
|
+
* parameterised interpolation. Never string-concatenate user input into the
|
|
3212
|
+
* clause; that's the one SQL-injection surface in this file.
|
|
3213
|
+
*/
|
|
3214
|
+
/** The CHECKs the engine's own migrations put on `rows`; any other is plugin fan-out. */
|
|
3215
|
+
const ENGINE_CHECKS = [
|
|
3216
|
+
"rows_created_by_kind_check",
|
|
3217
|
+
"rows_updated_by_kind_check",
|
|
3218
|
+
"rows_id_uuid_format"
|
|
3219
|
+
];
|
|
3220
|
+
/**
|
|
3221
|
+
* `N open index recommendation(s)` while the index advisor has any, else
|
|
3222
|
+
* nothing. Reads `_system/IndexRecommendation` from this schema's rows, so a
|
|
3223
|
+
* `_system` held elsewhere (memory) reports none.
|
|
3224
|
+
*/
|
|
3225
|
+
async function openRecommendations(sql) {
|
|
3226
|
+
const [row] = await sql`
|
|
3227
|
+
SELECT count(*)::int AS n FROM rows
|
|
3228
|
+
WHERE namespace = '_system' AND type = 'IndexRecommendation'
|
|
3229
|
+
AND deleted_at IS NULL AND body->>'state' = 'open'`;
|
|
3230
|
+
const n = row?.n ?? 0;
|
|
3231
|
+
return n === 0 ? [] : [`${n} open index recommendation${n === 1 ? "" : "s"}: query _system/IndexRecommendation, then apply or dismiss each with propose_index`];
|
|
3232
|
+
}
|
|
3233
|
+
/** Per pool, so every dataset on it reports what the others deferred. */
|
|
3234
|
+
const deferredBuilds = /* @__PURE__ */ new WeakMap();
|
|
3235
|
+
const RESERVED_DELETED_AT = "_deletedAt";
|
|
3236
|
+
/** Share of `rows` changed since ANALYZE past which its indexes' row counts are stale (autoanalyze's default). */
|
|
3237
|
+
const INDEX_COUNT_STALE = .1;
|
|
3238
|
+
/**
|
|
3239
|
+
* `dropEngineIndex`'s bound on each lock wait. A request transaction ends in
|
|
3240
|
+
* well under this; one still open after it is a long export or a caller
|
|
3241
|
+
* waiting on the drop itself, and the drop holds the table's index-build lock
|
|
3242
|
+
* and two connections while it waits, so it gives up and the caller retries.
|
|
3243
|
+
*/
|
|
3244
|
+
const DROP_LOCK_TIMEOUT_MS = 1e4;
|
|
3245
|
+
/**
|
|
3246
|
+
* DROP INDEX CONCURRENTLY `name` (of `where`, for messages) on `conn`, each
|
|
3247
|
+
* lock wait bounded by `timeoutMs`; a wait over it throws `index_drop_timeout`.
|
|
3248
|
+
*/
|
|
3249
|
+
async function dropIndexBounded(conn, name, where, timeoutMs) {
|
|
3250
|
+
const timeout = Math.max(1, Math.trunc(timeoutMs));
|
|
3251
|
+
try {
|
|
3252
|
+
await conn.unsafe(`SET lock_timeout = ${timeout}`);
|
|
3253
|
+
await conn.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${ident(name)}`);
|
|
3254
|
+
} catch (err) {
|
|
3255
|
+
if (err.code !== "55P03") throw err;
|
|
3256
|
+
throw new IndexDropRefusedError("index_drop_timeout", `dropping index ${name} on ${where} waited over ${timeout} ms on another transaction; retry once it ends`);
|
|
3257
|
+
} finally {
|
|
3258
|
+
await conn.unsafe("RESET lock_timeout");
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
3261
|
+
/** Rows the legacy `typeStats` sample aims for. */
|
|
3262
|
+
const SAMPLE_ROWS = 1e4;
|
|
3263
|
+
/**
|
|
3264
|
+
* Engine-owned partial indexes on a legacy `rows` above which `storageReport`
|
|
3265
|
+
* warns: the planner checks every partial index's predicate on every query of
|
|
3266
|
+
* `rows`, about 4-5 ms of planning at 50 on PG 18, against ~1 ms to run a page.
|
|
3267
|
+
*/
|
|
3268
|
+
const LEGACY_PARTIAL_INDEX_LIMIT = 50;
|
|
3269
|
+
/** The value of a one-value LIST partition bound as `pg_get_expr` prints it. */
|
|
3270
|
+
const listBound = (b) => b === null ? void 0 : /^FOR VALUES IN \('(.*)'\)$/s.exec(b)?.[1]?.replace(/''/g, "'");
|
|
3271
|
+
var PgDataset = class PgDataset {
|
|
3272
|
+
namespace;
|
|
3273
|
+
capabilities = {
|
|
3274
|
+
operators: fullBuiltinOperators(),
|
|
3275
|
+
orderBy: true,
|
|
3276
|
+
cursor: true
|
|
3277
|
+
};
|
|
3278
|
+
txScopeId;
|
|
3279
|
+
#sql;
|
|
3280
|
+
#inTx;
|
|
3281
|
+
#shortIds;
|
|
3282
|
+
#edgesStore;
|
|
3283
|
+
/** Stored so tx-bound and forNamespace clones inherit it. */
|
|
3284
|
+
#fieldDefLookup;
|
|
3285
|
+
/**
|
|
3286
|
+
* Validated once at construction. The raw map is kept so clones inherit it
|
|
3287
|
+
* rather than reverting to the default.
|
|
3288
|
+
*/
|
|
3289
|
+
#promotedColumns;
|
|
3290
|
+
#promotedFor;
|
|
3291
|
+
#layoutCache;
|
|
3292
|
+
#typeDefLookup;
|
|
3293
|
+
#diskGuard;
|
|
3294
|
+
constructor(options) {
|
|
3295
|
+
this.namespace = options.namespace;
|
|
3296
|
+
this.#sql = options.sql;
|
|
3297
|
+
this.#inTx = "inTx" in options ? options.inTx : false;
|
|
3298
|
+
const explicit = "txScopeId" in options ? options.txScopeId : void 0;
|
|
3299
|
+
this.txScopeId = explicit ?? txScopeIdFor(options.sql);
|
|
3300
|
+
this.#shortIds = new PgShortIdStore(this.#sql);
|
|
3301
|
+
const edgesOpt = options.edgesStore;
|
|
3302
|
+
if (edgesOpt !== void 0 && edgesOpt.txScopeId !== this.txScopeId) throw new Error(`PgDataset: edgesStore txScopeId '${edgesOpt.txScopeId}' does not match dataset txScopeId '${this.txScopeId}' — both must wrap the same Sql pool`);
|
|
3303
|
+
this.#edgesStore = edgesOpt ?? null;
|
|
3304
|
+
this.#fieldDefLookup = options.fieldDefLookup ?? null;
|
|
3305
|
+
this.#promotedColumns = options.promotedColumns ?? {};
|
|
3306
|
+
this.#promotedFor = resolvePromotedColumns(this.#promotedColumns);
|
|
3307
|
+
this.#layoutCache = ("layoutCache" in options ? options.layoutCache : void 0) ?? {};
|
|
3308
|
+
this.#typeDefLookup = options.typeDefLookup ?? null;
|
|
3309
|
+
this.#diskGuard = options.diskGuard;
|
|
3310
|
+
}
|
|
3311
|
+
/**
|
|
3312
|
+
* Reads rows of `type` per the §5.2 filter contract (where / orderBy /
|
|
3313
|
+
* cursor / limit). Every value is bound through a parameterised slot.
|
|
3314
|
+
*
|
|
3315
|
+
* Body values extract as text and are cast for comparison only; the filter
|
|
3316
|
+
* validator has already rejected type mismatches.
|
|
3317
|
+
*/
|
|
3318
|
+
/**
|
|
3319
|
+
* `namespace = '<ns>' [AND type = '<type>']` with both inlined as literals,
|
|
3320
|
+
* so plans prune partitions at plan time and a generic plan can still match
|
|
3321
|
+
* a legacy partial index's `namespace = … AND type = …` predicate.
|
|
3322
|
+
*/
|
|
3323
|
+
#scope(type, alias = "") {
|
|
3324
|
+
const ns = `${alias}namespace = ${sqlStringLiteral(this.namespace)}`;
|
|
3325
|
+
return this.#sql.unsafe(type === void 0 ? ns : `${ns} AND ${alias}type = ${sqlStringLiteral(type)}`);
|
|
3326
|
+
}
|
|
3327
|
+
async read(type, params) {
|
|
3328
|
+
const sql = this.#sql;
|
|
3329
|
+
const includeDeleted = params.includeDeleted === true;
|
|
3330
|
+
const limit = typeof params.limit === "number" && Number.isFinite(params.limit) && params.limit >= 0 ? Math.floor(params.limit) : void 0;
|
|
3331
|
+
if (limit === 0) return {
|
|
3332
|
+
rows: [],
|
|
3333
|
+
hasMore: false
|
|
3334
|
+
};
|
|
3335
|
+
const idList = [];
|
|
3336
|
+
let hasIdFilter = false;
|
|
3337
|
+
if (params.id !== void 0) {
|
|
3338
|
+
idList.push(idToText(params.id));
|
|
3339
|
+
hasIdFilter = true;
|
|
3340
|
+
}
|
|
3341
|
+
if (params.ids !== void 0) {
|
|
3342
|
+
for (const id of params.ids) idList.push(idToText(id));
|
|
3343
|
+
hasIdFilter = true;
|
|
3344
|
+
}
|
|
3345
|
+
const clauses = [];
|
|
3346
|
+
clauses.push(this.#scope(type, "r."));
|
|
3347
|
+
if (hasIdFilter) clauses.push(sql`r.id = ANY(${idList})`);
|
|
3348
|
+
if (!includeDeleted) clauses.push(sql`r.deleted_at IS NULL`);
|
|
3349
|
+
const bodyCastFor = (field) => {
|
|
3350
|
+
if (this.#fieldDefLookup === null) return null;
|
|
3351
|
+
return resolveBodyOrderByCast(this.#fieldDefLookup(type, field));
|
|
3352
|
+
};
|
|
3353
|
+
let extMeta = null;
|
|
3354
|
+
if (params.where !== void 0) {
|
|
3355
|
+
const detailed = await preResolveExtensionOperatorsDetailed(params.where, {
|
|
3356
|
+
namespace: this.namespace,
|
|
3357
|
+
type,
|
|
3358
|
+
k: limit,
|
|
3359
|
+
...fieldRequested(params.fields, "_vector") ? { returnVector: true } : {}
|
|
3360
|
+
});
|
|
3361
|
+
extMeta = detailed.meta;
|
|
3362
|
+
const whereFragment = buildWhere(sql, detailed.where, bodyCastFor, this.#promotedFor);
|
|
3363
|
+
if (whereFragment !== null) clauses.push(whereFragment);
|
|
3364
|
+
}
|
|
3365
|
+
const orderBy = normalizeOrderBy(params.orderBy);
|
|
3366
|
+
const orderByCastFor = bodyCastFor;
|
|
3367
|
+
let whereSql = combineAnd(sql, clauses);
|
|
3368
|
+
let tailSql = null;
|
|
3369
|
+
if (params.cursor !== void 0 && params.cursor !== "") {
|
|
3370
|
+
const cursor = decodeCursor(params.cursor, orderBy);
|
|
3371
|
+
const cursorFragment = buildCursorCompare(sql, orderBy, cursor.orderByValues, cursor.id, orderByCastFor, this.#promotedFor);
|
|
3372
|
+
const tail = buildCursorNullTail(sql, orderBy, cursor.orderByValues, orderByCastFor, this.#promotedFor);
|
|
3373
|
+
if (tail !== null) tailSql = combineAnd(sql, [...clauses, tail]);
|
|
3374
|
+
if (cursorFragment !== null) whereSql = combineAnd(sql, [...clauses, cursorFragment]);
|
|
3375
|
+
}
|
|
3376
|
+
const orderBySql = buildOrderBy(sql, orderBy, orderByCastFor, this.#promotedFor);
|
|
3377
|
+
const fetchLimit = limit !== void 0 ? limit + 1 : void 0;
|
|
3378
|
+
const bodyProjection = buildBodyProjection(sql, params.fields);
|
|
3379
|
+
const { fragment: sortKeyProjection, aliases: sortKeyAliases } = buildSortKeyProjection(sql, orderBy, bodyCastFor, this.#promotedFor);
|
|
3380
|
+
const joinShortIds = whereReferencesField(params.where, "shortId") || orderBy.some(([f]) => f === "shortId");
|
|
3381
|
+
const inlineLimit = (n) => n !== void 0 ? sql.unsafe(`LIMIT ${BigInt(n)}`) : sql``;
|
|
3382
|
+
const page = (where, n) => joinShortIds ? sql`
|
|
3383
|
+
SELECT r.id, ${bodyProjection} AS body, r.deleted_at, s.short_id,
|
|
3384
|
+
r.created_at, r.updated_at,
|
|
3385
|
+
r.created_by_kind, r.created_by_id,
|
|
3386
|
+
r.updated_by_kind, r.updated_by_id, r.seq${sortKeyProjection}
|
|
3387
|
+
FROM rows r
|
|
3388
|
+
LEFT JOIN short_ids s
|
|
3389
|
+
ON s.namespace = r.namespace
|
|
3390
|
+
AND s.type = r.type
|
|
3391
|
+
AND s.row_id = r.id
|
|
3392
|
+
WHERE ${where}
|
|
3393
|
+
${orderBySql}
|
|
3394
|
+
${n !== void 0 ? sql`LIMIT ${n}` : sql``}
|
|
3395
|
+
` : sql`
|
|
3396
|
+
SELECT p.*, s.short_id
|
|
3397
|
+
FROM (
|
|
3398
|
+
SELECT r.id, ${bodyProjection} AS body, r.deleted_at,
|
|
3399
|
+
r.created_at, r.updated_at,
|
|
3400
|
+
r.created_by_kind, r.created_by_id,
|
|
3401
|
+
r.updated_by_kind, r.updated_by_id, r.seq,
|
|
3402
|
+
r.namespace, r.type${sortKeyProjection}
|
|
3403
|
+
FROM rows r
|
|
3404
|
+
WHERE ${where}
|
|
3405
|
+
${orderBySql}
|
|
3406
|
+
${inlineLimit(n)}
|
|
3407
|
+
) p
|
|
3408
|
+
LEFT JOIN LATERAL (
|
|
3409
|
+
SELECT s.short_id FROM short_ids s
|
|
3410
|
+
WHERE s.namespace = p.namespace AND s.type = p.type AND s.row_id = p.id
|
|
3411
|
+
LIMIT 1
|
|
3412
|
+
) s ON true
|
|
3413
|
+
${buildProjectedOrderBy(sql, orderBy, sortKeyAliases, orderByCastFor, this.#promotedFor, "p")}
|
|
3414
|
+
`;
|
|
3415
|
+
let rows = await page(whereSql, fetchLimit);
|
|
3416
|
+
if (tailSql !== null && (fetchLimit === void 0 || rows.length < fetchLimit)) {
|
|
3417
|
+
const unseen = sql`r.id <> ALL(${rows.map((r) => r.id)})`;
|
|
3418
|
+
const tail = await page(combineAnd(sql, [tailSql, unseen]), fetchLimit);
|
|
3419
|
+
rows = rows.concat(fetchLimit === void 0 ? tail : tail.slice(0, fetchLimit - rows.length));
|
|
3420
|
+
}
|
|
3421
|
+
const materialised = rows.map((r) => this.materialize(r));
|
|
3422
|
+
let hasMore = false;
|
|
3423
|
+
let sliced = materialised;
|
|
3424
|
+
if (limit !== void 0 && materialised.length > limit) {
|
|
3425
|
+
hasMore = true;
|
|
3426
|
+
sliced = materialised.slice(0, limit);
|
|
3427
|
+
}
|
|
3428
|
+
const out = {
|
|
3429
|
+
rows: sliced,
|
|
3430
|
+
hasMore
|
|
3431
|
+
};
|
|
3432
|
+
const last = sliced[sliced.length - 1];
|
|
3433
|
+
const lastRaw = rows[sliced.length - 1];
|
|
3434
|
+
if (hasMore && last !== void 0) out.nextCursor = encodeCursor({
|
|
3435
|
+
orderByValues: orderBy.filter(([f]) => f !== "id").map(([field]) => {
|
|
3436
|
+
const alias = sortKeyAliases.get(field);
|
|
3437
|
+
return alias === void 0 ? null : lastRaw?.[alias] ?? null;
|
|
3438
|
+
}),
|
|
3439
|
+
id: last.id,
|
|
3440
|
+
order: orderBy
|
|
3441
|
+
});
|
|
3442
|
+
if (extMeta !== null) out.rows = applyExtensionResultMeta(out.rows, extMeta, {
|
|
3443
|
+
hasExplicitOrderBy: params.orderBy !== void 0 && params.orderBy.length > 0,
|
|
3444
|
+
fields: params.fields
|
|
3445
|
+
});
|
|
3446
|
+
return out;
|
|
3447
|
+
}
|
|
3448
|
+
/**
|
|
3449
|
+
* Upsert `rows`. A row without an `id` gets one from the database's
|
|
3450
|
+
* `uuidv7()`, returned in `ids` in input order. Limits of a v7 id: it is
|
|
3451
|
+
* never a happened-before signal; ids sort in creation order only within one
|
|
3452
|
+
* connection (across connections within a millisecond, and for
|
|
3453
|
+
* client-supplied ids, order is arbitrary); it reveals its creation time;
|
|
3454
|
+
* and deciding who won (last-writer-wins, `conditionalUpdate`) uses the
|
|
3455
|
+
* database's transaction time or a per-row version, never the id.
|
|
3456
|
+
*
|
|
3457
|
+
* A supplied id repeated in one batch: under `insertOnly` the first
|
|
3458
|
+
* occurrence lands and the rest are skipped; otherwise the batch is refused.
|
|
3459
|
+
*
|
|
3460
|
+
* On an `ordering: 'visible'` type each inserted row gets the next `seq`,
|
|
3461
|
+
* consecutive in input order, under a per-type transaction lock held until
|
|
3462
|
+
* commit, so seq order is commit order. A row that already exists keeps its
|
|
3463
|
+
* seq, and one `insertOnly` skips gets none.
|
|
3464
|
+
*/
|
|
3465
|
+
async write(type, rows, options) {
|
|
3466
|
+
if (rows.length === 0) return {
|
|
3467
|
+
written: 0,
|
|
3468
|
+
ids: [],
|
|
3469
|
+
rows: []
|
|
3470
|
+
};
|
|
3471
|
+
if (options?.insertOnly !== true) assertNoRepeatedIds(type, rows);
|
|
3472
|
+
const visible = this.#typeDefLookup?.(type, this.namespace)?.ordering === "visible";
|
|
3473
|
+
if (visible && !this.#inTx) return this.withTransaction((tx) => tx.write(type, rows, options));
|
|
3474
|
+
const sql = this.#sql;
|
|
3475
|
+
if (visible) await sql`SELECT pg_advisory_xact_lock(hashtext('substrate-visible-seq:' || current_schema() || ':' || ${this.namespace} || '/' || ${type}))`;
|
|
3476
|
+
const fallbackNow = (options?.now ?? /* @__PURE__ */ new Date()).toISOString();
|
|
3477
|
+
const fallbackActor = options?.identity ?? null;
|
|
3478
|
+
const namespaces = [];
|
|
3479
|
+
const types = [];
|
|
3480
|
+
const idsCol = [];
|
|
3481
|
+
const bodies = [];
|
|
3482
|
+
const createdAts = [];
|
|
3483
|
+
const updatedAts = [];
|
|
3484
|
+
const createdByKinds = [];
|
|
3485
|
+
const createdByIds = [];
|
|
3486
|
+
const updatedByKinds = [];
|
|
3487
|
+
const updatedByIds = [];
|
|
3488
|
+
for (const row of rows) {
|
|
3489
|
+
const meta = row._metadata ?? {
|
|
3490
|
+
createdAt: fallbackNow,
|
|
3491
|
+
updatedAt: fallbackNow,
|
|
3492
|
+
createdBy: fallbackActor,
|
|
3493
|
+
updatedBy: fallbackActor
|
|
3494
|
+
};
|
|
3495
|
+
const sanitised = {};
|
|
3496
|
+
for (const [k, v] of Object.entries(row)) {
|
|
3497
|
+
if (k === RESERVED_DELETED_AT) continue;
|
|
3498
|
+
if (k === "_metadata") continue;
|
|
3499
|
+
if (k === "id" && v == null) continue;
|
|
3500
|
+
sanitised[k] = v;
|
|
3501
|
+
}
|
|
3502
|
+
namespaces.push(this.namespace);
|
|
3503
|
+
types.push(type);
|
|
3504
|
+
idsCol.push(row.id == null ? null : idToText(row.id));
|
|
3505
|
+
bodies.push(JSON.stringify(sanitised));
|
|
3506
|
+
createdAts.push(meta.createdAt);
|
|
3507
|
+
updatedAts.push(meta.updatedAt);
|
|
3508
|
+
createdByKinds.push(meta.createdBy?.kind ?? null);
|
|
3509
|
+
createdByIds.push(meta.createdBy?.id ?? null);
|
|
3510
|
+
updatedByKinds.push(meta.updatedBy?.kind ?? null);
|
|
3511
|
+
updatedByIds.push(meta.updatedBy?.id ?? null);
|
|
3512
|
+
}
|
|
3513
|
+
const insertColumnParts = [];
|
|
3514
|
+
const insertValueParts = [];
|
|
3515
|
+
const conflictParts = [];
|
|
3516
|
+
for (const [field, meta] of Object.entries(this.#promotedColumns)) {
|
|
3517
|
+
const colRef = sql.unsafe(meta.column);
|
|
3518
|
+
const absentDefault = meta.kind === "boolean" ? sql`false` : sql`NULL`;
|
|
3519
|
+
const cast = meta.kind === "boolean" ? sql`::boolean` : sql``;
|
|
3520
|
+
insertColumnParts.push(sql`, ${colRef}`);
|
|
3521
|
+
insertValueParts.push(sql`, CASE WHEN bod::jsonb ? ${field} THEN (bod::jsonb->>${field})${cast} ELSE ${absentDefault} END`);
|
|
3522
|
+
conflictParts.push(sql`, ${colRef} = CASE WHEN EXCLUDED.body ? ${field} THEN (EXCLUDED.body->>${field})${cast} ELSE ${sql.unsafe(`rows.${meta.column}`)} END`);
|
|
3523
|
+
}
|
|
3524
|
+
const insertColumnList = insertColumnParts.length === 0 ? sql`` : insertColumnParts;
|
|
3525
|
+
const insertColumnValues = insertValueParts.length === 0 ? sql`` : insertValueParts;
|
|
3526
|
+
const conflictColumnUpdates = conflictParts.length === 0 ? sql`` : conflictParts;
|
|
3527
|
+
const onConflict = options?.insertOnly === true ? sql`ON CONFLICT (namespace, type, id) DO NOTHING` : sql`ON CONFLICT (namespace, type, id)
|
|
3528
|
+
DO UPDATE SET
|
|
3529
|
+
body = rows.body || EXCLUDED.body,
|
|
3530
|
+
created_at = COALESCE(rows.created_at, EXCLUDED.created_at),
|
|
3531
|
+
updated_at = COALESCE(EXCLUDED.updated_at, rows.updated_at),
|
|
3532
|
+
created_by_kind = COALESCE(rows.created_by_kind, EXCLUDED.created_by_kind),
|
|
3533
|
+
created_by_id = COALESCE(rows.created_by_id, EXCLUDED.created_by_id),
|
|
3534
|
+
updated_by_kind = COALESCE(EXCLUDED.updated_by_kind, rows.updated_by_kind),
|
|
3535
|
+
updated_by_id = COALESCE(EXCLUDED.updated_by_id, rows.updated_by_id)${conflictColumnUpdates}`;
|
|
3536
|
+
const seqCtes = visible ? sql`,
|
|
3537
|
+
seqd AS (
|
|
3538
|
+
SELECT f.*, CASE WHEN f.fresh THEN
|
|
3539
|
+
COALESCE((SELECT v.last_seq FROM visible_seqs v WHERE ${this.#scope(type, "v.")}), 0)
|
|
3540
|
+
+ count(*) FILTER (WHERE f.fresh) OVER (ORDER BY f.ord)
|
|
3541
|
+
END AS sq
|
|
3542
|
+
FROM (
|
|
3543
|
+
SELECT i.*, i.ord = min(i.ord) OVER (PARTITION BY i.rid) AS first,
|
|
3544
|
+
i.ord = min(i.ord) OVER (PARTITION BY i.rid) AND NOT EXISTS (
|
|
3545
|
+
SELECT 1 FROM rows r WHERE ${this.#scope(type, "r.")} AND r.id = i.rid
|
|
3546
|
+
) AS fresh
|
|
3547
|
+
FROM input i
|
|
3548
|
+
) f
|
|
3549
|
+
),
|
|
3550
|
+
counted AS (
|
|
3551
|
+
INSERT INTO visible_seqs (namespace, type, last_seq)
|
|
3552
|
+
SELECT ns, ty, max(sq) FROM seqd WHERE sq IS NOT NULL GROUP BY ns, ty
|
|
3553
|
+
ON CONFLICT (namespace, type) DO UPDATE SET last_seq = EXCLUDED.last_seq
|
|
3554
|
+
)` : sql``;
|
|
3555
|
+
const insertSource = visible ? sql`seqd WHERE first` : sql`input`;
|
|
3556
|
+
const merged = (await sql`
|
|
3557
|
+
WITH input AS (
|
|
3558
|
+
SELECT ns, ty, COALESCE(rid, uuidv7()::text) AS rid, bod,
|
|
3559
|
+
ca, ua, cbk, cbi, ubk, ubi, ord
|
|
3560
|
+
FROM UNNEST(
|
|
3561
|
+
${namespaces}::text[],
|
|
3562
|
+
${types}::text[],
|
|
3563
|
+
${idsCol}::text[],
|
|
3564
|
+
${bodies}::text[],
|
|
3565
|
+
${createdAts}::text[],
|
|
3566
|
+
${updatedAts}::text[],
|
|
3567
|
+
${createdByKinds}::text[],
|
|
3568
|
+
${createdByIds}::text[],
|
|
3569
|
+
${updatedByKinds}::text[],
|
|
3570
|
+
${updatedByIds}::text[]
|
|
3571
|
+
) WITH ORDINALITY AS t(ns, ty, rid, bod, ca, ua, cbk, cbi, ubk, ubi, ord)
|
|
3572
|
+
)${seqCtes},
|
|
3573
|
+
upserted AS (
|
|
3574
|
+
INSERT INTO rows (namespace, type, id, body, deleted_at,
|
|
3575
|
+
created_at, updated_at, created_by_kind, created_by_id,
|
|
3576
|
+
updated_by_kind, updated_by_id${insertColumnList}${visible ? sql`, seq` : sql``})
|
|
3577
|
+
SELECT
|
|
3578
|
+
ns, ty, rid, bod::jsonb, NULL::timestamptz,
|
|
3579
|
+
ca::timestamptz, ua::timestamptz, cbk, cbi, ubk, ubi${insertColumnValues}${visible ? sql`, sq` : sql``}
|
|
3580
|
+
FROM ${insertSource}
|
|
3581
|
+
${onConflict}
|
|
3582
|
+
RETURNING namespace, type, id, body, deleted_at,
|
|
3583
|
+
created_at, updated_at, created_by_kind, created_by_id,
|
|
3584
|
+
updated_by_kind, updated_by_id, seq
|
|
3585
|
+
)
|
|
3586
|
+
SELECT u.id, u.body, u.deleted_at, s.short_id,
|
|
3587
|
+
u.created_at, u.updated_at,
|
|
3588
|
+
u.created_by_kind, u.created_by_id,
|
|
3589
|
+
u.updated_by_kind, u.updated_by_id, u.seq
|
|
3590
|
+
FROM upserted u
|
|
3591
|
+
JOIN (SELECT rid, min(ord) AS ord FROM input GROUP BY rid) i ON i.rid = u.id
|
|
3592
|
+
LEFT JOIN short_ids s
|
|
3593
|
+
ON s.namespace = u.namespace
|
|
3594
|
+
AND s.type = u.type
|
|
3595
|
+
AND s.row_id = u.id
|
|
3596
|
+
ORDER BY i.ord
|
|
3597
|
+
`).map((r) => this.materialize(r));
|
|
3598
|
+
return {
|
|
3599
|
+
written: merged.length,
|
|
3600
|
+
ids: merged.map((r) => r.id),
|
|
3601
|
+
rows: merged
|
|
3602
|
+
};
|
|
3603
|
+
}
|
|
3604
|
+
async tombstone(type, ids) {
|
|
3605
|
+
if (ids.length === 0) return {
|
|
3606
|
+
deleted: 0,
|
|
3607
|
+
ids: []
|
|
3608
|
+
};
|
|
3609
|
+
const textIds = ids.map(idToText);
|
|
3610
|
+
const stamped = (await this.#sql`
|
|
3611
|
+
UPDATE rows
|
|
3612
|
+
SET deleted_at = now()
|
|
3613
|
+
WHERE ${this.#scope(type)}
|
|
3614
|
+
AND id = ANY(${textIds})
|
|
3615
|
+
AND deleted_at IS NULL
|
|
3616
|
+
RETURNING id, body
|
|
3617
|
+
`).map((r) => this.extractOriginalId(r.id, r.body));
|
|
3618
|
+
return {
|
|
3619
|
+
deleted: stamped.length,
|
|
3620
|
+
ids: stamped
|
|
3621
|
+
};
|
|
3622
|
+
}
|
|
3623
|
+
async purge(type, ids) {
|
|
3624
|
+
if (ids.length === 0) return {
|
|
3625
|
+
deleted: 0,
|
|
3626
|
+
ids: []
|
|
3627
|
+
};
|
|
3628
|
+
const textIds = ids.map(idToText);
|
|
3629
|
+
const removed = (await this.#sql`
|
|
3630
|
+
DELETE FROM rows
|
|
3631
|
+
WHERE ${this.#scope(type)}
|
|
3632
|
+
AND id = ANY(${textIds})
|
|
3633
|
+
RETURNING id, body
|
|
3634
|
+
`).map((r) => this.extractOriginalId(r.id, r.body));
|
|
3635
|
+
return {
|
|
3636
|
+
deleted: removed.length,
|
|
3637
|
+
ids: removed
|
|
3638
|
+
};
|
|
3639
|
+
}
|
|
3640
|
+
/**
|
|
3641
|
+
* Hard-deletes tombstoned rows whose `deleted_at` is older than `olderThan`,
|
|
3642
|
+
* across ALL namespaces (the table is substrate-shared). Called by the sweep
|
|
3643
|
+
* loop. Returns the number of rows deleted.
|
|
3644
|
+
*
|
|
3645
|
+
* Short-id mappings are left in place, as in `purge()`: a row id can
|
|
3646
|
+
* reappear after a tombstone + rewrite.
|
|
3647
|
+
*/
|
|
3648
|
+
async purgeTombstonedRows(opts) {
|
|
3649
|
+
return { deleted: (await this.#sql`
|
|
3650
|
+
DELETE FROM rows
|
|
3651
|
+
WHERE deleted_at IS NOT NULL
|
|
3652
|
+
AND deleted_at < ${opts.olderThan}
|
|
3653
|
+
RETURNING id
|
|
3654
|
+
`).length };
|
|
3655
|
+
}
|
|
3656
|
+
/**
|
|
3657
|
+
* Hard-delete every row for `type` in this namespace, including tombstoned
|
|
3658
|
+
* rows (`deleted_at IS NOT NULL`). Short-id mappings for that type are
|
|
3659
|
+
* removed in the same logical operation. Both deletes run through `this.#sql`
|
|
3660
|
+
* so they participate in any surrounding `withTransaction` boundary.
|
|
3661
|
+
*/
|
|
3662
|
+
async dropType(type) {
|
|
3663
|
+
return {
|
|
3664
|
+
rowsDeleted: (await this.#sql`
|
|
3665
|
+
DELETE FROM rows
|
|
3666
|
+
WHERE ${this.#scope(type)}
|
|
3667
|
+
RETURNING id
|
|
3668
|
+
`).length,
|
|
3669
|
+
shortIdsDeleted: await this.#shortIds.dropType(this.namespace, type)
|
|
3670
|
+
};
|
|
3671
|
+
}
|
|
3672
|
+
/**
|
|
3673
|
+
* Hard-delete every row across all types in this namespace, including
|
|
3674
|
+
* tombstoned rows. Short-id mappings for all types are removed too.
|
|
3675
|
+
*/
|
|
3676
|
+
async dropNamespace() {
|
|
3677
|
+
return {
|
|
3678
|
+
rowsDeleted: (await this.#sql`
|
|
3679
|
+
DELETE FROM rows
|
|
3680
|
+
WHERE ${this.#scope()}
|
|
3681
|
+
RETURNING id
|
|
3682
|
+
`).length,
|
|
3683
|
+
shortIdsDeleted: await this.#shortIds.dropNamespace(this.namespace)
|
|
3684
|
+
};
|
|
3685
|
+
}
|
|
3686
|
+
/**
|
|
3687
|
+
* Compare-and-update (spec §5.12): merges `updates` into the live row only
|
|
3688
|
+
* if every `match` field equals its expected value. Returns `matched` and
|
|
3689
|
+
* the updated row.
|
|
3690
|
+
*
|
|
3691
|
+
* One `UPDATE ... WHERE <pk> AND <match>`, so the row lock makes exactly one
|
|
3692
|
+
* of N concurrent callers match. `_deletedAt` and `id` are dropped from
|
|
3693
|
+
* `updates`.
|
|
3694
|
+
*/
|
|
3695
|
+
async conditionalUpdate(type, id, match, updates, actor, now) {
|
|
3696
|
+
const sql = this.#sql;
|
|
3697
|
+
const sanitisedUpdates = {};
|
|
3698
|
+
for (const [k, v] of Object.entries(updates)) {
|
|
3699
|
+
if (k === RESERVED_DELETED_AT) continue;
|
|
3700
|
+
if (k === "id") continue;
|
|
3701
|
+
sanitisedUpdates[k] = v;
|
|
3702
|
+
}
|
|
3703
|
+
const matchClauses = [];
|
|
3704
|
+
for (const [field, expected] of Object.entries(match)) matchClauses.push(conditionalMatchFragment(sql, field, expected));
|
|
3705
|
+
const matchSql = matchClauses.length === 0 ? sql`TRUE` : combineAnd(sql, matchClauses);
|
|
3706
|
+
const updatesJson = sql.json(asJsonValue(sanitisedUpdates));
|
|
3707
|
+
const deletedGuard = sql`deleted_at IS NULL`;
|
|
3708
|
+
const updatedBy = actor ?? {
|
|
3709
|
+
kind: "system",
|
|
3710
|
+
id: "anonymous"
|
|
3711
|
+
};
|
|
3712
|
+
const columnMirrors = [];
|
|
3713
|
+
for (const field of Object.keys(this.#promotedColumns)) {
|
|
3714
|
+
if (!(field in sanitisedUpdates)) continue;
|
|
3715
|
+
const meta = this.#promotedFor(field);
|
|
3716
|
+
if (meta === void 0) continue;
|
|
3717
|
+
const raw = sanitisedUpdates[field];
|
|
3718
|
+
const colIdent = sql(meta.column);
|
|
3719
|
+
if (raw === null) columnMirrors.push(sql`${colIdent} = NULL`);
|
|
3720
|
+
else if (meta.kind === "boolean") columnMirrors.push(sql`${colIdent} = ${String(raw)}::text::boolean`);
|
|
3721
|
+
else columnMirrors.push(sql`${colIdent} = ${String(raw)}::text`);
|
|
3722
|
+
}
|
|
3723
|
+
const setFragments = [
|
|
3724
|
+
sql`body = body || ${updatesJson}`,
|
|
3725
|
+
now === void 0 ? sql`updated_at = now()` : sql`updated_at = ${now}`,
|
|
3726
|
+
sql`updated_by_kind = ${updatedBy.kind}`,
|
|
3727
|
+
sql`updated_by_id = ${updatedBy.id}`,
|
|
3728
|
+
...columnMirrors
|
|
3729
|
+
];
|
|
3730
|
+
const commaFrag = sql`,`;
|
|
3731
|
+
const interleaved = [setFragments[0]];
|
|
3732
|
+
for (let i = 1; i < setFragments.length; i++) interleaved.push(commaFrag, setFragments[i]);
|
|
3733
|
+
const row = (await sql`
|
|
3734
|
+
WITH updated AS (
|
|
3735
|
+
UPDATE rows
|
|
3736
|
+
SET ${interleaved}
|
|
3737
|
+
WHERE ${this.#scope(type)}
|
|
3738
|
+
AND id = ${idToText(id)}
|
|
3739
|
+
AND ${deletedGuard}
|
|
3740
|
+
AND ${matchSql}
|
|
3741
|
+
RETURNING namespace, type, id, body, deleted_at, created_at, updated_at,
|
|
3742
|
+
created_by_kind, created_by_id, updated_by_kind, updated_by_id, seq
|
|
3743
|
+
)
|
|
3744
|
+
SELECT u.id, u.body, u.deleted_at, s.short_id,
|
|
3745
|
+
u.created_at, u.updated_at,
|
|
3746
|
+
u.created_by_kind, u.created_by_id,
|
|
3747
|
+
u.updated_by_kind, u.updated_by_id, u.seq
|
|
3748
|
+
FROM updated u
|
|
3749
|
+
LEFT JOIN short_ids s
|
|
3750
|
+
ON s.namespace = u.namespace
|
|
3751
|
+
AND s.type = u.type
|
|
3752
|
+
AND s.row_id = u.id
|
|
3753
|
+
`)[0];
|
|
3754
|
+
if (row === void 0) return {
|
|
3755
|
+
matched: false,
|
|
3756
|
+
row: null
|
|
3757
|
+
};
|
|
3758
|
+
return {
|
|
3759
|
+
matched: true,
|
|
3760
|
+
row: this.materialize(row)
|
|
3761
|
+
};
|
|
3762
|
+
}
|
|
3763
|
+
async updateField(ref, fieldName, value) {
|
|
3764
|
+
if (ref.ns !== void 0 && ref.ns !== this.namespace) throw new NamespaceMismatch(this.namespace, ref.ns);
|
|
3765
|
+
if (fieldName === RESERVED_DELETED_AT) throw new ReservedField(fieldName);
|
|
3766
|
+
if (fieldName === "_metadata") {
|
|
3767
|
+
const meta = value;
|
|
3768
|
+
await this.#sql`
|
|
3769
|
+
UPDATE rows
|
|
3770
|
+
SET created_at = ${meta?.createdAt ?? null},
|
|
3771
|
+
updated_at = ${meta?.updatedAt ?? null},
|
|
3772
|
+
created_by_kind = ${meta?.createdBy?.kind ?? null},
|
|
3773
|
+
created_by_id = ${meta?.createdBy?.id ?? null},
|
|
3774
|
+
updated_by_kind = ${meta?.updatedBy?.kind ?? null},
|
|
3775
|
+
updated_by_id = ${meta?.updatedBy?.id ?? null}
|
|
3776
|
+
WHERE ${this.#scope(ref.type)}
|
|
3777
|
+
AND id = ${idToText(ref.id)}
|
|
3778
|
+
`;
|
|
3779
|
+
return;
|
|
3780
|
+
}
|
|
3781
|
+
const setFragments = [value === null || value === void 0 ? this.#sql`body = jsonb_set(body, ${[fieldName]}, 'null'::jsonb, true)` : this.#sql`body = jsonb_set(body, ${[fieldName]}, ${this.#sql.json(asJsonValue(value))}, true)`];
|
|
3782
|
+
const promotedMeta = this.#promotedFor(fieldName);
|
|
3783
|
+
if (promotedMeta !== void 0) {
|
|
3784
|
+
const meta = promotedMeta;
|
|
3785
|
+
const colIdent = this.#sql(meta.column);
|
|
3786
|
+
if (value === null || value === void 0) setFragments.push(this.#sql`${colIdent} = NULL`);
|
|
3787
|
+
else if (meta.kind === "boolean") setFragments.push(this.#sql`${colIdent} = ${String(value)}::text::boolean`);
|
|
3788
|
+
else setFragments.push(this.#sql`${colIdent} = ${String(value)}::text`);
|
|
3789
|
+
}
|
|
3790
|
+
const commaFrag = this.#sql`,`;
|
|
3791
|
+
const interleaved = [setFragments[0]];
|
|
3792
|
+
for (let i = 1; i < setFragments.length; i++) interleaved.push(commaFrag, setFragments[i]);
|
|
3793
|
+
await this.#sql`
|
|
3794
|
+
UPDATE rows
|
|
3795
|
+
SET ${interleaved}
|
|
3796
|
+
WHERE ${this.#scope(ref.type)}
|
|
3797
|
+
AND id = ${idToText(ref.id)}
|
|
3798
|
+
`;
|
|
3799
|
+
}
|
|
3800
|
+
/**
|
|
3801
|
+
* `sql.begin`-backed atomicity. Spec §5.3 / G1.
|
|
3802
|
+
*
|
|
3803
|
+
* Nested calls flatten: postgres.js's `begin` inside a `begin` would create
|
|
3804
|
+
* a savepoint (inner rolls back independently of outer), which is stricter
|
|
3805
|
+
* than the G1 contract promises. To match the memory impl — where an inner
|
|
3806
|
+
* throw tears down outer writes too — we detect the re-entry via `#inTx`
|
|
3807
|
+
* and skip the nested begin entirely. The inner throw then propagates to
|
|
3808
|
+
* the outer `sql.begin`, which rolls back the whole thing.
|
|
3809
|
+
*/
|
|
3810
|
+
async withTransaction(fn) {
|
|
3811
|
+
if (this.#inTx) return fn(this);
|
|
3812
|
+
return this.#sql.begin(async (txSql) => {
|
|
3813
|
+
return fn(new PgDataset({
|
|
3814
|
+
namespace: this.namespace,
|
|
3815
|
+
sql: txSql,
|
|
3816
|
+
inTx: true,
|
|
3817
|
+
txScopeId: this.txScopeId,
|
|
3818
|
+
...this.#fieldDefLookup !== null ? { fieldDefLookup: this.#fieldDefLookup } : {},
|
|
3819
|
+
promotedColumns: this.#promotedColumns,
|
|
3820
|
+
layoutCache: this.#layoutCache,
|
|
3821
|
+
...this.#typeDefLookup !== null ? { typeDefLookup: this.#typeDefLookup } : {}
|
|
3822
|
+
}));
|
|
3823
|
+
});
|
|
3824
|
+
}
|
|
3825
|
+
/**
|
|
3826
|
+
* Runs `fn` in one pg transaction with a tx-bound dataset per requested
|
|
3827
|
+
* namespace (and the edges store, when wired), all on one `TransactionSql`.
|
|
3828
|
+
* A throw rolls back every namespace. Spec §5.3.
|
|
3829
|
+
*
|
|
3830
|
+
* Nested calls flatten onto the current transaction rather than opening a
|
|
3831
|
+
* savepoint.
|
|
3832
|
+
*/
|
|
3833
|
+
async withScopedTransaction(namespaces, fn) {
|
|
3834
|
+
const buildHandles = (handle) => {
|
|
3835
|
+
const datasets = /* @__PURE__ */ new Map();
|
|
3836
|
+
for (const ns of namespaces) datasets.set(ns, new PgDataset({
|
|
3837
|
+
namespace: ns,
|
|
3838
|
+
sql: handle,
|
|
3839
|
+
inTx: true,
|
|
3840
|
+
txScopeId: this.txScopeId,
|
|
3841
|
+
...this.#edgesStore !== null ? { edgesStore: this.#edgesStore } : {},
|
|
3842
|
+
...this.#fieldDefLookup !== null ? { fieldDefLookup: this.#fieldDefLookup } : {},
|
|
3843
|
+
promotedColumns: this.#promotedColumns,
|
|
3844
|
+
layoutCache: this.#layoutCache,
|
|
3845
|
+
...this.#typeDefLookup !== null ? { typeDefLookup: this.#typeDefLookup } : {}
|
|
3846
|
+
}));
|
|
3847
|
+
return {
|
|
3848
|
+
datasets,
|
|
3849
|
+
edges: this.#edgesStore !== null ? this.#edgesStore._enlistInTx(handle) : null
|
|
3850
|
+
};
|
|
3851
|
+
};
|
|
3852
|
+
if (this.#inTx) return fn(buildHandles(this.#sql));
|
|
3853
|
+
return this.#sql.begin(async (txSql) => {
|
|
3854
|
+
return fn(buildHandles(txSql));
|
|
3855
|
+
});
|
|
3856
|
+
}
|
|
3857
|
+
/**
|
|
3858
|
+
* See `Dataset.withAdvisoryLock`. Excludes every session on the
|
|
3859
|
+
* same schema, across processes. On a tx-bound instance the lock is
|
|
3860
|
+
* `pg_advisory_xact_lock` on the transaction's own connection, held until
|
|
3861
|
+
* it commits or rolls back: a session lock would need a second connection
|
|
3862
|
+
* beside the open transaction.
|
|
3863
|
+
*/
|
|
3864
|
+
async withAdvisoryLock(key, fn) {
|
|
3865
|
+
if (!this.#inTx) return withPgAdvisoryLock(this.#sql, key, fn);
|
|
3866
|
+
await this.#sql`SELECT pg_advisory_xact_lock(hashtext(current_schema() || ':' || ${key}))`;
|
|
3867
|
+
return fn();
|
|
3868
|
+
}
|
|
3869
|
+
async allocateShortId(type) {
|
|
3870
|
+
return this.#shortIds.allocateUnbound(this.namespace, type);
|
|
3871
|
+
}
|
|
3872
|
+
async bindShortId(type, shortId, rowId) {
|
|
3873
|
+
await this.#shortIds.bind(this.namespace, type, shortId, rowId);
|
|
3874
|
+
}
|
|
3875
|
+
async resolveShortId(type, shortId) {
|
|
3876
|
+
return this.#shortIds.resolveShortId(this.namespace, type, shortId);
|
|
3877
|
+
}
|
|
3878
|
+
async getShortIdForRow(type, rowId) {
|
|
3879
|
+
return this.#shortIds.getShortIdForRow(this.namespace, type, rowId);
|
|
3880
|
+
}
|
|
3881
|
+
forNamespace(namespace) {
|
|
3882
|
+
if (this.#inTx) throw new Error("PgDataset.forNamespace cannot be called on a tx-bound instance");
|
|
3883
|
+
return new PgDataset({
|
|
3884
|
+
namespace,
|
|
3885
|
+
sql: this.#sql,
|
|
3886
|
+
inTx: false,
|
|
3887
|
+
...this.#edgesStore !== null ? { edgesStore: this.#edgesStore } : {},
|
|
3888
|
+
...this.#fieldDefLookup !== null ? { fieldDefLookup: this.#fieldDefLookup } : {},
|
|
3889
|
+
promotedColumns: this.#promotedColumns,
|
|
3890
|
+
layoutCache: this.#layoutCache,
|
|
3891
|
+
...this.#typeDefLookup !== null ? { typeDefLookup: this.#typeDefLookup } : {},
|
|
3892
|
+
...this.#diskGuard !== void 0 ? { diskGuard: this.#diskGuard } : {}
|
|
3893
|
+
});
|
|
3894
|
+
}
|
|
3895
|
+
/** The storage layout of `rows`, detected once; `'absent'` is not cached. */
|
|
3896
|
+
async #layout() {
|
|
3897
|
+
if (this.#layoutCache.layout !== void 0) return this.#layoutCache.layout;
|
|
3898
|
+
const layout = await detectLayout(this.#sql);
|
|
3899
|
+
if (layout !== "absent") this.#layoutCache.layout = layout;
|
|
3900
|
+
return layout;
|
|
3901
|
+
}
|
|
3902
|
+
/**
|
|
3903
|
+
* See `Dataset.ensureTypeStorage`. On the partitioned layout, creates this
|
|
3904
|
+
* namespace's partition and the type's, then reconciles the type's indexes
|
|
3905
|
+
* on it; on legacy, reconciles only its declared `indexes`. Index builds
|
|
3906
|
+
* run CONCURRENTLY, outside any transaction, so a tx-bound instance refuses.
|
|
3907
|
+
* Logs each index it created, adopted or swapped, and records each one
|
|
3908
|
+
* left pending, building or failed for `describeStorage` and `storageReport`.
|
|
3909
|
+
*/
|
|
3910
|
+
async ensureTypeStorage(type, opts = {}) {
|
|
3911
|
+
const layout = await this.#layout();
|
|
3912
|
+
if (layout === "absent") return;
|
|
3913
|
+
if (this.#inTx) throw new Error("PgDataset.ensureTypeStorage cannot be called on a tx-bound instance");
|
|
3914
|
+
const sql = this.#sql;
|
|
3915
|
+
if (layout === "partitioned") await ensureTypeStorage(sql, this.namespace, type);
|
|
3916
|
+
const builds = this.#deferredBuilds();
|
|
3917
|
+
const qualified = `${this.namespace}/${type}`;
|
|
3918
|
+
const record = (s) => {
|
|
3919
|
+
if (s.state === "pending" || s.state === "failed") builds.set(s.name, {
|
|
3920
|
+
state: s.state,
|
|
3921
|
+
type: qualified,
|
|
3922
|
+
...s.error !== void 0 ? { error: s.error } : {},
|
|
3923
|
+
...s.reason !== void 0 ? { reason: s.reason } : {}
|
|
3924
|
+
});
|
|
3925
|
+
else builds.delete(s.name);
|
|
3926
|
+
};
|
|
3927
|
+
let states;
|
|
3928
|
+
try {
|
|
3929
|
+
states = await reconcileTypeIndexes(sql, this.#indexArgs(type, this.namespace, layout), {
|
|
3930
|
+
only: opts.indexes ?? "all",
|
|
3931
|
+
onBuild: (name) => builds.set(name, {
|
|
3932
|
+
state: "building",
|
|
3933
|
+
type: qualified
|
|
3934
|
+
}),
|
|
3935
|
+
onState: record,
|
|
3936
|
+
...opts.signal !== void 0 ? { signal: opts.signal } : {},
|
|
3937
|
+
...this.#diskGuard !== void 0 ? { diskGuard: this.#diskGuard } : {}
|
|
3938
|
+
});
|
|
3939
|
+
} catch (err) {
|
|
3940
|
+
const code = err.code;
|
|
3941
|
+
const reason = `${typeof code === "string" ? code : "error"}: ${(err instanceof Error ? err.message : String(err)).split("\n")[0]}`;
|
|
3942
|
+
for (const b of builds.values()) if (b.type === qualified && b.state !== "failed") {
|
|
3943
|
+
b.state = "failed";
|
|
3944
|
+
b.error = reason;
|
|
3945
|
+
delete b.reason;
|
|
3946
|
+
}
|
|
3947
|
+
throw err;
|
|
3948
|
+
}
|
|
3949
|
+
for (const s of states) {
|
|
3950
|
+
record(s);
|
|
3951
|
+
if (s.outcome === "unchanged") continue;
|
|
3952
|
+
process.stdout.write(`[substrate] index ${s.name} on ${this.namespace}/${type}: ${s.outcome} (${s.state})\n`);
|
|
3953
|
+
}
|
|
3954
|
+
}
|
|
3955
|
+
#deferredBuilds() {
|
|
3956
|
+
const builds = deferredBuilds.get(this.#sql) ?? /* @__PURE__ */ new Map();
|
|
3957
|
+
deferredBuilds.set(this.#sql, builds);
|
|
3958
|
+
return builds;
|
|
3959
|
+
}
|
|
3960
|
+
/** Each index's catalog state, overridden by what this process knows of its build. */
|
|
3961
|
+
#withBuilds(states) {
|
|
3962
|
+
const builds = this.#deferredBuilds();
|
|
3963
|
+
return states.map((s) => {
|
|
3964
|
+
const build = builds.get(s.name);
|
|
3965
|
+
if (build === void 0) return s;
|
|
3966
|
+
return {
|
|
3967
|
+
...s,
|
|
3968
|
+
state: build.state,
|
|
3969
|
+
...build.error !== void 0 ? { error: build.error } : {},
|
|
3970
|
+
...build.reason !== void 0 ? { reason: build.reason } : {}
|
|
3971
|
+
};
|
|
3972
|
+
});
|
|
3973
|
+
}
|
|
3974
|
+
#indexArgs(type, namespace, layout) {
|
|
3975
|
+
const def = this.#typeDefLookup?.(type, namespace);
|
|
3976
|
+
return {
|
|
3977
|
+
namespace,
|
|
3978
|
+
type,
|
|
3979
|
+
layout,
|
|
3980
|
+
fields: def?.fields ?? {},
|
|
3981
|
+
indexes: def?.indexes ?? [],
|
|
3982
|
+
orderedBy: naturalOrderOf(def ?? {}),
|
|
3983
|
+
...def?.ordering !== void 0 ? { ordering: def.ordering } : {},
|
|
3984
|
+
promotedColumns: this.#promotedColumns
|
|
3985
|
+
};
|
|
3986
|
+
}
|
|
3987
|
+
/**
|
|
3988
|
+
* The type's non-unique indexes the next build pass would build or swap,
|
|
3989
|
+
* each with the disk guard's estimate. Empty when the store or the type's
|
|
3990
|
+
* storage does not exist yet. Changes nothing.
|
|
3991
|
+
*/
|
|
3992
|
+
async pendingIndexBuilds(type) {
|
|
3993
|
+
const layout = await this.#layout();
|
|
3994
|
+
if (layout === "absent") return [];
|
|
3995
|
+
const args = this.#indexArgs(type, this.namespace, layout);
|
|
3996
|
+
if (layout === "partitioned" && await typePartition(this.#sql, this.namespace, type) === void 0) return [];
|
|
3997
|
+
return pendingIndexBuilds(this.#sql, args);
|
|
3998
|
+
}
|
|
3999
|
+
/** See `Dataset.describeStorage`. Reads the catalog only, in a fixed number of queries. */
|
|
4000
|
+
async describeStorage(type) {
|
|
4001
|
+
const layout = await this.#layout() === "partitioned" ? "partitioned" : "legacy";
|
|
4002
|
+
const sql = this.#sql;
|
|
4003
|
+
const args = this.#indexArgs(type, this.namespace, layout);
|
|
4004
|
+
if (layout === "legacy") return {
|
|
4005
|
+
layout,
|
|
4006
|
+
indexes: this.#withBuilds(describeTypeIndexes(args, await heldIndexes(sql, ["rows"])))
|
|
4007
|
+
};
|
|
4008
|
+
const partition = await typePartition(sql, this.namespace, type);
|
|
4009
|
+
if (partition === void 0) return {
|
|
4010
|
+
layout,
|
|
4011
|
+
indexes: []
|
|
4012
|
+
};
|
|
4013
|
+
return {
|
|
4014
|
+
layout,
|
|
4015
|
+
partition,
|
|
4016
|
+
indexes: this.#withBuilds(describeTypeIndexes(args, await heldIndexes(sql, [partition])))
|
|
4017
|
+
};
|
|
4018
|
+
}
|
|
4019
|
+
/**
|
|
4020
|
+
* See `Dataset.storageReport`. Spans every namespace of the database. Reads
|
|
4021
|
+
* the partition tree and every index on it in one query each, so its cost
|
|
4022
|
+
* does not grow with the number of partitions.
|
|
4023
|
+
*/
|
|
4024
|
+
async storageReport(opts) {
|
|
4025
|
+
const layout = await this.#layout();
|
|
4026
|
+
const building = [...this.#deferredBuilds()].sort(([a], [b]) => a.localeCompare(b)).map(([name, b]) => `index ${name} on ${b.type} is ${b.state}${b.error !== void 0 ? ` (${b.error})` : ""}${b.reason !== void 0 ? ` (${b.reason})` : ""}: filters it would serve scan until it is valid`);
|
|
4027
|
+
if (this.#diskGuard !== void 0 && this.#diskGuard.path === void 0) building.push("free-disk checks before index builds are off: set SUBSTRATE_INDEX_DISK_PATH to the database volume mount, or to off when the database has no local volume");
|
|
4028
|
+
if (layout === "absent") return {
|
|
4029
|
+
layout: "legacy",
|
|
4030
|
+
warnings: building
|
|
4031
|
+
};
|
|
4032
|
+
const sql = this.#sql;
|
|
4033
|
+
const info = await openRecommendations(sql);
|
|
4034
|
+
if (layout !== "partitioned") {
|
|
4035
|
+
const warnings = [...building];
|
|
4036
|
+
const held = await heldIndexes(sql, ["rows"]);
|
|
4037
|
+
const partial = held.filter((i) => isEngineOwned(i) && i.predicate !== null).length;
|
|
4038
|
+
if (partial > 50) warnings.push(`${partial} engine-owned partial indexes on the shared rows table, over ${50}: every query on rows pays planning time for each; the partitioned layout gives each type its own table and indexes`);
|
|
4039
|
+
const types = /* @__PURE__ */ new Map();
|
|
4040
|
+
for (const i of held) {
|
|
4041
|
+
const scope = legacyScope(i.predicate);
|
|
4042
|
+
if (scope !== void 0) types.set(JSON.stringify([scope.namespace, scope.type]), scope);
|
|
4043
|
+
}
|
|
4044
|
+
for (const t of types.values()) this.#undeclared(warnings, info, t.namespace, t.type, "legacy", "rows", held);
|
|
4045
|
+
return {
|
|
4046
|
+
layout: "legacy",
|
|
4047
|
+
warnings,
|
|
4048
|
+
info
|
|
4049
|
+
};
|
|
4050
|
+
}
|
|
4051
|
+
const warnings = [...await storageWarnings(sql), ...building];
|
|
4052
|
+
const tree = await sql`
|
|
4053
|
+
SELECT c.relname AS name, p.relname AS parent, pg_get_expr(c.relpartbound, c.oid) AS bound
|
|
4054
|
+
FROM pg_partition_tree('rows') t
|
|
4055
|
+
JOIN pg_class c ON c.oid = t.relid
|
|
4056
|
+
LEFT JOIN pg_class p ON p.oid = t.parentrelid`;
|
|
4057
|
+
const indexes = await sql`
|
|
4058
|
+
SELECT t.relname AS "table", c.relname AS name, i.indisprimary AS "primary"
|
|
4059
|
+
FROM pg_partition_tree('rows') pt
|
|
4060
|
+
JOIN pg_index i ON i.indrelid = pt.relid
|
|
4061
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
4062
|
+
JOIN pg_class t ON t.oid = pt.relid`;
|
|
4063
|
+
const checks = await sql`
|
|
4064
|
+
SELECT conname AS name FROM pg_constraint
|
|
4065
|
+
WHERE contype = 'c' AND conrelid = 'rows'::regclass AND NOT (conname = ANY(${ENGINE_CHECKS}))
|
|
4066
|
+
ORDER BY conname`;
|
|
4067
|
+
for (const c of checks) warnings.push(`check ${c.name} is on the rows parent: every partition carries it, and every write evaluates it`);
|
|
4068
|
+
for (const i of indexes) if (i.table === "rows" && !i.primary) warnings.push(`index ${i.name} is on the rows parent: every partition carries a copy of it`);
|
|
4069
|
+
const nsOf = new Map(tree.map((t) => [t.name, listBound(t.bound)]));
|
|
4070
|
+
const typeParts = tree.flatMap((part) => {
|
|
4071
|
+
const type = listBound(part.bound);
|
|
4072
|
+
const ns = part.parent === null || part.parent === "rows" ? void 0 : nsOf.get(part.parent);
|
|
4073
|
+
return type === void 0 || ns === void 0 ? [] : [{
|
|
4074
|
+
ns,
|
|
4075
|
+
type,
|
|
4076
|
+
table: part.name
|
|
4077
|
+
}];
|
|
4078
|
+
});
|
|
4079
|
+
const held = await heldIndexes(sql, typeParts.map((t) => t.table));
|
|
4080
|
+
for (const t of typeParts) this.#undeclared(warnings, info, t.ns, t.type, "partitioned", t.table, held);
|
|
4081
|
+
const n = tree.length - 1;
|
|
4082
|
+
if (n > opts.partitionBudget) warnings.push(`${n} partitions exceed the budget of ${opts.partitionBudget}: move a namespace to its own instance behind a bridge`);
|
|
4083
|
+
return {
|
|
4084
|
+
layout: "partitioned",
|
|
4085
|
+
warnings,
|
|
4086
|
+
info
|
|
4087
|
+
};
|
|
4088
|
+
}
|
|
4089
|
+
/**
|
|
4090
|
+
* Notes, as info, each engine-owned index of a known type that its
|
|
4091
|
+
* declaration no longer asks for: the index advisor recommends its removal
|
|
4092
|
+
* on evidence, so it is not a fault. A type that cannot be checked warns.
|
|
4093
|
+
*/
|
|
4094
|
+
#undeclared(warnings, info, ns, type, layout, table, held) {
|
|
4095
|
+
if (this.#typeDefLookup === null) return;
|
|
4096
|
+
const own = held.filter((i) => belongsToType(i, ns, type, layout, table));
|
|
4097
|
+
if (this.#typeDefLookup(type, ns) === void 0) {
|
|
4098
|
+
for (const i of own) {
|
|
4099
|
+
if (!isEngineOwned(i)) continue;
|
|
4100
|
+
info.push(`orphaned index ${i.name} on ${ns}/${type}: no registered type owns it; ${i.unique ? "it is unique, so drop it by hand" : "a signed propose_index remove drops it"}`);
|
|
4101
|
+
}
|
|
4102
|
+
return;
|
|
4103
|
+
}
|
|
4104
|
+
try {
|
|
4105
|
+
for (const name of undeclaredIndexes(this.#indexArgs(type, ns, layout), own)) info.push(`undeclared index ${name} on ${ns}/${type}: kept until a signed remove drops it (the index advisor recommends one when it goes unused)`);
|
|
4106
|
+
} catch (err) {
|
|
4107
|
+
warnings.push(`indexes of ${ns}/${type} could not be checked: ${err instanceof Error ? err.message : String(err)}`);
|
|
4108
|
+
}
|
|
4109
|
+
}
|
|
4110
|
+
/** See `Dataset.indexPlan`. */
|
|
4111
|
+
async indexPlan(type, namespace) {
|
|
4112
|
+
const def = this.#typeDefLookup?.(type, namespace);
|
|
4113
|
+
const layout = await this.#layout();
|
|
4114
|
+
if (def === void 0 || layout === "absent") return void 0;
|
|
4115
|
+
return {
|
|
4116
|
+
indexes: desiredIndexKeys(this.#indexArgs(type, namespace, layout)),
|
|
4117
|
+
...def.expectedSize !== void 0 ? { expectedSize: def.expectedSize } : {}
|
|
4118
|
+
};
|
|
4119
|
+
}
|
|
4120
|
+
/**
|
|
4121
|
+
* See `Dataset.dropEngineIndex`. Checks and drops under the table's index
|
|
4122
|
+
* build lock, so a reconcile cannot rebuild or swap the index in between.
|
|
4123
|
+
* Ownership is the reconciler's marker and generated name on an index of
|
|
4124
|
+
* this type (`typeIndexesOf`); the wanted set is the reconciler's own
|
|
4125
|
+
* (`desiredIndexKeys`), so only a leftover or a retracted declaration goes.
|
|
4126
|
+
* Refuses with `IndexDropRefusedError`.
|
|
4127
|
+
*
|
|
4128
|
+
* The drop waits for every transaction holding a lock on the table, and pg
|
|
4129
|
+
* cannot see a cycle through a caller that waits on this call, so each wait
|
|
4130
|
+
* is bounded by `lockTimeoutMs` (default `DROP_LOCK_TIMEOUT_MS`).
|
|
4131
|
+
*/
|
|
4132
|
+
async dropEngineIndex(type, name, opts = {}) {
|
|
4133
|
+
const layout = await this.#layout();
|
|
4134
|
+
if (this.#inTx) throw new Error("PgDataset.dropEngineIndex cannot be called on a tx-bound instance");
|
|
4135
|
+
const where = `${this.namespace}/${type}`;
|
|
4136
|
+
if (layout === "absent") throw new IndexDropRefusedError("unknown_index", `no index ${name} on ${where}`);
|
|
4137
|
+
const sql = this.#sql;
|
|
4138
|
+
const find = async () => (await typeIndexesOf(sql, this.namespace, type, layout)).find((i) => i.name === name);
|
|
4139
|
+
const seen = await find();
|
|
4140
|
+
if (seen === void 0) throw new IndexDropRefusedError("unknown_index", `no index ${name} on ${where}`);
|
|
4141
|
+
const conn = await sql.reserve();
|
|
4142
|
+
try {
|
|
4143
|
+
await withTableIndexLock(sql, conn, seen.table, void 0, async () => {
|
|
4144
|
+
const index = await find();
|
|
4145
|
+
if (index === void 0) throw new IndexDropRefusedError("unknown_index", `no index ${name} on ${where}`);
|
|
4146
|
+
if (!index.engineOwned) throw new IndexDropRefusedError("not_engine_owned", `index ${name} on ${where} was not created by substrate; drop it by hand if it must go`);
|
|
4147
|
+
if (index.unique) throw new IndexDropRefusedError("unique_index", `index ${name} on ${where} is unique; retract its declaration instead`);
|
|
4148
|
+
const lookup = this.#typeDefLookup;
|
|
4149
|
+
const orphan = lookup !== null && lookup(type, this.namespace) === void 0;
|
|
4150
|
+
if (lookup === null || !orphan && (name.endsWith("_base") || desiredIndexKeys(this.#indexArgs(type, this.namespace, layout)).some((d) => d.name === name))) throw new IndexDropRefusedError("still_desired", `index ${name} on ${where} is one the engine builds for this type; dropping it would only rebuild it`);
|
|
4151
|
+
await opts.beforeDrop?.();
|
|
4152
|
+
await dropIndexBounded(conn, name, where, opts.lockTimeoutMs ?? DROP_LOCK_TIMEOUT_MS);
|
|
4153
|
+
});
|
|
4154
|
+
} finally {
|
|
4155
|
+
conn.release();
|
|
4156
|
+
}
|
|
4157
|
+
}
|
|
4158
|
+
/**
|
|
4159
|
+
* See `Dataset.dropNamespaceIndexes`. On legacy, the engine-owned partial
|
|
4160
|
+
* indexes on `rows` scoped to this namespace; on partitioned, the
|
|
4161
|
+
* engine-owned indexes on its type partitions, which a namespace drop
|
|
4162
|
+
* empties but keeps. Unique ones only when `opts.unique`. Each table's under
|
|
4163
|
+
* its index-build lock, CONCURRENTLY; the wait for that lock and each lock
|
|
4164
|
+
* wait of a drop are bounded by `lockTimeoutMs` (default
|
|
4165
|
+
* `DROP_LOCK_TIMEOUT_MS`), past which it throws `index_drop_timeout`.
|
|
4166
|
+
*/
|
|
4167
|
+
async dropNamespaceIndexes(opts = {}) {
|
|
4168
|
+
const layout = await this.#layout();
|
|
4169
|
+
if (layout === "absent") return [];
|
|
4170
|
+
if (this.#inTx) throw new Error("PgDataset.dropNamespaceIndexes cannot be called on a tx-bound instance");
|
|
4171
|
+
const sql = this.#sql;
|
|
4172
|
+
const ns = this.namespace;
|
|
4173
|
+
const tables = layout === "legacy" ? ["rows"] : await typePartitions(sql, ns);
|
|
4174
|
+
const timeout = opts.lockTimeoutMs ?? DROP_LOCK_TIMEOUT_MS;
|
|
4175
|
+
const dropped = [];
|
|
4176
|
+
const conn = await sql.reserve();
|
|
4177
|
+
try {
|
|
4178
|
+
for (const table of tables) try {
|
|
4179
|
+
await withTableIndexLock(sql, conn, table, AbortSignal.timeout(timeout), async () => {
|
|
4180
|
+
for (const i of await heldIndexes(conn, [table])) {
|
|
4181
|
+
if (!isEngineOwned(i) || i.unique && opts.unique !== true) continue;
|
|
4182
|
+
if (layout === "legacy" && legacyScope(i.predicate)?.namespace !== ns) continue;
|
|
4183
|
+
await dropIndexBounded(conn, i.name, ns, timeout);
|
|
4184
|
+
dropped.push(i.name);
|
|
4185
|
+
}
|
|
4186
|
+
});
|
|
4187
|
+
} catch (err) {
|
|
4188
|
+
if (!(err instanceof IndexLockAborted)) throw err;
|
|
4189
|
+
throw new IndexDropRefusedError("index_drop_timeout", `the index-build lock of ${table} stayed held for over ${timeout} ms; indexes of ${ns} left in place`);
|
|
4190
|
+
}
|
|
4191
|
+
} finally {
|
|
4192
|
+
conn.release();
|
|
4193
|
+
}
|
|
4194
|
+
return dropped;
|
|
4195
|
+
}
|
|
4196
|
+
/** See `Dataset.indexDeclarationOf`. */
|
|
4197
|
+
async indexDeclarationOf(type, name) {
|
|
4198
|
+
const layout = await this.#layout();
|
|
4199
|
+
if (layout === "absent") return void 0;
|
|
4200
|
+
const args = this.#indexArgs(type, this.namespace, layout);
|
|
4201
|
+
const others = new Set(desiredIndexKeys({
|
|
4202
|
+
...args,
|
|
4203
|
+
indexes: []
|
|
4204
|
+
}).map((d) => d.name));
|
|
4205
|
+
return args.indexes.find((decl) => desiredIndexKeys({
|
|
4206
|
+
...args,
|
|
4207
|
+
indexes: [decl]
|
|
4208
|
+
}).some((d) => d.name === name && !others.has(d.name)));
|
|
4209
|
+
}
|
|
4210
|
+
/**
|
|
4211
|
+
* See `Dataset.typeStats`. Spans every namespace of the database, in a fixed
|
|
4212
|
+
* number of queries. Partitioned: each type partition's own statistics;
|
|
4213
|
+
* `liveRows` is `reltuples` (or `n_live_tup` before the first ANALYZE) and
|
|
4214
|
+
* counts tombstoned rows too. Legacy: see the shortcut below.
|
|
4215
|
+
*/
|
|
4216
|
+
async typeStats() {
|
|
4217
|
+
const layout = await this.#layout();
|
|
4218
|
+
if (layout === "absent") return [];
|
|
4219
|
+
const sql = this.#sql;
|
|
4220
|
+
const toStats = (t, held) => ({
|
|
4221
|
+
namespace: t.namespace,
|
|
4222
|
+
type: t.type,
|
|
4223
|
+
liveRows: Math.round(t.liveRows),
|
|
4224
|
+
insertsTotal: Math.round(t.insertsTotal),
|
|
4225
|
+
updatesTotal: Math.round(t.updatesTotal),
|
|
4226
|
+
deletesTotal: Math.round(t.deletesTotal),
|
|
4227
|
+
indexes: held.filter((i) => belongsToType(i, t.namespace, t.type, layout, t.table)).map((i) => ({
|
|
4228
|
+
name: i.name,
|
|
4229
|
+
scans: i.scans,
|
|
4230
|
+
lastScanAt: i.lastScanAt,
|
|
4231
|
+
sizeBytes: i.sizeBytes,
|
|
4232
|
+
unique: i.unique,
|
|
4233
|
+
engineOwned: isEngineOwned(i)
|
|
4234
|
+
}))
|
|
4235
|
+
});
|
|
4236
|
+
if (layout === "partitioned") {
|
|
4237
|
+
const types = (await sql`
|
|
4238
|
+
SELECT c.relname AS "table",
|
|
4239
|
+
pg_get_expr(p.relpartbound, p.oid) AS "nsBound",
|
|
4240
|
+
pg_get_expr(c.relpartbound, c.oid) AS bound,
|
|
4241
|
+
(CASE WHEN c.reltuples >= 0 THEN c.reltuples ELSE COALESCE(s.n_live_tup, 0) END)::float8
|
|
4242
|
+
AS "liveRows",
|
|
4243
|
+
COALESCE(s.n_tup_ins, 0)::float8 AS "insertsTotal",
|
|
4244
|
+
COALESCE(s.n_tup_upd, 0)::float8 AS "updatesTotal",
|
|
4245
|
+
COALESCE(s.n_tup_del, 0)::float8 AS "deletesTotal"
|
|
4246
|
+
FROM pg_partition_tree('rows') t
|
|
4247
|
+
JOIN pg_class c ON c.oid = t.relid
|
|
4248
|
+
JOIN pg_class p ON p.oid = t.parentrelid
|
|
4249
|
+
LEFT JOIN pg_stat_user_tables s ON s.relid = c.oid
|
|
4250
|
+
WHERE t.level = 2`).flatMap((p) => {
|
|
4251
|
+
const namespace = listBound(p.nsBound);
|
|
4252
|
+
const type = listBound(p.bound);
|
|
4253
|
+
return namespace === void 0 || type === void 0 ? [] : [{
|
|
4254
|
+
...p,
|
|
4255
|
+
namespace,
|
|
4256
|
+
type
|
|
4257
|
+
}];
|
|
4258
|
+
});
|
|
4259
|
+
const held = await heldIndexes(sql, types.map((t) => t.table));
|
|
4260
|
+
return types.map((t) => toStats(t, held));
|
|
4261
|
+
}
|
|
4262
|
+
const [table] = await sql`
|
|
4263
|
+
SELECT c.reltuples > 0
|
|
4264
|
+
AND COALESCE(s.n_mod_since_analyze, 0) <= ${INDEX_COUNT_STALE} * c.reltuples
|
|
4265
|
+
AS fresh,
|
|
4266
|
+
GREATEST(c.reltuples, COALESCE(s.n_live_tup, 0))::float8 AS size,
|
|
4267
|
+
COALESCE(s.n_tup_ins, 0)::float8 AS inserts,
|
|
4268
|
+
COALESCE(s.n_tup_upd, 0)::float8 AS updates,
|
|
4269
|
+
COALESCE(s.n_tup_del, 0)::float8 AS deletes
|
|
4270
|
+
FROM pg_class c LEFT JOIN pg_stat_user_tables s ON s.relid = c.oid
|
|
4271
|
+
WHERE c.oid = 'rows'::regclass`;
|
|
4272
|
+
const size = table?.size ?? 0;
|
|
4273
|
+
const sampled = size > SAMPLE_ROWS;
|
|
4274
|
+
const counts = await sql`
|
|
4275
|
+
SELECT namespace, type, count(*)::float8 AS "all",
|
|
4276
|
+
(count(*) FILTER (WHERE deleted_at IS NULL))::float8 AS live
|
|
4277
|
+
FROM ${sql.unsafe(sampled ? `rows TABLESAMPLE SYSTEM (${100 * SAMPLE_ROWS / size})` : "rows")} GROUP BY namespace, type`;
|
|
4278
|
+
const total = counts.reduce((n, c) => n + c.all, 0);
|
|
4279
|
+
const held = await heldIndexes(sql, ["rows"]);
|
|
4280
|
+
const live = /* @__PURE__ */ new Map();
|
|
4281
|
+
for (const c of counts) live.set(JSON.stringify([c.namespace, c.type]), {
|
|
4282
|
+
namespace: c.namespace,
|
|
4283
|
+
type: c.type,
|
|
4284
|
+
liveRows: sampled ? c.live * size / total : c.live
|
|
4285
|
+
});
|
|
4286
|
+
if (sampled && table?.fresh === true) {
|
|
4287
|
+
const fromIndex = /* @__PURE__ */ new Map();
|
|
4288
|
+
for (const i of held) {
|
|
4289
|
+
const scope = legacyLiveScope(i.predicate);
|
|
4290
|
+
if (scope === void 0 || i.reltuples <= 0) continue;
|
|
4291
|
+
const key = JSON.stringify([scope.namespace, scope.type]);
|
|
4292
|
+
const seen = fromIndex.get(key);
|
|
4293
|
+
if (seen === void 0 || i.reltuples > seen.liveRows) fromIndex.set(key, {
|
|
4294
|
+
...scope,
|
|
4295
|
+
liveRows: i.reltuples
|
|
4296
|
+
});
|
|
4297
|
+
}
|
|
4298
|
+
for (const [key, t] of fromIndex) live.set(key, t);
|
|
4299
|
+
}
|
|
4300
|
+
const liveTotal = Math.max(size, [...live.values()].reduce((n, t) => n + t.liveRows, 0));
|
|
4301
|
+
const share = (n, rows) => liveTotal > 0 ? (n ?? 0) * rows / liveTotal : 0;
|
|
4302
|
+
return [...live.values()].map((t) => toStats({
|
|
4303
|
+
...t,
|
|
4304
|
+
table: "rows",
|
|
4305
|
+
insertsTotal: share(table?.inserts, t.liveRows),
|
|
4306
|
+
updatesTotal: share(table?.updates, t.liveRows),
|
|
4307
|
+
deletesTotal: share(table?.deletes, t.liveRows)
|
|
4308
|
+
}, held));
|
|
4309
|
+
}
|
|
4310
|
+
materialize(record) {
|
|
4311
|
+
const body = record.body ?? {};
|
|
4312
|
+
const out = { id: this.extractOriginalId(record.id, body) };
|
|
4313
|
+
for (const [k, v] of Object.entries(body)) {
|
|
4314
|
+
if (k === "id") continue;
|
|
4315
|
+
if (k === RESERVED_DELETED_AT) continue;
|
|
4316
|
+
if (k === "_metadata") continue;
|
|
4317
|
+
out[k] = v;
|
|
4318
|
+
}
|
|
4319
|
+
if (record.deleted_at !== null) out._deletedAt = record.deleted_at;
|
|
4320
|
+
if (record.short_id !== null && record.short_id !== void 0) out.shortId = Number(record.short_id);
|
|
4321
|
+
const meta = assembleRowMetadata(record);
|
|
4322
|
+
if (meta !== void 0) out._metadata = meta;
|
|
4323
|
+
return out;
|
|
4324
|
+
}
|
|
4325
|
+
extractOriginalId(textId, body) {
|
|
4326
|
+
const bodyId = body.id;
|
|
4327
|
+
if (typeof bodyId === "number" || typeof bodyId === "string") return bodyId;
|
|
4328
|
+
return textId;
|
|
4329
|
+
}
|
|
4330
|
+
};
|
|
4331
|
+
|
|
4332
|
+
//#endregion
|
|
4333
|
+
export { tryWithPgAdvisoryLock as _, PLAIN_SCHEMA_NAME as a, assertStoreSchema as c, isDerivedMigration as d, pendingMigrations as f, detectLayout as g, UUID_REGEX as h, MIGRATE_STORE_MARKER_TABLE as i, deriveListIndexMigration as l, runMigrations as m, txScopeIdFor as n, UUID_CHECK_MIGRATION as o, runDerivedMigrations as p, seqName as r, assertNotMigrating as s, PgDataset as t, getMigrationList as u };
|