@mnmal-ai/substrate 1.0.0-alpha.2 → 1.0.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -11,7 +11,7 @@ It runs as a real install: the binary and its plugins resolve from one shared `n
11
11
  | `@mnmal-ai/substrate/sdk` | Plugin authoring: `defineNamespace`, the field DSL, autoCRUD, engine helpers, `defineBootstrap`, policy combinators, route helpers, and all shared types |
12
12
  | `@mnmal-ai/substrate/client` | `SubstrateClient`, `watchRows`, client errors, the admin API, and the token and grant helpers |
13
13
  | `@mnmal-ai/substrate/pack` | The MessagePack wire codec |
14
- | `@mnmal-ai/substrate/testing` | `compose`, `MemorySubstrateServer`, `MemoryDataset`, `MemoryEdgesStore`, `loadPlugins`, `FakeEmbedder`, and the Postgres test-database guards |
14
+ | `@mnmal-ai/substrate/testing` | `compose`, `MemorySubstrateServer`, `MemoryDataset`, `MemoryEdgesStore`, `loadPlugins`, `FakeEmbedder`, `PgDataset`, `PgEdgesStore`, the pg migration runners, and the Postgres test-database guards |
15
15
  | `@mnmal-ai/substrate/recall` | The recall plugin's primitives (`EmbeddingsStore`, `LexicalStore`, `embedAndStore`, …) |
16
16
  | `@mnmal-ai/substrate/proposals` | The proposals plugin's exports |
17
17
 
@@ -11,7 +11,7 @@ import { createHash, createPublicKey, generateKeyPairSync, randomUUID } from "no
11
11
  import postgres from "postgres";
12
12
 
13
13
  //#region package.json
14
- var version = "1.0.0-alpha.2";
14
+ var version = "1.0.0-alpha.3";
15
15
 
16
16
  //#endregion
17
17
  //#region src/admin/errors.ts
@@ -0,0 +1,381 @@
1
+ import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
+ import { n as txScopeIdFor } from "./src-CXKnqeTW.js";
3
+ import { randomUUID } from "node:crypto";
4
+
5
+ //#region ../../packages/substrate-dataset-pg/src/pg-edges-store.ts
6
+ /**
7
+ * PgEdgesStore — PostgreSQL-backed reference implementation of EdgesStore.
8
+ * Spec: §4.6, §6.2, §6.3.
9
+ *
10
+ * One table (`edges`) keyed by a text PK, no namespace scoping — edges
11
+ * are engine-scoped (§4.6), not per-ns. The caller owns the `postgres` Sql
12
+ * connection; this class never opens or closes pools.
13
+ *
14
+ * Edge ids are `crypto.randomUUID()`: unique, so cursor pagination works, but
15
+ * not time-ordered. The sweep walks the whole table, so it doesn't need order.
16
+ *
17
+ * Edge.from/to .id is `string | number`, stored and read back as text (unlike
18
+ * PgDataset, which preserves numeric ids).
19
+ *
20
+ * SQL safety: postgres.js tagged templates parameterise every value slot.
21
+ * Never string-concat caller input — the `update` dynamic column list uses a
22
+ * fixed allowlist of column names, not user-supplied keys.
23
+ */
24
+ const MUTABLE_PATCH_FIELDS = /* @__PURE__ */ new Set([
25
+ "id",
26
+ "lastSweptAt",
27
+ "blocked"
28
+ ]);
29
+ /**
30
+ * Batch size for chunked admin deletes (`dropEdgesByNamespace`,
31
+ * `dropEdgesByType`): bounds per-statement lock hold on large tables.
32
+ */
33
+ const DEFAULT_DROP_BATCH_SIZE = 5e3;
34
+ const VALID_POLICIES = /* @__PURE__ */ new Set([
35
+ "set_null",
36
+ "cascade",
37
+ "restrict",
38
+ "orphan"
39
+ ]);
40
+ const refToId = (id) => typeof id === "number" ? String(id) : id;
41
+ var PgEdgesStore = class PgEdgesStore {
42
+ #sql;
43
+ #now;
44
+ #inTx;
45
+ #dropBatchSize;
46
+ txScopeId;
47
+ get #edgeCols() {
48
+ return this.#sql`id, from_ns, from_type, from_id, from_bridged, to_ns, to_type, to_id, to_bridged, field_name, policy, created_at, deleted_at, last_swept_at, blocked`;
49
+ }
50
+ constructor(options) {
51
+ this.#sql = options.sql;
52
+ this.#now = options.now ?? (() => /* @__PURE__ */ new Date());
53
+ this.#inTx = "inTx" in options ? options.inTx : false;
54
+ const explicit = "txScopeId" in options ? options.txScopeId : void 0;
55
+ this.txScopeId = explicit ?? txScopeIdFor(options.sql);
56
+ const batch = options.dropBatchSize;
57
+ if (batch !== void 0 && (!Number.isFinite(batch) || batch <= 0)) throw new Error(`PgEdgesStore: dropBatchSize must be a positive finite number, got ${batch}`);
58
+ this.#dropBatchSize = batch ?? DEFAULT_DROP_BATCH_SIZE;
59
+ }
60
+ async write(input) {
61
+ const id = randomUUID();
62
+ const createdAt = this.#now();
63
+ const fromBridged = input.from.bridged === true;
64
+ const toBridged = input.to.bridged === true;
65
+ await this.#sql`
66
+ INSERT INTO edges (
67
+ id, from_ns, from_type, from_id, from_bridged,
68
+ to_ns, to_type, to_id, to_bridged,
69
+ field_name, policy, created_at, deleted_at, last_swept_at, blocked
70
+ )
71
+ VALUES (
72
+ ${id},
73
+ ${input.from.ns}, ${input.from.type}, ${refToId(input.from.id)}, ${fromBridged},
74
+ ${input.to.ns}, ${input.to.type}, ${refToId(input.to.id)}, ${toBridged},
75
+ ${input.fieldName}, ${input.policy}, ${createdAt},
76
+ NULL, NULL, false
77
+ )
78
+ `;
79
+ return {
80
+ id,
81
+ from: buildRef(input.from.ns, input.from.type, refToId(input.from.id), fromBridged),
82
+ to: buildRef(input.to.ns, input.to.type, refToId(input.to.id), toBridged),
83
+ fieldName: input.fieldName,
84
+ policy: input.policy,
85
+ createdAt
86
+ };
87
+ }
88
+ async tombstone(id, at) {
89
+ const stamp = at ?? this.#now();
90
+ await this.#sql`
91
+ UPDATE edges
92
+ SET deleted_at = ${stamp}
93
+ WHERE id = ${id}
94
+ AND deleted_at IS NULL
95
+ `;
96
+ }
97
+ /**
98
+ * Hard-deletes tombstoned edges whose `deleted_at` is older than
99
+ * `olderThan`. Called by the sweep loop. Returns the number deleted.
100
+ */
101
+ async purgeTombstonedEdges(opts) {
102
+ return { deleted: (await this.#sql`
103
+ DELETE FROM edges
104
+ WHERE deleted_at IS NOT NULL
105
+ AND deleted_at < ${opts.olderThan}
106
+ RETURNING id
107
+ `).length };
108
+ }
109
+ async update(patch) {
110
+ for (const key of Object.keys(patch)) if (!MUTABLE_PATCH_FIELDS.has(key)) throw new Error(`EdgesStore.update: field '${key}' is immutable`);
111
+ const hasLastSweptAt = Object.hasOwn(patch, "lastSweptAt");
112
+ const hasBlocked = Object.hasOwn(patch, "blocked");
113
+ if (!hasLastSweptAt && !hasBlocked) return;
114
+ if (hasLastSweptAt && hasBlocked) {
115
+ await this.#sql`
116
+ UPDATE edges
117
+ SET last_swept_at = ${patch.lastSweptAt ?? null},
118
+ blocked = ${patch.blocked ?? false}
119
+ WHERE id = ${patch.id}
120
+ `;
121
+ return;
122
+ }
123
+ if (hasLastSweptAt) {
124
+ await this.#sql`
125
+ UPDATE edges
126
+ SET last_swept_at = ${patch.lastSweptAt ?? null}
127
+ WHERE id = ${patch.id}
128
+ `;
129
+ return;
130
+ }
131
+ await this.#sql`
132
+ UPDATE edges
133
+ SET blocked = ${patch.blocked ?? false}
134
+ WHERE id = ${patch.id}
135
+ `;
136
+ }
137
+ async readNextBatch(cursorId, size) {
138
+ const edges = (cursorId === null ? await this.#sql`
139
+ SELECT ${this.#edgeCols}
140
+ FROM edges
141
+ ORDER BY id ASC
142
+ LIMIT ${size}
143
+ ` : await this.#sql`
144
+ SELECT ${this.#edgeCols}
145
+ FROM edges
146
+ WHERE id > ${cursorId}
147
+ ORDER BY id ASC
148
+ LIMIT ${size}
149
+ `).map(rowToEdge);
150
+ const last = edges[edges.length - 1];
151
+ return {
152
+ edges,
153
+ lastId: last ? last.id : null
154
+ };
155
+ }
156
+ async findEdgesTo(ref) {
157
+ return (await this.#sql`
158
+ SELECT ${this.#edgeCols}
159
+ FROM edges
160
+ WHERE to_ns = ${ref.ns}
161
+ AND to_type = ${ref.type}
162
+ AND to_id = ${refToId(ref.id)}
163
+ AND deleted_at IS NULL
164
+ `).map(rowToEdge);
165
+ }
166
+ async findEdgesToMany(refs) {
167
+ if (refs.length === 0) return [];
168
+ const sql = this.#sql;
169
+ const seenRef = /* @__PURE__ */ new Set();
170
+ const toNs = [];
171
+ const toType = [];
172
+ const toId = [];
173
+ for (const r of refs) {
174
+ const id = refToId(r.id);
175
+ const k = `${r.ns}\u0000${r.type}\u0000${id}`;
176
+ if (seenRef.has(k)) continue;
177
+ seenRef.add(k);
178
+ toNs.push(r.ns);
179
+ toType.push(r.type);
180
+ toId.push(id);
181
+ }
182
+ return (await sql`
183
+ SELECT ${this.#edgeCols}
184
+ FROM edges
185
+ JOIN unnest(${toNs}::text[], ${toType}::text[], ${toId}::text[])
186
+ AS seek(seek_ns, seek_type, seek_id)
187
+ ON to_ns = seek.seek_ns AND to_type = seek.seek_type AND to_id = seek.seek_id
188
+ WHERE deleted_at IS NULL
189
+ `).map(rowToEdge);
190
+ }
191
+ async findEdgesFrom(ref) {
192
+ return (await this.#sql`
193
+ SELECT ${this.#edgeCols}
194
+ FROM edges
195
+ WHERE from_ns = ${ref.ns}
196
+ AND from_type = ${ref.type}
197
+ AND from_id = ${refToId(ref.id)}
198
+ AND deleted_at IS NULL
199
+ `).map(rowToEdge);
200
+ }
201
+ async findEdgesTouchingNamespace(namespace, limit) {
202
+ return (await this.#sql`
203
+ SELECT ${this.#edgeCols}
204
+ FROM edges
205
+ WHERE deleted_at IS NULL
206
+ AND (from_ns = ${namespace} OR to_ns = ${namespace})
207
+ ORDER BY id ASC
208
+ LIMIT ${limit}
209
+ `).map(rowToEdge);
210
+ }
211
+ async findEdgesTouchingType(namespace, type, limit) {
212
+ return (await this.#sql`
213
+ SELECT ${this.#edgeCols}
214
+ FROM edges
215
+ WHERE deleted_at IS NULL
216
+ AND ((from_ns = ${namespace} AND from_type = ${type})
217
+ OR (to_ns = ${namespace} AND to_type = ${type}))
218
+ ORDER BY id ASC
219
+ LIMIT ${limit}
220
+ `).map(rowToEdge);
221
+ }
222
+ async namespacesLinkedTo(namespace, type) {
223
+ const fromMatch = type === void 0 ? this.#sql`from_ns = ${namespace}` : this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
224
+ const toMatch = type === void 0 ? this.#sql`to_ns = ${namespace}` : this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
225
+ return (await this.#sql`
226
+ SELECT ns FROM (
227
+ SELECT to_ns AS ns FROM edges WHERE ${fromMatch}
228
+ UNION
229
+ SELECT from_ns AS ns FROM edges WHERE ${toMatch}
230
+ ) linked
231
+ WHERE ns <> ${namespace}
232
+ ORDER BY ns
233
+ `).map((r) => r.ns);
234
+ }
235
+ /**
236
+ * Chunked physical delete of every edge touching `namespace`, at most
237
+ * `dropBatchSize` rows per statement, so a large delete does not hold every
238
+ * row lock at once. Returns the number deleted.
239
+ *
240
+ * Outside a tx each batch auto-commits. Inside a caller's tx the batches
241
+ * stay in it, so a throw still rolls back the whole drop.
242
+ */
243
+ async dropEdgesByNamespace(namespace, within) {
244
+ const batchSize = this.#dropBatchSize;
245
+ const farEnd = this.#farEndWithin(namespace, within, this.#sql`from_ns = ${namespace}`, this.#sql`to_ns = ${namespace}`);
246
+ let total = 0;
247
+ while (true) {
248
+ const rows = await this.#sql`
249
+ DELETE FROM edges
250
+ WHERE id IN (
251
+ SELECT id FROM edges
252
+ WHERE (from_ns = ${namespace} OR to_ns = ${namespace}) ${farEnd}
253
+ LIMIT ${batchSize}
254
+ )
255
+ RETURNING id
256
+ `;
257
+ total += rows.length;
258
+ if (rows.length < batchSize) return total;
259
+ await this.#yieldBetweenBatches();
260
+ }
261
+ }
262
+ /**
263
+ * Chunked physical delete of every edge touching `(namespace, type)`.
264
+ * See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
265
+ */
266
+ async dropEdgesByType(namespace, type, within) {
267
+ const batchSize = this.#dropBatchSize;
268
+ const fromMatch = this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
269
+ const toMatch = this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
270
+ const farEnd = this.#farEndWithin(namespace, within, fromMatch, toMatch);
271
+ let total = 0;
272
+ while (true) {
273
+ const rows = await this.#sql`
274
+ DELETE FROM edges
275
+ WHERE id IN (
276
+ SELECT id FROM edges
277
+ WHERE (${fromMatch} OR ${toMatch}) ${farEnd}
278
+ LIMIT ${batchSize}
279
+ )
280
+ RETURNING id
281
+ `;
282
+ total += rows.length;
283
+ if (rows.length < batchSize) return total;
284
+ await this.#yieldBetweenBatches();
285
+ }
286
+ }
287
+ /** `AND` clause keeping only edges whose far end is `namespace` or in `within`. */
288
+ #farEndWithin(namespace, within, fromMatch, toMatch) {
289
+ if (within === void 0) return this.#sql``;
290
+ const allowed = this.#sql([namespace, ...within]);
291
+ return this.#sql`AND ((${fromMatch} AND to_ns IN ${allowed}) OR (${toMatch} AND from_ns IN ${allowed}))`;
292
+ }
293
+ /**
294
+ * Yields to the event loop between drop batches outside a tx; no-op inside one.
295
+ */
296
+ async #yieldBetweenBatches() {
297
+ if (!this.#inTx) await new Promise((resolve) => setImmediate(resolve));
298
+ }
299
+ async canDelete(ref) {
300
+ const rows = await this.#sql`
301
+ SELECT ${this.#edgeCols}
302
+ FROM edges
303
+ WHERE to_ns = ${ref.ns}
304
+ AND to_type = ${ref.type}
305
+ AND to_id = ${refToId(ref.id)}
306
+ AND policy = 'restrict'
307
+ AND deleted_at IS NULL
308
+ `;
309
+ if (rows.length === 0) return { ok: true };
310
+ return {
311
+ ok: false,
312
+ blockedBy: rows.map(rowToEdge)
313
+ };
314
+ }
315
+ async findBlocked() {
316
+ return (await this.#sql`
317
+ SELECT ${this.#edgeCols}
318
+ FROM edges
319
+ WHERE blocked = true
320
+ AND deleted_at IS NULL
321
+ `).map(rowToEdge);
322
+ }
323
+ /**
324
+ * `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
325
+ * nested re-entry skips the inner begin and runs on the existing tx handle
326
+ * so an inner throw propagates to the outer `sql.begin` for rollback.
327
+ */
328
+ async withTransaction(fn) {
329
+ if (this.#inTx) return fn(this);
330
+ return this.#sql.begin(async (txSql) => {
331
+ return fn(new PgEdgesStore({
332
+ sql: txSql,
333
+ now: this.#now,
334
+ inTx: true,
335
+ txScopeId: this.txScopeId,
336
+ dropBatchSize: this.#dropBatchSize
337
+ }));
338
+ });
339
+ }
340
+ /**
341
+ * Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
342
+ * for `PgDataset.withScopedTransaction`. The caller has verified
343
+ * the txScopeIds match.
344
+ */
345
+ _enlistInTx(txSql) {
346
+ return new PgEdgesStore({
347
+ sql: txSql,
348
+ now: this.#now,
349
+ inTx: true,
350
+ txScopeId: this.txScopeId,
351
+ dropBatchSize: this.#dropBatchSize
352
+ });
353
+ }
354
+ };
355
+ function buildRef(ns, type, id, bridged) {
356
+ const ref = {
357
+ ns,
358
+ type,
359
+ id
360
+ };
361
+ if (bridged) ref.bridged = true;
362
+ return ref;
363
+ }
364
+ function rowToEdge(row) {
365
+ if (!VALID_POLICIES.has(row.policy)) throw new Error(`PgEdgesStore: unknown policy '${row.policy}' on edge id=${row.id}`);
366
+ const edge = {
367
+ id: row.id,
368
+ from: buildRef(row.from_ns, row.from_type, row.from_id, row.from_bridged),
369
+ to: buildRef(row.to_ns, row.to_type, row.to_id, row.to_bridged),
370
+ fieldName: row.field_name,
371
+ policy: row.policy,
372
+ createdAt: row.created_at
373
+ };
374
+ if (row.deleted_at !== null) edge.deletedAt = row.deleted_at;
375
+ if (row.last_swept_at !== null) edge.lastSweptAt = row.last_swept_at;
376
+ if (row.blocked) edge.blocked = true;
377
+ return edge;
378
+ }
379
+
380
+ //#endregion
381
+ export { PgEdgesStore as t };
package/dist/recall.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-DWm0iA88.js";
2
+ import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-B-wkCHYL.js";
3
3
  import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-QuJN9pG_.js";
4
4
 
5
5
  export { CLUSTERER_BACKEND, CLUSTERER_BACKEND_VERSION, CLUSTER_FIELDS, CLUSTER_MEMBER_FIELDS, CRYSTALLISE_IN_FLIGHT_CODE, DEFAULT_CRYSTALLISE_CONFIG, EMBEDDINGS_FETCH_MAX_REFS, EmbedSourceRegistry, InMemoryClusterStore, InMemoryEmbeddingsStore, InMemoryLexicalStore, PgClusterStore, PgEmbeddingsStore, PgLexicalStore, RECALL_VERSION, REINDEX_DEFAULT_LIMIT, REINDEX_MAX_LIMIT, RRF_K, SALIENCE_BOOST_MAX_TARGETS, SELECT_CANDIDATES_MAX_LIMIT, armRanks, buildClusterSurface, checkParams, createCrystalliseGuard, createEmbedStats, createRecallPlugin, crystalliseRouteHandler, embedAndStore, embedRowReporting, embedStatsRouteHandler, embeddingSimilarOperator, embeddingsFetchRouteHandler, embeddingsMigration, fuseByRank, handleCrystallise, handleEmbedStatsRequest, handleLexicalCoverage, handleRecallRequest, handleReindex, l2normalize, lexicalMigration, makeEmbedPostWrite, makeLexicalClassify, makeLexicalPostWrite, makeReembed, memberKey, recallClustersMigration, recallMcpTool, recallRouteHandler, reindexRouteHandler, runClusterer, salienceBoostRouteHandler, unknownParamMessage, unknownParams, validateCrystalliseInput, validateLexicalCoverageInput, validateReindexInput, validateSelector };
@@ -1,9 +1,9 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-QuJN9pG_.js";
3
2
  import { n as txScopeIdFor } from "./src-CXKnqeTW.js";
3
+ import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-QuJN9pG_.js";
4
4
 
5
5
  //#region ../../plugins/substrate-recall/package.json
6
- var version = "1.0.0-alpha.2";
6
+ var version = "1.0.0-alpha.3";
7
7
 
8
8
  //#endregion
9
9
  //#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
@@ -4,7 +4,7 @@ import { d as buildAuditRow, f as AUDIT_EVENT_TYPE } from "./src-CHp3QyqB.js";
4
4
  import { d as resolveDefaultKidPath, u as resolveDefaultKeyPath } from "./env-utils-CGNazUNt.js";
5
5
  import { E as describeGrant, I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./row-DZ0Fnpc6.js";
6
6
  import { S as reconcileIdentities } from "./src-DMzc3HfW.js";
7
- import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-BQ8egGl5.js";
7
+ import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-BBZuhrsL.js";
8
8
  import { existsSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
9
9
  import { dirname } from "node:path";
10
10
  import { createHash, randomUUID } from "node:crypto";
package/dist/substrate.js CHANGED
@@ -4,9 +4,10 @@ import { B as saveSweepState, G as registerExtensionOperator, I as effectiveFilt
4
4
  import { i as DEFAULT_CAPABILITIES, n as trimTrailingSlashes, s as makeAuthorizationProvider, t as parsePositiveInt } from "./env-utils-CGNazUNt.js";
5
5
  import { L as encodeList, T as SYSTEM_NAMESPACE, f as coversNamespace, m as grantedIn, p as grantOf } from "./row-DZ0Fnpc6.js";
6
6
  import { c as AuthError, d as AdminBackfillError, l as NetworkError, n as tokenAudience, r as SubstrateClient, t as noRedirectFetch } from "./src-DMzc3HfW.js";
7
- import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-DWm0iA88.js";
8
- import { a as runDerivedMigrations, n as txScopeIdFor, o as runMigrations, r as deriveListIndexMigration, s as tryWithPgAdvisoryLock, t as PgDataset } from "./src-CXKnqeTW.js";
9
- import { B as version, R as openSql, t as openPgDataset, z as AdminCliError } from "./dataset-BQ8egGl5.js";
7
+ import { a as runDerivedMigrations, o as runMigrations, r as deriveListIndexMigration, s as tryWithPgAdvisoryLock, t as PgDataset } from "./src-CXKnqeTW.js";
8
+ import { t as PgEdgesStore } from "./pg-edges-store-DKPYyd8l.js";
9
+ import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-B-wkCHYL.js";
10
+ import { B as version, R as openSql, t as openPgDataset, z as AdminCliError } from "./dataset-BBZuhrsL.js";
10
11
  import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
11
12
  import { homedir, hostname } from "node:os";
12
13
  import { basename, dirname, join, resolve } from "node:path";
@@ -292,382 +293,6 @@ var MemoryIdempotencyStore = class {
292
293
  }
293
294
  };
294
295
 
295
- //#endregion
296
- //#region ../../packages/substrate-dataset-pg/src/pg-edges-store.ts
297
- /**
298
- * PgEdgesStore — PostgreSQL-backed reference implementation of EdgesStore.
299
- * Spec: §4.6, §6.2, §6.3.
300
- *
301
- * One table (`edges`) keyed by a text PK, no namespace scoping — edges
302
- * are engine-scoped (§4.6), not per-ns. The caller owns the `postgres` Sql
303
- * connection; this class never opens or closes pools.
304
- *
305
- * Edge ids are `crypto.randomUUID()`: unique, so cursor pagination works, but
306
- * not time-ordered. The sweep walks the whole table, so it doesn't need order.
307
- *
308
- * Edge.from/to .id is `string | number`, stored and read back as text (unlike
309
- * PgDataset, which preserves numeric ids).
310
- *
311
- * SQL safety: postgres.js tagged templates parameterise every value slot.
312
- * Never string-concat caller input — the `update` dynamic column list uses a
313
- * fixed allowlist of column names, not user-supplied keys.
314
- */
315
- const MUTABLE_PATCH_FIELDS = /* @__PURE__ */ new Set([
316
- "id",
317
- "lastSweptAt",
318
- "blocked"
319
- ]);
320
- /**
321
- * Batch size for chunked admin deletes (`dropEdgesByNamespace`,
322
- * `dropEdgesByType`): bounds per-statement lock hold on large tables.
323
- */
324
- const DEFAULT_DROP_BATCH_SIZE = 5e3;
325
- const VALID_POLICIES = /* @__PURE__ */ new Set([
326
- "set_null",
327
- "cascade",
328
- "restrict",
329
- "orphan"
330
- ]);
331
- const refToId = (id) => typeof id === "number" ? String(id) : id;
332
- var PgEdgesStore = class PgEdgesStore {
333
- #sql;
334
- #now;
335
- #inTx;
336
- #dropBatchSize;
337
- txScopeId;
338
- get #edgeCols() {
339
- return this.#sql`id, from_ns, from_type, from_id, from_bridged, to_ns, to_type, to_id, to_bridged, field_name, policy, created_at, deleted_at, last_swept_at, blocked`;
340
- }
341
- constructor(options) {
342
- this.#sql = options.sql;
343
- this.#now = options.now ?? (() => /* @__PURE__ */ new Date());
344
- this.#inTx = "inTx" in options ? options.inTx : false;
345
- const explicit = "txScopeId" in options ? options.txScopeId : void 0;
346
- this.txScopeId = explicit ?? txScopeIdFor(options.sql);
347
- const batch = options.dropBatchSize;
348
- if (batch !== void 0 && (!Number.isFinite(batch) || batch <= 0)) throw new Error(`PgEdgesStore: dropBatchSize must be a positive finite number, got ${batch}`);
349
- this.#dropBatchSize = batch ?? DEFAULT_DROP_BATCH_SIZE;
350
- }
351
- async write(input) {
352
- const id = randomUUID();
353
- const createdAt = this.#now();
354
- const fromBridged = input.from.bridged === true;
355
- const toBridged = input.to.bridged === true;
356
- await this.#sql`
357
- INSERT INTO edges (
358
- id, from_ns, from_type, from_id, from_bridged,
359
- to_ns, to_type, to_id, to_bridged,
360
- field_name, policy, created_at, deleted_at, last_swept_at, blocked
361
- )
362
- VALUES (
363
- ${id},
364
- ${input.from.ns}, ${input.from.type}, ${refToId(input.from.id)}, ${fromBridged},
365
- ${input.to.ns}, ${input.to.type}, ${refToId(input.to.id)}, ${toBridged},
366
- ${input.fieldName}, ${input.policy}, ${createdAt},
367
- NULL, NULL, false
368
- )
369
- `;
370
- return {
371
- id,
372
- from: buildRef(input.from.ns, input.from.type, refToId(input.from.id), fromBridged),
373
- to: buildRef(input.to.ns, input.to.type, refToId(input.to.id), toBridged),
374
- fieldName: input.fieldName,
375
- policy: input.policy,
376
- createdAt
377
- };
378
- }
379
- async tombstone(id, at) {
380
- const stamp = at ?? this.#now();
381
- await this.#sql`
382
- UPDATE edges
383
- SET deleted_at = ${stamp}
384
- WHERE id = ${id}
385
- AND deleted_at IS NULL
386
- `;
387
- }
388
- /**
389
- * Hard-deletes tombstoned edges whose `deleted_at` is older than
390
- * `olderThan`. Called by the sweep loop. Returns the number deleted.
391
- */
392
- async purgeTombstonedEdges(opts) {
393
- return { deleted: (await this.#sql`
394
- DELETE FROM edges
395
- WHERE deleted_at IS NOT NULL
396
- AND deleted_at < ${opts.olderThan}
397
- RETURNING id
398
- `).length };
399
- }
400
- async update(patch) {
401
- for (const key of Object.keys(patch)) if (!MUTABLE_PATCH_FIELDS.has(key)) throw new Error(`EdgesStore.update: field '${key}' is immutable`);
402
- const hasLastSweptAt = Object.hasOwn(patch, "lastSweptAt");
403
- const hasBlocked = Object.hasOwn(patch, "blocked");
404
- if (!hasLastSweptAt && !hasBlocked) return;
405
- if (hasLastSweptAt && hasBlocked) {
406
- await this.#sql`
407
- UPDATE edges
408
- SET last_swept_at = ${patch.lastSweptAt ?? null},
409
- blocked = ${patch.blocked ?? false}
410
- WHERE id = ${patch.id}
411
- `;
412
- return;
413
- }
414
- if (hasLastSweptAt) {
415
- await this.#sql`
416
- UPDATE edges
417
- SET last_swept_at = ${patch.lastSweptAt ?? null}
418
- WHERE id = ${patch.id}
419
- `;
420
- return;
421
- }
422
- await this.#sql`
423
- UPDATE edges
424
- SET blocked = ${patch.blocked ?? false}
425
- WHERE id = ${patch.id}
426
- `;
427
- }
428
- async readNextBatch(cursorId, size) {
429
- const edges = (cursorId === null ? await this.#sql`
430
- SELECT ${this.#edgeCols}
431
- FROM edges
432
- ORDER BY id ASC
433
- LIMIT ${size}
434
- ` : await this.#sql`
435
- SELECT ${this.#edgeCols}
436
- FROM edges
437
- WHERE id > ${cursorId}
438
- ORDER BY id ASC
439
- LIMIT ${size}
440
- `).map(rowToEdge);
441
- const last = edges[edges.length - 1];
442
- return {
443
- edges,
444
- lastId: last ? last.id : null
445
- };
446
- }
447
- async findEdgesTo(ref) {
448
- return (await this.#sql`
449
- SELECT ${this.#edgeCols}
450
- FROM edges
451
- WHERE to_ns = ${ref.ns}
452
- AND to_type = ${ref.type}
453
- AND to_id = ${refToId(ref.id)}
454
- AND deleted_at IS NULL
455
- `).map(rowToEdge);
456
- }
457
- async findEdgesToMany(refs) {
458
- if (refs.length === 0) return [];
459
- const sql = this.#sql;
460
- const seenRef = /* @__PURE__ */ new Set();
461
- const toNs = [];
462
- const toType = [];
463
- const toId = [];
464
- for (const r of refs) {
465
- const id = refToId(r.id);
466
- const k = `${r.ns}\u0000${r.type}\u0000${id}`;
467
- if (seenRef.has(k)) continue;
468
- seenRef.add(k);
469
- toNs.push(r.ns);
470
- toType.push(r.type);
471
- toId.push(id);
472
- }
473
- return (await sql`
474
- SELECT ${this.#edgeCols}
475
- FROM edges
476
- JOIN unnest(${toNs}::text[], ${toType}::text[], ${toId}::text[])
477
- AS seek(seek_ns, seek_type, seek_id)
478
- ON to_ns = seek.seek_ns AND to_type = seek.seek_type AND to_id = seek.seek_id
479
- WHERE deleted_at IS NULL
480
- `).map(rowToEdge);
481
- }
482
- async findEdgesFrom(ref) {
483
- return (await this.#sql`
484
- SELECT ${this.#edgeCols}
485
- FROM edges
486
- WHERE from_ns = ${ref.ns}
487
- AND from_type = ${ref.type}
488
- AND from_id = ${refToId(ref.id)}
489
- AND deleted_at IS NULL
490
- `).map(rowToEdge);
491
- }
492
- async findEdgesTouchingNamespace(namespace, limit) {
493
- return (await this.#sql`
494
- SELECT ${this.#edgeCols}
495
- FROM edges
496
- WHERE deleted_at IS NULL
497
- AND (from_ns = ${namespace} OR to_ns = ${namespace})
498
- ORDER BY id ASC
499
- LIMIT ${limit}
500
- `).map(rowToEdge);
501
- }
502
- async findEdgesTouchingType(namespace, type, limit) {
503
- return (await this.#sql`
504
- SELECT ${this.#edgeCols}
505
- FROM edges
506
- WHERE deleted_at IS NULL
507
- AND ((from_ns = ${namespace} AND from_type = ${type})
508
- OR (to_ns = ${namespace} AND to_type = ${type}))
509
- ORDER BY id ASC
510
- LIMIT ${limit}
511
- `).map(rowToEdge);
512
- }
513
- async namespacesLinkedTo(namespace, type) {
514
- const fromMatch = type === void 0 ? this.#sql`from_ns = ${namespace}` : this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
515
- const toMatch = type === void 0 ? this.#sql`to_ns = ${namespace}` : this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
516
- return (await this.#sql`
517
- SELECT ns FROM (
518
- SELECT to_ns AS ns FROM edges WHERE ${fromMatch}
519
- UNION
520
- SELECT from_ns AS ns FROM edges WHERE ${toMatch}
521
- ) linked
522
- WHERE ns <> ${namespace}
523
- ORDER BY ns
524
- `).map((r) => r.ns);
525
- }
526
- /**
527
- * Chunked physical delete of every edge touching `namespace`, at most
528
- * `dropBatchSize` rows per statement, so a large delete does not hold every
529
- * row lock at once. Returns the number deleted.
530
- *
531
- * Outside a tx each batch auto-commits. Inside a caller's tx the batches
532
- * stay in it, so a throw still rolls back the whole drop.
533
- */
534
- async dropEdgesByNamespace(namespace, within) {
535
- const batchSize = this.#dropBatchSize;
536
- const farEnd = this.#farEndWithin(namespace, within, this.#sql`from_ns = ${namespace}`, this.#sql`to_ns = ${namespace}`);
537
- let total = 0;
538
- while (true) {
539
- const rows = await this.#sql`
540
- DELETE FROM edges
541
- WHERE id IN (
542
- SELECT id FROM edges
543
- WHERE (from_ns = ${namespace} OR to_ns = ${namespace}) ${farEnd}
544
- LIMIT ${batchSize}
545
- )
546
- RETURNING id
547
- `;
548
- total += rows.length;
549
- if (rows.length < batchSize) return total;
550
- await this.#yieldBetweenBatches();
551
- }
552
- }
553
- /**
554
- * Chunked physical delete of every edge touching `(namespace, type)`.
555
- * See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
556
- */
557
- async dropEdgesByType(namespace, type, within) {
558
- const batchSize = this.#dropBatchSize;
559
- const fromMatch = this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
560
- const toMatch = this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
561
- const farEnd = this.#farEndWithin(namespace, within, fromMatch, toMatch);
562
- let total = 0;
563
- while (true) {
564
- const rows = await this.#sql`
565
- DELETE FROM edges
566
- WHERE id IN (
567
- SELECT id FROM edges
568
- WHERE (${fromMatch} OR ${toMatch}) ${farEnd}
569
- LIMIT ${batchSize}
570
- )
571
- RETURNING id
572
- `;
573
- total += rows.length;
574
- if (rows.length < batchSize) return total;
575
- await this.#yieldBetweenBatches();
576
- }
577
- }
578
- /** `AND` clause keeping only edges whose far end is `namespace` or in `within`. */
579
- #farEndWithin(namespace, within, fromMatch, toMatch) {
580
- if (within === void 0) return this.#sql``;
581
- const allowed = this.#sql([namespace, ...within]);
582
- return this.#sql`AND ((${fromMatch} AND to_ns IN ${allowed}) OR (${toMatch} AND from_ns IN ${allowed}))`;
583
- }
584
- /**
585
- * Yields to the event loop between drop batches outside a tx; no-op inside one.
586
- */
587
- async #yieldBetweenBatches() {
588
- if (!this.#inTx) await new Promise((resolve) => setImmediate(resolve));
589
- }
590
- async canDelete(ref) {
591
- const rows = await this.#sql`
592
- SELECT ${this.#edgeCols}
593
- FROM edges
594
- WHERE to_ns = ${ref.ns}
595
- AND to_type = ${ref.type}
596
- AND to_id = ${refToId(ref.id)}
597
- AND policy = 'restrict'
598
- AND deleted_at IS NULL
599
- `;
600
- if (rows.length === 0) return { ok: true };
601
- return {
602
- ok: false,
603
- blockedBy: rows.map(rowToEdge)
604
- };
605
- }
606
- async findBlocked() {
607
- return (await this.#sql`
608
- SELECT ${this.#edgeCols}
609
- FROM edges
610
- WHERE blocked = true
611
- AND deleted_at IS NULL
612
- `).map(rowToEdge);
613
- }
614
- /**
615
- * `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
616
- * nested re-entry skips the inner begin and runs on the existing tx handle
617
- * so an inner throw propagates to the outer `sql.begin` for rollback.
618
- */
619
- async withTransaction(fn) {
620
- if (this.#inTx) return fn(this);
621
- return this.#sql.begin(async (txSql) => {
622
- return fn(new PgEdgesStore({
623
- sql: txSql,
624
- now: this.#now,
625
- inTx: true,
626
- txScopeId: this.txScopeId,
627
- dropBatchSize: this.#dropBatchSize
628
- }));
629
- });
630
- }
631
- /**
632
- * Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
633
- * for `PgDataset.withScopedTransaction`. The caller has verified
634
- * the txScopeIds match.
635
- */
636
- _enlistInTx(txSql) {
637
- return new PgEdgesStore({
638
- sql: txSql,
639
- now: this.#now,
640
- inTx: true,
641
- txScopeId: this.txScopeId,
642
- dropBatchSize: this.#dropBatchSize
643
- });
644
- }
645
- };
646
- function buildRef(ns, type, id, bridged) {
647
- const ref = {
648
- ns,
649
- type,
650
- id
651
- };
652
- if (bridged) ref.bridged = true;
653
- return ref;
654
- }
655
- function rowToEdge(row) {
656
- if (!VALID_POLICIES.has(row.policy)) throw new Error(`PgEdgesStore: unknown policy '${row.policy}' on edge id=${row.id}`);
657
- const edge = {
658
- id: row.id,
659
- from: buildRef(row.from_ns, row.from_type, row.from_id, row.from_bridged),
660
- to: buildRef(row.to_ns, row.to_type, row.to_id, row.to_bridged),
661
- fieldName: row.field_name,
662
- policy: row.policy,
663
- createdAt: row.created_at
664
- };
665
- if (row.deleted_at !== null) edge.deletedAt = row.deleted_at;
666
- if (row.last_swept_at !== null) edge.lastSweptAt = row.last_swept_at;
667
- if (row.blocked) edge.blocked = true;
668
- return edge;
669
- }
670
-
671
296
  //#endregion
672
297
  //#region ../../packages/substrate-dataset-pg/src/pg-idempotency-store.ts
673
298
  var PgIdempotencyStore = class {
package/dist/testing.d.ts CHANGED
@@ -1,8 +1,9 @@
1
1
  import { C as Id, E as Row, _ as EdgesStore, b as ActorIdentity, f as CanDeleteResult, g as EdgeWriteInput, h as EdgeRef, l as PackEnvelope, m as EdgePatch, n as FieldDef, p as Edge, u as Schema } from "./schema-g4FQLr0V.js";
2
2
  import { a as RuntimeErrors, n as ResolveObservation, r as ResolverStat, s as SlowRequest, t as ProcessStat } from "./runtime-metrics-DwpDJ8vz.js";
3
- import { $ as PolicyProbeContext, A as ScheduledTask, B as StartOpts, C as QualifiedFrame, D as RegistryEntryInput, E as RegistryEntry, F as HttpMethod, G as WsConnection, H as SubstrateResponse, I as PluginRouter, K as WsHandler, P as HttpHandler, Q as PolicyDecision, R as RequestObserver, St as WriteResult, T as RegistryBase, U as SubstrateServer, V as SubstrateRequest, Y as Identity, _t as ReadResult, a as EngineHandle, ct as Dataset, dt as DropResult, et as RequestContext, g as NamespaceConfig, gt as ReadParams, lt as DatasetRegistry, mt as FilterCapabilities, n as DropParticipant, o as EngineInput, p as KernelPlugin, st as ConditionalUpdateResult, t as CoordinatorPlugin, ut as DeleteResult, w as Registry$1, xt as WriteOptions, yt as UpdateFieldRef, z as RuntimeOp } from "./plugin-t2g2wX0b.js";
3
+ import { $ as PolicyProbeContext, A as ScheduledTask, B as StartOpts, C as QualifiedFrame, D as RegistryEntryInput, E as RegistryEntry, F as HttpMethod, G as WsConnection, H as SubstrateResponse, I as PluginRouter, K as WsHandler, P as HttpHandler, Q as PolicyDecision, R as RequestObserver, St as WriteResult, T as RegistryBase, U as SubstrateServer, V as SubstrateRequest, Y as Identity, _t as ReadResult, a as EngineHandle, ct as Dataset, dt as DropResult, et as RequestContext, g as NamespaceConfig, gt as ReadParams, lt as DatasetRegistry, mt as FilterCapabilities, n as DropParticipant, o as EngineInput, p as KernelPlugin, st as ConditionalUpdateResult, t as CoordinatorPlugin, ut as DeleteResult, vt as TxScopeHandles, w as Registry$1, xt as WriteOptions, yt as UpdateFieldRef, z as RuntimeOp } from "./plugin-t2g2wX0b.js";
4
4
  import { a as WsResponseFrame, f as IdempotencyStore, l as SweepDeps, s as BridgeRegistry, u as SweepState } from "./ws-C72iVS5T.js";
5
5
  import { t as Embedder } from "./types-8c0gtnd2.js";
6
+ import { Sql, TransactionSql } from "postgres";
6
7
  //#region ../../packages/substrate-compose/src/auth.d.ts
7
8
  type JwtAlg = 'RS256' | 'ES256' | 'HS256' | 'EdDSA';
8
9
  interface AuthKey {
@@ -1022,6 +1023,315 @@ interface ComposeResult {
1022
1023
  }
1023
1024
  export declare function compose(opts: ComposeOpts): Promise<ComposeResult>;
1024
1025
  //#endregion
1026
+ //#region ../../packages/substrate-dataset-pg/src/migrate.d.ts
1027
+ /**
1028
+ * Public migration shape accepted by the runner. Either supply both
1029
+ * `txStatements` + (optional) `concurrentStatements`, OR a legacy
1030
+ * `statements` bucket — the runner treats `statements` as `txStatements`
1031
+ * with no concurrent statements. Mixing `statements` and `txStatements`
1032
+ * is a configuration error.
1033
+ */
1034
+ type MigrationInput = {
1035
+ readonly version: string;
1036
+ readonly txStatements: ReadonlyArray<string>;
1037
+ readonly concurrentStatements?: ReadonlyArray<string>;
1038
+ readonly statements?: never;
1039
+ } | {
1040
+ readonly version: string;
1041
+ readonly statements: ReadonlyArray<string>;
1042
+ readonly txStatements?: never;
1043
+ readonly concurrentStatements?: never;
1044
+ };
1045
+ interface RunMigrationsOptions {
1046
+ /**
1047
+ * When true, drop the `migrations` tracking table before running.
1048
+ * Forces every migration to re-apply on this call. Intended for tests
1049
+ * that DROP schema objects between runs and need the framework to
1050
+ * recreate them; production paths should never set this.
1051
+ */
1052
+ force?: boolean;
1053
+ /**
1054
+ * Override the `SUBSTRATE_ENFORCE_UUID_CHECK=true` env-var gate for the gated
1055
+ * UUID-format CHECK constraint migration (0002). Defaults to whatever the
1056
+ * env says. Tests that need to exercise the constrained path explicitly
1057
+ * pass `true`; tests that need to opt out of an env-set gate pass `false`.
1058
+ * Production bootstraps shouldn't pass this — let the env decide.
1059
+ */
1060
+ enforceUuidCheck?: boolean;
1061
+ }
1062
+ export declare function runMigrations(sql: Sql, opts?: RunMigrationsOptions): Promise<void>;
1063
+ /**
1064
+ * One `field.list({ indexed: true })` or `field.json({ indexed: true })` field
1065
+ * that should be backed by a GIN over `(body -> field)`.
1066
+ */
1067
+ interface IndexedListField {
1068
+ readonly namespace: string;
1069
+ readonly type: string;
1070
+ readonly field: string;
1071
+ }
1072
+ /**
1073
+ * Derives the tracked, idempotent CONCURRENTLY GIN migration for one indexed
1074
+ * field. The version key uses the raw ns/type/field.
1075
+ */
1076
+ export declare function deriveListIndexMigration(f: IndexedListField): MigrationInput;
1077
+ /**
1078
+ * Applies migrations derived from field definitions through the same tracked,
1079
+ * advisory-locked machinery as the static list. Called after plugins load,
1080
+ * since `runMigrations` has no field-def knowledge. Idempotent: applied
1081
+ * versions are skipped.
1082
+ */
1083
+ export declare function runDerivedMigrations(sql: Sql, derived: ReadonlyArray<MigrationInput>): Promise<void>;
1084
+ //#endregion
1085
+ //#region ../../packages/substrate-dataset-pg/src/tx-scope.d.ts
1086
+ type PgHandle = Sql | TransactionSql;
1087
+ //#endregion
1088
+ //#region ../../packages/substrate-dataset-pg/src/pg-edges-store.d.ts
1089
+ interface PgEdgesStoreOptions {
1090
+ sql: Sql;
1091
+ /** Injected clock for tests. Defaults to `() => new Date()`. */
1092
+ now?: () => Date;
1093
+ /**
1094
+ * Per-statement batch size for chunked admin deletes. Defaults to
1095
+ * {@link DEFAULT_DROP_BATCH_SIZE}.
1096
+ */
1097
+ dropBatchSize?: number;
1098
+ }
1099
+ interface InternalPgEdgesStoreOptions {
1100
+ sql: PgHandle;
1101
+ now: () => Date;
1102
+ inTx: boolean;
1103
+ /**
1104
+ * Tx-bound instances inherit their parent's txScopeId. The public
1105
+ * constructor derives it from the Sql reference.
1106
+ */
1107
+ txScopeId?: string;
1108
+ dropBatchSize?: number;
1109
+ }
1110
+ export declare class PgEdgesStore implements EdgesStore {
1111
+ #private;
1112
+ readonly txScopeId: string;
1113
+ constructor(options: PgEdgesStoreOptions | InternalPgEdgesStoreOptions);
1114
+ write(input: EdgeWriteInput): Promise<Edge>;
1115
+ tombstone(id: string, at?: Date): Promise<void>;
1116
+ /**
1117
+ * Hard-deletes tombstoned edges whose `deleted_at` is older than
1118
+ * `olderThan`. Called by the sweep loop. Returns the number deleted.
1119
+ */
1120
+ purgeTombstonedEdges(opts: {
1121
+ olderThan: Date;
1122
+ }): Promise<{
1123
+ deleted: number;
1124
+ }>;
1125
+ update(patch: EdgePatch): Promise<void>;
1126
+ readNextBatch(cursorId: string | null, size: number): Promise<{
1127
+ edges: Edge[];
1128
+ lastId: string | null;
1129
+ }>;
1130
+ findEdgesTo(ref: EdgeRef): Promise<Edge[]>;
1131
+ findEdgesToMany(refs: readonly EdgeRef[]): Promise<Edge[]>;
1132
+ findEdgesFrom(ref: EdgeRef): Promise<Edge[]>;
1133
+ findEdgesTouchingNamespace(namespace: string, limit: number): Promise<Edge[]>;
1134
+ findEdgesTouchingType(namespace: string, type: string, limit: number): Promise<Edge[]>;
1135
+ namespacesLinkedTo(namespace: string, type?: string): Promise<string[]>;
1136
+ /**
1137
+ * Chunked physical delete of every edge touching `namespace`, at most
1138
+ * `dropBatchSize` rows per statement, so a large delete does not hold every
1139
+ * row lock at once. Returns the number deleted.
1140
+ *
1141
+ * Outside a tx each batch auto-commits. Inside a caller's tx the batches
1142
+ * stay in it, so a throw still rolls back the whole drop.
1143
+ */
1144
+ dropEdgesByNamespace(namespace: string, within?: readonly string[]): Promise<number>;
1145
+ /**
1146
+ * Chunked physical delete of every edge touching `(namespace, type)`.
1147
+ * See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
1148
+ */
1149
+ dropEdgesByType(namespace: string, type: string, within?: readonly string[]): Promise<number>;
1150
+ canDelete(ref: EdgeRef): Promise<CanDeleteResult>;
1151
+ findBlocked(): Promise<Edge[]>;
1152
+ /**
1153
+ * `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
1154
+ * nested re-entry skips the inner begin and runs on the existing tx handle
1155
+ * so an inner throw propagates to the outer `sql.begin` for rollback.
1156
+ */
1157
+ withTransaction<T>(fn: (tx: EdgesStore) => Promise<T>): Promise<T>;
1158
+ /**
1159
+ * Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
1160
+ * for `PgDataset.withScopedTransaction`. The caller has verified
1161
+ * the txScopeIds match.
1162
+ */
1163
+ _enlistInTx(txSql: PgHandle): PgEdgesStore;
1164
+ }
1165
+ //#endregion
1166
+ //#region ../../packages/substrate-dataset-pg/src/pg-sql/primitives.d.ts
1167
+ /**
1168
+ * A body key mirrored into a dedicated column on `rows`, so filters and
1169
+ * sorts on it hit an index instead of extracting from JSONB.
1170
+ *
1171
+ * `column` is a BARE column name, not a qualified `r.<col>` fragment: taking a
1172
+ * fragment from a caller would put caller text into `sql.unsafe`.
1173
+ */
1174
+ interface PromotedBodyColumn {
1175
+ column: string;
1176
+ kind: 'boolean' | 'text';
1177
+ }
1178
+ type PromotedBodyColumnMap = Readonly<Record<string, PromotedBodyColumn>>;
1179
+ //#endregion
1180
+ //#region ../../packages/substrate-dataset-pg/src/pg-dataset.d.ts
1181
+ /**
1182
+ * Returns the declared FieldDef for a type's field, or undefined when unknown
1183
+ * (body-key sorts then fall back to text compare). Lets the SQL builders cast
1184
+ * body values by declared kind, so `count` sorts 2 before 10 — see
1185
+ * `resolveBodyOrderByCast`.
1186
+ */
1187
+ type FieldDefLookup = (type: string, field: string) => FieldDef | undefined;
1188
+ interface PgDatasetOptions {
1189
+ namespace: string;
1190
+ sql: Sql;
1191
+ /**
1192
+ * Optional sibling `PgEdgesStore` to enlist into this dataset's substrate
1193
+ * transactions. When supplied, MUST share the same `Sql` pool (verified
1194
+ * via `txScopeId === txScopeIdFor(sql)`); the dataset throws on
1195
+ * construction otherwise.
1196
+ *
1197
+ * When supplied, `withScopedTransaction` yields a tx-bound edges store
1198
+ * on the same `TransactionSql` as the tx-bound datasets. When omitted, it
1199
+ * yields `edges: null` and the engine runs edge ops in a separate tx.
1200
+ */
1201
+ edgesStore?: PgEdgesStore;
1202
+ /**
1203
+ * Optional field-def lookup so orderBy + cursor compares on body fields can
1204
+ * cast to the declared scalar pg type (numeric / timestamptz / boolean)
1205
+ * instead of falling back to lexicographic text compare. When omitted, every
1206
+ * body-key orderBy compares as text.
1207
+ *
1208
+ * A cast sort cannot use an index on the uncast `(body ->> 'field')`
1209
+ * expression; it needs a matching cast expression index.
1210
+ */
1211
+ fieldDefLookup?: FieldDefLookup;
1212
+ /**
1213
+ * Body keys mirrored into dedicated columns on `rows`, so filters and
1214
+ * sorts on them use an index instead of extracting from JSONB.
1215
+ *
1216
+ * None by default. A namespace declares its own (`NamespaceDefinition`'s
1217
+ * `promotedColumns`), and the plugin that declares them adds the columns in
1218
+ * its kernel migrations: the engine creates none. Three things that bite:
1219
+ *
1220
+ * - Filters and sorts route to the column and ONLY the column, so a column
1221
+ * added over existing rows must be backfilled from the body first, or
1222
+ * those rows miss every filter. Reads return the body.
1223
+ * - `column` is a BARE name (`'severity'`, not `'r.severity'`) and is
1224
+ * validated as an identifier at construction, because it is interpolated
1225
+ * into SQL as an identifier.
1226
+ * - Text columns sort and range-compare `COLLATE "C"`, so an index meant
1227
+ * to serve those must be built `COLLATE "C"` too.
1228
+ */
1229
+ promotedColumns?: PromotedBodyColumnMap;
1230
+ }
1231
+ interface InternalPgSubstrateDatasetOptions {
1232
+ namespace: string;
1233
+ sql: PgHandle;
1234
+ inTx: boolean;
1235
+ /**
1236
+ * Tx-bound instances reuse the root's txScopeId so one substrate never
1237
+ * advertises two ids. The public constructor derives it from the Sql.
1238
+ */
1239
+ txScopeId?: string;
1240
+ edgesStore?: PgEdgesStore;
1241
+ fieldDefLookup?: FieldDefLookup;
1242
+ /** See `PgDatasetOptions.promotedColumns`. */
1243
+ promotedColumns?: PromotedBodyColumnMap;
1244
+ }
1245
+ export declare class PgDataset implements Dataset {
1246
+ #private;
1247
+ readonly namespace: string;
1248
+ readonly capabilities: FilterCapabilities;
1249
+ readonly txScopeId: string;
1250
+ constructor(options: PgDatasetOptions | InternalPgSubstrateDatasetOptions);
1251
+ /**
1252
+ * Reads rows of `type` per the §5.2 filter contract (where / orderBy /
1253
+ * cursor / limit). Every value is bound through a parameterised slot.
1254
+ *
1255
+ * Body values extract as text and are cast for comparison only; the filter
1256
+ * validator has already rejected type mismatches.
1257
+ */
1258
+ read(type: string, params: ReadParams): Promise<ReadResult>;
1259
+ write(type: string, rows: Row[], options?: WriteOptions): Promise<WriteResult>;
1260
+ tombstone(type: string, ids: Id[]): Promise<DeleteResult>;
1261
+ purge(type: string, ids: Id[]): Promise<DeleteResult>;
1262
+ /**
1263
+ * Hard-deletes tombstoned rows whose `deleted_at` is older than `olderThan`,
1264
+ * across ALL namespaces (the table is substrate-shared). Called by the sweep
1265
+ * loop. Returns the number of rows deleted.
1266
+ *
1267
+ * Short-id mappings are left in place, as in `purge()`: a row id can
1268
+ * reappear after a tombstone + rewrite.
1269
+ */
1270
+ purgeTombstonedRows(opts: {
1271
+ olderThan: Date;
1272
+ }): Promise<{
1273
+ deleted: number;
1274
+ }>;
1275
+ /**
1276
+ * Hard-delete every row for `type` in this namespace, including tombstoned
1277
+ * rows (`deleted_at IS NOT NULL`). Short-id mappings for that type are
1278
+ * removed in the same logical operation. Both deletes run through `this.#sql`
1279
+ * so they participate in any surrounding `withTransaction` boundary.
1280
+ */
1281
+ dropType(type: string): Promise<DropResult>;
1282
+ /**
1283
+ * Hard-delete every row across all types in this namespace, including
1284
+ * tombstoned rows. Short-id mappings for all types are removed too.
1285
+ */
1286
+ dropNamespace(): Promise<DropResult>;
1287
+ /**
1288
+ * Compare-and-update (spec §5.12): merges `updates` into the live row only
1289
+ * if every `match` field equals its expected value. Returns `matched` and
1290
+ * the updated row.
1291
+ *
1292
+ * One `UPDATE ... WHERE <pk> AND <match>`, so the row lock makes exactly one
1293
+ * of N concurrent callers match. `_deletedAt` and `id` are dropped from
1294
+ * `updates`.
1295
+ */
1296
+ conditionalUpdate(type: string, id: Id, match: Record<string, unknown>, updates: Record<string, unknown>, actor?: ActorIdentity, now?: Date): Promise<ConditionalUpdateResult>;
1297
+ updateField(ref: UpdateFieldRef, fieldName: string, value: unknown): Promise<void>;
1298
+ /**
1299
+ * `sql.begin`-backed atomicity. Spec §5.3 / G1.
1300
+ *
1301
+ * Nested calls flatten: postgres.js's `begin` inside a `begin` would create
1302
+ * a savepoint (inner rolls back independently of outer), which is stricter
1303
+ * than the G1 contract promises. To match the memory impl — where an inner
1304
+ * throw tears down outer writes too — we detect the re-entry via `#inTx`
1305
+ * and skip the nested begin entirely. The inner throw then propagates to
1306
+ * the outer `sql.begin`, which rolls back the whole thing.
1307
+ */
1308
+ withTransaction<T>(fn: (tx: Dataset) => Promise<T>): Promise<T>;
1309
+ /**
1310
+ * Runs `fn` in one pg transaction with a tx-bound dataset per requested
1311
+ * namespace (and the edges store, when wired), all on one `TransactionSql`.
1312
+ * A throw rolls back every namespace. Spec §5.3.
1313
+ *
1314
+ * Nested calls flatten onto the current transaction rather than opening a
1315
+ * savepoint.
1316
+ */
1317
+ withScopedTransaction<T>(namespaces: ReadonlyArray<string>, fn: (handles: TxScopeHandles) => Promise<T>): Promise<T>;
1318
+ /**
1319
+ * See `Dataset.withAdvisoryLock`. Excludes every session on the
1320
+ * database, across processes. On a tx-bound instance the lock is
1321
+ * `pg_advisory_xact_lock` on the transaction's own connection, held until
1322
+ * it commits or rolls back: a session lock would need a second connection
1323
+ * beside the open transaction.
1324
+ */
1325
+ withAdvisoryLock<T>(key: string, fn: () => Promise<T>): Promise<T>;
1326
+ allocateShortId(type: string): Promise<number>;
1327
+ bindShortId(type: string, shortId: number, rowId: string): Promise<void>;
1328
+ resolveShortId(type: string, shortId: number): Promise<string | null>;
1329
+ getShortIdForRow(type: string, rowId: string): Promise<number | null>;
1330
+ forNamespace(namespace: string): Dataset;
1331
+ private materialize;
1332
+ private extractOriginalId;
1333
+ }
1334
+ //#endregion
1025
1335
  //#region ../../packages/substrate-dataset-pg/src/testing.d.ts
1026
1336
  /**
1027
1337
  * Safety rails for pg integration tests. Shared by every package whose suites
@@ -1242,4 +1552,4 @@ export declare class MemorySubstrateServer implements SubstrateServer {
1242
1552
  */
1243
1553
  export declare function matchRoute(registered: string, actual: string): Record<string, string> | null;
1244
1554
  //#endregion
1245
- export type { AuthConfig, ComposeResult, MemoryWsClient };
1555
+ export type { AuthConfig, ComposeResult, FieldDefLookup, IndexedListField, MemoryWsClient, PgDatasetOptions, PgEdgesStoreOptions };
package/dist/testing.js CHANGED
@@ -1,5 +1,7 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
2
  import { N as MemoryEdgesStore, P as loadPlugins, at as WS_REASON_SERVER_RESTART, c as MemoryDataset, rt as signHs256, s as compose } from "./src-CHp3QyqB.js";
3
+ import { a as runDerivedMigrations, o as runMigrations, r as deriveListIndexMigration, t as PgDataset } from "./src-CXKnqeTW.js";
4
+ import { t as PgEdgesStore } from "./pg-edges-store-DKPYyd8l.js";
3
5
  import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
4
6
  import { randomUUID } from "node:crypto";
5
7
  import postgres from "postgres";
@@ -699,4 +701,4 @@ var MemorySubstrateServer = class {
699
701
  };
700
702
 
701
703
  //#endregion
702
- export { AlreadyStartedError, FakeEmbedder, MemoryDataset, MemoryEdgesStore, MemorySubstrateServer, UnknownWsPathError, WsClosedError, announcePgGating, assertCannotWriteOutsideTestDatabases, assertNotSuperuser, assertTestDatabase, compose, createMemoryWsPair, interpretSuperuserProbe, loadPlugins, matchRoute, pgTestSchemaName, pgTestSchemaUrl, preparePgTestSchema, publicLeftoverWarning, reachFailureMessage, signHs256 };
704
+ export { AlreadyStartedError, FakeEmbedder, MemoryDataset, MemoryEdgesStore, MemorySubstrateServer, PgDataset, PgEdgesStore, UnknownWsPathError, WsClosedError, announcePgGating, assertCannotWriteOutsideTestDatabases, assertNotSuperuser, assertTestDatabase, compose, createMemoryWsPair, deriveListIndexMigration, interpretSuperuserProbe, loadPlugins, matchRoute, pgTestSchemaName, pgTestSchemaUrl, preparePgTestSchema, publicLeftoverWarning, reachFailureMessage, runDerivedMigrations, runMigrations, signHs256 };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mnmal-ai/substrate",
3
- "version": "1.0.0-alpha.2",
3
+ "version": "1.0.0-alpha.3",
4
4
  "description": "Substrate — an agent-first typed-data server: the substrate and substrate-admin binaries, the built-in plugins and the SDK subpaths.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -36,24 +36,24 @@
36
36
  },
37
37
  "devDependencies": {
38
38
  "@inquirer/prompts": "^8.7.2",
39
- "@mnmal-ai/substrate-bootstrap": "1.0.0-alpha.2",
40
- "@mnmal-ai/substrate-client": "1.0.0-alpha.2",
41
- "@mnmal-ai/substrate-compose": "1.0.0-alpha.2",
42
- "@mnmal-ai/substrate-config": "1.0.0-alpha.2",
43
- "@mnmal-ai/substrate-core": "1.0.0-alpha.2",
44
- "@mnmal-ai/substrate-dataset-memory": "1.0.0-alpha.2",
45
- "@mnmal-ai/substrate-dataset-pg": "1.0.0-alpha.2",
46
- "@mnmal-ai/substrate-embedder": "1.0.0-alpha.2",
47
- "@mnmal-ai/substrate-filter": "1.0.0-alpha.2",
48
- "@mnmal-ai/substrate-pack": "1.0.0-alpha.2",
49
- "@mnmal-ai/substrate-policy": "1.0.0-alpha.2",
50
- "@mnmal-ai/substrate-proposals": "1.0.0-alpha.2",
51
- "@mnmal-ai/substrate-recall": "1.0.0-alpha.2",
52
- "@mnmal-ai/substrate-sdk": "1.0.0-alpha.2",
53
- "@mnmal-ai/substrate-sentinel-client": "1.0.0-alpha.2",
54
- "@mnmal-ai/substrate-server-fastify": "1.0.0-alpha.2",
55
- "@mnmal-ai/substrate-server-memory": "1.0.0-alpha.2",
56
- "@mnmal-ai/substrate-types": "1.0.0-alpha.2",
39
+ "@mnmal-ai/substrate-bootstrap": "1.0.0-alpha.3",
40
+ "@mnmal-ai/substrate-client": "1.0.0-alpha.3",
41
+ "@mnmal-ai/substrate-compose": "1.0.0-alpha.3",
42
+ "@mnmal-ai/substrate-config": "1.0.0-alpha.3",
43
+ "@mnmal-ai/substrate-core": "1.0.0-alpha.3",
44
+ "@mnmal-ai/substrate-dataset-memory": "1.0.0-alpha.3",
45
+ "@mnmal-ai/substrate-dataset-pg": "1.0.0-alpha.3",
46
+ "@mnmal-ai/substrate-embedder": "1.0.0-alpha.3",
47
+ "@mnmal-ai/substrate-filter": "1.0.0-alpha.3",
48
+ "@mnmal-ai/substrate-pack": "1.0.0-alpha.3",
49
+ "@mnmal-ai/substrate-policy": "1.0.0-alpha.3",
50
+ "@mnmal-ai/substrate-proposals": "1.0.0-alpha.3",
51
+ "@mnmal-ai/substrate-recall": "1.0.0-alpha.3",
52
+ "@mnmal-ai/substrate-sdk": "1.0.0-alpha.3",
53
+ "@mnmal-ai/substrate-sentinel-client": "1.0.0-alpha.3",
54
+ "@mnmal-ai/substrate-server-fastify": "1.0.0-alpha.3",
55
+ "@mnmal-ai/substrate-server-memory": "1.0.0-alpha.3",
56
+ "@mnmal-ai/substrate-types": "1.0.0-alpha.3",
57
57
  "@types/node": "^26.6.4",
58
58
  "@types/semver": "^7.5.0",
59
59
  "@types/ws": "^8.18.1",