@mnmal-ai/substrate 1.0.0-alpha.2 → 1.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{dataset-BQ8egGl5.js → dataset-BBZuhrsL.js} +1 -1
- package/dist/pg-edges-store-DKPYyd8l.js +381 -0
- package/dist/recall.js +1 -1
- package/dist/{src-DWm0iA88.js → src-B-wkCHYL.js} +2 -2
- package/dist/substrate-admin.js +1 -1
- package/dist/substrate.js +4 -379
- package/dist/testing.d.ts +312 -2
- package/dist/testing.js +3 -1
- package/package.json +19 -19
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ It runs as a real install: the binary and its plugins resolve from one shared `n
|
|
|
11
11
|
| `@mnmal-ai/substrate/sdk` | Plugin authoring: `defineNamespace`, the field DSL, autoCRUD, engine helpers, `defineBootstrap`, policy combinators, route helpers, and all shared types |
|
|
12
12
|
| `@mnmal-ai/substrate/client` | `SubstrateClient`, `watchRows`, client errors, the admin API, and the token and grant helpers |
|
|
13
13
|
| `@mnmal-ai/substrate/pack` | The MessagePack wire codec |
|
|
14
|
-
| `@mnmal-ai/substrate/testing` | `compose`, `MemorySubstrateServer`, `MemoryDataset`, `MemoryEdgesStore`, `loadPlugins`, `FakeEmbedder`, and the Postgres test-database guards |
|
|
14
|
+
| `@mnmal-ai/substrate/testing` | `compose`, `MemorySubstrateServer`, `MemoryDataset`, `MemoryEdgesStore`, `loadPlugins`, `FakeEmbedder`, `PgDataset`, `PgEdgesStore`, the pg migration runners, and the Postgres test-database guards |
|
|
15
15
|
| `@mnmal-ai/substrate/recall` | The recall plugin's primitives (`EmbeddingsStore`, `LexicalStore`, `embedAndStore`, …) |
|
|
16
16
|
| `@mnmal-ai/substrate/proposals` | The proposals plugin's exports |
|
|
17
17
|
|
|
@@ -11,7 +11,7 @@ import { createHash, createPublicKey, generateKeyPairSync, randomUUID } from "no
|
|
|
11
11
|
import postgres from "postgres";
|
|
12
12
|
|
|
13
13
|
//#region package.json
|
|
14
|
-
var version = "1.0.0-alpha.
|
|
14
|
+
var version = "1.0.0-alpha.3";
|
|
15
15
|
|
|
16
16
|
//#endregion
|
|
17
17
|
//#region src/admin/errors.ts
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { n as txScopeIdFor } from "./src-CXKnqeTW.js";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
|
|
5
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-edges-store.ts
|
|
6
|
+
/**
|
|
7
|
+
* PgEdgesStore — PostgreSQL-backed reference implementation of EdgesStore.
|
|
8
|
+
* Spec: §4.6, §6.2, §6.3.
|
|
9
|
+
*
|
|
10
|
+
* One table (`edges`) keyed by a text PK, no namespace scoping — edges
|
|
11
|
+
* are engine-scoped (§4.6), not per-ns. The caller owns the `postgres` Sql
|
|
12
|
+
* connection; this class never opens or closes pools.
|
|
13
|
+
*
|
|
14
|
+
* Edge ids are `crypto.randomUUID()`: unique, so cursor pagination works, but
|
|
15
|
+
* not time-ordered. The sweep walks the whole table, so it doesn't need order.
|
|
16
|
+
*
|
|
17
|
+
* Edge.from/to .id is `string | number`, stored and read back as text (unlike
|
|
18
|
+
* PgDataset, which preserves numeric ids).
|
|
19
|
+
*
|
|
20
|
+
* SQL safety: postgres.js tagged templates parameterise every value slot.
|
|
21
|
+
* Never string-concat caller input — the `update` dynamic column list uses a
|
|
22
|
+
* fixed allowlist of column names, not user-supplied keys.
|
|
23
|
+
*/
|
|
24
|
+
const MUTABLE_PATCH_FIELDS = /* @__PURE__ */ new Set([
|
|
25
|
+
"id",
|
|
26
|
+
"lastSweptAt",
|
|
27
|
+
"blocked"
|
|
28
|
+
]);
|
|
29
|
+
/**
|
|
30
|
+
* Batch size for chunked admin deletes (`dropEdgesByNamespace`,
|
|
31
|
+
* `dropEdgesByType`): bounds per-statement lock hold on large tables.
|
|
32
|
+
*/
|
|
33
|
+
const DEFAULT_DROP_BATCH_SIZE = 5e3;
|
|
34
|
+
const VALID_POLICIES = /* @__PURE__ */ new Set([
|
|
35
|
+
"set_null",
|
|
36
|
+
"cascade",
|
|
37
|
+
"restrict",
|
|
38
|
+
"orphan"
|
|
39
|
+
]);
|
|
40
|
+
const refToId = (id) => typeof id === "number" ? String(id) : id;
|
|
41
|
+
var PgEdgesStore = class PgEdgesStore {
|
|
42
|
+
#sql;
|
|
43
|
+
#now;
|
|
44
|
+
#inTx;
|
|
45
|
+
#dropBatchSize;
|
|
46
|
+
txScopeId;
|
|
47
|
+
get #edgeCols() {
|
|
48
|
+
return this.#sql`id, from_ns, from_type, from_id, from_bridged, to_ns, to_type, to_id, to_bridged, field_name, policy, created_at, deleted_at, last_swept_at, blocked`;
|
|
49
|
+
}
|
|
50
|
+
constructor(options) {
|
|
51
|
+
this.#sql = options.sql;
|
|
52
|
+
this.#now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
53
|
+
this.#inTx = "inTx" in options ? options.inTx : false;
|
|
54
|
+
const explicit = "txScopeId" in options ? options.txScopeId : void 0;
|
|
55
|
+
this.txScopeId = explicit ?? txScopeIdFor(options.sql);
|
|
56
|
+
const batch = options.dropBatchSize;
|
|
57
|
+
if (batch !== void 0 && (!Number.isFinite(batch) || batch <= 0)) throw new Error(`PgEdgesStore: dropBatchSize must be a positive finite number, got ${batch}`);
|
|
58
|
+
this.#dropBatchSize = batch ?? DEFAULT_DROP_BATCH_SIZE;
|
|
59
|
+
}
|
|
60
|
+
async write(input) {
|
|
61
|
+
const id = randomUUID();
|
|
62
|
+
const createdAt = this.#now();
|
|
63
|
+
const fromBridged = input.from.bridged === true;
|
|
64
|
+
const toBridged = input.to.bridged === true;
|
|
65
|
+
await this.#sql`
|
|
66
|
+
INSERT INTO edges (
|
|
67
|
+
id, from_ns, from_type, from_id, from_bridged,
|
|
68
|
+
to_ns, to_type, to_id, to_bridged,
|
|
69
|
+
field_name, policy, created_at, deleted_at, last_swept_at, blocked
|
|
70
|
+
)
|
|
71
|
+
VALUES (
|
|
72
|
+
${id},
|
|
73
|
+
${input.from.ns}, ${input.from.type}, ${refToId(input.from.id)}, ${fromBridged},
|
|
74
|
+
${input.to.ns}, ${input.to.type}, ${refToId(input.to.id)}, ${toBridged},
|
|
75
|
+
${input.fieldName}, ${input.policy}, ${createdAt},
|
|
76
|
+
NULL, NULL, false
|
|
77
|
+
)
|
|
78
|
+
`;
|
|
79
|
+
return {
|
|
80
|
+
id,
|
|
81
|
+
from: buildRef(input.from.ns, input.from.type, refToId(input.from.id), fromBridged),
|
|
82
|
+
to: buildRef(input.to.ns, input.to.type, refToId(input.to.id), toBridged),
|
|
83
|
+
fieldName: input.fieldName,
|
|
84
|
+
policy: input.policy,
|
|
85
|
+
createdAt
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
async tombstone(id, at) {
|
|
89
|
+
const stamp = at ?? this.#now();
|
|
90
|
+
await this.#sql`
|
|
91
|
+
UPDATE edges
|
|
92
|
+
SET deleted_at = ${stamp}
|
|
93
|
+
WHERE id = ${id}
|
|
94
|
+
AND deleted_at IS NULL
|
|
95
|
+
`;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Hard-deletes tombstoned edges whose `deleted_at` is older than
|
|
99
|
+
* `olderThan`. Called by the sweep loop. Returns the number deleted.
|
|
100
|
+
*/
|
|
101
|
+
async purgeTombstonedEdges(opts) {
|
|
102
|
+
return { deleted: (await this.#sql`
|
|
103
|
+
DELETE FROM edges
|
|
104
|
+
WHERE deleted_at IS NOT NULL
|
|
105
|
+
AND deleted_at < ${opts.olderThan}
|
|
106
|
+
RETURNING id
|
|
107
|
+
`).length };
|
|
108
|
+
}
|
|
109
|
+
async update(patch) {
|
|
110
|
+
for (const key of Object.keys(patch)) if (!MUTABLE_PATCH_FIELDS.has(key)) throw new Error(`EdgesStore.update: field '${key}' is immutable`);
|
|
111
|
+
const hasLastSweptAt = Object.hasOwn(patch, "lastSweptAt");
|
|
112
|
+
const hasBlocked = Object.hasOwn(patch, "blocked");
|
|
113
|
+
if (!hasLastSweptAt && !hasBlocked) return;
|
|
114
|
+
if (hasLastSweptAt && hasBlocked) {
|
|
115
|
+
await this.#sql`
|
|
116
|
+
UPDATE edges
|
|
117
|
+
SET last_swept_at = ${patch.lastSweptAt ?? null},
|
|
118
|
+
blocked = ${patch.blocked ?? false}
|
|
119
|
+
WHERE id = ${patch.id}
|
|
120
|
+
`;
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (hasLastSweptAt) {
|
|
124
|
+
await this.#sql`
|
|
125
|
+
UPDATE edges
|
|
126
|
+
SET last_swept_at = ${patch.lastSweptAt ?? null}
|
|
127
|
+
WHERE id = ${patch.id}
|
|
128
|
+
`;
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
await this.#sql`
|
|
132
|
+
UPDATE edges
|
|
133
|
+
SET blocked = ${patch.blocked ?? false}
|
|
134
|
+
WHERE id = ${patch.id}
|
|
135
|
+
`;
|
|
136
|
+
}
|
|
137
|
+
async readNextBatch(cursorId, size) {
|
|
138
|
+
const edges = (cursorId === null ? await this.#sql`
|
|
139
|
+
SELECT ${this.#edgeCols}
|
|
140
|
+
FROM edges
|
|
141
|
+
ORDER BY id ASC
|
|
142
|
+
LIMIT ${size}
|
|
143
|
+
` : await this.#sql`
|
|
144
|
+
SELECT ${this.#edgeCols}
|
|
145
|
+
FROM edges
|
|
146
|
+
WHERE id > ${cursorId}
|
|
147
|
+
ORDER BY id ASC
|
|
148
|
+
LIMIT ${size}
|
|
149
|
+
`).map(rowToEdge);
|
|
150
|
+
const last = edges[edges.length - 1];
|
|
151
|
+
return {
|
|
152
|
+
edges,
|
|
153
|
+
lastId: last ? last.id : null
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
async findEdgesTo(ref) {
|
|
157
|
+
return (await this.#sql`
|
|
158
|
+
SELECT ${this.#edgeCols}
|
|
159
|
+
FROM edges
|
|
160
|
+
WHERE to_ns = ${ref.ns}
|
|
161
|
+
AND to_type = ${ref.type}
|
|
162
|
+
AND to_id = ${refToId(ref.id)}
|
|
163
|
+
AND deleted_at IS NULL
|
|
164
|
+
`).map(rowToEdge);
|
|
165
|
+
}
|
|
166
|
+
async findEdgesToMany(refs) {
|
|
167
|
+
if (refs.length === 0) return [];
|
|
168
|
+
const sql = this.#sql;
|
|
169
|
+
const seenRef = /* @__PURE__ */ new Set();
|
|
170
|
+
const toNs = [];
|
|
171
|
+
const toType = [];
|
|
172
|
+
const toId = [];
|
|
173
|
+
for (const r of refs) {
|
|
174
|
+
const id = refToId(r.id);
|
|
175
|
+
const k = `${r.ns}\u0000${r.type}\u0000${id}`;
|
|
176
|
+
if (seenRef.has(k)) continue;
|
|
177
|
+
seenRef.add(k);
|
|
178
|
+
toNs.push(r.ns);
|
|
179
|
+
toType.push(r.type);
|
|
180
|
+
toId.push(id);
|
|
181
|
+
}
|
|
182
|
+
return (await sql`
|
|
183
|
+
SELECT ${this.#edgeCols}
|
|
184
|
+
FROM edges
|
|
185
|
+
JOIN unnest(${toNs}::text[], ${toType}::text[], ${toId}::text[])
|
|
186
|
+
AS seek(seek_ns, seek_type, seek_id)
|
|
187
|
+
ON to_ns = seek.seek_ns AND to_type = seek.seek_type AND to_id = seek.seek_id
|
|
188
|
+
WHERE deleted_at IS NULL
|
|
189
|
+
`).map(rowToEdge);
|
|
190
|
+
}
|
|
191
|
+
async findEdgesFrom(ref) {
|
|
192
|
+
return (await this.#sql`
|
|
193
|
+
SELECT ${this.#edgeCols}
|
|
194
|
+
FROM edges
|
|
195
|
+
WHERE from_ns = ${ref.ns}
|
|
196
|
+
AND from_type = ${ref.type}
|
|
197
|
+
AND from_id = ${refToId(ref.id)}
|
|
198
|
+
AND deleted_at IS NULL
|
|
199
|
+
`).map(rowToEdge);
|
|
200
|
+
}
|
|
201
|
+
async findEdgesTouchingNamespace(namespace, limit) {
|
|
202
|
+
return (await this.#sql`
|
|
203
|
+
SELECT ${this.#edgeCols}
|
|
204
|
+
FROM edges
|
|
205
|
+
WHERE deleted_at IS NULL
|
|
206
|
+
AND (from_ns = ${namespace} OR to_ns = ${namespace})
|
|
207
|
+
ORDER BY id ASC
|
|
208
|
+
LIMIT ${limit}
|
|
209
|
+
`).map(rowToEdge);
|
|
210
|
+
}
|
|
211
|
+
async findEdgesTouchingType(namespace, type, limit) {
|
|
212
|
+
return (await this.#sql`
|
|
213
|
+
SELECT ${this.#edgeCols}
|
|
214
|
+
FROM edges
|
|
215
|
+
WHERE deleted_at IS NULL
|
|
216
|
+
AND ((from_ns = ${namespace} AND from_type = ${type})
|
|
217
|
+
OR (to_ns = ${namespace} AND to_type = ${type}))
|
|
218
|
+
ORDER BY id ASC
|
|
219
|
+
LIMIT ${limit}
|
|
220
|
+
`).map(rowToEdge);
|
|
221
|
+
}
|
|
222
|
+
async namespacesLinkedTo(namespace, type) {
|
|
223
|
+
const fromMatch = type === void 0 ? this.#sql`from_ns = ${namespace}` : this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
|
|
224
|
+
const toMatch = type === void 0 ? this.#sql`to_ns = ${namespace}` : this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
|
|
225
|
+
return (await this.#sql`
|
|
226
|
+
SELECT ns FROM (
|
|
227
|
+
SELECT to_ns AS ns FROM edges WHERE ${fromMatch}
|
|
228
|
+
UNION
|
|
229
|
+
SELECT from_ns AS ns FROM edges WHERE ${toMatch}
|
|
230
|
+
) linked
|
|
231
|
+
WHERE ns <> ${namespace}
|
|
232
|
+
ORDER BY ns
|
|
233
|
+
`).map((r) => r.ns);
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Chunked physical delete of every edge touching `namespace`, at most
|
|
237
|
+
* `dropBatchSize` rows per statement, so a large delete does not hold every
|
|
238
|
+
* row lock at once. Returns the number deleted.
|
|
239
|
+
*
|
|
240
|
+
* Outside a tx each batch auto-commits. Inside a caller's tx the batches
|
|
241
|
+
* stay in it, so a throw still rolls back the whole drop.
|
|
242
|
+
*/
|
|
243
|
+
async dropEdgesByNamespace(namespace, within) {
|
|
244
|
+
const batchSize = this.#dropBatchSize;
|
|
245
|
+
const farEnd = this.#farEndWithin(namespace, within, this.#sql`from_ns = ${namespace}`, this.#sql`to_ns = ${namespace}`);
|
|
246
|
+
let total = 0;
|
|
247
|
+
while (true) {
|
|
248
|
+
const rows = await this.#sql`
|
|
249
|
+
DELETE FROM edges
|
|
250
|
+
WHERE id IN (
|
|
251
|
+
SELECT id FROM edges
|
|
252
|
+
WHERE (from_ns = ${namespace} OR to_ns = ${namespace}) ${farEnd}
|
|
253
|
+
LIMIT ${batchSize}
|
|
254
|
+
)
|
|
255
|
+
RETURNING id
|
|
256
|
+
`;
|
|
257
|
+
total += rows.length;
|
|
258
|
+
if (rows.length < batchSize) return total;
|
|
259
|
+
await this.#yieldBetweenBatches();
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Chunked physical delete of every edge touching `(namespace, type)`.
|
|
264
|
+
* See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
|
|
265
|
+
*/
|
|
266
|
+
async dropEdgesByType(namespace, type, within) {
|
|
267
|
+
const batchSize = this.#dropBatchSize;
|
|
268
|
+
const fromMatch = this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
|
|
269
|
+
const toMatch = this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
|
|
270
|
+
const farEnd = this.#farEndWithin(namespace, within, fromMatch, toMatch);
|
|
271
|
+
let total = 0;
|
|
272
|
+
while (true) {
|
|
273
|
+
const rows = await this.#sql`
|
|
274
|
+
DELETE FROM edges
|
|
275
|
+
WHERE id IN (
|
|
276
|
+
SELECT id FROM edges
|
|
277
|
+
WHERE (${fromMatch} OR ${toMatch}) ${farEnd}
|
|
278
|
+
LIMIT ${batchSize}
|
|
279
|
+
)
|
|
280
|
+
RETURNING id
|
|
281
|
+
`;
|
|
282
|
+
total += rows.length;
|
|
283
|
+
if (rows.length < batchSize) return total;
|
|
284
|
+
await this.#yieldBetweenBatches();
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
/** `AND` clause keeping only edges whose far end is `namespace` or in `within`. */
|
|
288
|
+
#farEndWithin(namespace, within, fromMatch, toMatch) {
|
|
289
|
+
if (within === void 0) return this.#sql``;
|
|
290
|
+
const allowed = this.#sql([namespace, ...within]);
|
|
291
|
+
return this.#sql`AND ((${fromMatch} AND to_ns IN ${allowed}) OR (${toMatch} AND from_ns IN ${allowed}))`;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Yields to the event loop between drop batches outside a tx; no-op inside one.
|
|
295
|
+
*/
|
|
296
|
+
async #yieldBetweenBatches() {
|
|
297
|
+
if (!this.#inTx) await new Promise((resolve) => setImmediate(resolve));
|
|
298
|
+
}
|
|
299
|
+
async canDelete(ref) {
|
|
300
|
+
const rows = await this.#sql`
|
|
301
|
+
SELECT ${this.#edgeCols}
|
|
302
|
+
FROM edges
|
|
303
|
+
WHERE to_ns = ${ref.ns}
|
|
304
|
+
AND to_type = ${ref.type}
|
|
305
|
+
AND to_id = ${refToId(ref.id)}
|
|
306
|
+
AND policy = 'restrict'
|
|
307
|
+
AND deleted_at IS NULL
|
|
308
|
+
`;
|
|
309
|
+
if (rows.length === 0) return { ok: true };
|
|
310
|
+
return {
|
|
311
|
+
ok: false,
|
|
312
|
+
blockedBy: rows.map(rowToEdge)
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
async findBlocked() {
|
|
316
|
+
return (await this.#sql`
|
|
317
|
+
SELECT ${this.#edgeCols}
|
|
318
|
+
FROM edges
|
|
319
|
+
WHERE blocked = true
|
|
320
|
+
AND deleted_at IS NULL
|
|
321
|
+
`).map(rowToEdge);
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
|
|
325
|
+
* nested re-entry skips the inner begin and runs on the existing tx handle
|
|
326
|
+
* so an inner throw propagates to the outer `sql.begin` for rollback.
|
|
327
|
+
*/
|
|
328
|
+
async withTransaction(fn) {
|
|
329
|
+
if (this.#inTx) return fn(this);
|
|
330
|
+
return this.#sql.begin(async (txSql) => {
|
|
331
|
+
return fn(new PgEdgesStore({
|
|
332
|
+
sql: txSql,
|
|
333
|
+
now: this.#now,
|
|
334
|
+
inTx: true,
|
|
335
|
+
txScopeId: this.txScopeId,
|
|
336
|
+
dropBatchSize: this.#dropBatchSize
|
|
337
|
+
}));
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
|
|
342
|
+
* for `PgDataset.withScopedTransaction`. The caller has verified
|
|
343
|
+
* the txScopeIds match.
|
|
344
|
+
*/
|
|
345
|
+
_enlistInTx(txSql) {
|
|
346
|
+
return new PgEdgesStore({
|
|
347
|
+
sql: txSql,
|
|
348
|
+
now: this.#now,
|
|
349
|
+
inTx: true,
|
|
350
|
+
txScopeId: this.txScopeId,
|
|
351
|
+
dropBatchSize: this.#dropBatchSize
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
function buildRef(ns, type, id, bridged) {
|
|
356
|
+
const ref = {
|
|
357
|
+
ns,
|
|
358
|
+
type,
|
|
359
|
+
id
|
|
360
|
+
};
|
|
361
|
+
if (bridged) ref.bridged = true;
|
|
362
|
+
return ref;
|
|
363
|
+
}
|
|
364
|
+
function rowToEdge(row) {
|
|
365
|
+
if (!VALID_POLICIES.has(row.policy)) throw new Error(`PgEdgesStore: unknown policy '${row.policy}' on edge id=${row.id}`);
|
|
366
|
+
const edge = {
|
|
367
|
+
id: row.id,
|
|
368
|
+
from: buildRef(row.from_ns, row.from_type, row.from_id, row.from_bridged),
|
|
369
|
+
to: buildRef(row.to_ns, row.to_type, row.to_id, row.to_bridged),
|
|
370
|
+
fieldName: row.field_name,
|
|
371
|
+
policy: row.policy,
|
|
372
|
+
createdAt: row.created_at
|
|
373
|
+
};
|
|
374
|
+
if (row.deleted_at !== null) edge.deletedAt = row.deleted_at;
|
|
375
|
+
if (row.last_swept_at !== null) edge.lastSweptAt = row.last_swept_at;
|
|
376
|
+
if (row.blocked) edge.blocked = true;
|
|
377
|
+
return edge;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
//#endregion
|
|
381
|
+
export { PgEdgesStore as t };
|
package/dist/recall.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-
|
|
2
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-B-wkCHYL.js";
|
|
3
3
|
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-QuJN9pG_.js";
|
|
4
4
|
|
|
5
5
|
export { CLUSTERER_BACKEND, CLUSTERER_BACKEND_VERSION, CLUSTER_FIELDS, CLUSTER_MEMBER_FIELDS, CRYSTALLISE_IN_FLIGHT_CODE, DEFAULT_CRYSTALLISE_CONFIG, EMBEDDINGS_FETCH_MAX_REFS, EmbedSourceRegistry, InMemoryClusterStore, InMemoryEmbeddingsStore, InMemoryLexicalStore, PgClusterStore, PgEmbeddingsStore, PgLexicalStore, RECALL_VERSION, REINDEX_DEFAULT_LIMIT, REINDEX_MAX_LIMIT, RRF_K, SALIENCE_BOOST_MAX_TARGETS, SELECT_CANDIDATES_MAX_LIMIT, armRanks, buildClusterSurface, checkParams, createCrystalliseGuard, createEmbedStats, createRecallPlugin, crystalliseRouteHandler, embedAndStore, embedRowReporting, embedStatsRouteHandler, embeddingSimilarOperator, embeddingsFetchRouteHandler, embeddingsMigration, fuseByRank, handleCrystallise, handleEmbedStatsRequest, handleLexicalCoverage, handleRecallRequest, handleReindex, l2normalize, lexicalMigration, makeEmbedPostWrite, makeLexicalClassify, makeLexicalPostWrite, makeReembed, memberKey, recallClustersMigration, recallMcpTool, recallRouteHandler, reindexRouteHandler, runClusterer, salienceBoostRouteHandler, unknownParamMessage, unknownParams, validateCrystalliseInput, validateLexicalCoverageInput, validateReindexInput, validateSelector };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-QuJN9pG_.js";
|
|
3
2
|
import { n as txScopeIdFor } from "./src-CXKnqeTW.js";
|
|
3
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-QuJN9pG_.js";
|
|
4
4
|
|
|
5
5
|
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
-
var version = "1.0.0-alpha.
|
|
6
|
+
var version = "1.0.0-alpha.3";
|
|
7
7
|
|
|
8
8
|
//#endregion
|
|
9
9
|
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
package/dist/substrate-admin.js
CHANGED
|
@@ -4,7 +4,7 @@ import { d as buildAuditRow, f as AUDIT_EVENT_TYPE } from "./src-CHp3QyqB.js";
|
|
|
4
4
|
import { d as resolveDefaultKidPath, u as resolveDefaultKeyPath } from "./env-utils-CGNazUNt.js";
|
|
5
5
|
import { E as describeGrant, I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./row-DZ0Fnpc6.js";
|
|
6
6
|
import { S as reconcileIdentities } from "./src-DMzc3HfW.js";
|
|
7
|
-
import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-
|
|
7
|
+
import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-BBZuhrsL.js";
|
|
8
8
|
import { existsSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
9
9
|
import { dirname } from "node:path";
|
|
10
10
|
import { createHash, randomUUID } from "node:crypto";
|
package/dist/substrate.js
CHANGED
|
@@ -4,9 +4,10 @@ import { B as saveSweepState, G as registerExtensionOperator, I as effectiveFilt
|
|
|
4
4
|
import { i as DEFAULT_CAPABILITIES, n as trimTrailingSlashes, s as makeAuthorizationProvider, t as parsePositiveInt } from "./env-utils-CGNazUNt.js";
|
|
5
5
|
import { L as encodeList, T as SYSTEM_NAMESPACE, f as coversNamespace, m as grantedIn, p as grantOf } from "./row-DZ0Fnpc6.js";
|
|
6
6
|
import { c as AuthError, d as AdminBackfillError, l as NetworkError, n as tokenAudience, r as SubstrateClient, t as noRedirectFetch } from "./src-DMzc3HfW.js";
|
|
7
|
-
import { a as
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
7
|
+
import { a as runDerivedMigrations, o as runMigrations, r as deriveListIndexMigration, s as tryWithPgAdvisoryLock, t as PgDataset } from "./src-CXKnqeTW.js";
|
|
8
|
+
import { t as PgEdgesStore } from "./pg-edges-store-DKPYyd8l.js";
|
|
9
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore } from "./src-B-wkCHYL.js";
|
|
10
|
+
import { B as version, R as openSql, t as openPgDataset, z as AdminCliError } from "./dataset-BBZuhrsL.js";
|
|
10
11
|
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
11
12
|
import { homedir, hostname } from "node:os";
|
|
12
13
|
import { basename, dirname, join, resolve } from "node:path";
|
|
@@ -292,382 +293,6 @@ var MemoryIdempotencyStore = class {
|
|
|
292
293
|
}
|
|
293
294
|
};
|
|
294
295
|
|
|
295
|
-
//#endregion
|
|
296
|
-
//#region ../../packages/substrate-dataset-pg/src/pg-edges-store.ts
|
|
297
|
-
/**
|
|
298
|
-
* PgEdgesStore — PostgreSQL-backed reference implementation of EdgesStore.
|
|
299
|
-
* Spec: §4.6, §6.2, §6.3.
|
|
300
|
-
*
|
|
301
|
-
* One table (`edges`) keyed by a text PK, no namespace scoping — edges
|
|
302
|
-
* are engine-scoped (§4.6), not per-ns. The caller owns the `postgres` Sql
|
|
303
|
-
* connection; this class never opens or closes pools.
|
|
304
|
-
*
|
|
305
|
-
* Edge ids are `crypto.randomUUID()`: unique, so cursor pagination works, but
|
|
306
|
-
* not time-ordered. The sweep walks the whole table, so it doesn't need order.
|
|
307
|
-
*
|
|
308
|
-
* Edge.from/to .id is `string | number`, stored and read back as text (unlike
|
|
309
|
-
* PgDataset, which preserves numeric ids).
|
|
310
|
-
*
|
|
311
|
-
* SQL safety: postgres.js tagged templates parameterise every value slot.
|
|
312
|
-
* Never string-concat caller input — the `update` dynamic column list uses a
|
|
313
|
-
* fixed allowlist of column names, not user-supplied keys.
|
|
314
|
-
*/
|
|
315
|
-
const MUTABLE_PATCH_FIELDS = /* @__PURE__ */ new Set([
|
|
316
|
-
"id",
|
|
317
|
-
"lastSweptAt",
|
|
318
|
-
"blocked"
|
|
319
|
-
]);
|
|
320
|
-
/**
|
|
321
|
-
* Batch size for chunked admin deletes (`dropEdgesByNamespace`,
|
|
322
|
-
* `dropEdgesByType`): bounds per-statement lock hold on large tables.
|
|
323
|
-
*/
|
|
324
|
-
const DEFAULT_DROP_BATCH_SIZE = 5e3;
|
|
325
|
-
const VALID_POLICIES = /* @__PURE__ */ new Set([
|
|
326
|
-
"set_null",
|
|
327
|
-
"cascade",
|
|
328
|
-
"restrict",
|
|
329
|
-
"orphan"
|
|
330
|
-
]);
|
|
331
|
-
const refToId = (id) => typeof id === "number" ? String(id) : id;
|
|
332
|
-
var PgEdgesStore = class PgEdgesStore {
|
|
333
|
-
#sql;
|
|
334
|
-
#now;
|
|
335
|
-
#inTx;
|
|
336
|
-
#dropBatchSize;
|
|
337
|
-
txScopeId;
|
|
338
|
-
get #edgeCols() {
|
|
339
|
-
return this.#sql`id, from_ns, from_type, from_id, from_bridged, to_ns, to_type, to_id, to_bridged, field_name, policy, created_at, deleted_at, last_swept_at, blocked`;
|
|
340
|
-
}
|
|
341
|
-
constructor(options) {
|
|
342
|
-
this.#sql = options.sql;
|
|
343
|
-
this.#now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
344
|
-
this.#inTx = "inTx" in options ? options.inTx : false;
|
|
345
|
-
const explicit = "txScopeId" in options ? options.txScopeId : void 0;
|
|
346
|
-
this.txScopeId = explicit ?? txScopeIdFor(options.sql);
|
|
347
|
-
const batch = options.dropBatchSize;
|
|
348
|
-
if (batch !== void 0 && (!Number.isFinite(batch) || batch <= 0)) throw new Error(`PgEdgesStore: dropBatchSize must be a positive finite number, got ${batch}`);
|
|
349
|
-
this.#dropBatchSize = batch ?? DEFAULT_DROP_BATCH_SIZE;
|
|
350
|
-
}
|
|
351
|
-
async write(input) {
|
|
352
|
-
const id = randomUUID();
|
|
353
|
-
const createdAt = this.#now();
|
|
354
|
-
const fromBridged = input.from.bridged === true;
|
|
355
|
-
const toBridged = input.to.bridged === true;
|
|
356
|
-
await this.#sql`
|
|
357
|
-
INSERT INTO edges (
|
|
358
|
-
id, from_ns, from_type, from_id, from_bridged,
|
|
359
|
-
to_ns, to_type, to_id, to_bridged,
|
|
360
|
-
field_name, policy, created_at, deleted_at, last_swept_at, blocked
|
|
361
|
-
)
|
|
362
|
-
VALUES (
|
|
363
|
-
${id},
|
|
364
|
-
${input.from.ns}, ${input.from.type}, ${refToId(input.from.id)}, ${fromBridged},
|
|
365
|
-
${input.to.ns}, ${input.to.type}, ${refToId(input.to.id)}, ${toBridged},
|
|
366
|
-
${input.fieldName}, ${input.policy}, ${createdAt},
|
|
367
|
-
NULL, NULL, false
|
|
368
|
-
)
|
|
369
|
-
`;
|
|
370
|
-
return {
|
|
371
|
-
id,
|
|
372
|
-
from: buildRef(input.from.ns, input.from.type, refToId(input.from.id), fromBridged),
|
|
373
|
-
to: buildRef(input.to.ns, input.to.type, refToId(input.to.id), toBridged),
|
|
374
|
-
fieldName: input.fieldName,
|
|
375
|
-
policy: input.policy,
|
|
376
|
-
createdAt
|
|
377
|
-
};
|
|
378
|
-
}
|
|
379
|
-
async tombstone(id, at) {
|
|
380
|
-
const stamp = at ?? this.#now();
|
|
381
|
-
await this.#sql`
|
|
382
|
-
UPDATE edges
|
|
383
|
-
SET deleted_at = ${stamp}
|
|
384
|
-
WHERE id = ${id}
|
|
385
|
-
AND deleted_at IS NULL
|
|
386
|
-
`;
|
|
387
|
-
}
|
|
388
|
-
/**
|
|
389
|
-
* Hard-deletes tombstoned edges whose `deleted_at` is older than
|
|
390
|
-
* `olderThan`. Called by the sweep loop. Returns the number deleted.
|
|
391
|
-
*/
|
|
392
|
-
async purgeTombstonedEdges(opts) {
|
|
393
|
-
return { deleted: (await this.#sql`
|
|
394
|
-
DELETE FROM edges
|
|
395
|
-
WHERE deleted_at IS NOT NULL
|
|
396
|
-
AND deleted_at < ${opts.olderThan}
|
|
397
|
-
RETURNING id
|
|
398
|
-
`).length };
|
|
399
|
-
}
|
|
400
|
-
async update(patch) {
|
|
401
|
-
for (const key of Object.keys(patch)) if (!MUTABLE_PATCH_FIELDS.has(key)) throw new Error(`EdgesStore.update: field '${key}' is immutable`);
|
|
402
|
-
const hasLastSweptAt = Object.hasOwn(patch, "lastSweptAt");
|
|
403
|
-
const hasBlocked = Object.hasOwn(patch, "blocked");
|
|
404
|
-
if (!hasLastSweptAt && !hasBlocked) return;
|
|
405
|
-
if (hasLastSweptAt && hasBlocked) {
|
|
406
|
-
await this.#sql`
|
|
407
|
-
UPDATE edges
|
|
408
|
-
SET last_swept_at = ${patch.lastSweptAt ?? null},
|
|
409
|
-
blocked = ${patch.blocked ?? false}
|
|
410
|
-
WHERE id = ${patch.id}
|
|
411
|
-
`;
|
|
412
|
-
return;
|
|
413
|
-
}
|
|
414
|
-
if (hasLastSweptAt) {
|
|
415
|
-
await this.#sql`
|
|
416
|
-
UPDATE edges
|
|
417
|
-
SET last_swept_at = ${patch.lastSweptAt ?? null}
|
|
418
|
-
WHERE id = ${patch.id}
|
|
419
|
-
`;
|
|
420
|
-
return;
|
|
421
|
-
}
|
|
422
|
-
await this.#sql`
|
|
423
|
-
UPDATE edges
|
|
424
|
-
SET blocked = ${patch.blocked ?? false}
|
|
425
|
-
WHERE id = ${patch.id}
|
|
426
|
-
`;
|
|
427
|
-
}
|
|
428
|
-
async readNextBatch(cursorId, size) {
|
|
429
|
-
const edges = (cursorId === null ? await this.#sql`
|
|
430
|
-
SELECT ${this.#edgeCols}
|
|
431
|
-
FROM edges
|
|
432
|
-
ORDER BY id ASC
|
|
433
|
-
LIMIT ${size}
|
|
434
|
-
` : await this.#sql`
|
|
435
|
-
SELECT ${this.#edgeCols}
|
|
436
|
-
FROM edges
|
|
437
|
-
WHERE id > ${cursorId}
|
|
438
|
-
ORDER BY id ASC
|
|
439
|
-
LIMIT ${size}
|
|
440
|
-
`).map(rowToEdge);
|
|
441
|
-
const last = edges[edges.length - 1];
|
|
442
|
-
return {
|
|
443
|
-
edges,
|
|
444
|
-
lastId: last ? last.id : null
|
|
445
|
-
};
|
|
446
|
-
}
|
|
447
|
-
async findEdgesTo(ref) {
|
|
448
|
-
return (await this.#sql`
|
|
449
|
-
SELECT ${this.#edgeCols}
|
|
450
|
-
FROM edges
|
|
451
|
-
WHERE to_ns = ${ref.ns}
|
|
452
|
-
AND to_type = ${ref.type}
|
|
453
|
-
AND to_id = ${refToId(ref.id)}
|
|
454
|
-
AND deleted_at IS NULL
|
|
455
|
-
`).map(rowToEdge);
|
|
456
|
-
}
|
|
457
|
-
async findEdgesToMany(refs) {
|
|
458
|
-
if (refs.length === 0) return [];
|
|
459
|
-
const sql = this.#sql;
|
|
460
|
-
const seenRef = /* @__PURE__ */ new Set();
|
|
461
|
-
const toNs = [];
|
|
462
|
-
const toType = [];
|
|
463
|
-
const toId = [];
|
|
464
|
-
for (const r of refs) {
|
|
465
|
-
const id = refToId(r.id);
|
|
466
|
-
const k = `${r.ns}\u0000${r.type}\u0000${id}`;
|
|
467
|
-
if (seenRef.has(k)) continue;
|
|
468
|
-
seenRef.add(k);
|
|
469
|
-
toNs.push(r.ns);
|
|
470
|
-
toType.push(r.type);
|
|
471
|
-
toId.push(id);
|
|
472
|
-
}
|
|
473
|
-
return (await sql`
|
|
474
|
-
SELECT ${this.#edgeCols}
|
|
475
|
-
FROM edges
|
|
476
|
-
JOIN unnest(${toNs}::text[], ${toType}::text[], ${toId}::text[])
|
|
477
|
-
AS seek(seek_ns, seek_type, seek_id)
|
|
478
|
-
ON to_ns = seek.seek_ns AND to_type = seek.seek_type AND to_id = seek.seek_id
|
|
479
|
-
WHERE deleted_at IS NULL
|
|
480
|
-
`).map(rowToEdge);
|
|
481
|
-
}
|
|
482
|
-
async findEdgesFrom(ref) {
|
|
483
|
-
return (await this.#sql`
|
|
484
|
-
SELECT ${this.#edgeCols}
|
|
485
|
-
FROM edges
|
|
486
|
-
WHERE from_ns = ${ref.ns}
|
|
487
|
-
AND from_type = ${ref.type}
|
|
488
|
-
AND from_id = ${refToId(ref.id)}
|
|
489
|
-
AND deleted_at IS NULL
|
|
490
|
-
`).map(rowToEdge);
|
|
491
|
-
}
|
|
492
|
-
async findEdgesTouchingNamespace(namespace, limit) {
|
|
493
|
-
return (await this.#sql`
|
|
494
|
-
SELECT ${this.#edgeCols}
|
|
495
|
-
FROM edges
|
|
496
|
-
WHERE deleted_at IS NULL
|
|
497
|
-
AND (from_ns = ${namespace} OR to_ns = ${namespace})
|
|
498
|
-
ORDER BY id ASC
|
|
499
|
-
LIMIT ${limit}
|
|
500
|
-
`).map(rowToEdge);
|
|
501
|
-
}
|
|
502
|
-
async findEdgesTouchingType(namespace, type, limit) {
|
|
503
|
-
return (await this.#sql`
|
|
504
|
-
SELECT ${this.#edgeCols}
|
|
505
|
-
FROM edges
|
|
506
|
-
WHERE deleted_at IS NULL
|
|
507
|
-
AND ((from_ns = ${namespace} AND from_type = ${type})
|
|
508
|
-
OR (to_ns = ${namespace} AND to_type = ${type}))
|
|
509
|
-
ORDER BY id ASC
|
|
510
|
-
LIMIT ${limit}
|
|
511
|
-
`).map(rowToEdge);
|
|
512
|
-
}
|
|
513
|
-
async namespacesLinkedTo(namespace, type) {
|
|
514
|
-
const fromMatch = type === void 0 ? this.#sql`from_ns = ${namespace}` : this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
|
|
515
|
-
const toMatch = type === void 0 ? this.#sql`to_ns = ${namespace}` : this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
|
|
516
|
-
return (await this.#sql`
|
|
517
|
-
SELECT ns FROM (
|
|
518
|
-
SELECT to_ns AS ns FROM edges WHERE ${fromMatch}
|
|
519
|
-
UNION
|
|
520
|
-
SELECT from_ns AS ns FROM edges WHERE ${toMatch}
|
|
521
|
-
) linked
|
|
522
|
-
WHERE ns <> ${namespace}
|
|
523
|
-
ORDER BY ns
|
|
524
|
-
`).map((r) => r.ns);
|
|
525
|
-
}
|
|
526
|
-
/**
|
|
527
|
-
* Chunked physical delete of every edge touching `namespace`, at most
|
|
528
|
-
* `dropBatchSize` rows per statement, so a large delete does not hold every
|
|
529
|
-
* row lock at once. Returns the number deleted.
|
|
530
|
-
*
|
|
531
|
-
* Outside a tx each batch auto-commits. Inside a caller's tx the batches
|
|
532
|
-
* stay in it, so a throw still rolls back the whole drop.
|
|
533
|
-
*/
|
|
534
|
-
async dropEdgesByNamespace(namespace, within) {
|
|
535
|
-
const batchSize = this.#dropBatchSize;
|
|
536
|
-
const farEnd = this.#farEndWithin(namespace, within, this.#sql`from_ns = ${namespace}`, this.#sql`to_ns = ${namespace}`);
|
|
537
|
-
let total = 0;
|
|
538
|
-
while (true) {
|
|
539
|
-
const rows = await this.#sql`
|
|
540
|
-
DELETE FROM edges
|
|
541
|
-
WHERE id IN (
|
|
542
|
-
SELECT id FROM edges
|
|
543
|
-
WHERE (from_ns = ${namespace} OR to_ns = ${namespace}) ${farEnd}
|
|
544
|
-
LIMIT ${batchSize}
|
|
545
|
-
)
|
|
546
|
-
RETURNING id
|
|
547
|
-
`;
|
|
548
|
-
total += rows.length;
|
|
549
|
-
if (rows.length < batchSize) return total;
|
|
550
|
-
await this.#yieldBetweenBatches();
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
/**
|
|
554
|
-
* Chunked physical delete of every edge touching `(namespace, type)`.
|
|
555
|
-
* See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
|
|
556
|
-
*/
|
|
557
|
-
async dropEdgesByType(namespace, type, within) {
|
|
558
|
-
const batchSize = this.#dropBatchSize;
|
|
559
|
-
const fromMatch = this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
|
|
560
|
-
const toMatch = this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
|
|
561
|
-
const farEnd = this.#farEndWithin(namespace, within, fromMatch, toMatch);
|
|
562
|
-
let total = 0;
|
|
563
|
-
while (true) {
|
|
564
|
-
const rows = await this.#sql`
|
|
565
|
-
DELETE FROM edges
|
|
566
|
-
WHERE id IN (
|
|
567
|
-
SELECT id FROM edges
|
|
568
|
-
WHERE (${fromMatch} OR ${toMatch}) ${farEnd}
|
|
569
|
-
LIMIT ${batchSize}
|
|
570
|
-
)
|
|
571
|
-
RETURNING id
|
|
572
|
-
`;
|
|
573
|
-
total += rows.length;
|
|
574
|
-
if (rows.length < batchSize) return total;
|
|
575
|
-
await this.#yieldBetweenBatches();
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
/** `AND` clause keeping only edges whose far end is `namespace` or in `within`. */
|
|
579
|
-
#farEndWithin(namespace, within, fromMatch, toMatch) {
|
|
580
|
-
if (within === void 0) return this.#sql``;
|
|
581
|
-
const allowed = this.#sql([namespace, ...within]);
|
|
582
|
-
return this.#sql`AND ((${fromMatch} AND to_ns IN ${allowed}) OR (${toMatch} AND from_ns IN ${allowed}))`;
|
|
583
|
-
}
|
|
584
|
-
/**
|
|
585
|
-
* Yields to the event loop between drop batches outside a tx; no-op inside one.
|
|
586
|
-
*/
|
|
587
|
-
async #yieldBetweenBatches() {
|
|
588
|
-
if (!this.#inTx) await new Promise((resolve) => setImmediate(resolve));
|
|
589
|
-
}
|
|
590
|
-
async canDelete(ref) {
|
|
591
|
-
const rows = await this.#sql`
|
|
592
|
-
SELECT ${this.#edgeCols}
|
|
593
|
-
FROM edges
|
|
594
|
-
WHERE to_ns = ${ref.ns}
|
|
595
|
-
AND to_type = ${ref.type}
|
|
596
|
-
AND to_id = ${refToId(ref.id)}
|
|
597
|
-
AND policy = 'restrict'
|
|
598
|
-
AND deleted_at IS NULL
|
|
599
|
-
`;
|
|
600
|
-
if (rows.length === 0) return { ok: true };
|
|
601
|
-
return {
|
|
602
|
-
ok: false,
|
|
603
|
-
blockedBy: rows.map(rowToEdge)
|
|
604
|
-
};
|
|
605
|
-
}
|
|
606
|
-
async findBlocked() {
|
|
607
|
-
return (await this.#sql`
|
|
608
|
-
SELECT ${this.#edgeCols}
|
|
609
|
-
FROM edges
|
|
610
|
-
WHERE blocked = true
|
|
611
|
-
AND deleted_at IS NULL
|
|
612
|
-
`).map(rowToEdge);
|
|
613
|
-
}
|
|
614
|
-
/**
|
|
615
|
-
* `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
|
|
616
|
-
* nested re-entry skips the inner begin and runs on the existing tx handle
|
|
617
|
-
* so an inner throw propagates to the outer `sql.begin` for rollback.
|
|
618
|
-
*/
|
|
619
|
-
async withTransaction(fn) {
|
|
620
|
-
if (this.#inTx) return fn(this);
|
|
621
|
-
return this.#sql.begin(async (txSql) => {
|
|
622
|
-
return fn(new PgEdgesStore({
|
|
623
|
-
sql: txSql,
|
|
624
|
-
now: this.#now,
|
|
625
|
-
inTx: true,
|
|
626
|
-
txScopeId: this.txScopeId,
|
|
627
|
-
dropBatchSize: this.#dropBatchSize
|
|
628
|
-
}));
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
/**
|
|
632
|
-
* Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
|
|
633
|
-
* for `PgDataset.withScopedTransaction`. The caller has verified
|
|
634
|
-
* the txScopeIds match.
|
|
635
|
-
*/
|
|
636
|
-
_enlistInTx(txSql) {
|
|
637
|
-
return new PgEdgesStore({
|
|
638
|
-
sql: txSql,
|
|
639
|
-
now: this.#now,
|
|
640
|
-
inTx: true,
|
|
641
|
-
txScopeId: this.txScopeId,
|
|
642
|
-
dropBatchSize: this.#dropBatchSize
|
|
643
|
-
});
|
|
644
|
-
}
|
|
645
|
-
};
|
|
646
|
-
function buildRef(ns, type, id, bridged) {
|
|
647
|
-
const ref = {
|
|
648
|
-
ns,
|
|
649
|
-
type,
|
|
650
|
-
id
|
|
651
|
-
};
|
|
652
|
-
if (bridged) ref.bridged = true;
|
|
653
|
-
return ref;
|
|
654
|
-
}
|
|
655
|
-
function rowToEdge(row) {
|
|
656
|
-
if (!VALID_POLICIES.has(row.policy)) throw new Error(`PgEdgesStore: unknown policy '${row.policy}' on edge id=${row.id}`);
|
|
657
|
-
const edge = {
|
|
658
|
-
id: row.id,
|
|
659
|
-
from: buildRef(row.from_ns, row.from_type, row.from_id, row.from_bridged),
|
|
660
|
-
to: buildRef(row.to_ns, row.to_type, row.to_id, row.to_bridged),
|
|
661
|
-
fieldName: row.field_name,
|
|
662
|
-
policy: row.policy,
|
|
663
|
-
createdAt: row.created_at
|
|
664
|
-
};
|
|
665
|
-
if (row.deleted_at !== null) edge.deletedAt = row.deleted_at;
|
|
666
|
-
if (row.last_swept_at !== null) edge.lastSweptAt = row.last_swept_at;
|
|
667
|
-
if (row.blocked) edge.blocked = true;
|
|
668
|
-
return edge;
|
|
669
|
-
}
|
|
670
|
-
|
|
671
296
|
//#endregion
|
|
672
297
|
//#region ../../packages/substrate-dataset-pg/src/pg-idempotency-store.ts
|
|
673
298
|
var PgIdempotencyStore = class {
|
package/dist/testing.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { C as Id, E as Row, _ as EdgesStore, b as ActorIdentity, f as CanDeleteResult, g as EdgeWriteInput, h as EdgeRef, l as PackEnvelope, m as EdgePatch, n as FieldDef, p as Edge, u as Schema } from "./schema-g4FQLr0V.js";
|
|
2
2
|
import { a as RuntimeErrors, n as ResolveObservation, r as ResolverStat, s as SlowRequest, t as ProcessStat } from "./runtime-metrics-DwpDJ8vz.js";
|
|
3
|
-
import { $ as PolicyProbeContext, A as ScheduledTask, B as StartOpts, C as QualifiedFrame, D as RegistryEntryInput, E as RegistryEntry, F as HttpMethod, G as WsConnection, H as SubstrateResponse, I as PluginRouter, K as WsHandler, P as HttpHandler, Q as PolicyDecision, R as RequestObserver, St as WriteResult, T as RegistryBase, U as SubstrateServer, V as SubstrateRequest, Y as Identity, _t as ReadResult, a as EngineHandle, ct as Dataset, dt as DropResult, et as RequestContext, g as NamespaceConfig, gt as ReadParams, lt as DatasetRegistry, mt as FilterCapabilities, n as DropParticipant, o as EngineInput, p as KernelPlugin, st as ConditionalUpdateResult, t as CoordinatorPlugin, ut as DeleteResult, w as Registry$1, xt as WriteOptions, yt as UpdateFieldRef, z as RuntimeOp } from "./plugin-t2g2wX0b.js";
|
|
3
|
+
import { $ as PolicyProbeContext, A as ScheduledTask, B as StartOpts, C as QualifiedFrame, D as RegistryEntryInput, E as RegistryEntry, F as HttpMethod, G as WsConnection, H as SubstrateResponse, I as PluginRouter, K as WsHandler, P as HttpHandler, Q as PolicyDecision, R as RequestObserver, St as WriteResult, T as RegistryBase, U as SubstrateServer, V as SubstrateRequest, Y as Identity, _t as ReadResult, a as EngineHandle, ct as Dataset, dt as DropResult, et as RequestContext, g as NamespaceConfig, gt as ReadParams, lt as DatasetRegistry, mt as FilterCapabilities, n as DropParticipant, o as EngineInput, p as KernelPlugin, st as ConditionalUpdateResult, t as CoordinatorPlugin, ut as DeleteResult, vt as TxScopeHandles, w as Registry$1, xt as WriteOptions, yt as UpdateFieldRef, z as RuntimeOp } from "./plugin-t2g2wX0b.js";
|
|
4
4
|
import { a as WsResponseFrame, f as IdempotencyStore, l as SweepDeps, s as BridgeRegistry, u as SweepState } from "./ws-C72iVS5T.js";
|
|
5
5
|
import { t as Embedder } from "./types-8c0gtnd2.js";
|
|
6
|
+
import { Sql, TransactionSql } from "postgres";
|
|
6
7
|
//#region ../../packages/substrate-compose/src/auth.d.ts
|
|
7
8
|
type JwtAlg = 'RS256' | 'ES256' | 'HS256' | 'EdDSA';
|
|
8
9
|
interface AuthKey {
|
|
@@ -1022,6 +1023,315 @@ interface ComposeResult {
|
|
|
1022
1023
|
}
|
|
1023
1024
|
export declare function compose(opts: ComposeOpts): Promise<ComposeResult>;
|
|
1024
1025
|
//#endregion
|
|
1026
|
+
//#region ../../packages/substrate-dataset-pg/src/migrate.d.ts
|
|
1027
|
+
/**
|
|
1028
|
+
* Public migration shape accepted by the runner. Either supply both
|
|
1029
|
+
* `txStatements` + (optional) `concurrentStatements`, OR a legacy
|
|
1030
|
+
* `statements` bucket — the runner treats `statements` as `txStatements`
|
|
1031
|
+
* with no concurrent statements. Mixing `statements` and `txStatements`
|
|
1032
|
+
* is a configuration error.
|
|
1033
|
+
*/
|
|
1034
|
+
type MigrationInput = {
|
|
1035
|
+
readonly version: string;
|
|
1036
|
+
readonly txStatements: ReadonlyArray<string>;
|
|
1037
|
+
readonly concurrentStatements?: ReadonlyArray<string>;
|
|
1038
|
+
readonly statements?: never;
|
|
1039
|
+
} | {
|
|
1040
|
+
readonly version: string;
|
|
1041
|
+
readonly statements: ReadonlyArray<string>;
|
|
1042
|
+
readonly txStatements?: never;
|
|
1043
|
+
readonly concurrentStatements?: never;
|
|
1044
|
+
};
|
|
1045
|
+
interface RunMigrationsOptions {
|
|
1046
|
+
/**
|
|
1047
|
+
* When true, drop the `migrations` tracking table before running.
|
|
1048
|
+
* Forces every migration to re-apply on this call. Intended for tests
|
|
1049
|
+
* that DROP schema objects between runs and need the framework to
|
|
1050
|
+
* recreate them; production paths should never set this.
|
|
1051
|
+
*/
|
|
1052
|
+
force?: boolean;
|
|
1053
|
+
/**
|
|
1054
|
+
* Override the `SUBSTRATE_ENFORCE_UUID_CHECK=true` env-var gate for the gated
|
|
1055
|
+
* UUID-format CHECK constraint migration (0002). Defaults to whatever the
|
|
1056
|
+
* env says. Tests that need to exercise the constrained path explicitly
|
|
1057
|
+
* pass `true`; tests that need to opt out of an env-set gate pass `false`.
|
|
1058
|
+
* Production bootstraps shouldn't pass this — let the env decide.
|
|
1059
|
+
*/
|
|
1060
|
+
enforceUuidCheck?: boolean;
|
|
1061
|
+
}
|
|
1062
|
+
export declare function runMigrations(sql: Sql, opts?: RunMigrationsOptions): Promise<void>;
|
|
1063
|
+
/**
|
|
1064
|
+
* One `field.list({ indexed: true })` or `field.json({ indexed: true })` field
|
|
1065
|
+
* that should be backed by a GIN over `(body -> field)`.
|
|
1066
|
+
*/
|
|
1067
|
+
interface IndexedListField {
|
|
1068
|
+
readonly namespace: string;
|
|
1069
|
+
readonly type: string;
|
|
1070
|
+
readonly field: string;
|
|
1071
|
+
}
|
|
1072
|
+
/**
|
|
1073
|
+
* Derives the tracked, idempotent CONCURRENTLY GIN migration for one indexed
|
|
1074
|
+
* field. The version key uses the raw ns/type/field.
|
|
1075
|
+
*/
|
|
1076
|
+
export declare function deriveListIndexMigration(f: IndexedListField): MigrationInput;
|
|
1077
|
+
/**
|
|
1078
|
+
* Applies migrations derived from field definitions through the same tracked,
|
|
1079
|
+
* advisory-locked machinery as the static list. Called after plugins load,
|
|
1080
|
+
* since `runMigrations` has no field-def knowledge. Idempotent: applied
|
|
1081
|
+
* versions are skipped.
|
|
1082
|
+
*/
|
|
1083
|
+
export declare function runDerivedMigrations(sql: Sql, derived: ReadonlyArray<MigrationInput>): Promise<void>;
|
|
1084
|
+
//#endregion
|
|
1085
|
+
//#region ../../packages/substrate-dataset-pg/src/tx-scope.d.ts
|
|
1086
|
+
type PgHandle = Sql | TransactionSql;
|
|
1087
|
+
//#endregion
|
|
1088
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-edges-store.d.ts
|
|
1089
|
+
interface PgEdgesStoreOptions {
|
|
1090
|
+
sql: Sql;
|
|
1091
|
+
/** Injected clock for tests. Defaults to `() => new Date()`. */
|
|
1092
|
+
now?: () => Date;
|
|
1093
|
+
/**
|
|
1094
|
+
* Per-statement batch size for chunked admin deletes. Defaults to
|
|
1095
|
+
* {@link DEFAULT_DROP_BATCH_SIZE}.
|
|
1096
|
+
*/
|
|
1097
|
+
dropBatchSize?: number;
|
|
1098
|
+
}
|
|
1099
|
+
interface InternalPgEdgesStoreOptions {
|
|
1100
|
+
sql: PgHandle;
|
|
1101
|
+
now: () => Date;
|
|
1102
|
+
inTx: boolean;
|
|
1103
|
+
/**
|
|
1104
|
+
* Tx-bound instances inherit their parent's txScopeId. The public
|
|
1105
|
+
* constructor derives it from the Sql reference.
|
|
1106
|
+
*/
|
|
1107
|
+
txScopeId?: string;
|
|
1108
|
+
dropBatchSize?: number;
|
|
1109
|
+
}
|
|
1110
|
+
export declare class PgEdgesStore implements EdgesStore {
|
|
1111
|
+
#private;
|
|
1112
|
+
readonly txScopeId: string;
|
|
1113
|
+
constructor(options: PgEdgesStoreOptions | InternalPgEdgesStoreOptions);
|
|
1114
|
+
write(input: EdgeWriteInput): Promise<Edge>;
|
|
1115
|
+
tombstone(id: string, at?: Date): Promise<void>;
|
|
1116
|
+
/**
|
|
1117
|
+
* Hard-deletes tombstoned edges whose `deleted_at` is older than
|
|
1118
|
+
* `olderThan`. Called by the sweep loop. Returns the number deleted.
|
|
1119
|
+
*/
|
|
1120
|
+
purgeTombstonedEdges(opts: {
|
|
1121
|
+
olderThan: Date;
|
|
1122
|
+
}): Promise<{
|
|
1123
|
+
deleted: number;
|
|
1124
|
+
}>;
|
|
1125
|
+
update(patch: EdgePatch): Promise<void>;
|
|
1126
|
+
readNextBatch(cursorId: string | null, size: number): Promise<{
|
|
1127
|
+
edges: Edge[];
|
|
1128
|
+
lastId: string | null;
|
|
1129
|
+
}>;
|
|
1130
|
+
findEdgesTo(ref: EdgeRef): Promise<Edge[]>;
|
|
1131
|
+
findEdgesToMany(refs: readonly EdgeRef[]): Promise<Edge[]>;
|
|
1132
|
+
findEdgesFrom(ref: EdgeRef): Promise<Edge[]>;
|
|
1133
|
+
findEdgesTouchingNamespace(namespace: string, limit: number): Promise<Edge[]>;
|
|
1134
|
+
findEdgesTouchingType(namespace: string, type: string, limit: number): Promise<Edge[]>;
|
|
1135
|
+
namespacesLinkedTo(namespace: string, type?: string): Promise<string[]>;
|
|
1136
|
+
/**
|
|
1137
|
+
* Chunked physical delete of every edge touching `namespace`, at most
|
|
1138
|
+
* `dropBatchSize` rows per statement, so a large delete does not hold every
|
|
1139
|
+
* row lock at once. Returns the number deleted.
|
|
1140
|
+
*
|
|
1141
|
+
* Outside a tx each batch auto-commits. Inside a caller's tx the batches
|
|
1142
|
+
* stay in it, so a throw still rolls back the whole drop.
|
|
1143
|
+
*/
|
|
1144
|
+
dropEdgesByNamespace(namespace: string, within?: readonly string[]): Promise<number>;
|
|
1145
|
+
/**
|
|
1146
|
+
* Chunked physical delete of every edge touching `(namespace, type)`.
|
|
1147
|
+
* See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
|
|
1148
|
+
*/
|
|
1149
|
+
dropEdgesByType(namespace: string, type: string, within?: readonly string[]): Promise<number>;
|
|
1150
|
+
canDelete(ref: EdgeRef): Promise<CanDeleteResult>;
|
|
1151
|
+
findBlocked(): Promise<Edge[]>;
|
|
1152
|
+
/**
|
|
1153
|
+
* `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
|
|
1154
|
+
* nested re-entry skips the inner begin and runs on the existing tx handle
|
|
1155
|
+
* so an inner throw propagates to the outer `sql.begin` for rollback.
|
|
1156
|
+
*/
|
|
1157
|
+
withTransaction<T>(fn: (tx: EdgesStore) => Promise<T>): Promise<T>;
|
|
1158
|
+
/**
|
|
1159
|
+
* Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
|
|
1160
|
+
* for `PgDataset.withScopedTransaction`. The caller has verified
|
|
1161
|
+
* the txScopeIds match.
|
|
1162
|
+
*/
|
|
1163
|
+
_enlistInTx(txSql: PgHandle): PgEdgesStore;
|
|
1164
|
+
}
|
|
1165
|
+
//#endregion
|
|
1166
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/primitives.d.ts
|
|
1167
|
+
/**
|
|
1168
|
+
* A body key mirrored into a dedicated column on `rows`, so filters and
|
|
1169
|
+
* sorts on it hit an index instead of extracting from JSONB.
|
|
1170
|
+
*
|
|
1171
|
+
* `column` is a BARE column name, not a qualified `r.<col>` fragment: taking a
|
|
1172
|
+
* fragment from a caller would put caller text into `sql.unsafe`.
|
|
1173
|
+
*/
|
|
1174
|
+
interface PromotedBodyColumn {
|
|
1175
|
+
column: string;
|
|
1176
|
+
kind: 'boolean' | 'text';
|
|
1177
|
+
}
|
|
1178
|
+
type PromotedBodyColumnMap = Readonly<Record<string, PromotedBodyColumn>>;
|
|
1179
|
+
//#endregion
|
|
1180
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-dataset.d.ts
|
|
1181
|
+
/**
|
|
1182
|
+
* Returns the declared FieldDef for a type's field, or undefined when unknown
|
|
1183
|
+
* (body-key sorts then fall back to text compare). Lets the SQL builders cast
|
|
1184
|
+
* body values by declared kind, so `count` sorts 2 before 10 — see
|
|
1185
|
+
* `resolveBodyOrderByCast`.
|
|
1186
|
+
*/
|
|
1187
|
+
type FieldDefLookup = (type: string, field: string) => FieldDef | undefined;
|
|
1188
|
+
interface PgDatasetOptions {
|
|
1189
|
+
namespace: string;
|
|
1190
|
+
sql: Sql;
|
|
1191
|
+
/**
|
|
1192
|
+
* Optional sibling `PgEdgesStore` to enlist into this dataset's substrate
|
|
1193
|
+
* transactions. When supplied, MUST share the same `Sql` pool (verified
|
|
1194
|
+
* via `txScopeId === txScopeIdFor(sql)`); the dataset throws on
|
|
1195
|
+
* construction otherwise.
|
|
1196
|
+
*
|
|
1197
|
+
* When supplied, `withScopedTransaction` yields a tx-bound edges store
|
|
1198
|
+
* on the same `TransactionSql` as the tx-bound datasets. When omitted, it
|
|
1199
|
+
* yields `edges: null` and the engine runs edge ops in a separate tx.
|
|
1200
|
+
*/
|
|
1201
|
+
edgesStore?: PgEdgesStore;
|
|
1202
|
+
/**
|
|
1203
|
+
* Optional field-def lookup so orderBy + cursor compares on body fields can
|
|
1204
|
+
* cast to the declared scalar pg type (numeric / timestamptz / boolean)
|
|
1205
|
+
* instead of falling back to lexicographic text compare. When omitted, every
|
|
1206
|
+
* body-key orderBy compares as text.
|
|
1207
|
+
*
|
|
1208
|
+
* A cast sort cannot use an index on the uncast `(body ->> 'field')`
|
|
1209
|
+
* expression; it needs a matching cast expression index.
|
|
1210
|
+
*/
|
|
1211
|
+
fieldDefLookup?: FieldDefLookup;
|
|
1212
|
+
/**
|
|
1213
|
+
* Body keys mirrored into dedicated columns on `rows`, so filters and
|
|
1214
|
+
* sorts on them use an index instead of extracting from JSONB.
|
|
1215
|
+
*
|
|
1216
|
+
* None by default. A namespace declares its own (`NamespaceDefinition`'s
|
|
1217
|
+
* `promotedColumns`), and the plugin that declares them adds the columns in
|
|
1218
|
+
* its kernel migrations: the engine creates none. Three things that bite:
|
|
1219
|
+
*
|
|
1220
|
+
* - Filters and sorts route to the column and ONLY the column, so a column
|
|
1221
|
+
* added over existing rows must be backfilled from the body first, or
|
|
1222
|
+
* those rows miss every filter. Reads return the body.
|
|
1223
|
+
* - `column` is a BARE name (`'severity'`, not `'r.severity'`) and is
|
|
1224
|
+
* validated as an identifier at construction, because it is interpolated
|
|
1225
|
+
* into SQL as an identifier.
|
|
1226
|
+
* - Text columns sort and range-compare `COLLATE "C"`, so an index meant
|
|
1227
|
+
* to serve those must be built `COLLATE "C"` too.
|
|
1228
|
+
*/
|
|
1229
|
+
promotedColumns?: PromotedBodyColumnMap;
|
|
1230
|
+
}
|
|
1231
|
+
interface InternalPgSubstrateDatasetOptions {
|
|
1232
|
+
namespace: string;
|
|
1233
|
+
sql: PgHandle;
|
|
1234
|
+
inTx: boolean;
|
|
1235
|
+
/**
|
|
1236
|
+
* Tx-bound instances reuse the root's txScopeId so one substrate never
|
|
1237
|
+
* advertises two ids. The public constructor derives it from the Sql.
|
|
1238
|
+
*/
|
|
1239
|
+
txScopeId?: string;
|
|
1240
|
+
edgesStore?: PgEdgesStore;
|
|
1241
|
+
fieldDefLookup?: FieldDefLookup;
|
|
1242
|
+
/** See `PgDatasetOptions.promotedColumns`. */
|
|
1243
|
+
promotedColumns?: PromotedBodyColumnMap;
|
|
1244
|
+
}
|
|
1245
|
+
export declare class PgDataset implements Dataset {
|
|
1246
|
+
#private;
|
|
1247
|
+
readonly namespace: string;
|
|
1248
|
+
readonly capabilities: FilterCapabilities;
|
|
1249
|
+
readonly txScopeId: string;
|
|
1250
|
+
constructor(options: PgDatasetOptions | InternalPgSubstrateDatasetOptions);
|
|
1251
|
+
/**
|
|
1252
|
+
* Reads rows of `type` per the §5.2 filter contract (where / orderBy /
|
|
1253
|
+
* cursor / limit). Every value is bound through a parameterised slot.
|
|
1254
|
+
*
|
|
1255
|
+
* Body values extract as text and are cast for comparison only; the filter
|
|
1256
|
+
* validator has already rejected type mismatches.
|
|
1257
|
+
*/
|
|
1258
|
+
read(type: string, params: ReadParams): Promise<ReadResult>;
|
|
1259
|
+
write(type: string, rows: Row[], options?: WriteOptions): Promise<WriteResult>;
|
|
1260
|
+
tombstone(type: string, ids: Id[]): Promise<DeleteResult>;
|
|
1261
|
+
purge(type: string, ids: Id[]): Promise<DeleteResult>;
|
|
1262
|
+
/**
|
|
1263
|
+
* Hard-deletes tombstoned rows whose `deleted_at` is older than `olderThan`,
|
|
1264
|
+
* across ALL namespaces (the table is substrate-shared). Called by the sweep
|
|
1265
|
+
* loop. Returns the number of rows deleted.
|
|
1266
|
+
*
|
|
1267
|
+
* Short-id mappings are left in place, as in `purge()`: a row id can
|
|
1268
|
+
* reappear after a tombstone + rewrite.
|
|
1269
|
+
*/
|
|
1270
|
+
purgeTombstonedRows(opts: {
|
|
1271
|
+
olderThan: Date;
|
|
1272
|
+
}): Promise<{
|
|
1273
|
+
deleted: number;
|
|
1274
|
+
}>;
|
|
1275
|
+
/**
|
|
1276
|
+
* Hard-delete every row for `type` in this namespace, including tombstoned
|
|
1277
|
+
* rows (`deleted_at IS NOT NULL`). Short-id mappings for that type are
|
|
1278
|
+
* removed in the same logical operation. Both deletes run through `this.#sql`
|
|
1279
|
+
* so they participate in any surrounding `withTransaction` boundary.
|
|
1280
|
+
*/
|
|
1281
|
+
dropType(type: string): Promise<DropResult>;
|
|
1282
|
+
/**
|
|
1283
|
+
* Hard-delete every row across all types in this namespace, including
|
|
1284
|
+
* tombstoned rows. Short-id mappings for all types are removed too.
|
|
1285
|
+
*/
|
|
1286
|
+
dropNamespace(): Promise<DropResult>;
|
|
1287
|
+
/**
|
|
1288
|
+
* Compare-and-update (spec §5.12): merges `updates` into the live row only
|
|
1289
|
+
* if every `match` field equals its expected value. Returns `matched` and
|
|
1290
|
+
* the updated row.
|
|
1291
|
+
*
|
|
1292
|
+
* One `UPDATE ... WHERE <pk> AND <match>`, so the row lock makes exactly one
|
|
1293
|
+
* of N concurrent callers match. `_deletedAt` and `id` are dropped from
|
|
1294
|
+
* `updates`.
|
|
1295
|
+
*/
|
|
1296
|
+
conditionalUpdate(type: string, id: Id, match: Record<string, unknown>, updates: Record<string, unknown>, actor?: ActorIdentity, now?: Date): Promise<ConditionalUpdateResult>;
|
|
1297
|
+
updateField(ref: UpdateFieldRef, fieldName: string, value: unknown): Promise<void>;
|
|
1298
|
+
/**
|
|
1299
|
+
* `sql.begin`-backed atomicity. Spec §5.3 / G1.
|
|
1300
|
+
*
|
|
1301
|
+
* Nested calls flatten: postgres.js's `begin` inside a `begin` would create
|
|
1302
|
+
* a savepoint (inner rolls back independently of outer), which is stricter
|
|
1303
|
+
* than the G1 contract promises. To match the memory impl — where an inner
|
|
1304
|
+
* throw tears down outer writes too — we detect the re-entry via `#inTx`
|
|
1305
|
+
* and skip the nested begin entirely. The inner throw then propagates to
|
|
1306
|
+
* the outer `sql.begin`, which rolls back the whole thing.
|
|
1307
|
+
*/
|
|
1308
|
+
withTransaction<T>(fn: (tx: Dataset) => Promise<T>): Promise<T>;
|
|
1309
|
+
/**
|
|
1310
|
+
* Runs `fn` in one pg transaction with a tx-bound dataset per requested
|
|
1311
|
+
* namespace (and the edges store, when wired), all on one `TransactionSql`.
|
|
1312
|
+
* A throw rolls back every namespace. Spec §5.3.
|
|
1313
|
+
*
|
|
1314
|
+
* Nested calls flatten onto the current transaction rather than opening a
|
|
1315
|
+
* savepoint.
|
|
1316
|
+
*/
|
|
1317
|
+
withScopedTransaction<T>(namespaces: ReadonlyArray<string>, fn: (handles: TxScopeHandles) => Promise<T>): Promise<T>;
|
|
1318
|
+
/**
|
|
1319
|
+
* See `Dataset.withAdvisoryLock`. Excludes every session on the
|
|
1320
|
+
* database, across processes. On a tx-bound instance the lock is
|
|
1321
|
+
* `pg_advisory_xact_lock` on the transaction's own connection, held until
|
|
1322
|
+
* it commits or rolls back: a session lock would need a second connection
|
|
1323
|
+
* beside the open transaction.
|
|
1324
|
+
*/
|
|
1325
|
+
withAdvisoryLock<T>(key: string, fn: () => Promise<T>): Promise<T>;
|
|
1326
|
+
allocateShortId(type: string): Promise<number>;
|
|
1327
|
+
bindShortId(type: string, shortId: number, rowId: string): Promise<void>;
|
|
1328
|
+
resolveShortId(type: string, shortId: number): Promise<string | null>;
|
|
1329
|
+
getShortIdForRow(type: string, rowId: string): Promise<number | null>;
|
|
1330
|
+
forNamespace(namespace: string): Dataset;
|
|
1331
|
+
private materialize;
|
|
1332
|
+
private extractOriginalId;
|
|
1333
|
+
}
|
|
1334
|
+
//#endregion
|
|
1025
1335
|
//#region ../../packages/substrate-dataset-pg/src/testing.d.ts
|
|
1026
1336
|
/**
|
|
1027
1337
|
* Safety rails for pg integration tests. Shared by every package whose suites
|
|
@@ -1242,4 +1552,4 @@ export declare class MemorySubstrateServer implements SubstrateServer {
|
|
|
1242
1552
|
*/
|
|
1243
1553
|
export declare function matchRoute(registered: string, actual: string): Record<string, string> | null;
|
|
1244
1554
|
//#endregion
|
|
1245
|
-
export type { AuthConfig, ComposeResult, MemoryWsClient };
|
|
1555
|
+
export type { AuthConfig, ComposeResult, FieldDefLookup, IndexedListField, MemoryWsClient, PgDatasetOptions, PgEdgesStoreOptions };
|
package/dist/testing.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
2
|
import { N as MemoryEdgesStore, P as loadPlugins, at as WS_REASON_SERVER_RESTART, c as MemoryDataset, rt as signHs256, s as compose } from "./src-CHp3QyqB.js";
|
|
3
|
+
import { a as runDerivedMigrations, o as runMigrations, r as deriveListIndexMigration, t as PgDataset } from "./src-CXKnqeTW.js";
|
|
4
|
+
import { t as PgEdgesStore } from "./pg-edges-store-DKPYyd8l.js";
|
|
3
5
|
import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
|
|
4
6
|
import { randomUUID } from "node:crypto";
|
|
5
7
|
import postgres from "postgres";
|
|
@@ -699,4 +701,4 @@ var MemorySubstrateServer = class {
|
|
|
699
701
|
};
|
|
700
702
|
|
|
701
703
|
//#endregion
|
|
702
|
-
export { AlreadyStartedError, FakeEmbedder, MemoryDataset, MemoryEdgesStore, MemorySubstrateServer, UnknownWsPathError, WsClosedError, announcePgGating, assertCannotWriteOutsideTestDatabases, assertNotSuperuser, assertTestDatabase, compose, createMemoryWsPair, interpretSuperuserProbe, loadPlugins, matchRoute, pgTestSchemaName, pgTestSchemaUrl, preparePgTestSchema, publicLeftoverWarning, reachFailureMessage, signHs256 };
|
|
704
|
+
export { AlreadyStartedError, FakeEmbedder, MemoryDataset, MemoryEdgesStore, MemorySubstrateServer, PgDataset, PgEdgesStore, UnknownWsPathError, WsClosedError, announcePgGating, assertCannotWriteOutsideTestDatabases, assertNotSuperuser, assertTestDatabase, compose, createMemoryWsPair, deriveListIndexMigration, interpretSuperuserProbe, loadPlugins, matchRoute, pgTestSchemaName, pgTestSchemaUrl, preparePgTestSchema, publicLeftoverWarning, reachFailureMessage, runDerivedMigrations, runMigrations, signHs256 };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mnmal-ai/substrate",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.3",
|
|
4
4
|
"description": "Substrate — an agent-first typed-data server: the substrate and substrate-admin binaries, the built-in plugins and the SDK subpaths.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -36,24 +36,24 @@
|
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@inquirer/prompts": "^8.7.2",
|
|
39
|
-
"@mnmal-ai/substrate-bootstrap": "1.0.0-alpha.
|
|
40
|
-
"@mnmal-ai/substrate-client": "1.0.0-alpha.
|
|
41
|
-
"@mnmal-ai/substrate-compose": "1.0.0-alpha.
|
|
42
|
-
"@mnmal-ai/substrate-config": "1.0.0-alpha.
|
|
43
|
-
"@mnmal-ai/substrate-core": "1.0.0-alpha.
|
|
44
|
-
"@mnmal-ai/substrate-dataset-memory": "1.0.0-alpha.
|
|
45
|
-
"@mnmal-ai/substrate-dataset-pg": "1.0.0-alpha.
|
|
46
|
-
"@mnmal-ai/substrate-embedder": "1.0.0-alpha.
|
|
47
|
-
"@mnmal-ai/substrate-filter": "1.0.0-alpha.
|
|
48
|
-
"@mnmal-ai/substrate-pack": "1.0.0-alpha.
|
|
49
|
-
"@mnmal-ai/substrate-policy": "1.0.0-alpha.
|
|
50
|
-
"@mnmal-ai/substrate-proposals": "1.0.0-alpha.
|
|
51
|
-
"@mnmal-ai/substrate-recall": "1.0.0-alpha.
|
|
52
|
-
"@mnmal-ai/substrate-sdk": "1.0.0-alpha.
|
|
53
|
-
"@mnmal-ai/substrate-sentinel-client": "1.0.0-alpha.
|
|
54
|
-
"@mnmal-ai/substrate-server-fastify": "1.0.0-alpha.
|
|
55
|
-
"@mnmal-ai/substrate-server-memory": "1.0.0-alpha.
|
|
56
|
-
"@mnmal-ai/substrate-types": "1.0.0-alpha.
|
|
39
|
+
"@mnmal-ai/substrate-bootstrap": "1.0.0-alpha.3",
|
|
40
|
+
"@mnmal-ai/substrate-client": "1.0.0-alpha.3",
|
|
41
|
+
"@mnmal-ai/substrate-compose": "1.0.0-alpha.3",
|
|
42
|
+
"@mnmal-ai/substrate-config": "1.0.0-alpha.3",
|
|
43
|
+
"@mnmal-ai/substrate-core": "1.0.0-alpha.3",
|
|
44
|
+
"@mnmal-ai/substrate-dataset-memory": "1.0.0-alpha.3",
|
|
45
|
+
"@mnmal-ai/substrate-dataset-pg": "1.0.0-alpha.3",
|
|
46
|
+
"@mnmal-ai/substrate-embedder": "1.0.0-alpha.3",
|
|
47
|
+
"@mnmal-ai/substrate-filter": "1.0.0-alpha.3",
|
|
48
|
+
"@mnmal-ai/substrate-pack": "1.0.0-alpha.3",
|
|
49
|
+
"@mnmal-ai/substrate-policy": "1.0.0-alpha.3",
|
|
50
|
+
"@mnmal-ai/substrate-proposals": "1.0.0-alpha.3",
|
|
51
|
+
"@mnmal-ai/substrate-recall": "1.0.0-alpha.3",
|
|
52
|
+
"@mnmal-ai/substrate-sdk": "1.0.0-alpha.3",
|
|
53
|
+
"@mnmal-ai/substrate-sentinel-client": "1.0.0-alpha.3",
|
|
54
|
+
"@mnmal-ai/substrate-server-fastify": "1.0.0-alpha.3",
|
|
55
|
+
"@mnmal-ai/substrate-server-memory": "1.0.0-alpha.3",
|
|
56
|
+
"@mnmal-ai/substrate-types": "1.0.0-alpha.3",
|
|
57
57
|
"@types/node": "^26.6.4",
|
|
58
58
|
"@types/semver": "^7.5.0",
|
|
59
59
|
"@types/ws": "^8.18.1",
|