@mnmal-ai/substrate 1.0.0-alpha.2 → 1.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{dataset-BQ8egGl5.js → dataset-Bd18RP3O.js} +1 -1
- package/dist/pg-edges-store-DKPYyd8l.js +381 -0
- package/dist/{plugin-QuJN9pG_.js → plugin-BG8E04wi.js} +42 -5
- package/dist/recall-bootstrap.js +1 -1
- package/dist/recall.d.ts +8 -0
- package/dist/recall.js +2 -2
- package/dist/{src-DWm0iA88.js → src-DQ1U9Yhs.js} +2 -2
- package/dist/substrate-admin.js +1 -1
- package/dist/substrate.js +6 -380
- package/dist/testing.d.ts +312 -2
- package/dist/testing.js +3 -1
- package/package.json +19 -19
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ It runs as a real install: the binary and its plugins resolve from one shared `n
|
|
|
11
11
|
| `@mnmal-ai/substrate/sdk` | Plugin authoring: `defineNamespace`, the field DSL, autoCRUD, engine helpers, `defineBootstrap`, policy combinators, route helpers, and all shared types |
|
|
12
12
|
| `@mnmal-ai/substrate/client` | `SubstrateClient`, `watchRows`, client errors, the admin API, and the token and grant helpers |
|
|
13
13
|
| `@mnmal-ai/substrate/pack` | The MessagePack wire codec |
|
|
14
|
-
| `@mnmal-ai/substrate/testing` | `compose`, `MemorySubstrateServer`, `MemoryDataset`, `MemoryEdgesStore`, `loadPlugins`, `FakeEmbedder`, and the Postgres test-database guards |
|
|
14
|
+
| `@mnmal-ai/substrate/testing` | `compose`, `MemorySubstrateServer`, `MemoryDataset`, `MemoryEdgesStore`, `loadPlugins`, `FakeEmbedder`, `PgDataset`, `PgEdgesStore`, the pg migration runners, and the Postgres test-database guards |
|
|
15
15
|
| `@mnmal-ai/substrate/recall` | The recall plugin's primitives (`EmbeddingsStore`, `LexicalStore`, `embedAndStore`, …) |
|
|
16
16
|
| `@mnmal-ai/substrate/proposals` | The proposals plugin's exports |
|
|
17
17
|
|
|
@@ -11,7 +11,7 @@ import { createHash, createPublicKey, generateKeyPairSync, randomUUID } from "no
|
|
|
11
11
|
import postgres from "postgres";
|
|
12
12
|
|
|
13
13
|
//#region package.json
|
|
14
|
-
var version = "1.0.0-alpha.
|
|
14
|
+
var version = "1.0.0-alpha.4";
|
|
15
15
|
|
|
16
16
|
//#endregion
|
|
17
17
|
//#region src/admin/errors.ts
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { n as txScopeIdFor } from "./src-CXKnqeTW.js";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
|
|
5
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-edges-store.ts
|
|
6
|
+
/**
|
|
7
|
+
* PgEdgesStore — PostgreSQL-backed reference implementation of EdgesStore.
|
|
8
|
+
* Spec: §4.6, §6.2, §6.3.
|
|
9
|
+
*
|
|
10
|
+
* One table (`edges`) keyed by a text PK, no namespace scoping — edges
|
|
11
|
+
* are engine-scoped (§4.6), not per-ns. The caller owns the `postgres` Sql
|
|
12
|
+
* connection; this class never opens or closes pools.
|
|
13
|
+
*
|
|
14
|
+
* Edge ids are `crypto.randomUUID()`: unique, so cursor pagination works, but
|
|
15
|
+
* not time-ordered. The sweep walks the whole table, so it doesn't need order.
|
|
16
|
+
*
|
|
17
|
+
* Edge.from/to .id is `string | number`, stored and read back as text (unlike
|
|
18
|
+
* PgDataset, which preserves numeric ids).
|
|
19
|
+
*
|
|
20
|
+
* SQL safety: postgres.js tagged templates parameterise every value slot.
|
|
21
|
+
* Never string-concat caller input — the `update` dynamic column list uses a
|
|
22
|
+
* fixed allowlist of column names, not user-supplied keys.
|
|
23
|
+
*/
|
|
24
|
+
const MUTABLE_PATCH_FIELDS = /* @__PURE__ */ new Set([
|
|
25
|
+
"id",
|
|
26
|
+
"lastSweptAt",
|
|
27
|
+
"blocked"
|
|
28
|
+
]);
|
|
29
|
+
/**
|
|
30
|
+
* Batch size for chunked admin deletes (`dropEdgesByNamespace`,
|
|
31
|
+
* `dropEdgesByType`): bounds per-statement lock hold on large tables.
|
|
32
|
+
*/
|
|
33
|
+
const DEFAULT_DROP_BATCH_SIZE = 5e3;
|
|
34
|
+
const VALID_POLICIES = /* @__PURE__ */ new Set([
|
|
35
|
+
"set_null",
|
|
36
|
+
"cascade",
|
|
37
|
+
"restrict",
|
|
38
|
+
"orphan"
|
|
39
|
+
]);
|
|
40
|
+
const refToId = (id) => typeof id === "number" ? String(id) : id;
|
|
41
|
+
var PgEdgesStore = class PgEdgesStore {
|
|
42
|
+
#sql;
|
|
43
|
+
#now;
|
|
44
|
+
#inTx;
|
|
45
|
+
#dropBatchSize;
|
|
46
|
+
txScopeId;
|
|
47
|
+
get #edgeCols() {
|
|
48
|
+
return this.#sql`id, from_ns, from_type, from_id, from_bridged, to_ns, to_type, to_id, to_bridged, field_name, policy, created_at, deleted_at, last_swept_at, blocked`;
|
|
49
|
+
}
|
|
50
|
+
constructor(options) {
|
|
51
|
+
this.#sql = options.sql;
|
|
52
|
+
this.#now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
53
|
+
this.#inTx = "inTx" in options ? options.inTx : false;
|
|
54
|
+
const explicit = "txScopeId" in options ? options.txScopeId : void 0;
|
|
55
|
+
this.txScopeId = explicit ?? txScopeIdFor(options.sql);
|
|
56
|
+
const batch = options.dropBatchSize;
|
|
57
|
+
if (batch !== void 0 && (!Number.isFinite(batch) || batch <= 0)) throw new Error(`PgEdgesStore: dropBatchSize must be a positive finite number, got ${batch}`);
|
|
58
|
+
this.#dropBatchSize = batch ?? DEFAULT_DROP_BATCH_SIZE;
|
|
59
|
+
}
|
|
60
|
+
async write(input) {
|
|
61
|
+
const id = randomUUID();
|
|
62
|
+
const createdAt = this.#now();
|
|
63
|
+
const fromBridged = input.from.bridged === true;
|
|
64
|
+
const toBridged = input.to.bridged === true;
|
|
65
|
+
await this.#sql`
|
|
66
|
+
INSERT INTO edges (
|
|
67
|
+
id, from_ns, from_type, from_id, from_bridged,
|
|
68
|
+
to_ns, to_type, to_id, to_bridged,
|
|
69
|
+
field_name, policy, created_at, deleted_at, last_swept_at, blocked
|
|
70
|
+
)
|
|
71
|
+
VALUES (
|
|
72
|
+
${id},
|
|
73
|
+
${input.from.ns}, ${input.from.type}, ${refToId(input.from.id)}, ${fromBridged},
|
|
74
|
+
${input.to.ns}, ${input.to.type}, ${refToId(input.to.id)}, ${toBridged},
|
|
75
|
+
${input.fieldName}, ${input.policy}, ${createdAt},
|
|
76
|
+
NULL, NULL, false
|
|
77
|
+
)
|
|
78
|
+
`;
|
|
79
|
+
return {
|
|
80
|
+
id,
|
|
81
|
+
from: buildRef(input.from.ns, input.from.type, refToId(input.from.id), fromBridged),
|
|
82
|
+
to: buildRef(input.to.ns, input.to.type, refToId(input.to.id), toBridged),
|
|
83
|
+
fieldName: input.fieldName,
|
|
84
|
+
policy: input.policy,
|
|
85
|
+
createdAt
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
async tombstone(id, at) {
|
|
89
|
+
const stamp = at ?? this.#now();
|
|
90
|
+
await this.#sql`
|
|
91
|
+
UPDATE edges
|
|
92
|
+
SET deleted_at = ${stamp}
|
|
93
|
+
WHERE id = ${id}
|
|
94
|
+
AND deleted_at IS NULL
|
|
95
|
+
`;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Hard-deletes tombstoned edges whose `deleted_at` is older than
|
|
99
|
+
* `olderThan`. Called by the sweep loop. Returns the number deleted.
|
|
100
|
+
*/
|
|
101
|
+
async purgeTombstonedEdges(opts) {
|
|
102
|
+
return { deleted: (await this.#sql`
|
|
103
|
+
DELETE FROM edges
|
|
104
|
+
WHERE deleted_at IS NOT NULL
|
|
105
|
+
AND deleted_at < ${opts.olderThan}
|
|
106
|
+
RETURNING id
|
|
107
|
+
`).length };
|
|
108
|
+
}
|
|
109
|
+
async update(patch) {
|
|
110
|
+
for (const key of Object.keys(patch)) if (!MUTABLE_PATCH_FIELDS.has(key)) throw new Error(`EdgesStore.update: field '${key}' is immutable`);
|
|
111
|
+
const hasLastSweptAt = Object.hasOwn(patch, "lastSweptAt");
|
|
112
|
+
const hasBlocked = Object.hasOwn(patch, "blocked");
|
|
113
|
+
if (!hasLastSweptAt && !hasBlocked) return;
|
|
114
|
+
if (hasLastSweptAt && hasBlocked) {
|
|
115
|
+
await this.#sql`
|
|
116
|
+
UPDATE edges
|
|
117
|
+
SET last_swept_at = ${patch.lastSweptAt ?? null},
|
|
118
|
+
blocked = ${patch.blocked ?? false}
|
|
119
|
+
WHERE id = ${patch.id}
|
|
120
|
+
`;
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
if (hasLastSweptAt) {
|
|
124
|
+
await this.#sql`
|
|
125
|
+
UPDATE edges
|
|
126
|
+
SET last_swept_at = ${patch.lastSweptAt ?? null}
|
|
127
|
+
WHERE id = ${patch.id}
|
|
128
|
+
`;
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
await this.#sql`
|
|
132
|
+
UPDATE edges
|
|
133
|
+
SET blocked = ${patch.blocked ?? false}
|
|
134
|
+
WHERE id = ${patch.id}
|
|
135
|
+
`;
|
|
136
|
+
}
|
|
137
|
+
async readNextBatch(cursorId, size) {
|
|
138
|
+
const edges = (cursorId === null ? await this.#sql`
|
|
139
|
+
SELECT ${this.#edgeCols}
|
|
140
|
+
FROM edges
|
|
141
|
+
ORDER BY id ASC
|
|
142
|
+
LIMIT ${size}
|
|
143
|
+
` : await this.#sql`
|
|
144
|
+
SELECT ${this.#edgeCols}
|
|
145
|
+
FROM edges
|
|
146
|
+
WHERE id > ${cursorId}
|
|
147
|
+
ORDER BY id ASC
|
|
148
|
+
LIMIT ${size}
|
|
149
|
+
`).map(rowToEdge);
|
|
150
|
+
const last = edges[edges.length - 1];
|
|
151
|
+
return {
|
|
152
|
+
edges,
|
|
153
|
+
lastId: last ? last.id : null
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
async findEdgesTo(ref) {
|
|
157
|
+
return (await this.#sql`
|
|
158
|
+
SELECT ${this.#edgeCols}
|
|
159
|
+
FROM edges
|
|
160
|
+
WHERE to_ns = ${ref.ns}
|
|
161
|
+
AND to_type = ${ref.type}
|
|
162
|
+
AND to_id = ${refToId(ref.id)}
|
|
163
|
+
AND deleted_at IS NULL
|
|
164
|
+
`).map(rowToEdge);
|
|
165
|
+
}
|
|
166
|
+
async findEdgesToMany(refs) {
|
|
167
|
+
if (refs.length === 0) return [];
|
|
168
|
+
const sql = this.#sql;
|
|
169
|
+
const seenRef = /* @__PURE__ */ new Set();
|
|
170
|
+
const toNs = [];
|
|
171
|
+
const toType = [];
|
|
172
|
+
const toId = [];
|
|
173
|
+
for (const r of refs) {
|
|
174
|
+
const id = refToId(r.id);
|
|
175
|
+
const k = `${r.ns}\u0000${r.type}\u0000${id}`;
|
|
176
|
+
if (seenRef.has(k)) continue;
|
|
177
|
+
seenRef.add(k);
|
|
178
|
+
toNs.push(r.ns);
|
|
179
|
+
toType.push(r.type);
|
|
180
|
+
toId.push(id);
|
|
181
|
+
}
|
|
182
|
+
return (await sql`
|
|
183
|
+
SELECT ${this.#edgeCols}
|
|
184
|
+
FROM edges
|
|
185
|
+
JOIN unnest(${toNs}::text[], ${toType}::text[], ${toId}::text[])
|
|
186
|
+
AS seek(seek_ns, seek_type, seek_id)
|
|
187
|
+
ON to_ns = seek.seek_ns AND to_type = seek.seek_type AND to_id = seek.seek_id
|
|
188
|
+
WHERE deleted_at IS NULL
|
|
189
|
+
`).map(rowToEdge);
|
|
190
|
+
}
|
|
191
|
+
async findEdgesFrom(ref) {
|
|
192
|
+
return (await this.#sql`
|
|
193
|
+
SELECT ${this.#edgeCols}
|
|
194
|
+
FROM edges
|
|
195
|
+
WHERE from_ns = ${ref.ns}
|
|
196
|
+
AND from_type = ${ref.type}
|
|
197
|
+
AND from_id = ${refToId(ref.id)}
|
|
198
|
+
AND deleted_at IS NULL
|
|
199
|
+
`).map(rowToEdge);
|
|
200
|
+
}
|
|
201
|
+
async findEdgesTouchingNamespace(namespace, limit) {
|
|
202
|
+
return (await this.#sql`
|
|
203
|
+
SELECT ${this.#edgeCols}
|
|
204
|
+
FROM edges
|
|
205
|
+
WHERE deleted_at IS NULL
|
|
206
|
+
AND (from_ns = ${namespace} OR to_ns = ${namespace})
|
|
207
|
+
ORDER BY id ASC
|
|
208
|
+
LIMIT ${limit}
|
|
209
|
+
`).map(rowToEdge);
|
|
210
|
+
}
|
|
211
|
+
async findEdgesTouchingType(namespace, type, limit) {
|
|
212
|
+
return (await this.#sql`
|
|
213
|
+
SELECT ${this.#edgeCols}
|
|
214
|
+
FROM edges
|
|
215
|
+
WHERE deleted_at IS NULL
|
|
216
|
+
AND ((from_ns = ${namespace} AND from_type = ${type})
|
|
217
|
+
OR (to_ns = ${namespace} AND to_type = ${type}))
|
|
218
|
+
ORDER BY id ASC
|
|
219
|
+
LIMIT ${limit}
|
|
220
|
+
`).map(rowToEdge);
|
|
221
|
+
}
|
|
222
|
+
async namespacesLinkedTo(namespace, type) {
|
|
223
|
+
const fromMatch = type === void 0 ? this.#sql`from_ns = ${namespace}` : this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
|
|
224
|
+
const toMatch = type === void 0 ? this.#sql`to_ns = ${namespace}` : this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
|
|
225
|
+
return (await this.#sql`
|
|
226
|
+
SELECT ns FROM (
|
|
227
|
+
SELECT to_ns AS ns FROM edges WHERE ${fromMatch}
|
|
228
|
+
UNION
|
|
229
|
+
SELECT from_ns AS ns FROM edges WHERE ${toMatch}
|
|
230
|
+
) linked
|
|
231
|
+
WHERE ns <> ${namespace}
|
|
232
|
+
ORDER BY ns
|
|
233
|
+
`).map((r) => r.ns);
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Chunked physical delete of every edge touching `namespace`, at most
|
|
237
|
+
* `dropBatchSize` rows per statement, so a large delete does not hold every
|
|
238
|
+
* row lock at once. Returns the number deleted.
|
|
239
|
+
*
|
|
240
|
+
* Outside a tx each batch auto-commits. Inside a caller's tx the batches
|
|
241
|
+
* stay in it, so a throw still rolls back the whole drop.
|
|
242
|
+
*/
|
|
243
|
+
async dropEdgesByNamespace(namespace, within) {
|
|
244
|
+
const batchSize = this.#dropBatchSize;
|
|
245
|
+
const farEnd = this.#farEndWithin(namespace, within, this.#sql`from_ns = ${namespace}`, this.#sql`to_ns = ${namespace}`);
|
|
246
|
+
let total = 0;
|
|
247
|
+
while (true) {
|
|
248
|
+
const rows = await this.#sql`
|
|
249
|
+
DELETE FROM edges
|
|
250
|
+
WHERE id IN (
|
|
251
|
+
SELECT id FROM edges
|
|
252
|
+
WHERE (from_ns = ${namespace} OR to_ns = ${namespace}) ${farEnd}
|
|
253
|
+
LIMIT ${batchSize}
|
|
254
|
+
)
|
|
255
|
+
RETURNING id
|
|
256
|
+
`;
|
|
257
|
+
total += rows.length;
|
|
258
|
+
if (rows.length < batchSize) return total;
|
|
259
|
+
await this.#yieldBetweenBatches();
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Chunked physical delete of every edge touching `(namespace, type)`.
|
|
264
|
+
* See {@link dropEdgesByNamespace} for the lock-hold + tx semantics.
|
|
265
|
+
*/
|
|
266
|
+
async dropEdgesByType(namespace, type, within) {
|
|
267
|
+
const batchSize = this.#dropBatchSize;
|
|
268
|
+
const fromMatch = this.#sql`(from_ns = ${namespace} AND from_type = ${type})`;
|
|
269
|
+
const toMatch = this.#sql`(to_ns = ${namespace} AND to_type = ${type})`;
|
|
270
|
+
const farEnd = this.#farEndWithin(namespace, within, fromMatch, toMatch);
|
|
271
|
+
let total = 0;
|
|
272
|
+
while (true) {
|
|
273
|
+
const rows = await this.#sql`
|
|
274
|
+
DELETE FROM edges
|
|
275
|
+
WHERE id IN (
|
|
276
|
+
SELECT id FROM edges
|
|
277
|
+
WHERE (${fromMatch} OR ${toMatch}) ${farEnd}
|
|
278
|
+
LIMIT ${batchSize}
|
|
279
|
+
)
|
|
280
|
+
RETURNING id
|
|
281
|
+
`;
|
|
282
|
+
total += rows.length;
|
|
283
|
+
if (rows.length < batchSize) return total;
|
|
284
|
+
await this.#yieldBetweenBatches();
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
/** `AND` clause keeping only edges whose far end is `namespace` or in `within`. */
|
|
288
|
+
#farEndWithin(namespace, within, fromMatch, toMatch) {
|
|
289
|
+
if (within === void 0) return this.#sql``;
|
|
290
|
+
const allowed = this.#sql([namespace, ...within]);
|
|
291
|
+
return this.#sql`AND ((${fromMatch} AND to_ns IN ${allowed}) OR (${toMatch} AND from_ns IN ${allowed}))`;
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Yields to the event loop between drop batches outside a tx; no-op inside one.
|
|
295
|
+
*/
|
|
296
|
+
async #yieldBetweenBatches() {
|
|
297
|
+
if (!this.#inTx) await new Promise((resolve) => setImmediate(resolve));
|
|
298
|
+
}
|
|
299
|
+
async canDelete(ref) {
|
|
300
|
+
const rows = await this.#sql`
|
|
301
|
+
SELECT ${this.#edgeCols}
|
|
302
|
+
FROM edges
|
|
303
|
+
WHERE to_ns = ${ref.ns}
|
|
304
|
+
AND to_type = ${ref.type}
|
|
305
|
+
AND to_id = ${refToId(ref.id)}
|
|
306
|
+
AND policy = 'restrict'
|
|
307
|
+
AND deleted_at IS NULL
|
|
308
|
+
`;
|
|
309
|
+
if (rows.length === 0) return { ok: true };
|
|
310
|
+
return {
|
|
311
|
+
ok: false,
|
|
312
|
+
blockedBy: rows.map(rowToEdge)
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
async findBlocked() {
|
|
316
|
+
return (await this.#sql`
|
|
317
|
+
SELECT ${this.#edgeCols}
|
|
318
|
+
FROM edges
|
|
319
|
+
WHERE blocked = true
|
|
320
|
+
AND deleted_at IS NULL
|
|
321
|
+
`).map(rowToEdge);
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* `sql.begin`-backed atomicity. Same flattening pattern as PgDataset —
|
|
325
|
+
* nested re-entry skips the inner begin and runs on the existing tx handle
|
|
326
|
+
* so an inner throw propagates to the outer `sql.begin` for rollback.
|
|
327
|
+
*/
|
|
328
|
+
async withTransaction(fn) {
|
|
329
|
+
if (this.#inTx) return fn(this);
|
|
330
|
+
return this.#sql.begin(async (txSql) => {
|
|
331
|
+
return fn(new PgEdgesStore({
|
|
332
|
+
sql: txSql,
|
|
333
|
+
now: this.#now,
|
|
334
|
+
inTx: true,
|
|
335
|
+
txScopeId: this.txScopeId,
|
|
336
|
+
dropBatchSize: this.#dropBatchSize
|
|
337
|
+
}));
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Engine-only. Returns a tx-bound clone on the supplied `TransactionSql`,
|
|
342
|
+
* for `PgDataset.withScopedTransaction`. The caller has verified
|
|
343
|
+
* the txScopeIds match.
|
|
344
|
+
*/
|
|
345
|
+
_enlistInTx(txSql) {
|
|
346
|
+
return new PgEdgesStore({
|
|
347
|
+
sql: txSql,
|
|
348
|
+
now: this.#now,
|
|
349
|
+
inTx: true,
|
|
350
|
+
txScopeId: this.txScopeId,
|
|
351
|
+
dropBatchSize: this.#dropBatchSize
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
};
|
|
355
|
+
function buildRef(ns, type, id, bridged) {
|
|
356
|
+
const ref = {
|
|
357
|
+
ns,
|
|
358
|
+
type,
|
|
359
|
+
id
|
|
360
|
+
};
|
|
361
|
+
if (bridged) ref.bridged = true;
|
|
362
|
+
return ref;
|
|
363
|
+
}
|
|
364
|
+
function rowToEdge(row) {
|
|
365
|
+
if (!VALID_POLICIES.has(row.policy)) throw new Error(`PgEdgesStore: unknown policy '${row.policy}' on edge id=${row.id}`);
|
|
366
|
+
const edge = {
|
|
367
|
+
id: row.id,
|
|
368
|
+
from: buildRef(row.from_ns, row.from_type, row.from_id, row.from_bridged),
|
|
369
|
+
to: buildRef(row.to_ns, row.to_type, row.to_id, row.to_bridged),
|
|
370
|
+
fieldName: row.field_name,
|
|
371
|
+
policy: row.policy,
|
|
372
|
+
createdAt: row.created_at
|
|
373
|
+
};
|
|
374
|
+
if (row.deleted_at !== null) edge.deletedAt = row.deleted_at;
|
|
375
|
+
if (row.last_swept_at !== null) edge.lastSweptAt = row.last_swept_at;
|
|
376
|
+
if (row.blocked) edge.blocked = true;
|
|
377
|
+
return edge;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
//#endregion
|
|
381
|
+
export { PgEdgesStore as t };
|
|
@@ -949,6 +949,11 @@ async function handleRecallRequest(input, deps) {
|
|
|
949
949
|
const problem = rowFilterProblem(where);
|
|
950
950
|
if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"where" ${problem}`), { code: "bad_request" });
|
|
951
951
|
}
|
|
952
|
+
const rowFields = input.rowFields;
|
|
953
|
+
if (rowFields !== void 0) {
|
|
954
|
+
const problem = rowFieldsProblem(rowFields);
|
|
955
|
+
if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"rowFields" ${problem}`), { code: "bad_request" });
|
|
956
|
+
}
|
|
952
957
|
const type = input.type;
|
|
953
958
|
if (type === void 0 || type === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `type`, the type to search. There is no default."), { code: "bad_request" });
|
|
954
959
|
if (typeof input.query !== "string" || input.query.trim() === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `query`, the text to search for."), { code: "bad_request" });
|
|
@@ -1166,7 +1171,7 @@ async function handleRecallRequest(input, deps) {
|
|
|
1166
1171
|
}
|
|
1167
1172
|
const t3 = performance.now();
|
|
1168
1173
|
return {
|
|
1169
|
-
rows: fusedRows,
|
|
1174
|
+
rows: rowFields === true ? fusedRows : fusedRows.map((r) => projectRow(r, rowFields)),
|
|
1170
1175
|
model: denseRan ? embedder?.modelId ?? "" : "",
|
|
1171
1176
|
arms: {
|
|
1172
1177
|
dense: denseRan,
|
|
@@ -1212,6 +1217,23 @@ function extractTargets(namespace, type, rows) {
|
|
|
1212
1217
|
}
|
|
1213
1218
|
return out;
|
|
1214
1219
|
}
|
|
1220
|
+
/** Why `rowFields` is malformed, or `null` when it is `true` or an object of `field: true`. */
|
|
1221
|
+
function rowFieldsProblem(value) {
|
|
1222
|
+
if (value === true) return null;
|
|
1223
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return "must be true (the whole body) or an object of field: true, e.g. { \"body\": true }";
|
|
1224
|
+
for (const [field, v] of Object.entries(value)) if (v !== true) return `field "${field}" must be true`;
|
|
1225
|
+
return null;
|
|
1226
|
+
}
|
|
1227
|
+
/** Narrow a hit's `row` to `id` plus `rowFields`, or plus the handle fields when absent. */
|
|
1228
|
+
function projectRow(hit, rowFields) {
|
|
1229
|
+
const row = {};
|
|
1230
|
+
const keep = rowFields === void 0 ? HANDLE_FIELDS : Object.keys(rowFields);
|
|
1231
|
+
for (const field of ["id", ...keep]) if (Object.hasOwn(hit.row, field)) row[field] = hit.row[field];
|
|
1232
|
+
return {
|
|
1233
|
+
...hit,
|
|
1234
|
+
row
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1215
1237
|
|
|
1216
1238
|
//#endregion
|
|
1217
1239
|
//#region ../../plugins/substrate-recall/src/params.ts
|
|
@@ -1265,7 +1287,8 @@ const ROUTE_PARAMS = [
|
|
|
1265
1287
|
"returnVector",
|
|
1266
1288
|
"lexical",
|
|
1267
1289
|
"explain",
|
|
1268
|
-
"where"
|
|
1290
|
+
"where",
|
|
1291
|
+
"rowFields"
|
|
1269
1292
|
];
|
|
1270
1293
|
function parseBody$3(req) {
|
|
1271
1294
|
const raw = req.body;
|
|
@@ -1325,6 +1348,7 @@ function parseBody$3(req) {
|
|
|
1325
1348
|
input.explain = obj.explain;
|
|
1326
1349
|
}
|
|
1327
1350
|
if (obj.where !== void 0) input.where = obj.where;
|
|
1351
|
+
if (obj.rowFields !== void 0) input.rowFields = obj.rowFields;
|
|
1328
1352
|
return input;
|
|
1329
1353
|
}
|
|
1330
1354
|
/**
|
|
@@ -2466,7 +2490,8 @@ const MCP_PARAMS = [
|
|
|
2466
2490
|
"returnVector",
|
|
2467
2491
|
"lexical",
|
|
2468
2492
|
"explain",
|
|
2469
|
-
"where"
|
|
2493
|
+
"where",
|
|
2494
|
+
"rowFields"
|
|
2470
2495
|
];
|
|
2471
2496
|
function recallMcpTool(cfg) {
|
|
2472
2497
|
return {
|
|
@@ -2514,6 +2539,10 @@ function recallMcpTool(cfg) {
|
|
|
2514
2539
|
where: {
|
|
2515
2540
|
type: "object",
|
|
2516
2541
|
description: "Row-field filter applied inside every candidate source, so k fills with matching rows: field -> { eq: scalar | null } or { in: [scalar | null, ...] }, every field must match. eq null matches a missing or null field. Example: { \"project\": { \"in\": [\"substrate\", null] } }."
|
|
2542
|
+
},
|
|
2543
|
+
rowFields: {
|
|
2544
|
+
type: ["object", "boolean"],
|
|
2545
|
+
description: "Row-body projection. Omitted, each hit's `row` carries `id` plus its handle fields (name, title, topic, synopsis, description), enough to decide on. field -> true keeps just those fields plus `id`, e.g. { \"synopsis\": true, \"body\": true }; true returns the whole body."
|
|
2517
2546
|
}
|
|
2518
2547
|
},
|
|
2519
2548
|
required: ["query", "type"]
|
|
@@ -3035,12 +3064,14 @@ function rethrowCoded(err) {
|
|
|
3035
3064
|
throw err;
|
|
3036
3065
|
}
|
|
3037
3066
|
/**
|
|
3038
|
-
* Keep only the
|
|
3067
|
+
* Keep only the selected keys of `fields`; `undefined` fields = passthrough.
|
|
3039
3068
|
* `fields: {}` strips every key, matching the compose layer's projectEnvelope.
|
|
3069
|
+
* A nested `row: { … }` selection keeps `row`, which the core has already
|
|
3070
|
+
* narrowed through `rowFields`.
|
|
3040
3071
|
*/
|
|
3041
3072
|
function projectHit(row, fields) {
|
|
3042
3073
|
if (fields === void 0) return row;
|
|
3043
|
-
const keep = Object.entries(fields).filter(([, v]) => v === true).map(([k]) => k);
|
|
3074
|
+
const keep = Object.entries(fields).filter(([k, v]) => v === true || k === "row" && isPlainObject(v)).map(([k]) => k);
|
|
3044
3075
|
const out = {};
|
|
3045
3076
|
for (const k of keep) if (k in row) out[k] = row[k];
|
|
3046
3077
|
return out;
|
|
@@ -3063,6 +3094,9 @@ function buildRecallFrameSurface(deps) {
|
|
|
3063
3094
|
const salience = deps.salience ?? DEFAULT_SALIENCE_CONFIG;
|
|
3064
3095
|
const hitResolver = async (ctx, args) => {
|
|
3065
3096
|
const input = parseHitParams(args.params);
|
|
3097
|
+
const rowSelection = args.fields?.row;
|
|
3098
|
+
if (rowSelection === true) input.rowFields = true;
|
|
3099
|
+
else if (isPlainObject(rowSelection)) input.rowFields = rowSelection;
|
|
3066
3100
|
const bound = bindPolicy(deps.checkReadPolicy, callerFromContext(ctx));
|
|
3067
3101
|
const boundWrite = bindPolicy(deps.checkWritePolicy, callerFromContext(ctx));
|
|
3068
3102
|
let out;
|
|
@@ -3144,6 +3178,9 @@ function buildRecallFrameSurface(deps) {
|
|
|
3144
3178
|
}
|
|
3145
3179
|
};
|
|
3146
3180
|
}
|
|
3181
|
+
function isPlainObject(v) {
|
|
3182
|
+
return v !== null && typeof v === "object" && !Array.isArray(v);
|
|
3183
|
+
}
|
|
3147
3184
|
|
|
3148
3185
|
//#endregion
|
|
3149
3186
|
//#region ../../plugins/substrate-recall/src/lexical-reindex-core.ts
|
package/dist/recall-bootstrap.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
2
|
import { Pt as defineBootstrap } from "./src-CHp3QyqB.js";
|
|
3
3
|
import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
|
|
4
|
-
import { t as createRecallPlugin } from "./plugin-
|
|
4
|
+
import { t as createRecallPlugin } from "./plugin-BG8E04wi.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
|
|
7
7
|
//#region ../../packages/substrate-embedder/src/types.ts
|
package/dist/recall.d.ts
CHANGED
|
@@ -724,6 +724,14 @@ interface RecallInput {
|
|
|
724
724
|
* or null field. A malformed filter is refused with `bad_request`.
|
|
725
725
|
*/
|
|
726
726
|
where?: KnnRowFilter;
|
|
727
|
+
/**
|
|
728
|
+
* Row-body projection. Field -> `true` keeps those keys plus `id`; `true`
|
|
729
|
+
* keeps the whole body. Absent → `id` plus the handle fields
|
|
730
|
+
* (`HANDLE_FIELDS`) the row has. Applied after row filtering and policy,
|
|
731
|
+
* which still see the full body. A malformed value is refused with
|
|
732
|
+
* `bad_request`.
|
|
733
|
+
*/
|
|
734
|
+
rowFields?: Record<string, true> | true;
|
|
727
735
|
}
|
|
728
736
|
interface RecallRow {
|
|
729
737
|
/**
|
package/dist/recall.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-
|
|
3
|
-
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-
|
|
2
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-DQ1U9Yhs.js";
|
|
3
|
+
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-BG8E04wi.js";
|
|
4
4
|
|
|
5
5
|
export { CLUSTERER_BACKEND, CLUSTERER_BACKEND_VERSION, CLUSTER_FIELDS, CLUSTER_MEMBER_FIELDS, CRYSTALLISE_IN_FLIGHT_CODE, DEFAULT_CRYSTALLISE_CONFIG, EMBEDDINGS_FETCH_MAX_REFS, EmbedSourceRegistry, InMemoryClusterStore, InMemoryEmbeddingsStore, InMemoryLexicalStore, PgClusterStore, PgEmbeddingsStore, PgLexicalStore, RECALL_VERSION, REINDEX_DEFAULT_LIMIT, REINDEX_MAX_LIMIT, RRF_K, SALIENCE_BOOST_MAX_TARGETS, SELECT_CANDIDATES_MAX_LIMIT, armRanks, buildClusterSurface, checkParams, createCrystalliseGuard, createEmbedStats, createRecallPlugin, crystalliseRouteHandler, embedAndStore, embedRowReporting, embedStatsRouteHandler, embeddingSimilarOperator, embeddingsFetchRouteHandler, embeddingsMigration, fuseByRank, handleCrystallise, handleEmbedStatsRequest, handleLexicalCoverage, handleRecallRequest, handleReindex, l2normalize, lexicalMigration, makeEmbedPostWrite, makeLexicalClassify, makeLexicalPostWrite, makeReembed, memberKey, recallClustersMigration, recallMcpTool, recallRouteHandler, reindexRouteHandler, runClusterer, salienceBoostRouteHandler, unknownParamMessage, unknownParams, validateCrystalliseInput, validateLexicalCoverageInput, validateReindexInput, validateSelector };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-QuJN9pG_.js";
|
|
3
2
|
import { n as txScopeIdFor } from "./src-CXKnqeTW.js";
|
|
3
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-BG8E04wi.js";
|
|
4
4
|
|
|
5
5
|
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
-
var version = "1.0.0-alpha.
|
|
6
|
+
var version = "1.0.0-alpha.4";
|
|
7
7
|
|
|
8
8
|
//#endregion
|
|
9
9
|
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
package/dist/substrate-admin.js
CHANGED
|
@@ -4,7 +4,7 @@ import { d as buildAuditRow, f as AUDIT_EVENT_TYPE } from "./src-CHp3QyqB.js";
|
|
|
4
4
|
import { d as resolveDefaultKidPath, u as resolveDefaultKeyPath } from "./env-utils-CGNazUNt.js";
|
|
5
5
|
import { E as describeGrant, I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./row-DZ0Fnpc6.js";
|
|
6
6
|
import { S as reconcileIdentities } from "./src-DMzc3HfW.js";
|
|
7
|
-
import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-
|
|
7
|
+
import { A as describeAdminTarget, B as version, C as CLI_ACTOR_KID, D as listActiveAdminKeys, E as generateAdminKeypair, F as RESTART_NOTE, I as USAGE, L as argvError, M as resolveAdminTarget, N as ARGV_SPEC, O as readEd25519PublicKey, P as validateArgv, S as BootstrapError, T as forUpdate, _ as REVOKE_EXIT_CODES, a as formatRegisterOutput, b as runKeyPatchViaRoute, c as runRegisterViaRoute, d as formatDoctorReport, f as runDoctor, g as bootstrapKidPath, h as runBootstrap, i as runOnboard, j as resolveAdminAuth, k as readKeyById, l as grantLines, m as parseBootstrapArgs, n as formatChecklist, o as parseRegisterArgs, p as formatBootstrapOutput, r as onboardCommand, s as runRegister, t as openPgDataset, u as doctorCommand, v as UPDATE_EXIT_CODES, w as base64Url, x as ACTOR_HINT, y as runKeyListViaRoute, z as AdminCliError } from "./dataset-Bd18RP3O.js";
|
|
8
8
|
import { existsSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
9
9
|
import { dirname } from "node:path";
|
|
10
10
|
import { createHash, randomUUID } from "node:crypto";
|