@mnmal-ai/substrate 1.0.0-alpha.0 → 1.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -9
- package/dist/boot-store-BVqfFxh1.js +4060 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-Cp4PaIYO.d.ts} +2 -87
- package/dist/cli-hSCBywhE.js +3126 -0
- package/dist/client.d.ts +130 -8
- package/dist/client.js +6 -5
- package/dist/{plugin-t2g2wX0b.d.ts → dataset-CejF-Fr_.d.ts} +1640 -1320
- package/dist/{agent-token-BIboWOU0.js → env-utils-CGNazUNt.js} +31 -2
- package/dist/{errors-D4vKuwz8.d.ts → errors-7mycvNGq.d.ts} +1 -1
- package/dist/{index-D7mSOFlM.d.ts → index-DgSfwx7M.d.ts} +1 -1
- package/dist/node-version-C5FSmwJX.js +26 -0
- package/dist/pack.d.ts +2 -2
- package/dist/pack.js +1 -1
- package/dist/pg-edges-store-Lxsl5ZnK.js +381 -0
- package/dist/{plugin-j1jt1l3n.js → plugin-CtwOINCM.js} +171 -18
- package/dist/proposals-bootstrap.js +6 -4
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +15 -4
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -2
- package/dist/recall-bootstrap.js +13 -4
- package/dist/recall.config.schema.json +12 -0
- package/dist/recall.d.ts +54 -9
- package/dist/recall.js +2 -2
- package/dist/{repl-C52wyfMs.js → repl-DqfHelK7.js} +62 -29
- package/dist/{row-ByLFE2Wx.d.ts → row-BKKCXIG6.d.ts} +5 -3
- package/dist/run-B5dTbZ3g.js +1997 -0
- package/dist/{schema-g4FQLr0V.d.ts → schema-DoSc-4ig.d.ts} +44 -7
- package/dist/script-C6DSa_Na.js +134 -0
- package/dist/sdk.d.ts +90 -12
- package/dist/sdk.js +5 -110
- package/dist/{src-D91yij6C.js → src-5SJ5GEIM.js} +1557 -327
- package/dist/{src-DuopV8u7.js → src-7CUSMK66.js} +148 -19
- package/dist/{src-CrEgiMJQ.js → src-Bi849yhn.js} +39 -15
- package/dist/{src-D88LvCX8.js → src-CqT-QWZ_.js} +697 -57
- package/dist/src-CyvV7dCK.js +4333 -0
- package/dist/{row-DK7lPZVe.js → src-Da0t6si5.js} +11 -11
- package/dist/{src-C4DzaIQZ.js → src-hZb8wfvg.js} +12 -1
- package/dist/substrate-admin.js +17 -781
- package/dist/substrate.js +4 -5471
- package/dist/testing.d.ts +557 -8
- package/dist/testing.js +39 -3
- package/dist/{ws-C72iVS5T.d.ts → ws-ByZyHk7m.d.ts} +3 -3
- package/package.json +21 -20
- package/dist/dataset-BXkus75W.js +0 -1527
- package/dist/env-utils-B7SGmoeJ.js +0 -21
- package/dist/src-CzxvmhFn.js +0 -2327
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
2
|
+
import { J as applyObligations, Vt as ResolverFrameError, ft as buildContext, mt as jsonResponse } from "./src-5SJ5GEIM.js";
|
|
3
|
+
import "./src-hZb8wfvg.js";
|
|
3
4
|
import { Worker } from "node:worker_threads";
|
|
4
5
|
|
|
5
6
|
//#region ../../plugins/substrate-recall/src/candidate-selection.ts
|
|
@@ -949,8 +950,14 @@ async function handleRecallRequest(input, deps) {
|
|
|
949
950
|
const problem = rowFilterProblem(where);
|
|
950
951
|
if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"where" ${problem}`), { code: "bad_request" });
|
|
951
952
|
}
|
|
953
|
+
const rowFields = input.rowFields;
|
|
954
|
+
if (rowFields !== void 0) {
|
|
955
|
+
const problem = rowFieldsProblem(rowFields);
|
|
956
|
+
if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"rowFields" ${problem}`), { code: "bad_request" });
|
|
957
|
+
}
|
|
952
958
|
const type = input.type;
|
|
953
959
|
if (type === void 0 || type === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `type`, the type to search. There is no default."), { code: "bad_request" });
|
|
960
|
+
if (typeof input.query !== "string" || input.query.trim() === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `query`, the text to search for."), { code: "bad_request" });
|
|
954
961
|
if (input.query.length > 8192) throw Object.assign(/* @__PURE__ */ new Error(`recall: query is longer than ${MAX_RECALL_QUERY_CHARS} characters`), { code: "bad_request" });
|
|
955
962
|
const k = clamp(input.k ?? 10, 1, 100);
|
|
956
963
|
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
@@ -976,7 +983,8 @@ async function handleRecallRequest(input, deps) {
|
|
|
976
983
|
type,
|
|
977
984
|
model: embedder.modelId,
|
|
978
985
|
vector,
|
|
979
|
-
k
|
|
986
|
+
k,
|
|
987
|
+
salienceHalfLifeHours: salience.halfLifeHours
|
|
980
988
|
};
|
|
981
989
|
if (threshold > 0) knnInput.threshold = threshold;
|
|
982
990
|
if (returnVector) knnInput.returnVector = true;
|
|
@@ -1164,7 +1172,7 @@ async function handleRecallRequest(input, deps) {
|
|
|
1164
1172
|
}
|
|
1165
1173
|
const t3 = performance.now();
|
|
1166
1174
|
return {
|
|
1167
|
-
rows: fusedRows,
|
|
1175
|
+
rows: rowFields === true ? fusedRows : fusedRows.map((r) => projectRow(r, rowFields)),
|
|
1168
1176
|
model: denseRan ? embedder?.modelId ?? "" : "",
|
|
1169
1177
|
arms: {
|
|
1170
1178
|
dense: denseRan,
|
|
@@ -1210,6 +1218,23 @@ function extractTargets(namespace, type, rows) {
|
|
|
1210
1218
|
}
|
|
1211
1219
|
return out;
|
|
1212
1220
|
}
|
|
1221
|
+
/** Why `rowFields` is malformed, or `null` when it is `true` or an object of `field: true`. */
|
|
1222
|
+
function rowFieldsProblem(value) {
|
|
1223
|
+
if (value === true) return null;
|
|
1224
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return "must be true (the whole body) or an object of field: true, e.g. { \"body\": true }";
|
|
1225
|
+
for (const [field, v] of Object.entries(value)) if (v !== true) return `field "${field}" must be true`;
|
|
1226
|
+
return null;
|
|
1227
|
+
}
|
|
1228
|
+
/** Narrow a hit's `row` to `id` plus `rowFields`, or plus the handle fields when absent. */
|
|
1229
|
+
function projectRow(hit, rowFields) {
|
|
1230
|
+
const row = {};
|
|
1231
|
+
const keep = rowFields === void 0 ? HANDLE_FIELDS : Object.keys(rowFields);
|
|
1232
|
+
for (const field of ["id", ...keep]) if (Object.hasOwn(hit.row, field)) row[field] = hit.row[field];
|
|
1233
|
+
return {
|
|
1234
|
+
...hit,
|
|
1235
|
+
row
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1213
1238
|
|
|
1214
1239
|
//#endregion
|
|
1215
1240
|
//#region ../../plugins/substrate-recall/src/params.ts
|
|
@@ -1263,7 +1288,8 @@ const ROUTE_PARAMS = [
|
|
|
1263
1288
|
"returnVector",
|
|
1264
1289
|
"lexical",
|
|
1265
1290
|
"explain",
|
|
1266
|
-
"where"
|
|
1291
|
+
"where",
|
|
1292
|
+
"rowFields"
|
|
1267
1293
|
];
|
|
1268
1294
|
function parseBody$3(req) {
|
|
1269
1295
|
const raw = req.body;
|
|
@@ -1323,6 +1349,7 @@ function parseBody$3(req) {
|
|
|
1323
1349
|
input.explain = obj.explain;
|
|
1324
1350
|
}
|
|
1325
1351
|
if (obj.where !== void 0) input.where = obj.where;
|
|
1352
|
+
if (obj.rowFields !== void 0) input.rowFields = obj.rowFields;
|
|
1326
1353
|
return input;
|
|
1327
1354
|
}
|
|
1328
1355
|
/**
|
|
@@ -1502,14 +1529,35 @@ function rethrowCoded$1(err) {
|
|
|
1502
1529
|
* answers both visibility and counts; `check` is cached per scope.
|
|
1503
1530
|
*/
|
|
1504
1531
|
async function resolveClusterVisibility(record, deps, ctx) {
|
|
1505
|
-
if (deps.checkReadPolicy === void 0) return
|
|
1532
|
+
if (deps.checkReadPolicy === void 0) return storedVisibility(record);
|
|
1533
|
+
const members = await deps.clusters.listMembers(record.id, { includeDeparted: true });
|
|
1534
|
+
return visibilityOf(members, await readableMembers(members, deps, ctx));
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
* `resolveClusterVisibility` for a whole listing: one member read and one
|
|
1538
|
+
* readability check across every cluster, rather than two per cluster.
|
|
1539
|
+
*/
|
|
1540
|
+
async function resolveListVisibility(records, deps, ctx) {
|
|
1541
|
+
if (deps.checkReadPolicy === void 0 || records.length === 0) return records.map(storedVisibility);
|
|
1542
|
+
const members = await deps.clusters.listMembers(records.map((r) => r.id), { includeDeparted: true });
|
|
1543
|
+
const readable = await readableMembers(members, deps, ctx);
|
|
1544
|
+
const byCluster = /* @__PURE__ */ new Map();
|
|
1545
|
+
for (const m of members) {
|
|
1546
|
+
const list = byCluster.get(m.clusterId);
|
|
1547
|
+
if (list === void 0) byCluster.set(m.clusterId, [m]);
|
|
1548
|
+
else list.push(m);
|
|
1549
|
+
}
|
|
1550
|
+
return records.map((r) => visibilityOf(byCluster.get(r.id) ?? [], readable));
|
|
1551
|
+
}
|
|
1552
|
+
function storedVisibility(record) {
|
|
1553
|
+
return {
|
|
1506
1554
|
visible: true,
|
|
1507
1555
|
memberCount: record.memberCount,
|
|
1508
1556
|
departedCount: record.departedCount,
|
|
1509
1557
|
labelReadable: true
|
|
1510
1558
|
};
|
|
1511
|
-
|
|
1512
|
-
|
|
1559
|
+
}
|
|
1560
|
+
function visibilityOf(members, readable) {
|
|
1513
1561
|
let liveTotal = 0;
|
|
1514
1562
|
let memberCount = 0;
|
|
1515
1563
|
let departedCount = 0;
|
|
@@ -1559,10 +1607,11 @@ function buildClusterSurface(deps) {
|
|
|
1559
1607
|
return [project(clusterRow(one, vis), args.fields)];
|
|
1560
1608
|
}
|
|
1561
1609
|
const rows = await deps.clusters.listClusters(p.includeRetired === true ? { includeRetired: true } : {});
|
|
1610
|
+
const visibility = await resolveListVisibility(rows, deps, ctx);
|
|
1562
1611
|
const visible = [];
|
|
1563
|
-
for (const c of rows) {
|
|
1564
|
-
const vis =
|
|
1565
|
-
if (vis
|
|
1612
|
+
for (const [i, c] of rows.entries()) {
|
|
1613
|
+
const vis = visibility[i];
|
|
1614
|
+
if (vis?.visible === true) visible.push(project(clusterRow(c, vis), args.fields));
|
|
1566
1615
|
}
|
|
1567
1616
|
return visible;
|
|
1568
1617
|
};
|
|
@@ -1739,7 +1788,7 @@ async function embedRowReporting(row, opts) {
|
|
|
1739
1788
|
if (opts.serializeRow) source = await opts.serializeRow(row);
|
|
1740
1789
|
else if (opts.fields && opts.fields.length > 0) source = opts.fields.map((f) => row[f]).filter((v) => typeof v === "string" && v.length > 0).join("\n");
|
|
1741
1790
|
else source = "";
|
|
1742
|
-
if (source === null || source === void 0) {
|
|
1791
|
+
if (source === null || source === void 0 || source.trim() === "") {
|
|
1743
1792
|
await opts.store.delete({
|
|
1744
1793
|
rowNamespace: opts.namespace,
|
|
1745
1794
|
rowType: opts.type,
|
|
@@ -2442,7 +2491,8 @@ const MCP_PARAMS = [
|
|
|
2442
2491
|
"returnVector",
|
|
2443
2492
|
"lexical",
|
|
2444
2493
|
"explain",
|
|
2445
|
-
"where"
|
|
2494
|
+
"where",
|
|
2495
|
+
"rowFields"
|
|
2446
2496
|
];
|
|
2447
2497
|
function recallMcpTool(cfg) {
|
|
2448
2498
|
return {
|
|
@@ -2490,6 +2540,10 @@ function recallMcpTool(cfg) {
|
|
|
2490
2540
|
where: {
|
|
2491
2541
|
type: "object",
|
|
2492
2542
|
description: "Row-field filter applied inside every candidate source, so k fills with matching rows: field -> { eq: scalar | null } or { in: [scalar | null, ...] }, every field must match. eq null matches a missing or null field. Example: { \"project\": { \"in\": [\"substrate\", null] } }."
|
|
2543
|
+
},
|
|
2544
|
+
rowFields: {
|
|
2545
|
+
type: ["object", "boolean"],
|
|
2546
|
+
description: "Row-body projection. Omitted, each hit's `row` carries `id` plus its handle fields (name, title, topic, synopsis, description), enough to decide on. field -> true keeps just those fields plus `id`, e.g. { \"synopsis\": true, \"body\": true }; true returns the whole body."
|
|
2493
2547
|
}
|
|
2494
2548
|
},
|
|
2495
2549
|
required: ["query", "type"]
|
|
@@ -2725,7 +2779,7 @@ async function alignVectorDimensions(s, opts) {
|
|
|
2725
2779
|
const { dimension } = opts;
|
|
2726
2780
|
if (!Number.isInteger(dimension) || dimension <= 0) throw new Error(`alignVectorDimensions: dimension must be a positive integer (got ${dimension})`);
|
|
2727
2781
|
return await s.begin(async (tx) => {
|
|
2728
|
-
await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align'))`;
|
|
2782
|
+
await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align:' || current_schema()))`;
|
|
2729
2783
|
return alignLocked(tx, opts);
|
|
2730
2784
|
});
|
|
2731
2785
|
}
|
|
@@ -2752,6 +2806,50 @@ async function alignLocked(s, opts) {
|
|
|
2752
2806
|
};
|
|
2753
2807
|
}
|
|
2754
2808
|
|
|
2809
|
+
//#endregion
|
|
2810
|
+
//#region ../../plugins/substrate-recall/src/embed-backfill.ts
|
|
2811
|
+
async function runEmbedBackfill(deps) {
|
|
2812
|
+
const result = {
|
|
2813
|
+
embedded: 0,
|
|
2814
|
+
unindexable: 0,
|
|
2815
|
+
failed: 0
|
|
2816
|
+
};
|
|
2817
|
+
const staleRowIds = deps.store.staleRowIds?.bind(deps.store);
|
|
2818
|
+
if (staleRowIds === void 0) return result;
|
|
2819
|
+
for (const { namespace, type } of deps.registry.types()) {
|
|
2820
|
+
const reembed = deps.registry.get(namespace, type)?.reembed;
|
|
2821
|
+
const dataset = deps.resolveDataset(namespace);
|
|
2822
|
+
if (reembed === void 0 || dataset === null) continue;
|
|
2823
|
+
const key = `${namespace}/${type}`;
|
|
2824
|
+
const ids = await staleRowIds({
|
|
2825
|
+
namespace,
|
|
2826
|
+
type,
|
|
2827
|
+
afterId: deps.cursors.get(key) ?? "",
|
|
2828
|
+
limit: deps.perTypeLimit
|
|
2829
|
+
});
|
|
2830
|
+
if (ids.length < deps.perTypeLimit) deps.cursors.delete(key);
|
|
2831
|
+
else deps.cursors.set(key, ids[ids.length - 1]);
|
|
2832
|
+
if (ids.length === 0) continue;
|
|
2833
|
+
const { rows } = await dataset.read(type, {
|
|
2834
|
+
ids,
|
|
2835
|
+
limit: ids.length
|
|
2836
|
+
});
|
|
2837
|
+
for (const row of rows) {
|
|
2838
|
+
const outcome = await reembed(row);
|
|
2839
|
+
if (outcome.status === "embedded") result.embedded++;
|
|
2840
|
+
else if (outcome.status === "unindexable") result.unindexable++;
|
|
2841
|
+
else if (outcome.status === "failed") {
|
|
2842
|
+
result.failed++;
|
|
2843
|
+
if (outcome.code === "unavailable") return {
|
|
2844
|
+
...result,
|
|
2845
|
+
stopped: outcome.error
|
|
2846
|
+
};
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
}
|
|
2850
|
+
return result;
|
|
2851
|
+
}
|
|
2852
|
+
|
|
2755
2853
|
//#endregion
|
|
2756
2854
|
//#region ../../plugins/substrate-recall/src/expansion-resolver.ts
|
|
2757
2855
|
function makeExpansionResolver(registry, edges, cfg) {
|
|
@@ -3011,12 +3109,14 @@ function rethrowCoded(err) {
|
|
|
3011
3109
|
throw err;
|
|
3012
3110
|
}
|
|
3013
3111
|
/**
|
|
3014
|
-
* Keep only the
|
|
3112
|
+
* Keep only the selected keys of `fields`; `undefined` fields = passthrough.
|
|
3015
3113
|
* `fields: {}` strips every key, matching the compose layer's projectEnvelope.
|
|
3114
|
+
* A nested `row: { … }` selection keeps `row`, which the core has already
|
|
3115
|
+
* narrowed through `rowFields`.
|
|
3016
3116
|
*/
|
|
3017
3117
|
function projectHit(row, fields) {
|
|
3018
3118
|
if (fields === void 0) return row;
|
|
3019
|
-
const keep = Object.entries(fields).filter(([, v]) => v === true).map(([k]) => k);
|
|
3119
|
+
const keep = Object.entries(fields).filter(([k, v]) => v === true || k === "row" && isPlainObject(v)).map(([k]) => k);
|
|
3020
3120
|
const out = {};
|
|
3021
3121
|
for (const k of keep) if (k in row) out[k] = row[k];
|
|
3022
3122
|
return out;
|
|
@@ -3039,6 +3139,9 @@ function buildRecallFrameSurface(deps) {
|
|
|
3039
3139
|
const salience = deps.salience ?? DEFAULT_SALIENCE_CONFIG;
|
|
3040
3140
|
const hitResolver = async (ctx, args) => {
|
|
3041
3141
|
const input = parseHitParams(args.params);
|
|
3142
|
+
const rowSelection = args.fields?.row;
|
|
3143
|
+
if (rowSelection === true) input.rowFields = true;
|
|
3144
|
+
else if (isPlainObject(rowSelection)) input.rowFields = rowSelection;
|
|
3042
3145
|
const bound = bindPolicy(deps.checkReadPolicy, callerFromContext(ctx));
|
|
3043
3146
|
const boundWrite = bindPolicy(deps.checkWritePolicy, callerFromContext(ctx));
|
|
3044
3147
|
let out;
|
|
@@ -3120,6 +3223,9 @@ function buildRecallFrameSurface(deps) {
|
|
|
3120
3223
|
}
|
|
3121
3224
|
};
|
|
3122
3225
|
}
|
|
3226
|
+
function isPlainObject(v) {
|
|
3227
|
+
return v !== null && typeof v === "object" && !Array.isArray(v);
|
|
3228
|
+
}
|
|
3123
3229
|
|
|
3124
3230
|
//#endregion
|
|
3125
3231
|
//#region ../../plugins/substrate-recall/src/lexical-reindex-core.ts
|
|
@@ -3593,9 +3699,13 @@ function makeSpreadResolver(registry, edges) {
|
|
|
3593
3699
|
//#endregion
|
|
3594
3700
|
//#region ../../plugins/substrate-recall/src/plugin.ts
|
|
3595
3701
|
const DEFAULT_CLUSTER_DIMENSION = 768;
|
|
3702
|
+
const DEFAULT_EMBED_BACKFILL_INTERVAL_MS = 6e4;
|
|
3703
|
+
/** Rows per embedded type per pass, so one pass stays a few seconds of embed work. */
|
|
3704
|
+
const EMBED_BACKFILL_PER_TYPE = 50;
|
|
3596
3705
|
const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3e4;
|
|
3706
|
+
const DEFAULT_WRITE_EMBED_TIMEOUT_MS = 3e4;
|
|
3597
3707
|
/** `embedder` with `timeoutMs` applied to every call that does not set its own. */
|
|
3598
|
-
function
|
|
3708
|
+
function withEmbedTimeout(embedder, timeoutMs) {
|
|
3599
3709
|
return {
|
|
3600
3710
|
modelId: embedder.modelId,
|
|
3601
3711
|
dimension: embedder.dimension,
|
|
@@ -3634,7 +3744,7 @@ function makeEmbedSourceContext(resolveDataset) {
|
|
|
3634
3744
|
}
|
|
3635
3745
|
function createRecallPlugin(cfg) {
|
|
3636
3746
|
const stats = cfg.stats ?? createEmbedStats();
|
|
3637
|
-
const queryEmbedder = cfg.embedder === void 0 ? void 0 :
|
|
3747
|
+
const queryEmbedder = cfg.embedder === void 0 ? void 0 : withEmbedTimeout(cfg.embedder, cfg.queryEmbedTimeoutMs ?? 3e4);
|
|
3638
3748
|
const spreadRegistry = new SpreadEdgeRegistry();
|
|
3639
3749
|
const embedSourceRegistry = new EmbedSourceRegistry();
|
|
3640
3750
|
const lexicalSourceRegistry = new LexicalSourceRegistry();
|
|
@@ -3669,11 +3779,19 @@ function createRecallPlugin(cfg) {
|
|
|
3669
3779
|
...cfg.checkReadPolicy !== void 0 ? { checkReadPolicy: cfg.checkReadPolicy } : {},
|
|
3670
3780
|
...cfg.runClusterer !== void 0 ? { runClusterer: cfg.runClusterer } : {}
|
|
3671
3781
|
};
|
|
3782
|
+
let backfillTimer;
|
|
3672
3783
|
return {
|
|
3673
3784
|
name: "substrate-recall",
|
|
3674
3785
|
contractVersion: ">=1.0.0-alpha.0 <2.0.0",
|
|
3786
|
+
storeTables: [
|
|
3787
|
+
"embeddings",
|
|
3788
|
+
"lexical",
|
|
3789
|
+
"recall_clusters",
|
|
3790
|
+
"recall_cluster_members"
|
|
3791
|
+
],
|
|
3675
3792
|
async onKernelReady(ctx) {
|
|
3676
3793
|
if (cfg.embedder === void 0 || ctx.sql === null) return;
|
|
3794
|
+
let dimensionMismatch = false;
|
|
3677
3795
|
const report = await alignVectorDimensions(ctx.sql, {
|
|
3678
3796
|
dimension: cfg.embedder.dimension,
|
|
3679
3797
|
...cfg.migration?.hnswM !== void 0 ? { hnswM: cfg.migration.hnswM } : {},
|
|
@@ -3681,6 +3799,7 @@ function createRecallPlugin(cfg) {
|
|
|
3681
3799
|
});
|
|
3682
3800
|
for (const [column, state] of [["embeddings.embedding", report.embeddings], ["recall_clusters.centroid", report.centroids]]) {
|
|
3683
3801
|
if (state !== "populated") continue;
|
|
3802
|
+
dimensionMismatch = true;
|
|
3684
3803
|
const found = column.startsWith("embeddings") ? report.found.embeddings : report.found.centroids;
|
|
3685
3804
|
ctx.metric("recall_dimension_mismatch", {
|
|
3686
3805
|
column,
|
|
@@ -3690,6 +3809,37 @@ function createRecallPlugin(cfg) {
|
|
|
3690
3809
|
});
|
|
3691
3810
|
process.emitWarning(`${column} is vector(${found}) and holds rows, but the embedder is ${cfg.embedder.modelId} at ${cfg.embedder.dimension}. Writes to it will fail until the rows are cleared (they are derived) and re-embedded: delete them, restart, then POST /substrate/v1/embed/reindex.`, { code: "RECALL_DIMENSION_MISMATCH" });
|
|
3692
3811
|
}
|
|
3812
|
+
const intervalMs = cfg.embedBackfillIntervalMs ?? DEFAULT_EMBED_BACKFILL_INTERVAL_MS;
|
|
3813
|
+
const resolveDataset = cfg.resolveDataset;
|
|
3814
|
+
if (dimensionMismatch || intervalMs === 0 || resolveDataset === void 0) return;
|
|
3815
|
+
if (!ctx.background) return;
|
|
3816
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
3817
|
+
let running = false;
|
|
3818
|
+
clearInterval(backfillTimer);
|
|
3819
|
+
backfillTimer = setInterval(() => {
|
|
3820
|
+
if (running) return;
|
|
3821
|
+
running = true;
|
|
3822
|
+
runEmbedBackfill({
|
|
3823
|
+
registry: embedSourceRegistry,
|
|
3824
|
+
store: cfg.store,
|
|
3825
|
+
resolveDataset,
|
|
3826
|
+
cursors,
|
|
3827
|
+
perTypeLimit: EMBED_BACKFILL_PER_TYPE
|
|
3828
|
+
}).then((r) => {
|
|
3829
|
+
if (r.embedded + r.failed === 0 && r.stopped === void 0) return;
|
|
3830
|
+
ctx.metric("recall_embed_backfill", r);
|
|
3831
|
+
process.stdout.write(`recall.embed_backfill ${JSON.stringify(r)}\n`);
|
|
3832
|
+
}).catch((err) => {
|
|
3833
|
+
process.stderr.write(`recall.embed_backfill failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
3834
|
+
}).finally(() => {
|
|
3835
|
+
running = false;
|
|
3836
|
+
});
|
|
3837
|
+
}, intervalMs);
|
|
3838
|
+
backfillTimer.unref();
|
|
3839
|
+
},
|
|
3840
|
+
onKernelStop() {
|
|
3841
|
+
clearInterval(backfillTimer);
|
|
3842
|
+
backfillTimer = void 0;
|
|
3693
3843
|
},
|
|
3694
3844
|
migrations: [
|
|
3695
3845
|
embeddingsMigration(cfg.migration ?? {}),
|
|
@@ -3826,7 +3976,10 @@ function createRecallPlugin(cfg) {
|
|
|
3826
3976
|
stats,
|
|
3827
3977
|
...embedSourceContext !== void 0 ? { embedSourceContext } : {}
|
|
3828
3978
|
};
|
|
3829
|
-
const hook = makeEmbedPostWrite(
|
|
3979
|
+
const hook = makeEmbedPostWrite({
|
|
3980
|
+
...binding,
|
|
3981
|
+
embedder: withEmbedTimeout(binding.embedder, cfg.writeEmbedTimeoutMs ?? 3e4)
|
|
3982
|
+
});
|
|
3830
3983
|
out[typeName] = composeHooks(out[typeName], hook);
|
|
3831
3984
|
embedSourceRegistry.add({
|
|
3832
3985
|
namespace: namespace.namespace,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
3
|
-
import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-
|
|
2
|
+
import { Qt as defineBootstrap } from "./src-5SJ5GEIM.js";
|
|
3
|
+
import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-CqT-QWZ_.js";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
|
|
6
6
|
//#region ../../plugins/substrate-proposals/src/bootstrap.ts
|
|
@@ -22,14 +22,16 @@ import { z } from "zod";
|
|
|
22
22
|
*/
|
|
23
23
|
const ProposalsBootstrapConfigSchema = z.object({
|
|
24
24
|
ownedNamespaces: z.array(z.string().min(1)).default([]),
|
|
25
|
-
proposeCapability: z.string().min(1).default("substrate.propose")
|
|
25
|
+
proposeCapability: z.string().min(1).default("substrate.propose"),
|
|
26
|
+
indexCapability: z.string().min(1).default("substrate.index")
|
|
26
27
|
}).strict();
|
|
27
28
|
const bootstrap = defineBootstrap.coordinator({
|
|
28
29
|
schema: ProposalsBootstrapConfigSchema,
|
|
29
30
|
build: (config) => createProposalsPlugin({
|
|
30
31
|
ownedNamespaces: config.ownedNamespaces,
|
|
31
32
|
pubsub: createNoopPubSub(),
|
|
32
|
-
proposeCapability: config.proposeCapability
|
|
33
|
+
proposeCapability: config.proposeCapability,
|
|
34
|
+
indexCapability: config.indexCapability
|
|
33
35
|
})
|
|
34
36
|
});
|
|
35
37
|
|
package/dist/proposals.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { a as ResolverFrameError } from "./errors-
|
|
1
|
+
import { x as OnDelete } from "./schema-DoSc-4ig.js";
|
|
2
|
+
import { k as CoordinatorPlugin, n as Dataset } from "./dataset-CejF-Fr_.js";
|
|
3
|
+
import { a as ResolverFrameError } from "./errors-7mycvNGq.js";
|
|
4
4
|
//#region ../../plugins/substrate-proposals/src/config.d.ts
|
|
5
5
|
/**
|
|
6
6
|
* Multi-instance invalidation transport used by the plugin to broadcast
|
|
@@ -41,6 +41,13 @@ interface ProposalsPluginConfig {
|
|
|
41
41
|
* propose authority under a differently-named cap.
|
|
42
42
|
*/
|
|
43
43
|
proposeCapability?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Capability name required to invoke `propose_index`, in the namespace whose
|
|
46
|
+
* indexes it changes. Defaults to `'substrate.index'`: separate from the
|
|
47
|
+
* propose capability, so only the operator or a designated reviewer changes
|
|
48
|
+
* indexes.
|
|
49
|
+
*/
|
|
50
|
+
indexCapability?: string;
|
|
44
51
|
/**
|
|
45
52
|
* Lookup function the `propose_namespace` resolver uses to find the dataset
|
|
46
53
|
* for a hot-added namespace. The host (typically the substrate binary) wires
|
|
@@ -57,8 +64,12 @@ interface ProposalsPluginConfig {
|
|
|
57
64
|
*/
|
|
58
65
|
export declare function createNoopPubSub(): PubSub;
|
|
59
66
|
//#endregion
|
|
67
|
+
//#region ../../plugins/substrate-proposals/src/engine-write.d.ts
|
|
68
|
+
/** Resolves once every storage preparation started so far has settled. */
|
|
69
|
+
export declare function storageSettled(): Promise<void>;
|
|
70
|
+
//#endregion
|
|
60
71
|
//#region ../../plugins/substrate-proposals/src/errors.d.ts
|
|
61
|
-
type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired';
|
|
72
|
+
type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired' | 'index_on_code_defined_type' | 'unknown_index' | 'not_engine_owned' | 'unique_index' | 'still_desired' | 'unknown_recommendation' | 'recommendation_closed' | 'recommendation_stale' | 'propose_index_not_alone' | 'index_drop_timeout';
|
|
62
73
|
/**
|
|
63
74
|
* Structured error for proposal operations with stable error codes.
|
|
64
75
|
*
|
package/dist/proposals.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import { n as createNoopPubSub, r as
|
|
2
|
+
import { i as ProposalError, n as createNoopPubSub, r as storageSettled, t as createProposalsPlugin } from "./src-CqT-QWZ_.js";
|
|
3
3
|
|
|
4
|
-
export { ProposalError, createNoopPubSub, createProposalsPlugin };
|
|
4
|
+
export { ProposalError, createNoopPubSub, createProposalsPlugin, storageSettled };
|
package/dist/recall-bootstrap.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
2
|
+
import { Qt as defineBootstrap } from "./src-5SJ5GEIM.js";
|
|
3
3
|
import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
|
|
4
|
-
import { t as createRecallPlugin } from "./plugin-
|
|
4
|
+
import { t as createRecallPlugin } from "./plugin-CtwOINCM.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
|
|
7
7
|
//#region ../../packages/substrate-embedder/src/types.ts
|
|
@@ -49,7 +49,7 @@ var OllamaEmbedder = class {
|
|
|
49
49
|
async embed(text, opts) {
|
|
50
50
|
const timeoutMs = opts?.timeoutMs ?? this.timeoutMs;
|
|
51
51
|
const contextLength = await this.resolveContextLength(timeoutMs);
|
|
52
|
-
const url = `${this.baseUrl
|
|
52
|
+
const url = `${trimTrailingSlashes(this.baseUrl)}/api/embed`;
|
|
53
53
|
const controller = new AbortController();
|
|
54
54
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
55
55
|
let res;
|
|
@@ -98,7 +98,7 @@ var OllamaEmbedder = class {
|
|
|
98
98
|
return this.contextLength;
|
|
99
99
|
}
|
|
100
100
|
async fetchContextLength(timeoutMs) {
|
|
101
|
-
const url = `${this.baseUrl
|
|
101
|
+
const url = `${trimTrailingSlashes(this.baseUrl)}/api/show`;
|
|
102
102
|
const controller = new AbortController();
|
|
103
103
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
104
104
|
let body;
|
|
@@ -137,6 +137,11 @@ var OllamaEmbedder = class {
|
|
|
137
137
|
}
|
|
138
138
|
};
|
|
139
139
|
/** `unavailable` for a failed request, naming the timeout when it was the abort. */
|
|
140
|
+
function trimTrailingSlashes(url) {
|
|
141
|
+
let end = url.length;
|
|
142
|
+
while (end > 0 && url.charCodeAt(end - 1) === 47) end--;
|
|
143
|
+
return url.slice(0, end);
|
|
144
|
+
}
|
|
140
145
|
function transportError(url, controller, timeoutMs, err) {
|
|
141
146
|
const detail = controller.signal.aborted ? `timed out after ${timeoutMs}ms` : `failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
142
147
|
return new EmbedError("unavailable", `OllamaEmbedder: POST ${url} ${detail}`, { cause: err });
|
|
@@ -378,6 +383,8 @@ const CrystalliseSchema = z.object({
|
|
|
378
383
|
const RecallBootstrapConfigSchema = z.object({
|
|
379
384
|
embedder: EmbedderSchema.optional(),
|
|
380
385
|
queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
386
|
+
writeEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
387
|
+
embedBackfillIntervalMs: z.number().int().nonnegative().default(6e4),
|
|
381
388
|
hnswM: z.number().int().positive().default(16),
|
|
382
389
|
hnswEfConstruction: z.number().int().positive().default(64),
|
|
383
390
|
salience: SalienceSchema.default({
|
|
@@ -426,6 +433,8 @@ const bootstrap = defineBootstrap.kernel({
|
|
|
426
433
|
return createRecallPlugin({
|
|
427
434
|
...embedder !== null ? { embedder } : {},
|
|
428
435
|
queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
|
|
436
|
+
writeEmbedTimeoutMs: config.writeEmbedTimeoutMs,
|
|
437
|
+
embedBackfillIntervalMs: config.embedBackfillIntervalMs,
|
|
429
438
|
...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
|
|
430
439
|
store: deps.store,
|
|
431
440
|
...deps.stats !== void 0 ? { stats: deps.stats } : {},
|
|
@@ -118,6 +118,18 @@
|
|
|
118
118
|
"exclusiveMinimum": 0,
|
|
119
119
|
"maximum": 9007199254740991
|
|
120
120
|
},
|
|
121
|
+
"writeEmbedTimeoutMs": {
|
|
122
|
+
"default": 30000,
|
|
123
|
+
"type": "integer",
|
|
124
|
+
"exclusiveMinimum": 0,
|
|
125
|
+
"maximum": 9007199254740991
|
|
126
|
+
},
|
|
127
|
+
"embedBackfillIntervalMs": {
|
|
128
|
+
"default": 60000,
|
|
129
|
+
"type": "integer",
|
|
130
|
+
"minimum": 0,
|
|
131
|
+
"maximum": 9007199254740991
|
|
132
|
+
},
|
|
121
133
|
"hnswM": {
|
|
122
134
|
"default": 16,
|
|
123
135
|
"type": "integer",
|
package/dist/recall.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { b as EdgesStore, k as Row, r as FieldDef } from "./schema-DoSc-4ig.js";
|
|
2
|
+
import { Dt as PolicyDecision, F as FilterOperatorDef, H as KernelPlugin, Ot as PolicyProbeContext, W as McpToolRegistration, Z as PostWriteHook, j as EmbedSourceContext, jt as ResolverArgs, kt as RequestContext, lt as HttpHandler, n as Dataset, wt as Identity, z as KernelMigration } from "./dataset-CejF-Fr_.js";
|
|
3
3
|
import { t as Embedder } from "./types-8c0gtnd2.js";
|
|
4
4
|
import postgres, { Sql } from "postgres";
|
|
5
5
|
//#region ../../plugins/substrate-recall/src/candidate-selection.d.ts
|
|
@@ -150,8 +150,11 @@ interface ClusterStore {
|
|
|
150
150
|
includeRetired?: boolean;
|
|
151
151
|
}): Promise<ClusterRecord[]>;
|
|
152
152
|
getCluster(id: string): Promise<ClusterRecord | null>;
|
|
153
|
-
/**
|
|
154
|
-
|
|
153
|
+
/**
|
|
154
|
+
* Members of one cluster, or of each of several in one read. Departed
|
|
155
|
+
* members excluded unless asked for.
|
|
156
|
+
*/
|
|
157
|
+
listMembers(clusterId: string | readonly string[], opts?: {
|
|
155
158
|
includeDeparted?: boolean;
|
|
156
159
|
}): Promise<ClusterMemberRecord[]>;
|
|
157
160
|
/**
|
|
@@ -242,6 +245,11 @@ interface EmbeddingsStoreKnnInput {
|
|
|
242
245
|
* store shared by several still fills k. Equality family only.
|
|
243
246
|
*/
|
|
244
247
|
rowFilter?: KnnRowFilter;
|
|
248
|
+
/**
|
|
249
|
+
* Return each row's salience decayed to now by this half-life, the decay
|
|
250
|
+
* `boostSalience` applies. Absent: the stored value, as of its last boost.
|
|
251
|
+
*/
|
|
252
|
+
salienceHalfLifeHours?: number;
|
|
245
253
|
}
|
|
246
254
|
type KnnScalar = string | number | boolean | null;
|
|
247
255
|
/** Field -> `{ eq }` or `{ in }`; every field must match. `eq: null` matches a missing or null field. */
|
|
@@ -297,6 +305,17 @@ interface EmbeddingsStore {
|
|
|
297
305
|
* mismatched vector surfaces as a substrate-level error.
|
|
298
306
|
*/
|
|
299
307
|
upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
|
|
308
|
+
/**
|
|
309
|
+
* Ids of live rows of (namespace, type) whose vector is missing or older than
|
|
310
|
+
* the row, after `afterId` in id order: the writes whose embed failed. Optional;
|
|
311
|
+
* a store that cannot see the rows table omits it and nothing is backfilled.
|
|
312
|
+
*/
|
|
313
|
+
staleRowIds?(input: {
|
|
314
|
+
namespace: string;
|
|
315
|
+
type: string;
|
|
316
|
+
afterId: string;
|
|
317
|
+
limit: number;
|
|
318
|
+
}): Promise<string[]>;
|
|
300
319
|
/**
|
|
301
320
|
* Remove the embedding for one row, leaving the row itself untouched. Used
|
|
302
321
|
* to un-index a row whose `embed.source` returns the null sentinel.
|
|
@@ -716,6 +735,14 @@ interface RecallInput {
|
|
|
716
735
|
* or null field. A malformed filter is refused with `bad_request`.
|
|
717
736
|
*/
|
|
718
737
|
where?: KnnRowFilter;
|
|
738
|
+
/**
|
|
739
|
+
* Row-body projection. Field -> `true` keeps those keys plus `id`; `true`
|
|
740
|
+
* keeps the whole body. Absent → `id` plus the handle fields
|
|
741
|
+
* (`HANDLE_FIELDS`) the row has. Applied after row filtering and policy,
|
|
742
|
+
* which still see the full body. A malformed value is refused with
|
|
743
|
+
* `bad_request`.
|
|
744
|
+
*/
|
|
745
|
+
rowFields?: Record<string, true> | true;
|
|
719
746
|
}
|
|
720
747
|
interface RecallRow {
|
|
721
748
|
/**
|
|
@@ -1451,7 +1478,7 @@ export declare class InMemoryClusterStore implements ClusterStore {
|
|
|
1451
1478
|
includeRetired?: boolean;
|
|
1452
1479
|
}): Promise<ClusterRecord[]>;
|
|
1453
1480
|
getCluster(id: string): Promise<ClusterRecord | null>;
|
|
1454
|
-
listMembers(clusterId: string, opts?: {
|
|
1481
|
+
listMembers(clusterId: string | readonly string[], opts?: {
|
|
1455
1482
|
includeDeparted?: boolean;
|
|
1456
1483
|
}): Promise<ClusterMemberRecord[]>;
|
|
1457
1484
|
listAllMembers(): Promise<ClusterMemberRecord[]>;
|
|
@@ -1639,7 +1666,7 @@ export declare class PgClusterStore implements ClusterStore {
|
|
|
1639
1666
|
constructor(opts: PgClusterStoreOptions);
|
|
1640
1667
|
/**
|
|
1641
1668
|
* Runs `fn` holding a session advisory lock that excludes formation in every
|
|
1642
|
-
* process on this
|
|
1669
|
+
* process on this schema; throws `crystallise_in_flight` if it is held.
|
|
1643
1670
|
*
|
|
1644
1671
|
* Lock and unlock share one reserved connection, since through the pool the
|
|
1645
1672
|
* unlock can land on another session and fail to release.
|
|
@@ -1651,7 +1678,7 @@ export declare class PgClusterStore implements ClusterStore {
|
|
|
1651
1678
|
includeRetired?: boolean;
|
|
1652
1679
|
}): Promise<ClusterRecord[]>;
|
|
1653
1680
|
getCluster(id: string): Promise<ClusterRecord | null>;
|
|
1654
|
-
listMembers(clusterId: string, opts?: {
|
|
1681
|
+
listMembers(clusterId: string | readonly string[], opts?: {
|
|
1655
1682
|
includeDeparted?: boolean;
|
|
1656
1683
|
}): Promise<ClusterMemberRecord[]>;
|
|
1657
1684
|
listAllMembers(): Promise<ClusterMemberRecord[]>;
|
|
@@ -1688,6 +1715,12 @@ export declare class PgEmbeddingsStore implements EmbeddingsStore {
|
|
|
1688
1715
|
*
|
|
1689
1716
|
* @returns number of orphaned embeddings deleted.
|
|
1690
1717
|
*/
|
|
1718
|
+
staleRowIds(input: {
|
|
1719
|
+
namespace: string;
|
|
1720
|
+
type: string;
|
|
1721
|
+
afterId: string;
|
|
1722
|
+
limit: number;
|
|
1723
|
+
}): Promise<string[]>;
|
|
1691
1724
|
purgeOrphans(): Promise<number>;
|
|
1692
1725
|
}
|
|
1693
1726
|
//#endregion
|
|
@@ -1731,10 +1764,22 @@ export declare class PgLexicalStore implements LexicalStore {
|
|
|
1731
1764
|
interface RecallPluginConfig {
|
|
1732
1765
|
embedder?: Embedder;
|
|
1733
1766
|
/**
|
|
1734
|
-
* Timeout for embedding a recall or `embeddingSimilar` query, in ms.
|
|
1735
|
-
*
|
|
1767
|
+
* Timeout for embedding a recall or `embeddingSimilar` query, in ms.
|
|
1768
|
+
* Reindex embeds keep the embedder's own timeout. Default 30000.
|
|
1736
1769
|
*/
|
|
1737
1770
|
queryEmbedTimeoutMs?: number;
|
|
1771
|
+
/**
|
|
1772
|
+
* Timeout for the embed that follows a write, in ms. The engine awaits
|
|
1773
|
+
* after-commit work before it replies, so this bounds how long an already
|
|
1774
|
+
* committed write can be held by a slow embedder. A timed-out embed keeps
|
|
1775
|
+
* the row's previous vector until its next write or a reindex. Default 30000.
|
|
1776
|
+
*/
|
|
1777
|
+
writeEmbedTimeoutMs?: number;
|
|
1778
|
+
/**
|
|
1779
|
+
* How often to re-embed rows whose write-path embed failed or timed out (their
|
|
1780
|
+
* vector is missing or older than the row). 0 disables. Default 60000.
|
|
1781
|
+
*/
|
|
1782
|
+
embedBackfillIntervalMs?: number;
|
|
1738
1783
|
store: EmbeddingsStore;
|
|
1739
1784
|
/**
|
|
1740
1785
|
* Store for the lexical arm. Absent → types declaring `lexical.source`
|