@mnmal-ai/substrate 1.0.0-alpha.2 → 1.0.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -14
- package/dist/boot-store-BaDqbBS-.js +4062 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-BhiJKwfF.d.ts} +2 -87
- package/dist/cli-RRFphMtQ.js +3366 -0
- package/dist/client.browser.js +2485 -0
- package/dist/client.d.ts +273 -135
- package/dist/client.js +6 -6
- package/dist/cluster-worker.js +0 -1
- package/dist/{content-type-C7T28xNb.js → content-type-wLljot5M.js} +0 -1
- package/dist/{env-utils-CGNazUNt.js → env-utils-s25Z1YSA.js} +0 -1
- package/dist/{errors-D4vKuwz8.d.ts → errors-CYOaWZ8l.d.ts} +44 -4
- package/dist/{fake-DUVtKVSE.js → fake-DdXtb-m8.js} +0 -1
- package/dist/{index-D7mSOFlM.d.ts → index-ZKeYFHDD.d.ts} +7 -11
- package/dist/msgpackr-Dq18g6Ds.js +16 -0
- package/dist/node-version-BtK_Nq6z.js +32 -0
- package/dist/pack.d.ts +75 -11
- package/dist/pack.js +4 -4
- package/dist/pg-edges-store-C-IRv-O9.js +443 -0
- package/dist/{plugin-QuJN9pG_.js → plugin-MSbSZ6Al.js} +132 -8
- package/dist/proposals-bootstrap.js +6 -5
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +17 -5
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -3
- package/dist/recall-bootstrap.js +5 -4
- package/dist/recall.config.schema.json +6 -0
- package/dist/recall.d.ts +35 -6
- package/dist/recall.js +2 -3
- package/dist/{repl-C52wyfMs.js → repl-Ob5OyQqj.js} +65 -33
- package/dist/{row-ByLFE2Wx.d.ts → row-CfsHSEfB.d.ts} +92 -12
- package/dist/run-B438GGJK.js +2435 -0
- package/dist/{plugin-t2g2wX0b.d.ts → schema-laGLMvMC.d.ts} +1287 -63
- package/dist/script-A9dxMRZA.js +143 -0
- package/dist/script-runner.js +77 -0
- package/dist/sdk.d.ts +227 -38
- package/dist/sdk.js +5 -110
- package/dist/semver-D_zdq0hi.js +17 -0
- package/dist/{src-DMzc3HfW.js → src-BNdhH_Hx.js} +745 -851
- package/dist/{src-CG7giuHr.js → src-BSL2yyVy.js} +779 -75
- package/dist/{src-BLgbLq2k.js → src-B_kHkbsr.js} +780 -330
- package/dist/{src-DWm0iA88.js → src-De9j33jv.js} +188 -87
- package/dist/src-DkxrpQOz.js +7709 -0
- package/dist/{row-DZ0Fnpc6.js → src-DmiUWOLp.js} +2 -1
- package/dist/{src-CHp3QyqB.js → src-S84ef4e3.js} +9177 -3377
- package/dist/substrate-admin.js +17 -782
- package/dist/substrate.js +4 -5487
- package/dist/testing.d.ts +730 -53
- package/dist/testing.js +20 -163
- package/dist/{ws-C72iVS5T.d.ts → ws-BT8kL1IF.d.ts} +4 -5
- package/package.json +26 -26
- package/dist/dataset-BQ8egGl5.js +0 -1535
- package/dist/schema-g4FQLr0V.d.ts +0 -433
- package/dist/src-CXKnqeTW.js +0 -2339
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
1
|
+
import { Lt as buildContext, tt as applyObligations, xn as ResolverFrameError, zt as jsonResponse } from "./src-S84ef4e3.js";
|
|
2
|
+
import "./src-B_kHkbsr.js";
|
|
3
3
|
import { Worker } from "node:worker_threads";
|
|
4
4
|
|
|
5
5
|
//#region ../../plugins/substrate-recall/src/candidate-selection.ts
|
|
@@ -949,6 +949,11 @@ async function handleRecallRequest(input, deps) {
|
|
|
949
949
|
const problem = rowFilterProblem(where);
|
|
950
950
|
if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"where" ${problem}`), { code: "bad_request" });
|
|
951
951
|
}
|
|
952
|
+
const rowFields = input.rowFields;
|
|
953
|
+
if (rowFields !== void 0) {
|
|
954
|
+
const problem = rowFieldsProblem(rowFields);
|
|
955
|
+
if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"rowFields" ${problem}`), { code: "bad_request" });
|
|
956
|
+
}
|
|
952
957
|
const type = input.type;
|
|
953
958
|
if (type === void 0 || type === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `type`, the type to search. There is no default."), { code: "bad_request" });
|
|
954
959
|
if (typeof input.query !== "string" || input.query.trim() === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `query`, the text to search for."), { code: "bad_request" });
|
|
@@ -1166,7 +1171,7 @@ async function handleRecallRequest(input, deps) {
|
|
|
1166
1171
|
}
|
|
1167
1172
|
const t3 = performance.now();
|
|
1168
1173
|
return {
|
|
1169
|
-
rows: fusedRows,
|
|
1174
|
+
rows: rowFields === true ? fusedRows : fusedRows.map((r) => projectRow(r, rowFields)),
|
|
1170
1175
|
model: denseRan ? embedder?.modelId ?? "" : "",
|
|
1171
1176
|
arms: {
|
|
1172
1177
|
dense: denseRan,
|
|
@@ -1212,6 +1217,23 @@ function extractTargets(namespace, type, rows) {
|
|
|
1212
1217
|
}
|
|
1213
1218
|
return out;
|
|
1214
1219
|
}
|
|
1220
|
+
/** Why `rowFields` is malformed, or `null` when it is `true` or an object of `field: true`. */
|
|
1221
|
+
function rowFieldsProblem(value) {
|
|
1222
|
+
if (value === true) return null;
|
|
1223
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return "must be true (the whole body) or an object of field: true, e.g. { \"body\": true }";
|
|
1224
|
+
for (const [field, v] of Object.entries(value)) if (v !== true) return `field "${field}" must be true`;
|
|
1225
|
+
return null;
|
|
1226
|
+
}
|
|
1227
|
+
/** Narrow a hit's `row` to `id` plus `rowFields`, or plus the handle fields when absent. */
|
|
1228
|
+
function projectRow(hit, rowFields) {
|
|
1229
|
+
const row = {};
|
|
1230
|
+
const keep = rowFields === void 0 ? HANDLE_FIELDS : Object.keys(rowFields);
|
|
1231
|
+
for (const field of ["id", ...keep]) if (Object.hasOwn(hit.row, field)) row[field] = hit.row[field];
|
|
1232
|
+
return {
|
|
1233
|
+
...hit,
|
|
1234
|
+
row
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1215
1237
|
|
|
1216
1238
|
//#endregion
|
|
1217
1239
|
//#region ../../plugins/substrate-recall/src/params.ts
|
|
@@ -1265,7 +1287,8 @@ const ROUTE_PARAMS = [
|
|
|
1265
1287
|
"returnVector",
|
|
1266
1288
|
"lexical",
|
|
1267
1289
|
"explain",
|
|
1268
|
-
"where"
|
|
1290
|
+
"where",
|
|
1291
|
+
"rowFields"
|
|
1269
1292
|
];
|
|
1270
1293
|
function parseBody$3(req) {
|
|
1271
1294
|
const raw = req.body;
|
|
@@ -1325,6 +1348,7 @@ function parseBody$3(req) {
|
|
|
1325
1348
|
input.explain = obj.explain;
|
|
1326
1349
|
}
|
|
1327
1350
|
if (obj.where !== void 0) input.where = obj.where;
|
|
1351
|
+
if (obj.rowFields !== void 0) input.rowFields = obj.rowFields;
|
|
1328
1352
|
return input;
|
|
1329
1353
|
}
|
|
1330
1354
|
/**
|
|
@@ -2466,7 +2490,8 @@ const MCP_PARAMS = [
|
|
|
2466
2490
|
"returnVector",
|
|
2467
2491
|
"lexical",
|
|
2468
2492
|
"explain",
|
|
2469
|
-
"where"
|
|
2493
|
+
"where",
|
|
2494
|
+
"rowFields"
|
|
2470
2495
|
];
|
|
2471
2496
|
function recallMcpTool(cfg) {
|
|
2472
2497
|
return {
|
|
@@ -2514,6 +2539,10 @@ function recallMcpTool(cfg) {
|
|
|
2514
2539
|
where: {
|
|
2515
2540
|
type: "object",
|
|
2516
2541
|
description: "Row-field filter applied inside every candidate source, so k fills with matching rows: field -> { eq: scalar | null } or { in: [scalar | null, ...] }, every field must match. eq null matches a missing or null field. Example: { \"project\": { \"in\": [\"substrate\", null] } }."
|
|
2542
|
+
},
|
|
2543
|
+
rowFields: {
|
|
2544
|
+
type: ["object", "boolean"],
|
|
2545
|
+
description: "Row-body projection. Omitted, each hit's `row` carries `id` plus its handle fields (name, title, topic, synopsis, description), enough to decide on. field -> true keeps just those fields plus `id`, e.g. { \"synopsis\": true, \"body\": true }; true returns the whole body."
|
|
2517
2546
|
}
|
|
2518
2547
|
},
|
|
2519
2548
|
required: ["query", "type"]
|
|
@@ -2749,7 +2778,7 @@ async function alignVectorDimensions(s, opts) {
|
|
|
2749
2778
|
const { dimension } = opts;
|
|
2750
2779
|
if (!Number.isInteger(dimension) || dimension <= 0) throw new Error(`alignVectorDimensions: dimension must be a positive integer (got ${dimension})`);
|
|
2751
2780
|
return await s.begin(async (tx) => {
|
|
2752
|
-
await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align'))`;
|
|
2781
|
+
await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align:' || current_schema()))`;
|
|
2753
2782
|
return alignLocked(tx, opts);
|
|
2754
2783
|
});
|
|
2755
2784
|
}
|
|
@@ -2776,6 +2805,50 @@ async function alignLocked(s, opts) {
|
|
|
2776
2805
|
};
|
|
2777
2806
|
}
|
|
2778
2807
|
|
|
2808
|
+
//#endregion
|
|
2809
|
+
//#region ../../plugins/substrate-recall/src/embed-backfill.ts
|
|
2810
|
+
async function runEmbedBackfill(deps) {
|
|
2811
|
+
const result = {
|
|
2812
|
+
embedded: 0,
|
|
2813
|
+
unindexable: 0,
|
|
2814
|
+
failed: 0
|
|
2815
|
+
};
|
|
2816
|
+
const staleRowIds = deps.store.staleRowIds?.bind(deps.store);
|
|
2817
|
+
if (staleRowIds === void 0) return result;
|
|
2818
|
+
for (const { namespace, type } of deps.registry.types()) {
|
|
2819
|
+
const reembed = deps.registry.get(namespace, type)?.reembed;
|
|
2820
|
+
const dataset = deps.resolveDataset(namespace);
|
|
2821
|
+
if (reembed === void 0 || dataset === null) continue;
|
|
2822
|
+
const key = `${namespace}/${type}`;
|
|
2823
|
+
const ids = await staleRowIds({
|
|
2824
|
+
namespace,
|
|
2825
|
+
type,
|
|
2826
|
+
afterId: deps.cursors.get(key) ?? "",
|
|
2827
|
+
limit: deps.perTypeLimit
|
|
2828
|
+
});
|
|
2829
|
+
if (ids.length < deps.perTypeLimit) deps.cursors.delete(key);
|
|
2830
|
+
else deps.cursors.set(key, ids[ids.length - 1]);
|
|
2831
|
+
if (ids.length === 0) continue;
|
|
2832
|
+
const { rows } = await dataset.read(type, {
|
|
2833
|
+
ids,
|
|
2834
|
+
limit: ids.length
|
|
2835
|
+
});
|
|
2836
|
+
for (const row of rows) {
|
|
2837
|
+
const outcome = await reembed(row);
|
|
2838
|
+
if (outcome.status === "embedded") result.embedded++;
|
|
2839
|
+
else if (outcome.status === "unindexable") result.unindexable++;
|
|
2840
|
+
else if (outcome.status === "failed") {
|
|
2841
|
+
result.failed++;
|
|
2842
|
+
if (outcome.code === "unavailable") return {
|
|
2843
|
+
...result,
|
|
2844
|
+
stopped: outcome.error
|
|
2845
|
+
};
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
return result;
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2779
2852
|
//#endregion
|
|
2780
2853
|
//#region ../../plugins/substrate-recall/src/expansion-resolver.ts
|
|
2781
2854
|
function makeExpansionResolver(registry, edges, cfg) {
|
|
@@ -3035,12 +3108,14 @@ function rethrowCoded(err) {
|
|
|
3035
3108
|
throw err;
|
|
3036
3109
|
}
|
|
3037
3110
|
/**
|
|
3038
|
-
* Keep only the
|
|
3111
|
+
* Keep only the selected keys of `fields`; `undefined` fields = passthrough.
|
|
3039
3112
|
* `fields: {}` strips every key, matching the compose layer's projectEnvelope.
|
|
3113
|
+
* A nested `row: { … }` selection keeps `row`, which the core has already
|
|
3114
|
+
* narrowed through `rowFields`.
|
|
3040
3115
|
*/
|
|
3041
3116
|
function projectHit(row, fields) {
|
|
3042
3117
|
if (fields === void 0) return row;
|
|
3043
|
-
const keep = Object.entries(fields).filter(([, v]) => v === true).map(([k]) => k);
|
|
3118
|
+
const keep = Object.entries(fields).filter(([k, v]) => v === true || k === "row" && isPlainObject(v)).map(([k]) => k);
|
|
3044
3119
|
const out = {};
|
|
3045
3120
|
for (const k of keep) if (k in row) out[k] = row[k];
|
|
3046
3121
|
return out;
|
|
@@ -3063,6 +3138,9 @@ function buildRecallFrameSurface(deps) {
|
|
|
3063
3138
|
const salience = deps.salience ?? DEFAULT_SALIENCE_CONFIG;
|
|
3064
3139
|
const hitResolver = async (ctx, args) => {
|
|
3065
3140
|
const input = parseHitParams(args.params);
|
|
3141
|
+
const rowSelection = args.fields?.row;
|
|
3142
|
+
if (rowSelection === true) input.rowFields = true;
|
|
3143
|
+
else if (isPlainObject(rowSelection)) input.rowFields = rowSelection;
|
|
3066
3144
|
const bound = bindPolicy(deps.checkReadPolicy, callerFromContext(ctx));
|
|
3067
3145
|
const boundWrite = bindPolicy(deps.checkWritePolicy, callerFromContext(ctx));
|
|
3068
3146
|
let out;
|
|
@@ -3144,6 +3222,9 @@ function buildRecallFrameSurface(deps) {
|
|
|
3144
3222
|
}
|
|
3145
3223
|
};
|
|
3146
3224
|
}
|
|
3225
|
+
function isPlainObject(v) {
|
|
3226
|
+
return v !== null && typeof v === "object" && !Array.isArray(v);
|
|
3227
|
+
}
|
|
3147
3228
|
|
|
3148
3229
|
//#endregion
|
|
3149
3230
|
//#region ../../plugins/substrate-recall/src/lexical-reindex-core.ts
|
|
@@ -3617,6 +3698,9 @@ function makeSpreadResolver(registry, edges) {
|
|
|
3617
3698
|
//#endregion
|
|
3618
3699
|
//#region ../../plugins/substrate-recall/src/plugin.ts
|
|
3619
3700
|
const DEFAULT_CLUSTER_DIMENSION = 768;
|
|
3701
|
+
const DEFAULT_EMBED_BACKFILL_INTERVAL_MS = 6e4;
|
|
3702
|
+
/** Rows per embedded type per pass, so one pass stays a few seconds of embed work. */
|
|
3703
|
+
const EMBED_BACKFILL_PER_TYPE = 50;
|
|
3620
3704
|
const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3e4;
|
|
3621
3705
|
const DEFAULT_WRITE_EMBED_TIMEOUT_MS = 3e4;
|
|
3622
3706
|
/** `embedder` with `timeoutMs` applied to every call that does not set its own. */
|
|
@@ -3694,11 +3778,19 @@ function createRecallPlugin(cfg) {
|
|
|
3694
3778
|
...cfg.checkReadPolicy !== void 0 ? { checkReadPolicy: cfg.checkReadPolicy } : {},
|
|
3695
3779
|
...cfg.runClusterer !== void 0 ? { runClusterer: cfg.runClusterer } : {}
|
|
3696
3780
|
};
|
|
3781
|
+
let backfillTimer;
|
|
3697
3782
|
return {
|
|
3698
3783
|
name: "substrate-recall",
|
|
3699
3784
|
contractVersion: ">=1.0.0-alpha.0 <2.0.0",
|
|
3785
|
+
storeTables: [
|
|
3786
|
+
"embeddings",
|
|
3787
|
+
"lexical",
|
|
3788
|
+
"recall_clusters",
|
|
3789
|
+
"recall_cluster_members"
|
|
3790
|
+
],
|
|
3700
3791
|
async onKernelReady(ctx) {
|
|
3701
3792
|
if (cfg.embedder === void 0 || ctx.sql === null) return;
|
|
3793
|
+
let dimensionMismatch = false;
|
|
3702
3794
|
const report = await alignVectorDimensions(ctx.sql, {
|
|
3703
3795
|
dimension: cfg.embedder.dimension,
|
|
3704
3796
|
...cfg.migration?.hnswM !== void 0 ? { hnswM: cfg.migration.hnswM } : {},
|
|
@@ -3706,6 +3798,7 @@ function createRecallPlugin(cfg) {
|
|
|
3706
3798
|
});
|
|
3707
3799
|
for (const [column, state] of [["embeddings.embedding", report.embeddings], ["recall_clusters.centroid", report.centroids]]) {
|
|
3708
3800
|
if (state !== "populated") continue;
|
|
3801
|
+
dimensionMismatch = true;
|
|
3709
3802
|
const found = column.startsWith("embeddings") ? report.found.embeddings : report.found.centroids;
|
|
3710
3803
|
ctx.metric("recall_dimension_mismatch", {
|
|
3711
3804
|
column,
|
|
@@ -3715,6 +3808,37 @@ function createRecallPlugin(cfg) {
|
|
|
3715
3808
|
});
|
|
3716
3809
|
process.emitWarning(`${column} is vector(${found}) and holds rows, but the embedder is ${cfg.embedder.modelId} at ${cfg.embedder.dimension}. Writes to it will fail until the rows are cleared (they are derived) and re-embedded: delete them, restart, then POST /substrate/v1/embed/reindex.`, { code: "RECALL_DIMENSION_MISMATCH" });
|
|
3717
3810
|
}
|
|
3811
|
+
const intervalMs = cfg.embedBackfillIntervalMs ?? DEFAULT_EMBED_BACKFILL_INTERVAL_MS;
|
|
3812
|
+
const resolveDataset = cfg.resolveDataset;
|
|
3813
|
+
if (dimensionMismatch || intervalMs === 0 || resolveDataset === void 0) return;
|
|
3814
|
+
if (!ctx.background) return;
|
|
3815
|
+
const cursors = /* @__PURE__ */ new Map();
|
|
3816
|
+
let running = false;
|
|
3817
|
+
clearInterval(backfillTimer);
|
|
3818
|
+
backfillTimer = setInterval(() => {
|
|
3819
|
+
if (running) return;
|
|
3820
|
+
running = true;
|
|
3821
|
+
runEmbedBackfill({
|
|
3822
|
+
registry: embedSourceRegistry,
|
|
3823
|
+
store: cfg.store,
|
|
3824
|
+
resolveDataset,
|
|
3825
|
+
cursors,
|
|
3826
|
+
perTypeLimit: EMBED_BACKFILL_PER_TYPE
|
|
3827
|
+
}).then((r) => {
|
|
3828
|
+
if (r.embedded + r.failed === 0 && r.stopped === void 0) return;
|
|
3829
|
+
ctx.metric("recall_embed_backfill", r);
|
|
3830
|
+
process.stdout.write(`recall.embed_backfill ${JSON.stringify(r)}\n`);
|
|
3831
|
+
}).catch((err) => {
|
|
3832
|
+
process.stderr.write(`recall.embed_backfill failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
3833
|
+
}).finally(() => {
|
|
3834
|
+
running = false;
|
|
3835
|
+
});
|
|
3836
|
+
}, intervalMs);
|
|
3837
|
+
backfillTimer.unref();
|
|
3838
|
+
},
|
|
3839
|
+
onKernelStop() {
|
|
3840
|
+
clearInterval(backfillTimer);
|
|
3841
|
+
backfillTimer = void 0;
|
|
3718
3842
|
},
|
|
3719
3843
|
migrations: [
|
|
3720
3844
|
embeddingsMigration(cfg.migration ?? {}),
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-CG7giuHr.js";
|
|
1
|
+
import { Un as defineBootstrap } from "./src-S84ef4e3.js";
|
|
2
|
+
import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-BSL2yyVy.js";
|
|
4
3
|
import { z } from "zod";
|
|
5
4
|
|
|
6
5
|
//#region ../../plugins/substrate-proposals/src/bootstrap.ts
|
|
@@ -22,14 +21,16 @@ import { z } from "zod";
|
|
|
22
21
|
*/
|
|
23
22
|
const ProposalsBootstrapConfigSchema = z.object({
|
|
24
23
|
ownedNamespaces: z.array(z.string().min(1)).default([]),
|
|
25
|
-
proposeCapability: z.string().min(1).default("substrate.propose")
|
|
24
|
+
proposeCapability: z.string().min(1).default("substrate.propose"),
|
|
25
|
+
indexCapability: z.string().min(1).default("substrate.index")
|
|
26
26
|
}).strict();
|
|
27
27
|
const bootstrap = defineBootstrap.coordinator({
|
|
28
28
|
schema: ProposalsBootstrapConfigSchema,
|
|
29
29
|
build: (config) => createProposalsPlugin({
|
|
30
30
|
ownedNamespaces: config.ownedNamespaces,
|
|
31
31
|
pubsub: createNoopPubSub(),
|
|
32
|
-
proposeCapability: config.proposeCapability
|
|
32
|
+
proposeCapability: config.proposeCapability,
|
|
33
|
+
indexCapability: config.indexCapability
|
|
33
34
|
})
|
|
34
35
|
});
|
|
35
36
|
|
package/dist/proposals.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { a as ResolverFrameError } from "./errors-D4vKuwz8.js";
|
|
1
|
+
import { kn as OnDelete, kt as Dataset, x as CoordinatorPlugin } from "./schema-laGLMvMC.js";
|
|
2
|
+
import { u as ResolverFrameError } from "./errors-CYOaWZ8l.js";
|
|
4
3
|
//#region ../../plugins/substrate-proposals/src/config.d.ts
|
|
5
4
|
/**
|
|
6
5
|
* Multi-instance invalidation transport used by the plugin to broadcast
|
|
@@ -24,6 +23,8 @@ interface PubSub {
|
|
|
24
23
|
notify(channel: string, payload: string): Promise<void>;
|
|
25
24
|
}
|
|
26
25
|
interface ProposalsPluginConfig {
|
|
26
|
+
/** Where a one-line warning goes, such as a stored proposal skipped at replay. Defaults to `console.warn`. */
|
|
27
|
+
log?: (line: string) => void;
|
|
27
28
|
/**
|
|
28
29
|
* Namespaces this plugin owns. Propose mutations outside this list are
|
|
29
30
|
* rejected at resolver entry. A proposed crossRef must target a type in an
|
|
@@ -41,6 +42,13 @@ interface ProposalsPluginConfig {
|
|
|
41
42
|
* propose authority under a differently-named cap.
|
|
42
43
|
*/
|
|
43
44
|
proposeCapability?: string;
|
|
45
|
+
/**
|
|
46
|
+
* Capability name required to invoke `propose_index`, in the namespace whose
|
|
47
|
+
* indexes it changes. Defaults to `'substrate.index'`: separate from the
|
|
48
|
+
* propose capability, so only the operator or a designated reviewer changes
|
|
49
|
+
* indexes.
|
|
50
|
+
*/
|
|
51
|
+
indexCapability?: string;
|
|
44
52
|
/**
|
|
45
53
|
* Lookup function the `propose_namespace` resolver uses to find the dataset
|
|
46
54
|
* for a hot-added namespace. The host (typically the substrate binary) wires
|
|
@@ -57,8 +65,12 @@ interface ProposalsPluginConfig {
|
|
|
57
65
|
*/
|
|
58
66
|
export declare function createNoopPubSub(): PubSub;
|
|
59
67
|
//#endregion
|
|
68
|
+
//#region ../../plugins/substrate-proposals/src/engine-write.d.ts
|
|
69
|
+
/** Resolves once every storage preparation started so far has settled. */
|
|
70
|
+
export declare function storageSettled(): Promise<void>;
|
|
71
|
+
//#endregion
|
|
60
72
|
//#region ../../plugins/substrate-proposals/src/errors.d.ts
|
|
61
|
-
type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired';
|
|
73
|
+
type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired' | 'index_on_code_defined_type' | 'unknown_index' | 'not_engine_owned' | 'unique_index' | 'still_desired' | 'unknown_recommendation' | 'recommendation_closed' | 'recommendation_stale' | 'propose_index_not_alone' | 'index_drop_timeout';
|
|
62
74
|
/**
|
|
63
75
|
* Structured error for proposal operations with stable error codes.
|
|
64
76
|
*
|
|
@@ -80,7 +92,7 @@ export declare class ProposalError extends ResolverFrameError {
|
|
|
80
92
|
}
|
|
81
93
|
//#endregion
|
|
82
94
|
//#region ../../plugins/substrate-proposals/src/field-spec-json.d.ts
|
|
83
|
-
type ScalarType = 'string' | 'number' | 'boolean' | 'date' | 'json';
|
|
95
|
+
type ScalarType = 'string' | 'number' | 'int16' | 'int32' | 'int64' | 'boolean' | 'date' | 'json';
|
|
84
96
|
interface ScalarSpec {
|
|
85
97
|
kind: 'scalar';
|
|
86
98
|
type: ScalarType;
|
package/dist/proposals.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { n as createNoopPubSub, r as ProposalError, t as createProposalsPlugin } from "./src-CG7giuHr.js";
|
|
1
|
+
import { i as ProposalError, n as createNoopPubSub, r as storageSettled, t as createProposalsPlugin } from "./src-BSL2yyVy.js";
|
|
3
2
|
|
|
4
|
-
export { ProposalError, createNoopPubSub, createProposalsPlugin };
|
|
3
|
+
export { ProposalError, createNoopPubSub, createProposalsPlugin, storageSettled };
|
package/dist/recall-bootstrap.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { t as
|
|
4
|
-
import { t as createRecallPlugin } from "./plugin-QuJN9pG_.js";
|
|
1
|
+
import { Un as defineBootstrap } from "./src-S84ef4e3.js";
|
|
2
|
+
import { t as FakeEmbedder } from "./fake-DdXtb-m8.js";
|
|
3
|
+
import { t as createRecallPlugin } from "./plugin-MSbSZ6Al.js";
|
|
5
4
|
import { z } from "zod";
|
|
6
5
|
|
|
7
6
|
//#region ../../packages/substrate-embedder/src/types.ts
|
|
@@ -384,6 +383,7 @@ const RecallBootstrapConfigSchema = z.object({
|
|
|
384
383
|
embedder: EmbedderSchema.optional(),
|
|
385
384
|
queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
386
385
|
writeEmbedTimeoutMs: z.number().int().positive().default(3e4),
|
|
386
|
+
embedBackfillIntervalMs: z.number().int().nonnegative().default(6e4),
|
|
387
387
|
hnswM: z.number().int().positive().default(16),
|
|
388
388
|
hnswEfConstruction: z.number().int().positive().default(64),
|
|
389
389
|
salience: SalienceSchema.default({
|
|
@@ -433,6 +433,7 @@ const bootstrap = defineBootstrap.kernel({
|
|
|
433
433
|
...embedder !== null ? { embedder } : {},
|
|
434
434
|
queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
|
|
435
435
|
writeEmbedTimeoutMs: config.writeEmbedTimeoutMs,
|
|
436
|
+
embedBackfillIntervalMs: config.embedBackfillIntervalMs,
|
|
436
437
|
...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
|
|
437
438
|
store: deps.store,
|
|
438
439
|
...deps.stats !== void 0 ? { stats: deps.stats } : {},
|
|
@@ -124,6 +124,12 @@
|
|
|
124
124
|
"exclusiveMinimum": 0,
|
|
125
125
|
"maximum": 9007199254740991
|
|
126
126
|
},
|
|
127
|
+
"embedBackfillIntervalMs": {
|
|
128
|
+
"default": 60000,
|
|
129
|
+
"type": "integer",
|
|
130
|
+
"minimum": 0,
|
|
131
|
+
"maximum": 9007199254740991
|
|
132
|
+
},
|
|
127
133
|
"hnswM": {
|
|
128
134
|
"default": 16,
|
|
129
135
|
"type": "integer",
|
package/dist/recall.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { $ as PolicyProbeContext, P as HttpHandler, Q as PolicyDecision, S as PostWriteHook, Y as Identity, ct as Dataset, et as RequestContext, h as McpToolRegistration, nt as ResolverArgs, p as KernelPlugin, r as EmbedSourceContext, s as FilterOperatorDef, u as KernelMigration } from "./plugin-t2g2wX0b.js";
|
|
1
|
+
import { C as EmbedSourceContext, D as FilterOperatorDef, I as McpToolRegistration, Ln as Row, On as EdgesStore, P as KernelPlugin, U as PostWriteHook, _t as PolicyDecision, i as FieldDef, j as KernelMigration, kt as Dataset, mt as Identity, tt as HttpHandler, vt as PolicyProbeContext, xt as ResolverArgs, yt as RequestContext } from "./schema-laGLMvMC.js";
|
|
3
2
|
import { t as Embedder } from "./types-8c0gtnd2.js";
|
|
4
3
|
import postgres, { Sql } from "postgres";
|
|
5
4
|
//#region ../../plugins/substrate-recall/src/candidate-selection.d.ts
|
|
@@ -305,6 +304,17 @@ interface EmbeddingsStore {
|
|
|
305
304
|
* mismatched vector surfaces as a substrate-level error.
|
|
306
305
|
*/
|
|
307
306
|
upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
|
|
307
|
+
/**
|
|
308
|
+
* Ids of live rows of (namespace, type) whose vector is missing or older than
|
|
309
|
+
* the row, after `afterId` in id order: the writes whose embed failed. Optional;
|
|
310
|
+
* a store that cannot see the rows table omits it and nothing is backfilled.
|
|
311
|
+
*/
|
|
312
|
+
staleRowIds?(input: {
|
|
313
|
+
namespace: string;
|
|
314
|
+
type: string;
|
|
315
|
+
afterId: string;
|
|
316
|
+
limit: number;
|
|
317
|
+
}): Promise<string[]>;
|
|
308
318
|
/**
|
|
309
319
|
* Remove the embedding for one row, leaving the row itself untouched. Used
|
|
310
320
|
* to un-index a row whose `embed.source` returns the null sentinel.
|
|
@@ -724,6 +734,14 @@ interface RecallInput {
|
|
|
724
734
|
* or null field. A malformed filter is refused with `bad_request`.
|
|
725
735
|
*/
|
|
726
736
|
where?: KnnRowFilter;
|
|
737
|
+
/**
|
|
738
|
+
* Row-body projection. Field -> `true` keeps those keys plus `id`; `true`
|
|
739
|
+
* keeps the whole body. Absent → `id` plus the handle fields
|
|
740
|
+
* (`HANDLE_FIELDS`) the row has. Applied after row filtering and policy,
|
|
741
|
+
* which still see the full body. A malformed value is refused with
|
|
742
|
+
* `bad_request`.
|
|
743
|
+
*/
|
|
744
|
+
rowFields?: Record<string, true> | true;
|
|
727
745
|
}
|
|
728
746
|
interface RecallRow {
|
|
729
747
|
/**
|
|
@@ -1498,7 +1516,7 @@ export declare class InMemoryEmbeddingsStore implements EmbeddingsStore {
|
|
|
1498
1516
|
//#endregion
|
|
1499
1517
|
//#region ../../plugins/substrate-recall/src/in-memory-lexical-store.d.ts
|
|
1500
1518
|
/**
|
|
1501
|
-
* Resolves a row body, mirroring pg's join against
|
|
1519
|
+
* Resolves a row body, mirroring pg's join against the type's table.
|
|
1502
1520
|
*
|
|
1503
1521
|
* Signature matches `InMemoryEmbeddingsStoreOptions.datasetFor` deliberately:
|
|
1504
1522
|
* a host wires both from the same registry lookup.
|
|
@@ -1647,7 +1665,7 @@ export declare class PgClusterStore implements ClusterStore {
|
|
|
1647
1665
|
constructor(opts: PgClusterStoreOptions);
|
|
1648
1666
|
/**
|
|
1649
1667
|
* Runs `fn` holding a session advisory lock that excludes formation in every
|
|
1650
|
-
* process on this
|
|
1668
|
+
* process on this schema; throws `crystallise_in_flight` if it is held.
|
|
1651
1669
|
*
|
|
1652
1670
|
* Lock and unlock share one reserved connection, since through the pool the
|
|
1653
1671
|
* unlock can land on another session and fail to release.
|
|
@@ -1688,9 +1706,15 @@ export declare class PgEmbeddingsStore implements EmbeddingsStore {
|
|
|
1688
1706
|
id: string;
|
|
1689
1707
|
}>, model: string): Promise<ReadonlyArray<EmbeddingsStoreFetchResult>>;
|
|
1690
1708
|
selectCandidates(selector: CandidateSelector, model: string): Promise<Candidate[]>;
|
|
1709
|
+
staleRowIds(input: {
|
|
1710
|
+
namespace: string;
|
|
1711
|
+
type: string;
|
|
1712
|
+
afterId: string;
|
|
1713
|
+
limit: number;
|
|
1714
|
+
}): Promise<string[]>;
|
|
1691
1715
|
/**
|
|
1692
1716
|
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
1693
|
-
* Needed because `embeddings` has no FK to
|
|
1717
|
+
* Needed because `embeddings` has no FK to the type tables and no
|
|
1694
1718
|
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
1695
1719
|
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
1696
1720
|
*
|
|
@@ -1716,7 +1740,7 @@ export declare class PgLexicalStore implements LexicalStore {
|
|
|
1716
1740
|
/**
|
|
1717
1741
|
* Deletes every lexical row whose backing row is missing or soft-deleted,
|
|
1718
1742
|
* as `PgEmbeddingsStore.purgeOrphans` does for vectors: `lexical` has no FK
|
|
1719
|
-
* to
|
|
1743
|
+
* to the type tables, so a drop, purge or tombstone would otherwise leave the text.
|
|
1720
1744
|
*
|
|
1721
1745
|
* @returns number of orphaned lexical rows deleted.
|
|
1722
1746
|
*/
|
|
@@ -1750,6 +1774,11 @@ interface RecallPluginConfig {
|
|
|
1750
1774
|
* the row's previous vector until its next write or a reindex. Default 30000.
|
|
1751
1775
|
*/
|
|
1752
1776
|
writeEmbedTimeoutMs?: number;
|
|
1777
|
+
/**
|
|
1778
|
+
* How often to re-embed rows whose write-path embed failed or timed out (their
|
|
1779
|
+
* vector is missing or older than the row). 0 disables. Default 60000.
|
|
1780
|
+
*/
|
|
1781
|
+
embedBackfillIntervalMs?: number;
|
|
1753
1782
|
store: EmbeddingsStore;
|
|
1754
1783
|
/**
|
|
1755
1784
|
* Store for the lexical arm. Absent → types declaring `lexical.source`
|
package/dist/recall.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { a as
|
|
3
|
-
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-QuJN9pG_.js";
|
|
1
|
+
import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-De9j33jv.js";
|
|
2
|
+
import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-MSbSZ6Al.js";
|
|
4
3
|
|
|
5
4
|
export { CLUSTERER_BACKEND, CLUSTERER_BACKEND_VERSION, CLUSTER_FIELDS, CLUSTER_MEMBER_FIELDS, CRYSTALLISE_IN_FLIGHT_CODE, DEFAULT_CRYSTALLISE_CONFIG, EMBEDDINGS_FETCH_MAX_REFS, EmbedSourceRegistry, InMemoryClusterStore, InMemoryEmbeddingsStore, InMemoryLexicalStore, PgClusterStore, PgEmbeddingsStore, PgLexicalStore, RECALL_VERSION, REINDEX_DEFAULT_LIMIT, REINDEX_MAX_LIMIT, RRF_K, SALIENCE_BOOST_MAX_TARGETS, SELECT_CANDIDATES_MAX_LIMIT, armRanks, buildClusterSurface, checkParams, createCrystalliseGuard, createEmbedStats, createRecallPlugin, crystalliseRouteHandler, embedAndStore, embedRowReporting, embedStatsRouteHandler, embeddingSimilarOperator, embeddingsFetchRouteHandler, embeddingsMigration, fuseByRank, handleCrystallise, handleEmbedStatsRequest, handleLexicalCoverage, handleRecallRequest, handleReindex, l2normalize, lexicalMigration, makeEmbedPostWrite, makeLexicalClassify, makeLexicalPostWrite, makeReembed, memberKey, recallClustersMigration, recallMcpTool, recallRouteHandler, reindexRouteHandler, runClusterer, salienceBoostRouteHandler, unknownParamMessage, unknownParams, validateCrystalliseInput, validateLexicalCoverageInput, validateReindexInput, validateSelector };
|