@mnmal-ai/substrate 1.0.0-alpha.0 → 1.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +51 -9
  2. package/dist/boot-store-BVqfFxh1.js +4060 -0
  3. package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-Cp4PaIYO.d.ts} +2 -87
  4. package/dist/cli-hSCBywhE.js +3126 -0
  5. package/dist/client.d.ts +130 -8
  6. package/dist/client.js +6 -5
  7. package/dist/{plugin-t2g2wX0b.d.ts → dataset-CejF-Fr_.d.ts} +1640 -1320
  8. package/dist/{agent-token-BIboWOU0.js → env-utils-CGNazUNt.js} +31 -2
  9. package/dist/{errors-D4vKuwz8.d.ts → errors-7mycvNGq.d.ts} +1 -1
  10. package/dist/{index-D7mSOFlM.d.ts → index-DgSfwx7M.d.ts} +1 -1
  11. package/dist/node-version-C5FSmwJX.js +26 -0
  12. package/dist/pack.d.ts +2 -2
  13. package/dist/pack.js +1 -1
  14. package/dist/pg-edges-store-Lxsl5ZnK.js +381 -0
  15. package/dist/{plugin-j1jt1l3n.js → plugin-CtwOINCM.js} +171 -18
  16. package/dist/proposals-bootstrap.js +6 -4
  17. package/dist/proposals.config.schema.json +5 -0
  18. package/dist/proposals.d.ts +15 -4
  19. package/dist/proposals.default-config.json +2 -1
  20. package/dist/proposals.js +2 -2
  21. package/dist/recall-bootstrap.js +13 -4
  22. package/dist/recall.config.schema.json +12 -0
  23. package/dist/recall.d.ts +54 -9
  24. package/dist/recall.js +2 -2
  25. package/dist/{repl-C52wyfMs.js → repl-DqfHelK7.js} +62 -29
  26. package/dist/{row-ByLFE2Wx.d.ts → row-BKKCXIG6.d.ts} +5 -3
  27. package/dist/run-B5dTbZ3g.js +1997 -0
  28. package/dist/{schema-g4FQLr0V.d.ts → schema-DoSc-4ig.d.ts} +44 -7
  29. package/dist/script-C6DSa_Na.js +134 -0
  30. package/dist/sdk.d.ts +90 -12
  31. package/dist/sdk.js +5 -110
  32. package/dist/{src-D91yij6C.js → src-5SJ5GEIM.js} +1557 -327
  33. package/dist/{src-DuopV8u7.js → src-7CUSMK66.js} +148 -19
  34. package/dist/{src-CrEgiMJQ.js → src-Bi849yhn.js} +39 -15
  35. package/dist/{src-D88LvCX8.js → src-CqT-QWZ_.js} +697 -57
  36. package/dist/src-CyvV7dCK.js +4333 -0
  37. package/dist/{row-DK7lPZVe.js → src-Da0t6si5.js} +11 -11
  38. package/dist/{src-C4DzaIQZ.js → src-hZb8wfvg.js} +12 -1
  39. package/dist/substrate-admin.js +17 -781
  40. package/dist/substrate.js +4 -5471
  41. package/dist/testing.d.ts +557 -8
  42. package/dist/testing.js +39 -3
  43. package/dist/{ws-C72iVS5T.d.ts → ws-ByZyHk7m.d.ts} +3 -3
  44. package/package.json +21 -20
  45. package/dist/dataset-BXkus75W.js +0 -1527
  46. package/dist/env-utils-B7SGmoeJ.js +0 -21
  47. package/dist/src-CzxvmhFn.js +0 -2327
@@ -1,5 +1,6 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { L as applyObligations, Q as buildContext, et as jsonResponse, wt as ResolverFrameError } from "./src-D91yij6C.js";
2
+ import { J as applyObligations, Vt as ResolverFrameError, ft as buildContext, mt as jsonResponse } from "./src-5SJ5GEIM.js";
3
+ import "./src-hZb8wfvg.js";
3
4
  import { Worker } from "node:worker_threads";
4
5
 
5
6
  //#region ../../plugins/substrate-recall/src/candidate-selection.ts
@@ -949,8 +950,14 @@ async function handleRecallRequest(input, deps) {
949
950
  const problem = rowFilterProblem(where);
950
951
  if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"where" ${problem}`), { code: "bad_request" });
951
952
  }
953
+ const rowFields = input.rowFields;
954
+ if (rowFields !== void 0) {
955
+ const problem = rowFieldsProblem(rowFields);
956
+ if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"rowFields" ${problem}`), { code: "bad_request" });
957
+ }
952
958
  const type = input.type;
953
959
  if (type === void 0 || type === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `type`, the type to search. There is no default."), { code: "bad_request" });
960
+ if (typeof input.query !== "string" || input.query.trim() === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `query`, the text to search for."), { code: "bad_request" });
954
961
  if (input.query.length > 8192) throw Object.assign(/* @__PURE__ */ new Error(`recall: query is longer than ${MAX_RECALL_QUERY_CHARS} characters`), { code: "bad_request" });
955
962
  const k = clamp(input.k ?? 10, 1, 100);
956
963
  const threshold = typeof input.threshold === "number" ? input.threshold : 0;
@@ -976,7 +983,8 @@ async function handleRecallRequest(input, deps) {
976
983
  type,
977
984
  model: embedder.modelId,
978
985
  vector,
979
- k
986
+ k,
987
+ salienceHalfLifeHours: salience.halfLifeHours
980
988
  };
981
989
  if (threshold > 0) knnInput.threshold = threshold;
982
990
  if (returnVector) knnInput.returnVector = true;
@@ -1164,7 +1172,7 @@ async function handleRecallRequest(input, deps) {
1164
1172
  }
1165
1173
  const t3 = performance.now();
1166
1174
  return {
1167
- rows: fusedRows,
1175
+ rows: rowFields === true ? fusedRows : fusedRows.map((r) => projectRow(r, rowFields)),
1168
1176
  model: denseRan ? embedder?.modelId ?? "" : "",
1169
1177
  arms: {
1170
1178
  dense: denseRan,
@@ -1210,6 +1218,23 @@ function extractTargets(namespace, type, rows) {
1210
1218
  }
1211
1219
  return out;
1212
1220
  }
1221
+ /** Why `rowFields` is malformed, or `null` when it is `true` or an object of `field: true`. */
1222
+ function rowFieldsProblem(value) {
1223
+ if (value === true) return null;
1224
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return "must be true (the whole body) or an object of field: true, e.g. { \"body\": true }";
1225
+ for (const [field, v] of Object.entries(value)) if (v !== true) return `field "${field}" must be true`;
1226
+ return null;
1227
+ }
1228
+ /** Narrow a hit's `row` to `id` plus `rowFields`, or plus the handle fields when absent. */
1229
+ function projectRow(hit, rowFields) {
1230
+ const row = {};
1231
+ const keep = rowFields === void 0 ? HANDLE_FIELDS : Object.keys(rowFields);
1232
+ for (const field of ["id", ...keep]) if (Object.hasOwn(hit.row, field)) row[field] = hit.row[field];
1233
+ return {
1234
+ ...hit,
1235
+ row
1236
+ };
1237
+ }
1213
1238
 
1214
1239
  //#endregion
1215
1240
  //#region ../../plugins/substrate-recall/src/params.ts
@@ -1263,7 +1288,8 @@ const ROUTE_PARAMS = [
1263
1288
  "returnVector",
1264
1289
  "lexical",
1265
1290
  "explain",
1266
- "where"
1291
+ "where",
1292
+ "rowFields"
1267
1293
  ];
1268
1294
  function parseBody$3(req) {
1269
1295
  const raw = req.body;
@@ -1323,6 +1349,7 @@ function parseBody$3(req) {
1323
1349
  input.explain = obj.explain;
1324
1350
  }
1325
1351
  if (obj.where !== void 0) input.where = obj.where;
1352
+ if (obj.rowFields !== void 0) input.rowFields = obj.rowFields;
1326
1353
  return input;
1327
1354
  }
1328
1355
  /**
@@ -1502,14 +1529,35 @@ function rethrowCoded$1(err) {
1502
1529
  * answers both visibility and counts; `check` is cached per scope.
1503
1530
  */
1504
1531
  async function resolveClusterVisibility(record, deps, ctx) {
1505
- if (deps.checkReadPolicy === void 0) return {
1532
+ if (deps.checkReadPolicy === void 0) return storedVisibility(record);
1533
+ const members = await deps.clusters.listMembers(record.id, { includeDeparted: true });
1534
+ return visibilityOf(members, await readableMembers(members, deps, ctx));
1535
+ }
1536
+ /**
1537
+ * `resolveClusterVisibility` for a whole listing: one member read and one
1538
+ * readability check across every cluster, rather than two per cluster.
1539
+ */
1540
+ async function resolveListVisibility(records, deps, ctx) {
1541
+ if (deps.checkReadPolicy === void 0 || records.length === 0) return records.map(storedVisibility);
1542
+ const members = await deps.clusters.listMembers(records.map((r) => r.id), { includeDeparted: true });
1543
+ const readable = await readableMembers(members, deps, ctx);
1544
+ const byCluster = /* @__PURE__ */ new Map();
1545
+ for (const m of members) {
1546
+ const list = byCluster.get(m.clusterId);
1547
+ if (list === void 0) byCluster.set(m.clusterId, [m]);
1548
+ else list.push(m);
1549
+ }
1550
+ return records.map((r) => visibilityOf(byCluster.get(r.id) ?? [], readable));
1551
+ }
1552
+ function storedVisibility(record) {
1553
+ return {
1506
1554
  visible: true,
1507
1555
  memberCount: record.memberCount,
1508
1556
  departedCount: record.departedCount,
1509
1557
  labelReadable: true
1510
1558
  };
1511
- const members = await deps.clusters.listMembers(record.id, { includeDeparted: true });
1512
- const readable = await readableMembers(members, deps, ctx);
1559
+ }
1560
+ function visibilityOf(members, readable) {
1513
1561
  let liveTotal = 0;
1514
1562
  let memberCount = 0;
1515
1563
  let departedCount = 0;
@@ -1559,10 +1607,11 @@ function buildClusterSurface(deps) {
1559
1607
  return [project(clusterRow(one, vis), args.fields)];
1560
1608
  }
1561
1609
  const rows = await deps.clusters.listClusters(p.includeRetired === true ? { includeRetired: true } : {});
1610
+ const visibility = await resolveListVisibility(rows, deps, ctx);
1562
1611
  const visible = [];
1563
- for (const c of rows) {
1564
- const vis = await resolveClusterVisibility(c, deps, ctx);
1565
- if (vis.visible) visible.push(project(clusterRow(c, vis), args.fields));
1612
+ for (const [i, c] of rows.entries()) {
1613
+ const vis = visibility[i];
1614
+ if (vis?.visible === true) visible.push(project(clusterRow(c, vis), args.fields));
1566
1615
  }
1567
1616
  return visible;
1568
1617
  };
@@ -1739,7 +1788,7 @@ async function embedRowReporting(row, opts) {
1739
1788
  if (opts.serializeRow) source = await opts.serializeRow(row);
1740
1789
  else if (opts.fields && opts.fields.length > 0) source = opts.fields.map((f) => row[f]).filter((v) => typeof v === "string" && v.length > 0).join("\n");
1741
1790
  else source = "";
1742
- if (source === null || source === void 0) {
1791
+ if (source === null || source === void 0 || source.trim() === "") {
1743
1792
  await opts.store.delete({
1744
1793
  rowNamespace: opts.namespace,
1745
1794
  rowType: opts.type,
@@ -2442,7 +2491,8 @@ const MCP_PARAMS = [
2442
2491
  "returnVector",
2443
2492
  "lexical",
2444
2493
  "explain",
2445
- "where"
2494
+ "where",
2495
+ "rowFields"
2446
2496
  ];
2447
2497
  function recallMcpTool(cfg) {
2448
2498
  return {
@@ -2490,6 +2540,10 @@ function recallMcpTool(cfg) {
2490
2540
  where: {
2491
2541
  type: "object",
2492
2542
  description: "Row-field filter applied inside every candidate source, so k fills with matching rows: field -> { eq: scalar | null } or { in: [scalar | null, ...] }, every field must match. eq null matches a missing or null field. Example: { \"project\": { \"in\": [\"substrate\", null] } }."
2543
+ },
2544
+ rowFields: {
2545
+ type: ["object", "boolean"],
2546
+ description: "Row-body projection. Omitted, each hit's `row` carries `id` plus its handle fields (name, title, topic, synopsis, description), enough to decide on. field -> true keeps just those fields plus `id`, e.g. { \"synopsis\": true, \"body\": true }; true returns the whole body."
2493
2547
  }
2494
2548
  },
2495
2549
  required: ["query", "type"]
@@ -2725,7 +2779,7 @@ async function alignVectorDimensions(s, opts) {
2725
2779
  const { dimension } = opts;
2726
2780
  if (!Number.isInteger(dimension) || dimension <= 0) throw new Error(`alignVectorDimensions: dimension must be a positive integer (got ${dimension})`);
2727
2781
  return await s.begin(async (tx) => {
2728
- await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align'))`;
2782
+ await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align:' || current_schema()))`;
2729
2783
  return alignLocked(tx, opts);
2730
2784
  });
2731
2785
  }
@@ -2752,6 +2806,50 @@ async function alignLocked(s, opts) {
2752
2806
  };
2753
2807
  }
2754
2808
 
2809
+ //#endregion
2810
+ //#region ../../plugins/substrate-recall/src/embed-backfill.ts
2811
+ async function runEmbedBackfill(deps) {
2812
+ const result = {
2813
+ embedded: 0,
2814
+ unindexable: 0,
2815
+ failed: 0
2816
+ };
2817
+ const staleRowIds = deps.store.staleRowIds?.bind(deps.store);
2818
+ if (staleRowIds === void 0) return result;
2819
+ for (const { namespace, type } of deps.registry.types()) {
2820
+ const reembed = deps.registry.get(namespace, type)?.reembed;
2821
+ const dataset = deps.resolveDataset(namespace);
2822
+ if (reembed === void 0 || dataset === null) continue;
2823
+ const key = `${namespace}/${type}`;
2824
+ const ids = await staleRowIds({
2825
+ namespace,
2826
+ type,
2827
+ afterId: deps.cursors.get(key) ?? "",
2828
+ limit: deps.perTypeLimit
2829
+ });
2830
+ if (ids.length < deps.perTypeLimit) deps.cursors.delete(key);
2831
+ else deps.cursors.set(key, ids[ids.length - 1]);
2832
+ if (ids.length === 0) continue;
2833
+ const { rows } = await dataset.read(type, {
2834
+ ids,
2835
+ limit: ids.length
2836
+ });
2837
+ for (const row of rows) {
2838
+ const outcome = await reembed(row);
2839
+ if (outcome.status === "embedded") result.embedded++;
2840
+ else if (outcome.status === "unindexable") result.unindexable++;
2841
+ else if (outcome.status === "failed") {
2842
+ result.failed++;
2843
+ if (outcome.code === "unavailable") return {
2844
+ ...result,
2845
+ stopped: outcome.error
2846
+ };
2847
+ }
2848
+ }
2849
+ }
2850
+ return result;
2851
+ }
2852
+
2755
2853
  //#endregion
2756
2854
  //#region ../../plugins/substrate-recall/src/expansion-resolver.ts
2757
2855
  function makeExpansionResolver(registry, edges, cfg) {
@@ -3011,12 +3109,14 @@ function rethrowCoded(err) {
3011
3109
  throw err;
3012
3110
  }
3013
3111
  /**
3014
- * Keep only the `=== true` keys of `fields`; `undefined` fields = passthrough.
3112
+ * Keep only the selected keys of `fields`; `undefined` fields = passthrough.
3015
3113
  * `fields: {}` strips every key, matching the compose layer's projectEnvelope.
3114
+ * A nested `row: { … }` selection keeps `row`, which the core has already
3115
+ * narrowed through `rowFields`.
3016
3116
  */
3017
3117
  function projectHit(row, fields) {
3018
3118
  if (fields === void 0) return row;
3019
- const keep = Object.entries(fields).filter(([, v]) => v === true).map(([k]) => k);
3119
+ const keep = Object.entries(fields).filter(([k, v]) => v === true || k === "row" && isPlainObject(v)).map(([k]) => k);
3020
3120
  const out = {};
3021
3121
  for (const k of keep) if (k in row) out[k] = row[k];
3022
3122
  return out;
@@ -3039,6 +3139,9 @@ function buildRecallFrameSurface(deps) {
3039
3139
  const salience = deps.salience ?? DEFAULT_SALIENCE_CONFIG;
3040
3140
  const hitResolver = async (ctx, args) => {
3041
3141
  const input = parseHitParams(args.params);
3142
+ const rowSelection = args.fields?.row;
3143
+ if (rowSelection === true) input.rowFields = true;
3144
+ else if (isPlainObject(rowSelection)) input.rowFields = rowSelection;
3042
3145
  const bound = bindPolicy(deps.checkReadPolicy, callerFromContext(ctx));
3043
3146
  const boundWrite = bindPolicy(deps.checkWritePolicy, callerFromContext(ctx));
3044
3147
  let out;
@@ -3120,6 +3223,9 @@ function buildRecallFrameSurface(deps) {
3120
3223
  }
3121
3224
  };
3122
3225
  }
3226
+ function isPlainObject(v) {
3227
+ return v !== null && typeof v === "object" && !Array.isArray(v);
3228
+ }
3123
3229
 
3124
3230
  //#endregion
3125
3231
  //#region ../../plugins/substrate-recall/src/lexical-reindex-core.ts
@@ -3593,9 +3699,13 @@ function makeSpreadResolver(registry, edges) {
3593
3699
  //#endregion
3594
3700
  //#region ../../plugins/substrate-recall/src/plugin.ts
3595
3701
  const DEFAULT_CLUSTER_DIMENSION = 768;
3702
+ const DEFAULT_EMBED_BACKFILL_INTERVAL_MS = 6e4;
3703
+ /** Rows per embedded type per pass, so one pass stays a few seconds of embed work. */
3704
+ const EMBED_BACKFILL_PER_TYPE = 50;
3596
3705
  const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3e4;
3706
+ const DEFAULT_WRITE_EMBED_TIMEOUT_MS = 3e4;
3597
3707
  /** `embedder` with `timeoutMs` applied to every call that does not set its own. */
3598
- function withQueryTimeout(embedder, timeoutMs) {
3708
+ function withEmbedTimeout(embedder, timeoutMs) {
3599
3709
  return {
3600
3710
  modelId: embedder.modelId,
3601
3711
  dimension: embedder.dimension,
@@ -3634,7 +3744,7 @@ function makeEmbedSourceContext(resolveDataset) {
3634
3744
  }
3635
3745
  function createRecallPlugin(cfg) {
3636
3746
  const stats = cfg.stats ?? createEmbedStats();
3637
- const queryEmbedder = cfg.embedder === void 0 ? void 0 : withQueryTimeout(cfg.embedder, cfg.queryEmbedTimeoutMs ?? 3e4);
3747
+ const queryEmbedder = cfg.embedder === void 0 ? void 0 : withEmbedTimeout(cfg.embedder, cfg.queryEmbedTimeoutMs ?? 3e4);
3638
3748
  const spreadRegistry = new SpreadEdgeRegistry();
3639
3749
  const embedSourceRegistry = new EmbedSourceRegistry();
3640
3750
  const lexicalSourceRegistry = new LexicalSourceRegistry();
@@ -3669,11 +3779,19 @@ function createRecallPlugin(cfg) {
3669
3779
  ...cfg.checkReadPolicy !== void 0 ? { checkReadPolicy: cfg.checkReadPolicy } : {},
3670
3780
  ...cfg.runClusterer !== void 0 ? { runClusterer: cfg.runClusterer } : {}
3671
3781
  };
3782
+ let backfillTimer;
3672
3783
  return {
3673
3784
  name: "substrate-recall",
3674
3785
  contractVersion: ">=1.0.0-alpha.0 <2.0.0",
3786
+ storeTables: [
3787
+ "embeddings",
3788
+ "lexical",
3789
+ "recall_clusters",
3790
+ "recall_cluster_members"
3791
+ ],
3675
3792
  async onKernelReady(ctx) {
3676
3793
  if (cfg.embedder === void 0 || ctx.sql === null) return;
3794
+ let dimensionMismatch = false;
3677
3795
  const report = await alignVectorDimensions(ctx.sql, {
3678
3796
  dimension: cfg.embedder.dimension,
3679
3797
  ...cfg.migration?.hnswM !== void 0 ? { hnswM: cfg.migration.hnswM } : {},
@@ -3681,6 +3799,7 @@ function createRecallPlugin(cfg) {
3681
3799
  });
3682
3800
  for (const [column, state] of [["embeddings.embedding", report.embeddings], ["recall_clusters.centroid", report.centroids]]) {
3683
3801
  if (state !== "populated") continue;
3802
+ dimensionMismatch = true;
3684
3803
  const found = column.startsWith("embeddings") ? report.found.embeddings : report.found.centroids;
3685
3804
  ctx.metric("recall_dimension_mismatch", {
3686
3805
  column,
@@ -3690,6 +3809,37 @@ function createRecallPlugin(cfg) {
3690
3809
  });
3691
3810
  process.emitWarning(`${column} is vector(${found}) and holds rows, but the embedder is ${cfg.embedder.modelId} at ${cfg.embedder.dimension}. Writes to it will fail until the rows are cleared (they are derived) and re-embedded: delete them, restart, then POST /substrate/v1/embed/reindex.`, { code: "RECALL_DIMENSION_MISMATCH" });
3692
3811
  }
3812
+ const intervalMs = cfg.embedBackfillIntervalMs ?? DEFAULT_EMBED_BACKFILL_INTERVAL_MS;
3813
+ const resolveDataset = cfg.resolveDataset;
3814
+ if (dimensionMismatch || intervalMs === 0 || resolveDataset === void 0) return;
3815
+ if (!ctx.background) return;
3816
+ const cursors = /* @__PURE__ */ new Map();
3817
+ let running = false;
3818
+ clearInterval(backfillTimer);
3819
+ backfillTimer = setInterval(() => {
3820
+ if (running) return;
3821
+ running = true;
3822
+ runEmbedBackfill({
3823
+ registry: embedSourceRegistry,
3824
+ store: cfg.store,
3825
+ resolveDataset,
3826
+ cursors,
3827
+ perTypeLimit: EMBED_BACKFILL_PER_TYPE
3828
+ }).then((r) => {
3829
+ if (r.embedded + r.failed === 0 && r.stopped === void 0) return;
3830
+ ctx.metric("recall_embed_backfill", r);
3831
+ process.stdout.write(`recall.embed_backfill ${JSON.stringify(r)}\n`);
3832
+ }).catch((err) => {
3833
+ process.stderr.write(`recall.embed_backfill failed: ${err instanceof Error ? err.message : String(err)}\n`);
3834
+ }).finally(() => {
3835
+ running = false;
3836
+ });
3837
+ }, intervalMs);
3838
+ backfillTimer.unref();
3839
+ },
3840
+ onKernelStop() {
3841
+ clearInterval(backfillTimer);
3842
+ backfillTimer = void 0;
3693
3843
  },
3694
3844
  migrations: [
3695
3845
  embeddingsMigration(cfg.migration ?? {}),
@@ -3826,7 +3976,10 @@ function createRecallPlugin(cfg) {
3826
3976
  stats,
3827
3977
  ...embedSourceContext !== void 0 ? { embedSourceContext } : {}
3828
3978
  };
3829
- const hook = makeEmbedPostWrite(binding);
3979
+ const hook = makeEmbedPostWrite({
3980
+ ...binding,
3981
+ embedder: withEmbedTimeout(binding.embedder, cfg.writeEmbedTimeoutMs ?? 3e4)
3982
+ });
3830
3983
  out[typeName] = composeHooks(out[typeName], hook);
3831
3984
  embedSourceRegistry.add({
3832
3985
  namespace: namespace.namespace,
@@ -1,6 +1,6 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { Pt as defineBootstrap } from "./src-D91yij6C.js";
3
- import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-D88LvCX8.js";
2
+ import { Qt as defineBootstrap } from "./src-5SJ5GEIM.js";
3
+ import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-CqT-QWZ_.js";
4
4
  import { z } from "zod";
5
5
 
6
6
  //#region ../../plugins/substrate-proposals/src/bootstrap.ts
@@ -22,14 +22,16 @@ import { z } from "zod";
22
22
  */
23
23
  const ProposalsBootstrapConfigSchema = z.object({
24
24
  ownedNamespaces: z.array(z.string().min(1)).default([]),
25
- proposeCapability: z.string().min(1).default("substrate.propose")
25
+ proposeCapability: z.string().min(1).default("substrate.propose"),
26
+ indexCapability: z.string().min(1).default("substrate.index")
26
27
  }).strict();
27
28
  const bootstrap = defineBootstrap.coordinator({
28
29
  schema: ProposalsBootstrapConfigSchema,
29
30
  build: (config) => createProposalsPlugin({
30
31
  ownedNamespaces: config.ownedNamespaces,
31
32
  pubsub: createNoopPubSub(),
32
- proposeCapability: config.proposeCapability
33
+ proposeCapability: config.proposeCapability,
34
+ indexCapability: config.indexCapability
33
35
  })
34
36
  });
35
37
 
@@ -14,6 +14,11 @@
14
14
  "default": "substrate.propose",
15
15
  "type": "string",
16
16
  "minLength": 1
17
+ },
18
+ "indexCapability": {
19
+ "default": "substrate.index",
20
+ "type": "string",
21
+ "minLength": 1
17
22
  }
18
23
  },
19
24
  "additionalProperties": false
@@ -1,6 +1,6 @@
1
- import { v as OnDelete } from "./schema-g4FQLr0V.js";
2
- import { ct as Dataset, t as CoordinatorPlugin } from "./plugin-t2g2wX0b.js";
3
- import { a as ResolverFrameError } from "./errors-D4vKuwz8.js";
1
+ import { x as OnDelete } from "./schema-DoSc-4ig.js";
2
+ import { k as CoordinatorPlugin, n as Dataset } from "./dataset-CejF-Fr_.js";
3
+ import { a as ResolverFrameError } from "./errors-7mycvNGq.js";
4
4
  //#region ../../plugins/substrate-proposals/src/config.d.ts
5
5
  /**
6
6
  * Multi-instance invalidation transport used by the plugin to broadcast
@@ -41,6 +41,13 @@ interface ProposalsPluginConfig {
41
41
  * propose authority under a differently-named cap.
42
42
  */
43
43
  proposeCapability?: string;
44
+ /**
45
+ * Capability name required to invoke `propose_index`, in the namespace whose
46
+ * indexes it changes. Defaults to `'substrate.index'`: separate from the
47
+ * propose capability, so only the operator or a designated reviewer changes
48
+ * indexes.
49
+ */
50
+ indexCapability?: string;
44
51
  /**
45
52
  * Lookup function the `propose_namespace` resolver uses to find the dataset
46
53
  * for a hot-added namespace. The host (typically the substrate binary) wires
@@ -57,8 +64,12 @@ interface ProposalsPluginConfig {
57
64
  */
58
65
  export declare function createNoopPubSub(): PubSub;
59
66
  //#endregion
67
+ //#region ../../plugins/substrate-proposals/src/engine-write.d.ts
68
+ /** Resolves once every storage preparation started so far has settled. */
69
+ export declare function storageSettled(): Promise<void>;
70
+ //#endregion
60
71
  //#region ../../plugins/substrate-proposals/src/errors.d.ts
61
- type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired';
72
+ type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired' | 'index_on_code_defined_type' | 'unknown_index' | 'not_engine_owned' | 'unique_index' | 'still_desired' | 'unknown_recommendation' | 'recommendation_closed' | 'recommendation_stale' | 'propose_index_not_alone' | 'index_drop_timeout';
62
73
  /**
63
74
  * Structured error for proposal operations with stable error codes.
64
75
  *
@@ -1,4 +1,5 @@
1
1
  {
2
2
  "ownedNamespaces": [],
3
- "proposeCapability": "substrate.propose"
3
+ "proposeCapability": "substrate.propose",
4
+ "indexCapability": "substrate.index"
4
5
  }
package/dist/proposals.js CHANGED
@@ -1,4 +1,4 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { n as createNoopPubSub, r as ProposalError, t as createProposalsPlugin } from "./src-D88LvCX8.js";
2
+ import { i as ProposalError, n as createNoopPubSub, r as storageSettled, t as createProposalsPlugin } from "./src-CqT-QWZ_.js";
3
3
 
4
- export { ProposalError, createNoopPubSub, createProposalsPlugin };
4
+ export { ProposalError, createNoopPubSub, createProposalsPlugin, storageSettled };
@@ -1,7 +1,7 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { Pt as defineBootstrap } from "./src-D91yij6C.js";
2
+ import { Qt as defineBootstrap } from "./src-5SJ5GEIM.js";
3
3
  import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
4
- import { t as createRecallPlugin } from "./plugin-j1jt1l3n.js";
4
+ import { t as createRecallPlugin } from "./plugin-CtwOINCM.js";
5
5
  import { z } from "zod";
6
6
 
7
7
  //#region ../../packages/substrate-embedder/src/types.ts
@@ -49,7 +49,7 @@ var OllamaEmbedder = class {
49
49
  async embed(text, opts) {
50
50
  const timeoutMs = opts?.timeoutMs ?? this.timeoutMs;
51
51
  const contextLength = await this.resolveContextLength(timeoutMs);
52
- const url = `${this.baseUrl.replace(/\/+$/, "")}/api/embed`;
52
+ const url = `${trimTrailingSlashes(this.baseUrl)}/api/embed`;
53
53
  const controller = new AbortController();
54
54
  const timer = setTimeout(() => controller.abort(), timeoutMs);
55
55
  let res;
@@ -98,7 +98,7 @@ var OllamaEmbedder = class {
98
98
  return this.contextLength;
99
99
  }
100
100
  async fetchContextLength(timeoutMs) {
101
- const url = `${this.baseUrl.replace(/\/+$/, "")}/api/show`;
101
+ const url = `${trimTrailingSlashes(this.baseUrl)}/api/show`;
102
102
  const controller = new AbortController();
103
103
  const timer = setTimeout(() => controller.abort(), timeoutMs);
104
104
  let body;
@@ -137,6 +137,11 @@ var OllamaEmbedder = class {
137
137
  }
138
138
  };
139
139
  /** `unavailable` for a failed request, naming the timeout when it was the abort. */
140
+ function trimTrailingSlashes(url) {
141
+ let end = url.length;
142
+ while (end > 0 && url.charCodeAt(end - 1) === 47) end--;
143
+ return url.slice(0, end);
144
+ }
140
145
  function transportError(url, controller, timeoutMs, err) {
141
146
  const detail = controller.signal.aborted ? `timed out after ${timeoutMs}ms` : `failed: ${err instanceof Error ? err.message : String(err)}`;
142
147
  return new EmbedError("unavailable", `OllamaEmbedder: POST ${url} ${detail}`, { cause: err });
@@ -378,6 +383,8 @@ const CrystalliseSchema = z.object({
378
383
  const RecallBootstrapConfigSchema = z.object({
379
384
  embedder: EmbedderSchema.optional(),
380
385
  queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
386
+ writeEmbedTimeoutMs: z.number().int().positive().default(3e4),
387
+ embedBackfillIntervalMs: z.number().int().nonnegative().default(6e4),
381
388
  hnswM: z.number().int().positive().default(16),
382
389
  hnswEfConstruction: z.number().int().positive().default(64),
383
390
  salience: SalienceSchema.default({
@@ -426,6 +433,8 @@ const bootstrap = defineBootstrap.kernel({
426
433
  return createRecallPlugin({
427
434
  ...embedder !== null ? { embedder } : {},
428
435
  queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
436
+ writeEmbedTimeoutMs: config.writeEmbedTimeoutMs,
437
+ embedBackfillIntervalMs: config.embedBackfillIntervalMs,
429
438
  ...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
430
439
  store: deps.store,
431
440
  ...deps.stats !== void 0 ? { stats: deps.stats } : {},
@@ -118,6 +118,18 @@
118
118
  "exclusiveMinimum": 0,
119
119
  "maximum": 9007199254740991
120
120
  },
121
+ "writeEmbedTimeoutMs": {
122
+ "default": 30000,
123
+ "type": "integer",
124
+ "exclusiveMinimum": 0,
125
+ "maximum": 9007199254740991
126
+ },
127
+ "embedBackfillIntervalMs": {
128
+ "default": 60000,
129
+ "type": "integer",
130
+ "minimum": 0,
131
+ "maximum": 9007199254740991
132
+ },
121
133
  "hnswM": {
122
134
  "default": 16,
123
135
  "type": "integer",
package/dist/recall.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { E as Row, _ as EdgesStore, n as FieldDef } from "./schema-g4FQLr0V.js";
2
- import { $ as PolicyProbeContext, P as HttpHandler, Q as PolicyDecision, S as PostWriteHook, Y as Identity, ct as Dataset, et as RequestContext, h as McpToolRegistration, nt as ResolverArgs, p as KernelPlugin, r as EmbedSourceContext, s as FilterOperatorDef, u as KernelMigration } from "./plugin-t2g2wX0b.js";
1
+ import { b as EdgesStore, k as Row, r as FieldDef } from "./schema-DoSc-4ig.js";
2
+ import { Dt as PolicyDecision, F as FilterOperatorDef, H as KernelPlugin, Ot as PolicyProbeContext, W as McpToolRegistration, Z as PostWriteHook, j as EmbedSourceContext, jt as ResolverArgs, kt as RequestContext, lt as HttpHandler, n as Dataset, wt as Identity, z as KernelMigration } from "./dataset-CejF-Fr_.js";
3
3
  import { t as Embedder } from "./types-8c0gtnd2.js";
4
4
  import postgres, { Sql } from "postgres";
5
5
  //#region ../../plugins/substrate-recall/src/candidate-selection.d.ts
@@ -150,8 +150,11 @@ interface ClusterStore {
150
150
  includeRetired?: boolean;
151
151
  }): Promise<ClusterRecord[]>;
152
152
  getCluster(id: string): Promise<ClusterRecord | null>;
153
- /** Members of one cluster. Departed members excluded unless asked for. */
154
- listMembers(clusterId: string, opts?: {
153
+ /**
154
+ * Members of one cluster, or of each of several in one read. Departed
155
+ * members excluded unless asked for.
156
+ */
157
+ listMembers(clusterId: string | readonly string[], opts?: {
155
158
  includeDeparted?: boolean;
156
159
  }): Promise<ClusterMemberRecord[]>;
157
160
  /**
@@ -242,6 +245,11 @@ interface EmbeddingsStoreKnnInput {
242
245
  * store shared by several still fills k. Equality family only.
243
246
  */
244
247
  rowFilter?: KnnRowFilter;
248
+ /**
249
+ * Return each row's salience decayed to now by this half-life, the decay
250
+ * `boostSalience` applies. Absent: the stored value, as of its last boost.
251
+ */
252
+ salienceHalfLifeHours?: number;
245
253
  }
246
254
  type KnnScalar = string | number | boolean | null;
247
255
  /** Field -> `{ eq }` or `{ in }`; every field must match. `eq: null` matches a missing or null field. */
@@ -297,6 +305,17 @@ interface EmbeddingsStore {
297
305
  * mismatched vector surfaces as a substrate-level error.
298
306
  */
299
307
  upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
308
+ /**
309
+ * Ids of live rows of (namespace, type) whose vector is missing or older than
310
+ * the row, after `afterId` in id order: the writes whose embed failed. Optional;
311
+ * a store that cannot see the rows table omits it and nothing is backfilled.
312
+ */
313
+ staleRowIds?(input: {
314
+ namespace: string;
315
+ type: string;
316
+ afterId: string;
317
+ limit: number;
318
+ }): Promise<string[]>;
300
319
  /**
301
320
  * Remove the embedding for one row, leaving the row itself untouched. Used
302
321
  * to un-index a row whose `embed.source` returns the null sentinel.
@@ -716,6 +735,14 @@ interface RecallInput {
716
735
  * or null field. A malformed filter is refused with `bad_request`.
717
736
  */
718
737
  where?: KnnRowFilter;
738
+ /**
739
+ * Row-body projection. Field -> `true` keeps those keys plus `id`; `true`
740
+ * keeps the whole body. Absent → `id` plus the handle fields
741
+ * (`HANDLE_FIELDS`) the row has. Applied after row filtering and policy,
742
+ * which still see the full body. A malformed value is refused with
743
+ * `bad_request`.
744
+ */
745
+ rowFields?: Record<string, true> | true;
719
746
  }
720
747
  interface RecallRow {
721
748
  /**
@@ -1451,7 +1478,7 @@ export declare class InMemoryClusterStore implements ClusterStore {
1451
1478
  includeRetired?: boolean;
1452
1479
  }): Promise<ClusterRecord[]>;
1453
1480
  getCluster(id: string): Promise<ClusterRecord | null>;
1454
- listMembers(clusterId: string, opts?: {
1481
+ listMembers(clusterId: string | readonly string[], opts?: {
1455
1482
  includeDeparted?: boolean;
1456
1483
  }): Promise<ClusterMemberRecord[]>;
1457
1484
  listAllMembers(): Promise<ClusterMemberRecord[]>;
@@ -1639,7 +1666,7 @@ export declare class PgClusterStore implements ClusterStore {
1639
1666
  constructor(opts: PgClusterStoreOptions);
1640
1667
  /**
1641
1668
  * Runs `fn` holding a session advisory lock that excludes formation in every
1642
- * process on this database; throws `crystallise_in_flight` if it is held.
1669
+ * process on this schema; throws `crystallise_in_flight` if it is held.
1643
1670
  *
1644
1671
  * Lock and unlock share one reserved connection, since through the pool the
1645
1672
  * unlock can land on another session and fail to release.
@@ -1651,7 +1678,7 @@ export declare class PgClusterStore implements ClusterStore {
1651
1678
  includeRetired?: boolean;
1652
1679
  }): Promise<ClusterRecord[]>;
1653
1680
  getCluster(id: string): Promise<ClusterRecord | null>;
1654
- listMembers(clusterId: string, opts?: {
1681
+ listMembers(clusterId: string | readonly string[], opts?: {
1655
1682
  includeDeparted?: boolean;
1656
1683
  }): Promise<ClusterMemberRecord[]>;
1657
1684
  listAllMembers(): Promise<ClusterMemberRecord[]>;
@@ -1688,6 +1715,12 @@ export declare class PgEmbeddingsStore implements EmbeddingsStore {
1688
1715
  *
1689
1716
  * @returns number of orphaned embeddings deleted.
1690
1717
  */
1718
+ staleRowIds(input: {
1719
+ namespace: string;
1720
+ type: string;
1721
+ afterId: string;
1722
+ limit: number;
1723
+ }): Promise<string[]>;
1691
1724
  purgeOrphans(): Promise<number>;
1692
1725
  }
1693
1726
  //#endregion
@@ -1731,10 +1764,22 @@ export declare class PgLexicalStore implements LexicalStore {
1731
1764
  interface RecallPluginConfig {
1732
1765
  embedder?: Embedder;
1733
1766
  /**
1734
- * Timeout for embedding a recall or `embeddingSimilar` query, in ms. Write
1735
- * and reindex embeds keep the embedder's own timeout. Default 30000.
1767
+ * Timeout for embedding a recall or `embeddingSimilar` query, in ms.
1768
+ * Reindex embeds keep the embedder's own timeout. Default 30000.
1736
1769
  */
1737
1770
  queryEmbedTimeoutMs?: number;
1771
+ /**
1772
+ * Timeout for the embed that follows a write, in ms. The engine awaits
1773
+ * after-commit work before it replies, so this bounds how long an already
1774
+ * committed write can be held by a slow embedder. A timed-out embed keeps
1775
+ * the row's previous vector until its next write or a reindex. Default 30000.
1776
+ */
1777
+ writeEmbedTimeoutMs?: number;
1778
+ /**
1779
+ * How often to re-embed rows whose write-path embed failed or timed out (their
1780
+ * vector is missing or older than the row). 0 disables. Default 60000.
1781
+ */
1782
+ embedBackfillIntervalMs?: number;
1738
1783
  store: EmbeddingsStore;
1739
1784
  /**
1740
1785
  * Store for the lexical arm. Absent → types declaring `lexical.source`