@mnmal-ai/substrate 1.0.0-alpha.2 → 1.0.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +64 -14
  2. package/dist/boot-store-Dz6PGwNY.js +4079 -0
  3. package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-BeVye_Oj.d.ts} +2 -87
  4. package/dist/cli-CH7M6ZTQ.js +3383 -0
  5. package/dist/client.browser.js +2485 -0
  6. package/dist/client.d.ts +273 -135
  7. package/dist/client.js +6 -6
  8. package/dist/cluster-worker.js +0 -1
  9. package/dist/{content-type-C7T28xNb.js → content-type-wLljot5M.js} +0 -1
  10. package/dist/{env-utils-CGNazUNt.js → env-utils-s25Z1YSA.js} +0 -1
  11. package/dist/{errors-D4vKuwz8.d.ts → errors-CoHLKAX0.d.ts} +44 -4
  12. package/dist/{fake-DUVtKVSE.js → fake-DdXtb-m8.js} +0 -1
  13. package/dist/{index-D7mSOFlM.d.ts → index-D7XRAGC6.d.ts} +7 -11
  14. package/dist/msgpackr-Dq18g6Ds.js +16 -0
  15. package/dist/node-version-BtK_Nq6z.js +32 -0
  16. package/dist/pack.d.ts +75 -11
  17. package/dist/pack.js +4 -4
  18. package/dist/pg-edges-store-Dnaxesmy.js +443 -0
  19. package/dist/{plugin-QuJN9pG_.js → plugin-BIz4Pr5-.js} +132 -8
  20. package/dist/proposals-bootstrap.js +6 -5
  21. package/dist/proposals.config.schema.json +5 -0
  22. package/dist/proposals.d.ts +17 -5
  23. package/dist/proposals.default-config.json +2 -1
  24. package/dist/proposals.js +2 -3
  25. package/dist/recall-bootstrap.js +5 -4
  26. package/dist/recall.config.schema.json +6 -0
  27. package/dist/recall.d.ts +35 -6
  28. package/dist/recall.js +2 -3
  29. package/dist/{repl-C52wyfMs.js → repl-BUkd9DPZ.js} +65 -33
  30. package/dist/{row-ByLFE2Wx.d.ts → row-aEWVHEsf.d.ts} +92 -12
  31. package/dist/run-Cu9w5Y10.js +2435 -0
  32. package/dist/{plugin-t2g2wX0b.d.ts → schema-DwnXxdjk.d.ts} +1293 -63
  33. package/dist/script-A9dxMRZA.js +143 -0
  34. package/dist/script-runner.js +77 -0
  35. package/dist/sdk.d.ts +227 -38
  36. package/dist/sdk.js +5 -110
  37. package/dist/semver-D_zdq0hi.js +17 -0
  38. package/dist/{src-DWm0iA88.js → src--accZ5Ka.js} +188 -87
  39. package/dist/{src-DMzc3HfW.js → src-BNdhH_Hx.js} +745 -851
  40. package/dist/{src-CG7giuHr.js → src-BQnDrs_E.js} +793 -75
  41. package/dist/{src-BLgbLq2k.js → src-B_kHkbsr.js} +780 -330
  42. package/dist/src-BbCUeMsI.js +7723 -0
  43. package/dist/{src-CHp3QyqB.js → src-BmJD4ScT.js} +9215 -3376
  44. package/dist/{row-DZ0Fnpc6.js → src-DmiUWOLp.js} +2 -1
  45. package/dist/substrate-admin.js +17 -782
  46. package/dist/substrate.js +4 -5487
  47. package/dist/testing.d.ts +730 -53
  48. package/dist/testing.js +20 -163
  49. package/dist/{ws-C72iVS5T.d.ts → ws-DJaKruX-.d.ts} +4 -5
  50. package/package.json +26 -26
  51. package/dist/dataset-BQ8egGl5.js +0 -1535
  52. package/dist/schema-g4FQLr0V.d.ts +0 -433
  53. package/dist/src-CXKnqeTW.js +0 -2339
@@ -1,5 +1,5 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { L as applyObligations, Q as buildContext, et as jsonResponse, wt as ResolverFrameError } from "./src-CHp3QyqB.js";
1
+ import { Lt as buildContext, tt as applyObligations, xn as ResolverFrameError, zt as jsonResponse } from "./src-BmJD4ScT.js";
2
+ import "./src-B_kHkbsr.js";
3
3
  import { Worker } from "node:worker_threads";
4
4
 
5
5
  //#region ../../plugins/substrate-recall/src/candidate-selection.ts
@@ -949,6 +949,11 @@ async function handleRecallRequest(input, deps) {
949
949
  const problem = rowFilterProblem(where);
950
950
  if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"where" ${problem}`), { code: "bad_request" });
951
951
  }
952
+ const rowFields = input.rowFields;
953
+ if (rowFields !== void 0) {
954
+ const problem = rowFieldsProblem(rowFields);
955
+ if (problem !== null) throw Object.assign(/* @__PURE__ */ new Error(`"rowFields" ${problem}`), { code: "bad_request" });
956
+ }
952
957
  const type = input.type;
953
958
  if (type === void 0 || type === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `type`, the type to search. There is no default."), { code: "bad_request" });
954
959
  if (typeof input.query !== "string" || input.query.trim() === "") throw Object.assign(/* @__PURE__ */ new Error("recall: pass `query`, the text to search for."), { code: "bad_request" });
@@ -1166,7 +1171,7 @@ async function handleRecallRequest(input, deps) {
1166
1171
  }
1167
1172
  const t3 = performance.now();
1168
1173
  return {
1169
- rows: fusedRows,
1174
+ rows: rowFields === true ? fusedRows : fusedRows.map((r) => projectRow(r, rowFields)),
1170
1175
  model: denseRan ? embedder?.modelId ?? "" : "",
1171
1176
  arms: {
1172
1177
  dense: denseRan,
@@ -1212,6 +1217,23 @@ function extractTargets(namespace, type, rows) {
1212
1217
  }
1213
1218
  return out;
1214
1219
  }
1220
+ /** Why `rowFields` is malformed, or `null` when it is `true` or an object of `field: true`. */
1221
+ function rowFieldsProblem(value) {
1222
+ if (value === true) return null;
1223
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return "must be true (the whole body) or an object of field: true, e.g. { \"body\": true }";
1224
+ for (const [field, v] of Object.entries(value)) if (v !== true) return `field "${field}" must be true`;
1225
+ return null;
1226
+ }
1227
+ /** Narrow a hit's `row` to `id` plus `rowFields`, or plus the handle fields when absent. */
1228
+ function projectRow(hit, rowFields) {
1229
+ const row = {};
1230
+ const keep = rowFields === void 0 ? HANDLE_FIELDS : Object.keys(rowFields);
1231
+ for (const field of ["id", ...keep]) if (Object.hasOwn(hit.row, field)) row[field] = hit.row[field];
1232
+ return {
1233
+ ...hit,
1234
+ row
1235
+ };
1236
+ }
1215
1237
 
1216
1238
  //#endregion
1217
1239
  //#region ../../plugins/substrate-recall/src/params.ts
@@ -1265,7 +1287,8 @@ const ROUTE_PARAMS = [
1265
1287
  "returnVector",
1266
1288
  "lexical",
1267
1289
  "explain",
1268
- "where"
1290
+ "where",
1291
+ "rowFields"
1269
1292
  ];
1270
1293
  function parseBody$3(req) {
1271
1294
  const raw = req.body;
@@ -1325,6 +1348,7 @@ function parseBody$3(req) {
1325
1348
  input.explain = obj.explain;
1326
1349
  }
1327
1350
  if (obj.where !== void 0) input.where = obj.where;
1351
+ if (obj.rowFields !== void 0) input.rowFields = obj.rowFields;
1328
1352
  return input;
1329
1353
  }
1330
1354
  /**
@@ -2466,7 +2490,8 @@ const MCP_PARAMS = [
2466
2490
  "returnVector",
2467
2491
  "lexical",
2468
2492
  "explain",
2469
- "where"
2493
+ "where",
2494
+ "rowFields"
2470
2495
  ];
2471
2496
  function recallMcpTool(cfg) {
2472
2497
  return {
@@ -2514,6 +2539,10 @@ function recallMcpTool(cfg) {
2514
2539
  where: {
2515
2540
  type: "object",
2516
2541
  description: "Row-field filter applied inside every candidate source, so k fills with matching rows: field -> { eq: scalar | null } or { in: [scalar | null, ...] }, every field must match. eq null matches a missing or null field. Example: { \"project\": { \"in\": [\"substrate\", null] } }."
2542
+ },
2543
+ rowFields: {
2544
+ type: ["object", "boolean"],
2545
+ description: "Row-body projection. Omitted, each hit's `row` carries `id` plus its handle fields (name, title, topic, synopsis, description), enough to decide on. field -> true keeps just those fields plus `id`, e.g. { \"synopsis\": true, \"body\": true }; true returns the whole body."
2517
2546
  }
2518
2547
  },
2519
2548
  required: ["query", "type"]
@@ -2749,7 +2778,7 @@ async function alignVectorDimensions(s, opts) {
2749
2778
  const { dimension } = opts;
2750
2779
  if (!Number.isInteger(dimension) || dimension <= 0) throw new Error(`alignVectorDimensions: dimension must be a positive integer (got ${dimension})`);
2751
2780
  return await s.begin(async (tx) => {
2752
- await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align'))`;
2781
+ await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align:' || current_schema()))`;
2753
2782
  return alignLocked(tx, opts);
2754
2783
  });
2755
2784
  }
@@ -2776,6 +2805,50 @@ async function alignLocked(s, opts) {
2776
2805
  };
2777
2806
  }
2778
2807
 
2808
+ //#endregion
2809
+ //#region ../../plugins/substrate-recall/src/embed-backfill.ts
2810
+ async function runEmbedBackfill(deps) {
2811
+ const result = {
2812
+ embedded: 0,
2813
+ unindexable: 0,
2814
+ failed: 0
2815
+ };
2816
+ const staleRowIds = deps.store.staleRowIds?.bind(deps.store);
2817
+ if (staleRowIds === void 0) return result;
2818
+ for (const { namespace, type } of deps.registry.types()) {
2819
+ const reembed = deps.registry.get(namespace, type)?.reembed;
2820
+ const dataset = deps.resolveDataset(namespace);
2821
+ if (reembed === void 0 || dataset === null) continue;
2822
+ const key = `${namespace}/${type}`;
2823
+ const ids = await staleRowIds({
2824
+ namespace,
2825
+ type,
2826
+ afterId: deps.cursors.get(key) ?? "",
2827
+ limit: deps.perTypeLimit
2828
+ });
2829
+ if (ids.length < deps.perTypeLimit) deps.cursors.delete(key);
2830
+ else deps.cursors.set(key, ids[ids.length - 1]);
2831
+ if (ids.length === 0) continue;
2832
+ const { rows } = await dataset.read(type, {
2833
+ ids,
2834
+ limit: ids.length
2835
+ });
2836
+ for (const row of rows) {
2837
+ const outcome = await reembed(row);
2838
+ if (outcome.status === "embedded") result.embedded++;
2839
+ else if (outcome.status === "unindexable") result.unindexable++;
2840
+ else if (outcome.status === "failed") {
2841
+ result.failed++;
2842
+ if (outcome.code === "unavailable") return {
2843
+ ...result,
2844
+ stopped: outcome.error
2845
+ };
2846
+ }
2847
+ }
2848
+ }
2849
+ return result;
2850
+ }
2851
+
2779
2852
  //#endregion
2780
2853
  //#region ../../plugins/substrate-recall/src/expansion-resolver.ts
2781
2854
  function makeExpansionResolver(registry, edges, cfg) {
@@ -3035,12 +3108,14 @@ function rethrowCoded(err) {
3035
3108
  throw err;
3036
3109
  }
3037
3110
  /**
3038
- * Keep only the `=== true` keys of `fields`; `undefined` fields = passthrough.
3111
+ * Keep only the selected keys of `fields`; `undefined` fields = passthrough.
3039
3112
  * `fields: {}` strips every key, matching the compose layer's projectEnvelope.
3113
+ * A nested `row: { … }` selection keeps `row`, which the core has already
3114
+ * narrowed through `rowFields`.
3040
3115
  */
3041
3116
  function projectHit(row, fields) {
3042
3117
  if (fields === void 0) return row;
3043
- const keep = Object.entries(fields).filter(([, v]) => v === true).map(([k]) => k);
3118
+ const keep = Object.entries(fields).filter(([k, v]) => v === true || k === "row" && isPlainObject(v)).map(([k]) => k);
3044
3119
  const out = {};
3045
3120
  for (const k of keep) if (k in row) out[k] = row[k];
3046
3121
  return out;
@@ -3063,6 +3138,9 @@ function buildRecallFrameSurface(deps) {
3063
3138
  const salience = deps.salience ?? DEFAULT_SALIENCE_CONFIG;
3064
3139
  const hitResolver = async (ctx, args) => {
3065
3140
  const input = parseHitParams(args.params);
3141
+ const rowSelection = args.fields?.row;
3142
+ if (rowSelection === true) input.rowFields = true;
3143
+ else if (isPlainObject(rowSelection)) input.rowFields = rowSelection;
3066
3144
  const bound = bindPolicy(deps.checkReadPolicy, callerFromContext(ctx));
3067
3145
  const boundWrite = bindPolicy(deps.checkWritePolicy, callerFromContext(ctx));
3068
3146
  let out;
@@ -3144,6 +3222,9 @@ function buildRecallFrameSurface(deps) {
3144
3222
  }
3145
3223
  };
3146
3224
  }
3225
+ function isPlainObject(v) {
3226
+ return v !== null && typeof v === "object" && !Array.isArray(v);
3227
+ }
3147
3228
 
3148
3229
  //#endregion
3149
3230
  //#region ../../plugins/substrate-recall/src/lexical-reindex-core.ts
@@ -3617,6 +3698,9 @@ function makeSpreadResolver(registry, edges) {
3617
3698
  //#endregion
3618
3699
  //#region ../../plugins/substrate-recall/src/plugin.ts
3619
3700
  const DEFAULT_CLUSTER_DIMENSION = 768;
3701
+ const DEFAULT_EMBED_BACKFILL_INTERVAL_MS = 6e4;
3702
+ /** Rows per embedded type per pass, so one pass stays a few seconds of embed work. */
3703
+ const EMBED_BACKFILL_PER_TYPE = 50;
3620
3704
  const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3e4;
3621
3705
  const DEFAULT_WRITE_EMBED_TIMEOUT_MS = 3e4;
3622
3706
  /** `embedder` with `timeoutMs` applied to every call that does not set its own. */
@@ -3694,11 +3778,19 @@ function createRecallPlugin(cfg) {
3694
3778
  ...cfg.checkReadPolicy !== void 0 ? { checkReadPolicy: cfg.checkReadPolicy } : {},
3695
3779
  ...cfg.runClusterer !== void 0 ? { runClusterer: cfg.runClusterer } : {}
3696
3780
  };
3781
+ let backfillTimer;
3697
3782
  return {
3698
3783
  name: "substrate-recall",
3699
3784
  contractVersion: ">=1.0.0-alpha.0 <2.0.0",
3785
+ storeTables: [
3786
+ "embeddings",
3787
+ "lexical",
3788
+ "recall_clusters",
3789
+ "recall_cluster_members"
3790
+ ],
3700
3791
  async onKernelReady(ctx) {
3701
3792
  if (cfg.embedder === void 0 || ctx.sql === null) return;
3793
+ let dimensionMismatch = false;
3702
3794
  const report = await alignVectorDimensions(ctx.sql, {
3703
3795
  dimension: cfg.embedder.dimension,
3704
3796
  ...cfg.migration?.hnswM !== void 0 ? { hnswM: cfg.migration.hnswM } : {},
@@ -3706,6 +3798,7 @@ function createRecallPlugin(cfg) {
3706
3798
  });
3707
3799
  for (const [column, state] of [["embeddings.embedding", report.embeddings], ["recall_clusters.centroid", report.centroids]]) {
3708
3800
  if (state !== "populated") continue;
3801
+ dimensionMismatch = true;
3709
3802
  const found = column.startsWith("embeddings") ? report.found.embeddings : report.found.centroids;
3710
3803
  ctx.metric("recall_dimension_mismatch", {
3711
3804
  column,
@@ -3715,6 +3808,37 @@ function createRecallPlugin(cfg) {
3715
3808
  });
3716
3809
  process.emitWarning(`${column} is vector(${found}) and holds rows, but the embedder is ${cfg.embedder.modelId} at ${cfg.embedder.dimension}. Writes to it will fail until the rows are cleared (they are derived) and re-embedded: delete them, restart, then POST /substrate/v1/embed/reindex.`, { code: "RECALL_DIMENSION_MISMATCH" });
3717
3810
  }
3811
+ const intervalMs = cfg.embedBackfillIntervalMs ?? DEFAULT_EMBED_BACKFILL_INTERVAL_MS;
3812
+ const resolveDataset = cfg.resolveDataset;
3813
+ if (dimensionMismatch || intervalMs === 0 || resolveDataset === void 0) return;
3814
+ if (!ctx.background) return;
3815
+ const cursors = /* @__PURE__ */ new Map();
3816
+ let running = false;
3817
+ clearInterval(backfillTimer);
3818
+ backfillTimer = setInterval(() => {
3819
+ if (running) return;
3820
+ running = true;
3821
+ runEmbedBackfill({
3822
+ registry: embedSourceRegistry,
3823
+ store: cfg.store,
3824
+ resolveDataset,
3825
+ cursors,
3826
+ perTypeLimit: EMBED_BACKFILL_PER_TYPE
3827
+ }).then((r) => {
3828
+ if (r.embedded + r.failed === 0 && r.stopped === void 0) return;
3829
+ ctx.metric("recall_embed_backfill", r);
3830
+ process.stdout.write(`recall.embed_backfill ${JSON.stringify(r)}\n`);
3831
+ }).catch((err) => {
3832
+ process.stderr.write(`recall.embed_backfill failed: ${err instanceof Error ? err.message : String(err)}\n`);
3833
+ }).finally(() => {
3834
+ running = false;
3835
+ });
3836
+ }, intervalMs);
3837
+ backfillTimer.unref();
3838
+ },
3839
+ onKernelStop() {
3840
+ clearInterval(backfillTimer);
3841
+ backfillTimer = void 0;
3718
3842
  },
3719
3843
  migrations: [
3720
3844
  embeddingsMigration(cfg.migration ?? {}),
@@ -1,6 +1,5 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { Pt as defineBootstrap } from "./src-CHp3QyqB.js";
3
- import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-CG7giuHr.js";
1
+ import { Un as defineBootstrap } from "./src-BmJD4ScT.js";
2
+ import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-BQnDrs_E.js";
4
3
  import { z } from "zod";
5
4
 
6
5
  //#region ../../plugins/substrate-proposals/src/bootstrap.ts
@@ -22,14 +21,16 @@ import { z } from "zod";
22
21
  */
23
22
  const ProposalsBootstrapConfigSchema = z.object({
24
23
  ownedNamespaces: z.array(z.string().min(1)).default([]),
25
- proposeCapability: z.string().min(1).default("substrate.propose")
24
+ proposeCapability: z.string().min(1).default("substrate.propose"),
25
+ indexCapability: z.string().min(1).default("substrate.index")
26
26
  }).strict();
27
27
  const bootstrap = defineBootstrap.coordinator({
28
28
  schema: ProposalsBootstrapConfigSchema,
29
29
  build: (config) => createProposalsPlugin({
30
30
  ownedNamespaces: config.ownedNamespaces,
31
31
  pubsub: createNoopPubSub(),
32
- proposeCapability: config.proposeCapability
32
+ proposeCapability: config.proposeCapability,
33
+ indexCapability: config.indexCapability
33
34
  })
34
35
  });
35
36
 
@@ -14,6 +14,11 @@
14
14
  "default": "substrate.propose",
15
15
  "type": "string",
16
16
  "minLength": 1
17
+ },
18
+ "indexCapability": {
19
+ "default": "substrate.index",
20
+ "type": "string",
21
+ "minLength": 1
17
22
  }
18
23
  },
19
24
  "additionalProperties": false
@@ -1,6 +1,5 @@
1
- import { v as OnDelete } from "./schema-g4FQLr0V.js";
2
- import { ct as Dataset, t as CoordinatorPlugin } from "./plugin-t2g2wX0b.js";
3
- import { a as ResolverFrameError } from "./errors-D4vKuwz8.js";
1
+ import { kn as OnDelete, kt as Dataset, x as CoordinatorPlugin } from "./schema-DwnXxdjk.js";
2
+ import { u as ResolverFrameError } from "./errors-CoHLKAX0.js";
4
3
  //#region ../../plugins/substrate-proposals/src/config.d.ts
5
4
  /**
6
5
  * Multi-instance invalidation transport used by the plugin to broadcast
@@ -24,6 +23,8 @@ interface PubSub {
24
23
  notify(channel: string, payload: string): Promise<void>;
25
24
  }
26
25
  interface ProposalsPluginConfig {
26
+ /** Where a one-line warning goes, such as a stored proposal skipped at replay. Defaults to `console.warn`. */
27
+ log?: (line: string) => void;
27
28
  /**
28
29
  * Namespaces this plugin owns. Propose mutations outside this list are
29
30
  * rejected at resolver entry. A proposed crossRef must target a type in an
@@ -41,6 +42,13 @@ interface ProposalsPluginConfig {
41
42
  * propose authority under a differently-named cap.
42
43
  */
43
44
  proposeCapability?: string;
45
+ /**
46
+ * Capability name required to invoke `propose_index`, in the namespace whose
47
+ * indexes it changes. Defaults to `'substrate.index'`: separate from the
48
+ * propose capability, so only the operator or a designated reviewer changes
49
+ * indexes.
50
+ */
51
+ indexCapability?: string;
44
52
  /**
45
53
  * Lookup function the `propose_namespace` resolver uses to find the dataset
46
54
  * for a hot-added namespace. The host (typically the substrate binary) wires
@@ -57,8 +65,12 @@ interface ProposalsPluginConfig {
57
65
  */
58
66
  export declare function createNoopPubSub(): PubSub;
59
67
  //#endregion
68
+ //#region ../../plugins/substrate-proposals/src/engine-write.d.ts
69
+ /** Resolves once every storage preparation started so far has settled. */
70
+ export declare function storageSettled(): Promise<void>;
71
+ //#endregion
60
72
  //#region ../../plugins/substrate-proposals/src/errors.d.ts
61
- type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired';
73
+ type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired' | 'index_on_code_defined_type' | 'unknown_index' | 'not_engine_owned' | 'unique_index' | 'still_desired' | 'unknown_recommendation' | 'recommendation_closed' | 'recommendation_stale' | 'propose_index_not_alone' | 'index_drop_timeout';
62
74
  /**
63
75
  * Structured error for proposal operations with stable error codes.
64
76
  *
@@ -80,7 +92,7 @@ export declare class ProposalError extends ResolverFrameError {
80
92
  }
81
93
  //#endregion
82
94
  //#region ../../plugins/substrate-proposals/src/field-spec-json.d.ts
83
- type ScalarType = 'string' | 'number' | 'boolean' | 'date' | 'json';
95
+ type ScalarType = 'string' | 'number' | 'int16' | 'int32' | 'int64' | 'boolean' | 'date' | 'json';
84
96
  interface ScalarSpec {
85
97
  kind: 'scalar';
86
98
  type: ScalarType;
@@ -1,4 +1,5 @@
1
1
  {
2
2
  "ownedNamespaces": [],
3
- "proposeCapability": "substrate.propose"
3
+ "proposeCapability": "substrate.propose",
4
+ "indexCapability": "substrate.index"
4
5
  }
package/dist/proposals.js CHANGED
@@ -1,4 +1,3 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { n as createNoopPubSub, r as ProposalError, t as createProposalsPlugin } from "./src-CG7giuHr.js";
1
+ import { i as ProposalError, n as createNoopPubSub, r as storageSettled, t as createProposalsPlugin } from "./src-BQnDrs_E.js";
3
2
 
4
- export { ProposalError, createNoopPubSub, createProposalsPlugin };
3
+ export { ProposalError, createNoopPubSub, createProposalsPlugin, storageSettled };
@@ -1,7 +1,6 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { Pt as defineBootstrap } from "./src-CHp3QyqB.js";
3
- import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
4
- import { t as createRecallPlugin } from "./plugin-QuJN9pG_.js";
1
+ import { Un as defineBootstrap } from "./src-BmJD4ScT.js";
2
+ import { t as FakeEmbedder } from "./fake-DdXtb-m8.js";
3
+ import { t as createRecallPlugin } from "./plugin-BIz4Pr5-.js";
5
4
  import { z } from "zod";
6
5
 
7
6
  //#region ../../packages/substrate-embedder/src/types.ts
@@ -384,6 +383,7 @@ const RecallBootstrapConfigSchema = z.object({
384
383
  embedder: EmbedderSchema.optional(),
385
384
  queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
386
385
  writeEmbedTimeoutMs: z.number().int().positive().default(3e4),
386
+ embedBackfillIntervalMs: z.number().int().nonnegative().default(6e4),
387
387
  hnswM: z.number().int().positive().default(16),
388
388
  hnswEfConstruction: z.number().int().positive().default(64),
389
389
  salience: SalienceSchema.default({
@@ -433,6 +433,7 @@ const bootstrap = defineBootstrap.kernel({
433
433
  ...embedder !== null ? { embedder } : {},
434
434
  queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
435
435
  writeEmbedTimeoutMs: config.writeEmbedTimeoutMs,
436
+ embedBackfillIntervalMs: config.embedBackfillIntervalMs,
436
437
  ...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
437
438
  store: deps.store,
438
439
  ...deps.stats !== void 0 ? { stats: deps.stats } : {},
@@ -124,6 +124,12 @@
124
124
  "exclusiveMinimum": 0,
125
125
  "maximum": 9007199254740991
126
126
  },
127
+ "embedBackfillIntervalMs": {
128
+ "default": 60000,
129
+ "type": "integer",
130
+ "minimum": 0,
131
+ "maximum": 9007199254740991
132
+ },
127
133
  "hnswM": {
128
134
  "default": 16,
129
135
  "type": "integer",
package/dist/recall.d.ts CHANGED
@@ -1,5 +1,4 @@
1
- import { E as Row, _ as EdgesStore, n as FieldDef } from "./schema-g4FQLr0V.js";
2
- import { $ as PolicyProbeContext, P as HttpHandler, Q as PolicyDecision, S as PostWriteHook, Y as Identity, ct as Dataset, et as RequestContext, h as McpToolRegistration, nt as ResolverArgs, p as KernelPlugin, r as EmbedSourceContext, s as FilterOperatorDef, u as KernelMigration } from "./plugin-t2g2wX0b.js";
1
+ import { C as EmbedSourceContext, D as FilterOperatorDef, I as McpToolRegistration, Ln as Row, On as EdgesStore, P as KernelPlugin, U as PostWriteHook, _t as PolicyDecision, i as FieldDef, j as KernelMigration, kt as Dataset, mt as Identity, tt as HttpHandler, vt as PolicyProbeContext, xt as ResolverArgs, yt as RequestContext } from "./schema-DwnXxdjk.js";
3
2
  import { t as Embedder } from "./types-8c0gtnd2.js";
4
3
  import postgres, { Sql } from "postgres";
5
4
  //#region ../../plugins/substrate-recall/src/candidate-selection.d.ts
@@ -305,6 +304,17 @@ interface EmbeddingsStore {
305
304
  * mismatched vector surfaces as a substrate-level error.
306
305
  */
307
306
  upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
307
+ /**
308
+ * Ids of live rows of (namespace, type) whose vector is missing or older than
309
+ * the row, after `afterId` in id order: the writes whose embed failed. Optional;
310
+ * a store that cannot see the rows table omits it and nothing is backfilled.
311
+ */
312
+ staleRowIds?(input: {
313
+ namespace: string;
314
+ type: string;
315
+ afterId: string;
316
+ limit: number;
317
+ }): Promise<string[]>;
308
318
  /**
309
319
  * Remove the embedding for one row, leaving the row itself untouched. Used
310
320
  * to un-index a row whose `embed.source` returns the null sentinel.
@@ -724,6 +734,14 @@ interface RecallInput {
724
734
  * or null field. A malformed filter is refused with `bad_request`.
725
735
  */
726
736
  where?: KnnRowFilter;
737
+ /**
738
+ * Row-body projection. Field -> `true` keeps those keys plus `id`; `true`
739
+ * keeps the whole body. Absent → `id` plus the handle fields
740
+ * (`HANDLE_FIELDS`) the row has. Applied after row filtering and policy,
741
+ * which still see the full body. A malformed value is refused with
742
+ * `bad_request`.
743
+ */
744
+ rowFields?: Record<string, true> | true;
727
745
  }
728
746
  interface RecallRow {
729
747
  /**
@@ -1498,7 +1516,7 @@ export declare class InMemoryEmbeddingsStore implements EmbeddingsStore {
1498
1516
  //#endregion
1499
1517
  //#region ../../plugins/substrate-recall/src/in-memory-lexical-store.d.ts
1500
1518
  /**
1501
- * Resolves a row body, mirroring pg's join against `rows`.
1519
+ * Resolves a row body, mirroring pg's join against the type's table.
1502
1520
  *
1503
1521
  * Signature matches `InMemoryEmbeddingsStoreOptions.datasetFor` deliberately:
1504
1522
  * a host wires both from the same registry lookup.
@@ -1647,7 +1665,7 @@ export declare class PgClusterStore implements ClusterStore {
1647
1665
  constructor(opts: PgClusterStoreOptions);
1648
1666
  /**
1649
1667
  * Runs `fn` holding a session advisory lock that excludes formation in every
1650
- * process on this database; throws `crystallise_in_flight` if it is held.
1668
+ * process on this schema; throws `crystallise_in_flight` if it is held.
1651
1669
  *
1652
1670
  * Lock and unlock share one reserved connection, since through the pool the
1653
1671
  * unlock can land on another session and fail to release.
@@ -1688,9 +1706,15 @@ export declare class PgEmbeddingsStore implements EmbeddingsStore {
1688
1706
  id: string;
1689
1707
  }>, model: string): Promise<ReadonlyArray<EmbeddingsStoreFetchResult>>;
1690
1708
  selectCandidates(selector: CandidateSelector, model: string): Promise<Candidate[]>;
1709
+ staleRowIds(input: {
1710
+ namespace: string;
1711
+ type: string;
1712
+ afterId: string;
1713
+ limit: number;
1714
+ }): Promise<string[]>;
1691
1715
  /**
1692
1716
  * Deletes every embedding whose backing row is missing or soft-deleted.
1693
- * Needed because `embeddings` has no FK to `rows` and no
1717
+ * Needed because `embeddings` has no FK to the type tables and no
1694
1718
  * dataset delete path touches it; tombstones are terminal, so soft-deleted
1695
1719
  * rows are safe to collect immediately. Run as a sweep stage by the binary.
1696
1720
  *
@@ -1716,7 +1740,7 @@ export declare class PgLexicalStore implements LexicalStore {
1716
1740
  /**
1717
1741
  * Deletes every lexical row whose backing row is missing or soft-deleted,
1718
1742
  * as `PgEmbeddingsStore.purgeOrphans` does for vectors: `lexical` has no FK
1719
- * to `rows`, so a drop, purge or tombstone would otherwise leave the text.
1743
+ * to the type tables, so a drop, purge or tombstone would otherwise leave the text.
1720
1744
  *
1721
1745
  * @returns number of orphaned lexical rows deleted.
1722
1746
  */
@@ -1750,6 +1774,11 @@ interface RecallPluginConfig {
1750
1774
  * the row's previous vector until its next write or a reindex. Default 30000.
1751
1775
  */
1752
1776
  writeEmbedTimeoutMs?: number;
1777
+ /**
1778
+ * How often to re-embed rows whose write-path embed failed or timed out (their
1779
+ * vector is missing or older than the row). 0 disables. Default 60000.
1780
+ */
1781
+ embedBackfillIntervalMs?: number;
1753
1782
  store: EmbeddingsStore;
1754
1783
  /**
1755
1784
  * Store for the lexical arm. Absent → types declaring `lexical.source`
package/dist/recall.js CHANGED
@@ -1,5 +1,4 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src-DWm0iA88.js";
3
- import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-QuJN9pG_.js";
1
+ import { a as InMemoryLexicalStore, i as PgClusterStore, n as PgLexicalStore, o as InMemoryEmbeddingsStore, r as PgEmbeddingsStore, s as InMemoryClusterStore, t as RECALL_VERSION } from "./src--accZ5Ka.js";
2
+ import { $ as CLUSTERER_BACKEND_VERSION, A as CLUSTER_MEMBER_FIELDS, C as EmbedSourceRegistry, D as makeReembed, E as makeEmbedPostWrite, F as unknownParams, G as DEFAULT_CRYSTALLISE_CONFIG, H as CRYSTALLISE_IN_FLIGHT_CODE, I as handleRecallRequest, J as runClusterer, K as handleCrystallise, L as RRF_K, M as recallRouteHandler, N as checkParams, O as crystalliseRouteHandler, P as unknownParamMessage, Q as CLUSTERER_BACKEND, R as armRanks, S as createEmbedStats, T as embedRowReporting, U as createCrystalliseGuard, X as l2normalize, _ as EMBEDDINGS_FETCH_MAX_REFS, a as REINDEX_MAX_LIMIT, b as handleLexicalCoverage, c as SALIENCE_BOOST_MAX_TARGETS, d as embeddingsMigration, et as memberKey, f as recallMcpTool, g as embeddingsFetchRouteHandler, h as embeddingSimilarOperator, i as REINDEX_DEFAULT_LIMIT, j as buildClusterSurface, k as CLUSTER_FIELDS, l as lexicalMigration, m as makeLexicalPostWrite, n as salienceBoostRouteHandler, nt as validateSelector, o as handleReindex, p as makeLexicalClassify, q as validateCrystalliseInput, r as reindexRouteHandler, s as validateReindexInput, t as createRecallPlugin, tt as SELECT_CANDIDATES_MAX_LIMIT, u as recallClustersMigration, v as embedStatsRouteHandler, w as embedAndStore, x as validateLexicalCoverageInput, y as handleEmbedStatsRequest, z as fuseByRank } from "./plugin-BIz4Pr5-.js";
4
3
 
5
4
  export { CLUSTERER_BACKEND, CLUSTERER_BACKEND_VERSION, CLUSTER_FIELDS, CLUSTER_MEMBER_FIELDS, CRYSTALLISE_IN_FLIGHT_CODE, DEFAULT_CRYSTALLISE_CONFIG, EMBEDDINGS_FETCH_MAX_REFS, EmbedSourceRegistry, InMemoryClusterStore, InMemoryEmbeddingsStore, InMemoryLexicalStore, PgClusterStore, PgEmbeddingsStore, PgLexicalStore, RECALL_VERSION, REINDEX_DEFAULT_LIMIT, REINDEX_MAX_LIMIT, RRF_K, SALIENCE_BOOST_MAX_TARGETS, SELECT_CANDIDATES_MAX_LIMIT, armRanks, buildClusterSurface, checkParams, createCrystalliseGuard, createEmbedStats, createRecallPlugin, crystalliseRouteHandler, embedAndStore, embedRowReporting, embedStatsRouteHandler, embeddingSimilarOperator, embeddingsFetchRouteHandler, embeddingsMigration, fuseByRank, handleCrystallise, handleEmbedStatsRequest, handleLexicalCoverage, handleRecallRequest, handleReindex, l2normalize, lexicalMigration, makeEmbedPostWrite, makeLexicalClassify, makeLexicalPostWrite, makeReembed, memberKey, recallClustersMigration, recallMcpTool, recallRouteHandler, reindexRouteHandler, runClusterer, salienceBoostRouteHandler, unknownParamMessage, unknownParams, validateCrystalliseInput, validateLexicalCoverageInput, validateReindexInput, validateSelector };