@mnmal-ai/substrate 1.0.0-alpha.8 → 1.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +32 -14
  2. package/dist/boot-store-BMQUbHKv.js +4086 -0
  3. package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-DnToJedN.d.ts} +2 -87
  4. package/dist/cli-DModIbnN.js +3390 -0
  5. package/dist/client.browser.js +2507 -0
  6. package/dist/client.d.ts +222 -171
  7. package/dist/client.js +6 -7
  8. package/dist/cluster-worker.js +0 -1
  9. package/dist/{content-type-C7T28xNb.js → content-type-wLljot5M.js} +0 -1
  10. package/dist/{env-utils-CGNazUNt.js → env-utils-s25Z1YSA.js} +0 -1
  11. package/dist/{errors-D4vKuwz8.d.ts → errors-Cmkp1qxf.d.ts} +44 -4
  12. package/dist/{fake-DUVtKVSE.js → fake-DdXtb-m8.js} +0 -1
  13. package/dist/{index-D7mSOFlM.d.ts → index-BI4KqFiR.d.ts} +7 -11
  14. package/dist/msgpackr-Dq18g6Ds.js +16 -0
  15. package/dist/node-version-BtK_Nq6z.js +32 -0
  16. package/dist/pack.d.ts +75 -11
  17. package/dist/pack.js +4 -4
  18. package/dist/{plugin-B3NxUCVn.js → plugin-CEOhrEme.js} +149 -6
  19. package/dist/proposals-bootstrap.js +6 -5
  20. package/dist/proposals.config.schema.json +5 -0
  21. package/dist/proposals.d.ts +17 -5
  22. package/dist/proposals.default-config.json +2 -1
  23. package/dist/proposals.js +2 -3
  24. package/dist/recall-bootstrap.js +5 -4
  25. package/dist/recall.config.schema.json +6 -0
  26. package/dist/recall.d.ts +27 -6
  27. package/dist/recall.js +2 -3
  28. package/dist/{repl-DeKzfvNV.js → repl-BvDZy-Ut.js} +52 -26
  29. package/dist/{row-ByLFE2Wx.d.ts → row-D0If4qif.d.ts} +92 -12
  30. package/dist/run-BT123RV1.js +2443 -0
  31. package/dist/{plugin-t2g2wX0b.d.ts → schema-aIkY8sR3.d.ts} +1293 -63
  32. package/dist/{script-C6DSa_Na.js → script-ByQ9pzhC.js} +54 -26
  33. package/dist/script-runner.js +78 -0
  34. package/dist/sdk.d.ts +236 -38
  35. package/dist/sdk.js +5 -110
  36. package/dist/semver-D_zdq0hi.js +17 -0
  37. package/dist/src-BMD2OE--.js +8289 -0
  38. package/dist/{src-BLgbLq2k.js → src-B_kHkbsr.js} +780 -330
  39. package/dist/{src-DjIuRjzF.js → src-CKSJ13p8.js} +10231 -4237
  40. package/dist/{src-CkZw6xVD.js → src-CW7eRDss.js} +188 -87
  41. package/dist/{row-DZ0Fnpc6.js → src-DmiUWOLp.js} +2 -1
  42. package/dist/{src-DCy7Iy2l.js → src-DmjwFMu8.js} +900 -77
  43. package/dist/{src-DiNTSaFk.js → src-FXO5JPyk.js} +686 -856
  44. package/dist/substrate-admin.js +17 -782
  45. package/dist/substrate.js +4 -5206
  46. package/dist/testing.d.ts +473 -96
  47. package/dist/testing.js +19 -165
  48. package/dist/{ws-C72iVS5T.d.ts → ws-DW8ng02H.d.ts} +4 -5
  49. package/package.json +26 -26
  50. package/dist/dataset-jZS_4ci4.js +0 -1535
  51. package/dist/pg-edges-store-KH_FQCxx.js +0 -381
  52. package/dist/schema-g4FQLr0V.d.ts +0 -433
  53. package/dist/src-BD27zeGF.js +0 -2339
@@ -1,5 +1,5 @@
1
- import { l as PackEnvelope, n as FieldDef, s as FieldInput, u as Schema } from "./schema-g4FQLr0V.js";
2
- import { pack as packFrame, unpack as unpackFrame } from "msgpackr";
1
+ import { f as PackEnvelope, i as FieldDef, l as FieldInput, m as Schema } from "./schema-aIkY8sR3.js";
2
+ import * as msgpackr from "msgpackr";
3
3
  //#region ../../packages/substrate-pack/src/decoder.d.ts
4
4
  interface DecodeContext {
5
5
  schemas?: Record<string, Schema>;
@@ -12,7 +12,7 @@ interface DecodeEnvelopeCtx {
12
12
  * A key listed here is decoded using `decodePositionalFromFieldDef` instead
13
13
  * of the client-sent FieldInput.
14
14
  */
15
- returnsByKey?: Record<string, FieldDef>;
15
+ returnsByKey?: Record<string, FieldDef | readonly FieldDef[]>;
16
16
  typeByKey: Record<string, string>;
17
17
  schemas: Record<string, Schema>;
18
18
  currentNsByKey: Record<string, string>;
@@ -58,12 +58,8 @@ declare function decodePositional<T = unknown>(encoded: unknown, fields: FieldIn
58
58
  declare function decodePositionalFromFieldDef<T = unknown>(encoded: unknown, def: FieldDef, schema: Schema, ctx: DecodeContext, path?: string[]): T;
59
59
  declare function decodeEnvelope(bytes: Buffer | Uint8Array, ctx: DecodeEnvelopeCtx): PackEnvelope;
60
60
  //#endregion
61
- //#region ../../packages/substrate-pack/src/index.d.ts
62
- /**
63
- * WebSocket subprotocol advertised in `Sec-WebSocket-Protocol` at handshake
64
- * time. Connections negotiating this subprotocol receive msgpack frames with
65
- * positionally-encoded `data`; anything else falls back to JSON.
66
- */
67
- declare const WS_SUBPROTOCOL_SUBSTRATE_PACK = "substrate-pack";
61
+ //#region ../../packages/substrate-pack/src/msgpackr.d.ts
62
+ declare const pack: (...args: Parameters<typeof msgpackr.pack>) => ReturnType<typeof msgpackr.pack>;
63
+ declare const unpack: (...args: Parameters<typeof msgpackr.unpack>) => ReturnType<typeof msgpackr.unpack>;
68
64
  //#endregion
69
- export { DecodeEnvelopeCtx as a, decodeEnvelope as c, DecodeContext as i, decodePositional as l, packFrame as n, PackDecodeError as o, unpackFrame as r, SchemaDriftError as s, WS_SUBPROTOCOL_SUBSTRATE_PACK as t, decodePositionalFromFieldDef as u };
65
+ export { PackDecodeError as a, decodePositional as c, DecodeEnvelopeCtx as i, decodePositionalFromFieldDef as l, unpack as n, SchemaDriftError as o, DecodeContext as r, decodeEnvelope as s, pack as t };
@@ -0,0 +1,16 @@
1
+ import * as msgpackr from "msgpackr";
2
+
3
+ //#region ../../packages/substrate-pack/src/msgpackr.ts
4
+ /**
5
+ * The one module that imports msgpackr. The published bundle gives it a chunk
6
+ * of its own, so a consumer that never packs or unpacks drops msgpackr, which
7
+ * does not declare itself free of load-time side effects. These are wrappers,
8
+ * not re-exports: a bundler resolves a re-export straight to msgpackr, which
9
+ * would leave every importer of this module importing msgpackr itself.
10
+ */
11
+ const pack = (...args) => msgpackr.pack(...args);
12
+ const unpack = (...args) => msgpackr.unpack(...args);
13
+ const createUnpackr = (options) => new msgpackr.Unpackr(options);
14
+
15
+ //#endregion
16
+ export { pack as n, unpack as r, createUnpackr as t };
@@ -0,0 +1,32 @@
1
+ //#region src/node-version.ts
2
+ /** The oldest Node the binaries run on; matches `engines.node` in package.json. */
3
+ const MIN_NODE_VERSION = "26.13.1";
4
+ /** Returns a refusal message when `version` (e.g. `process.versions.node`) is below the minimum, else null. */
5
+ function nodeVersionRefusal(version) {
6
+ const have = version.split(".").map(Number);
7
+ const want = MIN_NODE_VERSION.split(".").map(Number);
8
+ for (let i = 0; i < 3; i++) {
9
+ const a = have[i] ?? 0;
10
+ const b = want[i] ?? 0;
11
+ if (a > b) return null;
12
+ if (a < b) return `substrate requires Node ${MIN_NODE_VERSION} or newer; this is Node ${version}`;
13
+ }
14
+ return null;
15
+ }
16
+ /**
17
+ * The refusal for this runtime, or null. Bun is exempt: it reports a Node
18
+ * compatibility version (24.x) that says nothing about its own security fixes.
19
+ */
20
+ function runtimeRefusal(versions) {
21
+ return versions.bun !== void 0 ? null : nodeVersionRefusal(versions.node);
22
+ }
23
+ /** Exits the process with a refusal when the running Node is below the minimum. */
24
+ function assertNodeVersion() {
25
+ const refusal = runtimeRefusal(process.versions);
26
+ if (refusal === null) return;
27
+ process.stderr.write(`${refusal}\n`);
28
+ process.exit(1);
29
+ }
30
+
31
+ //#endregion
32
+ export { assertNodeVersion as t };
package/dist/pack.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { l as PackEnvelope, n as FieldDef, s as FieldInput, u as Schema } from "./schema-g4FQLr0V.js";
2
- import { a as DecodeEnvelopeCtx, c as decodeEnvelope, i as DecodeContext, l as decodePositional, n as packFrame, o as PackDecodeError, r as unpackFrame, s as SchemaDriftError, t as WS_SUBPROTOCOL_SUBSTRATE_PACK, u as decodePositionalFromFieldDef } from "./index-D7mSOFlM.js";
1
+ import { f as PackEnvelope, i as FieldDef, l as FieldInput, m as Schema, u as MutationSchema } from "./schema-aIkY8sR3.js";
2
+ import { a as PackDecodeError, c as decodePositional, i as DecodeEnvelopeCtx, l as decodePositionalFromFieldDef, n as unpack, o as SchemaDriftError, r as DecodeContext, s as decodeEnvelope, t as pack } from "./index-BI4KqFiR.js";
3
3
  //#region ../../packages/substrate-pack/src/content-type.d.ts
4
4
  /**
5
5
  * Content-type negotiation. Spec: §7.0.
@@ -26,9 +26,10 @@ interface EncodeEnvelopeCtx {
26
26
  /**
27
27
  * Mutation-path return-shape map — keyed by envelope key. Mutation responses
28
28
  * don't carry a client-sent FieldInput; their shape is declared at registry
29
- * time as `MutationSchema.returns` and wired through here.
29
+ * time as `MutationSchema.returns` and wired through here. An array gives a
30
+ * batched key one shape per frame.
30
31
  */
31
- returnsByKey?: Record<string, FieldDef>;
32
+ returnsByKey?: Record<string, FieldDef | readonly FieldDef[]>;
32
33
  /** Query-path type-name map — keyed by envelope key. */
33
34
  typeByKey: Record<string, string>;
34
35
  schemas: Record<string, Schema>;
@@ -61,6 +62,31 @@ export declare function encodePositional(value: unknown, fields: FieldInput, sch
61
62
  */
62
63
  export declare function encodePositionalFromFieldDef(value: unknown, def: FieldDef, schema: Schema, ctx: EncodeContext, path?: string[]): unknown;
63
64
  export declare function encodeEnvelope(envelope: PackEnvelope, ctx: EncodeEnvelopeCtx): Buffer;
65
+ /**
66
+ * The wire form of `envelope.data[key]`, or of `value` in its place: rows as
67
+ * positional tuples, or the value as-is when nothing positions it. A `list`
68
+ * value's rows encode one by one, so a slice of its rows gives that slice of
69
+ * the whole value's wire form.
70
+ */
71
+ export declare function positionValue(envelope: PackEnvelope, ctx: EncodeEnvelopeCtx, key: string, value?: unknown): unknown;
72
+ /** `envelope` with `data` replaced by its wire form; `pack` of it is the response body. */
73
+ export declare function withData(envelope: PackEnvelope, data: Record<string, unknown>): PackEnvelope;
74
+ /**
75
+ * `pack(wire)` in pieces, a long row list `rowsPerPart` rows per piece, for a
76
+ * `wire` envelope from `withData`. msgpackr's default `pack` writes every
77
+ * object as a map16 (`de`) and records no shared state between values, so
78
+ * the pieces concatenate to the same bytes.
79
+ */
80
+ export declare function packEnvelopeParts(wire: PackEnvelope, rowsPerPart: number): Generator<Uint8Array>;
81
+ //#endregion
82
+ //#region ../../packages/substrate-pack/src/mutation-returns.d.ts
83
+ /** What a row-writing mutation answers when the frame names no `fields`. */
84
+ export declare const MUTATION_ACK_FIELDS: {
85
+ readonly id: true;
86
+ readonly shortId: true;
87
+ };
88
+ /** The `returns` a mutation's result is positioned by, given the frame's `fields` (`true` selects everything). */
89
+ export declare function mutationReturns(mutation: Pick<MutationSchema, 'returns' | 'writesRows'>, fields: FieldInput | true | undefined): FieldDef;
64
90
  //#endregion
65
91
  //#region ../../packages/substrate-pack/src/stream-chunk.d.ts
66
92
  /**
@@ -119,14 +145,13 @@ export declare function decodeStreamChunk<T = unknown>(bytes: Buffer | Uint8Arra
119
145
  //#region ../../packages/substrate-pack/src/wire.d.ts
120
146
  type WireMode = 'json' | 'pack';
121
147
  /**
122
- * Encode an outbound WS frame. Pack mode returns msgpack bytes; JSON mode
123
- * returns stringified text. The `mode` argument is the wire-format discriminant
124
- * — call sites can route the result to the right `WebSocket.send` overload via
125
- * a typed `typeof wire === 'string'` branch.
148
+ * Encode an outbound frame. Pack mode returns msgpack bytes; JSON mode
149
+ * returns stringified text. The `mode` argument is the wire-format discriminant;
150
+ * callers branch on `typeof wire === 'string'`.
126
151
  */
127
152
  export declare function encodeWsFrame(value: unknown, mode: WireMode): string | Uint8Array;
128
153
  /**
129
- * Decode an inbound WS message envelope (the outer `{ id, kind, data?, ... }`
154
+ * Decode an inbound frame envelope (the outer `{ id, kind, data?, ... }`
130
155
  * shape — positional `data` reconstruction is a separate concern handled by
131
156
  * `applyDecodePlanInPlace`).
132
157
  *
@@ -147,7 +172,7 @@ export declare function decodeWsEnvelope<T = unknown>(raw: unknown, mode: WireMo
147
172
  */
148
173
  export declare function isPlainRequestData(value: unknown, depth?: number): boolean;
149
174
  /**
150
- * A single precomputed entry in a WS pack-decode plan. All per-key lookups
175
+ * A single precomputed entry in a pack-decode plan. All per-key lookups
151
176
  * (parseQualifiedKey, schema probe, fields narrow) happen once at plan-build
152
177
  * time; per-frame decoding then iterates the plan directly.
153
178
  *
@@ -189,4 +214,43 @@ export declare function buildDecodePlan(frameSpec: Record<string, FrameSpecEntry
189
214
  */
190
215
  export declare function applyDecodePlanInPlace(envelope: unknown, plan: DecodePlanEntry[]): void;
191
216
  //#endregion
192
- export { type DecodeContext, type DecodeEnvelopeCtx, type DecodePlanEntry, type DecodeStreamChunkInput, type EncodeContext, type EncodeEnvelopeCtx, type EncodeStreamChunkInput, type FrameSpecEntry, PackDecodeError, type PackStreamChunk, SchemaDriftError, WS_SUBPROTOCOL_SUBSTRATE_PACK, type WireMode, decodeEnvelope, decodePositional, decodePositionalFromFieldDef, packFrame, unpackFrame };
217
+ //#region ../../packages/substrate-pack/src/stream-frames.d.ts
218
+ export declare const FRAMES_JSON_MIME = "application/vnd.substrate.frames+json";
219
+ export declare const FRAMES_PACK_MIME = "application/vnd.substrate.frames+pack";
220
+ /** Largest payload a decoder accepts unless told otherwise. */
221
+ export declare const DEFAULT_MAX_FRAME_BYTES: number;
222
+ /** A frame of length 0: keeps an idle stream alive, and decoders skip it. */
223
+ export declare const HEARTBEAT_FRAME: Uint8Array;
224
+ type FrameStreamErrorCode = 'frame_too_large' | 'truncated_frame' | 'stream_idle';
225
+ export declare class FrameStreamError extends Error {
226
+ readonly code: FrameStreamErrorCode;
227
+ constructor(code: FrameStreamErrorCode, message: string);
228
+ }
229
+ /** A little over three of the server's 20 s heartbeats. */
230
+ export declare const DEFAULT_STREAM_IDLE_MS = 65000;
231
+ /**
232
+ * `body`, failing with `stream_idle` when a read waits `ms` for bytes,
233
+ * heartbeats included. It times only reads the consumer is waiting on, so a
234
+ * slow consumer never trips it; a connection that died without closing does.
235
+ * The underlying reader is cancelled on timeout, which settles its pending read.
236
+ */
237
+ export declare function withIdleTimeout(body: ReadableStream<Uint8Array>, ms: number): ReadableStream<Uint8Array>;
238
+ /** The frames encoding an `Accept` header asks for, or undefined when it asks for none. */
239
+ export declare function framesEncodingFromAccept(accept: string | undefined): WireMode | undefined;
240
+ /**
241
+ * One frame: u32 BE length, then `value` as JSON or msgpack.
242
+ *
243
+ * @throws FrameStreamError `frame_too_large` when the payload is over `maxFrameBytes`,
244
+ * which a decoder with the same limit would refuse
245
+ */
246
+ export declare function encodeStreamFrame(value: unknown, mode: WireMode, maxFrameBytes?: number): Uint8Array;
247
+ /**
248
+ * The payloads of a frame stream, in order, heartbeats skipped. A length above
249
+ * `maxFrameBytes` throws `frame_too_large` before any of that payload is read
250
+ * or allocated; a body that ends inside a frame throws `truncated_frame`.
251
+ */
252
+ export declare function decodeStreamFrames(body: AsyncIterable<Uint8Array> | ReadableStream<Uint8Array>, opts?: {
253
+ maxFrameBytes?: number;
254
+ }): AsyncGenerator<Uint8Array>;
255
+ //#endregion
256
+ export { type DecodeContext, type DecodeEnvelopeCtx, type DecodePlanEntry, type DecodeStreamChunkInput, type EncodeContext, type EncodeEnvelopeCtx, type EncodeStreamChunkInput, type FrameSpecEntry, type FrameStreamErrorCode, PackDecodeError, type PackStreamChunk, SchemaDriftError, type WireMode, decodeEnvelope, decodePositional, decodePositionalFromFieldDef, pack as packFrame, unpack as unpackFrame };
package/dist/pack.js CHANGED
@@ -1,5 +1,5 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { _ as decodeEnvelope, a as buildDecodePlan, c as isPlainRequestData, d as PackEncodeError, f as encodeEnvelope, g as SchemaDriftError, h as PackDecodeError, i as applyDecodePlanInPlace, l as decodeStreamChunk, m as encodePositionalFromFieldDef, n as packFrame, o as decodeWsEnvelope, p as encodePositional, r as unpackFrame, s as encodeWsFrame, t as WS_SUBPROTOCOL_SUBSTRATE_PACK, u as encodeStreamChunk, v as decodePositional, y as decodePositionalFromFieldDef } from "./src-BLgbLq2k.js";
3
- import { n as acceptsPack, t as PACK_MIME } from "./content-type-C7T28xNb.js";
1
+ import { A as positionValue, C as decodePositional, D as encodePositional, E as encodeEnvelope, O as encodePositionalFromFieldDef, S as decodeEnvelope, T as PackEncodeError, _ as encodeStreamChunk, a as FrameStreamError, b as PackDecodeError, c as encodeStreamFrame, d as applyDecodePlanInPlace, f as buildDecodePlan, g as decodeStreamChunk, h as isPlainRequestData, i as FRAMES_PACK_MIME, j as withData, k as packEnvelopeParts, l as framesEncodingFromAccept, m as encodeWsFrame, n as DEFAULT_STREAM_IDLE_MS, o as HEARTBEAT_FRAME, p as decodeWsEnvelope, r as FRAMES_JSON_MIME, s as decodeStreamFrames, t as DEFAULT_MAX_FRAME_BYTES, u as withIdleTimeout, v as MUTATION_ACK_FIELDS, w as decodePositionalFromFieldDef, x as SchemaDriftError, y as mutationReturns } from "./src-B_kHkbsr.js";
2
+ import { n as acceptsPack, t as PACK_MIME } from "./content-type-wLljot5M.js";
3
+ import { n as pack, r as unpack } from "./msgpackr-Dq18g6Ds.js";
4
4
 
5
- export { PACK_MIME, PackDecodeError, PackEncodeError, SchemaDriftError, WS_SUBPROTOCOL_SUBSTRATE_PACK, acceptsPack, applyDecodePlanInPlace, buildDecodePlan, decodeEnvelope, decodePositional, decodePositionalFromFieldDef, decodeStreamChunk, decodeWsEnvelope, encodeEnvelope, encodePositional, encodePositionalFromFieldDef, encodeStreamChunk, encodeWsFrame, isPlainRequestData, packFrame, unpackFrame };
5
+ export { DEFAULT_MAX_FRAME_BYTES, DEFAULT_STREAM_IDLE_MS, FRAMES_JSON_MIME, FRAMES_PACK_MIME, FrameStreamError, HEARTBEAT_FRAME, MUTATION_ACK_FIELDS, PACK_MIME, PackDecodeError, PackEncodeError, SchemaDriftError, acceptsPack, applyDecodePlanInPlace, buildDecodePlan, decodeEnvelope, decodePositional, decodePositionalFromFieldDef, decodeStreamChunk, decodeStreamFrames, decodeWsEnvelope, encodeEnvelope, encodePositional, encodePositionalFromFieldDef, encodeStreamChunk, encodeStreamFrame, encodeWsFrame, framesEncodingFromAccept, isPlainRequestData, mutationReturns, packEnvelopeParts, pack as packFrame, positionValue, unpack as unpackFrame, withData, withIdleTimeout };
@@ -1,5 +1,6 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { L as applyObligations, Q as buildContext, et as jsonResponse, wt as ResolverFrameError } from "./src-DjIuRjzF.js";
1
+ import { Bt as jsonResponse, Rt as buildContext, tt as applyObligations, wn as ResolverFrameError } from "./src-CKSJ13p8.js";
2
+ import "./src-B_kHkbsr.js";
3
+ import { w as queryTimeoutError } from "./src-BMD2OE--.js";
3
4
  import { Worker } from "node:worker_threads";
4
5
 
5
6
  //#region ../../plugins/substrate-recall/src/candidate-selection.ts
@@ -819,6 +820,37 @@ function createCrystalliseGuard(opts = {}) {
819
820
  } };
820
821
  }
821
822
 
823
+ //#endregion
824
+ //#region ../../plugins/substrate-recall/src/query-timeout.ts
825
+ /**
826
+ * Recall's SQL (the embeddings, lexical and cluster stores, the row filter)
827
+ * cancelled by a role- or database-level statement_timeout: refused
828
+ * `query_timeout` on every surface instead of a 500 (#1836). The mapping is
829
+ * dataset-pg's, shared with its reads and the edges store.
830
+ */
831
+ /** A 57014 as `query_timeout`, else null. */
832
+ function recallQueryTimeout(err) {
833
+ return queryTimeoutError("recall", err);
834
+ }
835
+ /** A 57014 as a 503 `query_timeout` response, else null. */
836
+ function queryTimeoutResponse(err) {
837
+ const refused = recallQueryTimeout(err);
838
+ return refused === null ? null : jsonResponse(503, { error: {
839
+ code: refused.code,
840
+ message: refused.message
841
+ } });
842
+ }
843
+ /** `fn` with a 57014 it throws refused `query_timeout`. */
844
+ function timeoutMapped(fn) {
845
+ return async (...args) => {
846
+ try {
847
+ return await fn(...args);
848
+ } catch (err) {
849
+ throw recallQueryTimeout(err) ?? err;
850
+ }
851
+ };
852
+ }
853
+
822
854
  //#endregion
823
855
  //#region ../../plugins/substrate-recall/src/embeddings-store.ts
824
856
  /** Field names a row filter may name: plain identifiers, so they are safe to address in SQL. */
@@ -1393,6 +1425,8 @@ function recallRouteHandler(cfg) {
1393
1425
  });
1394
1426
  return jsonResponse(200, out);
1395
1427
  } catch (err) {
1428
+ const timedOut = queryTimeoutResponse(err);
1429
+ if (timedOut !== null) return timedOut;
1396
1430
  const code = err?.code;
1397
1431
  const message = err instanceof Error ? err.message : String(err);
1398
1432
  if (code === "bad_request") return jsonResponse(400, {
@@ -1511,6 +1545,8 @@ function memberRow(m) {
1511
1545
  };
1512
1546
  }
1513
1547
  function rethrowCoded$1(err) {
1548
+ const timedOut = recallQueryTimeout(err);
1549
+ if (timedOut !== null) throw timedOut;
1514
1550
  const code = err?.code;
1515
1551
  const message = err instanceof Error ? err.message : String(err);
1516
1552
  if (code === "policy_denied" || code === "bad_request" || code === "crystallise_in_flight") throw new ResolverFrameError(code, message);
@@ -1668,11 +1704,11 @@ function buildClusterSurface(deps) {
1668
1704
  types: {
1669
1705
  Cluster: {
1670
1706
  fields: CLUSTER_FIELDS,
1671
- resolver: clusterResolver
1707
+ resolver: timeoutMapped(clusterResolver)
1672
1708
  },
1673
1709
  ClusterMember: {
1674
1710
  fields: CLUSTER_MEMBER_FIELDS,
1675
- resolver: memberResolver
1711
+ resolver: timeoutMapped(memberResolver)
1676
1712
  }
1677
1713
  },
1678
1714
  mutations: {
@@ -1748,6 +1784,8 @@ function crystalliseRouteHandler(cfg) {
1748
1784
  }));
1749
1785
  return jsonResponse(200, out);
1750
1786
  } catch (err) {
1787
+ const timedOut = queryTimeoutResponse(err);
1788
+ if (timedOut !== null) return timedOut;
1751
1789
  const code = err?.code;
1752
1790
  const message = err instanceof Error ? err.message : String(err);
1753
1791
  if (code === "policy_denied") return jsonResponse(403, { error: {
@@ -2111,6 +2149,8 @@ async function coverageResponse(base, lexicalRegistry, composition) {
2111
2149
  lexicalCoverage: coverage.types
2112
2150
  });
2113
2151
  } catch (err) {
2152
+ const timedOut = queryTimeoutResponse(err);
2153
+ if (timedOut !== null) return timedOut;
2114
2154
  const code = err?.code;
2115
2155
  const message = err instanceof Error ? err.message : String(err);
2116
2156
  if (code === "unknown_type") return jsonResponse(400, { error: {
@@ -2300,6 +2340,8 @@ function embeddingsFetchRouteHandler(cfg) {
2300
2340
  });
2301
2341
  return jsonResponse(200, out);
2302
2342
  } catch (err) {
2343
+ const timedOut = queryTimeoutResponse(err);
2344
+ if (timedOut !== null) return timedOut;
2303
2345
  const code = err?.code;
2304
2346
  const message = err instanceof Error ? err.message : String(err);
2305
2347
  if (code === "bad_request") return jsonResponse(400, {
@@ -2575,6 +2617,11 @@ function recallMcpTool(cfg) {
2575
2617
  caller
2576
2618
  });
2577
2619
  } catch (err) {
2620
+ const timedOut = recallQueryTimeout(err);
2621
+ if (timedOut !== null) return { error: {
2622
+ code: timedOut.code,
2623
+ message: timedOut.message
2624
+ } };
2578
2625
  const code = err?.code;
2579
2626
  if (code === "bad_request" || code === "policy_denied" || code === "embedder_unavailable" || code === "no_retrieval_arm") return { error: {
2580
2627
  code,
@@ -2778,7 +2825,7 @@ async function alignVectorDimensions(s, opts) {
2778
2825
  const { dimension } = opts;
2779
2826
  if (!Number.isInteger(dimension) || dimension <= 0) throw new Error(`alignVectorDimensions: dimension must be a positive integer (got ${dimension})`);
2780
2827
  return await s.begin(async (tx) => {
2781
- await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align'))`;
2828
+ await tx`SELECT pg_advisory_xact_lock(hashtext('substrate-recall-align:' || current_schema()))`;
2782
2829
  return alignLocked(tx, opts);
2783
2830
  });
2784
2831
  }
@@ -2805,6 +2852,50 @@ async function alignLocked(s, opts) {
2805
2852
  };
2806
2853
  }
2807
2854
 
2855
+ //#endregion
2856
+ //#region ../../plugins/substrate-recall/src/embed-backfill.ts
2857
+ async function runEmbedBackfill(deps) {
2858
+ const result = {
2859
+ embedded: 0,
2860
+ unindexable: 0,
2861
+ failed: 0
2862
+ };
2863
+ const staleRowIds = deps.store.staleRowIds?.bind(deps.store);
2864
+ if (staleRowIds === void 0) return result;
2865
+ for (const { namespace, type } of deps.registry.types()) {
2866
+ const reembed = deps.registry.get(namespace, type)?.reembed;
2867
+ const dataset = deps.resolveDataset(namespace);
2868
+ if (reembed === void 0 || dataset === null) continue;
2869
+ const key = `${namespace}/${type}`;
2870
+ const ids = await staleRowIds({
2871
+ namespace,
2872
+ type,
2873
+ afterId: deps.cursors.get(key) ?? "",
2874
+ limit: deps.perTypeLimit
2875
+ });
2876
+ if (ids.length < deps.perTypeLimit) deps.cursors.delete(key);
2877
+ else deps.cursors.set(key, ids[ids.length - 1]);
2878
+ if (ids.length === 0) continue;
2879
+ const { rows } = await dataset.read(type, {
2880
+ ids,
2881
+ limit: ids.length
2882
+ });
2883
+ for (const row of rows) {
2884
+ const outcome = await reembed(row);
2885
+ if (outcome.status === "embedded") result.embedded++;
2886
+ else if (outcome.status === "unindexable") result.unindexable++;
2887
+ else if (outcome.status === "failed") {
2888
+ result.failed++;
2889
+ if (outcome.code === "unavailable") return {
2890
+ ...result,
2891
+ stopped: outcome.error
2892
+ };
2893
+ }
2894
+ }
2895
+ }
2896
+ return result;
2897
+ }
2898
+
2808
2899
  //#endregion
2809
2900
  //#region ../../plugins/substrate-recall/src/expansion-resolver.ts
2810
2901
  function makeExpansionResolver(registry, edges, cfg) {
@@ -3053,9 +3144,12 @@ function parseHitParams(params) {
3053
3144
  * Re-raise a coded core throw as a `ResolverFrameError` so the engine surfaces
3054
3145
  * the same code the HTTP route maps to a status. `policy_denied` / `bad_request`
3055
3146
  * pass through verbatim; embedder `unavailable` / `invalid_response` become
3056
- * `embedder_*` (route 503 parity). Anything uncoded rethrows raw.
3147
+ * `embedder_*` (route 503 parity), and a statement-timeout cancel becomes
3148
+ * `query_timeout`. Anything uncoded rethrows raw.
3057
3149
  */
3058
3150
  function rethrowCoded(err) {
3151
+ const timedOut = recallQueryTimeout(err);
3152
+ if (timedOut !== null) throw timedOut;
3059
3153
  const code = err?.code;
3060
3154
  const message = err instanceof Error ? err.message : String(err);
3061
3155
  if (code === "policy_denied" || code === "bad_request") throw new ResolverFrameError(code, message);
@@ -3300,6 +3394,8 @@ function lexicalReindexRouteHandler(cfg) {
3300
3394
  try {
3301
3395
  return jsonResponse(200, await handleLexicalReindex(validateLexicalReindexInput(parsed), cfg));
3302
3396
  } catch (err) {
3397
+ const timedOut = queryTimeoutResponse(err);
3398
+ if (timedOut !== null) return timedOut;
3303
3399
  const code = err?.code;
3304
3400
  const message = err instanceof Error ? err.message : String(err);
3305
3401
  if (code === "bad_request" || code === "unknown_type") return jsonResponse(400, { error: {
@@ -3467,6 +3563,8 @@ function reindexRouteHandler(cfg) {
3467
3563
  try {
3468
3564
  return jsonResponse(200, await handleReindex(validateReindexInput(parsed), cfg));
3469
3565
  } catch (err) {
3566
+ const timedOut = queryTimeoutResponse(err);
3567
+ if (timedOut !== null) return timedOut;
3470
3568
  const code = err?.code;
3471
3569
  const message = err instanceof Error ? err.message : String(err);
3472
3570
  if (code === "bad_request" || code === "unknown_type") return jsonResponse(400, { error: {
@@ -3547,6 +3645,8 @@ function salienceBoostRouteHandler(cfg) {
3547
3645
  });
3548
3646
  return jsonResponse(200, out);
3549
3647
  } catch (err) {
3648
+ const timedOut = queryTimeoutResponse(err);
3649
+ if (timedOut !== null) return timedOut;
3550
3650
  const code = err?.code;
3551
3651
  const message = err instanceof Error ? err.message : String(err);
3552
3652
  if (code === "policy_denied") return jsonResponse(403, { error: {
@@ -3654,6 +3754,9 @@ function makeSpreadResolver(registry, edges) {
3654
3754
  //#endregion
3655
3755
  //#region ../../plugins/substrate-recall/src/plugin.ts
3656
3756
  const DEFAULT_CLUSTER_DIMENSION = 768;
3757
+ const DEFAULT_EMBED_BACKFILL_INTERVAL_MS = 6e4;
3758
+ /** Rows per embedded type per pass, so one pass stays a few seconds of embed work. */
3759
+ const EMBED_BACKFILL_PER_TYPE = 50;
3657
3760
  const DEFAULT_QUERY_EMBED_TIMEOUT_MS = 3e4;
3658
3761
  const DEFAULT_WRITE_EMBED_TIMEOUT_MS = 3e4;
3659
3762
  /** `embedder` with `timeoutMs` applied to every call that does not set its own. */
@@ -3731,11 +3834,19 @@ function createRecallPlugin(cfg) {
3731
3834
  ...cfg.checkReadPolicy !== void 0 ? { checkReadPolicy: cfg.checkReadPolicy } : {},
3732
3835
  ...cfg.runClusterer !== void 0 ? { runClusterer: cfg.runClusterer } : {}
3733
3836
  };
3837
+ let backfillTimer;
3734
3838
  return {
3735
3839
  name: "substrate-recall",
3736
3840
  contractVersion: ">=1.0.0-alpha.0 <2.0.0",
3841
+ storeTables: [
3842
+ "embeddings",
3843
+ "lexical",
3844
+ "recall_clusters",
3845
+ "recall_cluster_members"
3846
+ ],
3737
3847
  async onKernelReady(ctx) {
3738
3848
  if (cfg.embedder === void 0 || ctx.sql === null) return;
3849
+ let dimensionMismatch = false;
3739
3850
  const report = await alignVectorDimensions(ctx.sql, {
3740
3851
  dimension: cfg.embedder.dimension,
3741
3852
  ...cfg.migration?.hnswM !== void 0 ? { hnswM: cfg.migration.hnswM } : {},
@@ -3743,6 +3854,7 @@ function createRecallPlugin(cfg) {
3743
3854
  });
3744
3855
  for (const [column, state] of [["embeddings.embedding", report.embeddings], ["recall_clusters.centroid", report.centroids]]) {
3745
3856
  if (state !== "populated") continue;
3857
+ dimensionMismatch = true;
3746
3858
  const found = column.startsWith("embeddings") ? report.found.embeddings : report.found.centroids;
3747
3859
  ctx.metric("recall_dimension_mismatch", {
3748
3860
  column,
@@ -3752,6 +3864,37 @@ function createRecallPlugin(cfg) {
3752
3864
  });
3753
3865
  process.emitWarning(`${column} is vector(${found}) and holds rows, but the embedder is ${cfg.embedder.modelId} at ${cfg.embedder.dimension}. Writes to it will fail until the rows are cleared (they are derived) and re-embedded: delete them, restart, then POST /substrate/v1/embed/reindex.`, { code: "RECALL_DIMENSION_MISMATCH" });
3754
3866
  }
3867
+ const intervalMs = cfg.embedBackfillIntervalMs ?? DEFAULT_EMBED_BACKFILL_INTERVAL_MS;
3868
+ const resolveDataset = cfg.resolveDataset;
3869
+ if (dimensionMismatch || intervalMs === 0 || resolveDataset === void 0) return;
3870
+ if (!ctx.background) return;
3871
+ const cursors = /* @__PURE__ */ new Map();
3872
+ let running = false;
3873
+ clearInterval(backfillTimer);
3874
+ backfillTimer = setInterval(() => {
3875
+ if (running) return;
3876
+ running = true;
3877
+ runEmbedBackfill({
3878
+ registry: embedSourceRegistry,
3879
+ store: cfg.store,
3880
+ resolveDataset,
3881
+ cursors,
3882
+ perTypeLimit: EMBED_BACKFILL_PER_TYPE
3883
+ }).then((r) => {
3884
+ if (r.embedded + r.failed === 0 && r.stopped === void 0) return;
3885
+ ctx.metric("recall_embed_backfill", r);
3886
+ process.stdout.write(`recall.embed_backfill ${JSON.stringify(r)}\n`);
3887
+ }).catch((err) => {
3888
+ process.stderr.write(`recall.embed_backfill failed: ${err instanceof Error ? err.message : String(err)}\n`);
3889
+ }).finally(() => {
3890
+ running = false;
3891
+ });
3892
+ }, intervalMs);
3893
+ backfillTimer.unref();
3894
+ },
3895
+ onKernelStop() {
3896
+ clearInterval(backfillTimer);
3897
+ backfillTimer = void 0;
3755
3898
  },
3756
3899
  migrations: [
3757
3900
  embeddingsMigration(cfg.migration ?? {}),
@@ -1,6 +1,5 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { Pt as defineBootstrap } from "./src-DjIuRjzF.js";
3
- import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-DCy7Iy2l.js";
1
+ import { Kn as defineBootstrap } from "./src-CKSJ13p8.js";
2
+ import { n as createNoopPubSub, t as createProposalsPlugin } from "./src-DmjwFMu8.js";
4
3
  import { z } from "zod";
5
4
 
6
5
  //#region ../../plugins/substrate-proposals/src/bootstrap.ts
@@ -22,14 +21,16 @@ import { z } from "zod";
22
21
  */
23
22
  const ProposalsBootstrapConfigSchema = z.object({
24
23
  ownedNamespaces: z.array(z.string().min(1)).default([]),
25
- proposeCapability: z.string().min(1).default("substrate.propose")
24
+ proposeCapability: z.string().min(1).default("substrate.propose"),
25
+ indexCapability: z.string().min(1).default("substrate.index")
26
26
  }).strict();
27
27
  const bootstrap = defineBootstrap.coordinator({
28
28
  schema: ProposalsBootstrapConfigSchema,
29
29
  build: (config) => createProposalsPlugin({
30
30
  ownedNamespaces: config.ownedNamespaces,
31
31
  pubsub: createNoopPubSub(),
32
- proposeCapability: config.proposeCapability
32
+ proposeCapability: config.proposeCapability,
33
+ indexCapability: config.indexCapability
33
34
  })
34
35
  });
35
36
 
@@ -14,6 +14,11 @@
14
14
  "default": "substrate.propose",
15
15
  "type": "string",
16
16
  "minLength": 1
17
+ },
18
+ "indexCapability": {
19
+ "default": "substrate.index",
20
+ "type": "string",
21
+ "minLength": 1
17
22
  }
18
23
  },
19
24
  "additionalProperties": false
@@ -1,6 +1,5 @@
1
- import { v as OnDelete } from "./schema-g4FQLr0V.js";
2
- import { ct as Dataset, t as CoordinatorPlugin } from "./plugin-t2g2wX0b.js";
3
- import { a as ResolverFrameError } from "./errors-D4vKuwz8.js";
1
+ import { kn as OnDelete, kt as Dataset, x as CoordinatorPlugin } from "./schema-aIkY8sR3.js";
2
+ import { u as ResolverFrameError } from "./errors-Cmkp1qxf.js";
4
3
  //#region ../../plugins/substrate-proposals/src/config.d.ts
5
4
  /**
6
5
  * Multi-instance invalidation transport used by the plugin to broadcast
@@ -24,6 +23,8 @@ interface PubSub {
24
23
  notify(channel: string, payload: string): Promise<void>;
25
24
  }
26
25
  interface ProposalsPluginConfig {
26
+ /** Where a one-line warning goes, such as a stored proposal skipped at replay. Defaults to `console.warn`. */
27
+ log?: (line: string) => void;
27
28
  /**
28
29
  * Namespaces this plugin owns. Propose mutations outside this list are
29
30
  * rejected at resolver entry. A proposed crossRef must target a type in an
@@ -41,6 +42,13 @@ interface ProposalsPluginConfig {
41
42
  * propose authority under a differently-named cap.
42
43
  */
43
44
  proposeCapability?: string;
45
+ /**
46
+ * Capability name required to invoke `propose_index`, in the namespace whose
47
+ * indexes it changes. Defaults to `'substrate.index'`: separate from the
48
+ * propose capability, so only the operator or a designated reviewer changes
49
+ * indexes.
50
+ */
51
+ indexCapability?: string;
44
52
  /**
45
53
  * Lookup function the `propose_namespace` resolver uses to find the dataset
46
54
  * for a hot-added namespace. The host (typically the substrate binary) wires
@@ -57,8 +65,12 @@ interface ProposalsPluginConfig {
57
65
  */
58
66
  export declare function createNoopPubSub(): PubSub;
59
67
  //#endregion
68
+ //#region ../../plugins/substrate-proposals/src/engine-write.d.ts
69
+ /** Resolves once every storage preparation started so far has settled. */
70
+ export declare function storageSettled(): Promise<void>;
71
+ //#endregion
60
72
  //#region ../../plugins/substrate-proposals/src/errors.d.ts
61
- type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired';
73
+ type ProposalErrorCode = 'type_already_exists' | 'type_already_proposed' | 'field_already_exists' | 'destructive_change_required' | 'unknown_type' | 'unowned_namespace' | 'invalid_field_spec' | 'invalid_params' | 'invalid_namespace' | 'namespace_already_proposed' | 'namespace_retired' | 'dataset_not_provisioned' | 'engine_not_wired' | 'index_on_code_defined_type' | 'unknown_index' | 'not_engine_owned' | 'unique_index' | 'still_desired' | 'unknown_recommendation' | 'recommendation_closed' | 'recommendation_stale' | 'propose_index_not_alone' | 'index_drop_timeout';
62
74
  /**
63
75
  * Structured error for proposal operations with stable error codes.
64
76
  *
@@ -80,7 +92,7 @@ export declare class ProposalError extends ResolverFrameError {
80
92
  }
81
93
  //#endregion
82
94
  //#region ../../plugins/substrate-proposals/src/field-spec-json.d.ts
83
- type ScalarType = 'string' | 'number' | 'boolean' | 'date' | 'json';
95
+ type ScalarType = 'string' | 'number' | 'int16' | 'int32' | 'int64' | 'boolean' | 'date' | 'json';
84
96
  interface ScalarSpec {
85
97
  kind: 'scalar';
86
98
  type: ScalarType;
@@ -1,4 +1,5 @@
1
1
  {
2
2
  "ownedNamespaces": [],
3
- "proposeCapability": "substrate.propose"
3
+ "proposeCapability": "substrate.propose",
4
+ "indexCapability": "substrate.index"
4
5
  }
package/dist/proposals.js CHANGED
@@ -1,4 +1,3 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { n as createNoopPubSub, r as ProposalError, t as createProposalsPlugin } from "./src-DCy7Iy2l.js";
1
+ import { i as ProposalError, n as createNoopPubSub, r as storageSettled, t as createProposalsPlugin } from "./src-DmjwFMu8.js";
3
2
 
4
- export { ProposalError, createNoopPubSub, createProposalsPlugin };
3
+ export { ProposalError, createNoopPubSub, createProposalsPlugin, storageSettled };
@@ -1,7 +1,6 @@
1
- import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { Pt as defineBootstrap } from "./src-DjIuRjzF.js";
3
- import { t as FakeEmbedder } from "./fake-DUVtKVSE.js";
4
- import { t as createRecallPlugin } from "./plugin-B3NxUCVn.js";
1
+ import { Kn as defineBootstrap } from "./src-CKSJ13p8.js";
2
+ import { t as FakeEmbedder } from "./fake-DdXtb-m8.js";
3
+ import { t as createRecallPlugin } from "./plugin-CEOhrEme.js";
5
4
  import { z } from "zod";
6
5
 
7
6
  //#region ../../packages/substrate-embedder/src/types.ts
@@ -384,6 +383,7 @@ const RecallBootstrapConfigSchema = z.object({
384
383
  embedder: EmbedderSchema.optional(),
385
384
  queryEmbedTimeoutMs: z.number().int().positive().default(3e4),
386
385
  writeEmbedTimeoutMs: z.number().int().positive().default(3e4),
386
+ embedBackfillIntervalMs: z.number().int().nonnegative().default(6e4),
387
387
  hnswM: z.number().int().positive().default(16),
388
388
  hnswEfConstruction: z.number().int().positive().default(64),
389
389
  salience: SalienceSchema.default({
@@ -433,6 +433,7 @@ const bootstrap = defineBootstrap.kernel({
433
433
  ...embedder !== null ? { embedder } : {},
434
434
  queryEmbedTimeoutMs: config.queryEmbedTimeoutMs,
435
435
  writeEmbedTimeoutMs: config.writeEmbedTimeoutMs,
436
+ embedBackfillIntervalMs: config.embedBackfillIntervalMs,
436
437
  ...deps.lexicalStore !== void 0 ? { lexicalStore: deps.lexicalStore } : {},
437
438
  store: deps.store,
438
439
  ...deps.stats !== void 0 ? { stats: deps.stats } : {},