@mnmal-ai/substrate 1.0.0-alpha.10 → 1.0.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/{boot-store-BVqfFxh1.js → boot-store-C5vNZJ3i.js} +13 -8
  2. package/dist/{bridge-Cp4PaIYO.d.ts → bridge-COu0Tg_h.d.ts} +1 -1
  3. package/dist/{cli-hSCBywhE.js → cli-BrwW4tjO.js} +41 -23
  4. package/dist/client.d.ts +3 -4
  5. package/dist/client.js +3 -3
  6. package/dist/{errors-7mycvNGq.d.ts → errors-BOllGhVx.d.ts} +15 -2
  7. package/dist/{index-DgSfwx7M.d.ts → index-Cp74OcmS.d.ts} +1 -1
  8. package/dist/pack.d.ts +2 -2
  9. package/dist/pack.js +1 -1
  10. package/dist/{pg-edges-store-Lxsl5ZnK.js → pg-edges-store-BLsbcXkk.js} +1 -1
  11. package/dist/{plugin-CtwOINCM.js → plugin-DIYm2cZE.js} +2 -2
  12. package/dist/proposals-bootstrap.js +2 -2
  13. package/dist/proposals.d.ts +2 -3
  14. package/dist/proposals.js +1 -1
  15. package/dist/recall-bootstrap.js +2 -2
  16. package/dist/recall.d.ts +1 -2
  17. package/dist/recall.js +2 -2
  18. package/dist/{repl-DqfHelK7.js → repl-B8gy0YM7.js} +22 -22
  19. package/dist/{row-BKKCXIG6.d.ts → row-DwaDU4b-.d.ts} +57 -10
  20. package/dist/{run-B5dTbZ3g.js → run-B18qtGKc.js} +23 -8
  21. package/dist/{dataset-CejF-Fr_.d.ts → schema-CijHRfuk.d.ts} +2225 -1604
  22. package/dist/sdk.d.ts +21 -11
  23. package/dist/sdk.js +4 -4
  24. package/dist/{src-7CUSMK66.js → src-Br6aDUti.js} +7 -3
  25. package/dist/{src-CqT-QWZ_.js → src-D041j6GS.js} +3 -3
  26. package/dist/{src-5SJ5GEIM.js → src-DK7VEQT6.js} +2572 -742
  27. package/dist/{src-hZb8wfvg.js → src-Dbvuel6w.js} +383 -284
  28. package/dist/{src-Da0t6si5.js → src-DpPdw2oL.js} +1 -1
  29. package/dist/{src-Bi849yhn.js → src-DptNGNYQ.js} +3 -3
  30. package/dist/{src-CyvV7dCK.js → src-gz4I1F29.js} +1027 -263
  31. package/dist/substrate-admin.js +1 -1
  32. package/dist/substrate.js +1 -1
  33. package/dist/testing.d.ts +28 -4
  34. package/dist/testing.js +4 -4
  35. package/dist/{ws-ByZyHk7m.d.ts → ws-DC8OhMaV.d.ts} +2 -3
  36. package/package.json +20 -19
  37. package/dist/schema-DoSc-4ig.d.ts +0 -470
@@ -1,1888 +1,2509 @@
1
- import { C as ActorIdentity, E as Id, M as WriteRow, b as EdgesStore, d as PackEnvelope, f as Schema, k as Row, r as FieldDef, u as OrderedBy } from "./schema-DoSc-4ig.js";
2
- //#region ../../packages/substrate-types/src/policy.d.ts
3
- interface Identity {
4
- id: string;
5
- roles?: string[];
6
- scopes?: string[];
1
+ //#region ../../packages/substrate-types/src/core.d.ts
2
+ /**
3
+ * Core substrate-neutral types. Spec: §4.5 Dataset, §4.6 Edge.
4
+ */
5
+ type Id = string | number;
6
+ interface Row {
7
+ id: Id;
8
+ _deletedAt?: Date | null;
7
9
  /**
8
- * Per-namespace capabilities — the only grant surface. `{ notes: ['read'],
9
- * tasks: ['read','mutate'] }`; `'*'` is a legal key. Read it through the
10
- * `identity-grant.ts` helpers.
11
- *
12
- * A plain object rather than a Map: `Identity` is JSON-serialised into the
13
- * SSE stream ticket, and a Map would arrive as `{}` on the far side.
14
- *
15
- * A bare `{ id }` with no grant (direct-call tests, system actors) is the
16
- * no-claim case: `grantOf` resolves it to `null`, and the engine's grant
17
- * gate then admits every namespace. An authenticator for remote callers
18
- * must therefore always set `grants`, `{}` for none.
10
+ * Server-managed row-lifecycle metadata, set by autoCRUD on create/update
11
+ * and projected on every read alongside `id`. Each dataset stores it in
12
+ * its own way.
19
13
  */
20
- grants: Record<string, string[]>;
21
- [k: string]: unknown;
14
+ _metadata?: RowMetadata;
15
+ /** Per-row walk aggregates (`params.agg`). */
16
+ _agg?: Record<string, unknown>;
17
+ /** Nested selections whose cap was hit, by name. */
18
+ _truncated?: Record<string, true>;
19
+ [field: string]: unknown;
22
20
  }
23
21
  /**
24
- * Semantic verb stamped on autoCRUD-registered mutation entries.
25
- * Absent on type entries and custom mutations — policies needing those use
26
- * `RequestContext.entry` / `operation` instead.
22
+ * A row as passed to `Dataset.write`. `id` may be omitted: the dataset then
23
+ * assigns a version-7 UUID and returns it in `WriteResult.ids`.
27
24
  */
28
- type AutoCrudVerb = 'get' | 'create' | 'update' | 'delete';
29
- interface RequestContext {
30
- identity: Identity | null;
31
- namespace: string;
32
- operation: 'query' | 'mutate' | 'subscribe' | 'stream';
33
- now: Date;
34
- headers: Record<string, string>;
35
- meta: Record<string, unknown>;
25
+ interface WriteRow {
26
+ id?: Id;
27
+ _deletedAt?: Date | null;
28
+ _metadata?: RowMetadata;
29
+ [field: string]: unknown;
30
+ }
31
+ /**
32
+ * Refuse an upsert batch that names one id twice: the datasets would
33
+ * otherwise disagree (pg cannot update a row twice in one statement).
34
+ */
35
+ declare function assertNoRepeatedIds(type: string, rows: readonly WriteRow[]): void;
36
+ type IdentityKind = 'agent' | 'user' | 'system';
37
+ /**
38
+ * Row-attribution actor — records who created or last updated a row.
39
+ * Distinct from the auth `Identity` in `policy.ts` (JWT principal).
40
+ */
41
+ interface ActorIdentity {
42
+ readonly kind: IdentityKind;
43
+ readonly id: string;
44
+ }
45
+ interface RowMetadata {
46
+ readonly createdAt: string | null;
47
+ readonly updatedAt: string | null;
48
+ readonly createdBy: ActorIdentity | null;
49
+ readonly updatedBy: ActorIdentity | null;
36
50
  /**
37
- * Trust-gated client IP computed by the HTTP route handler from the request's
38
- * `peerIp` + the operator-configured trusted-proxy allowlist. When the peer
39
- * is in the allowlist, this is the leftmost `x-forwarded-for` entry; when
40
- * untrusted, this is the peer IP itself. Undefined when the transport doesn't
41
- * expose peerIp (direct engine unit tests, pre-upgrade WS paths).
51
+ * The row's place in its type's commit order, on a type declared
52
+ * `ordering: 'visible'`; absent on every other type. Assigned on insert only.
42
53
  */
43
- clientIp?: string;
54
+ readonly seq?: number;
55
+ }
56
+ declare const CLIENT_SUPPLIED_SERVER_FIELD: 'client_supplied_server_field';
57
+ type ClientSuppliedServerFieldError = typeof CLIENT_SUPPLIED_SERVER_FIELD;
58
+ declare const ROW_METADATA_FIELDS: readonly ['createdAt', 'updatedAt', 'createdBy', 'updatedBy'];
59
+ type RowMetadataField = (typeof ROW_METADATA_FIELDS)[number];
60
+ //#endregion
61
+ //#region ../../packages/substrate-types/src/edge.d.ts
62
+ type OnDelete = 'set_null' | 'cascade' | 'restrict' | 'orphan';
63
+ type OnMissing = 'reject' | 'allow';
64
+ interface EdgeRef {
65
+ ns: string;
66
+ type: string;
67
+ id: Id;
68
+ bridged?: boolean;
69
+ }
70
+ interface Edge {
71
+ id: string;
72
+ from: EdgeRef;
73
+ to: EdgeRef;
74
+ fieldName: string;
75
+ policy: OnDelete;
76
+ createdAt: Date;
77
+ deletedAt?: Date;
78
+ lastSweptAt?: Date;
79
+ blocked?: boolean;
80
+ }
81
+ /**
82
+ * Narrow patch type for `EdgesStore.update`. Immutable fields (from, to,
83
+ * policy, fieldName, createdAt, id-as-identity) cannot be changed post-write.
84
+ * `deletedAt` is managed exclusively by `tombstone(id)`.
85
+ */
86
+ type EdgePatch = Pick<Edge, 'id'> & Partial<Pick<Edge, 'lastSweptAt' | 'blocked'>>;
87
+ type EdgeWriteInput = Omit<Edge, 'id' | 'createdAt' | 'lastSweptAt' | 'deletedAt' | 'blocked'>;
88
+ type CanDeleteResult = {
89
+ ok: true;
90
+ } | {
91
+ ok: false;
92
+ blockedBy: Edge[];
93
+ };
94
+ interface EdgesStore {
44
95
  /**
45
- * Whether the deployment verifies credentials at all, as the transport knows
46
- * it, so a refusal for want of an identity can say whether auth is off
47
- * rather than implying the caller's key is wrong.
96
+ * Stable identifier for the storage substrate this edges store wraps.
97
+ * Parallels `Dataset.txScopeId` (spec §5.3). When an `EdgesStore`
98
+ * and a `Dataset` share a `txScopeId`, the engine enlists edge
99
+ * mutations into the dataset's `withScopedTransaction`, making
100
+ * row-tombstone + edge-tombstone within one frame atomic.
48
101
  *
49
- * `undefined` means the transport did not say (direct engine unit tests,
50
- * routes with no auth config in scope). Treat it as unknown, never as `false`.
51
- */
52
- authEnabled?: boolean;
53
- /**
54
- * The registry entry name this dispatch targets — the `<name>` half of the
55
- * qualified frame key (e.g. `deleteTodo`, `Todo`). Threaded by the engine
56
- * at every dispatch site (query/mutate/stream/subscribe) so type- and
57
- * plugin-level policies can discriminate between resolvers that otherwise
58
- * present identically (`get<T>` and `delete<T>` both arrive as
59
- * operation:'mutate' with id-only params). Undefined only in direct-call
60
- * unit tests that construct a bare ctx.
61
- */
62
- entry?: string;
63
- /**
64
- * Semantic verb of the targeted entry when autoCRUD registered it.
65
- * Prefer this over `entry`-name matching for intent-level policies
66
- * ("deny deletes") — it survives entry renames. Absent for type entries
67
- * and custom mutations.
102
+ * Convention:
103
+ * - pg: derived from the `postgres.Sql` pool reference identity. A
104
+ * `PgEdgesStore` constructed against the same `Sql` as a
105
+ * `PgDataset` advertises the same `txScopeId`.
106
+ * - memory: unique per `MemoryEdgesStore` instance. A memory edges
107
+ * store can never coalesce into a dataset's substrate tx; the engine
108
+ * falls back to nested independent transactions for that case.
68
109
  */
69
- verb?: AutoCrudVerb;
70
- }
71
- /** The caller's request context for a policy check made outside dispatch. */
72
- type PolicyProbeContext = Partial<Pick<RequestContext, 'now' | 'headers' | 'meta' | 'clientIp' | 'authEnabled'>>;
73
- interface ResolverArgs {
74
- params?: Record<string, unknown>;
75
- fields?: Record<string, unknown>;
76
- rows?: Row[];
110
+ readonly txScopeId: string;
111
+ write(input: EdgeWriteInput): Promise<Edge>;
112
+ tombstone(id: string, at?: Date): Promise<void>;
113
+ update(patch: EdgePatch): Promise<void>;
114
+ readNextBatch(cursorId: string | null, size: number): Promise<{
115
+ edges: Edge[];
116
+ lastId: string | null;
117
+ }>;
118
+ findEdgesTo(ref: EdgeRef): Promise<Edge[]>;
77
119
  /**
78
- * Engine-injected per-call dataset handle. On `engine.mutate`, this is the
79
- * transaction-scoped `tx` view (spec §5.3, G1); on `engine.query` it's the
80
- * plain namespace dataset. Resolvers that perform I/O MUST use this handle
81
- * — using a dataset captured at registration time bypasses the active
82
- * transaction and breaks atomicity. Optional purely to keep direct-resolver
83
- * unit tests concise; the engine always supplies it.
120
+ * Batched `findEdgesTo`: all live (non-tombstoned) inbound edges whose `to`
121
+ * matches ANY of `refs`, in a single round-trip. Empty `refs` yields an
122
+ * empty result. Callers group the flat result by `edge.to` themselves.
84
123
  */
85
- dataset?: Dataset;
124
+ findEdgesToMany(refs: readonly EdgeRef[]): Promise<Edge[]>;
125
+ findEdgesFrom(ref: EdgeRef): Promise<Edge[]>;
126
+ canDelete(ref: EdgeRef): Promise<CanDeleteResult>;
127
+ findBlocked(): Promise<Edge[]>;
86
128
  /**
87
- * Engine-injected per-call edges store (§4.6). Resolvers that touch edges
88
- * MUST use this handle; the closed-over store from registration is a
89
- * fallback for direct-call tests. Unlike `dataset`, this is not namespace-
90
- * scoped — edges are inherently cross-namespace, and a single engine-scoped
91
- * `EdgesStore` is threaded through every call.
129
+ * List live (non-tombstoned) edges where `from_ns === namespace` OR
130
+ * `to_ns === namespace`. Used by drop-namespace to detect dangling refs
131
+ * when `cascade === 'refuse'`. Bounded by `limit` — callers only need
132
+ * enough to decide + surface a sample.
92
133
  */
93
- edges?: EdgesStore;
134
+ findEdgesTouchingNamespace(namespace: string, limit: number): Promise<Edge[]>;
94
135
  /**
95
- * Engine-injected post-commit write hook (§5.4). Write-path resolvers
96
- * (autoCRUD create/update/delete, or custom mutation resolvers) call this
97
- * AFTER their substrate write lands to announce the change. The engine
98
- * collects calls into a per-transaction batch and fans them out to the
99
- * subscription bus only after the enclosing `withTransaction` commits — a
100
- * rolled-back tx emits nothing. Direct-call unit tests may pass `onWrite` to
101
- * observe resolver output without spinning up an engine.
136
+ * List live (non-tombstoned) edges where either endpoint matches
137
+ * `(namespace, type)`. Used by drop-type for the same cascade-refuse check.
102
138
  */
103
- onWrite?: (event: WriteEvent) => void;
139
+ findEdgesTouchingType(namespace: string, type: string, limit: number): Promise<Edge[]>;
104
140
  /**
105
- * Queue work to run once the surrounding transaction has settled, whether it
106
- * committed or rolled back: a record that must outlive a rollback, such as
107
- * an audit row for the refusal that caused it. The work is told which
108
- * (`committed`), so post-commit work can skip a rollback. Absent outside a
109
- * transaction, where the caller can simply do the work.
141
+ * Distinct namespaces other than `namespace` at the far end of any edge
142
+ * (tombstoned or live) touching `namespace`, or `(namespace, type)` when
143
+ * `type` is given: the namespaces `dropEdgesByNamespace` / `dropEdgesByType`
144
+ * would delete edges from.
110
145
  */
111
- afterTx?: (fn: AfterTxWork) => void;
146
+ namespacesLinkedTo(namespace: string, type?: string): Promise<string[]>;
112
147
  /**
113
- * Engine-injected cancellation signal (§5.5). Supplied on `stream` dispatch
114
- * so generator resolvers can abort mid-iteration when the client cancels or
115
- * the socket closes. Streaming resolvers SHOULD observe this signal — either
116
- * by calling `signal.throwIfAborted()` at yield points, or by listening for
117
- * `abort` to release cursors / close descriptors in their `finally` block.
118
- * The engine cannot force-stop a generator; cooperation is the contract.
119
- * Absent on query/mutate dispatch — non-streaming resolvers should not see a
120
- * signal they have no way to honour.
148
+ * Physically delete every edge (tombstoned or live) where `from_ns === namespace`
149
+ * OR `to_ns === namespace`. Used by drop-namespace when `cascade === 'delete'`.
150
+ * Returns the exact count physically removed.
151
+ *
152
+ * Same atomicity rules as `tombstone` — MUST run inside a caller-supplied
153
+ * `withTransaction` for multi-store atomicity.
121
154
  */
122
- signal?: AbortSignal;
155
+ dropEdgesByNamespace(namespace: string, within?: readonly string[]): Promise<number>;
123
156
  /**
124
- * Engine-injected cross-namespace dataset resolver (committed, non-tx view).
125
- * Write-path resolvers use it to verify crossRef referents that live in a
126
- * DIFFERENT namespace (onMissing:'reject'). Same-namespace targets should use
127
- * the tx-scoped `dataset` handle instead. Absent on the query/subscribe path.
157
+ * Physically delete every edge (tombstoned or live) where either endpoint
158
+ * matches `(namespace, type)`. Used by drop-type when `cascade === 'delete'`.
159
+ * `within` restricts both drops to edges whose far end is `namespace` or one
160
+ * of the listed namespaces.
128
161
  */
129
- resolveDataset?: (ns: string) => Dataset | undefined;
162
+ dropEdgesByType(namespace: string, type: string, within?: readonly string[]): Promise<number>;
130
163
  /**
131
- * Engine-injected on mutate: how many frame entries share this call's
132
- * transaction, itself included. A resolver whose effect waits on other
133
- * transactions (a concurrent index drop) refuses to share one.
164
+ * Execute `fn` atomically. All ops performed on the `tx` argument run inside
165
+ * one substrate transaction. On throw, every mutation inside the callback
166
+ * rolls back; on resolve, the transaction commits and the value is returned.
167
+ *
168
+ * Nested calls flatten into the outer tx (savepoints out of scope).
169
+ * Cross-store atomicity with a sibling `Dataset.withTransaction` is the
170
+ * engine's concern; each store manages its own tx independently here.
171
+ * For enlisted cross-store atomicity, see `Dataset.withScopedTransaction`
172
+ * which yields a tx-bound `EdgesStore` alongside the tx-bound datasets.
173
+ *
174
+ * Mirrors `Dataset.withTransaction` (§4.5, G1 §5.3).
134
175
  */
135
- txEntries?: number;
176
+ withTransaction<T>(fn: (tx: EdgesStore) => Promise<T>): Promise<T>;
136
177
  }
137
- /**
138
- * Post-commit write announcement consumed by the engine's subscription bus
139
- * (§5.4). Emitted by write-path resolvers after the substrate operation
140
- * commits inside the current transaction. `row` is the post-commit canonical
141
- * row for creates and updates; for deletes it's the pre-tombstone snapshot
142
- * (absent if the resolver couldn't read the row before deleting). The engine
143
- * filters matching subscriptions by `(ns, type)`, tests the `row` against each
144
- * subscription's `where` predicate, and emits `add` / `update` / `remove`.
145
- */
146
- interface WriteEvent {
147
- ns: string;
148
- type: string;
149
- id: Row['id'];
150
- operation: 'create' | 'update' | 'delete';
151
- row?: Row;
178
+ //#endregion
179
+ //#region ../../packages/substrate-types/src/runtime-metrics.d.ts
180
+ /** Per-key resolver latency and error counts, as served by GET /substrate/v1/metrics. */
181
+ interface ResolverStat {
182
+ kind: 'query' | 'mutate';
183
+ calls: number;
184
+ errors: number;
185
+ p50Ms: number;
186
+ p99Ms: number;
187
+ maxMs: number;
188
+ lastAt: string;
152
189
  }
153
- interface Obligation {
154
- kind: string;
155
- [k: string]: unknown;
190
+ interface ProcessStat {
191
+ rssMB: number;
192
+ heapUsedMB: number;
193
+ cpuUserMs: number;
194
+ cpuSystemMs: number;
195
+ eventLoopLagP99Ms: number;
196
+ inFlight: number;
156
197
  }
157
- /**
158
- * Row-filter obligation: emitted by `row<T>(predicate)` policies. The engine
159
- * scans accumulated obligations after resolver execution and applies each
160
- * predicate as a silent filter.
161
- */
162
- interface RowFilterObligation<T = unknown> extends Obligation {
163
- kind: 'row-filter';
164
- predicate: (ctx: RequestContext, row: T) => boolean;
198
+ interface SlowRequest {
199
+ key: string;
200
+ kind: 'query' | 'mutate';
201
+ durationMs: number;
202
+ at: string;
165
203
  }
166
- interface PolicyDecision {
167
- allow: boolean;
168
- reason?: string;
169
- obligations?: Obligation[];
204
+ interface RuntimeErrorEntry {
205
+ at: string;
206
+ op: 'query' | 'mutate';
207
+ key: string;
208
+ code: string;
209
+ message: string;
210
+ /** The verified identity's id, or `'anonymous'`. */
211
+ caller?: string;
212
+ /** Present when consecutive identical `(unknown)` failures were merged into this entry. */
213
+ count?: number;
214
+ }
215
+ interface RuntimeErrors {
216
+ since: string;
217
+ errors: RuntimeErrorEntry[];
218
+ }
219
+ interface ResolveObservation {
220
+ key: string;
221
+ kind: 'query' | 'mutate';
222
+ durationMs: number;
223
+ ok: boolean;
224
+ code?: string;
225
+ message?: string;
226
+ /** The verified identity's id; absent for an anonymous caller. */
227
+ caller?: string;
170
228
  }
171
- type Policy = (ctx: RequestContext, args: ResolverArgs) => PolicyDecision;
172
- /**
173
- * Engine-facing resolver shape. Spec: §4.3, §5.2, §5.5.
174
- *
175
- * The engine treats any resolver's return as opaque rows. A `ReadResult` return
176
- * signals a paged list; the engine unpacks `rows` into the envelope and copies
177
- * `nextCursor`/`hasMore` into `_pagination[key]`. An `AsyncIterable<Row>`
178
- * return is the §5.5 stream-dispatch shape (duck-typed via
179
- * `Symbol.asyncIterator` so it can't collide with `Row[]`).
180
- *
181
- * This is what the registry stores and what the engine dispatches to. Plugin
182
- * authors declaring custom resolvers use the typed-authoring form
183
- * `TypedResolver<T>` below; the engine erases to `Resolver` at registration
184
- * time.
185
- */
186
- type Resolver = (ctx: RequestContext, args: ResolverArgs) => Promise<Row | Row[] | ReadResult | AsyncIterable<Row> | null> | AsyncIterable<Row>;
187
- /**
188
- * Typed-authoring resolver shape. Used by autoCRUD and hand-authored resolvers
189
- * that want to preserve the concrete return type before erasing to `Resolver`
190
- * at registration. `T` is unconstrained at the type level — the engine
191
- * discriminates on the runtime value shape regardless of static type.
192
- */
193
- type TypedResolver<T> = (ctx: RequestContext, args: ResolverArgs) => Promise<T>;
194
229
  /**
195
- * Engine-visible return shape for a resolver. `ReadResult` means "paged list";
196
- * `Row[]` means "full list, no pagination"; `Row` means "single"; `null` means
197
- * "single, not found". The engine discriminates at runtime.
230
+ * One type's table and index statistics, for the index advisor. Counters are
231
+ * cumulative since the server's statistics were last reset. Row counts are
232
+ * estimates. A type ABSENT from a `typeStats()` result is unknown, not empty:
233
+ * never read its absence as 0 rows (a sampled store can miss a small type).
198
234
  */
199
- type ResolverReturn = Row | Row[] | ReadResult | null;
200
- /** Work queued through `ResolverArgs.afterTx`, told whether the transaction committed. */
201
- type AfterTxWork = (outcome: {
202
- committed: boolean;
203
- }) => Promise<void>;
204
- //#endregion
205
- //#region ../../packages/substrate-types/src/server.d.ts
206
- type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
207
- interface SubstrateRequest {
208
- method: HttpMethod;
209
- path: string;
210
- headers: Record<string, string>;
211
- query: Record<string, string>;
212
- params: Record<string, string>;
213
- body: Buffer | null;
214
- identity?: Identity;
215
- /**
216
- * Remote peer IP observed at the transport layer. Populated by adapters
217
- * (Fastify: `request.ip`; memory: caller supplies via `inject({ peerIp })`).
218
- * Absent when transport doesn't expose it. Consumers that gate on peer IP
219
- * (e.g. `x-forwarded-for` trust) treat `undefined` as "untrusted peer".
220
- */
221
- peerIp?: string;
222
- /**
223
- * Client IP after the trusted-proxy check, stamped by the host before plugin
224
- * and kernel route handlers run. Never read from the client.
225
- */
226
- clientIp?: string;
227
- /** Whether the deployment verifies credentials, stamped with `clientIp`. */
228
- authEnabled?: boolean;
229
- /**
230
- * Per-request abort signal. Production transports (Fastify) wire it to fire
231
- * on socket close (client disconnect) and on `abortAllRequests()` during
232
- * graceful shutdown. Route handlers thread it into `engine.query/mutate`
233
- * via `signal:` so resolvers can cooperatively bail. Optional because
234
- * test fixtures and embedded `inject()` callers may not need cancellation.
235
- */
236
- signal?: AbortSignal;
237
- }
238
- interface SubstrateResponse {
239
- status: number;
240
- headers?: Record<string, string>;
241
- body?: Buffer | Uint8Array | string | ReadableStream;
242
- }
243
- type HttpHandler = (req: SubstrateRequest) => SubstrateResponse | Promise<SubstrateResponse>;
244
- /** Request op classes counted by the runtime-metrics surface (§ engine-metrics spec). */
245
- type RuntimeOp = 'query' | 'mutate';
246
- /** One observed HTTP request at the transport seam. */
247
- interface RequestObservation {
248
- readonly op: RuntimeOp;
249
- /** Transport-level success — HTTP status < 400. Frame-level envelope errors still count ok. */
250
- readonly ok: boolean;
251
- /** Wall-clock elapsed from handler entry to response, measured via Date.now(). Always >= 0. */
252
- readonly durationMs: number;
253
- }
254
- type RequestObserver = (o: RequestObservation) => void;
255
- interface WsConnection {
256
- readonly id: string;
257
- send(frame: Buffer | Uint8Array | string): void;
258
- /**
259
- * Transport-level back-pressure seam. Resolves once the
260
- * socket's write buffer is below its high-water mark — immediately when it
261
- * already is. The stream route awaits this between chunks so a fast producer
262
- * paces to a slow consumer instead of growing the send buffer unbounded.
263
- * Optional: transports without buffer introspection (memory tests) omit it,
264
- * and the route treats a missing hook as "always drained".
265
- */
266
- drain?(): Promise<void>;
267
- onMessage(cb: (frame: Buffer) => void): void;
268
- onClose(cb: (code?: number) => void): void;
269
- close(code?: number, reason?: string): void;
270
- /**
271
- * Register a callback invoked synchronously during graceful server shutdown,
272
- * BEFORE the underlying socket is closed. Higher layers use this to flush
273
- * structured frames (e.g. `{ kind: 'end', reason: 'server_restart' }` per
274
- * active subscription) so clients can short-circuit reconnect backoff
275
- * instead of seeing an abrupt 1006 drop. Optional — transports without a
276
- * pre-close hook (memory tests) may omit it; the route layer should
277
- * tolerate undefined.
278
- */
279
- onShutdown?(cb: (reason: string) => void): void;
280
- }
281
- type WsHandler = (req: SubstrateRequest, conn: WsConnection) => void | Promise<void>;
282
- interface TlsConfig {
283
- cert: string | Buffer;
284
- key: string | Buffer;
285
- ca?: string | Buffer;
286
- }
287
- interface StartOpts {
288
- host?: string;
289
- port?: number;
290
- tls?: TlsConfig;
291
- }
292
- interface PluginRouter {
293
- get(path: string, handler: HttpHandler): void;
294
- post(path: string, handler: HttpHandler): void;
295
- put(path: string, handler: HttpHandler): void;
296
- delete(path: string, handler: HttpHandler): void;
297
- patch(path: string, handler: HttpHandler): void;
298
- websocket(path: string, handler: WsHandler): void;
299
- }
300
- interface SubstrateServer {
301
- start(opts?: StartOpts): Promise<{
302
- address: string;
303
- port: number;
304
- }>;
305
- stop(opts?: {
306
- grace?: number;
307
- abortInFlight?: boolean;
308
- }): Promise<void>;
309
- route(method: HttpMethod, path: string, handler: HttpHandler): void;
310
- websocket(path: string, handler: WsHandler): void;
311
- onError?(hook: (err: Error, req?: SubstrateRequest) => void): void;
312
- createPluginRouter(plugin: {
235
+ interface TypeStats {
236
+ namespace: string;
237
+ type: string;
238
+ liveRows: number;
239
+ insertsTotal: number;
240
+ updatesTotal: number;
241
+ deletesTotal: number;
242
+ indexes: Array<{
313
243
  name: string;
314
- }): PluginRouter;
315
- /**
316
- * Abort every in-flight HTTP request's `req.signal` so resolvers cooperatively
317
- * bail. Used during graceful shutdown to give the engine a chance to roll
318
- * back mutations before fastify's drain timeout fires. WS connections are
319
- * not affected by this call; their teardown rides on socket close.
320
- */
321
- abortAllRequests(reason?: string): void;
322
- /**
323
- * Register a callback invoked synchronously at the START of `stop()` —
324
- * before in-flight requests are aborted and before the transport drains.
325
- *
326
- * The HTTP analogue of `WsConnection.onShutdown`, for routes that hold
327
- * a response open indefinitely (SSE): handlers flush a terminal frame and end
328
- * their response, or the drain would wait forever.
329
- *
330
- * Additive, not last-one-wins (unlike `onError`): a dropped registration is
331
- * a leaked stream. Optional — transports with no graceful phase may omit it,
332
- * and callers must tolerate undefined.
333
- */
334
- onShutdown?(cb: (reason: string) => void): void;
335
- /**
336
- * Test-only: invoke the server's handler pipeline without opening a socket.
337
- * Networked implementations may leave this undefined.
338
- */
339
- inject?(req: Partial<SubstrateRequest>): Promise<SubstrateResponse>;
340
- /**
341
- * Register a per-request observer for op throughput + latency. Optional —
342
- * transports without instrumentation may omit it. At most one observer; a
343
- * second call replaces the first. The transport invokes it only for the
344
- * `/substrate/v1/query` and `/substrate/v1/mutate` routes.
345
- */
346
- observeRequests?(observer: RequestObserver): void;
347
- /** Optional: told when a query/mutate HTTP request starts (+1) and ends (-1), including when it throws. */
348
- observeInFlight?(fn: (op: RuntimeOp, delta: 1 | -1) => void): void;
244
+ scans: number;
245
+ /** The last scan (ISO), or null when none since the statistics were last reset. */
246
+ lastScanAt: string | null;
247
+ sizeBytes: number;
248
+ unique: boolean;
249
+ /** Built or adopted by the engine's index reconciler for this type. */
250
+ engineOwned: boolean;
251
+ }>;
252
+ }
253
+ /** What GET /substrate/v1/metrics returns. The new fields are optional so an older server still type-checks. */
254
+ interface RuntimeMetrics {
255
+ since: string;
256
+ requests: {
257
+ query: {
258
+ total: number;
259
+ ok: number;
260
+ error: number;
261
+ durationMsSum: number;
262
+ durationMsMax: number;
263
+ };
264
+ mutate: {
265
+ total: number;
266
+ ok: number;
267
+ error: number;
268
+ durationMsSum: number;
269
+ durationMsMax: number;
270
+ };
271
+ };
272
+ subscriptions: {
273
+ active: number;
274
+ maxQueueDepth?: number;
275
+ entries?: {
276
+ id: string;
277
+ ns: string;
278
+ type: string;
279
+ queueLength: number;
280
+ }[];
281
+ };
282
+ events: Record<string, number>;
283
+ resolvers?: Record<string, ResolverStat>;
284
+ process?: ProcessStat;
285
+ slowest?: SlowRequest[];
349
286
  }
350
287
  //#endregion
351
- //#region ../../packages/substrate-types/src/plugin.d.ts
352
- /** Whether a registry entry is a top-level addressable type or a mutation. */
353
- type RegistryEntryKind = 'type' | 'mutation';
288
+ //#region ../../packages/substrate-types/src/dataset.d.ts
354
289
  /**
355
- * postWrite hook signature. Receives the written row and request context;
356
- * may perform side-effects (e.g. embedding, index updates). Returned promise
357
- * is awaited before the engine acks the write to the caller.
290
+ * Handles yielded by `Dataset.withScopedTransaction`. Contains the
291
+ * tx-bound dataset for every requested namespace plus (optionally) a
292
+ * tx-bound edges store enlisted into the same substrate transaction.
293
+ * See `withScopedTransaction` JSDoc for the `edges` contract.
358
294
  */
359
- type PostWriteHook = (row: Row, ctx: RequestContext) => Promise<void>;
295
+ interface TxScopeHandles {
296
+ datasets: Map<string, Dataset>;
297
+ edges: EdgesStore | null;
298
+ }
360
299
  /**
361
- * Resolver signature stored in a `RegistryEntry`. Identical to the canonical
362
- * engine-facing `Resolver` in `./policy.ts`; redeclared inline here to keep
363
- * the registry surface self-describing for the types package. Importers that
364
- * want the canonical alias should import `Resolver` from this package.
300
+ * A leaf predicate on a single field. At most one operator per call-site is
301
+ * used in practice (e.g. `{ age: { gt: 18 } }`), but the shape admits several
302
+ * operators so `{ age: { gte: 10, lt: 100 } }` can express a half-open range
303
+ * without the caller reaching for an explicit `AND`. Semantics: the leaf
304
+ * matches when EVERY specified operator matches (implicit AND within a leaf).
305
+ *
306
+ * `eq`/`neq` accept `null` to address IS NULL / IS NOT NULL semantics at the
307
+ * substrate layer — the validator enforces `null` only on those two operators.
365
308
  */
366
- type RegistryEntryResolver = (ctx: RequestContext, args: ResolverArgs) => Promise<Row | Row[] | ReadResult | AsyncIterable<Row> | null> | AsyncIterable<Row>;
367
- /** Input shape for `Registry.register` / `Registry.replace`. */
368
- interface RegistryEntryInput {
369
- resolver: RegistryEntryResolver;
370
- policy?: Policy;
371
- fields?: Record<string, FieldDef>;
372
- kind?: RegistryEntryKind;
373
- params?: Record<string, FieldDef>;
374
- returns?: FieldDef;
375
- /** Natural order of a `kind: 'type'` entry; published on its `TypeSchema`. */
376
- orderedBy?: OrderedBy;
377
- /** Declared indexes of a `kind: 'type'` entry installed at runtime (a proposed type). */
378
- indexes?: IndexDecl[];
379
- /** A proposed type's size hint; `large` lowers the index advisor's row minimum. */
380
- expectedSize?: 'small' | 'large';
381
- /**
382
- * Semantic verb for autoCRUD-registered mutations. The engine
383
- * threads this into `RequestContext.verb` at dispatch so type-/plugin-level
384
- * policies can discriminate get/create/update/delete without entry-name
385
- * string matching. Omit for type entries and custom mutations.
386
- */
387
- verb?: AutoCrudVerb;
309
+ interface FieldPredicate {
310
+ eq?: number | string | boolean | Date | null | RefOperand;
311
+ neq?: number | string | boolean | Date | null | RefOperand;
312
+ gt?: number | string | Date | RefOperand;
313
+ gte?: number | string | Date | RefOperand;
314
+ lt?: number | string | Date | RefOperand;
315
+ lte?: number | string | Date | RefOperand;
316
+ between?: [number | string | Date, number | string | Date];
317
+ in?: Array<number | string | boolean | Date>;
318
+ nin?: Array<number | string | boolean | Date>;
319
+ contains?: string;
320
+ startsWith?: string;
321
+ endsWith?: string;
322
+ has?: number | string | boolean | Date | null;
323
+ path?: string[];
324
+ }
325
+ /**
326
+ * A where clause is either a logical combinator OR a map of field names to
327
+ * predicates / nested where clauses (for struct fields). Field names and the
328
+ * three combinator keys (AND/OR/NOT) share a namespace — `AND`/`OR`/`NOT` are
329
+ * reserved and must not collide with field names declared in any type.
330
+ */
331
+ type WhereClause = {
332
+ AND: WhereClause[];
333
+ } | {
334
+ OR: WhereClause[];
335
+ } | {
336
+ NOT: WhereClause;
337
+ } | {
338
+ _walk: WalkNode;
339
+ } | WalkAggPredicate | {
340
+ _walk?: never;
341
+ _agg?: never;
342
+ [field: string]: FieldPredicate | WhereClause | unknown;
343
+ };
344
+ type WalkPick = {
345
+ latest: string;
346
+ } | {
347
+ max: string;
348
+ } | {
349
+ min: string;
350
+ };
351
+ interface WalkNode {
352
+ as?: string;
353
+ /** `'<ns>/<Type>.<field>'`: inbound edge. */
354
+ in?: string;
355
+ /** `'<field>'`: outbound edge. */
356
+ out?: string;
357
+ where?: WhereClause;
358
+ quantifier?: 'any' | 'none' | 'all';
359
+ pick?: WalkPick;
360
+ }
361
+ /** `'$.f'` | `'<as>.f'` | `'_agg.<name>'` */
362
+ interface RefOperand {
363
+ $ref: string;
364
+ times?: number;
365
+ plus?: number;
366
+ }
367
+ type AggFn = {
368
+ count: true;
369
+ } | {
370
+ avg: string;
371
+ } | {
372
+ sum: string;
373
+ } | {
374
+ min: string;
375
+ } | {
376
+ max: string;
377
+ } | {
378
+ latest: {
379
+ by: string;
380
+ value: string;
381
+ };
382
+ };
383
+ type WalkAgg = {
384
+ in?: string;
385
+ out?: string;
386
+ where?: WhereClause;
387
+ } & AggFn;
388
+ type FrameAgg = Exclude<AggFn, {
389
+ latest: unknown;
390
+ }>;
391
+ interface BucketSpec {
392
+ field: string;
393
+ by: 'day' | 'hour';
394
+ }
395
+ type DeriveSpec = {
396
+ diff: [string, string];
397
+ } | {
398
+ ratio: [string, string];
399
+ };
400
+ interface NestedSelection {
401
+ _walk: {
402
+ in?: string;
403
+ out?: string;
404
+ where?: WhereClause;
405
+ };
406
+ fields: Record<string, unknown>;
407
+ where?: WhereClause;
408
+ orderBy?: OrderByClause[];
409
+ limit?: number;
388
410
  }
411
+ /** An `_agg` comparison operand: a number, a date or string (by the aggregate's kind), or a `$ref`. */
412
+ type WalkAggOperand = number | string | RefOperand;
413
+ /** `{ _agg: <walk aggregate>, <cmp>: <operand> }` */
414
+ type WalkAggPredicate = {
415
+ _agg: WalkAgg;
416
+ } & ({
417
+ eq: WalkAggOperand;
418
+ } | {
419
+ neq: WalkAggOperand;
420
+ } | {
421
+ gt: WalkAggOperand;
422
+ } | {
423
+ gte: WalkAggOperand;
424
+ } | {
425
+ lt: WalkAggOperand;
426
+ } | {
427
+ lte: WalkAggOperand;
428
+ });
389
429
  /**
390
- * Stored shape returned by `Registry.get` and `Registry.all`. Mirrors the
391
- * concrete `RegistryEntry` in `@mnmal-ai/substrate-core` — the engine reads
392
- * `policy`, `kind`, `resolver`, `fields`, `params`, and `returns` off this
393
- * shape on every dispatch.
430
+ * OrderBy clause. One entry per field, direction per field. Engine appends an
431
+ * implicit `{ id: 'asc' }` when callers omit `id` as the final tie-breaker — a
432
+ * deterministic tie-break is what makes cursor pagination stable across pages.
394
433
  */
395
- interface RegistryEntry {
396
- resolver: RegistryEntryResolver;
397
- policy: Policy;
398
- fields: Record<string, FieldDef>;
399
- kind: RegistryEntryKind;
400
- /** Always present — `{}` for `kind: 'type'` entries where the concept doesn't apply. */
401
- params: Record<string, FieldDef>;
402
- /** Present only for `kind: 'mutation'` entries that declared a return shape. */
403
- returns?: FieldDef;
404
- /** Mirrors `RegistryEntryInput.orderedBy`. */
405
- orderedBy?: OrderedBy;
406
- /** Mirrors `RegistryEntryInput.indexes`. */
407
- indexes?: IndexDecl[];
408
- /** Mirrors `RegistryEntryInput.expectedSize`. */
409
- expectedSize?: 'small' | 'large';
410
- /** Mirrors `RegistryEntryInput.verb` — present when the registrant stamped one. */
411
- verb?: AutoCrudVerb;
434
+ interface OrderByClause {
435
+ /** A `$`-prefixed key orders by a `params.derive` entry. */
436
+ [field: string]: 'asc' | 'desc';
412
437
  }
413
- /**
414
- * Canonical per-namespace registry surface, implemented by the concrete
415
- * `Registry` class in `@mnmal-ai/substrate-core`.
416
- */
417
- interface Registry {
418
- register(key: string | {
419
- readonly name: string;
420
- }, input: RegistryEntryInput): void;
421
- replace(key: string | {
422
- readonly name: string;
423
- }, input: RegistryEntryInput): void;
424
- /**
425
- * Read back a registered entry. Returns undefined when the entry is absent.
426
- * The full `RegistryEntry` shape (resolver, policy, kind, fields, params,
427
- * returns) is exposed because the engine reads every field on dispatch.
428
- */
429
- get(key: string | {
430
- readonly name: string;
431
- }): RegistryEntry | undefined;
432
- /** Presence check. Returns true iff `key` is registered. */
433
- has(key: string | {
434
- readonly name: string;
435
- }): boolean;
436
- /** Iterate every `[name, entry]` pair. Used by the schema publisher. */
437
- all(): IterableIterator<[string, RegistryEntry]>;
438
+ interface ReadParams {
439
+ /** Aborts the read. Substrates that can (PgDataset) cancel the in-flight statement; others MAY ignore it. */
440
+ signal?: AbortSignal;
441
+ id?: Id;
442
+ ids?: Id[];
443
+ includeDeleted?: boolean;
444
+ where?: WhereClause;
445
+ orderBy?: OrderByClause[];
446
+ limit?: number;
447
+ /** Return a match count (`ReadResult.count`). */
448
+ count?: boolean;
449
+ aggregate?: Record<string, FrameAgg>;
450
+ bucket?: BucketSpec;
451
+ derive?: Record<string, DeriveSpec>;
452
+ /** Per-row walk aggregates, lifted out of `fields` by the list resolver. */
453
+ agg?: Record<string, WalkAgg>;
454
+ /** Per-row nested selections, lifted out of `fields` by the list resolver. */
455
+ nested?: Record<string, NestedSelection>;
438
456
  /**
439
- * Remove an entry. Optional: only a registry that can take back a live
440
- * apply (a schema proposal rolled back with its transaction) offers it.
457
+ * Opaque cursor produced by a prior `read()` call. Base64url-encoded JSON
458
+ * (server-built, server-consumed). Callers should treat it as a black box.
441
459
  */
442
- unregister?(key: string | {
443
- readonly name: string;
444
- }): void;
460
+ cursor?: string;
445
461
  /**
446
- * Monotonic mutation counter — incremented by every `register` / `replace` / `unregister`.
447
- * Lets a consumer cache anything derived from the registry (the published
448
- * schema, above all) and detect staleness from the data, without relying on
449
- * every registrant remembering to call `engine.invalidateSchemaCaches`.
462
+ * Optional caller-supplied projection hint. Substrates MAY use this to
463
+ * narrow the body keys they materialise from storage — e.g. PgDataset
464
+ * builds a `jsonb_build_object(...)` projection over only the requested
465
+ * top-level body keys, saving wire + parse bandwidth.
450
466
  *
451
- * Optional so third-party `Registry` implementations remain valid; consumers
452
- * fall back to callback-driven invalidation when it is absent.
467
+ * When present, ONLY top-level entries are honoured (the value can be `true`
468
+ * or a nested FieldInput-shaped object — the substrate treats both as
469
+ * "include this key"). Nested projection inside JSONB sub-trees is intentionally
470
+ * out of scope; the pack codec already prunes deeper. Row-metadata-style keys
471
+ * (`_metadata`, `shortId`, `_deletedAt`, `id`) are populated from dedicated
472
+ * columns regardless of `fields` and never need to be included here.
473
+ *
474
+ * When absent, substrates return the full body, for raw `dataset.read()`
475
+ * callers (sweep, compose registry, migration scripts) that don't know which
476
+ * body keys they'll need downstream.
477
+ *
478
+ * Substrates MAY ignore this hint — it's a perf optimisation, not a contract.
479
+ * Memory dataset, for instance, has no wire to save and ignores it.
453
480
  */
454
- readonly epoch?: number | undefined;
481
+ fields?: {
482
+ [field: string]: true | unknown;
483
+ };
455
484
  }
456
485
  /**
457
- * Base class binding the `epoch` bookkeeping so an implementation cannot forget
458
- * it. Subclasses supply storage via `performRegister` / `performReplace` and
459
- * never implement `register` / `replace` themselves, so the counter is advanced
460
- * by construction rather than by discipline.
461
- *
462
- * A throw from `performRegister` leaves the epoch untouched, which is correct:
463
- * a rejected registration changed nothing.
464
- *
465
- * Decorators cannot extend this — they wrap an instance rather than subclass it
466
- * — so a wrapping registry must forward `epoch` from the wrapped instance. See
467
- * `substrate-compose`'s `_system` install for the worked example.
486
+ * Paged result of a `read()` call. `rows` is always present; `nextCursor` is
487
+ * present iff `hasMore` is `true` and the current page is non-empty.
468
488
  */
469
- declare abstract class RegistryBase implements Registry {
470
- #private;
471
- get epoch(): number;
472
- register(key: string | {
473
- readonly name: string;
474
- }, input: RegistryEntryInput): void;
475
- replace(key: string | {
476
- readonly name: string;
477
- }, input: RegistryEntryInput): void;
478
- unregister(key: string | {
479
- readonly name: string;
480
- }): void;
481
- /** Store a new entry. Throw to reject; the epoch will not advance. */
482
- protected abstract performRegister(key: string | {
483
- readonly name: string;
484
- }, input: RegistryEntryInput): void;
485
- /** Overwrite an existing entry. Throw to reject; the epoch will not advance. */
486
- protected abstract performReplace(key: string | {
487
- readonly name: string;
488
- }, input: RegistryEntryInput): void;
489
- /** Remove an entry. Not abstract, so existing subclasses stay valid; they refuse. */
490
- protected performUnregister(key: string | {
491
- readonly name: string;
492
- }): void;
493
- abstract get(key: string | {
494
- readonly name: string;
495
- }): RegistryEntry | undefined;
496
- abstract has(key: string | {
497
- readonly name: string;
498
- }): boolean;
499
- abstract all(): IterableIterator<[string, RegistryEntry]>;
500
- }
501
- /** See `EngineHandle.addDropParticipant`. */
502
- interface DropParticipant {
503
- /** Namespaces whose datasets `onDrop` writes, opened in the drop's transaction. */
504
- readonly namespaces: readonly string[];
505
- /** `datasetFor` returns the transaction-bound dataset of one of `namespaces`. */
506
- onDrop(ns: string, datasetFor: (ns: string) => Dataset): Promise<void>;
489
+ interface ReadResult {
490
+ rows: Row[];
491
+ nextCursor?: string;
492
+ hasMore: boolean;
493
+ count?: number;
494
+ groups?: Array<Record<string, unknown>>;
495
+ truncated?: boolean;
507
496
  }
508
- interface NamespaceInstallContext {
509
- registry: Registry;
510
- dataset: Dataset;
511
- edges: EdgesStore;
497
+ interface WriteResult {
498
+ written: number;
512
499
  /**
513
- * Plugin-global policy composed from the owning plugin's `policies` array
514
- * (undefined when the plugin declares none). Namespace install
515
- * implementations SHOULD compose this with any namespace-scoped or per-type
516
- * policies via `and()` so plugin-level invariants apply across every
517
- * resolver the namespace contributes. `defineNamespace` in
518
- * `@mnmal-ai/substrate-sdk` does this automatically.
500
+ * One per written row, in input order: a supplied id verbatim, otherwise
501
+ * the id the dataset assigned. Under `insertOnly`, skipped rows are absent.
519
502
  */
520
- pluginPolicy?: Policy;
503
+ ids: Id[];
521
504
  /**
522
- * Substrate-plugin contributions to per-type postWrite chains. Map
523
- * `typeName → PostWriteHook[]`. Engine populates by walking each
524
- * composed substrate plugin's `contributePostWrites(namespace)` and
525
- * merging results. Hooks run AFTER the type's inline postWrite in
526
- * declaration order. Empty map / undefined = no contributions.
505
+ * Merged rows after upsert — substrates populate this with the post-merge
506
+ * row state so autoCRUD update can skip its post-read. Order
507
+ * matches `ids` (one entry per input row). Substrates that can't compute
508
+ * the merged shape cheaply may emit input rows verbatim; consumers must
509
+ * not rely on body keys absent from the input being filled in.
527
510
  */
528
- postWriteContributions?: Record<string, PostWriteHook[]>;
511
+ rows: Row[];
529
512
  }
530
513
  /**
531
- * Read-side context passed to `NamespaceTypeDef.embed.source` when the
532
- * substrate composing the embedding (e.g. substrate-recall) has a dataset
533
- * resolver wired. Lets the source function pull related-entity context
534
- * via crossRefs (e.g. inline a linked Concept's name into the embedded
535
- * text so semantic recall surfaces relevant rows).
514
+ * Optional 3rd-arg to `dataset.write` / `tx.write`. When the incoming row's
515
+ * `_metadata` is absent, the substrate auto-fills it from these hints:
516
+ * - `now` → both `createdAt` and `updatedAt` (defaults to `new Date()`)
517
+ * - `identity` → both `createdBy` and `updatedBy` (defaults to `null`)
536
518
  *
537
- * Minimal by design — `read` is the only method, and it returns the
538
- * first matching row or `null`. Source functions that don't need
539
- * crossRef enrichment ignore the parameter.
519
+ * On upsert collision, `created_*` columns preserve existing values (immutability);
520
+ * `updated_*` columns adopt the new values (bump intent). Engine path
521
+ * (`autocrud-create`/`autocrud-update`) constructs full `_metadata` itself —
522
+ * this auto-fill is purely a safety net for raw-write bypass paths.
540
523
  */
541
- interface EmbedSourceContext {
542
- read: (namespace: string, type: string, params: {
543
- id: string;
544
- }) => Promise<Row | null>;
524
+ interface WriteOptions {
525
+ readonly identity?: ActorIdentity;
526
+ readonly now?: Date;
527
+ /**
528
+ * Insert only: a row whose id already exists (live or tombstoned) is left
529
+ * untouched and omitted from the result's `ids` and `rows`. The existence
530
+ * check and the insert are one atomic step.
531
+ */
532
+ readonly insertOnly?: boolean;
533
+ }
534
+ interface DeleteResult {
535
+ deleted: number;
536
+ ids: Id[];
537
+ }
538
+ interface DropResult {
539
+ /** Number of rows physically removed. */
540
+ rowsDeleted: number;
541
+ /** Number of short-id allocations physically removed. */
542
+ shortIdsDeleted: number;
545
543
  }
546
544
  /**
547
- * A declared index over a type's scalar or crossRef fields, built by the
548
- * engine with the same expressions the filter compiler emits.
549
- *
550
- * - `fields`: key columns in order; `dir` defaults to `asc`. Both directions
551
- * sort NULLs last, as `orderBy` does.
552
- * - `unique`: refused at build time while duplicate live values exist.
553
- * - `where`: always ANDed with "not deleted". `'live'` adds nothing more; a
554
- * record adds equality to a boolean (or IS NULL for `null`) on the type's
555
- * own fields. Only those operands compile to literals, which a generic plan
556
- * needs to match the index predicate.
557
- * - `name`: the pg index name to own or adopt; generated when omitted. It may
558
- * not end in `__swap`, the reconciler's temporary suffix.
559
- */
560
- interface IndexDecl {
561
- fields: Array<string | {
562
- field: string;
563
- dir: 'asc' | 'desc';
564
- }>;
565
- unique?: boolean;
566
- where?: 'live' | Record<string, boolean | null>;
567
- name?: string;
545
+ * Dataset-declared filter capabilities. §5.2 capability negotiation: the engine
546
+ * validates every where-clause leaf against `operators[fieldKind]` and rejects
547
+ * ops the dataset can't execute rather than returning a silently wrong result.
548
+ */
549
+ type FieldKind = 'scalar.number' | 'scalar.string' | 'scalar.boolean' | 'scalar.date' | 'scalar.json' | 'crossRef' | 'list';
550
+ interface FilterCapabilities {
551
+ operators: Record<FieldKind, string[]>;
552
+ /** Does the dataset honour `orderBy`? */
553
+ orderBy: boolean;
554
+ /** Does the dataset honour cursor-based pagination? */
555
+ cursor: boolean;
556
+ /**
557
+ * Does the dataset answer the typed-edge-walk grammar? The engine refuses a
558
+ * walk frame on a dataset without it (`walks_unsupported_store`).
559
+ */
560
+ walks?: boolean;
561
+ }
562
+ interface UpdateFieldRef {
563
+ ns?: string;
564
+ type: string;
565
+ id: Id;
568
566
  }
569
567
  /**
570
- * Structural shape of a type definition, mirroring `TypeInput` from
571
- * @mnmal-ai/substrate-sdk (redeclared to avoid a substrate-types → substrate-sdk import
572
- * cycle). Substrate plugins read it structurally; slots not named here pass
573
- * through opaquely.
568
+ * Outcome of a `conditionalUpdate` call (spec §5.12 Phase 2 / Dispatch B).
569
+ *
570
+ * `matched: true` when the row existed AND every key in `match` equalled the
571
+ * row's stored value at the moment of the update — the substrate applied
572
+ * `updates` and returned the merged row. `matched: false` when the row was
573
+ * missing or at least one match condition failed; in both cases no write landed
574
+ * and `row` is `null`.
574
575
  */
575
- interface NamespaceTypeDef {
576
- fields: Record<string, FieldDef>;
577
- postWrite?: (row: Row, ctx: RequestContext) => Promise<void>;
578
- embed?: {
579
- /**
580
- * Produce the source text fed to the embedder for this row, or `null` /
581
- * `undefined` to declare the row un-indexable.
582
- *
583
- * The null sentinel is a RETRIEVAL POLICY, not an absence of text: it
584
- * means "this row exists and must be referenceable, but must never enter
585
- * the recall candidate set". The composing substrate both declines to
586
- * index the row AND removes any embedding it already has, so a row that
587
- * flips to un-indexable after being written stops being recallable
588
- * without being deleted.
589
- *
590
- * `''` is NOT the sentinel — an empty string embeds normally. Returning
591
- * `''` to mean "skip" indexes every such row at the same point in the
592
- * vector space.
593
- *
594
- * `ctx` is optional — when the composing substrate has a dataset
595
- * resolver wired, source functions may call `ctx.read(ns, type,
596
- * { id })` to inline related-entity context. When `ctx` is undefined,
597
- * the source must work from `row` alone. Async return is supported
598
- * so source functions can await crossRef reads.
599
- */
600
- source: (row: Row, ctx?: EmbedSourceContext) => string | null | undefined | Promise<string | null | undefined>;
601
- };
576
+ interface ConditionalUpdateResult {
577
+ matched: boolean;
578
+ row: Row | null;
579
+ }
580
+ interface Dataset {
581
+ readonly namespace: string;
602
582
  /**
603
- * Produce the text fed to the LEXICAL (keyword) index for this row, with the
604
- * same `null` / `undefined` un-indexable sentinel as `embed.source`.
583
+ * Stable identifier for the storage substrate this dataset wraps. Two
584
+ * datasets share a substrate when their `txScopeId` values are `===`
585
+ * — they can participate in a single transaction via the engine's
586
+ * substrate-coalescing path (spec §5.3, enabling cross-namespace G1
587
+ * batched mutations when all frames target the same substrate).
605
588
  *
606
- * A sibling of `embed` rather than a flag on it, because the two indexes
607
- * want different text: `embed.source` conventionally takes a short handle
608
- * (`synopsis || body`), which drops proper nouns a keyword index needs.
609
- * `lexical.source` is where the long form goes.
589
+ * Convention:
590
+ * - pg: derived from the `postgres.Sql` pool reference identity. All
591
+ * `PgDataset` instances constructed against the same `sql`
592
+ * handle share one `txScopeId`.
593
+ * - memory: unique per `MemoryDataset` instance by default (each
594
+ * instance owns its own `Map`). Test harnesses that want shared-
595
+ * memory semantics can opt in via an explicit constructor option.
610
596
  *
611
- * A type may declare either, both, or neither. Both is the expected shape for
612
- * a store whose handle is shorter than its content.
597
+ * Datasets with different `txScopeId` values CANNOT participate in a
598
+ * single transaction — the engine refuses mixed-substrate batched
599
+ * mutates with `cross_tx_scope_refused`.
613
600
  */
614
- lexical?: {
615
- source: (row: Row) => string | null | undefined | Promise<string | null | undefined>;
616
- };
617
- /** Composite and unique indexes the engine builds for this type. See {@link IndexDecl}. */
618
- indexes?: IndexDecl[];
619
- /** The type's natural order, the default read order. See {@link OrderedBy}. */
620
- orderedBy?: OrderedBy;
621
- /** See the SDK's `TypeInput.ordering`. */
622
- ordering?: 'visible';
623
- /** Size hint: `large` lowers the index advisor's row minimum for this type. */
624
- expectedSize?: 'small' | 'large';
625
- [key: string]: unknown;
626
- }
627
- interface NamespaceDefinition {
628
- namespace: string;
601
+ readonly txScopeId: string;
602
+ read(type: string, params: ReadParams): Promise<ReadResult>;
603
+ write(type: string, rows: WriteRow[], options?: WriteOptions): Promise<WriteResult>;
604
+ tombstone(type: string, ids: Id[]): Promise<DeleteResult>;
605
+ purge(type: string, ids: Id[]): Promise<DeleteResult>;
629
606
  /**
630
- * Type definitions registered under this namespace, keyed by type name.
631
- * Read-only metadata exposed for substrate plugins that need to walk
632
- * the type surface at coordinator-namespace install time (e.g.
633
- * substrate-recall reads `types[T].embed?.source` to contribute embedding
634
- * postWrite hooks). Populated by `defineNamespace`; absent on hand-crafted
635
- * `NamespaceDefinition` objects that bypass the SDK helper.
607
+ * Physically delete every row of `type` within this dataset's namespace,
608
+ * plus every short-id allocation for that type. Ignores `_deletedAt` /
609
+ * `archived` state — drop is hard-delete regardless of row status.
610
+ *
611
+ * MUST run inside a caller-supplied `withTransaction` if atomicity with
612
+ * edge cleanup / audit emission is required. When called standalone,
613
+ * implementations MUST still execute the row-delete + short-id-delete as
614
+ * a single atomic step against the substrate.
615
+ *
616
+ * Returns the exact counts physically removed. If the type has no rows,
617
+ * returns zeros without error.
618
+ *
619
+ * Storage-level primitive: operates on any (namespace, type) rows present
620
+ * in storage regardless of whether the type is currently known to the
621
+ * engine's registry.
636
622
  */
637
- types?: Readonly<Record<string, NamespaceTypeDef>>;
623
+ dropType(type: string): Promise<DropResult>;
638
624
  /**
639
- * Opaque per-consumer metadata, keyed by consumer (e.g. `nl`). Core never
640
- * interprets it; it is passed through to the namespace schema unchanged.
625
+ * Physically delete every row AND short-id for `this.namespace`, across
626
+ * every type. Equivalent to calling `dropType` for each distinct type
627
+ * seen in storage, but substrate impls SHOULD use a single bulk DELETE
628
+ * where available (pg: `DELETE FROM rows WHERE namespace = $1`).
629
+ *
630
+ * Same atomicity rules as `dropType`. Same "ignores archived state".
641
631
  */
642
- annotations?: Readonly<Record<string, unknown>>;
632
+ dropNamespace(): Promise<DropResult>;
633
+ updateField(ref: UpdateFieldRef, fieldName: string, value: unknown): Promise<void>;
643
634
  /**
644
- * Body keys a SQL dataset mirrors into dedicated `rows` columns, so filters
645
- * and sorts on them hit an index. The declaring plugin creates the columns in
646
- * its kernel migrations; the engine creates none. `column` is a bare
647
- * identifier.
635
+ * Atomic compare-and-update by PK. If the row `(type, id)` exists AND every
636
+ * `(field, value)` pair in `match` equals the stored row's field, apply the
637
+ * `updates` map and return `{ matched: true, row: mergedRow }`. Otherwise
638
+ * write nothing and return `{ matched: false, row: null }`.
639
+ *
640
+ * Semantics:
641
+ * - Equality is strict (`===`) for scalars; `Date` compares by
642
+ * `.getTime()`; `null` matches only a stored `null` (including
643
+ * intentionally-null fields set via `updateField`), never an absent field.
644
+ * - A missing row fails the match even when `match` is empty — the
645
+ * operation is "update this existing row if predicate holds".
646
+ * - `updates` is merged onto the stored row the same way `write()` upserts
647
+ * merge: each key overwrites; omitted keys are preserved. `_deletedAt`
648
+ * is reserved and silently skipped (matches `write()`'s sanitisation).
649
+ * - The whole check+write is atomic against other dataset operations.
650
+ * Implementations MUST NOT yield between the predicate test and the
651
+ * write. Pg uses `UPDATE ... WHERE <match> RETURNING *` inside the
652
+ * caller's enclosing `withTransaction`; memory runs the whole body
653
+ * synchronously within the `async` microtask.
654
+ *
655
+ * Motivating use case: the `_system` register route consumes a one-shot
656
+ * invite — the invite row's `consumedAt` must transition from `null` to a
657
+ * timestamp exactly once, even under concurrent registrations.
658
+ *
659
+ * Metadata: a successful match unconditionally bumps `updatedAt` to `now`
660
+ * (a server-set `now()` when omitted) and stamps `updatedBy` with the supplied `actor` (or
661
+ * a `{ kind: 'system', id: 'anonymous' }` sentinel when omitted) — same
662
+ * semantics as `write()`'s upsert path, so recency-ranked recall sees the
663
+ * change. `createdAt` / `createdBy` are preserved.
648
664
  */
649
- promotedColumns?: Readonly<Record<string, {
650
- column: string;
651
- kind: 'boolean' | 'text';
652
- }>>;
665
+ conditionalUpdate(type: string, id: Id, match: Record<string, unknown>, updates: Record<string, unknown>, actor?: ActorIdentity, now?: Date): Promise<ConditionalUpdateResult>;
666
+ readonly capabilities: FilterCapabilities;
653
667
  /**
654
- * Install types + mutations + policies into the registry. Invoked once at
655
- * registration time, BEFORE the engine is constructed. The plugin owns the
656
- * dataset handle it uses for resolver closures; wiring datasets into the
657
- * engine's `DatasetRegistry` is the app composition root's job.
668
+ * Run `fn` while holding a mutual-exclusion lock named `key`, waiting until
669
+ * it is free. Excludes every other holder of the same key on this substrate:
670
+ * pg holds a session advisory lock, so the exclusion spans processes sharing
671
+ * the store's database schema; memory excludes holders on this dataset instance only.
672
+ * Released when `fn` settles, whether it resolves or throws, except on a
673
+ * pg tx-bound dataset, where it is held until the transaction ends so the
674
+ * next holder sees what this one committed. Not re-entrant:
675
+ * re-acquiring a held key inside `fn` waits forever.
676
+ *
677
+ * OPTIONAL. Callers feature-detect and fall back to a process-local lock.
678
+ *
679
+ * @returns whatever `fn` resolves to
680
+ * @throws whatever `fn` throws, after the lock is released
658
681
  */
659
- install(ctx: NamespaceInstallContext): void | Promise<void>;
660
- }
661
- interface PluginLifecycleContext {
682
+ withAdvisoryLock?<T>(key: string, fn: () => Promise<T>): Promise<T>;
662
683
  /**
663
- * Tagged-union discriminant. Differentiates this context from
664
- * the structurally-similar `KernelLifecycleContext` so a coordinator's
665
- * `onStart` handler is type-incompatible with a substrate's
666
- * `onKernelReady` handler — the composition root sets this, and plugins
667
- * can also branch on it at runtime if a single helper handles both
668
- * lifecycle stages.
684
+ * Prepare dedicated storage for `type` in this namespace, moving any rows
685
+ * already written for it there. Idempotent, and safe beside concurrent
686
+ * reads and writes: no row is lost, duplicated or seen twice. That holds for
687
+ * READ COMMITTED readers; a REPEATABLE READ snapshot taken before the move
688
+ * can miss the moved rows. Also builds the type's indexes, including its
689
+ * declared `indexes`, and throws when a unique one cannot be built.
690
+ *
691
+ * `indexes` picks which: `unique` builds only the unique ones and leaves the
692
+ * rest `pending`, for a later `nonUnique` call to build. `signal` cancels an
693
+ * in-flight non-unique build, which the next call retries. Default `all`.
694
+ *
695
+ * OPTIONAL. Backends without per-type storage omit it.
669
696
  */
670
- readonly lifecycle: 'coordinator';
697
+ ensureTypeStorage?(type: string, opts?: {
698
+ indexes?: 'all' | 'unique' | 'nonUnique';
699
+ signal?: AbortSignal;
700
+ }): Promise<void>;
671
701
  /**
672
- * Structured metric/log sink — the hosting binary picks the concrete surface
673
- * (stdout JSON, Prometheus counter, OpenTelemetry span, …). Plugins MUST NOT
674
- * assume the sink is synchronous or observable; the contract is fire-and-
675
- * forget per call.
702
+ * Where `type`'s rows live and how its declared indexes stand, read from the
703
+ * catalog without building anything.
704
+ *
705
+ * OPTIONAL. Backends without per-type storage omit it.
676
706
  */
677
- metric(name: string, payload?: unknown): void;
707
+ describeStorage?(type: string): Promise<TypeStorageReport>;
678
708
  /**
679
- * Engine handle threaded into `onStart` so plugins that mutate their
680
- * registry post-construction (e.g. substrate-proposals' applyType /
681
- * applyField paths) can wire `engine.invalidateSchemaCaches(ns)` into
682
- * their replace flow. Engine construction completes before `onStart`
683
- * fires, so the handle is always live here.
709
+ * The database-wide storage layout and what an operator should look at:
710
+ * rows parked in DEFAULT partitions, indexes and CHECK constraints on the
711
+ * `rows` parent, indexes this process has yet to build or failed to build,
712
+ * and a partition count over `partitionBudget`; and, as `info`, engine
713
+ * indexes no longer declared and open index recommendations. Never changes
714
+ * anything.
715
+ *
716
+ * OPTIONAL. Backends without per-type storage omit it.
684
717
  */
685
- engine: EngineHandle;
686
- }
687
- /**
688
- * Engine-facing input shape for `query` / `mutate` / `subscribe` / `stream`.
689
- * Declared here (rather than in substrate-core) so plugin surfaces can reference
690
- * the engine's call shape as a type without importing the engine package —
691
- * substrate-types stays zero-runtime.
692
- */
693
- interface FrameEntry {
694
- params?: Record<string, unknown>;
695
- fields?: Record<string, unknown>;
718
+ storageReport?(opts: {
719
+ partitionBudget: number;
720
+ }): Promise<StorageReport>;
696
721
  /**
697
- * §5.13 dedup key. Present on a mutate frame entry opts the whole frame
698
- * into the engine's idempotency store; a repeat call with the same key
699
- * inside the TTL returns the cached envelope verbatim.
722
+ * Table and index statistics for every type the store holds, every
723
+ * namespace, read from the database's own statistics. Estimates, never a
724
+ * count. Never changes anything.
725
+ *
726
+ * OPTIONAL. Backends without such statistics omit it.
700
727
  */
701
- idempotencyKey?: string;
702
- }
703
- /**
704
- * Qualified frame (§4.7). Keys are `<ns>/<name>`. `<name>` is the type entry
705
- * name for query, or the mutation name for mutate.
706
- *
707
- * Batched mutate (§5.13): a mutate op-key MAY carry an array of `FrameEntry`
708
- * values. All frames under one op-key run serially inside the SAME
709
- * transaction — all-or-nothing per op-key, committed independently of other
710
- * op-keys. Single-frame op-keys share one transaction when they share a
711
- * substrate and are refused (`cross_tx_scope_refused`) when they do not.
712
- * Query / subscribe / stream op-keys reject array values.
713
- */
714
- type QualifiedFrame = Record<string, FrameEntry | Array<FrameEntry>>;
715
- interface EngineInput {
716
- frame: QualifiedFrame;
717
- ctx: Partial<RequestContext>;
728
+ typeStats?(): Promise<TypeStats[]>;
718
729
  /**
719
- * §5.13 cancellation. Threaded into each resolver via `args.signal` so
720
- * resolvers observe cancel by polling `signal.aborted` or registering an
721
- * `abort` listener.
730
+ * The indexes the engine wants for `namespace`/`type` (any namespace, not
731
+ * only this one), with each one's leading keys, and the type's size hint.
732
+ * Undefined when the type is unknown. Never changes anything.
733
+ *
734
+ * OPTIONAL. Backends that build no indexes omit it.
722
735
  */
723
- signal?: AbortSignal;
736
+ indexPlan?(type: string, namespace: string): Promise<IndexPlan | undefined>;
724
737
  /**
725
- * Page bounds for list reads, set by a transport serving remote callers:
726
- * a type read with no `limit` gets `defaultLimit`, one above `maxLimit` is
727
- * refused. Absent for in-process callers, which read what they ask for.
738
+ * Drop index `name` of this namespace's `type`, a signed `propose_index`
739
+ * remove. Only an engine-owned, non-unique index the engine no longer wants
740
+ * (a leftover or a retracted declaration) is dropped; anything else is
741
+ * refused with an error whose `code` is `unknown_index`, `not_engine_owned`,
742
+ * `unique_index` or `still_desired`, and nothing is dropped. A drop that
743
+ * waits longer than `lockTimeoutMs` on another transaction is abandoned with
744
+ * `index_drop_timeout`; the index may then be left invalid, and a retry drops it.
745
+ *
746
+ * OPTIONAL. Backends that build no indexes omit it (the memory dataset):
747
+ * there, `propose_index remove` has no index to find, so it is refused
748
+ * `unknown_index`, and a remove recommendation it names is superseded.
728
749
  */
729
- pageLimits?: {
730
- readonly defaultLimit: number;
731
- readonly maxLimit: number;
732
- };
733
- }
734
- /**
735
- * Per-namespace engine configuration. substrate-core imports this type rather
736
- * than redeclaring it.
737
- */
738
- interface NamespaceConfig {
739
- registry: Registry;
750
+ dropEngineIndex?(type: string, name: string, opts?: {
751
+ /** Runs once every check has passed, just before the drop; a throw drops nothing. */
752
+ beforeDrop?: () => Promise<void>;
753
+ /** The drop's bound on each lock wait, in ms. Defaults to the backend's own. */
754
+ lockTimeoutMs?: number;
755
+ }): Promise<void>;
740
756
  /**
741
- * Marks a substrate-contributed namespace. Virtual namespaces
742
- * have NO dataset: the engine skips the dataset-presence check on
743
- * dispatch and leaves `args.dataset` undefined — resolvers compute from
744
- * `params`/`fields` and MUST NOT reach for a dataset handle. They are
745
- * structurally incapable of autoCRUD/storage — coordinators own data,
746
- * substrate surfaces own computation. Subscribe refuses these namespaces
747
- * with `subscription_unsupported_frame`.
757
+ * Drop the engine-owned indexes of this namespace, after a namespace drop
758
+ * has committed: its types leave the engine, so nothing would remove them.
759
+ * Non-unique ones only, unless `unique`, which the caller passes only when
760
+ * the namespace is declared nowhere (in no code and no live proposal): a
761
+ * declared namespace's types come back, and until the reconciler rebuilds a
762
+ * unique index, writes would not be held to it. Returns the names dropped.
763
+ * Never inside a transaction; a tx-bound instance throws. A lock wait that
764
+ * times out throws `index_drop_timeout`, leaving the rest for
765
+ * `storageReport` to list as orphaned.
766
+ *
767
+ * OPTIONAL. Backends that build no indexes omit it.
748
768
  */
749
- virtual?: boolean;
750
- /** From `NamespaceDefinition.annotations`; surfaced on the schema, never hashed. */
751
- annotations?: Readonly<Record<string, unknown>>;
752
- }
753
- /**
754
- * Narrow engine handle threaded into scheduled-task / plugin-route contexts.
755
- * The concrete `Engine` class in `@mnmal-ai/substrate-core` satisfies this
756
- * structurally; plugin code binds against this handle without importing the
757
- * engine package.
758
- */
759
- interface EngineHandle {
760
- query(input: EngineInput): Promise<PackEnvelope>;
761
- mutate(input: EngineInput): Promise<PackEnvelope>;
769
+ dropNamespaceIndexes?(opts?: {
770
+ unique?: boolean;
771
+ }): Promise<string[]>;
762
772
  /**
763
- * Evaluate the read policy for a type entry without dispatching it.
764
- * Out-of-band read surfaces (kernel routes returning row data
765
- * outside frame dispatch — e.g. /substrate/v1/recall) call this so the same
766
- * composed policies that gate `query` gate them too. Fails closed on
767
- * unknown namespace/type and tombstoned namespaces.
773
+ * The entry of this namespace's `type`'s declared `indexes` whose index is
774
+ * named `name`, or undefined when no declaration builds it (a derived or
775
+ * engine index, or none). `propose_index remove` retracts it when it is a
776
+ * proposed type's. Never changes anything.
777
+ *
778
+ * OPTIONAL. Backends that build no indexes omit it.
768
779
  */
769
- evaluateReadPolicy(ns: string, type: string, identity: Identity | null, probe?: PolicyProbeContext): PolicyDecision;
780
+ indexDeclarationOf?(type: string, name: string): Promise<IndexDecl | undefined>;
770
781
  /**
771
- * Invalidate cached schema outputs after an out-of-band registry mutation
772
- * (e.g. `applyTypeProposal` / `applyFieldProposal` calling
773
- * `Registry.replace`). Without this hook, `federatedSchema()` and
774
- * `schemaFor(ns)` keep returning pre-mutation views. Optional `ns` argument also drops the per-namespace `schemaFor` cache;
775
- * omit to invalidate only the federated outputs.
782
+ * Execute `fn` atomically. All ops performed on the `tx` argument run inside
783
+ * one substrate transaction. If `fn` throws or returns a rejected promise,
784
+ * every mutation inside the callback rolls back. If `fn` resolves, the
785
+ * transaction commits and the resolved value is returned.
786
+ *
787
+ * Scope rules:
788
+ * - `tx.namespace === this.namespace`
789
+ * - Calls on the outer dataset (`this`) inside `fn` bypass the transaction
790
+ * and are NOT rolled back on failure — this is a caller bug; implementations
791
+ * MUST NOT attempt to guard against it. All ops inside `fn` MUST go through
792
+ * `tx`.
793
+ * - Nested `tx.withTransaction(...)` is implementation-defined. The reference
794
+ * impls (memory, pg) flatten nested calls into the outermost transaction
795
+ * (no savepoints).
776
796
  */
777
- invalidateSchemaCaches(ns?: string): void;
797
+ withTransaction<T>(fn: (tx: Dataset) => Promise<T>): Promise<T>;
778
798
  /**
779
- * Register a new namespace at runtime. Throws on collision or on a
780
- * tombstoned name. Used by substrate-proposals' propose_namespace verb to provision
781
- * per-user / per-tenant namespaces without binary restart. Caller (typically
782
- * the resolver) is responsible for FPP semantics; this method is the
783
- * mechanical primitive.
799
+ * Engine-only. Open a substrate-level transaction spanning multiple
800
+ * namespaces on the same substrate, and invoke `fn` with a handles object
801
+ * containing tx-bound datasets + (optionally) a tx-bound `EdgesStore`.
802
+ * Called by the engine's mutate path (spec §5.3) for both single-ns and
803
+ * cross-ns frames — the engine calls with `[ns]` for the single-ns case.
804
+ *
805
+ * `namespaces` is the full set of namespaces the engine will dispatch
806
+ * frames to inside the tx. Implementations MUST return a tx-bound
807
+ * dataset for every requested namespace in `handles.datasets`; missing
808
+ * entries are a programming error on the caller side.
809
+ *
810
+ * `handles.edges` contract:
811
+ * - `non-null` when the implementation can enlist a shared `EdgesStore`
812
+ * into the same substrate tx (e.g. pg where the `PgEdgesStore` is wired
813
+ * in via constructor and shares the `Sql`). The engine routes edge
814
+ * mutations through this handle — true cross-store atomicity for
815
+ * row + edge within one frame.
816
+ * - `null` when the implementation can't enlist (e.g. memory datasets
817
+ * where edges live in a separate Map). The engine falls back to
818
+ * running edge ops in a nested independent tx and accepts the
819
+ * visibility-ordering quirk documented in `autocrud-delete.ts`.
820
+ *
821
+ * Scope rules mirror `withTransaction`:
822
+ * - All ops on values from `handles.datasets.get(ns)` and `handles.edges`
823
+ * run in the same tx.
824
+ * - Ops on the outer dataset (`this`) inside `fn` bypass the tx — caller
825
+ * bug; implementations do not guard.
826
+ * - A throw or rejection inside `fn` rolls back every mutation across
827
+ * every namespace + edges in the tx.
828
+ *
829
+ * OPTIONAL. Implementations that can't coalesce cross-namespace writes
830
+ * into one substrate transaction omit this method — the engine falls
831
+ * back to refusing cross-namespace batches with `cross_tx_scope_refused`
832
+ * (and loses edge-enlist for single-ns).
784
833
  */
785
- addNamespace(ns: string, config: NamespaceConfig): void;
834
+ withScopedTransaction?<T>(namespaces: ReadonlyArray<string>, fn: (handles: TxScopeHandles) => Promise<T>): Promise<T>;
786
835
  /**
787
- * Mark a namespace as tombstoned. After this call:
788
- * - schemaFor(ns) throws `unknown namespace`
789
- * - federatedSchema() omits the ns
790
- * - query / mutate / subscribe against the ns return a per-key
791
- * `namespace_tombstoned` error
792
- * - existing live subscriptions on the ns are closed with a
793
- * `namespace_tombstoned` error frame
836
+ * Allocate the next short id for the given type within this dataset's
837
+ * namespace. Monotonic per (namespace, type). Does NOT record any
838
+ * row-id mapping — call `bindShortId` after the row is written.
794
839
  *
795
- * Idempotent. The underlying NamespaceConfig stays in `#namespaces` for
796
- * concurrency safety (in-flight ops keep their references) — full physical
797
- * reclaim happens at engine restart, gated by whatever persistence layer
798
- * the host uses to decide which namespaces to re-install at boot. Tombstone
799
- * state itself is in-memory; restart wipes it.
840
+ * The implementation may use the dataset's current tx handle so the
841
+ * sequence advance rolls back on tx rollback (postgres sequences
842
+ * actually never roll back, but the surrounding tx semantics match
843
+ * what callers expect).
800
844
  */
801
- tombstoneNamespace(ns: string): void;
845
+ allocateShortId(type: string): Promise<number>;
802
846
  /**
803
- * Be told, after the fact, each time a namespace is tombstoned (a drop), so
804
- * a plugin that installed it can retire what it owns. A listener's throw is
805
- * reported as a metric and does not reach the dropper. Returns a disposer.
847
+ * Record the (shortId → rowId) pairing after the row is written.
848
+ * Called by autoCRUD create immediately after the dataset write()
849
+ * returns the new row id. Safe to call within the enclosing tx.
806
850
  */
807
- onNamespaceTombstoned(listener: (ns: string) => void): () => void;
851
+ bindShortId(type: string, shortId: number, rowId: string): Promise<void>;
808
852
  /**
809
- * Take part in every namespace drop's transaction: the drop route opens
810
- * `participant.namespaces` in its transaction too and runs `onDrop` after
811
- * the rows are gone and before it commits, so a throw rolls the drop back.
812
- * Returns a disposer.
853
+ * Resolve a short id to the underlying row id, or null if no such short
854
+ * id exists for this (namespace, type). Does not join `rows` —
855
+ * callers chain their own `read()` after resolution.
813
856
  */
814
- addDropParticipant(participant: DropParticipant): () => void;
815
- /** The participants `addDropParticipant` registered, for the drop route. */
816
- dropParticipants(): readonly DropParticipant[];
857
+ resolveShortId(type: string, shortId: number): Promise<string | null>;
817
858
  /**
818
- * Take a namespace off the engine and free its name for `addNamespace`.
819
- * Subscriptions on it close, its schema disappears, new ops answer
820
- * `unknown_namespace`; in-flight ops keep their resolved config. Undoes a
821
- * runtime add, e.g. a rolled-back propose_namespace. Idempotent.
859
+ * Inverse of `resolveShortId` — look up the short id assigned to a
860
+ * given row id. Returns null if no sidecar entry exists (e.g. rows written
861
+ * before short ids were allocated).
822
862
  */
823
- removeNamespace(ns: string): void;
824
- /** Whether `ns` is on the engine, and if so whether it is still served. */
825
- namespaceState(ns: string): 'absent' | 'live' | 'tombstoned';
863
+ getShortIdForRow(type: string, rowId: string): Promise<number | null>;
826
864
  /**
827
- * Construct a fresh empty Registry that the engine can accept via
828
- * `addNamespace`. Plugins call this to provision new namespaces at runtime
829
- * without taking a runtime dep on the concrete Registry class in substrate-core.
865
+ * Spawn a sibling dataset bound to the same backing store but scoped to
866
+ * a different namespace. Used by `compose()` to auto-allocate per-namespace
867
+ * datasets when a coordinator plugin claims a namespace the operator hasn't
868
+ * mapped explicitly.
869
+ *
870
+ * Substrate semantics:
871
+ * - Pg: returns a new `PgDataset` sharing this dataset's `sql` pool +
872
+ * `edgesStore`. The returned dataset has the SAME `txScopeId` (so it
873
+ * can participate in cross-namespace batched mutations on the shared pool).
874
+ * - Memory: returns a new `MemoryDataset` with its own backing `Map`
875
+ * per the default convention (each instance owns its own substrate). The
876
+ * returned dataset has a NEW `txScopeId` — it cannot share a transaction
877
+ * with the caller. Test harnesses that need shared-memory semantics across
878
+ * namespaces should construct each `MemoryDataset` explicitly with
879
+ * the relevant constructor option.
880
+ *
881
+ * Idempotency note: callers SHOULD NOT call `forNamespace(this.namespace)`;
882
+ * the resulting object's identity is implementation-defined. Use this only
883
+ * to derive datasets for *different* namespaces.
830
884
  */
831
- createRegistry(): Registry;
885
+ forNamespace(namespace: string): Dataset;
886
+ }
887
+ interface DatasetRegistry {
888
+ get(ns: string): Dataset | undefined;
889
+ has(ns: string): boolean;
890
+ set(ns: string, dataset: Dataset): void;
832
891
  /**
833
- * Look up the host-registered dataset for a namespace, or `undefined` if the
834
- * host has not provisioned one. Thin pass-through to the engine's
835
- * `DatasetRegistry.get`, surfaced on the handle so plugins running on
836
- * dynamically-added namespaces can resolve datasets without the host having
837
- * to plumb a `datasetFor` lookup through plugin config.
892
+ * Observe namespace lifecycle events (§6.4). Fires when a namespace is
893
+ * disposed — explicit unregister or substrate-signalled loss (memory-backed
894
+ * restart). Returns a disposer that unsubscribes when invoked.
838
895
  *
839
- * Substrate selection remains the host's responsibility — this method only
840
- * exposes the lookup. Plugins that hot-add namespaces (substrate-proposals'
841
- * `propose_namespace`) MUST require the host to register the dataset before
842
- * the call; this method returns `undefined` for unregistered namespaces.
896
+ * Optional: registries over persistent substrates (pg) never lose namespaces
897
+ * at runtime and can omit this surface entirely. Consumers MUST feature-
898
+ * detect before subscribing.
843
899
  */
844
- datasetFor(namespace: string): Dataset | undefined;
845
- }
846
- /**
847
- * Engine surface needed by substrate-pack-aware route helpers — extends the
848
- * dispatch handle with the two methods compose's `serializeEnvelope` /
849
- * `mutateRoute` need to drive positional encoding. The concrete
850
- * `Engine` class satisfies this structurally; tests + alt
851
- * implementations can satisfy it without depending on the engine package.
852
- */
853
- interface EncodeContext extends EngineHandle {
854
- /** Federated schema map (per-namespace Schema) for substrate-pack encoding. */
855
- federatedSchema(): Record<string, Schema>;
900
+ onNamespaceLost?(listener: (ns: string) => void): () => void;
856
901
  /**
857
- * Look up a namespace's mutation-return registry view. Returns undefined
858
- * when the namespace isn't registered. The mutate route uses this to
859
- * resolve each frame key's `returns` FieldDef for positional encoding.
902
+ * Explicitly drop `ns` from the registry and fire `onNamespaceLost`
903
+ * listeners synchronously. Disposing an unregistered namespace is a silent
904
+ * no-op — keeps graceful-shutdown replays idempotent.
905
+ *
906
+ * Optional: same rationale as `onNamespaceLost`.
860
907
  */
861
- registryFor(namespace: string): {
862
- get(name: string): {
863
- returns?: FieldDef;
864
- } | undefined;
865
- } | undefined;
908
+ dispose?(ns: string): void;
866
909
  }
867
910
  /**
868
- * Context threaded into every `ScheduledTask.handler` call.
911
+ * One declared index of a type and its state: as the catalog holds it, or,
912
+ * while this process defers or runs its build, `pending`, `building` or
913
+ * `failed`. Filters on an index that is not `valid` scan.
869
914
  */
870
- interface ScheduledTaskContext {
871
- /** The owning plugin's `name`. */
872
- pluginName: string;
873
- /** Engine handle for read/write work performed by the task. */
874
- engine: EngineHandle;
875
- /** Structured metric sink — fire-and-forget. */
876
- metric(name: string, payload?: unknown): void;
877
- }
878
- interface ScheduledTask {
879
- /** Unique identifier within the plugin. Used in metric labels. */
915
+ interface TypeIndexState {
880
916
  name: string;
881
- /** Fire interval in milliseconds. Minimum 100ms — smaller throws at register. */
882
- intervalMs: number;
883
- /** Fire once at registration in addition to the interval ticks. Default false. */
884
- runOnStart?: boolean;
885
- /** Handler — receives a context with engine access for read/write work. */
886
- handler(ctx: ScheduledTaskContext): void | Promise<void>;
917
+ fields: string[];
918
+ unique: boolean;
919
+ state: 'valid' | 'building' | 'pending' | 'invalid' | 'failed';
920
+ outcome: 'created' | 'adopted' | 'swapped' | 'unchanged';
921
+ /** Why a `failed` index failed: an error code, then a short reason. */
922
+ error?: string;
923
+ /** Why a `pending` index was not built: the free-disk check refused it. */
924
+ reason?: 'insufficient_disk';
887
925
  }
888
- /**
889
- * Context threaded into `CoordinatorPlugin.routes()` calls.
890
- */
891
- interface PluginRouteContext {
892
- /** The owning plugin's `name`. */
893
- pluginName: string;
894
- /** Engine handle for plugin-scoped routes that need to dispatch back through the engine. */
895
- engine: EngineHandle;
896
- /** Structured metric sink — fire-and-forget. */
897
- metric(name: string, payload?: unknown): void;
926
+ /** Where a type's rows live: `partition` is set on the partitioned layout once the type has one. */
927
+ interface TypeStorageReport {
928
+ layout: 'legacy' | 'partitioned';
929
+ partition?: string;
930
+ indexes: TypeIndexState[];
898
931
  }
899
- interface CoordinatorPlugin {
900
- name: string;
901
- namespaces: NamespaceDefinition[];
932
+ interface StorageReport {
933
+ /** `unknown` when the report itself failed. */
934
+ layout: 'legacy' | 'partitioned' | 'unknown';
935
+ warnings: string[];
936
+ /** Worth showing, not a fault: the doctor prints these without failing. */
937
+ info?: string[];
938
+ }
939
+ /** See `Dataset.indexPlan`. */
940
+ interface IndexPlan {
941
+ /** Each desired index, its keys in order: a derived index ends with the type's natural order. */
942
+ indexes: Array<{
943
+ name: string;
944
+ keys: Array<[string, 'asc' | 'desc']>;
945
+ unique: boolean;
946
+ /** A partial index's predicate beyond `live`: it serves only reads that `eq` each field to its value. */
947
+ only?: Record<string, boolean | null>;
948
+ }>;
949
+ expectedSize?: 'small' | 'large';
950
+ }
951
+ //#endregion
952
+ //#region ../../packages/substrate-types/src/policy.d.ts
953
+ interface Identity {
954
+ id: string;
955
+ roles?: string[];
956
+ scopes?: string[];
902
957
  /**
903
- * Plugin's bound contract version: a semver RANGE over the binary version,
904
- * e.g. `'>=4.1.0 <6.0.0'`.
958
+ * Per-namespace capabilities — the only grant surface. `{ notes: ['read'],
959
+ * tasks: ['read','mutate'] }`; `'*'` is a legal key. Read it through the
960
+ * `identity-grant.ts` helpers.
905
961
  *
906
- * The ceiling tracks actual compatibility, not the next major: it moves
907
- * only when a major genuinely breaks the plugin-facing interface. A ceiling
908
- * that admits a major which did break lets the plugin load and misbehave
909
- * silently — if you cannot say whether the last major broke the plugin
910
- * interface, do not widen it.
962
+ * A plain object rather than a Map: `Identity` is JSON-serialised into the
963
+ * SSE stream ticket, and a Map would arrive as `{}` on the far side.
911
964
  *
912
- * Enforced at boot: a plugin whose range excludes the running binary is
913
- * refused.
965
+ * A bare `{ id }` with no grant (direct-call tests, system actors) is the
966
+ * no-claim case: `grantOf` resolves it to `null`, and the engine's grant
967
+ * gate then admits every namespace. An authenticator for remote callers
968
+ * must therefore always set `grants`, `{}` for none.
914
969
  */
915
- contractVersion: string;
970
+ grants: Record<string, string[]>;
971
+ [k: string]: unknown;
972
+ }
973
+ /**
974
+ * Semantic verb stamped on autoCRUD-registered mutation entries.
975
+ * Absent on type entries and custom mutations — policies needing those use
976
+ * `RequestContext.entry` / `operation` instead.
977
+ */
978
+ type AutoCrudVerb = 'get' | 'create' | 'update' | 'delete';
979
+ interface RequestContext {
980
+ identity: Identity | null;
981
+ namespace: string;
982
+ operation: 'query' | 'mutate' | 'subscribe' | 'stream';
983
+ now: Date;
984
+ headers: Record<string, string>;
985
+ meta: Record<string, unknown>;
916
986
  /**
917
- * Plugin-global policies applied to every resolver in every namespace the
918
- * plugin contributes. Composed by the loader via `and(...)` — the combined
919
- * policy must allow. Useful for cross-cutting concerns (audit logging,
920
- * rate limits, identity enrichment) that span multiple namespaces in the
921
- * same plugin. The composed result is surfaced through
922
- * `NamespaceInstallContext.pluginPolicy`.
987
+ * Trust-gated client IP computed by the HTTP route handler from the request's
988
+ * `peerIp` + the operator-configured trusted-proxy allowlist. When the peer
989
+ * is in the allowlist, this is the leftmost `x-forwarded-for` entry; when
990
+ * untrusted, this is the peer IP itself. Undefined when the transport doesn't
991
+ * expose peerIp (direct engine unit tests, pre-upgrade WS paths).
923
992
  */
924
- policies?: Policy[];
993
+ clientIp?: string;
925
994
  /**
926
- * Post-register lifecycle hook. Fires after every plugin's `install()`
927
- * resolves — use this for resource priming that needs the full namespace
928
- * map (e.g. cross-plugin edge warmup).
995
+ * Whether the deployment verifies credentials at all, as the transport knows
996
+ * it, so a refusal for want of an identity can say whether auth is off
997
+ * rather than implying the caller's key is wrong.
998
+ *
999
+ * `undefined` means the transport did not say (direct engine unit tests,
1000
+ * routes with no auth config in scope). Treat it as unknown, never as `false`.
929
1001
  */
930
- onStart?(ctx: PluginLifecycleContext): void | Promise<void>;
1002
+ authEnabled?: boolean;
931
1003
  /**
932
- * Pre-shutdown lifecycle hook. Fires when the host calls `compose()`'s
933
- * returned `stop()` (which the binary's SIGTERM handler invokes inside
934
- * the `SUBSTRATE_SHUTDOWN_GRACE_MS` window). Hooks run in REVERSE
935
- * registration order so plugins tear down in the inverse of the order
936
- * they came up — late-loading plugins that depended on earlier ones get
937
- * to release their handles first.
938
- *
939
- * Use for resources the OS won't reap on process exit:
940
- * - subprocess children that should receive SIGTERM cleanly
941
- * - long-lived `setInterval` timers (`.unref()` keeps them from
942
- * blocking exit, but explicit teardown is still cleaner for tests)
943
- * - WebSocket subscriptions / outbound connections that should close
944
- * cooperatively rather than dropping mid-request
945
- *
946
- * Errors are caught + logged via the lifecycle metric sink under
947
- * `plugin_on_stop_error` — one plugin's failed teardown does NOT abort
948
- * other plugins' teardown. The engine + pg pool are still live during
949
- * onStop calls, so plugins can do final writes (e.g. an audit row
950
- * recording the shutdown). After all onStop hooks resolve, the server
951
- * stops accepting connections and the substrate's `closeStore()` runs.
952
- *
953
- * Symmetric to `onStart`. Skipped (no-op) when omitted.
1004
+ * The registry entry name this dispatch targets — the `<name>` half of the
1005
+ * qualified frame key (e.g. `deleteTodo`, `Todo`). Threaded by the engine
1006
+ * at every dispatch site (query/mutate/stream/subscribe) so type- and
1007
+ * plugin-level policies can discriminate between resolvers that otherwise
1008
+ * present identically (`get<T>` and `delete<T>` both arrive as
1009
+ * operation:'mutate' with id-only params). Undefined only in direct-call
1010
+ * unit tests that construct a bare ctx.
954
1011
  */
955
- onStop?(ctx: PluginLifecycleContext): void | Promise<void>;
1012
+ entry?: string;
1013
+ /**
1014
+ * Semantic verb of the targeted entry when autoCRUD registered it.
1015
+ * Prefer this over `entry`-name matching for intent-level policies
1016
+ * ("deny deletes") — it survives entry renames. Absent for type entries
1017
+ * and custom mutations.
1018
+ */
1019
+ verb?: AutoCrudVerb;
1020
+ }
1021
+ /** The caller's request context for a policy check made outside dispatch. */
1022
+ type PolicyProbeContext = Partial<Pick<RequestContext, 'now' | 'headers' | 'meta' | 'clientIp' | 'authEnabled'>>;
1023
+ interface ResolverArgs {
1024
+ params?: Record<string, unknown>;
1025
+ fields?: Record<string, unknown>;
1026
+ rows?: Row[];
1027
+ /**
1028
+ * Engine-injected per-call dataset handle. On `engine.mutate`, this is the
1029
+ * transaction-scoped `tx` view (spec §5.3, G1); on `engine.query` it's the
1030
+ * plain namespace dataset. Resolvers that perform I/O MUST use this handle
1031
+ * — using a dataset captured at registration time bypasses the active
1032
+ * transaction and breaks atomicity. Optional purely to keep direct-resolver
1033
+ * unit tests concise; the engine always supplies it.
1034
+ */
1035
+ dataset?: Dataset;
1036
+ /**
1037
+ * Engine-injected per-call edges store (§4.6). Resolvers that touch edges
1038
+ * MUST use this handle; the closed-over store from registration is a
1039
+ * fallback for direct-call tests. Unlike `dataset`, this is not namespace-
1040
+ * scoped — edges are inherently cross-namespace, and a single engine-scoped
1041
+ * `EdgesStore` is threaded through every call.
1042
+ */
1043
+ edges?: EdgesStore;
1044
+ /**
1045
+ * Engine-injected post-commit write hook (§5.4). Write-path resolvers
1046
+ * (autoCRUD create/update/delete, or custom mutation resolvers) call this
1047
+ * AFTER their substrate write lands to announce the change. The engine
1048
+ * collects calls into a per-transaction batch and fans them out to the
1049
+ * subscription bus only after the enclosing `withTransaction` commits — a
1050
+ * rolled-back tx emits nothing. Direct-call unit tests may pass `onWrite` to
1051
+ * observe resolver output without spinning up an engine.
1052
+ */
1053
+ onWrite?: (event: WriteEvent) => void;
1054
+ /**
1055
+ * Queue work to run once the surrounding transaction has settled, whether it
1056
+ * committed or rolled back: a record that must outlive a rollback, such as
1057
+ * an audit row for the refusal that caused it. The work is told which
1058
+ * (`committed`), so post-commit work can skip a rollback. Absent outside a
1059
+ * transaction, where the caller can simply do the work.
1060
+ */
1061
+ afterTx?: (fn: AfterTxWork) => void;
956
1062
  /**
957
- * Scheduled tasks — the binary ticks these on `.unref()`'d intervals.
958
- * Registered with the shared `TaskScheduler` at compose time and started
959
- * after the server's `start()` resolves.
1063
+ * Engine-injected cancellation signal (§5.5). Supplied on `stream` dispatch
1064
+ * so generator resolvers can abort mid-iteration when the client cancels or
1065
+ * the socket closes. Streaming resolvers SHOULD observe this signal — either
1066
+ * by calling `signal.throwIfAborted()` at yield points, or by listening for
1067
+ * `abort` to release cursors / close descriptors in their `finally` block.
1068
+ * The engine cannot force-stop a generator; cooperation is the contract.
1069
+ * Absent on query/mutate dispatch — non-streaming resolvers should not see a
1070
+ * signal they have no way to honour.
960
1071
  */
961
- schedule?: ScheduledTask[];
1072
+ signal?: AbortSignal;
962
1073
  /**
963
- * Mount plugin-scoped HTTP/WS routes. Called once at compose-time with a
964
- * `PluginRouter` pre-prefixed `/substrate/v1/plugins/<name>` by the server impl. The
965
- * router surface matches `PluginRouter` in `./server.ts`.
966
- *
967
- * With auth on, HTTP routes require `read` for GET and `mutate` otherwise,
968
- * granted in one of this plugin's `namespaces`; a plugin with no namespaces
969
- * accepts the capability granted in any namespace.
1074
+ * Engine-injected cross-namespace dataset resolver (committed, non-tx view).
1075
+ * Write-path resolvers use it to verify crossRef referents that live in a
1076
+ * DIFFERENT namespace (onMissing:'reject'). Same-namespace targets should use
1077
+ * the tx-scoped `dataset` handle instead. Absent on the query/subscribe path.
970
1078
  */
971
- routes?(router: PluginRouter, ctx: PluginRouteContext): void | Promise<void>;
1079
+ resolveDataset?: (ns: string) => Dataset | undefined;
972
1080
  /**
973
- * Optional onboarding contribution. When set, the hosting binary's
974
- * `GET /substrate/v1/onboard` aggregates this record under
975
- * `instructions.plugins.<name>` and enumerates the plugin's conventional
976
- * `/substrate/v1/plugins/<name>/onboard` route under `instructions.pluginRoutes`. Use
977
- * this to teach agents about the plugin's types, write flows, or any
978
- * plugin-specific grammar a cold-start agent would otherwise have to
979
- * discover per endpoint. The shape is free-form — the binary round-trips
980
- * it verbatim.
1081
+ * Engine-injected on mutate: how many frame entries share this call's
1082
+ * transaction, itself included. A resolver whose effect waits on other
1083
+ * transactions (a concurrent index drop) refuses to share one.
981
1084
  */
982
- onboardInstructions?: Record<string, unknown>;
1085
+ txEntries?: number;
983
1086
  }
984
1087
  /**
985
- * Substrate-tier plugin. Augments the kernel rather than the agent-facing
986
- * surface. Composes BEFORE coordinator plugins so coordinators can rely on
987
- * substrate registrations being live.
1088
+ * Post-commit write announcement consumed by the engine's subscription bus
1089
+ * (§5.4). Emitted by write-path resolvers after the substrate operation
1090
+ * commits inside the current transaction. `row` is the post-commit canonical
1091
+ * row for creates and updates; for deletes it's the pre-tombstone snapshot
1092
+ * (absent if the resolver couldn't read the row before deleting). The engine
1093
+ * filters matching subscriptions by `(ns, type)`, tests the `row` against each
1094
+ * subscription's `where` predicate, and emits `add` / `update` / `remove`.
1095
+ */
1096
+ interface WriteEvent {
1097
+ ns: string;
1098
+ type: string;
1099
+ id: Row['id'];
1100
+ operation: 'create' | 'update' | 'delete';
1101
+ row?: Row;
1102
+ }
1103
+ interface Obligation {
1104
+ kind: string;
1105
+ [k: string]: unknown;
1106
+ }
1107
+ /**
1108
+ * Row-filter obligation: emitted by `row<T>(predicate)` policies. The engine
1109
+ * scans accumulated obligations after resolver execution and applies each
1110
+ * predicate as a silent filter.
1111
+ */
1112
+ interface RowFilterObligation<T = unknown> extends Obligation {
1113
+ kind: 'row-filter';
1114
+ predicate: (ctx: RequestContext, row: T) => boolean;
1115
+ }
1116
+ interface PolicyDecision {
1117
+ allow: boolean;
1118
+ reason?: string;
1119
+ obligations?: Obligation[];
1120
+ }
1121
+ type Policy = (ctx: RequestContext, args: ResolverArgs) => PolicyDecision;
1122
+ /**
1123
+ * Engine-facing resolver shape. Spec: §4.3, §5.2, §5.5.
988
1124
  *
989
- * SubstratePlugins do NOT own data namespaces — that's a CoordinatorPlugin
990
- * concern (a substrate may claim a virtual `kernelNamespace`).
1125
+ * The engine treats any resolver's return as opaque rows. A `ReadResult` return
1126
+ * signals a paged list; the engine unpacks `rows` into the envelope and copies
1127
+ * `nextCursor`/`hasMore` into `_pagination[key]`. An `AsyncIterable<Row>`
1128
+ * return is the §5.5 stream-dispatch shape (duck-typed via
1129
+ * `Symbol.asyncIterator` so it can't collide with `Row[]`).
1130
+ *
1131
+ * This is what the registry stores and what the engine dispatches to. Plugin
1132
+ * authors declaring custom resolvers use the typed-authoring form
1133
+ * `TypedResolver<T>` below; the engine erases to `Resolver` at registration
1134
+ * time.
991
1135
  */
992
- interface KernelPlugin {
993
- /** Unique plugin name. Used in metric labels + collision detection. */
994
- name: string;
1136
+ type Resolver = (ctx: RequestContext, args: ResolverArgs) => Promise<Row | Row[] | ReadResult | AsyncIterable<Row> | null> | AsyncIterable<Row>;
1137
+ /**
1138
+ * Typed-authoring resolver shape. Used by autoCRUD and hand-authored resolvers
1139
+ * that want to preserve the concrete return type before erasing to `Resolver`
1140
+ * at registration. `T` is unconstrained at the type level — the engine
1141
+ * discriminates on the runtime value shape regardless of static type.
1142
+ */
1143
+ type TypedResolver<T> = (ctx: RequestContext, args: ResolverArgs) => Promise<T>;
1144
+ /**
1145
+ * Engine-visible return shape for a resolver. `ReadResult` means "paged list";
1146
+ * `Row[]` means "full list, no pagination"; `Row` means "single"; `null` means
1147
+ * "single, not found". The engine discriminates at runtime.
1148
+ */
1149
+ type ResolverReturn = Row | Row[] | ReadResult | null;
1150
+ /** Work queued through `ResolverArgs.afterTx`, told whether the transaction committed. */
1151
+ type AfterTxWork = (outcome: {
1152
+ committed: boolean;
1153
+ }) => Promise<void>;
1154
+ //#endregion
1155
+ //#region ../../packages/substrate-types/src/server.d.ts
1156
+ type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
1157
+ interface SubstrateRequest {
1158
+ method: HttpMethod;
1159
+ path: string;
1160
+ headers: Record<string, string>;
1161
+ query: Record<string, string>;
1162
+ params: Record<string, string>;
1163
+ body: Buffer | null;
1164
+ identity?: Identity;
995
1165
  /**
996
- * Plugin's bound contract version: a semver RANGE over the binary version,
997
- * e.g. `'>=4.1.0 <6.0.0'`. Required on substrate plugins. Checked by
998
- * `composeFromManifests` and REFUSED at boot on mismatch; the
999
- * ceiling tracks actual compatibility rather than the next major boundary.
1000
- * See the note on `CoordinatorPlugin.contractVersion` for both.
1166
+ * Remote peer IP observed at the transport layer. Populated by adapters
1167
+ * (Fastify: `request.ip`; memory: caller supplies via `inject({ peerIp })`).
1168
+ * Absent when transport doesn't expose it. Consumers that gate on peer IP
1169
+ * (e.g. `x-forwarded-for` trust) treat `undefined` as "untrusted peer".
1001
1170
  */
1002
- contractVersion: string;
1171
+ peerIp?: string;
1003
1172
  /**
1004
- * Schema migrations the plugin contributes. Applied in declared order
1005
- * on first install AND on every binary boot (idempotently, gated by
1006
- * the migrations tracking table seeded by the first KernelPlugin
1007
- * to populate this slot).
1173
+ * Client IP after the trusted-proxy check, stamped by the host before plugin
1174
+ * and kernel route handlers run. Never read from the client.
1008
1175
  */
1009
- migrations?: KernelMigration[];
1176
+ clientIp?: string;
1177
+ /** Whether the deployment verifies credentials, stamped with `clientIp`. */
1178
+ authEnabled?: boolean;
1010
1179
  /**
1011
- * Names of the tables (in the store's schema) this plugin keeps data in.
1012
- * `substrate-admin migrate-store` copies them alongside the engine's own
1013
- * tables and refuses a source table that nothing declares.
1180
+ * Per-request abort signal. Production transports (Fastify) wire it to fire
1181
+ * on socket close (client disconnect) and on `abortAllRequests()` during
1182
+ * graceful shutdown. Route handlers thread it into `engine.query/mutate`
1183
+ * via `signal:` so resolvers can cooperatively bail. Optional because
1184
+ * test fixtures and embedded `inject()` callers may not need cancellation.
1014
1185
  */
1015
- storeTables?: readonly string[];
1186
+ signal?: AbortSignal;
1187
+ }
1188
+ interface SubstrateResponse {
1189
+ status: number;
1190
+ headers?: Record<string, string>;
1191
+ body?: Buffer | Uint8Array | string | ReadableStream;
1192
+ }
1193
+ type HttpHandler = (req: SubstrateRequest) => SubstrateResponse | Promise<SubstrateResponse>;
1194
+ /** Request op classes counted by the runtime-metrics surface (§ engine-metrics spec). */
1195
+ type RuntimeOp = 'query' | 'mutate';
1196
+ /** One observed HTTP request at the transport seam. */
1197
+ interface RequestObservation {
1198
+ readonly op: RuntimeOp;
1199
+ /** Transport-level success — HTTP status < 400. Frame-level envelope errors still count ok. */
1200
+ readonly ok: boolean;
1201
+ /** Wall-clock elapsed from handler entry to response, measured via Date.now(). Always >= 0. */
1202
+ readonly durationMs: number;
1203
+ }
1204
+ type RequestObserver = (o: RequestObservation) => void;
1205
+ interface WsConnection {
1206
+ readonly id: string;
1207
+ send(frame: Buffer | Uint8Array | string): void;
1016
1208
  /**
1017
- * Filter operators contributed to the where-clause grammar. Each entry
1018
- * registers a custom operator (e.g. `embeddingSimilar`) the engine's
1019
- * query validator + substrate executor can dispatch to.
1209
+ * Transport-level back-pressure seam. Resolves once the
1210
+ * socket's write buffer is below its high-water mark — immediately when it
1211
+ * already is. The stream route awaits this between chunks so a fast producer
1212
+ * paces to a slow consumer instead of growing the send buffer unbounded.
1213
+ * Optional: transports without buffer introspection (memory tests) omit it,
1214
+ * and the route treats a missing hook as "always drained".
1020
1215
  */
1021
- filterOperators?: Record<string, FilterOperatorDef>;
1216
+ drain?(): Promise<void>;
1217
+ onMessage(cb: (frame: Buffer) => void): void;
1218
+ onClose(cb: (code?: number) => void): void;
1219
+ close(code?: number, reason?: string): void;
1022
1220
  /**
1023
- * Top-level kernel route claims. Plugin reserves a path under /substrate/v1/.
1024
- * First-claim wins; collision = boot refusal with actionable error.
1221
+ * Register a callback invoked synchronously during graceful server shutdown,
1222
+ * BEFORE the underlying socket is closed. Higher layers use this to flush
1223
+ * structured frames (e.g. `{ kind: 'end', reason: 'server_restart' }` per
1224
+ * active subscription) so clients can short-circuit reconnect backoff
1225
+ * instead of seeing an abrupt 1006 drop. Optional — transports without a
1226
+ * pre-close hook (memory tests) may omit it; the route layer should
1227
+ * tolerate undefined.
1228
+ */
1229
+ onShutdown?(cb: (reason: string) => void): void;
1230
+ }
1231
+ type WsHandler = (req: SubstrateRequest, conn: WsConnection) => void | Promise<void>;
1232
+ interface TlsConfig {
1233
+ cert: string | Buffer;
1234
+ key: string | Buffer;
1235
+ ca?: string | Buffer;
1236
+ }
1237
+ interface StartOpts {
1238
+ host?: string;
1239
+ port?: number;
1240
+ tls?: TlsConfig;
1241
+ }
1242
+ interface PluginRouter {
1243
+ get(path: string, handler: HttpHandler): void;
1244
+ post(path: string, handler: HttpHandler): void;
1245
+ put(path: string, handler: HttpHandler): void;
1246
+ delete(path: string, handler: HttpHandler): void;
1247
+ patch(path: string, handler: HttpHandler): void;
1248
+ websocket(path: string, handler: WsHandler): void;
1249
+ }
1250
+ interface SubstrateServer {
1251
+ start(opts?: StartOpts): Promise<{
1252
+ address: string;
1253
+ port: number;
1254
+ }>;
1255
+ stop(opts?: {
1256
+ grace?: number;
1257
+ abortInFlight?: boolean;
1258
+ }): Promise<void>;
1259
+ route(method: HttpMethod, path: string, handler: HttpHandler): void;
1260
+ websocket(path: string, handler: WsHandler): void;
1261
+ onError?(hook: (err: Error, req?: SubstrateRequest) => void): void;
1262
+ createPluginRouter(plugin: {
1263
+ name: string;
1264
+ }): PluginRouter;
1265
+ /**
1266
+ * Abort every in-flight HTTP request's `req.signal` so resolvers cooperatively
1267
+ * bail. Used during graceful shutdown to give the engine a chance to roll
1268
+ * back mutations before fastify's drain timeout fires. WS connections are
1269
+ * not affected by this call; their teardown rides on socket close.
1270
+ */
1271
+ abortAllRequests(reason?: string): void;
1272
+ /**
1273
+ * Register a callback invoked synchronously at the START of `stop()` —
1274
+ * before in-flight requests are aborted and before the transport drains.
1275
+ *
1276
+ * The HTTP analogue of `WsConnection.onShutdown`, for routes that hold
1277
+ * a response open indefinitely (SSE): handlers flush a terminal frame and end
1278
+ * their response, or the drain would wait forever.
1279
+ *
1280
+ * Additive, not last-one-wins (unlike `onError`): a dropped registration is
1281
+ * a leaked stream. Optional — transports with no graceful phase may omit it,
1282
+ * and callers must tolerate undefined.
1283
+ */
1284
+ onShutdown?(cb: (reason: string) => void): void;
1285
+ /**
1286
+ * Test-only: invoke the server's handler pipeline without opening a socket.
1287
+ * Networked implementations may leave this undefined.
1025
1288
  */
1026
- kernelRoutes?: KernelRouteClaim[];
1289
+ inject?(req: Partial<SubstrateRequest>): Promise<SubstrateResponse>;
1027
1290
  /**
1028
- * Top-level MCP tool registrations. Tool name registers globally with
1029
- * the MCP server (no /plugins/ prefix on MCP — tools are flat).
1291
+ * Register a per-request observer for op throughput + latency. Optional —
1292
+ * transports without instrumentation may omit it. At most one observer; a
1293
+ * second call replaces the first. The transport invokes it only for the
1294
+ * `/substrate/v1/query` and `/substrate/v1/mutate` routes.
1030
1295
  */
1031
- mcpTools?: McpToolRegistration[];
1296
+ observeRequests?(observer: RequestObserver): void;
1297
+ /** Optional: told when a query/mutate HTTP request starts (+1) and ends (-1), including when it throws. */
1298
+ observeInFlight?(fn: (op: RuntimeOp, delta: 1 | -1) => void): void;
1299
+ }
1300
+ //#endregion
1301
+ //#region ../../packages/substrate-types/src/plugin.d.ts
1302
+ /** Whether a registry entry is a top-level addressable type or a mutation. */
1303
+ type RegistryEntryKind = 'type' | 'mutation';
1304
+ /**
1305
+ * postWrite hook signature. Receives the written row and request context;
1306
+ * may perform side-effects (e.g. embedding, index updates). Returned promise
1307
+ * is awaited before the engine acks the write to the caller.
1308
+ */
1309
+ type PostWriteHook = (row: Row, ctx: RequestContext) => Promise<void>;
1310
+ /**
1311
+ * Resolver signature stored in a `RegistryEntry`. Identical to the canonical
1312
+ * engine-facing `Resolver` in `./policy.ts`; redeclared inline here to keep
1313
+ * the registry surface self-describing for the types package. Importers that
1314
+ * want the canonical alias should import `Resolver` from this package.
1315
+ */
1316
+ type RegistryEntryResolver = (ctx: RequestContext, args: ResolverArgs) => Promise<Row | Row[] | ReadResult | AsyncIterable<Row> | null> | AsyncIterable<Row>;
1317
+ /** Input shape for `Registry.register` / `Registry.replace`. */
1318
+ interface RegistryEntryInput {
1319
+ resolver: RegistryEntryResolver;
1320
+ policy?: Policy;
1321
+ fields?: Record<string, FieldDef>;
1322
+ kind?: RegistryEntryKind;
1323
+ params?: Record<string, FieldDef>;
1324
+ returns?: FieldDef;
1325
+ /** Natural order of a `kind: 'type'` entry; published on its `TypeSchema`. */
1326
+ orderedBy?: OrderedBy;
1327
+ /** Declared indexes of a `kind: 'type'` entry installed at runtime (a proposed type). */
1328
+ indexes?: IndexDecl[];
1329
+ /** A proposed type's size hint; `large` lowers the index advisor's row minimum. */
1330
+ expectedSize?: 'small' | 'large';
1032
1331
  /**
1033
- * Frame-surface contribution. Claims one namespace in the engine registry
1034
- * carrying resolver-backed query types + mutations — no dataset, no
1035
- * autoCRUD. Dispatch/policy/idempotency/pack/schema/NL/MCP all inherit from
1036
- * the normal frame path.
1332
+ * Semantic verb for autoCRUD-registered mutations. The engine
1333
+ * threads this into `RequestContext.verb` at dispatch so type-/plugin-level
1334
+ * policies can discriminate get/create/update/delete without entry-name
1335
+ * string matching. Omit for type entries and custom mutations.
1037
1336
  */
1038
- kernelNamespace?: KernelNamespaceContribution;
1337
+ verb?: AutoCrudVerb;
1338
+ }
1339
+ /**
1340
+ * Stored shape returned by `Registry.get` and `Registry.all`. Mirrors the
1341
+ * concrete `RegistryEntry` in `@mnmal-ai/substrate-core` — the engine reads
1342
+ * `policy`, `kind`, `resolver`, `fields`, `params`, and `returns` off this
1343
+ * shape on every dispatch.
1344
+ */
1345
+ interface RegistryEntry {
1346
+ resolver: RegistryEntryResolver;
1347
+ policy: Policy;
1348
+ fields: Record<string, FieldDef>;
1349
+ kind: RegistryEntryKind;
1350
+ /** Always present — `{}` for `kind: 'type'` entries where the concept doesn't apply. */
1351
+ params: Record<string, FieldDef>;
1352
+ /** Present only for `kind: 'mutation'` entries that declared a return shape. */
1353
+ returns?: FieldDef;
1354
+ /** Mirrors `RegistryEntryInput.orderedBy`. */
1355
+ orderedBy?: OrderedBy;
1356
+ /** Mirrors `RegistryEntryInput.indexes`. */
1357
+ indexes?: IndexDecl[];
1358
+ /** Mirrors `RegistryEntryInput.expectedSize`. */
1359
+ expectedSize?: 'small' | 'large';
1360
+ /** Mirrors `RegistryEntryInput.verb` — present when the registrant stamped one. */
1361
+ verb?: AutoCrudVerb;
1362
+ }
1363
+ /**
1364
+ * Canonical per-namespace registry surface, implemented by the concrete
1365
+ * `Registry` class in `@mnmal-ai/substrate-core`.
1366
+ */
1367
+ interface Registry {
1368
+ register(key: string | {
1369
+ readonly name: string;
1370
+ }, input: RegistryEntryInput): void;
1371
+ replace(key: string | {
1372
+ readonly name: string;
1373
+ }, input: RegistryEntryInput): void;
1039
1374
  /**
1040
- * Post-register lifecycle hook. Fires after every KernelPlugin's
1041
- * migrations + registrations resolve, BEFORE any coordinator plugin
1042
- * composes. Use for resource priming the kernel needs before
1043
- * coordinators arrive.
1375
+ * Read back a registered entry. Returns undefined when the entry is absent.
1376
+ * The full `RegistryEntry` shape (resolver, policy, kind, fields, params,
1377
+ * returns) is exposed because the engine reads every field on dispatch.
1044
1378
  */
1045
- onKernelReady?(ctx: KernelLifecycleContext): void | Promise<void>;
1379
+ get(key: string | {
1380
+ readonly name: string;
1381
+ }): RegistryEntry | undefined;
1382
+ /** Presence check. Returns true iff `key` is registered. */
1383
+ has(key: string | {
1384
+ readonly name: string;
1385
+ }): boolean;
1386
+ /** Iterate every `[name, entry]` pair. Used by the schema publisher. */
1387
+ all(): IterableIterator<[string, RegistryEntry]>;
1046
1388
  /**
1047
- * Stops whatever `onKernelReady` started (timers, workers). The host calls
1048
- * it before it closes the store, and may call it more than once.
1389
+ * Remove an entry. Optional: only a registry that can take back a live
1390
+ * apply (a schema proposal rolled back with its transaction) offers it.
1049
1391
  */
1050
- onKernelStop?(): void | Promise<void>;
1392
+ unregister?(key: string | {
1393
+ readonly name: string;
1394
+ }): void;
1051
1395
  /**
1052
- * Contribute postWrite hooks for a coordinator namespace at install time.
1053
- * Called by the engine ONCE per registered coordinator namespace, AFTER
1054
- * substrate composition completes. Returned hooks are merged into the
1055
- * type's effective postWrite chain alongside the type's own inline
1056
- * `postWrite` (if any). Substrate-supplied hooks run AFTER the type's
1057
- * inline hook in chain order.
1058
- *
1059
- * Use this to inspect `namespace.types[T].embed` and emit embedding
1060
- * hooks per type without coupling the binary to plugin internals.
1396
+ * Monotonic mutation counter — incremented by every `register` / `replace` / `unregister`.
1397
+ * Lets a consumer cache anything derived from the registry (the published
1398
+ * schema, above all) and detect staleness from the data, without relying on
1399
+ * every registrant remembering to call `engine.invalidateSchemaCaches`.
1061
1400
  *
1062
- * Returns: a record mapping type names to PostWriteHook contributions.
1063
- * Type names not in the record receive no contribution.
1401
+ * Optional so third-party `Registry` implementations remain valid; consumers
1402
+ * fall back to callback-driven invalidation when it is absent.
1064
1403
  */
1065
- contributePostWrites?: (namespace: NamespaceDefinition) => Record<string, PostWriteHook>;
1404
+ readonly epoch?: number | undefined;
1066
1405
  }
1067
- /** Single migration unit. */
1068
- interface KernelMigration {
1069
- /** Unique id within the plugin. Stored in migrations tracking table. */
1070
- id: string;
1071
- /** Apply migration. Called only when this migration's id is absent from the migrations table. */
1072
- apply(ctx: KernelMigrationContext): Promise<void>;
1073
- /** Inverse — drops what apply created. */
1074
- rollback?(ctx: KernelMigrationContext): Promise<void>;
1406
+ /**
1407
+ * Base class binding the `epoch` bookkeeping so an implementation cannot forget
1408
+ * it. Subclasses supply storage via `performRegister` / `performReplace` and
1409
+ * never implement `register` / `replace` themselves, so the counter is advanced
1410
+ * by construction rather than by discipline.
1411
+ *
1412
+ * A throw from `performRegister` leaves the epoch untouched, which is correct:
1413
+ * a rejected registration changed nothing.
1414
+ *
1415
+ * Decorators cannot extend this — they wrap an instance rather than subclass it
1416
+ * — so a wrapping registry must forward `epoch` from the wrapped instance. See
1417
+ * `substrate-compose`'s `_system` install for the worked example.
1418
+ */
1419
+ declare abstract class RegistryBase implements Registry {
1420
+ #private;
1421
+ get epoch(): number;
1422
+ register(key: string | {
1423
+ readonly name: string;
1424
+ }, input: RegistryEntryInput): void;
1425
+ replace(key: string | {
1426
+ readonly name: string;
1427
+ }, input: RegistryEntryInput): void;
1428
+ unregister(key: string | {
1429
+ readonly name: string;
1430
+ }): void;
1431
+ /** Store a new entry. Throw to reject; the epoch will not advance. */
1432
+ protected abstract performRegister(key: string | {
1433
+ readonly name: string;
1434
+ }, input: RegistryEntryInput): void;
1435
+ /** Overwrite an existing entry. Throw to reject; the epoch will not advance. */
1436
+ protected abstract performReplace(key: string | {
1437
+ readonly name: string;
1438
+ }, input: RegistryEntryInput): void;
1439
+ /** Remove an entry. Not abstract, so existing subclasses stay valid; they refuse. */
1440
+ protected performUnregister(key: string | {
1441
+ readonly name: string;
1442
+ }): void;
1443
+ abstract get(key: string | {
1444
+ readonly name: string;
1445
+ }): RegistryEntry | undefined;
1446
+ abstract has(key: string | {
1447
+ readonly name: string;
1448
+ }): boolean;
1449
+ abstract all(): IterableIterator<[string, RegistryEntry]>;
1075
1450
  }
1076
- interface KernelMigrationContext {
1451
+ /** See `EngineHandle.addDropParticipant`. */
1452
+ interface DropParticipant {
1453
+ /** Namespaces whose datasets `onDrop` writes, opened in the drop's transaction. */
1454
+ readonly namespaces: readonly string[];
1455
+ /** `datasetFor` returns the transaction-bound dataset of one of `namespaces`. */
1456
+ onDrop(ns: string, datasetFor: (ns: string) => Dataset): Promise<void>;
1457
+ }
1458
+ interface NamespaceInstallContext {
1459
+ registry: Registry;
1460
+ dataset: Dataset;
1461
+ edges: EdgesStore;
1077
1462
  /**
1078
- * Direct SQL handle for postgres-backed deployments; null when the
1079
- * binary is running against the in-memory dataset.
1463
+ * Plugin-global policy composed from the owning plugin's `policies` array
1464
+ * (undefined when the plugin declares none). Namespace install
1465
+ * implementations SHOULD compose this with any namespace-scoped or per-type
1466
+ * policies via `and()` so plugin-level invariants apply across every
1467
+ * resolver the namespace contributes. `defineNamespace` in
1468
+ * `@mnmal-ai/substrate-sdk` does this automatically.
1080
1469
  */
1081
- sql: unknown | null;
1470
+ pluginPolicy?: Policy;
1082
1471
  /**
1083
- * How `rows` is stored: `'partitioned'` (LIST(namespace) then LIST(type)) or
1084
- * `'legacy'` (one plain table). `null` on the in-memory dataset. PG refuses
1085
- * a unique index on a partitioned parent unless it includes every partition
1086
- * key column (`namespace`, `type`), so such DDL on `rows` must branch on this.
1472
+ * Substrate-plugin contributions to per-type postWrite chains. Map
1473
+ * `typeName → PostWriteHook[]`. Engine populates by walking each
1474
+ * composed substrate plugin's `contributePostWrites(namespace)` and
1475
+ * merging results. Hooks run AFTER the type's inline postWrite in
1476
+ * declaration order. Empty map / undefined = no contributions.
1087
1477
  */
1088
- layout: 'legacy' | 'partitioned' | null;
1089
- metric(name: string, payload?: unknown): void;
1478
+ postWriteContributions?: Record<string, PostWriteHook[]>;
1090
1479
  }
1091
1480
  /**
1092
- * Where-clause operator extension.
1093
- *
1094
- * The operator's name is the Record key it's registered under in
1095
- * `KernelPlugin.filterOperators`.
1481
+ * Read-side context passed to `NamespaceTypeDef.embed.source` when the
1482
+ * substrate composing the embedding (e.g. substrate-recall) has a dataset
1483
+ * resolver wired. Lets the source function pull related-entity context
1484
+ * via crossRefs (e.g. inline a linked Concept's name into the embedded
1485
+ * text so semantic recall surfaces relevant rows).
1096
1486
  *
1097
- * `execute` is awaited by the pre-resolve pass (`@mnmal-ai/substrate-filter`'s
1098
- * `preResolveExtensionOperatorsDetailed`) and must resolve to an ordered,
1099
- * scored `{ hits: [{ id: string; score: number; vector?: number[] }, ...] }`
1100
- * envelope; any other shape throws `ExtensionOperatorResultError`. Relevance
1101
- * order is preserved in the rewritten `id IN`, and score/vector are surfaced
1102
- * via the pre-pass meta side-channel for the datasets to stitch onto rows.
1487
+ * Minimal by design — `read` is the only method, and it returns the
1488
+ * first matching row or `null`. Source functions that don't need
1489
+ * crossRef enrichment ignore the parameter.
1103
1490
  */
1104
- interface FilterOperatorDef {
1105
- validate(value: unknown): {
1106
- ok: true;
1107
- value: unknown;
1108
- } | {
1109
- ok: false;
1110
- reason: string;
1111
- };
1112
- execute(value: unknown, context: unknown): Promise<unknown>;
1113
- /**
1114
- * Computed projection field names the operator contributes per hit (e.g.
1115
- * `['_score', '_vector']`). The engine refuses a projection of one of these
1116
- * fields unless its operator is in the where-clause.
1117
- */
1118
- computedFields?: string[];
1491
+ interface EmbedSourceContext {
1492
+ read: (namespace: string, type: string, params: {
1493
+ id: string;
1494
+ }) => Promise<Row | null>;
1119
1495
  }
1120
1496
  /**
1121
- * Query-only virtual type contributed by a substrate plugin.
1122
- * `fields` reuses the FieldDef DSL so schema publication + client typing
1123
- * work unchanged. The resolver receives the standard (ctx, args) pair; the
1124
- * engine leaves `args.dataset` undefined for virtual namespaces — resolvers
1125
- * compute from `params`/`fields` and MUST NOT reach for a dataset handle.
1497
+ * A declared index over a type's scalar or crossRef fields, built by the
1498
+ * engine with the same expressions the filter compiler emits.
1499
+ *
1500
+ * - `fields`: key columns in order; `dir` defaults to `asc`. Both directions
1501
+ * sort NULLs last, as `orderBy` does.
1502
+ * - `unique`: refused at build time while duplicate live values exist.
1503
+ * - `where`: always ANDed with "not deleted". `'live'` adds nothing more; a
1504
+ * record adds equality to a boolean (or IS NULL for `null`) on the type's
1505
+ * own fields. Only those operands compile to literals, which a generic plan
1506
+ * needs to match the index predicate.
1507
+ * - `name`: the pg index name to own or adopt; generated when omitted. It may
1508
+ * not end in `__swap`, the reconciler's temporary suffix.
1126
1509
  */
1127
- interface VirtualTypeDef {
1128
- fields: Record<string, FieldDef>;
1129
- resolver: RegistryEntryResolver;
1130
- }
1131
- /** Mutation contributed by a substrate plugin under its virtual namespace. */
1132
- interface VirtualMutationDef {
1133
- resolver: RegistryEntryResolver;
1134
- /** Optional params schema published into the namespace schema. */
1135
- params?: Record<string, FieldDef>;
1510
+ interface IndexDecl {
1511
+ fields: Array<string | {
1512
+ field: string;
1513
+ dir: 'asc' | 'desc';
1514
+ }>;
1515
+ unique?: boolean;
1516
+ where?: 'live' | Record<string, boolean | null>;
1517
+ name?: string;
1136
1518
  }
1137
1519
  /**
1138
- * A substrate plugin's claimed frame surface.
1139
- * Registered into the engine's namespaces map as `{ registry, virtual: true }`.
1140
- * Collision posture mirrors KernelRouteClaim: second claim = boot refusal.
1520
+ * Structural shape of a type definition, mirroring `TypeInput` from
1521
+ * @mnmal-ai/substrate-sdk (redeclared to avoid a substrate-types → substrate-sdk import
1522
+ * cycle). Substrate plugins read it structurally; slots not named here pass
1523
+ * through opaquely.
1141
1524
  */
1142
- interface KernelNamespaceContribution {
1143
- namespace: string;
1144
- types?: Record<string, VirtualTypeDef>;
1145
- mutations?: Record<string, VirtualMutationDef>;
1146
- }
1147
- /** Top-level kernel route reservation. */
1148
- interface KernelRouteClaim {
1149
- path: string;
1150
- method: HttpMethod;
1151
- handler: HttpHandler;
1525
+ interface NamespaceTypeDef {
1526
+ fields: Record<string, FieldDef>;
1527
+ postWrite?: (row: Row, ctx: RequestContext) => Promise<void>;
1528
+ embed?: {
1529
+ /**
1530
+ * Produce the source text fed to the embedder for this row, or `null` /
1531
+ * `undefined` to declare the row un-indexable.
1532
+ *
1533
+ * The null sentinel is a RETRIEVAL POLICY, not an absence of text: it
1534
+ * means "this row exists and must be referenceable, but must never enter
1535
+ * the recall candidate set". The composing substrate both declines to
1536
+ * index the row AND removes any embedding it already has, so a row that
1537
+ * flips to un-indexable after being written stops being recallable
1538
+ * without being deleted.
1539
+ *
1540
+ * `''` is NOT the sentinel — an empty string embeds normally. Returning
1541
+ * `''` to mean "skip" indexes every such row at the same point in the
1542
+ * vector space.
1543
+ *
1544
+ * `ctx` is optional — when the composing substrate has a dataset
1545
+ * resolver wired, source functions may call `ctx.read(ns, type,
1546
+ * { id })` to inline related-entity context. When `ctx` is undefined,
1547
+ * the source must work from `row` alone. Async return is supported
1548
+ * so source functions can await crossRef reads.
1549
+ */
1550
+ source: (row: Row, ctx?: EmbedSourceContext) => string | null | undefined | Promise<string | null | undefined>;
1551
+ };
1152
1552
  /**
1153
- * Capability the host must check before dispatching, when auth is enabled.
1154
- * REQUIRED — `'public'` is how a route opts out, so opting out is a
1155
- * declaration rather than an omission.
1553
+ * Produce the text fed to the LEXICAL (keyword) index for this row, with the
1554
+ * same `null` / `undefined` un-indexable sentinel as `embed.source`.
1156
1555
  *
1157
- * Do not make this optional: kernel routes mount via the host's raw
1158
- * `server.route`, which applies no gate of its own, so a route with no
1159
- * capability would get no auth wrapper at all.
1556
+ * A sibling of `embed` rather than a flag on it, because the two indexes
1557
+ * want different text: `embed.source` conventionally takes a short handle
1558
+ * (`synopsis || body`), which drops proper nouns a keyword index needs.
1559
+ * `lexical.source` is where the long form goes.
1560
+ *
1561
+ * A type may declare either, both, or neither. Both is the expected shape for
1562
+ * a store whose handle is shorter than its content.
1160
1563
  */
1161
- capability: 'read' | 'mutate' | 'admin' | 'public';
1162
- }
1163
- /** Top-level MCP tool registration. */
1164
- interface McpToolRegistration {
1165
- name: string;
1166
- description: string;
1167
- inputSchema: unknown;
1168
- handler(input: unknown): Promise<unknown>;
1564
+ lexical?: {
1565
+ source: (row: Row) => string | null | undefined | Promise<string | null | undefined>;
1566
+ };
1567
+ /** Composite and unique indexes the engine builds for this type. See {@link IndexDecl}. */
1568
+ indexes?: IndexDecl[];
1569
+ /** The type's natural order, the default read order. See {@link OrderedBy}. */
1570
+ orderedBy?: OrderedBy;
1571
+ /** See the SDK's `TypeInput.ordering`. */
1572
+ ordering?: 'visible';
1573
+ /** Size hint: `large` lowers the index advisor's row minimum for this type. */
1574
+ expectedSize?: 'small' | 'large';
1575
+ [key: string]: unknown;
1169
1576
  }
1170
- interface KernelLifecycleContext {
1171
- /**
1172
- * Tagged-union discriminant. Differentiates this context from the
1173
- * structurally-similar `PluginLifecycleContext`, which would otherwise be
1174
- * interchangeable under structural typing.
1175
- */
1176
- readonly lifecycle: 'kernel';
1577
+ interface NamespaceDefinition {
1578
+ namespace: string;
1177
1579
  /**
1178
- * Direct SQL handle; null when the binary runs against the in-memory
1179
- * dataset. Same shape and same null semantics as
1180
- * `KernelMigrationContext.sql`.
1181
- *
1182
- * For convergent, run-every-boot reconciliation: a migration runs only once
1183
- * per `id`, so it cannot re-scope objects it already created.
1580
+ * Type definitions registered under this namespace, keyed by type name.
1581
+ * Read-only metadata exposed for substrate plugins that need to walk
1582
+ * the type surface at coordinator-namespace install time (e.g.
1583
+ * substrate-recall reads `types[T].embed?.source` to contribute embedding
1584
+ * postWrite hooks). Populated by `defineNamespace`; absent on hand-crafted
1585
+ * `NamespaceDefinition` objects that bypass the SDK helper.
1184
1586
  */
1185
- sql: unknown | null;
1587
+ types?: Readonly<Record<string, NamespaceTypeDef>>;
1186
1588
  /**
1187
- * Namespaces DECLARED by the coordinator plugins this binary loaded, in
1188
- * load order. Empty when none are configured.
1189
- *
1190
- * Declared, not installed: coordinators compose AFTER substrate, so the
1191
- * engine has not yet registered them.
1192
- *
1193
- * Read these rather than adding a namespace knob to substrate config, which
1194
- * could silently disagree with what the coordinator uses.
1589
+ * Opaque per-consumer metadata, keyed by consumer (e.g. `nl`). Core never
1590
+ * interprets it; it is passed through to the namespace schema unchanged.
1195
1591
  */
1196
- coordinatorNamespaces: readonly string[];
1592
+ annotations?: Readonly<Record<string, unknown>>;
1197
1593
  /**
1198
- * The same namespaces, attributed to the coordinator that declared each.
1199
- * Declaration order preserved within a plugin.
1200
- *
1201
- * The two absent cases differ and consumers must branch on them:
1202
- * - `undefined` — this host does not supply attribution (e.g. an older
1203
- * binary); do not treat it as an empty list
1204
- * - `[]` — attribution supplied, and there are no coordinators
1205
- *
1206
- * Filtering on `plugin === 'some-name'` makes the coordinator's `name` a
1207
- * contract term: hold it in one constant shared with the coordinator and pin
1208
- * it with a test, or a rename makes the filter silently match nothing.
1594
+ * Body keys a SQL dataset mirrors into dedicated `rows` columns, so filters
1595
+ * and sorts on them hit an index. The declaring plugin creates the columns in
1596
+ * its kernel migrations; the engine creates none. `column` is a bare
1597
+ * identifier.
1209
1598
  */
1210
- coordinatorNamespacesByPlugin?: readonly {
1211
- plugin: string;
1212
- namespace: string;
1213
- }[];
1599
+ promotedColumns?: Readonly<Record<string, {
1600
+ column: string;
1601
+ kind: 'boolean' | 'text';
1602
+ }>>;
1214
1603
  /**
1215
- * Whether this boot may start background work (timers, backfills). False
1216
- * when the host boots the store only to prepare or copy into it, as
1217
- * `substrate-admin migrate-store` does: do convergent reconciliation, start
1218
- * nothing that writes later.
1219
- */
1220
- background: boolean;
1221
- metric(name: string, payload?: unknown): void;
1222
- }
1223
- //#endregion
1224
- //#region ../../packages/substrate-types/src/runtime-metrics.d.ts
1225
- /** Per-key resolver latency and error counts, as served by GET /substrate/v1/metrics. */
1226
- interface ResolverStat {
1227
- kind: 'query' | 'mutate';
1228
- calls: number;
1229
- errors: number;
1230
- p50Ms: number;
1231
- p99Ms: number;
1232
- maxMs: number;
1233
- lastAt: string;
1234
- }
1235
- interface ProcessStat {
1236
- rssMB: number;
1237
- heapUsedMB: number;
1238
- cpuUserMs: number;
1239
- cpuSystemMs: number;
1240
- eventLoopLagP99Ms: number;
1241
- inFlight: number;
1242
- }
1243
- interface SlowRequest {
1244
- key: string;
1245
- kind: 'query' | 'mutate';
1246
- durationMs: number;
1247
- at: string;
1248
- }
1249
- interface RuntimeErrorEntry {
1250
- at: string;
1251
- op: 'query' | 'mutate';
1252
- key: string;
1253
- code: string;
1254
- message: string;
1255
- /** The verified identity's id, or `'anonymous'`. */
1256
- caller?: string;
1257
- /** Present when consecutive identical `(unknown)` failures were merged into this entry. */
1258
- count?: number;
1259
- }
1260
- interface RuntimeErrors {
1261
- since: string;
1262
- errors: RuntimeErrorEntry[];
1263
- }
1264
- interface ResolveObservation {
1265
- key: string;
1266
- kind: 'query' | 'mutate';
1267
- durationMs: number;
1268
- ok: boolean;
1269
- code?: string;
1270
- message?: string;
1271
- /** The verified identity's id; absent for an anonymous caller. */
1272
- caller?: string;
1273
- }
1274
- /**
1275
- * One type's table and index statistics, for the index advisor. Counters are
1276
- * cumulative since the server's statistics were last reset. Row counts are
1277
- * estimates. A type ABSENT from a `typeStats()` result is unknown, not empty:
1278
- * never read its absence as 0 rows (a sampled store can miss a small type).
1279
- */
1280
- interface TypeStats {
1281
- namespace: string;
1282
- type: string;
1283
- liveRows: number;
1284
- insertsTotal: number;
1285
- updatesTotal: number;
1286
- deletesTotal: number;
1287
- indexes: Array<{
1288
- name: string;
1289
- scans: number;
1290
- /** The last scan (ISO), or null when none since the statistics were last reset. */
1291
- lastScanAt: string | null;
1292
- sizeBytes: number;
1293
- unique: boolean;
1294
- /** Built or adopted by the engine's index reconciler for this type. */
1295
- engineOwned: boolean;
1296
- }>;
1604
+ * Install types + mutations + policies into the registry. Invoked once at
1605
+ * registration time, BEFORE the engine is constructed. The plugin owns the
1606
+ * dataset handle it uses for resolver closures; wiring datasets into the
1607
+ * engine's `DatasetRegistry` is the app composition root's job.
1608
+ */
1609
+ install(ctx: NamespaceInstallContext): void | Promise<void>;
1297
1610
  }
1298
- /** What GET /substrate/v1/metrics returns. The new fields are optional so an older server still type-checks. */
1299
- interface RuntimeMetrics {
1300
- since: string;
1301
- requests: {
1302
- query: {
1303
- total: number;
1304
- ok: number;
1305
- error: number;
1306
- durationMsSum: number;
1307
- durationMsMax: number;
1308
- };
1309
- mutate: {
1310
- total: number;
1311
- ok: number;
1312
- error: number;
1313
- durationMsSum: number;
1314
- durationMsMax: number;
1315
- };
1316
- };
1317
- subscriptions: {
1318
- active: number;
1319
- maxQueueDepth?: number;
1320
- entries?: {
1321
- id: string;
1322
- ns: string;
1323
- type: string;
1324
- queueLength: number;
1325
- }[];
1326
- };
1327
- events: Record<string, number>;
1328
- resolvers?: Record<string, ResolverStat>;
1329
- process?: ProcessStat;
1330
- slowest?: SlowRequest[];
1611
+ interface PluginLifecycleContext {
1612
+ /**
1613
+ * Tagged-union discriminant. Differentiates this context from
1614
+ * the structurally-similar `KernelLifecycleContext` so a coordinator's
1615
+ * `onStart` handler is type-incompatible with a substrate's
1616
+ * `onKernelReady` handler — the composition root sets this, and plugins
1617
+ * can also branch on it at runtime if a single helper handles both
1618
+ * lifecycle stages.
1619
+ */
1620
+ readonly lifecycle: 'coordinator';
1621
+ /**
1622
+ * Structured metric/log sink — the hosting binary picks the concrete surface
1623
+ * (stdout JSON, Prometheus counter, OpenTelemetry span, …). Plugins MUST NOT
1624
+ * assume the sink is synchronous or observable; the contract is fire-and-
1625
+ * forget per call.
1626
+ */
1627
+ metric(name: string, payload?: unknown): void;
1628
+ /**
1629
+ * Engine handle threaded into `onStart` so plugins that mutate their
1630
+ * registry post-construction (e.g. substrate-proposals' applyType /
1631
+ * applyField paths) can wire `engine.invalidateSchemaCaches(ns)` into
1632
+ * their replace flow. Engine construction completes before `onStart`
1633
+ * fires, so the handle is always live here.
1634
+ */
1635
+ engine: EngineHandle;
1331
1636
  }
1332
- //#endregion
1333
- //#region ../../packages/substrate-types/src/dataset.d.ts
1334
1637
  /**
1335
- * Handles yielded by `Dataset.withScopedTransaction`. Contains the
1336
- * tx-bound dataset for every requested namespace plus (optionally) a
1337
- * tx-bound edges store enlisted into the same substrate transaction.
1338
- * See `withScopedTransaction` JSDoc for the `edges` contract.
1638
+ * Engine-facing input shape for `query` / `mutate` / `subscribe` / `stream`.
1639
+ * Declared here (rather than in substrate-core) so plugin surfaces can reference
1640
+ * the engine's call shape as a type without importing the engine package —
1641
+ * substrate-types stays zero-runtime.
1339
1642
  */
1340
- interface TxScopeHandles {
1341
- datasets: Map<string, Dataset>;
1342
- edges: EdgesStore | null;
1643
+ interface FrameEntry {
1644
+ params?: Record<string, unknown>;
1645
+ fields?: Record<string, unknown>;
1646
+ /**
1647
+ * §5.13 dedup key. Present on a mutate frame entry opts the whole frame
1648
+ * into the engine's idempotency store; a repeat call with the same key
1649
+ * inside the TTL returns the cached envelope verbatim.
1650
+ */
1651
+ idempotencyKey?: string;
1343
1652
  }
1344
1653
  /**
1345
- * A leaf predicate on a single field. At most one operator per call-site is
1346
- * used in practice (e.g. `{ age: { gt: 18 } }`), but the shape admits several
1347
- * operators so `{ age: { gte: 10, lt: 100 } }` can express a half-open range
1348
- * without the caller reaching for an explicit `AND`. Semantics: the leaf
1349
- * matches when EVERY specified operator matches (implicit AND within a leaf).
1654
+ * Qualified frame (§4.7). Keys are `<ns>/<name>`. `<name>` is the type entry
1655
+ * name for query, or the mutation name for mutate.
1350
1656
  *
1351
- * `eq`/`neq` accept `null` to address IS NULL / IS NOT NULL semantics at the
1352
- * substrate layer — the validator enforces `null` only on those two operators.
1657
+ * Batched mutate (§5.13): a mutate op-key MAY carry an array of `FrameEntry`
1658
+ * values. All frames under one op-key run serially inside the SAME
1659
+ * transaction — all-or-nothing per op-key, committed independently of other
1660
+ * op-keys. Single-frame op-keys share one transaction when they share a
1661
+ * substrate and are refused (`cross_tx_scope_refused`) when they do not.
1662
+ * Query / subscribe / stream op-keys reject array values.
1353
1663
  */
1354
- interface FieldPredicate {
1355
- eq?: number | string | boolean | Date | null;
1356
- neq?: number | string | boolean | Date | null;
1357
- gt?: number | string | Date;
1358
- gte?: number | string | Date;
1359
- lt?: number | string | Date;
1360
- lte?: number | string | Date;
1361
- between?: [number | string | Date, number | string | Date];
1362
- in?: Array<number | string | boolean | Date>;
1363
- nin?: Array<number | string | boolean | Date>;
1364
- contains?: string;
1365
- startsWith?: string;
1366
- endsWith?: string;
1367
- has?: number | string | boolean | Date | null;
1368
- path?: string[];
1664
+ type QualifiedFrame = Record<string, FrameEntry | Array<FrameEntry>>;
1665
+ interface EngineInput {
1666
+ frame: QualifiedFrame;
1667
+ ctx: Partial<RequestContext>;
1668
+ /**
1669
+ * §5.13 cancellation. Threaded into each resolver via `args.signal` so
1670
+ * resolvers observe cancel by polling `signal.aborted` or registering an
1671
+ * `abort` listener.
1672
+ */
1673
+ signal?: AbortSignal;
1674
+ /**
1675
+ * Page bounds for list reads, set by a transport serving remote callers:
1676
+ * a type read with no `limit` gets `defaultLimit`, one above `maxLimit` is
1677
+ * refused. Absent for in-process callers, which read what they ask for.
1678
+ */
1679
+ pageLimits?: {
1680
+ readonly defaultLimit: number;
1681
+ readonly maxLimit: number;
1682
+ };
1369
1683
  }
1370
1684
  /**
1371
- * A where clause is either a logical combinator OR a map of field names to
1372
- * predicates / nested where clauses (for struct fields). Field names and the
1373
- * three combinator keys (AND/OR/NOT) share a namespace — `AND`/`OR`/`NOT` are
1374
- * reserved and must not collide with field names declared in any type.
1685
+ * Per-namespace engine configuration. substrate-core imports this type rather
1686
+ * than redeclaring it.
1375
1687
  */
1376
- type WhereClause = {
1377
- AND: WhereClause[];
1378
- } | {
1379
- OR: WhereClause[];
1380
- } | {
1381
- NOT: WhereClause;
1382
- } | {
1383
- [field: string]: FieldPredicate | WhereClause | unknown;
1384
- };
1688
+ interface NamespaceConfig {
1689
+ registry: Registry;
1690
+ /**
1691
+ * Marks a substrate-contributed namespace. Virtual namespaces
1692
+ * have NO dataset: the engine skips the dataset-presence check on
1693
+ * dispatch and leaves `args.dataset` undefined — resolvers compute from
1694
+ * `params`/`fields` and MUST NOT reach for a dataset handle. They are
1695
+ * structurally incapable of autoCRUD/storage — coordinators own data,
1696
+ * substrate surfaces own computation. Subscribe refuses these namespaces
1697
+ * with `subscription_unsupported_frame`.
1698
+ */
1699
+ virtual?: boolean;
1700
+ /** From `NamespaceDefinition.annotations`; surfaced on the schema, never hashed. */
1701
+ annotations?: Readonly<Record<string, unknown>>;
1702
+ }
1385
1703
  /**
1386
- * OrderBy clause. One entry per field, direction per field. Engine appends an
1387
- * implicit `{ id: 'asc' }` when callers omit `id` as the final tie-breaker — a
1388
- * deterministic tie-break is what makes cursor pagination stable across pages.
1704
+ * Narrow engine handle threaded into scheduled-task / plugin-route contexts.
1705
+ * The concrete `Engine` class in `@mnmal-ai/substrate-core` satisfies this
1706
+ * structurally; plugin code binds against this handle without importing the
1707
+ * engine package.
1389
1708
  */
1390
- interface OrderByClause {
1391
- [field: string]: 'asc' | 'desc';
1392
- }
1393
- interface ReadParams {
1394
- id?: Id;
1395
- ids?: Id[];
1396
- includeDeleted?: boolean;
1397
- where?: WhereClause;
1398
- orderBy?: OrderByClause[];
1399
- limit?: number;
1709
+ interface EngineHandle {
1710
+ query(input: EngineInput): Promise<PackEnvelope>;
1711
+ mutate(input: EngineInput): Promise<PackEnvelope>;
1400
1712
  /**
1401
- * Opaque cursor produced by a prior `read()` call. Base64url-encoded JSON
1402
- * (server-built, server-consumed). Callers should treat it as a black box.
1713
+ * Evaluate the read policy for a type entry without dispatching it.
1714
+ * Out-of-band read surfaces (kernel routes returning row data
1715
+ * outside frame dispatch — e.g. /substrate/v1/recall) call this so the same
1716
+ * composed policies that gate `query` gate them too. Fails closed on
1717
+ * unknown namespace/type and tombstoned namespaces.
1403
1718
  */
1404
- cursor?: string;
1719
+ evaluateReadPolicy(ns: string, type: string, identity: Identity | null, probe?: PolicyProbeContext): PolicyDecision;
1405
1720
  /**
1406
- * Optional caller-supplied projection hint. Substrates MAY use this to
1407
- * narrow the body keys they materialise from storage — e.g. PgDataset
1408
- * builds a `jsonb_build_object(...)` projection over only the requested
1409
- * top-level body keys, saving wire + parse bandwidth.
1410
- *
1411
- * When present, ONLY top-level entries are honoured (the value can be `true`
1412
- * or a nested FieldInput-shaped object — the substrate treats both as
1413
- * "include this key"). Nested projection inside JSONB sub-trees is intentionally
1414
- * out of scope; the pack codec already prunes deeper. Row-metadata-style keys
1415
- * (`_metadata`, `shortId`, `_deletedAt`, `id`) are populated from dedicated
1416
- * columns regardless of `fields` and never need to be included here.
1417
- *
1418
- * When absent, substrates return the full body, for raw `dataset.read()`
1419
- * callers (sweep, compose registry, migration scripts) that don't know which
1420
- * body keys they'll need downstream.
1721
+ * Invalidate cached schema outputs after an out-of-band registry mutation
1722
+ * (e.g. `applyTypeProposal` / `applyFieldProposal` calling
1723
+ * `Registry.replace`). Without this hook, `federatedSchema()` and
1724
+ * `schemaFor(ns)` keep returning pre-mutation views. Optional `ns` argument also drops the per-namespace `schemaFor` cache;
1725
+ * omit to invalidate only the federated outputs.
1726
+ */
1727
+ invalidateSchemaCaches(ns?: string): void;
1728
+ /**
1729
+ * Register a new namespace at runtime. Throws on collision or on a
1730
+ * tombstoned name. Used by substrate-proposals' propose_namespace verb to provision
1731
+ * per-user / per-tenant namespaces without binary restart. Caller (typically
1732
+ * the resolver) is responsible for FPP semantics; this method is the
1733
+ * mechanical primitive.
1734
+ */
1735
+ addNamespace(ns: string, config: NamespaceConfig): void;
1736
+ /**
1737
+ * Mark a namespace as tombstoned. After this call:
1738
+ * - schemaFor(ns) throws `unknown namespace`
1739
+ * - federatedSchema() omits the ns
1740
+ * - query / mutate / subscribe against the ns return a per-key
1741
+ * `namespace_tombstoned` error
1742
+ * - existing live subscriptions on the ns are closed with a
1743
+ * `namespace_tombstoned` error frame
1421
1744
  *
1422
- * Substrates MAY ignore this hint — it's a perf optimisation, not a contract.
1423
- * Memory dataset, for instance, has no wire to save and ignores it.
1745
+ * Idempotent. The underlying NamespaceConfig stays in `#namespaces` for
1746
+ * concurrency safety (in-flight ops keep their references) — full physical
1747
+ * reclaim happens at engine restart, gated by whatever persistence layer
1748
+ * the host uses to decide which namespaces to re-install at boot. Tombstone
1749
+ * state itself is in-memory; restart wipes it.
1424
1750
  */
1425
- fields?: {
1426
- [field: string]: true | unknown;
1427
- };
1428
- }
1429
- /**
1430
- * Paged result of a `read()` call. `rows` is always present; `nextCursor` is
1431
- * present iff `hasMore` is `true` and the current page is non-empty.
1432
- */
1433
- interface ReadResult {
1434
- rows: Row[];
1435
- nextCursor?: string;
1436
- hasMore: boolean;
1437
- }
1438
- interface WriteResult {
1439
- written: number;
1751
+ tombstoneNamespace(ns: string): void;
1752
+ /**
1753
+ * Be told, after the fact, each time a namespace is tombstoned (a drop), so
1754
+ * a plugin that installed it can retire what it owns. A listener's throw is
1755
+ * reported as a metric and does not reach the dropper. Returns a disposer.
1756
+ */
1757
+ onNamespaceTombstoned(listener: (ns: string) => void): () => void;
1758
+ /**
1759
+ * Take part in every namespace drop's transaction: the drop route opens
1760
+ * `participant.namespaces` in its transaction too and runs `onDrop` after
1761
+ * the rows are gone and before it commits, so a throw rolls the drop back.
1762
+ * Returns a disposer.
1763
+ */
1764
+ addDropParticipant(participant: DropParticipant): () => void;
1765
+ /** The participants `addDropParticipant` registered, for the drop route. */
1766
+ dropParticipants(): readonly DropParticipant[];
1440
1767
  /**
1441
- * One per written row, in input order: a supplied id verbatim, otherwise
1442
- * the id the dataset assigned. Under `insertOnly`, skipped rows are absent.
1768
+ * Take a namespace off the engine and free its name for `addNamespace`.
1769
+ * Subscriptions on it close, its schema disappears, new ops answer
1770
+ * `unknown_namespace`; in-flight ops keep their resolved config. Undoes a
1771
+ * runtime add, e.g. a rolled-back propose_namespace. Idempotent.
1443
1772
  */
1444
- ids: Id[];
1773
+ removeNamespace(ns: string): void;
1774
+ /** Whether `ns` is on the engine, and if so whether it is still served. */
1775
+ namespaceState(ns: string): 'absent' | 'live' | 'tombstoned';
1445
1776
  /**
1446
- * Merged rows after upsert — substrates populate this with the post-merge
1447
- * row state so autoCRUD update can skip its post-read. Order
1448
- * matches `ids` (one entry per input row). Substrates that can't compute
1449
- * the merged shape cheaply may emit input rows verbatim; consumers must
1450
- * not rely on body keys absent from the input being filled in.
1777
+ * Construct a fresh empty Registry that the engine can accept via
1778
+ * `addNamespace`. Plugins call this to provision new namespaces at runtime
1779
+ * without taking a runtime dep on the concrete Registry class in substrate-core.
1451
1780
  */
1452
- rows: Row[];
1781
+ createRegistry(): Registry;
1782
+ /**
1783
+ * Look up the host-registered dataset for a namespace, or `undefined` if the
1784
+ * host has not provisioned one. Thin pass-through to the engine's
1785
+ * `DatasetRegistry.get`, surfaced on the handle so plugins running on
1786
+ * dynamically-added namespaces can resolve datasets without the host having
1787
+ * to plumb a `datasetFor` lookup through plugin config.
1788
+ *
1789
+ * Substrate selection remains the host's responsibility — this method only
1790
+ * exposes the lookup. Plugins that hot-add namespaces (substrate-proposals'
1791
+ * `propose_namespace`) MUST require the host to register the dataset before
1792
+ * the call; this method returns `undefined` for unregistered namespaces.
1793
+ */
1794
+ datasetFor(namespace: string): Dataset | undefined;
1453
1795
  }
1454
1796
  /**
1455
- * Optional 3rd-arg to `dataset.write` / `tx.write`. When the incoming row's
1456
- * `_metadata` is absent, the substrate auto-fills it from these hints:
1457
- * - `now` → both `createdAt` and `updatedAt` (defaults to `new Date()`)
1458
- * - `identity` → both `createdBy` and `updatedBy` (defaults to `null`)
1459
- *
1460
- * On upsert collision, `created_*` columns preserve existing values (immutability);
1461
- * `updated_*` columns adopt the new values (bump intent). Engine path
1462
- * (`autocrud-create`/`autocrud-update`) constructs full `_metadata` itself —
1463
- * this auto-fill is purely a safety net for raw-write bypass paths.
1797
+ * Engine surface needed by substrate-pack-aware route helpers — extends the
1798
+ * dispatch handle with the two methods compose's `serializeEnvelope` /
1799
+ * `mutateRoute` need to drive positional encoding. The concrete
1800
+ * `Engine` class satisfies this structurally; tests + alt
1801
+ * implementations can satisfy it without depending on the engine package.
1464
1802
  */
1465
- interface WriteOptions {
1466
- readonly identity?: ActorIdentity;
1467
- readonly now?: Date;
1803
+ interface EncodeContext extends EngineHandle {
1804
+ /** Federated schema map (per-namespace Schema) for substrate-pack encoding. */
1805
+ federatedSchema(): Record<string, Schema>;
1468
1806
  /**
1469
- * Insert only: a row whose id already exists (live or tombstoned) is left
1470
- * untouched and omitted from the result's `ids` and `rows`. The existence
1471
- * check and the insert are one atomic step.
1807
+ * Look up a namespace's mutation-return registry view. Returns undefined
1808
+ * when the namespace isn't registered. The mutate route uses this to
1809
+ * resolve each frame key's `returns` FieldDef for positional encoding.
1472
1810
  */
1473
- readonly insertOnly?: boolean;
1474
- }
1475
- interface DeleteResult {
1476
- deleted: number;
1477
- ids: Id[];
1478
- }
1479
- interface DropResult {
1480
- /** Number of rows physically removed. */
1481
- rowsDeleted: number;
1482
- /** Number of short-id allocations physically removed. */
1483
- shortIdsDeleted: number;
1811
+ registryFor(namespace: string): {
1812
+ get(name: string): {
1813
+ returns?: FieldDef;
1814
+ } | undefined;
1815
+ } | undefined;
1484
1816
  }
1485
1817
  /**
1486
- * Dataset-declared filter capabilities. §5.2 capability negotiation: the engine
1487
- * validates every where-clause leaf against `operators[fieldKind]` and rejects
1488
- * ops the dataset can't execute rather than returning a silently wrong result.
1818
+ * Context threaded into every `ScheduledTask.handler` call.
1489
1819
  */
1490
- type FieldKind = 'scalar.number' | 'scalar.string' | 'scalar.boolean' | 'scalar.date' | 'scalar.json' | 'crossRef' | 'list';
1491
- interface FilterCapabilities {
1492
- operators: Record<FieldKind, string[]>;
1493
- /** Does the dataset honour `orderBy`? */
1494
- orderBy: boolean;
1495
- /** Does the dataset honour cursor-based pagination? */
1496
- cursor: boolean;
1820
+ interface ScheduledTaskContext {
1821
+ /** The owning plugin's `name`. */
1822
+ pluginName: string;
1823
+ /** Engine handle for read/write work performed by the task. */
1824
+ engine: EngineHandle;
1825
+ /** Structured metric sink — fire-and-forget. */
1826
+ metric(name: string, payload?: unknown): void;
1497
1827
  }
1498
- interface UpdateFieldRef {
1499
- ns?: string;
1500
- type: string;
1501
- id: Id;
1828
+ interface ScheduledTask {
1829
+ /** Unique identifier within the plugin. Used in metric labels. */
1830
+ name: string;
1831
+ /** Fire interval in milliseconds. Minimum 100ms — smaller throws at register. */
1832
+ intervalMs: number;
1833
+ /** Fire once at registration in addition to the interval ticks. Default false. */
1834
+ runOnStart?: boolean;
1835
+ /** Handler — receives a context with engine access for read/write work. */
1836
+ handler(ctx: ScheduledTaskContext): void | Promise<void>;
1502
1837
  }
1503
1838
  /**
1504
- * Outcome of a `conditionalUpdate` call (spec §5.12 Phase 2 / Dispatch B).
1505
- *
1506
- * `matched: true` when the row existed AND every key in `match` equalled the
1507
- * row's stored value at the moment of the update — the substrate applied
1508
- * `updates` and returned the merged row. `matched: false` when the row was
1509
- * missing or at least one match condition failed; in both cases no write landed
1510
- * and `row` is `null`.
1839
+ * Context threaded into `CoordinatorPlugin.routes()` calls.
1511
1840
  */
1512
- interface ConditionalUpdateResult {
1513
- matched: boolean;
1514
- row: Row | null;
1841
+ interface PluginRouteContext {
1842
+ /** The owning plugin's `name`. */
1843
+ pluginName: string;
1844
+ /** Engine handle for plugin-scoped routes that need to dispatch back through the engine. */
1845
+ engine: EngineHandle;
1846
+ /** Structured metric sink — fire-and-forget. */
1847
+ metric(name: string, payload?: unknown): void;
1515
1848
  }
1516
- interface Dataset {
1517
- readonly namespace: string;
1849
+ interface CoordinatorPlugin {
1850
+ name: string;
1851
+ namespaces: NamespaceDefinition[];
1518
1852
  /**
1519
- * Stable identifier for the storage substrate this dataset wraps. Two
1520
- * datasets share a substrate when their `txScopeId` values are `===`
1521
- * — they can participate in a single transaction via the engine's
1522
- * substrate-coalescing path (spec §5.3, enabling cross-namespace G1
1523
- * batched mutations when all frames target the same substrate).
1853
+ * Plugin's bound contract version: a semver RANGE over the binary version,
1854
+ * e.g. `'>=4.1.0 <6.0.0'`.
1524
1855
  *
1525
- * Convention:
1526
- * - pg: derived from the `postgres.Sql` pool reference identity. All
1527
- * `PgDataset` instances constructed against the same `sql`
1528
- * handle share one `txScopeId`.
1529
- * - memory: unique per `MemoryDataset` instance by default (each
1530
- * instance owns its own `Map`). Test harnesses that want shared-
1531
- * memory semantics can opt in via an explicit constructor option.
1856
+ * The ceiling tracks actual compatibility, not the next major: it moves
1857
+ * only when a major genuinely breaks the plugin-facing interface. A ceiling
1858
+ * that admits a major which did break lets the plugin load and misbehave
1859
+ * silently — if you cannot say whether the last major broke the plugin
1860
+ * interface, do not widen it.
1532
1861
  *
1533
- * Datasets with different `txScopeId` values CANNOT participate in a
1534
- * single transaction — the engine refuses mixed-substrate batched
1535
- * mutates with `cross_tx_scope_refused`.
1862
+ * Enforced at boot: a plugin whose range excludes the running binary is
1863
+ * refused.
1536
1864
  */
1537
- readonly txScopeId: string;
1538
- read(type: string, params: ReadParams): Promise<ReadResult>;
1539
- write(type: string, rows: WriteRow[], options?: WriteOptions): Promise<WriteResult>;
1540
- tombstone(type: string, ids: Id[]): Promise<DeleteResult>;
1541
- purge(type: string, ids: Id[]): Promise<DeleteResult>;
1865
+ contractVersion: string;
1542
1866
  /**
1543
- * Physically delete every row of `type` within this dataset's namespace,
1544
- * plus every short-id allocation for that type. Ignores `_deletedAt` /
1545
- * `archived` state — drop is hard-delete regardless of row status.
1867
+ * Plugin-global policies applied to every resolver in every namespace the
1868
+ * plugin contributes. Composed by the loader via `and(...)` — the combined
1869
+ * policy must allow. Useful for cross-cutting concerns (audit logging,
1870
+ * rate limits, identity enrichment) that span multiple namespaces in the
1871
+ * same plugin. The composed result is surfaced through
1872
+ * `NamespaceInstallContext.pluginPolicy`.
1873
+ */
1874
+ policies?: Policy[];
1875
+ /**
1876
+ * Post-register lifecycle hook. Fires after every plugin's `install()`
1877
+ * resolves — use this for resource priming that needs the full namespace
1878
+ * map (e.g. cross-plugin edge warmup).
1879
+ */
1880
+ onStart?(ctx: PluginLifecycleContext): void | Promise<void>;
1881
+ /**
1882
+ * Pre-shutdown lifecycle hook. Fires when the host calls `compose()`'s
1883
+ * returned `stop()` (which the binary's SIGTERM handler invokes inside
1884
+ * the `SUBSTRATE_SHUTDOWN_GRACE_MS` window). Hooks run in REVERSE
1885
+ * registration order so plugins tear down in the inverse of the order
1886
+ * they came up — late-loading plugins that depended on earlier ones get
1887
+ * to release their handles first.
1546
1888
  *
1547
- * MUST run inside a caller-supplied `withTransaction` if atomicity with
1548
- * edge cleanup / audit emission is required. When called standalone,
1549
- * implementations MUST still execute the row-delete + short-id-delete as
1550
- * a single atomic step against the substrate.
1889
+ * Use for resources the OS won't reap on process exit:
1890
+ * - subprocess children that should receive SIGTERM cleanly
1891
+ * - long-lived `setInterval` timers (`.unref()` keeps them from
1892
+ * blocking exit, but explicit teardown is still cleaner for tests)
1893
+ * - WebSocket subscriptions / outbound connections that should close
1894
+ * cooperatively rather than dropping mid-request
1551
1895
  *
1552
- * Returns the exact counts physically removed. If the type has no rows,
1553
- * returns zeros without error.
1896
+ * Errors are caught + logged via the lifecycle metric sink under
1897
+ * `plugin_on_stop_error` — one plugin's failed teardown does NOT abort
1898
+ * other plugins' teardown. The engine + pg pool are still live during
1899
+ * onStop calls, so plugins can do final writes (e.g. an audit row
1900
+ * recording the shutdown). After all onStop hooks resolve, the server
1901
+ * stops accepting connections and the substrate's `closeStore()` runs.
1554
1902
  *
1555
- * Storage-level primitive: operates on any (namespace, type) rows present
1556
- * in storage regardless of whether the type is currently known to the
1557
- * engine's registry.
1903
+ * Symmetric to `onStart`. Skipped (no-op) when omitted.
1904
+ */
1905
+ onStop?(ctx: PluginLifecycleContext): void | Promise<void>;
1906
+ /**
1907
+ * Scheduled tasks — the binary ticks these on `.unref()`'d intervals.
1908
+ * Registered with the shared `TaskScheduler` at compose time and started
1909
+ * after the server's `start()` resolves.
1910
+ */
1911
+ schedule?: ScheduledTask[];
1912
+ /**
1913
+ * Mount plugin-scoped HTTP/WS routes. Called once at compose-time with a
1914
+ * `PluginRouter` pre-prefixed `/substrate/v1/plugins/<name>` by the server impl. The
1915
+ * router surface matches `PluginRouter` in `./server.ts`.
1916
+ *
1917
+ * With auth on, HTTP routes require `read` for GET and `mutate` otherwise,
1918
+ * granted in one of this plugin's `namespaces`; a plugin with no namespaces
1919
+ * accepts the capability granted in any namespace.
1920
+ */
1921
+ routes?(router: PluginRouter, ctx: PluginRouteContext): void | Promise<void>;
1922
+ /**
1923
+ * Optional onboarding contribution. When set, the hosting binary's
1924
+ * `GET /substrate/v1/onboard` aggregates this record under
1925
+ * `instructions.plugins.<name>` and enumerates the plugin's conventional
1926
+ * `/substrate/v1/plugins/<name>/onboard` route under `instructions.pluginRoutes`. Use
1927
+ * this to teach agents about the plugin's types, write flows, or any
1928
+ * plugin-specific grammar a cold-start agent would otherwise have to
1929
+ * discover per endpoint. The shape is free-form — the binary round-trips
1930
+ * it verbatim.
1931
+ */
1932
+ onboardInstructions?: Record<string, unknown>;
1933
+ }
1934
+ /**
1935
+ * Substrate-tier plugin. Augments the kernel rather than the agent-facing
1936
+ * surface. Composes BEFORE coordinator plugins so coordinators can rely on
1937
+ * substrate registrations being live.
1938
+ *
1939
+ * SubstratePlugins do NOT own data namespaces — that's a CoordinatorPlugin
1940
+ * concern (a substrate may claim a virtual `kernelNamespace`).
1941
+ */
1942
+ interface KernelPlugin {
1943
+ /** Unique plugin name. Used in metric labels + collision detection. */
1944
+ name: string;
1945
+ /**
1946
+ * Plugin's bound contract version: a semver RANGE over the binary version,
1947
+ * e.g. `'>=4.1.0 <6.0.0'`. Required on substrate plugins. Checked by
1948
+ * `composeFromManifests` and REFUSED at boot on mismatch; the
1949
+ * ceiling tracks actual compatibility rather than the next major boundary.
1950
+ * See the note on `CoordinatorPlugin.contractVersion` for both.
1558
1951
  */
1559
- dropType(type: string): Promise<DropResult>;
1952
+ contractVersion: string;
1560
1953
  /**
1561
- * Physically delete every row AND short-id for `this.namespace`, across
1562
- * every type. Equivalent to calling `dropType` for each distinct type
1563
- * seen in storage, but substrate impls SHOULD use a single bulk DELETE
1564
- * where available (pg: `DELETE FROM rows WHERE namespace = $1`).
1565
- *
1566
- * Same atomicity rules as `dropType`. Same "ignores archived state".
1954
+ * Schema migrations the plugin contributes. Applied in declared order
1955
+ * on first install AND on every binary boot (idempotently, gated by
1956
+ * the migrations tracking table seeded by the first KernelPlugin
1957
+ * to populate this slot).
1567
1958
  */
1568
- dropNamespace(): Promise<DropResult>;
1569
- updateField(ref: UpdateFieldRef, fieldName: string, value: unknown): Promise<void>;
1959
+ migrations?: KernelMigration[];
1570
1960
  /**
1571
- * Atomic compare-and-update by PK. If the row `(type, id)` exists AND every
1572
- * `(field, value)` pair in `match` equals the stored row's field, apply the
1573
- * `updates` map and return `{ matched: true, row: mergedRow }`. Otherwise
1574
- * write nothing and return `{ matched: false, row: null }`.
1575
- *
1576
- * Semantics:
1577
- * - Equality is strict (`===`) for scalars; `Date` compares by
1578
- * `.getTime()`; `null` matches only a stored `null` (including
1579
- * intentionally-null fields set via `updateField`), never an absent field.
1580
- * - A missing row fails the match even when `match` is empty — the
1581
- * operation is "update this existing row if predicate holds".
1582
- * - `updates` is merged onto the stored row the same way `write()` upserts
1583
- * merge: each key overwrites; omitted keys are preserved. `_deletedAt`
1584
- * is reserved and silently skipped (matches `write()`'s sanitisation).
1585
- * - The whole check+write is atomic against other dataset operations.
1586
- * Implementations MUST NOT yield between the predicate test and the
1587
- * write. Pg uses `UPDATE ... WHERE <match> RETURNING *` inside the
1588
- * caller's enclosing `withTransaction`; memory runs the whole body
1589
- * synchronously within the `async` microtask.
1590
- *
1591
- * Motivating use case: the `_system` register route consumes a one-shot
1592
- * invite — the invite row's `consumedAt` must transition from `null` to a
1593
- * timestamp exactly once, even under concurrent registrations.
1594
- *
1595
- * Metadata: a successful match unconditionally bumps `updatedAt` to `now`
1596
- * (a server-set `now()` when omitted) and stamps `updatedBy` with the supplied `actor` (or
1597
- * a `{ kind: 'system', id: 'anonymous' }` sentinel when omitted) — same
1598
- * semantics as `write()`'s upsert path, so recency-ranked recall sees the
1599
- * change. `createdAt` / `createdBy` are preserved.
1961
+ * Names of the tables (in the store's schema) this plugin keeps data in.
1962
+ * `substrate-admin migrate-store` copies them alongside the engine's own
1963
+ * tables and refuses a source table that nothing declares.
1600
1964
  */
1601
- conditionalUpdate(type: string, id: Id, match: Record<string, unknown>, updates: Record<string, unknown>, actor?: ActorIdentity, now?: Date): Promise<ConditionalUpdateResult>;
1602
- readonly capabilities: FilterCapabilities;
1965
+ storeTables?: readonly string[];
1603
1966
  /**
1604
- * Run `fn` while holding a mutual-exclusion lock named `key`, waiting until
1605
- * it is free. Excludes every other holder of the same key on this substrate:
1606
- * pg holds a session advisory lock, so the exclusion spans processes sharing
1607
- * the store's database schema; memory excludes holders on this dataset instance only.
1608
- * Released when `fn` settles, whether it resolves or throws, except on a
1609
- * pg tx-bound dataset, where it is held until the transaction ends so the
1610
- * next holder sees what this one committed. Not re-entrant:
1611
- * re-acquiring a held key inside `fn` waits forever.
1612
- *
1613
- * OPTIONAL. Callers feature-detect and fall back to a process-local lock.
1614
- *
1615
- * @returns whatever `fn` resolves to
1616
- * @throws whatever `fn` throws, after the lock is released
1967
+ * Filter operators contributed to the where-clause grammar. Each entry
1968
+ * registers a custom operator (e.g. `embeddingSimilar`) the engine's
1969
+ * query validator + substrate executor can dispatch to.
1617
1970
  */
1618
- withAdvisoryLock?<T>(key: string, fn: () => Promise<T>): Promise<T>;
1971
+ filterOperators?: Record<string, FilterOperatorDef>;
1619
1972
  /**
1620
- * Prepare dedicated storage for `type` in this namespace, moving any rows
1621
- * already written for it there. Idempotent, and safe beside concurrent
1622
- * reads and writes: no row is lost, duplicated or seen twice. That holds for
1623
- * READ COMMITTED readers; a REPEATABLE READ snapshot taken before the move
1624
- * can miss the moved rows. Also builds the type's indexes, including its
1625
- * declared `indexes`, and throws when a unique one cannot be built.
1626
- *
1627
- * `indexes` picks which: `unique` builds only the unique ones and leaves the
1628
- * rest `pending`, for a later `nonUnique` call to build. `signal` cancels an
1629
- * in-flight non-unique build, which the next call retries. Default `all`.
1630
- *
1631
- * OPTIONAL. Backends without per-type storage omit it.
1973
+ * Top-level kernel route claims. Plugin reserves a path under /substrate/v1/.
1974
+ * First-claim wins; collision = boot refusal with actionable error.
1632
1975
  */
1633
- ensureTypeStorage?(type: string, opts?: {
1634
- indexes?: 'all' | 'unique' | 'nonUnique';
1635
- signal?: AbortSignal;
1636
- }): Promise<void>;
1976
+ kernelRoutes?: KernelRouteClaim[];
1637
1977
  /**
1638
- * Where `type`'s rows live and how its declared indexes stand, read from the
1639
- * catalog without building anything.
1640
- *
1641
- * OPTIONAL. Backends without per-type storage omit it.
1978
+ * Top-level MCP tool registrations. Tool name registers globally with
1979
+ * the MCP server (no /plugins/ prefix on MCP — tools are flat).
1642
1980
  */
1643
- describeStorage?(type: string): Promise<TypeStorageReport>;
1981
+ mcpTools?: McpToolRegistration[];
1644
1982
  /**
1645
- * The database-wide storage layout and what an operator should look at:
1646
- * rows parked in DEFAULT partitions, indexes and CHECK constraints on the
1647
- * `rows` parent, indexes this process has yet to build or failed to build,
1648
- * and a partition count over `partitionBudget`; and, as `info`, engine
1649
- * indexes no longer declared and open index recommendations. Never changes
1650
- * anything.
1651
- *
1652
- * OPTIONAL. Backends without per-type storage omit it.
1983
+ * Frame-surface contribution. Claims one namespace in the engine registry
1984
+ * carrying resolver-backed query types + mutations — no dataset, no
1985
+ * autoCRUD. Dispatch/policy/idempotency/pack/schema/NL/MCP all inherit from
1986
+ * the normal frame path.
1653
1987
  */
1654
- storageReport?(opts: {
1655
- partitionBudget: number;
1656
- }): Promise<StorageReport>;
1988
+ kernelNamespace?: KernelNamespaceContribution;
1657
1989
  /**
1658
- * Table and index statistics for every type the store holds, every
1659
- * namespace, read from the database's own statistics. Estimates, never a
1660
- * count. Never changes anything.
1661
- *
1662
- * OPTIONAL. Backends without such statistics omit it.
1990
+ * Post-register lifecycle hook. Fires after every KernelPlugin's
1991
+ * migrations + registrations resolve, BEFORE any coordinator plugin
1992
+ * composes. Use for resource priming the kernel needs before
1993
+ * coordinators arrive.
1663
1994
  */
1664
- typeStats?(): Promise<TypeStats[]>;
1995
+ onKernelReady?(ctx: KernelLifecycleContext): void | Promise<void>;
1665
1996
  /**
1666
- * The indexes the engine wants for `namespace`/`type` (any namespace, not
1667
- * only this one), with each one's leading keys, and the type's size hint.
1668
- * Undefined when the type is unknown. Never changes anything.
1669
- *
1670
- * OPTIONAL. Backends that build no indexes omit it.
1997
+ * Stops whatever `onKernelReady` started (timers, workers). The host calls
1998
+ * it before it closes the store, and may call it more than once.
1671
1999
  */
1672
- indexPlan?(type: string, namespace: string): Promise<IndexPlan | undefined>;
2000
+ onKernelStop?(): void | Promise<void>;
1673
2001
  /**
1674
- * Drop index `name` of this namespace's `type`, a signed `propose_index`
1675
- * remove. Only an engine-owned, non-unique index the engine no longer wants
1676
- * (a leftover or a retracted declaration) is dropped; anything else is
1677
- * refused with an error whose `code` is `unknown_index`, `not_engine_owned`,
1678
- * `unique_index` or `still_desired`, and nothing is dropped. A drop that
1679
- * waits longer than `lockTimeoutMs` on another transaction is abandoned with
1680
- * `index_drop_timeout`; the index may then be left invalid, and a retry drops it.
2002
+ * Contribute postWrite hooks for a coordinator namespace at install time.
2003
+ * Called by the engine ONCE per registered coordinator namespace, AFTER
2004
+ * substrate composition completes. Returned hooks are merged into the
2005
+ * type's effective postWrite chain alongside the type's own inline
2006
+ * `postWrite` (if any). Substrate-supplied hooks run AFTER the type's
2007
+ * inline hook in chain order.
1681
2008
  *
1682
- * OPTIONAL. Backends that build no indexes omit it (the memory dataset):
1683
- * there, `propose_index remove` has no index to find, so it is refused
1684
- * `unknown_index`, and a remove recommendation it names is superseded.
2009
+ * Use this to inspect `namespace.types[T].embed` and emit embedding
2010
+ * hooks per type without coupling the binary to plugin internals.
2011
+ *
2012
+ * Returns: a record mapping type names to PostWriteHook contributions.
2013
+ * Type names not in the record receive no contribution.
1685
2014
  */
1686
- dropEngineIndex?(type: string, name: string, opts?: {
1687
- /** Runs once every check has passed, just before the drop; a throw drops nothing. */
1688
- beforeDrop?: () => Promise<void>;
1689
- /** The drop's bound on each lock wait, in ms. Defaults to the backend's own. */
1690
- lockTimeoutMs?: number;
1691
- }): Promise<void>;
2015
+ contributePostWrites?: (namespace: NamespaceDefinition) => Record<string, PostWriteHook>;
2016
+ }
2017
+ /** Single migration unit. */
2018
+ interface KernelMigration {
2019
+ /** Unique id within the plugin. Stored in migrations tracking table. */
2020
+ id: string;
2021
+ /** Apply migration. Called only when this migration's id is absent from the migrations table. */
2022
+ apply(ctx: KernelMigrationContext): Promise<void>;
2023
+ /** Inverse — drops what apply created. */
2024
+ rollback?(ctx: KernelMigrationContext): Promise<void>;
2025
+ }
2026
+ interface KernelMigrationContext {
1692
2027
  /**
1693
- * Drop the engine-owned indexes of this namespace, after a namespace drop
1694
- * has committed: its types leave the engine, so nothing would remove them.
1695
- * Non-unique ones only, unless `unique`, which the caller passes only when
1696
- * the namespace is declared nowhere (in no code and no live proposal): a
1697
- * declared namespace's types come back, and until the reconciler rebuilds a
1698
- * unique index, writes would not be held to it. Returns the names dropped.
1699
- * Never inside a transaction; a tx-bound instance throws. A lock wait that
1700
- * times out throws `index_drop_timeout`, leaving the rest for
1701
- * `storageReport` to list as orphaned.
2028
+ * Direct SQL handle for postgres-backed deployments; null when the
2029
+ * binary is running against the in-memory dataset.
2030
+ */
2031
+ sql: unknown | null;
2032
+ /**
2033
+ * How `rows` is stored: `'partitioned'` (LIST(namespace) then LIST(type)) or
2034
+ * `'legacy'` (one plain table). `null` on the in-memory dataset. PG refuses
2035
+ * a unique index on a partitioned parent unless it includes every partition
2036
+ * key column (`namespace`, `type`), so such DDL on `rows` must branch on this.
2037
+ */
2038
+ layout: 'legacy' | 'partitioned' | null;
2039
+ metric(name: string, payload?: unknown): void;
2040
+ }
2041
+ /**
2042
+ * Where-clause operator extension.
2043
+ *
2044
+ * The operator's name is the Record key it's registered under in
2045
+ * `KernelPlugin.filterOperators`.
2046
+ *
2047
+ * `execute` is awaited by the pre-resolve pass (`@mnmal-ai/substrate-filter`'s
2048
+ * `preResolveExtensionOperatorsDetailed`) and must resolve to an ordered,
2049
+ * scored `{ hits: [{ id: string; score: number; vector?: number[] }, ...] }`
2050
+ * envelope; any other shape throws `ExtensionOperatorResultError`. Relevance
2051
+ * order is preserved in the rewritten `id IN`, and score/vector are surfaced
2052
+ * via the pre-pass meta side-channel for the datasets to stitch onto rows.
2053
+ */
2054
+ interface FilterOperatorDef {
2055
+ validate(value: unknown): {
2056
+ ok: true;
2057
+ value: unknown;
2058
+ } | {
2059
+ ok: false;
2060
+ reason: string;
2061
+ };
2062
+ execute(value: unknown, context: unknown): Promise<unknown>;
2063
+ /**
2064
+ * Computed projection field names the operator contributes per hit (e.g.
2065
+ * `['_score', '_vector']`). The engine refuses a projection of one of these
2066
+ * fields unless its operator is in the where-clause.
2067
+ */
2068
+ computedFields?: string[];
2069
+ }
2070
+ /**
2071
+ * Query-only virtual type contributed by a substrate plugin.
2072
+ * `fields` reuses the FieldDef DSL so schema publication + client typing
2073
+ * work unchanged. The resolver receives the standard (ctx, args) pair; the
2074
+ * engine leaves `args.dataset` undefined for virtual namespaces — resolvers
2075
+ * compute from `params`/`fields` and MUST NOT reach for a dataset handle.
2076
+ */
2077
+ interface VirtualTypeDef {
2078
+ fields: Record<string, FieldDef>;
2079
+ resolver: RegistryEntryResolver;
2080
+ }
2081
+ /** Mutation contributed by a substrate plugin under its virtual namespace. */
2082
+ interface VirtualMutationDef {
2083
+ resolver: RegistryEntryResolver;
2084
+ /** Optional params schema published into the namespace schema. */
2085
+ params?: Record<string, FieldDef>;
2086
+ }
2087
+ /**
2088
+ * A substrate plugin's claimed frame surface.
2089
+ * Registered into the engine's namespaces map as `{ registry, virtual: true }`.
2090
+ * Collision posture mirrors KernelRouteClaim: second claim = boot refusal.
2091
+ */
2092
+ interface KernelNamespaceContribution {
2093
+ namespace: string;
2094
+ types?: Record<string, VirtualTypeDef>;
2095
+ mutations?: Record<string, VirtualMutationDef>;
2096
+ }
2097
+ /** Top-level kernel route reservation. */
2098
+ interface KernelRouteClaim {
2099
+ path: string;
2100
+ method: HttpMethod;
2101
+ handler: HttpHandler;
2102
+ /**
2103
+ * Capability the host must check before dispatching, when auth is enabled.
2104
+ * REQUIRED — `'public'` is how a route opts out, so opting out is a
2105
+ * declaration rather than an omission.
1702
2106
  *
1703
- * OPTIONAL. Backends that build no indexes omit it.
2107
+ * Do not make this optional: kernel routes mount via the host's raw
2108
+ * `server.route`, which applies no gate of its own, so a route with no
2109
+ * capability would get no auth wrapper at all.
1704
2110
  */
1705
- dropNamespaceIndexes?(opts?: {
1706
- unique?: boolean;
1707
- }): Promise<string[]>;
2111
+ capability: 'read' | 'mutate' | 'admin' | 'public';
2112
+ }
2113
+ /** Top-level MCP tool registration. */
2114
+ interface McpToolRegistration {
2115
+ name: string;
2116
+ description: string;
2117
+ inputSchema: unknown;
2118
+ handler(input: unknown): Promise<unknown>;
2119
+ }
2120
+ interface KernelLifecycleContext {
1708
2121
  /**
1709
- * The entry of this namespace's `type`'s declared `indexes` whose index is
1710
- * named `name`, or undefined when no declaration builds it (a derived or
1711
- * engine index, or none). `propose_index remove` retracts it when it is a
1712
- * proposed type's. Never changes anything.
1713
- *
1714
- * OPTIONAL. Backends that build no indexes omit it.
2122
+ * Tagged-union discriminant. Differentiates this context from the
2123
+ * structurally-similar `PluginLifecycleContext`, which would otherwise be
2124
+ * interchangeable under structural typing.
1715
2125
  */
1716
- indexDeclarationOf?(type: string, name: string): Promise<IndexDecl | undefined>;
2126
+ readonly lifecycle: 'kernel';
1717
2127
  /**
1718
- * Execute `fn` atomically. All ops performed on the `tx` argument run inside
1719
- * one substrate transaction. If `fn` throws or returns a rejected promise,
1720
- * every mutation inside the callback rolls back. If `fn` resolves, the
1721
- * transaction commits and the resolved value is returned.
2128
+ * Direct SQL handle; null when the binary runs against the in-memory
2129
+ * dataset. Same shape and same null semantics as
2130
+ * `KernelMigrationContext.sql`.
1722
2131
  *
1723
- * Scope rules:
1724
- * - `tx.namespace === this.namespace`
1725
- * - Calls on the outer dataset (`this`) inside `fn` bypass the transaction
1726
- * and are NOT rolled back on failure — this is a caller bug; implementations
1727
- * MUST NOT attempt to guard against it. All ops inside `fn` MUST go through
1728
- * `tx`.
1729
- * - Nested `tx.withTransaction(...)` is implementation-defined. The reference
1730
- * impls (memory, pg) flatten nested calls into the outermost transaction
1731
- * (no savepoints).
2132
+ * For convergent, run-every-boot reconciliation: a migration runs only once
2133
+ * per `id`, so it cannot re-scope objects it already created.
1732
2134
  */
1733
- withTransaction<T>(fn: (tx: Dataset) => Promise<T>): Promise<T>;
2135
+ sql: unknown | null;
1734
2136
  /**
1735
- * Engine-only. Open a substrate-level transaction spanning multiple
1736
- * namespaces on the same substrate, and invoke `fn` with a handles object
1737
- * containing tx-bound datasets + (optionally) a tx-bound `EdgesStore`.
1738
- * Called by the engine's mutate path (spec §5.3) for both single-ns and
1739
- * cross-ns frames — the engine calls with `[ns]` for the single-ns case.
2137
+ * Namespaces DECLARED by the coordinator plugins this binary loaded, in
2138
+ * load order. Empty when none are configured.
1740
2139
  *
1741
- * `namespaces` is the full set of namespaces the engine will dispatch
1742
- * frames to inside the tx. Implementations MUST return a tx-bound
1743
- * dataset for every requested namespace in `handles.datasets`; missing
1744
- * entries are a programming error on the caller side.
2140
+ * Declared, not installed: coordinators compose AFTER substrate, so the
2141
+ * engine has not yet registered them.
1745
2142
  *
1746
- * `handles.edges` contract:
1747
- * - `non-null` when the implementation can enlist a shared `EdgesStore`
1748
- * into the same substrate tx (e.g. pg where the `PgEdgesStore` is wired
1749
- * in via constructor and shares the `Sql`). The engine routes edge
1750
- * mutations through this handle — true cross-store atomicity for
1751
- * row + edge within one frame.
1752
- * - `null` when the implementation can't enlist (e.g. memory datasets
1753
- * where edges live in a separate Map). The engine falls back to
1754
- * running edge ops in a nested independent tx and accepts the
1755
- * visibility-ordering quirk documented in `autocrud-delete.ts`.
2143
+ * Read these rather than adding a namespace knob to substrate config, which
2144
+ * could silently disagree with what the coordinator uses.
2145
+ */
2146
+ coordinatorNamespaces: readonly string[];
2147
+ /**
2148
+ * The same namespaces, attributed to the coordinator that declared each.
2149
+ * Declaration order preserved within a plugin.
1756
2150
  *
1757
- * Scope rules mirror `withTransaction`:
1758
- * - All ops on values from `handles.datasets.get(ns)` and `handles.edges`
1759
- * run in the same tx.
1760
- * - Ops on the outer dataset (`this`) inside `fn` bypass the tx — caller
1761
- * bug; implementations do not guard.
1762
- * - A throw or rejection inside `fn` rolls back every mutation across
1763
- * every namespace + edges in the tx.
2151
+ * The two absent cases differ and consumers must branch on them:
2152
+ * - `undefined` — this host does not supply attribution (e.g. an older
2153
+ * binary); do not treat it as an empty list
2154
+ * - `[]` — attribution supplied, and there are no coordinators
1764
2155
  *
1765
- * OPTIONAL. Implementations that can't coalesce cross-namespace writes
1766
- * into one substrate transaction omit this method — the engine falls
1767
- * back to refusing cross-namespace batches with `cross_tx_scope_refused`
1768
- * (and loses edge-enlist for single-ns).
2156
+ * Filtering on `plugin === 'some-name'` makes the coordinator's `name` a
2157
+ * contract term: hold it in one constant shared with the coordinator and pin
2158
+ * it with a test, or a rename makes the filter silently match nothing.
1769
2159
  */
1770
- withScopedTransaction?<T>(namespaces: ReadonlyArray<string>, fn: (handles: TxScopeHandles) => Promise<T>): Promise<T>;
2160
+ coordinatorNamespacesByPlugin?: readonly {
2161
+ plugin: string;
2162
+ namespace: string;
2163
+ }[];
1771
2164
  /**
1772
- * Allocate the next short id for the given type within this dataset's
1773
- * namespace. Monotonic per (namespace, type). Does NOT record any
1774
- * row-id mapping — call `bindShortId` after the row is written.
2165
+ * Whether this boot may start background work (timers, backfills). False
2166
+ * when the host boots the store only to prepare or copy into it, as
2167
+ * `substrate-admin migrate-store` does: do convergent reconciliation, start
2168
+ * nothing that writes later.
2169
+ */
2170
+ background: boolean;
2171
+ metric(name: string, payload?: unknown): void;
2172
+ }
2173
+ //#endregion
2174
+ //#region ../../packages/substrate-types/src/schema.d.ts
2175
+ /**
2176
+ * One clause of a walk-grammar read, marked index-served or not (typed edge
2177
+ * walks, goal 7). `path` is inside the frame entry, e.g. `['where', '_walk']`.
2178
+ * An unserved clause names the index that would serve it in `recommend`, when
2179
+ * one can be declared.
2180
+ */
2181
+ interface PlanClause {
2182
+ path: string[];
2183
+ kind: 'hop' | 'filter' | 'order' | 'ref' | 'derive' | 'bucket' | 'agg';
2184
+ served: boolean;
2185
+ recommend?: IndexDecl;
2186
+ }
2187
+ /** Words of a natural-language question the parse did not use, and why. */
2188
+ interface Dropped {
2189
+ span: string;
2190
+ reason: string;
2191
+ /**
2192
+ * What the loss changes. `filter`: which rows match (a filter, comparison,
2193
+ * walk, time window or connective), so a count, aggregate, `$ref` or pick
2194
+ * reading that lost it is `unknown`. `order`: an ordering, limit or
2195
+ * projection; the matched rows are the same and the answer stands.
2196
+ */
2197
+ kind: 'filter' | 'order';
2198
+ }
2199
+ interface FieldDefScalar {
2200
+ kind: 'scalar';
2201
+ /**
2202
+ * `'json'` is an opaque body: msgpack-friendly nested object/array, value
2203
+ * round-trips unchanged, but no per-leaf filtering or ordering. Use it for
2204
+ * freeform payloads (audit details, debug bags). When you need filterable
2205
+ * inner keys, use `field.struct` instead.
2206
+ */
2207
+ type: 'string' | 'number' | 'boolean' | 'date' | 'json';
2208
+ /**
2209
+ * Tri-state. `true` builds an index (a btree; a GIN for json). Unset means
2210
+ * no index. `false` means no index AND not a search handle: long-form
2211
+ * prose that NL and similar consumers leave out. The field filters either
2212
+ * way. Physical consumers test `=== true`, so never normalise unset to
2213
+ * `false`.
2214
+ */
2215
+ indexed?: boolean;
2216
+ /** Metadata: when true, consumers SHOULD expect `null` as a legitimate value. */
2217
+ nullable?: boolean;
2218
+ /**
2219
+ * Metadata: when true, this field is populated by the engine (not the caller)
2220
+ * and write attempts that include it MUST be rejected. Used for the
2221
+ * auto-injected `shortId` field. Consumers (autoCRUD create, schema
2222
+ * publisher, client codegen) rely on this flag as a declarative contract —
2223
+ * do not imperatively guard readonly fields without also flipping this.
2224
+ */
2225
+ readonly?: boolean;
2226
+ /**
2227
+ * Server-managed date flag. Clients may not supply the field in the
2228
+ * indicated phase(s) — autoCRUD rejects it with `client_supplied_server_field`:
2229
+ * - 'create' — on create only
2230
+ * - 'update' — on update only
2231
+ * - 'both' — on create and update
2232
+ * Nothing stamps it automatically: server-side code (a custom resolver or a
2233
+ * system writer such as `applyUpdatePatch`) sets it. A non-nullable,
2234
+ * non-defaulted field gated on create is refused by autoCRUD at install,
2235
+ * since nothing could fill it. Only honoured on `type: 'date'`.
2236
+ */
2237
+ serverSet?: 'create' | 'update' | 'both';
2238
+ /**
2239
+ * Controlled vocabulary. When present, the value MUST be one of these
2240
+ * strings. Consumed by autoCRUD write-validation (covers memory + pg
2241
+ * uniformly) and surfaced verbatim in the published schema. `type` stays
2242
+ * 'string' — an enum is a constrained string, so pack / client decoder /
2243
+ * where-validation treat it identically with no new branch.
2244
+ */
2245
+ enum?: readonly string[];
2246
+ /**
2247
+ * Governed vocabulary: alias -> canonical value (each canonical value is in
2248
+ * `enum`). autoCRUD stores the canonical value when an alias is written, and
2249
+ * `where` filters on the field match it, so synonyms converge instead of
2250
+ * sprawling. Matching is case-insensitive. Published in the schema so every
2251
+ * client reads the same table.
2252
+ */
2253
+ aliases?: Readonly<Record<string, string>>;
2254
+ /**
2255
+ * `false` admits values outside `enum` (aliases are still normalised): a
2256
+ * transition mode for adopting a vocabulary over existing free-string data.
2257
+ * Absent means strict.
2258
+ */
2259
+ strict?: boolean;
2260
+ /**
2261
+ * Alias (or canonical value) -> sibling fields it implies, e.g.
2262
+ * `done -> { resolution: 'completed' }`. On write the siblings are filled
2263
+ * when the caller omits them and a contradicting value is refused; an
2264
+ * `eq` filter on the alias also matches the implied siblings. Keys match
2265
+ * case-insensitively.
2266
+ */
2267
+ implies?: Readonly<Record<string, Readonly<Record<string, unknown>>>>;
2268
+ /**
2269
+ * Default value applied by autoCRUD `create` when the caller omits this
2270
+ * field. Two effects: (1) the row is written with the default instead of
2271
+ * being absent; (2) it satisfies the non-nullable presence check, so a
2272
+ * non-nullable scalar WITHOUT a default that is omitted on create is
2273
+ * rejected (`missing_required_field`) — that is how "required" is expressed.
2274
+ * Only applied on create (update is a partial patch). The value should match
2275
+ * `type` (booleans for boolean, ISO string for date, etc.).
2276
+ */
2277
+ default?: string | number | boolean;
2278
+ }
2279
+ interface FieldDefCrossRef {
2280
+ kind: 'crossRef';
2281
+ ns: string;
2282
+ type: string;
2283
+ onDelete: OnDelete;
2284
+ /**
2285
+ * Write-time existence policy. 'reject' refuses a create/update
2286
+ * whose crossRef value points at a non-existent or tombstoned target;
2287
+ * 'allow' is the default when absent. Consulted only at write time, never
2288
+ * stored on the edge. Enforced only for locally-resolvable
2289
+ * target namespaces (compose rejects 'reject' on bridged/unknown targets).
2290
+ */
2291
+ onMissing?: OnMissing;
2292
+ /**
2293
+ * Substrate-level indexing hint for the edge-source column, mirroring
2294
+ * `FieldDefScalar.indexed`, so the dataset need not infer it from the
2295
+ * presence of the crossRef itself.
2296
+ */
2297
+ indexed?: boolean;
2298
+ /** Metadata: when true, consumers SHOULD expect `null` as a legitimate value. */
2299
+ nullable?: boolean;
2300
+ /**
2301
+ * Activation-spreading opt-in. When `spread: true`,
2302
+ * recall hits on a row touching this crossRef propagate a damped salience
2303
+ * boost to the linked row (forward direction, X → row X references) AND to
2304
+ * rows whose same field points at the boosted row (backward direction,
2305
+ * X ← rows that reference X). Default off — operators opt in per-edge to
2306
+ * avoid polluting attention with low-signal edges (e.g. Decision.sessionId).
1775
2307
  *
1776
- * The implementation may use the dataset's current tx handle so the
1777
- * sequence advance rolls back on tx rollback (postgres sequences
1778
- * actually never roll back, but the surrounding tx semantics match
1779
- * what callers expect).
2308
+ * Damping (typically 0.3) + depth bound (1) are configured on the recall
2309
+ * substrate, not per-field.
1780
2310
  */
1781
- allocateShortId(type: string): Promise<number>;
2311
+ spread?: boolean;
1782
2312
  /**
1783
- * Record the (shortId → rowId) pairing after the row is written.
1784
- * Called by autoCRUD create immediately after the dataset write()
1785
- * returns the new row id. Safe to call within the enclosing tx.
2313
+ * Cross-TYPE opt-in for the recall expansion path.
2314
+ * Only meaningful when `spread: true` (the SDK `field.crossRef` builder
2315
+ * rejects `expandCrossType` without `spread` at registration). When both
2316
+ * are set, BACKWARD walks via this edge are allowed to surface neighbours
2317
+ * whose type differs from the request's `scope.type` (e.g. a Concept-typed
2318
+ * query surfaces Decisions via inbound `Decision.establishesConcept`).
2319
+ *
2320
+ * Cross-NAMESPACE expansion is always rejected regardless of this flag.
2321
+ * Forward walks always honour scope strictly: allowing them lets
2322
+ * neighbour-type rows displace the rows that establish them in the ranking.
2323
+ * Default false.
1786
2324
  */
1787
- bindShortId(type: string, shortId: number, rowId: string): Promise<void>;
2325
+ expandCrossType?: boolean;
2326
+ }
2327
+ interface FieldDefStruct {
2328
+ kind: 'struct';
2329
+ fields: Record<string, FieldDef>;
2330
+ /** Metadata: when true, consumers SHOULD expect `null` as a legitimate value. */
2331
+ nullable?: boolean;
2332
+ }
2333
+ interface FieldDefList {
2334
+ kind: 'list';
2335
+ of: FieldDef;
1788
2336
  /**
1789
- * Resolve a short id to the underlying row id, or null if no such short
1790
- * id exists for this (namespace, type). Does not join `rows` —
1791
- * callers chain their own `read()` after resolution.
2337
+ * When true, the `has` (element membership) operator is backed by a GIN
2338
+ * index on `(body -> '<field>')`. List fields are stored as native JSONB
2339
+ * arrays. Default false — `has` on an unindexed list still works, by
2340
+ * scanning the type's rows. The substrate binary derives the matching GIN
2341
+ * as an idempotent migration at boot from this flag; no hand-authored
2342
+ * migration is needed per field.
1792
2343
  */
1793
- resolveShortId(type: string, shortId: number): Promise<string | null>;
2344
+ indexed?: boolean;
2345
+ /** Metadata: when true, consumers SHOULD expect `null` as a legitimate value. */
2346
+ nullable?: boolean;
2347
+ }
2348
+ type FieldDef = FieldDefScalar | FieldDefCrossRef | FieldDefStruct | FieldDefList;
2349
+ /**
2350
+ * Per-type schema entry. Fields may be declared at registration time; MVP emits
2351
+ * `fields: {}` for types registered without a field map (e.g. mutations whose
2352
+ * shape is inferred from the resolver's return).
2353
+ */
2354
+ interface TypeSchema {
2355
+ fields: Record<string, FieldDef>;
2356
+ /** The type's natural order; absent means {@link DEFAULT_ORDERED_BY}. */
2357
+ orderedBy?: OrderedBy;
1794
2358
  /**
1795
- * Inverse of `resolveShortId` — look up the short id assigned to a
1796
- * given row id. Returns null if no sidecar entry exists (e.g. rows written
1797
- * before short ids were allocated).
2359
+ * Same-namespace crossRefs that point at this type: `field` on `type`.
2360
+ * Where a walk can come from. Absent when none.
1798
2361
  */
1799
- getShortIdForRow(type: string, rowId: string): Promise<number | null>;
2362
+ inboundRefs?: Array<{
2363
+ type: string;
2364
+ field: string;
2365
+ }>;
2366
+ }
2367
+ /**
2368
+ * A type's natural order: exactly one `{ field: direction }` key, the order a
2369
+ * read with no `orderBy` returns. The datasets still append `id` as the tie-break.
2370
+ */
2371
+ type OrderedBy = Record<string, 'asc' | 'desc'>;
2372
+ /** Natural order of a type that declares none: newest first. */
2373
+ declare const DEFAULT_ORDERED_BY: OrderedBy;
2374
+ /** The declared natural order, or {@link DEFAULT_ORDERED_BY}. */
2375
+ declare function naturalOrderOf(schema: Pick<TypeSchema, 'orderedBy'>): OrderedBy;
2376
+ /**
2377
+ * Per-mutation schema entry. `params` describes the input shape a client
2378
+ * sends; `returns` describes the shape a client decodes on success. Both are
2379
+ * required — an empty param bag surfaces as `{}`, never as an absent key.
2380
+ *
2381
+ * `returns` is typically a `FieldDefStruct` describing the row shape the
2382
+ * mutation produces (e.g. `create<T>` returns the inserted row; `delete<T>`
2383
+ * returns `{ id, deleted }`). `get<T>` and `update<T>` mark the struct
2384
+ * `nullable: true` because an id-miss legitimately produces `null`.
2385
+ *
2386
+ * Mutations registered without explicit `returns` metadata receive a sentinel
2387
+ * fallback (`{ kind: 'scalar', type: 'string', nullable: true }`) so substrate-pack
2388
+ * can still round-trip the response — clients treating it as "unknown shape"
2389
+ * should fall back to JSON passthrough.
2390
+ */
2391
+ interface MutationSchema {
2392
+ params: Record<string, FieldDef>;
2393
+ returns: FieldDef;
2394
+ }
2395
+ /**
2396
+ * The published schema shape surfaced at `GET /substrate/v1/schema`. Spec: §4.2, §6.1,
2397
+ * §7.1. `schemaVersion` is the 12-char SHA-256 prefix over the canonicalized
2398
+ * schema JSON and travels on every response envelope (§7.0) so the client can
2399
+ * detect drift.
2400
+ */
2401
+ interface Schema {
2402
+ schemaVersion: string;
2403
+ types: Record<string, TypeSchema>;
2404
+ mutations: Record<string, MutationSchema>;
2405
+ /** Present (true) on substrate-contributed virtual namespaces: no autoCRUD, no storage, query/mutate one-shot only. */
2406
+ virtual?: true;
1800
2407
  /**
1801
- * Spawn a sibling dataset bound to the same backing store but scoped to
1802
- * a different namespace. Used by `compose()` to auto-allocate per-namespace
1803
- * datasets when a coordinator plugin claims a namespace the operator hasn't
1804
- * mapped explicitly.
1805
- *
1806
- * Substrate semantics:
1807
- * - Pg: returns a new `PgDataset` sharing this dataset's `sql` pool +
1808
- * `edgesStore`. The returned dataset has the SAME `txScopeId` (so it
1809
- * can participate in cross-namespace batched mutations on the shared pool).
1810
- * - Memory: returns a new `MemoryDataset` with its own backing `Map`
1811
- * per the default convention (each instance owns its own substrate). The
1812
- * returned dataset has a NEW `txScopeId` — it cannot share a transaction
1813
- * with the caller. Test harnesses that need shared-memory semantics across
1814
- * namespaces should construct each `MemoryDataset` explicitly with
1815
- * the relevant constructor option.
1816
- *
1817
- * Idempotency note: callers SHOULD NOT call `forNamespace(this.namespace)`;
1818
- * the resulting object's identity is implementation-defined. Use this only
1819
- * to derive datasets for *different* namespaces.
2408
+ * The namespace's opaque per-consumer metadata (`defineNamespace` option).
2409
+ * Not part of `schemaVersion`: an annotations-only change does not move it.
1820
2410
  */
1821
- forNamespace(namespace: string): Dataset;
2411
+ annotations?: Readonly<Record<string, unknown>>;
1822
2412
  }
1823
- interface DatasetRegistry {
1824
- get(ns: string): Dataset | undefined;
1825
- has(ns: string): boolean;
1826
- set(ns: string, dataset: Dataset): void;
2413
+ /**
2414
+ * Per-key shape. A single frame resolves to `'single' | 'list'`, or for a
2415
+ * walk read to `'count'` (`data[key]` is `{ count }`) or `'groups'`
2416
+ * (`data[key]` is `{ groups, truncated? }`). A batched mutate op-key (array
2417
+ * of frames) resolves to `Array<'single' | 'list'>` — one shape per frame, in
2418
+ * frame-array order. The decoder keys off the discriminator: string → single
2419
+ * frame; array → batched.
2420
+ */
2421
+ type EnvelopeShape = 'single' | 'list' | 'count' | 'groups' | Array<'single' | 'list'>;
2422
+ /**
2423
+ * The response envelope. Spec: §7.0.
2424
+ *
2425
+ * Query data values are positionally encoded using the client-sent `FieldInput`
2426
+ * selection; mutation data values are positionally encoded using the declared
2427
+ * `MutationSchema.returns` (§4.2) — mutations don't carry a FieldInput
2428
+ * since there's no field selection on a return row.
2429
+ */
2430
+ interface PackEnvelope {
2431
+ schemaVersion: string;
2432
+ data: Record<string, unknown>;
2433
+ shapes: Record<string, EnvelopeShape>;
2434
+ _meta?: Record<string, unknown>;
2435
+ _pagination?: Record<string, {
2436
+ nextCursor?: string;
2437
+ hasMore: boolean;
2438
+ }>;
1827
2439
  /**
1828
- * Observe namespace lifecycle events (§6.4). Fires when a namespace is
1829
- * disposed — explicit unregister or substrate-signalled loss (memory-backed
1830
- * restart). Returns a disposer that unsubscribes when invoked.
2440
+ * Per-top-level-key completeness. ABSENT MEANS COMPLETE — callers that
2441
+ * predate this field stay correct without change.
1831
2442
  *
1832
- * Optional: registries over persistent substrates (pg) never lose namespaces
1833
- * at runtime and can omit this surface entirely. Consumers MUST feature-
1834
- * detect before subscribing.
1835
- */
1836
- onNamespaceLost?(listener: (ns: string) => void): () => void;
2443
+ * Set when a bridged key could not be fully resolved (§6.5 "reads never
2444
+ * fail" means such a read still returns 200 with orphan markers). The
2445
+ * per-row `__orphan` marker says a row is missing; this says the RESULT SET
2446
+ * is partial, which a consumer cannot otherwise distinguish from an
2447
+ * genuinely empty result.
2448
+ */
2449
+ _completeness?: Record<string, {
2450
+ complete: boolean;
2451
+ unreachable?: string[];
2452
+ }>;
1837
2453
  /**
1838
- * Explicitly drop `ns` from the registry and fire `onNamespaceLost`
1839
- * listeners synchronously. Disposing an unregistered namespace is a silent
1840
- * no-op — keeps graceful-shutdown replays idempotent.
2454
+ * Non-fatal read diagnostics, keyed by top-level frame key. ABSENT MEANS
2455
+ * NOTHING TO REPORT — callers that predate this field stay correct without
2456
+ * change, same contract as `_completeness`.
1841
2457
  *
1842
- * Optional: same rationale as `onNamespaceLost`.
2458
+ * Deliberately NOT `errors`: a consumer doing `if (env.errors) throw` would
2459
+ * start failing on reads that succeeded.
2460
+ *
2461
+ * A client that maps rows into declared interfaces MUST surface this (log,
2462
+ * attach to the result, or throw in development): an unknown projection key
2463
+ * arrives as `null`, satisfies the interface, and the read looks correct.
1843
2464
  */
1844
- dispose?(ns: string): void;
2465
+ _diagnostics?: Record<string, {
2466
+ /**
2467
+ * Projection keys the type does not declare. The read still returned
2468
+ * every row; each unknown key materialises as `null` on all of them,
2469
+ * indistinguishable from a declared field that is null everywhere.
2470
+ */
2471
+ unknownProjectionFields?: string[];
2472
+ /**
2473
+ * The where-clause reached the substrate without being schema-validated,
2474
+ * because the type is registered but absent from its namespace's
2475
+ * published schema (e.g. runtime-created types). The result is still
2476
+ * correct — substrates refuse a predicate they cannot honour — but the
2477
+ * engine's operator and operand check did not run.
2478
+ */
2479
+ filterValidationSkipped?: string;
2480
+ /**
2481
+ * A walk-grammar read's clauses, each marked index-served or not
2482
+ * (typed edge walks, goal 7). Annotation only: nothing is refused on
2483
+ * it. Absent on ordinary reads, and when the store has no index plan.
2484
+ */
2485
+ plan?: PlanClause[];
2486
+ }>;
2487
+ errors?: Array<{
2488
+ message: string;
2489
+ path?: string[];
2490
+ code?: string;
2491
+ /**
2492
+ * Optional structured payload — populated when the underlying error class
2493
+ * carries one (e.g. `ResolverFrameError({ field: '...' })`).
2494
+ */
2495
+ details?: Record<string, unknown>;
2496
+ }>;
1845
2497
  }
1846
2498
  /**
1847
- * One declared index of a type and its state: as the catalog holds it, or,
1848
- * while this process defers or runs its build, `pending`, `building` or
1849
- * `failed`. Filters on an index that is not `valid` scan.
2499
+ * Client-sent field selection tree. Defined here rather than in
2500
+ * `@mnmal-ai/substrate-pack` so subscription and streaming machinery (§5.4, §5.5)
2501
+ * can reference the same shape without adding substrate-pack to their dependency
2502
+ * set — substrate-pack already imports from types, so moving the type upward
2503
+ * avoids the would-be cycle.
1850
2504
  */
1851
- interface TypeIndexState {
1852
- name: string;
1853
- fields: string[];
1854
- unique: boolean;
1855
- state: 'valid' | 'building' | 'pending' | 'invalid' | 'failed';
1856
- outcome: 'created' | 'adopted' | 'swapped' | 'unchanged';
1857
- /** Why a `failed` index failed: an error code, then a short reason. */
1858
- error?: string;
1859
- /** Why a `pending` index was not built: the free-disk check refused it. */
1860
- reason?: 'insufficient_disk';
1861
- }
1862
- /** Where a type's rows live: `partition` is set on the partitioned layout once the type has one. */
1863
- interface TypeStorageReport {
1864
- layout: 'legacy' | 'partitioned';
1865
- partition?: string;
1866
- indexes: TypeIndexState[];
1867
- }
1868
- interface StorageReport {
1869
- /** `unknown` when the report itself failed. */
1870
- layout: 'legacy' | 'partitioned' | 'unknown';
1871
- warnings: string[];
1872
- /** Worth showing, not a fault: the doctor prints these without failing. */
1873
- info?: string[];
1874
- }
1875
- /** See `Dataset.indexPlan`. */
1876
- interface IndexPlan {
1877
- /** Each desired index, its keys in order: a derived index ends with the type's natural order. */
1878
- indexes: Array<{
1879
- name: string;
1880
- keys: Array<[string, 'asc' | 'desc']>;
1881
- unique: boolean;
1882
- /** A partial index's predicate beyond `live`: it serves only reads that `eq` each field to its value. */
1883
- only?: Record<string, boolean | null>;
1884
- }>;
1885
- expectedSize?: 'small' | 'large';
1886
- }
2505
+ type FieldInput = {
2506
+ [field: string]: true | FieldInput;
2507
+ };
1887
2508
  //#endregion
1888
- export { Registry as $, DropParticipant as A, Resolver as At, KernelMigrationContext as B, ResolverStat as C, AutoCrudVerb as Ct, SlowRequest as D, PolicyDecision as Dt, RuntimeMetrics as E, Policy as Et, FilterOperatorDef as F, WriteEvent as Ft, NamespaceConfig as G, KernelPlugin as H, FrameEntry as I, NamespaceTypeDef as J, NamespaceDefinition as K, IndexDecl as L, EncodeContext as M, ResolverReturn as Mt, EngineHandle as N, RowFilterObligation as Nt, TypeStats as O, PolicyProbeContext as Ot, EngineInput as P, TypedResolver as Pt, QualifiedFrame as Q, KernelLifecycleContext as R, ResolveObservation as S, AfterTxWork as St, RuntimeErrors as T, Obligation as Tt, KernelRouteClaim as U, KernelNamespaceContribution as V, McpToolRegistration as W, PluginRouteContext as X, PluginLifecycleContext as Y, PostWriteHook as Z, UpdateFieldRef as _, SubstrateResponse as _t, DropResult as a, ScheduledTask as at, WriteResult as b, WsConnection as bt, FilterCapabilities as c, VirtualTypeDef as ct, ReadParams as d, PluginRouter as dt, RegistryBase as et, ReadResult as f, RequestObservation as ft, TypeStorageReport as g, SubstrateRequest as gt, TypeIndexState as h, StartOpts as ht, DeleteResult as i, RegistryEntryResolver as it, EmbedSourceContext as j, ResolverArgs as jt, CoordinatorPlugin as k, RequestContext as kt, IndexPlan as l, HttpHandler as lt, TxScopeHandles as m, RuntimeOp as mt, Dataset as n, RegistryEntryInput as nt, FieldKind as o, ScheduledTaskContext as ot, StorageReport as p, RequestObserver as pt, NamespaceInstallContext as q, DatasetRegistry as r, RegistryEntryKind as rt, FieldPredicate as s, VirtualMutationDef as st, ConditionalUpdateResult as t, RegistryEntry as tt, OrderByClause as u, HttpMethod as ut, WhereClause as v, SubstrateServer as vt, RuntimeErrorEntry as w, Identity as wt, ProcessStat as x, WsHandler as xt, WriteOptions as y, TlsConfig as yt, KernelMigration as z };
2509
+ export { PluginRouter as $, WriteOptions as $t, KernelPlugin as A, DeriveSpec as At, QualifiedFrame as B, ReadResult as Bt, FilterOperatorDef as C, Row as Cn, WriteEvent as Ct, KernelMigration as D, assertNoRepeatedIds as Dn, Dataset as Dt, KernelLifecycleContext as E, WriteRow as En, ConditionalUpdateResult as Et, NamespaceInstallContext as F, FrameAgg as Ft, RegistryEntryKind as G, TypeStorageReport as Gt, RegistryBase as H, StorageReport as Ht, NamespaceTypeDef as I, IndexPlan as It, ScheduledTaskContext as J, WalkAggOperand as Jt, RegistryEntryResolver as K, UpdateFieldRef as Kt, PluginLifecycleContext as L, NestedSelection as Lt, McpToolRegistration as M, FieldKind as Mt, NamespaceConfig as N, FieldPredicate as Nt, KernelMigrationContext as O, DatasetRegistry as Ot, NamespaceDefinition as P, FilterCapabilities as Pt, HttpMethod as Q, WhereClause as Qt, PluginRouteContext as R, OrderByClause as Rt, EngineInput as S, ROW_METADATA_FIELDS as Sn, TypedResolver as St, IndexDecl as T, RowMetadataField as Tn, BucketSpec as Tt, RegistryEntry as U, TxScopeHandles as Ut, Registry as V, RefOperand as Vt, RegistryEntryInput as W, TypeIndexState as Wt, VirtualTypeDef as X, WalkNode as Xt, VirtualMutationDef as Y, WalkAggPredicate as Yt, HttpHandler as Z, WalkPick as Zt, CoordinatorPlugin as _, ActorIdentity as _n, RequestContext as _t, FieldDefCrossRef as a, RuntimeErrors as an, SubstrateResponse as at, EncodeContext as b, Id as bn, ResolverReturn as bt, FieldDefStruct as c, TypeStats as cn, WsConnection as ct, OrderedBy as d, EdgePatch as dn, AutoCrudVerb as dt, WriteResult as en, RequestObservation as et, PackEnvelope as f, EdgeRef as fn, Identity as ft, naturalOrderOf as g, OnMissing as gn, PolicyProbeContext as gt, TypeSchema as h, OnDelete as hn, PolicyDecision as ht, FieldDef as i, RuntimeErrorEntry as in, SubstrateRequest as it, KernelRouteClaim as j, DropResult as jt, KernelNamespaceContribution as k, DeleteResult as kt, FieldInput as l, CanDeleteResult as ln, WsHandler as lt, Schema as m, EdgesStore as mn, Policy as mt, Dropped as n, ResolveObservation as nn, RuntimeOp as nt, FieldDefList as o, RuntimeMetrics as on, SubstrateServer as ot, PlanClause as p, EdgeWriteInput as pn, Obligation as pt, ScheduledTask as q, WalkAgg as qt, EnvelopeShape as r, ResolverStat as rn, StartOpts as rt, FieldDefScalar as s, SlowRequest as sn, TlsConfig as st, DEFAULT_ORDERED_BY as t, ProcessStat as tn, RequestObserver as tt, MutationSchema as u, Edge as un, AfterTxWork as ut, DropParticipant as v, CLIENT_SUPPLIED_SERVER_FIELD as vn, Resolver as vt, FrameEntry as w, RowMetadata as wn, AggFn as wt, EngineHandle as x, IdentityKind as xn, RowFilterObligation as xt, EmbedSourceContext as y, ClientSuppliedServerFieldError as yn, ResolverArgs as yt, PostWriteHook as z, ReadParams as zt };