@mnmal-ai/substrate 0.0.0-stage → 1.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +119 -2
- package/dist/agent-token-BIboWOU0.js +200 -0
- package/dist/client.d.ts +782 -0
- package/dist/client.js +7 -0
- package/dist/cluster-worker.js +35 -0
- package/dist/content-type-C7T28xNb.js +25 -0
- package/dist/dataset-BXkus75W.js +1527 -0
- package/dist/env-utils-B7SGmoeJ.js +21 -0
- package/dist/errors-D4vKuwz8.d.ts +130 -0
- package/dist/fake-DUVtKVSE.js +36 -0
- package/dist/index-D7mSOFlM.d.ts +69 -0
- package/dist/pack.d.ts +192 -0
- package/dist/pack.js +5 -0
- package/dist/plugin-j1jt1l3n.js +3844 -0
- package/dist/plugin-t2g2wX0b.d.ts +1568 -0
- package/dist/proposals-bootstrap.js +37 -0
- package/dist/proposals.config.schema.json +20 -0
- package/dist/proposals.d.ts +118 -0
- package/dist/proposals.default-config.json +4 -0
- package/dist/proposals.js +4 -0
- package/dist/recall-bootstrap.js +451 -0
- package/dist/recall.config.schema.json +232 -0
- package/dist/recall.d.ts +1981 -0
- package/dist/recall.default-config.json +4 -0
- package/dist/recall.js +5 -0
- package/dist/repl-C52wyfMs.js +2213 -0
- package/dist/row-ByLFE2Wx.d.ts +719 -0
- package/dist/row-DK7lPZVe.js +732 -0
- package/dist/runtime-metrics-DwpDJ8vz.d.ts +159 -0
- package/dist/schema-g4FQLr0V.d.ts +433 -0
- package/dist/sdk.d.ts +694 -0
- package/dist/sdk.js +112 -0
- package/dist/src-C4DzaIQZ.js +837 -0
- package/dist/src-CrEgiMJQ.js +1015 -0
- package/dist/src-CzxvmhFn.js +2327 -0
- package/dist/src-D88LvCX8.js +1797 -0
- package/dist/src-D91yij6C.js +13523 -0
- package/dist/src-DuopV8u7.js +2408 -0
- package/dist/substrate-admin.js +808 -0
- package/dist/substrate.js +5477 -0
- package/dist/testing.d.ts +1240 -0
- package/dist/testing.js +702 -0
- package/dist/types-8c0gtnd2.d.ts +35 -0
- package/dist/ws-C72iVS5T.d.ts +143 -0
- package/package.json +117 -3
package/dist/recall.d.ts
ADDED
|
@@ -0,0 +1,1981 @@
|
|
|
1
|
+
import { E as Row, _ as EdgesStore, n as FieldDef } from "./schema-g4FQLr0V.js";
|
|
2
|
+
import { $ as PolicyProbeContext, P as HttpHandler, Q as PolicyDecision, S as PostWriteHook, Y as Identity, ct as Dataset, et as RequestContext, h as McpToolRegistration, nt as ResolverArgs, p as KernelPlugin, r as EmbedSourceContext, s as FilterOperatorDef, u as KernelMigration } from "./plugin-t2g2wX0b.js";
|
|
3
|
+
import { t as Embedder } from "./types-8c0gtnd2.js";
|
|
4
|
+
import postgres, { Sql } from "postgres";
|
|
5
|
+
//#region ../../plugins/substrate-recall/src/candidate-selection.d.ts
|
|
6
|
+
/**
|
|
7
|
+
* Candidate selection for crystallisation.
|
|
8
|
+
*
|
|
9
|
+
* Recall does not hardcode what counts as junk — that is the consumer's
|
|
10
|
+
* policy — but it does require the filter to be explicit. There is no
|
|
11
|
+
* "cluster everything" default: near-duplicate fixtures form the densest
|
|
12
|
+
* regions and would dominate the clusters.
|
|
13
|
+
*
|
|
14
|
+
* `bad_request`-coded throws map to 400 on the route and to
|
|
15
|
+
* `ResolverFrameError('bad_request', …)` on the frame surface.
|
|
16
|
+
*/
|
|
17
|
+
/** One (namespace, type) pair to consider. */
|
|
18
|
+
interface SelectorScope {
|
|
19
|
+
namespace: string;
|
|
20
|
+
type: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Drop rows whose `body->>field` starts with `prefix`.
|
|
24
|
+
*/
|
|
25
|
+
interface TextPrefixExclusion {
|
|
26
|
+
field: string;
|
|
27
|
+
prefix: string;
|
|
28
|
+
}
|
|
29
|
+
interface CandidateSelector {
|
|
30
|
+
scopes: SelectorScope[];
|
|
31
|
+
excludeTextPrefixes: TextPrefixExclusion[];
|
|
32
|
+
limit: number;
|
|
33
|
+
}
|
|
34
|
+
interface Candidate {
|
|
35
|
+
namespace: string;
|
|
36
|
+
type: string;
|
|
37
|
+
id: string;
|
|
38
|
+
/** The stored vector, exactly as `embeddings` holds it. */
|
|
39
|
+
vector: Float32Array;
|
|
40
|
+
/** The full row body, used for labelling. */
|
|
41
|
+
row: Record<string, unknown>;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Ceiling on one selection, with headroom under `WasmClusterer`'s
|
|
45
|
+
* `WASM_MAX_ROWS` refusal limit.
|
|
46
|
+
*/
|
|
47
|
+
export declare const SELECT_CANDIDATES_MAX_LIMIT = 20000;
|
|
48
|
+
export declare function validateSelector(value: unknown): CandidateSelector;
|
|
49
|
+
//#endregion
|
|
50
|
+
//#region ../../plugins/substrate-recall/src/cluster-store.d.ts
|
|
51
|
+
/**
|
|
52
|
+
* ClusterStore — persistence for crystallised clusters and their membership.
|
|
53
|
+
* Implemented for pg and in-memory, like `EmbeddingsStore`.
|
|
54
|
+
*
|
|
55
|
+
* Membership is a row, not an edge: it carries `origin` (formation vs
|
|
56
|
+
* accretion), the similarity at attach time, and a `departedAt` stamp for
|
|
57
|
+
* mark-don't-remove.
|
|
58
|
+
*/
|
|
59
|
+
/**
|
|
60
|
+
* How a clustering was produced. Required because different backends, and the
|
|
61
|
+
* same backend on differently-rounded input, produce different assignments.
|
|
62
|
+
*/
|
|
63
|
+
interface ClusterProvenance {
|
|
64
|
+
/** `'wasm' | 'subprocess'` — which holomap-clusterer backend ran. */
|
|
65
|
+
backend: string;
|
|
66
|
+
/** The clusterer package version, e.g. `'0.1.0'`. */
|
|
67
|
+
backendVersion: string;
|
|
68
|
+
/**
|
|
69
|
+
* The float path the input took. `'normalised'` means the vectors were
|
|
70
|
+
* L2-normalised before the pipeline (which normalises again internally).
|
|
71
|
+
*/
|
|
72
|
+
regime: 'raw' | 'normalised';
|
|
73
|
+
seed: number;
|
|
74
|
+
params: {
|
|
75
|
+
nComponents: number;
|
|
76
|
+
nNeighbors: number;
|
|
77
|
+
minClusterSize: number;
|
|
78
|
+
};
|
|
79
|
+
/** The embedder model whose vector space this cluster lives in. */
|
|
80
|
+
model: string;
|
|
81
|
+
}
|
|
82
|
+
interface ClusterRecord {
|
|
83
|
+
id: string;
|
|
84
|
+
shortId: number;
|
|
85
|
+
/**
|
|
86
|
+
* Mean of member vectors in the ORIGINAL embedding space, never reduced
|
|
87
|
+
* space: a reduced centroid cannot be searched against the full index.
|
|
88
|
+
*/
|
|
89
|
+
centroid: Float32Array;
|
|
90
|
+
memberCount: number;
|
|
91
|
+
departedCount: number;
|
|
92
|
+
cohesion: number;
|
|
93
|
+
label: string;
|
|
94
|
+
labelledBy: 'medoid' | 'agent';
|
|
95
|
+
provenance: ClusterProvenance;
|
|
96
|
+
formedAt: Date;
|
|
97
|
+
lastAccretedAt: Date | null;
|
|
98
|
+
retiredAt: Date | null;
|
|
99
|
+
}
|
|
100
|
+
interface ClusterMemberRecord {
|
|
101
|
+
id: string;
|
|
102
|
+
shortId: number;
|
|
103
|
+
clusterId: string;
|
|
104
|
+
memberNs: string;
|
|
105
|
+
memberType: string;
|
|
106
|
+
memberId: string;
|
|
107
|
+
origin: 'formation' | 'accretion';
|
|
108
|
+
similarity: number;
|
|
109
|
+
attachedAt: Date;
|
|
110
|
+
departedAt: Date | null;
|
|
111
|
+
}
|
|
112
|
+
/** Identifies one source row. */
|
|
113
|
+
interface MemberRef {
|
|
114
|
+
namespace: string;
|
|
115
|
+
type: string;
|
|
116
|
+
id: string;
|
|
117
|
+
}
|
|
118
|
+
interface NewCluster {
|
|
119
|
+
id: string;
|
|
120
|
+
centroid: Float32Array;
|
|
121
|
+
cohesion: number;
|
|
122
|
+
label: string;
|
|
123
|
+
labelledBy: 'medoid' | 'agent';
|
|
124
|
+
provenance: ClusterProvenance;
|
|
125
|
+
/** Override for tests. Defaults to `new Date()` inside the impl. */
|
|
126
|
+
now?: Date;
|
|
127
|
+
}
|
|
128
|
+
interface NewMember {
|
|
129
|
+
id: string;
|
|
130
|
+
clusterId: string;
|
|
131
|
+
ref: MemberRef;
|
|
132
|
+
origin: 'formation' | 'accretion';
|
|
133
|
+
similarity: number;
|
|
134
|
+
now?: Date;
|
|
135
|
+
}
|
|
136
|
+
interface ClusterStore {
|
|
137
|
+
/** Parallels `EmbeddingsStore.txScopeId` / `Dataset.txScopeId`. */
|
|
138
|
+
readonly txScopeId: string;
|
|
139
|
+
/** Insert a freshly formed cluster with zero members. */
|
|
140
|
+
createCluster(input: NewCluster): Promise<ClusterRecord>;
|
|
141
|
+
/**
|
|
142
|
+
* Attach members and update the parent clusters' `memberCount` +
|
|
143
|
+
* `lastAccretedAt` in one atomic unit. Re-attaching an existing
|
|
144
|
+
* (cluster, member) pair is a silent no-op — accretion may legitimately
|
|
145
|
+
* see the same row twice across concurrent passes.
|
|
146
|
+
*/
|
|
147
|
+
attachMembers(inputs: readonly NewMember[]): Promise<void>;
|
|
148
|
+
/** Live clusters, oldest first. Retired clusters excluded unless asked for. */
|
|
149
|
+
listClusters(opts?: {
|
|
150
|
+
includeRetired?: boolean;
|
|
151
|
+
}): Promise<ClusterRecord[]>;
|
|
152
|
+
getCluster(id: string): Promise<ClusterRecord | null>;
|
|
153
|
+
/** Members of one cluster. Departed members excluded unless asked for. */
|
|
154
|
+
listMembers(clusterId: string, opts?: {
|
|
155
|
+
includeDeparted?: boolean;
|
|
156
|
+
}): Promise<ClusterMemberRecord[]>;
|
|
157
|
+
/**
|
|
158
|
+
* Every live (non-departed) membership across every cluster, in one query.
|
|
159
|
+
* Spans RETIRED clusters deliberately: the decay sweep must release
|
|
160
|
+
* survivors stranded on them.
|
|
161
|
+
*/
|
|
162
|
+
listAllMembers(): Promise<ClusterMemberRecord[]>;
|
|
163
|
+
/** The reverse walk: every membership row naming this source row. */
|
|
164
|
+
findClustersForMember(ref: MemberRef): Promise<ClusterMemberRecord[]>;
|
|
165
|
+
/**
|
|
166
|
+
* Of `refs`, which are already claimed by a live (non-departed) membership?
|
|
167
|
+
* Returned as `"ns/type/id"` keys. This is the never-reassign gate: both
|
|
168
|
+
* formation and accretion subtract this set before doing anything.
|
|
169
|
+
*/
|
|
170
|
+
claimedMembers(refs: readonly MemberRef[]): Promise<Set<string>>;
|
|
171
|
+
/**
|
|
172
|
+
* Mark memberships departed and re-derive the owning clusters' counts.
|
|
173
|
+
* Idempotent — a already-departed membership keeps its original
|
|
174
|
+
* `departedAt`. Returns how many rows transitioned.
|
|
175
|
+
*/
|
|
176
|
+
markDeparted(refs: readonly MemberRef[], now?: Date): Promise<number>;
|
|
177
|
+
/** Set `retiredAt`. Idempotent; a retired cluster keeps its original stamp. */
|
|
178
|
+
retire(clusterId: string, now?: Date): Promise<void>;
|
|
179
|
+
/** Replace the label. `labelledBy` becomes `'agent'`. Returns null if absent. */
|
|
180
|
+
relabel(clusterId: string, label: string): Promise<ClusterRecord | null>;
|
|
181
|
+
/**
|
|
182
|
+
* Run `fn` holding a lock that excludes formation in EVERY process sharing
|
|
183
|
+
* this store, or throw `crystallise_in_flight` at once if another holds it.
|
|
184
|
+
* Never waits. Omitted by stores that only one process can see (in-memory).
|
|
185
|
+
*/
|
|
186
|
+
withFormationLock?<T>(fn: () => Promise<T>): Promise<T>;
|
|
187
|
+
}
|
|
188
|
+
/** Canonical key for a member ref — the form `claimedMembers` returns. */
|
|
189
|
+
export declare function memberKey(ref: MemberRef): string;
|
|
190
|
+
//#endregion
|
|
191
|
+
//#region ../../plugins/substrate-recall/src/embeddings-store.d.ts
|
|
192
|
+
interface EmbeddingsStoreUpsertInput {
|
|
193
|
+
rowNamespace: string;
|
|
194
|
+
rowType: string;
|
|
195
|
+
rowId: string;
|
|
196
|
+
vector: Float32Array;
|
|
197
|
+
/**
|
|
198
|
+
* Embedder identifier (e.g. `some-model:768`). Stored alongside the
|
|
199
|
+
* vector so multi-model coexistence is possible — a `knnSearch` call
|
|
200
|
+
* filters on this so vectors generated by different embedders don't mix.
|
|
201
|
+
*/
|
|
202
|
+
model: string;
|
|
203
|
+
/**
|
|
204
|
+
* When the source row was written; defaults to now. An upsert older than
|
|
205
|
+
* the stored vector is ignored, so a slow embed cannot overwrite a newer one.
|
|
206
|
+
*/
|
|
207
|
+
now?: Date;
|
|
208
|
+
}
|
|
209
|
+
/** Identifies the single embedding a `delete` call removes. */
|
|
210
|
+
interface EmbeddingsStoreDeleteInput {
|
|
211
|
+
rowNamespace: string;
|
|
212
|
+
rowType: string;
|
|
213
|
+
rowId: string;
|
|
214
|
+
}
|
|
215
|
+
interface EmbeddingsStoreKnnInput {
|
|
216
|
+
/** Row scope. Most callers know exactly one (namespace, type) at recall time. */
|
|
217
|
+
namespace: string;
|
|
218
|
+
type: string;
|
|
219
|
+
/**
|
|
220
|
+
* Active embedder model. Vectors stored under a different model are
|
|
221
|
+
* filtered out — they live in a different vector space and would produce
|
|
222
|
+
* meaningless similarity scores.
|
|
223
|
+
*/
|
|
224
|
+
model: string;
|
|
225
|
+
/** The query vector. Must match the model's dimensionality. */
|
|
226
|
+
vector: Float32Array;
|
|
227
|
+
/** Max rows to return. Stores cap at their own ceiling (typically 100). */
|
|
228
|
+
k: number;
|
|
229
|
+
/**
|
|
230
|
+
* Cosine-similarity threshold (0..1). Rows with score below this are
|
|
231
|
+
* dropped. Default 0 (no threshold).
|
|
232
|
+
*/
|
|
233
|
+
threshold?: number;
|
|
234
|
+
/**
|
|
235
|
+
* When `true`, each result carries its stored embedding vector. Opt-in
|
|
236
|
+
* because vectors are large and most callers only need scores.
|
|
237
|
+
*/
|
|
238
|
+
returnVector?: boolean;
|
|
239
|
+
/**
|
|
240
|
+
* Restrict candidates to rows whose fields match, applied INSIDE the
|
|
241
|
+
* search rather than after it, so a scope like one project in a
|
|
242
|
+
* store shared by several still fills k. Equality family only.
|
|
243
|
+
*/
|
|
244
|
+
rowFilter?: KnnRowFilter;
|
|
245
|
+
}
|
|
246
|
+
type KnnScalar = string | number | boolean | null;
|
|
247
|
+
/** Field -> `{ eq }` or `{ in }`; every field must match. `eq: null` matches a missing or null field. */
|
|
248
|
+
type KnnRowFilter = Readonly<Record<string, {
|
|
249
|
+
readonly eq?: KnnScalar;
|
|
250
|
+
readonly in?: readonly KnnScalar[];
|
|
251
|
+
}>>;
|
|
252
|
+
interface EmbeddingsStoreKnnResult {
|
|
253
|
+
/** The full `rows.body` JSON for the matched row. */
|
|
254
|
+
row: Record<string, unknown>;
|
|
255
|
+
/** Cosine similarity in `[0, 1]` — higher is closer. */
|
|
256
|
+
score: number;
|
|
257
|
+
/**
|
|
258
|
+
* Per-row salience: a float counter incremented on each recall hit, with
|
|
259
|
+
* exponential decay applied at boost time. `0` for never-recalled rows.
|
|
260
|
+
*/
|
|
261
|
+
salience: number;
|
|
262
|
+
/**
|
|
263
|
+
* The row's stored embedding vector. Present iff the search was called
|
|
264
|
+
* with `returnVector: true`; absent otherwise.
|
|
265
|
+
*/
|
|
266
|
+
vector?: number[];
|
|
267
|
+
}
|
|
268
|
+
/** Identifies one row to bump in a `boostSalience` call. */
|
|
269
|
+
interface SalienceTarget {
|
|
270
|
+
namespace: string;
|
|
271
|
+
type: string;
|
|
272
|
+
id: string;
|
|
273
|
+
}
|
|
274
|
+
/** One row returned by `fetchEmbeddings` — body joined to the stored vector. */
|
|
275
|
+
interface EmbeddingsStoreFetchResult {
|
|
276
|
+
namespace: string;
|
|
277
|
+
type: string;
|
|
278
|
+
id: string;
|
|
279
|
+
row: Record<string, unknown>;
|
|
280
|
+
vector: Float32Array;
|
|
281
|
+
/** Per-row salience, the same signal `knnSearch` returns. `0` for never-reinforced rows. */
|
|
282
|
+
salience: number;
|
|
283
|
+
}
|
|
284
|
+
interface EmbeddingsStore {
|
|
285
|
+
/**
|
|
286
|
+
* Stable identifier for the storage substrate this store wraps; parallels
|
|
287
|
+
* `Dataset.txScopeId` (§5.3). Embedding writes are not enlisted in
|
|
288
|
+
* the substrate tx — they are a fire-and-forget post-write side effect.
|
|
289
|
+
*/
|
|
290
|
+
readonly txScopeId: string;
|
|
291
|
+
/**
|
|
292
|
+
* Insert or replace the embedding for a row. Idempotent — repeat calls
|
|
293
|
+
* with the same key tuple overwrite the prior vector + bump `updated_at`.
|
|
294
|
+
*
|
|
295
|
+
* Vector dimensionality is enforced by the substrate (pgvector column
|
|
296
|
+
* type). The store does NOT validate dimensionality client-side; a
|
|
297
|
+
* mismatched vector surfaces as a substrate-level error.
|
|
298
|
+
*/
|
|
299
|
+
upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
|
|
300
|
+
/**
|
|
301
|
+
* Remove the embedding for one row, leaving the row itself untouched. Used
|
|
302
|
+
* to un-index a row whose `embed.source` returns the null sentinel.
|
|
303
|
+
* Idempotent: deleting an absent embedding is a no-op.
|
|
304
|
+
*/
|
|
305
|
+
delete(input: EmbeddingsStoreDeleteInput): Promise<void>;
|
|
306
|
+
/**
|
|
307
|
+
* KNN search — return up to `k` rows whose embeddings are closest to
|
|
308
|
+
* `input.vector` (cosine similarity), filtered to the active model.
|
|
309
|
+
* Results are joined against the row body so callers don't need a
|
|
310
|
+
* second round-trip to read the matched rows.
|
|
311
|
+
*/
|
|
312
|
+
knnSearch(input: EmbeddingsStoreKnnInput): Promise<EmbeddingsStoreKnnResult[]>;
|
|
313
|
+
/**
|
|
314
|
+
* Apply a salience boost to each target row, with lazy decay:
|
|
315
|
+
* `salience_new = salience_old * 2^(-elapsed_hours / halfLifeHours) + bump`
|
|
316
|
+
* and `salience_updated_at = now()`. No periodic decay sweep is needed.
|
|
317
|
+
*
|
|
318
|
+
* `bump` defaults to `1` (direct recall hits); activation spreading passes
|
|
319
|
+
* `spreadDamping`. Targets without an embedding row are silent no-ops.
|
|
320
|
+
*
|
|
321
|
+
* Returns the targets that actually matched a stored embedding, in no
|
|
322
|
+
* specified order; callers wanting the misses should difference against
|
|
323
|
+
* what they sent rather than zipping by index.
|
|
324
|
+
*/
|
|
325
|
+
boostSalience(targets: ReadonlyArray<SalienceTarget>, halfLifeHours: number, bump?: number): Promise<ReadonlyArray<SalienceTarget>>;
|
|
326
|
+
/**
|
|
327
|
+
* Fetch stored embeddings for a list of row IDs, each joined to its row
|
|
328
|
+
* body. IDs without an embedding under `model` are silently absent. One
|
|
329
|
+
* batched lookup; callers should not loop calling this per ID.
|
|
330
|
+
*/
|
|
331
|
+
fetchEmbeddings(ids: ReadonlyArray<{
|
|
332
|
+
namespace: string;
|
|
333
|
+
type: string;
|
|
334
|
+
id: string;
|
|
335
|
+
}>, model: string): Promise<ReadonlyArray<EmbeddingsStoreFetchResult>>;
|
|
336
|
+
/**
|
|
337
|
+
* Select clustering candidates — every embedded row under the selector's
|
|
338
|
+
* scopes, minus its exclusions, with vector and body.
|
|
339
|
+
*
|
|
340
|
+
* Ordering is `(namespace, type, id)` ascending and is part of the
|
|
341
|
+
* contract: the clusterer's input feeds a seeded pipeline, so an unstable
|
|
342
|
+
* order would break deterministic assignments.
|
|
343
|
+
*
|
|
344
|
+
* The selector is REQUIRED and has no default. See `validateSelector`.
|
|
345
|
+
*/
|
|
346
|
+
selectCandidates(selector: CandidateSelector, model: string): Promise<Candidate[]>;
|
|
347
|
+
}
|
|
348
|
+
//#endregion
|
|
349
|
+
//#region ../../plugins/substrate-recall/src/row-visibility.d.ts
|
|
350
|
+
/** The caller's request context; the per-scope fields are filled per check. */
|
|
351
|
+
type CallerContext = Omit<RequestContext, 'namespace' | 'operation' | 'entry' | 'verb'>;
|
|
352
|
+
//#endregion
|
|
353
|
+
//#region ../../plugins/substrate-recall/src/run-clusterer.d.ts
|
|
354
|
+
/**
|
|
355
|
+
* Marshalling + worker orchestration for a clustering pass.
|
|
356
|
+
*
|
|
357
|
+
* The `spawn` seam keeps this module testable in-process: the flattening,
|
|
358
|
+
* validation and result-shape checks are ordinary unit-tested code, and the
|
|
359
|
+
* real `worker_threads` boundary is exercised separately by
|
|
360
|
+
* `test/cluster-worker-isolation.test.ts`.
|
|
361
|
+
*/
|
|
362
|
+
interface ClusterRunParams {
|
|
363
|
+
/** 0 = skip reduction (holomap protocol convention). */
|
|
364
|
+
nComponents: number;
|
|
365
|
+
nNeighbors: number;
|
|
366
|
+
minClusterSize: number;
|
|
367
|
+
seed: number;
|
|
368
|
+
}
|
|
369
|
+
interface ClusterWorkerRequest {
|
|
370
|
+
/** Row-major flattened vectors — one copy, negligible beside an O(N²·d) kNN. */
|
|
371
|
+
flat: Float32Array;
|
|
372
|
+
rows: number;
|
|
373
|
+
dims: number;
|
|
374
|
+
params: ClusterRunParams;
|
|
375
|
+
}
|
|
376
|
+
type ClusterSpawn = (payload: ClusterWorkerRequest) => Promise<Int32Array>;
|
|
377
|
+
interface RunClustererDeps {
|
|
378
|
+
spawn?: ClusterSpawn;
|
|
379
|
+
}
|
|
380
|
+
export declare function runClusterer(vectors: readonly Float32Array[], params: ClusterRunParams, deps?: RunClustererDeps): Promise<number[]>;
|
|
381
|
+
//#endregion
|
|
382
|
+
//#region ../../plugins/substrate-recall/src/crystallise-core.d.ts
|
|
383
|
+
interface CrystalliseConfig {
|
|
384
|
+
/** UMAP output dimensionality. 0 skips reduction (holomap convention). */
|
|
385
|
+
nComponents: number;
|
|
386
|
+
nNeighbors: number;
|
|
387
|
+
/** HDBSCAN floor — and the live-member floor below which a cluster retires. */
|
|
388
|
+
minClusterSize: number;
|
|
389
|
+
seed: number;
|
|
390
|
+
/**
|
|
391
|
+
* Cosine floor for accretion: how eagerly an unclaimed row joins an
|
|
392
|
+
* existing cluster. Tune it per corpus.
|
|
393
|
+
*/
|
|
394
|
+
accretionThreshold: number;
|
|
395
|
+
}
|
|
396
|
+
export declare const DEFAULT_CRYSTALLISE_CONFIG: CrystalliseConfig;
|
|
397
|
+
interface CrystalliseInput {
|
|
398
|
+
mode: 'form' | 'accrete';
|
|
399
|
+
select: CandidateSelector;
|
|
400
|
+
params?: Partial<ClusterRunParams>;
|
|
401
|
+
}
|
|
402
|
+
interface CrystalliseOutput {
|
|
403
|
+
mode: 'form' | 'accrete';
|
|
404
|
+
/** Rows the selector matched. */
|
|
405
|
+
candidates: number;
|
|
406
|
+
/** Of those, how many were unclaimed and therefore eligible. */
|
|
407
|
+
considered: number;
|
|
408
|
+
clustersFormed: number;
|
|
409
|
+
clustersAccretedInto: number;
|
|
410
|
+
membersAttached: number;
|
|
411
|
+
/** Candidates the clusterer assigned to no cluster (formation only). */
|
|
412
|
+
noise: number;
|
|
413
|
+
clusters: Array<{
|
|
414
|
+
id: string;
|
|
415
|
+
label: string;
|
|
416
|
+
memberCount: number;
|
|
417
|
+
cohesion: number;
|
|
418
|
+
}>;
|
|
419
|
+
/** Members whose source row has gone, marked departed this pass. */
|
|
420
|
+
departed: number;
|
|
421
|
+
/** Clusters that fell below `minClusterSize` live members this pass. */
|
|
422
|
+
retired: number;
|
|
423
|
+
}
|
|
424
|
+
type PolicyProbe = (namespace: string, type: string) => PolicyDecision;
|
|
425
|
+
interface CrystalliseCoreDeps {
|
|
426
|
+
embeddings: EmbeddingsStore;
|
|
427
|
+
clusters: ClusterStore;
|
|
428
|
+
/** Active embedder model — the vector space everything here lives in. */
|
|
429
|
+
model: string;
|
|
430
|
+
config: CrystalliseConfig;
|
|
431
|
+
/** Seam for tests; defaults to the real worker-backed runner. */
|
|
432
|
+
runClusterer?: (vectors: readonly Float32Array[], params: ClusterRunParams) => Promise<number[]>;
|
|
433
|
+
newId?: () => string;
|
|
434
|
+
now?: () => Date;
|
|
435
|
+
/** Identity-bound read-policy probe, as the recall route already threads. */
|
|
436
|
+
checkReadPolicy?: PolicyProbe;
|
|
437
|
+
/** The caller's request context, for row-filter obligations. Absent → anonymous. */
|
|
438
|
+
caller?: CallerContext;
|
|
439
|
+
/**
|
|
440
|
+
* Returns the `memberKey`s of those `refs` whose source row no longer
|
|
441
|
+
* exists, in one batched call per sweep.
|
|
442
|
+
*
|
|
443
|
+
* Report only rows POSITIVELY known to be gone: anything unresolved stays
|
|
444
|
+
* attached. Departure cannot be undone, so a partial answer must
|
|
445
|
+
* under-depart rather than over-depart.
|
|
446
|
+
*
|
|
447
|
+
* When absent, the decay sweep does not run at all.
|
|
448
|
+
*/
|
|
449
|
+
rowsMissing?: (refs: readonly MemberRef[]) => Promise<Set<string>>;
|
|
450
|
+
}
|
|
451
|
+
export declare function validateCrystalliseInput(value: unknown): CrystalliseInput;
|
|
452
|
+
export declare function handleCrystallise(input: CrystalliseInput, deps: CrystalliseCoreDeps): Promise<CrystalliseOutput>;
|
|
453
|
+
//#endregion
|
|
454
|
+
//#region ../../plugins/substrate-recall/src/crystallise-guard.d.ts
|
|
455
|
+
/**
|
|
456
|
+
* Mutual exclusion for crystallise passes: concurrent passes would each see the
|
|
457
|
+
* same rows as unclaimed and form duplicate clusters.
|
|
458
|
+
*
|
|
459
|
+
* One instance must be shared by both transports (the route and the
|
|
460
|
+
* `recall/crystallise` frame); `createRecallPlugin` builds it once. An
|
|
461
|
+
* in-process flag covers this binary, and the optional `crossProcess` lock
|
|
462
|
+
* (`ClusterStore.withFormationLock`) covers other binaries on the same store.
|
|
463
|
+
* Either refusing throws `crystallise_in_flight`.
|
|
464
|
+
*/
|
|
465
|
+
export declare const CRYSTALLISE_IN_FLIGHT_CODE = "crystallise_in_flight";
|
|
466
|
+
interface CrystalliseGuard {
|
|
467
|
+
/**
|
|
468
|
+
* Run `fn` if no other pass is currently running; otherwise throw a coded
|
|
469
|
+
* `crystallise_in_flight` error immediately (never queues). The flag is
|
|
470
|
+
* cleared in a `finally`, so a throw from `fn` itself still releases it.
|
|
471
|
+
*/
|
|
472
|
+
runExclusive<T>(fn: () => Promise<T>): Promise<T>;
|
|
473
|
+
}
|
|
474
|
+
interface CrystalliseGuardOptions {
|
|
475
|
+
/** Exclusion across processes; see `ClusterStore.withFormationLock`. */
|
|
476
|
+
crossProcess?: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
477
|
+
}
|
|
478
|
+
export declare function createCrystalliseGuard(opts?: CrystalliseGuardOptions): CrystalliseGuard;
|
|
479
|
+
//#endregion
|
|
480
|
+
//#region ../../plugins/substrate-recall/src/embed-stats.d.ts
|
|
481
|
+
/**
|
|
482
|
+
* Observable counter for embed write-path outcomes. Embed failures are
|
|
483
|
+
* swallowed to stderr so writes never fail on them; this counter makes them
|
|
484
|
+
* visible via a snapshot (GET /embed-stats).
|
|
485
|
+
*
|
|
486
|
+
* Deliberately process-local: counts reset on restart.
|
|
487
|
+
*/
|
|
488
|
+
interface EmbedStatsSnapshot {
|
|
489
|
+
/** Wall-clock ISO timestamp when the counter was initialised. */
|
|
490
|
+
readonly since: string;
|
|
491
|
+
/** Total writes observed (success + failure + skip). */
|
|
492
|
+
readonly writes: number;
|
|
493
|
+
/** Writes that produced a vector AND upserted into embeddings. */
|
|
494
|
+
readonly successes: number;
|
|
495
|
+
/** Writes whose embedder or sql call threw. */
|
|
496
|
+
readonly failures: number;
|
|
497
|
+
/**
|
|
498
|
+
* Writes whose `embed.source` returned the null sentinel — the row was
|
|
499
|
+
* declared un-indexable, so nothing was embedded and any existing
|
|
500
|
+
* embedding was removed. Neither a success nor a failure.
|
|
501
|
+
*/
|
|
502
|
+
readonly skips: number;
|
|
503
|
+
/** Most recent failure, or null when no failures have occurred since boot. */
|
|
504
|
+
readonly lastFailure: {
|
|
505
|
+
rowId: string;
|
|
506
|
+
error: string;
|
|
507
|
+
at: string;
|
|
508
|
+
} | null;
|
|
509
|
+
/** Caught exceptions in the expansion try/catch (resolver threw or fetchEmbeddings threw). */
|
|
510
|
+
readonly expansionFailures: number;
|
|
511
|
+
}
|
|
512
|
+
interface EmbedStats {
|
|
513
|
+
/** Record a successful embed → upsert cycle. */
|
|
514
|
+
recordSuccess(): void;
|
|
515
|
+
/** Record a failure (embedder or sql). `error` is the stringified error message. */
|
|
516
|
+
recordFailure(rowId: string, error: string): void;
|
|
517
|
+
/** Record a write whose source returned the null sentinel (row un-indexed). */
|
|
518
|
+
recordSkip(): void;
|
|
519
|
+
/** Record a caught exception in the expansion try/catch (resolver or fetchEmbeddings threw). */
|
|
520
|
+
recordExpansionFailure(): void;
|
|
521
|
+
/** Read-only snapshot of the current counts + last failure. */
|
|
522
|
+
snapshot(): EmbedStatsSnapshot;
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* Build a fresh EmbedStats instance. Each `createRecallPlugin(...)` call
|
|
526
|
+
* gets its own counter unless one is supplied.
|
|
527
|
+
*/
|
|
528
|
+
export declare function createEmbedStats(now?: () => Date): EmbedStats;
|
|
529
|
+
//#endregion
|
|
530
|
+
//#region ../../plugins/substrate-recall/src/spread-resolver.d.ts
|
|
531
|
+
/** Identifies a recall hit whose neighbors should be enumerated. */
|
|
532
|
+
interface SpreadSourceRow {
|
|
533
|
+
namespace: string;
|
|
534
|
+
type: string;
|
|
535
|
+
id: string;
|
|
536
|
+
/** Full row body (from `knnSearch.row`). Used to read forward field values. */
|
|
537
|
+
row: Record<string, unknown>;
|
|
538
|
+
}
|
|
539
|
+
//#endregion
|
|
540
|
+
//#region ../../plugins/substrate-recall/src/expansion-resolver.d.ts
|
|
541
|
+
interface ExpansionSourceRow {
|
|
542
|
+
namespace: string;
|
|
543
|
+
type: string;
|
|
544
|
+
id: string;
|
|
545
|
+
row: Record<string, unknown>;
|
|
546
|
+
}
|
|
547
|
+
interface ExpansionNeighbour {
|
|
548
|
+
namespace: string;
|
|
549
|
+
type: string;
|
|
550
|
+
id: string;
|
|
551
|
+
/** ID of the seed row that walked to this neighbour (highest-cosine seed wins). */
|
|
552
|
+
seedId: string;
|
|
553
|
+
/**
|
|
554
|
+
* True when this neighbour was reached via a BACKWARD walk on an edge
|
|
555
|
+
* whose spec has `expandCrossType: true`. The recall-route scope filter
|
|
556
|
+
* waives the TYPE check (never the namespace check) on these. Forward walks
|
|
557
|
+
* NEVER set this; forward always honours scope strictly.
|
|
558
|
+
*/
|
|
559
|
+
allowsCrossType: boolean;
|
|
560
|
+
}
|
|
561
|
+
//#endregion
|
|
562
|
+
//#region ../../plugins/substrate-recall/src/lexical-store.d.ts
|
|
563
|
+
/**
|
|
564
|
+
* Storage seam for the lexical arm — the keyword half of recall.
|
|
565
|
+
*
|
|
566
|
+
* Separate from `EmbeddingsStore` because the two indexes are independent: a
|
|
567
|
+
* row can be lexically indexed with no embedder configured, and vice versa.
|
|
568
|
+
*/
|
|
569
|
+
/** Identifies the row whose lexical text is being written. */
|
|
570
|
+
interface LexicalStoreUpsertInput {
|
|
571
|
+
rowNamespace: string;
|
|
572
|
+
rowType: string;
|
|
573
|
+
rowId: string;
|
|
574
|
+
/** The text to index. Tokenisation is the store's business, not the caller's. */
|
|
575
|
+
text: string;
|
|
576
|
+
/**
|
|
577
|
+
* When the source row was read for indexing; defaults to now. An upsert
|
|
578
|
+
* older than the stored text is ignored, so a late write cannot restore
|
|
579
|
+
* wording a newer one removed.
|
|
580
|
+
*/
|
|
581
|
+
now?: Date;
|
|
582
|
+
}
|
|
583
|
+
/** Identifies the single lexical row a `delete` call removes. */
|
|
584
|
+
interface LexicalStoreDeleteInput {
|
|
585
|
+
rowNamespace: string;
|
|
586
|
+
rowType: string;
|
|
587
|
+
rowId: string;
|
|
588
|
+
}
|
|
589
|
+
/** A lexical search request. Scoped like `EmbeddingsStoreKnnInput`. */
|
|
590
|
+
interface LexicalStoreSearchInput {
|
|
591
|
+
/** Row scope. Callers know exactly one (namespace, type) at recall time. */
|
|
592
|
+
namespace: string;
|
|
593
|
+
type: string;
|
|
594
|
+
/** Raw search text. Tokenisation is the store's business, not the caller's. */
|
|
595
|
+
query: string;
|
|
596
|
+
/** Max rows to return. */
|
|
597
|
+
k: number;
|
|
598
|
+
/**
|
|
599
|
+
* Restrict candidates to rows whose fields match, applied INSIDE the search
|
|
600
|
+
* (both the all-terms pass and the any-term fallback) so k fills with
|
|
601
|
+
* matching rows. Same semantics as `EmbeddingsStoreKnnInput.rowFilter`.
|
|
602
|
+
*/
|
|
603
|
+
rowFilter?: KnnRowFilter;
|
|
604
|
+
}
|
|
605
|
+
interface LexicalStoreSearchResult {
|
|
606
|
+
/**
|
|
607
|
+
* Hit identity, carried explicitly so fusion can join against the dense arm
|
|
608
|
+
* whether or not a row resolver is wired.
|
|
609
|
+
*/
|
|
610
|
+
rowNamespace: string;
|
|
611
|
+
rowType: string;
|
|
612
|
+
rowId: string;
|
|
613
|
+
/** The row body when a resolver is wired; `{}` otherwise. */
|
|
614
|
+
row: Record<string, unknown>;
|
|
615
|
+
/** Relevance, higher is better. Comparable WITHIN one search, not across. */
|
|
616
|
+
score: number;
|
|
617
|
+
}
|
|
618
|
+
interface LexicalStore {
|
|
619
|
+
/**
|
|
620
|
+
* Index one row's text, replacing whatever was indexed for it before.
|
|
621
|
+
* Idempotent on (rowNamespace, rowType, rowId).
|
|
622
|
+
*/
|
|
623
|
+
upsert(input: LexicalStoreUpsertInput): Promise<void>;
|
|
624
|
+
/**
|
|
625
|
+
* Remove one row's lexical text, so a row that becomes un-indexable stops
|
|
626
|
+
* being findable. Idempotent — removing an absent row is not an error.
|
|
627
|
+
*/
|
|
628
|
+
delete(input: LexicalStoreDeleteInput): Promise<void>;
|
|
629
|
+
/**
|
|
630
|
+
* Count indexed rows for one (namespace, type). The pg implementation counts
|
|
631
|
+
* only rows recall could return (not soft-deleted); the in-memory store
|
|
632
|
+
* counts entries.
|
|
633
|
+
*/
|
|
634
|
+
countIndexed(namespace: string, type: string): Promise<number>;
|
|
635
|
+
/**
|
|
636
|
+
* Rank indexed rows against `query`, best first, capped at `k`.
|
|
637
|
+
*
|
|
638
|
+
* Scores are comparable only within one call; backends produce different
|
|
639
|
+
* magnitudes. Fusion therefore consumes ranks, not scores.
|
|
640
|
+
*/
|
|
641
|
+
search(input: LexicalStoreSearchInput): Promise<LexicalStoreSearchResult[]>;
|
|
642
|
+
}
|
|
643
|
+
//#endregion
|
|
644
|
+
//#region ../../plugins/substrate-recall/src/recall/types.d.ts
|
|
645
|
+
/**
|
|
646
|
+
* Read-policy probe for the recall surfaces. Hosts wire this to
|
|
647
|
+
* `engine.evaluateReadPolicy` so recall honours the same policies that gate
|
|
648
|
+
* `engine.query`. Absent → open.
|
|
649
|
+
*
|
|
650
|
+
* The decision is taken per (namespace, type) with no query args, so a policy
|
|
651
|
+
* that allows reads only for some `where` scopes is evaluated with none. Its
|
|
652
|
+
* `row-filter` obligations are then evaluated per row, against the full
|
|
653
|
+
* stored body, with the caller's request context (`row-visibility.ts`).
|
|
654
|
+
*/
|
|
655
|
+
type ReadPolicyCheck = (ns: string, type: string, identity: Identity | null, probe?: PolicyProbeContext) => PolicyDecision;
|
|
656
|
+
/**
|
|
657
|
+
* Write-policy probe for the surfaces that mutate. Hosts wire this to
|
|
658
|
+
* `engine.evaluateWritePolicy`.
|
|
659
|
+
*
|
|
660
|
+
* Separate from `ReadPolicyCheck` because a namespace can gate writes while
|
|
661
|
+
* leaving reads open. Absent → the write does not happen (fails closed,
|
|
662
|
+
* unlike `ReadPolicyCheck`).
|
|
663
|
+
*/
|
|
664
|
+
type WritePolicyCheck = (ns: string, type: string, identity: Identity | null, probe?: PolicyProbeContext) => PolicyDecision;
|
|
665
|
+
interface RecallInput {
|
|
666
|
+
query: string;
|
|
667
|
+
k?: number;
|
|
668
|
+
/**
|
|
669
|
+
* Set `false` to turn the lexical arm off for this request. The arm is on
|
|
670
|
+
* whenever a `lexicalStore` is wired and the type declares `lexical.source`;
|
|
671
|
+
* this can only turn it off, never on.
|
|
672
|
+
*/
|
|
673
|
+
lexical?: boolean;
|
|
674
|
+
/** Return per-arm ranks and the pre-fusion dense score on each row. */
|
|
675
|
+
explain?: boolean;
|
|
676
|
+
namespace?: string;
|
|
677
|
+
type?: string;
|
|
678
|
+
/**
|
|
679
|
+
* Cosine floor (0..1) on the dense arm, and on lexical-only rows unless
|
|
680
|
+
* `lexicalThreshold` is set. Lexical-only rows are scored against their
|
|
681
|
+
* stored vectors; one with no stored vector is dropped. Applies only when
|
|
682
|
+
* the dense arm runs. Default 0.
|
|
683
|
+
*/
|
|
684
|
+
threshold?: number;
|
|
685
|
+
/**
|
|
686
|
+
* Cosine floor for lexical-only hits, scored against their stored vectors.
|
|
687
|
+
* Default: `threshold`. `0` leaves them unfloored, keeping rows that have no
|
|
688
|
+
* stored vector.
|
|
689
|
+
*/
|
|
690
|
+
lexicalThreshold?: number;
|
|
691
|
+
/**
|
|
692
|
+
* When `true` AND the plugin-level kill-switch is on, walks opt-in
|
|
693
|
+
* crossRefs from the knn top-K, fetches neighbour embeddings, computes
|
|
694
|
+
* fresh cosine vs the query vector, merges into the candidate pool, and
|
|
695
|
+
* re-ranks. Expanded rows are annotated with `_expandedFrom`. Default
|
|
696
|
+
* `false`.
|
|
697
|
+
*/
|
|
698
|
+
expand?: boolean;
|
|
699
|
+
/**
|
|
700
|
+
* When `false`, the recall runs normally (KNN + blend + scored envelope)
|
|
701
|
+
* but skips the fire-and-forget salience boost on the returned rows — for
|
|
702
|
+
* consumers that reinforce explicitly on use instead of on retrieval
|
|
703
|
+
* (`POST /substrate/v1/salience/boost`). Default `true`.
|
|
704
|
+
*/
|
|
705
|
+
boost?: boolean;
|
|
706
|
+
/**
|
|
707
|
+
* When `true`, each returned row carries its stored embedding vector
|
|
708
|
+
* (`RecallRow.vector`), saving a `/substrate/v1/embeddings/fetch` round-trip.
|
|
709
|
+
* Vectors ride the row's existing read-policy gate. Default `false`.
|
|
710
|
+
*/
|
|
711
|
+
returnVector?: boolean;
|
|
712
|
+
/**
|
|
713
|
+
* Row-field filter, field -> `{ eq }` or `{ in }`, every field must match.
|
|
714
|
+
* Applied inside every candidate source (dense kNN, lexical search,
|
|
715
|
+
* expansion), so k fills with matching rows. `eq: null` matches a missing
|
|
716
|
+
* or null field. A malformed filter is refused with `bad_request`.
|
|
717
|
+
*/
|
|
718
|
+
where?: KnnRowFilter;
|
|
719
|
+
}
|
|
720
|
+
interface RecallRow {
|
|
721
|
+
/**
|
|
722
|
+
* Per-arm ranks, present only under `explain: true`. `null` for an arm that
|
|
723
|
+
* did not return this row — distinct from a rank, which is 1-based.
|
|
724
|
+
*/
|
|
725
|
+
_arms?: {
|
|
726
|
+
dense: number | null;
|
|
727
|
+
lexical: number | null;
|
|
728
|
+
};
|
|
729
|
+
/**
|
|
730
|
+
* Pre-fusion dense score, present only under `explain: true`.
|
|
731
|
+
*
|
|
732
|
+
* `null` for a row the dense arm never ranked. Not `0`: that is a real
|
|
733
|
+
* cosine, and a consumer comparing against a floor would wrongly admit a
|
|
734
|
+
* lexical-only row.
|
|
735
|
+
*/
|
|
736
|
+
_denseScore?: number | null;
|
|
737
|
+
row: Record<string, unknown>;
|
|
738
|
+
/**
|
|
739
|
+
* Actual (namespace, type) of this row. Equals the query's ns/type for
|
|
740
|
+
* direct knn hits; for rows pulled in via cross-type expansion it is the
|
|
741
|
+
* neighbour's real ns/type.
|
|
742
|
+
*/
|
|
743
|
+
namespace: string;
|
|
744
|
+
type: string;
|
|
745
|
+
score: number;
|
|
746
|
+
/**
|
|
747
|
+
* Per-row salience signal feeding into ranking. `0` for never-recalled
|
|
748
|
+
* rows.
|
|
749
|
+
*/
|
|
750
|
+
_salience: number;
|
|
751
|
+
/**
|
|
752
|
+
* Score after `salience_weight` blend. Equals `score` when weight=0
|
|
753
|
+
* (default). When weight>0: `score * (1 + weight * log(1 + salience))`.
|
|
754
|
+
*/
|
|
755
|
+
_blendedScore: number;
|
|
756
|
+
/**
|
|
757
|
+
* Set on rows that entered the result pool via expansion. Records the seed (namespace, type, id) from which this row was reached.
|
|
758
|
+
* Absent for rows returned directly by knn.
|
|
759
|
+
*/
|
|
760
|
+
_expandedFrom?: {
|
|
761
|
+
namespace: string;
|
|
762
|
+
type: string;
|
|
763
|
+
id: string;
|
|
764
|
+
};
|
|
765
|
+
/**
|
|
766
|
+
* The row's stored embedding vector. Present iff the request set
|
|
767
|
+
* `returnVector: true` and the store had a vector for the row (knn hits
|
|
768
|
+
* always do; scored expansion rows carry the fetched neighbour vector).
|
|
769
|
+
*/
|
|
770
|
+
vector?: number[];
|
|
771
|
+
}
|
|
772
|
+
interface RecallTimings {
|
|
773
|
+
/** Embedder call. */
|
|
774
|
+
readonly embed_ms: number;
|
|
775
|
+
/** KNN scan against the EmbeddingsStore (e.g. pgvector HNSW). */
|
|
776
|
+
readonly knn_ms: number;
|
|
777
|
+
/** End-to-end handler latency. */
|
|
778
|
+
readonly total_ms: number;
|
|
779
|
+
}
|
|
780
|
+
interface RecallOutput {
|
|
781
|
+
rows: RecallRow[];
|
|
782
|
+
/**
|
|
783
|
+
* The dense arm's embedding model; empty string when no dense arm ran.
|
|
784
|
+
* Required rather than optional so `undefined` is never read as "server
|
|
785
|
+
* predates the field".
|
|
786
|
+
*/
|
|
787
|
+
model: string;
|
|
788
|
+
timings: RecallTimings;
|
|
789
|
+
/**
|
|
790
|
+
* Which arms actually ran: the effective mode, not the requested one
|
|
791
|
+
* (recall ignores unknown body fields, so the request is not evidence).
|
|
792
|
+
*
|
|
793
|
+
* Optional because this is a wire response: `undefined` means the server
|
|
794
|
+
* predates the field, not that no arm ran. Do not collapse the two.
|
|
795
|
+
*/
|
|
796
|
+
arms?: {
|
|
797
|
+
/**
|
|
798
|
+
* Did the dense arm run? False when no embedder is configured, equally
|
|
799
|
+
* when one is but no `EmbeddingsStore` is — knn needs both halves — and
|
|
800
|
+
* equally when the TYPE declares no `embed.source`, which is the case a
|
|
801
|
+
* caller cannot detect for itself. `denseWhy` says which.
|
|
802
|
+
*/
|
|
803
|
+
dense: boolean;
|
|
804
|
+
/**
|
|
805
|
+
* Why the dense arm did or did not run; the mirror of `lexicalWhy`.
|
|
806
|
+
* `no-source` means the type is never embedded, so an empty result is not
|
|
807
|
+
* "nothing matched".
|
|
808
|
+
*
|
|
809
|
+
* Optional, unlike `lexicalWhy`, so consumers that construct a
|
|
810
|
+
* `RecallOutput` keep compiling. The core always populates it; absent
|
|
811
|
+
* means the server predates the field and must not be read as `ran`.
|
|
812
|
+
*/
|
|
813
|
+
denseWhy?: 'ran' | 'no-embedder' | 'no-store' | 'no-source';
|
|
814
|
+
/**
|
|
815
|
+
* Did the lexical arm run? False when no `lexicalStore` is wired, when the
|
|
816
|
+
* type declares no `lexical.source`, or when the caller passed
|
|
817
|
+
* `lexical: false` — three different reasons, and `lexicalWhy` says which.
|
|
818
|
+
*/
|
|
819
|
+
lexical: boolean;
|
|
820
|
+
/**
|
|
821
|
+
* Why the lexical arm did or did not run. `no-source` means the arm cannot
|
|
822
|
+
* match for this type, which an empty result alone would not reveal.
|
|
823
|
+
*/
|
|
824
|
+
lexicalWhy: 'ran' | 'no-store' | 'no-source' | 'disabled-by-request';
|
|
825
|
+
};
|
|
826
|
+
}
|
|
827
|
+
/** Configuration for the salience blend in `handleRecallRequest`. */
|
|
828
|
+
interface SalienceConfig {
|
|
829
|
+
/**
|
|
830
|
+
* Blend weight. `0` (default) → ranking unchanged.
|
|
831
|
+
* `>0` re-ranks by `cosine * (1 + weight * log(1 + salience))`.
|
|
832
|
+
*/
|
|
833
|
+
weight: number;
|
|
834
|
+
/** Half-life used for the boost-on-recall decay. Hours. */
|
|
835
|
+
halfLifeHours: number;
|
|
836
|
+
/**
|
|
837
|
+
* Activation-spreading damping. When the route's
|
|
838
|
+
* `spreadResolver` returns opt-in crossRef neighbours, each gets a boost
|
|
839
|
+
* of `spreadDamping` × the primary +1. Default 0.3 keeps the spread
|
|
840
|
+
* signal modest; 0 disables spreading entirely even if a resolver is
|
|
841
|
+
* configured.
|
|
842
|
+
*/
|
|
843
|
+
spreadDamping: number;
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Spread-target resolver. Built by the plugin from
|
|
847
|
+
* the spread registry + an `EdgesStore`. recall-route invokes it with the
|
|
848
|
+
* boosted source rows; the resolver returns deduplicated forward + backward
|
|
849
|
+
* targets to apply a damped boost on. `null` short-circuits the spread path.
|
|
850
|
+
*/
|
|
851
|
+
type SpreadResolver = (sources: ReadonlyArray<SpreadSourceRow>) => Promise<SalienceTarget[]>;
|
|
852
|
+
/**
|
|
853
|
+
* Expansion resolver. Built by the plugin from the spread
|
|
854
|
+
* registry + an optional `EdgesStore`. recall-route invokes it with the knn
|
|
855
|
+
* top-K source rows; the resolver returns deduplicated (namespace, type, id)
|
|
856
|
+
* candidates to fetch + score. `null` short-circuits the expansion path.
|
|
857
|
+
*/
|
|
858
|
+
type ExpansionResolver = (sources: ReadonlyArray<ExpansionSourceRow>) => Promise<ExpansionNeighbour[]>;
|
|
859
|
+
/**
|
|
860
|
+
* Dependencies for `handleRecallRequest`. Every member is optional; an absent
|
|
861
|
+
* one defaults to the off/empty behaviour.
|
|
862
|
+
*/
|
|
863
|
+
interface RecallDeps {
|
|
864
|
+
/**
|
|
865
|
+
* Dense arm: both halves or neither. An embedder with no store is no dense
|
|
866
|
+
* arm, and `arms.dense` reports the conjunction. Optional because the
|
|
867
|
+
* lexical arm needs no model.
|
|
868
|
+
*/
|
|
869
|
+
embedder?: Embedder;
|
|
870
|
+
store?: EmbeddingsStore;
|
|
871
|
+
/**
|
|
872
|
+
* Lexical arm store. Absent → the arm never runs, and the response says so
|
|
873
|
+
* via `arms.lexicalWhy: 'no-store'` rather than leaving the caller to infer
|
|
874
|
+
* it from an unchanged result.
|
|
875
|
+
*/
|
|
876
|
+
lexicalStore?: LexicalStore;
|
|
877
|
+
/**
|
|
878
|
+
* Which types declare a `lexical.source` — satisfied by
|
|
879
|
+
* `LexicalSourceRegistry`.
|
|
880
|
+
*
|
|
881
|
+
* Absent or empty both run the arm. An empty registry is not yet populated
|
|
882
|
+
* (it fills at `contributePostWrites` time); treating it as authoritative
|
|
883
|
+
* would disable the arm for every type.
|
|
884
|
+
*/
|
|
885
|
+
lexicalSources?: {
|
|
886
|
+
get(namespace: string, type: string): unknown;
|
|
887
|
+
size(): number;
|
|
888
|
+
};
|
|
889
|
+
/**
|
|
890
|
+
* Which types declare an `embed.source` — satisfied by
|
|
891
|
+
* `EmbedSourceRegistry`. Lets the dense arm answer `denseWhy: 'no-source'`
|
|
892
|
+
* instead of running a knn that cannot match.
|
|
893
|
+
*
|
|
894
|
+
* Absent or empty both run the arm, as for `lexicalSources`.
|
|
895
|
+
*/
|
|
896
|
+
embedSources?: {
|
|
897
|
+
get(namespace: string, type: string): unknown;
|
|
898
|
+
size(): number;
|
|
899
|
+
};
|
|
900
|
+
salience?: SalienceConfig;
|
|
901
|
+
spreadResolver?: SpreadResolver | null;
|
|
902
|
+
expansionResolver?: ExpansionResolver | null;
|
|
903
|
+
expansionEnabled?: boolean;
|
|
904
|
+
stats?: EmbedStats;
|
|
905
|
+
/**
|
|
906
|
+
* Identity-bound read-policy check — the transport caller
|
|
907
|
+
* (HTTP route / MCP tool) binds its identity before delegating here.
|
|
908
|
+
* Denied primary scope throws `{ code: 'policy_denied' }` before any
|
|
909
|
+
* embedder/store work; denied expansion-neighbour types are silently
|
|
910
|
+
* dropped from the candidate pool. Absent → open.
|
|
911
|
+
*/
|
|
912
|
+
checkReadPolicy?: (ns: string, type: string) => PolicyDecision;
|
|
913
|
+
/** The caller's request context, for row-filter obligations. Absent → anonymous. */
|
|
914
|
+
caller?: CallerContext;
|
|
915
|
+
/**
|
|
916
|
+
* Identity-bound write-policy check for the salience boost only. A denial
|
|
917
|
+
* skips the boost but does not fail the recall. Absent → no boost (fails
|
|
918
|
+
* closed).
|
|
919
|
+
*/
|
|
920
|
+
checkWritePolicy?: (ns: string, type: string) => PolicyDecision;
|
|
921
|
+
}
|
|
922
|
+
//#endregion
|
|
923
|
+
//#region ../../plugins/substrate-recall/src/recall/handler.d.ts
|
|
924
|
+
export declare function handleRecallRequest(input: RecallInput, deps: RecallDeps): Promise<RecallOutput>;
|
|
925
|
+
//#endregion
|
|
926
|
+
//#region ../../plugins/substrate-recall/src/recall/route.d.ts
|
|
927
|
+
interface RecallRouteConfig {
|
|
928
|
+
embedder?: Embedder;
|
|
929
|
+
store?: EmbeddingsStore;
|
|
930
|
+
/** Lexical arm store. Absent → the arm never runs and `arms.lexicalWhy` says so. */
|
|
931
|
+
lexicalStore?: LexicalStore;
|
|
932
|
+
/**
|
|
933
|
+
* Which types declare a `lexical.source`. Forwarded to the core so it can
|
|
934
|
+
* answer `lexicalWhy: 'no-source'` instead of reporting `ran` for an arm
|
|
935
|
+
* that cannot match. Absent → the core cannot say, and runs the arm.
|
|
936
|
+
*/
|
|
937
|
+
lexicalSources?: {
|
|
938
|
+
get(namespace: string, type: string): unknown;
|
|
939
|
+
size(): number;
|
|
940
|
+
};
|
|
941
|
+
/**
|
|
942
|
+
* Which types declare an `embed.source`. Forwarded to the core so it can
|
|
943
|
+
* answer `denseWhy: 'no-source'` for a type with no vectors instead of
|
|
944
|
+
* reporting `dense: true` over a knn that cannot match. Absent → the core
|
|
945
|
+
* cannot say, and runs the arm.
|
|
946
|
+
*/
|
|
947
|
+
embedSources?: {
|
|
948
|
+
get(namespace: string, type: string): unknown;
|
|
949
|
+
size(): number;
|
|
950
|
+
};
|
|
951
|
+
/**
|
|
952
|
+
* Salience blend config. When omitted, defaults to
|
|
953
|
+
* `DEFAULT_SALIENCE_CONFIG` (weight=0 → ranking unchanged, boost still
|
|
954
|
+
* fires so the signal accumulates passively for future opt-in).
|
|
955
|
+
*/
|
|
956
|
+
salience?: SalienceConfig;
|
|
957
|
+
/**
|
|
958
|
+
* Spread-target resolver. The plugin builds this
|
|
959
|
+
* at compose time from the spread registry + an `EdgesStore`. Absent →
|
|
960
|
+
* the spread path is short-circuited (forward + backward boosts skip).
|
|
961
|
+
*/
|
|
962
|
+
spreadResolver?: SpreadResolver;
|
|
963
|
+
/**
|
|
964
|
+
* Expansion resolver. The plugin builds this at compose time
|
|
965
|
+
* from the spread registry + an optional `EdgesStore`. When set, enables
|
|
966
|
+
* `input.expand:true` to walk opt-in crossRefs from the knn top-K and
|
|
967
|
+
* merge scored neighbours into the candidate pool. Absent → expansion is
|
|
968
|
+
* short-circuited even if `input.expand:true`.
|
|
969
|
+
*/
|
|
970
|
+
expansionResolver?: ExpansionResolver;
|
|
971
|
+
/**
|
|
972
|
+
* Plugin-level kill-switch for expansion. Set to `false` to
|
|
973
|
+
* disable expansion regardless of `input.expand`. Defaults to `true`.
|
|
974
|
+
*/
|
|
975
|
+
expansionEnabled?: boolean;
|
|
976
|
+
/**
|
|
977
|
+
* Optional stats counter. When provided, the route
|
|
978
|
+
* increments `expansionFailures` on caught exceptions in the expansion path
|
|
979
|
+
* (resolver throws / fetchEmbeddings throws). Surfaces degraded paths on the
|
|
980
|
+
* `/substrate/v1/embed-stats` route.
|
|
981
|
+
*/
|
|
982
|
+
stats?: EmbedStats;
|
|
983
|
+
/**
|
|
984
|
+
* Unbound read-policy check. The route binds the request's
|
|
985
|
+
* verified identity (`req.identity ?? null` — populated by the auth layer)
|
|
986
|
+
* and threads the bound check into `handleRecallRequest`. Denied primary
|
|
987
|
+
* scope maps to a 403 `policy_denied` envelope. Absent → open.
|
|
988
|
+
*/
|
|
989
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
990
|
+
/** Unbound write-policy check — the route binds `req.identity`. */
|
|
991
|
+
checkWritePolicy?: WritePolicyCheck;
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* KernelRouteClaim handler factory.
|
|
995
|
+
*
|
|
996
|
+
* Returns an `HttpHandler` for `POST /substrate/v1/recall`. Pass to
|
|
997
|
+
* `KernelRouteClaim.handler` when building the substrate plugin's
|
|
998
|
+
* `kernelRoutes` array.
|
|
999
|
+
*
|
|
1000
|
+
* @example
|
|
1001
|
+
* ```ts
|
|
1002
|
+
* kernelRoutes: [
|
|
1003
|
+
* { path: 'recall', method: 'POST', handler: recallRouteHandler({ embedder, store }) }
|
|
1004
|
+
* ]
|
|
1005
|
+
* ```
|
|
1006
|
+
*/
|
|
1007
|
+
export declare function recallRouteHandler(cfg: RecallRouteConfig): HttpHandler;
|
|
1008
|
+
//#endregion
|
|
1009
|
+
//#region ../../plugins/substrate-recall/src/cluster-frame-surface.d.ts
|
|
1010
|
+
/**
|
|
1011
|
+
* Published shape of a `recall/Cluster` row.
|
|
1012
|
+
*
|
|
1013
|
+
* `centroid` is deliberately absent: it is a full embedding vector per row,
|
|
1014
|
+
* raw vector content that `/substrate/v1/embeddings/fetch` serves under its own gate.
|
|
1015
|
+
*/
|
|
1016
|
+
export declare const CLUSTER_FIELDS: Record<string, FieldDef>;
|
|
1017
|
+
export declare const CLUSTER_MEMBER_FIELDS: Record<string, FieldDef>;
|
|
1018
|
+
interface ClusterSurfaceDeps {
|
|
1019
|
+
embeddings: EmbeddingsStore;
|
|
1020
|
+
clusters: ClusterStore;
|
|
1021
|
+
model: string;
|
|
1022
|
+
config: CrystalliseConfig;
|
|
1023
|
+
runClusterer?: (vectors: readonly Float32Array[], params: ClusterRunParams) => Promise<number[]>;
|
|
1024
|
+
rowsMissing?: (refs: readonly MemberRef[]) => Promise<Set<string>>;
|
|
1025
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
1026
|
+
/**
|
|
1027
|
+
* Shared mutual-exclusion guard (see module doc); must be the instance the
|
|
1028
|
+
* route holds. Omitted = a private guard scoped to this surface only, which
|
|
1029
|
+
* is NOT sufficient for production wiring.
|
|
1030
|
+
*/
|
|
1031
|
+
guard?: CrystalliseGuard;
|
|
1032
|
+
}
|
|
1033
|
+
type ClusterTypeEntry = {
|
|
1034
|
+
fields: Record<string, FieldDef>;
|
|
1035
|
+
resolver: (ctx: RequestContext, args: ResolverArgs) => Promise<Row[]>;
|
|
1036
|
+
};
|
|
1037
|
+
type ClusterMutationEntry = {
|
|
1038
|
+
resolver: (ctx: RequestContext, args: ResolverArgs) => Promise<Row>;
|
|
1039
|
+
};
|
|
1040
|
+
export declare function buildClusterSurface(deps: ClusterSurfaceDeps): {
|
|
1041
|
+
types: {
|
|
1042
|
+
Cluster: ClusterTypeEntry;
|
|
1043
|
+
ClusterMember: ClusterTypeEntry;
|
|
1044
|
+
};
|
|
1045
|
+
mutations: {
|
|
1046
|
+
crystallise: ClusterMutationEntry;
|
|
1047
|
+
labelCluster: ClusterMutationEntry;
|
|
1048
|
+
};
|
|
1049
|
+
};
|
|
1050
|
+
//#endregion
|
|
1051
|
+
//#region ../../plugins/substrate-recall/src/clusterer-version.d.ts
|
|
1052
|
+
/**
|
|
1053
|
+
* Which clusterer produced a clustering. Recorded in every cluster's
|
|
1054
|
+
* provenance, because two backends produce different assignments on
|
|
1055
|
+
* identical input — and, decisively, so does the same backend on
|
|
1056
|
+
* differently-rounded input.
|
|
1057
|
+
*
|
|
1058
|
+
* The version is a hand-maintained constant rather than a read of the
|
|
1059
|
+
* dependency's package.json, because `@mnmal-ai/holomap-clusterer` exports
|
|
1060
|
+
* only `"."` — there is no `./package.json` subpath to require. Drift is
|
|
1061
|
+
* prevented by `test/clusterer-version.test.ts`, which reads the installed
|
|
1062
|
+
* package directly and fails if this constant disagrees.
|
|
1063
|
+
*/
|
|
1064
|
+
export declare const CLUSTERER_BACKEND = "wasm";
|
|
1065
|
+
export declare const CLUSTERER_BACKEND_VERSION = "0.2.1";
|
|
1066
|
+
//#endregion
|
|
1067
|
+
//#region ../../plugins/substrate-recall/src/crystallise-route.d.ts
|
|
1068
|
+
interface CrystalliseRouteConfig {
|
|
1069
|
+
embeddings: EmbeddingsStore;
|
|
1070
|
+
/** Absent = crystallisation is not wired; the route answers 503. */
|
|
1071
|
+
clusters?: ClusterStore;
|
|
1072
|
+
model: string;
|
|
1073
|
+
config: CrystalliseConfig;
|
|
1074
|
+
runClusterer?: (vectors: readonly Float32Array[], params: ClusterRunParams) => Promise<number[]>;
|
|
1075
|
+
rowsMissing?: (refs: readonly MemberRef[]) => Promise<Set<string>>;
|
|
1076
|
+
/** Unbound 3-arg check (the plugin config form) — the route binds identity. */
|
|
1077
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
1078
|
+
/**
|
|
1079
|
+
* Shared mutual-exclusion guard (see module doc); must be the instance the
|
|
1080
|
+
* frame surface holds. Omitted = a private guard scoped to this handler
|
|
1081
|
+
* only, which is NOT sufficient for production wiring.
|
|
1082
|
+
*/
|
|
1083
|
+
guard?: CrystalliseGuard;
|
|
1084
|
+
}
|
|
1085
|
+
export declare function crystalliseRouteHandler(cfg: CrystalliseRouteConfig): HttpHandler;
|
|
1086
|
+
//#endregion
|
|
1087
|
+
//#region ../../plugins/substrate-recall/src/embed-and-store.d.ts
|
|
1088
|
+
interface EmbedAndStoreOptions {
|
|
1089
|
+
namespace: string;
|
|
1090
|
+
type: string;
|
|
1091
|
+
/**
|
|
1092
|
+
* Optional custom serializer returning the source text to embed (sync or
|
|
1093
|
+
* async), or `null` / `undefined` to declare the row un-indexable.
|
|
1094
|
+
*
|
|
1095
|
+
* The null sentinel is distinct from `''`: an empty string still embeds,
|
|
1096
|
+
* whereas the sentinel embeds nothing and removes any existing embedding.
|
|
1097
|
+
*
|
|
1098
|
+
* When omitted, the source text joins the non-empty string values of
|
|
1099
|
+
* `fields` with `'\n'`; if both are omitted it is `''`.
|
|
1100
|
+
*/
|
|
1101
|
+
serializeRow?: (row: Record<string, unknown>) => string | null | undefined | Promise<string | null | undefined>;
|
|
1102
|
+
/**
|
|
1103
|
+
* Field names whose string values are joined with `'\n'` to build the
|
|
1104
|
+
* source text. Used only when `serializeRow` is not provided. Non-string
|
|
1105
|
+
* and empty values are filtered before joining.
|
|
1106
|
+
*/
|
|
1107
|
+
fields?: string[];
|
|
1108
|
+
embedder: Embedder;
|
|
1109
|
+
store: EmbeddingsStore;
|
|
1110
|
+
stats?: EmbedStats;
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* What became of one row's embedding attempt. Backfills report this; the
|
|
1114
|
+
* write path (`embedAndStore`) swallows it.
|
|
1115
|
+
*/
|
|
1116
|
+
type EmbedOutcome = {
|
|
1117
|
+
status: 'embedded';
|
|
1118
|
+
} |
|
|
1119
|
+
/** Source returned the null sentinel — any existing vector was deleted. */
|
|
1120
|
+
{
|
|
1121
|
+
status: 'unindexable';
|
|
1122
|
+
} |
|
|
1123
|
+
/** Row carried no string or number `id`; nothing was attempted. */
|
|
1124
|
+
{
|
|
1125
|
+
status: 'no-id';
|
|
1126
|
+
} |
|
|
1127
|
+
/** `code` is the `EmbedError` code when the embedder raised one. */
|
|
1128
|
+
{
|
|
1129
|
+
status: 'failed';
|
|
1130
|
+
error: string;
|
|
1131
|
+
code?: 'unavailable' | 'invalid_response';
|
|
1132
|
+
};
|
|
1133
|
+
/**
|
|
1134
|
+
* Embed one row and return what happened, without touching stats or stderr.
|
|
1135
|
+
* Never throws: failures come back as `{ status: 'failed' }`.
|
|
1136
|
+
*/
|
|
1137
|
+
export declare function embedRowReporting(row: Record<string, unknown>, opts: EmbedAndStoreOptions): Promise<EmbedOutcome>;
|
|
1138
|
+
export declare function embedAndStore(row: Record<string, unknown>, opts: EmbedAndStoreOptions): Promise<void>;
|
|
1139
|
+
interface EmbedBinding {
|
|
1140
|
+
namespace: string;
|
|
1141
|
+
type: string;
|
|
1142
|
+
source: (row: Row, ctx?: EmbedSourceContext) => string | null | undefined | Promise<string | null | undefined>;
|
|
1143
|
+
embedder: Embedder;
|
|
1144
|
+
store: EmbeddingsStore;
|
|
1145
|
+
stats?: EmbedStats;
|
|
1146
|
+
/** Optional context passed to `source(row, ctx)`; `undefined` when omitted. */
|
|
1147
|
+
embedSourceContext?: EmbedSourceContext;
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* Wrap `embedAndStore` into a PostWriteHook that embeds every successfully
|
|
1151
|
+
* written row. Never throws, so it cannot break the postWrite chain.
|
|
1152
|
+
*/
|
|
1153
|
+
export declare function makeEmbedPostWrite(args: EmbedBinding): PostWriteHook;
|
|
1154
|
+
/**
|
|
1155
|
+
* Returns a re-embed function built from the same binding as
|
|
1156
|
+
* `makeEmbedPostWrite`, so a backfill writes the same vector the write path
|
|
1157
|
+
* would. Reports each outcome instead of swallowing errors.
|
|
1158
|
+
*/
|
|
1159
|
+
export declare function makeReembed(args: EmbedBinding): (row: Row) => Promise<EmbedOutcome>;
|
|
1160
|
+
//#endregion
|
|
1161
|
+
//#region ../../plugins/substrate-recall/src/embed-source-registry.d.ts
|
|
1162
|
+
/** One embedded type: its identity, and the hook that (re)embeds one of its rows. */
|
|
1163
|
+
interface EmbedSourceEntry {
|
|
1164
|
+
namespace: string;
|
|
1165
|
+
type: string;
|
|
1166
|
+
/**
|
|
1167
|
+
* The post-write hook built from this type's `embed.source`. Invoking it
|
|
1168
|
+
* with a row re-embeds that row exactly as the write path would.
|
|
1169
|
+
*/
|
|
1170
|
+
hook: PostWriteHook;
|
|
1171
|
+
/**
|
|
1172
|
+
* Re-embed one row of this type and report the outcome; the admin reindex
|
|
1173
|
+
* route runs on this. Unlike `hook`, which swallows errors because the
|
|
1174
|
+
* write has already committed, it reports failures.
|
|
1175
|
+
*
|
|
1176
|
+
* `createRecallPlugin` always populates it; it is optional only to keep the
|
|
1177
|
+
* published interface non-breaking. Treat absence as "cannot report", never
|
|
1178
|
+
* as success.
|
|
1179
|
+
*/
|
|
1180
|
+
reembed?: (row: Row) => Promise<EmbedOutcome>;
|
|
1181
|
+
}
|
|
1182
|
+
/** Identity half of an entry — what `types()` returns. */
|
|
1183
|
+
type EmbeddedTypeRef = Pick<EmbedSourceEntry, 'namespace' | 'type'>;
|
|
1184
|
+
export declare class EmbedSourceRegistry {
|
|
1185
|
+
#private;
|
|
1186
|
+
/**
|
|
1187
|
+
* Register one embedded type. Idempotent on (namespace, type) — a re-install
|
|
1188
|
+
* overwrites rather than duplicating, because the LATEST registration
|
|
1189
|
+
* reflects the currently-composed namespace definition.
|
|
1190
|
+
*/
|
|
1191
|
+
add(entry: EmbedSourceEntry): void;
|
|
1192
|
+
/** The hook for one type, or undefined when the type carries no `embed.source`. */
|
|
1193
|
+
get(namespace: string, type: string): EmbedSourceEntry | undefined;
|
|
1194
|
+
/** Every embedded type, sorted by (namespace, type) so audit runs diff cleanly. */
|
|
1195
|
+
types(): ReadonlyArray<EmbeddedTypeRef>;
|
|
1196
|
+
size(): number;
|
|
1197
|
+
}
|
|
1198
|
+
//#endregion
|
|
1199
|
+
//#region ../../plugins/substrate-recall/src/lexical-index.d.ts
|
|
1200
|
+
interface LexicalPostWriteBinding {
|
|
1201
|
+
namespace: string;
|
|
1202
|
+
type: string;
|
|
1203
|
+
source: (row: Row) => string | null | undefined | Promise<string | null | undefined>;
|
|
1204
|
+
store: LexicalStore;
|
|
1205
|
+
}
|
|
1206
|
+
/** What became of one row's lexical indexing attempt. */
|
|
1207
|
+
type LexicalOutcome = {
|
|
1208
|
+
status: 'indexed';
|
|
1209
|
+
} |
|
|
1210
|
+
/** Source returned the null sentinel — any indexed text was removed. */
|
|
1211
|
+
{
|
|
1212
|
+
status: 'unindexable';
|
|
1213
|
+
} |
|
|
1214
|
+
/** Row carried no string `id`; nothing was attempted. */
|
|
1215
|
+
{
|
|
1216
|
+
status: 'no-id';
|
|
1217
|
+
} | {
|
|
1218
|
+
status: 'failed';
|
|
1219
|
+
error: string;
|
|
1220
|
+
};
|
|
1221
|
+
/** Build the post-write hook that keeps one type's lexical index current. */
|
|
1222
|
+
export declare function makeLexicalPostWrite(binding: LexicalPostWriteBinding): PostWriteHook;
|
|
1223
|
+
/**
|
|
1224
|
+
* Build the pure classifier a dry-run coverage scan runs on. Shares the hook's
|
|
1225
|
+
* binding; evaluates the source and reports, touching no store.
|
|
1226
|
+
*/
|
|
1227
|
+
export declare function makeLexicalClassify(binding: LexicalPostWriteBinding): (row: Row) => Promise<LexicalClassifyOutcome>;
|
|
1228
|
+
//#endregion
|
|
1229
|
+
//#region ../../plugins/substrate-recall/src/lexical-source-registry.d.ts
|
|
1230
|
+
/** One lexically-indexed type: its identity, and how to re-index one row. */
|
|
1231
|
+
interface LexicalSourceEntry {
|
|
1232
|
+
namespace: string;
|
|
1233
|
+
type: string;
|
|
1234
|
+
/** The post-write hook built from this type's `lexical.source`. */
|
|
1235
|
+
hook: PostWriteHook;
|
|
1236
|
+
/**
|
|
1237
|
+
* Re-index one row and report the outcome; the backfill runs on this.
|
|
1238
|
+
* Optional only to keep the published interface non-breaking. Treat absence
|
|
1239
|
+
* as "cannot report", never as success.
|
|
1240
|
+
*/
|
|
1241
|
+
reindex?: (row: Row) => Promise<LexicalOutcome>;
|
|
1242
|
+
/**
|
|
1243
|
+
* Classify one row WITHOUT writing — the read-only twin of `reindex`,
|
|
1244
|
+
* built from the same source function. Optional, like `reindex`. Absent →
|
|
1245
|
+
* coverage reports `unindexable: null` (unknown, not zero) and scans nothing.
|
|
1246
|
+
*/
|
|
1247
|
+
classify?: (row: Row) => Promise<LexicalClassifyOutcome>;
|
|
1248
|
+
}
|
|
1249
|
+
/** What became of one row's dry-run classification. Mirrors `LexicalOutcome` minus the writes. */
|
|
1250
|
+
type LexicalClassifyOutcome =
|
|
1251
|
+
/** Source returned text — the row belongs in the index. */
|
|
1252
|
+
{
|
|
1253
|
+
status: 'indexable';
|
|
1254
|
+
} |
|
|
1255
|
+
/** Source returned the null sentinel — the row is refused, by design. */
|
|
1256
|
+
{
|
|
1257
|
+
status: 'unindexable';
|
|
1258
|
+
} |
|
|
1259
|
+
/** Row carried no string `id`; nothing could be classified. */
|
|
1260
|
+
{
|
|
1261
|
+
status: 'no-id';
|
|
1262
|
+
} | {
|
|
1263
|
+
status: 'failed';
|
|
1264
|
+
error: string;
|
|
1265
|
+
};
|
|
1266
|
+
/** Identity half of an entry — what `types()` returns. */
|
|
1267
|
+
type LexicalTypeRef = Pick<LexicalSourceEntry, 'namespace' | 'type'>;
|
|
1268
|
+
declare class LexicalSourceRegistry {
|
|
1269
|
+
#private;
|
|
1270
|
+
/**
|
|
1271
|
+
* Register one lexically-indexed type. Idempotent on (namespace, type) — a
|
|
1272
|
+
* re-install overwrites, because the LATEST registration reflects the
|
|
1273
|
+
* currently-composed namespace definition.
|
|
1274
|
+
*/
|
|
1275
|
+
add(entry: LexicalSourceEntry): void;
|
|
1276
|
+
/** The entry for one type, or undefined when it declares no `lexical.source`. */
|
|
1277
|
+
get(namespace: string, type: string): LexicalSourceEntry | undefined;
|
|
1278
|
+
/** Every lexically-indexed type, sorted by (namespace, type) so audit runs diff cleanly. */
|
|
1279
|
+
types(): ReadonlyArray<LexicalTypeRef>;
|
|
1280
|
+
size(): number;
|
|
1281
|
+
}
|
|
1282
|
+
//#endregion
|
|
1283
|
+
//#region ../../plugins/substrate-recall/src/lexical-coverage-core.d.ts
|
|
1284
|
+
interface LexicalCoverageInput {
|
|
1285
|
+
/** Restrict to one type. Absent → every type in the registry. */
|
|
1286
|
+
namespace?: string;
|
|
1287
|
+
type?: string;
|
|
1288
|
+
}
|
|
1289
|
+
interface LexicalTypeCoverage {
|
|
1290
|
+
namespace: string;
|
|
1291
|
+
type: string;
|
|
1292
|
+
/** Dataset rows examined. 0 when the type could not be classified (see unindexable). */
|
|
1293
|
+
scanned: number;
|
|
1294
|
+
/** Rows currently in the lexical index (live COUNT, same liveness as search). */
|
|
1295
|
+
indexed: number;
|
|
1296
|
+
/**
|
|
1297
|
+
* Rows whose source returned the null sentinel.
|
|
1298
|
+
*
|
|
1299
|
+
* `null` (not 0) means UNKNOWN: the entry has no classifier, so no scan ran
|
|
1300
|
+
* and `scanned` is 0. Do not collapse with `?? 0`.
|
|
1301
|
+
*/
|
|
1302
|
+
unindexable: number | null;
|
|
1303
|
+
/** Source threw, or the row carried no string id. Detail lives in the reindex route. */
|
|
1304
|
+
failed: number;
|
|
1305
|
+
}
|
|
1306
|
+
interface LexicalCoverageResult {
|
|
1307
|
+
types: LexicalTypeCoverage[];
|
|
1308
|
+
}
|
|
1309
|
+
interface LexicalCoverageDeps {
|
|
1310
|
+
registry: LexicalSourceRegistry;
|
|
1311
|
+
/**
|
|
1312
|
+
* Row source. Absent → the core throws `unavailable` rather than reporting
|
|
1313
|
+
* zeros that would read as "uncovered".
|
|
1314
|
+
*/
|
|
1315
|
+
resolveDataset?: (ns: string) => Dataset | null;
|
|
1316
|
+
lexicalStore: LexicalStore;
|
|
1317
|
+
}
|
|
1318
|
+
export declare function validateLexicalCoverageInput(raw: unknown): LexicalCoverageInput;
|
|
1319
|
+
export declare function handleLexicalCoverage(input: LexicalCoverageInput, deps: LexicalCoverageDeps): Promise<LexicalCoverageResult>;
|
|
1320
|
+
//#endregion
|
|
1321
|
+
//#region ../../plugins/substrate-recall/src/embed-stats-route.d.ts
|
|
1322
|
+
/** Shape returned by the GET route. */
|
|
1323
|
+
interface EmbedStatsResponse extends EmbedStatsSnapshot {
|
|
1324
|
+
/** false when the plugin was composed without an embedder (recall off); counts will all be zero in that case. */
|
|
1325
|
+
readonly enabled: boolean;
|
|
1326
|
+
/**
|
|
1327
|
+
* Every type that declares an `embed.source`, sorted — the authoritative
|
|
1328
|
+
* answer to "what is embedded", including types with no rows yet.
|
|
1329
|
+
*
|
|
1330
|
+
* Optional because this is a wire response, and the two empty answers differ:
|
|
1331
|
+
*
|
|
1332
|
+
* `undefined` the server predates this field — it does not know
|
|
1333
|
+
* `[]` the server knows, and nothing is embedded
|
|
1334
|
+
*
|
|
1335
|
+
* Do not collapse them with `?? []`.
|
|
1336
|
+
*/
|
|
1337
|
+
readonly embeddedTypes?: ReadonlyArray<EmbeddedTypeRef>;
|
|
1338
|
+
/**
|
|
1339
|
+
* Every type that declares a `lexical.source`, sorted, including types with
|
|
1340
|
+
* no rows yet.
|
|
1341
|
+
*
|
|
1342
|
+
* Same two-empty-answers rule as `embeddedTypes`; do not collapse with
|
|
1343
|
+
* `?? []`. Unlike `embeddedTypes`, an unwired registry yields `undefined`
|
|
1344
|
+
* here rather than `[]`.
|
|
1345
|
+
*/
|
|
1346
|
+
readonly lexicalTypes?: ReadonlyArray<LexicalTypeRef>;
|
|
1347
|
+
/**
|
|
1348
|
+
* Did the plugin receive a `lexicalStore`? The lexical counterpart of
|
|
1349
|
+
* `enabled`. Distinguishes the two causes of `lexicalTypes: []`:
|
|
1350
|
+
*
|
|
1351
|
+
* store wired, nothing declared -> fix the CONSUMER's namespace definition
|
|
1352
|
+
* no store wired -> fix the HOST's composition
|
|
1353
|
+
*
|
|
1354
|
+
* `undefined` means the server does not say; `false` means the arm is off.
|
|
1355
|
+
* Do not read `!body.lexicalEnabled` as "off".
|
|
1356
|
+
*/
|
|
1357
|
+
readonly lexicalEnabled?: boolean;
|
|
1358
|
+
/**
|
|
1359
|
+
* Dry-run lexical coverage per (namespace, type) — `{ scanned, indexed,
|
|
1360
|
+
* unindexable, failed }`.
|
|
1361
|
+
*
|
|
1362
|
+
* Present only when the caller passes `?coverage=true`, since each type
|
|
1363
|
+
* costs a full dataset scan. Absent key = not asked (or server predates
|
|
1364
|
+
* the field).
|
|
1365
|
+
*
|
|
1366
|
+
* `unindexable: null` on a type means UNKNOWN (entry carries no pure
|
|
1367
|
+
* classifier), not zero. Do not collapse with `?? 0`.
|
|
1368
|
+
*/
|
|
1369
|
+
readonly lexicalCoverage?: ReadonlyArray<LexicalTypeCoverage>;
|
|
1370
|
+
}
|
|
1371
|
+
/**
|
|
1372
|
+
* Facts about composition that the registries cannot supply: a registry knows
|
|
1373
|
+
* what was registered, not what was wired.
|
|
1374
|
+
*/
|
|
1375
|
+
interface EmbedStatsComposition {
|
|
1376
|
+
/** Did the plugin receive a `lexicalStore`? Omit to say nothing. */
|
|
1377
|
+
lexicalEnabled?: boolean;
|
|
1378
|
+
/**
|
|
1379
|
+
* Store + row source for opt-in `?coverage=true`. Omit either and coverage
|
|
1380
|
+
* refuses `unavailable` rather than reporting zeros.
|
|
1381
|
+
*/
|
|
1382
|
+
lexicalStore?: LexicalStore;
|
|
1383
|
+
resolveDataset?: (ns: string) => Dataset | null;
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Pure handler. Inputs are the optional stats instance, source registries and
|
|
1387
|
+
* composition facts; output is the JSON body shape (without coverage).
|
|
1388
|
+
*/
|
|
1389
|
+
export declare function handleEmbedStatsRequest(stats: EmbedStats | undefined, registry?: EmbedSourceRegistry, lexicalRegistry?: LexicalSourceRegistry, composition?: EmbedStatsComposition): EmbedStatsResponse;
|
|
1390
|
+
/**
|
|
1391
|
+
* KernelRouteClaim handler factory.
|
|
1392
|
+
*
|
|
1393
|
+
* Returns an `HttpHandler` that serialises the current embed stats snapshot
|
|
1394
|
+
* as JSON. Pass to `KernelRouteClaim.handler` when building the substrate
|
|
1395
|
+
* plugin's `kernelRoutes` array.
|
|
1396
|
+
*
|
|
1397
|
+
* @example
|
|
1398
|
+
* ```ts
|
|
1399
|
+
* kernelRoutes: [
|
|
1400
|
+
* { path: 'embed-stats', method: 'GET', handler: embedStatsRouteHandler(stats) }
|
|
1401
|
+
* ]
|
|
1402
|
+
* ```
|
|
1403
|
+
*/
|
|
1404
|
+
export declare function embedStatsRouteHandler(stats: EmbedStats | undefined, registry?: EmbedSourceRegistry, lexicalRegistry?: LexicalSourceRegistry, composition?: EmbedStatsComposition): HttpHandler;
|
|
1405
|
+
//#endregion
|
|
1406
|
+
//#region ../../plugins/substrate-recall/src/embeddings-fetch-core.d.ts
|
|
1407
|
+
/**
|
|
1408
|
+
* Per-request ref cap. Bounds both the response payload and the store's
|
|
1409
|
+
* batched lookup.
|
|
1410
|
+
*/
|
|
1411
|
+
export declare const EMBEDDINGS_FETCH_MAX_REFS = 500;
|
|
1412
|
+
//#endregion
|
|
1413
|
+
//#region ../../plugins/substrate-recall/src/embeddings-fetch-route.d.ts
|
|
1414
|
+
interface EmbeddingsFetchRouteConfig {
|
|
1415
|
+
store: EmbeddingsStore;
|
|
1416
|
+
/** Supplies the default model id when the request omits `model`. */
|
|
1417
|
+
embedder?: Embedder;
|
|
1418
|
+
/** Supplies the boost decay half-life. Defaults to DEFAULT_SALIENCE_CONFIG. */
|
|
1419
|
+
salience?: SalienceConfig;
|
|
1420
|
+
/** Unbound read-policy check — the route binds `req.identity`. */
|
|
1421
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
1422
|
+
/** Unbound write-policy check — the route binds `req.identity`. */
|
|
1423
|
+
checkWritePolicy?: WritePolicyCheck;
|
|
1424
|
+
}
|
|
1425
|
+
/**
|
|
1426
|
+
* KernelRouteClaim handler factory for `POST /substrate/v1/embeddings/fetch`.
|
|
1427
|
+
* Registered by `createRecallPlugin` alongside the recall route. Thin
|
|
1428
|
+
* transport adapter over `handleEmbeddingsFetch` — JSON-parse + bind identity,
|
|
1429
|
+
* then map coded throws (bad_request → 400, policy_denied → 403) to envelopes.
|
|
1430
|
+
*/
|
|
1431
|
+
export declare function embeddingsFetchRouteHandler(cfg: EmbeddingsFetchRouteConfig): HttpHandler;
|
|
1432
|
+
//#endregion
|
|
1433
|
+
//#region ../../plugins/substrate-recall/src/filter-operator.d.ts
|
|
1434
|
+
interface EmbeddingSimilarOperatorDeps {
|
|
1435
|
+
embedder: Embedder;
|
|
1436
|
+
store: EmbeddingsStore;
|
|
1437
|
+
}
|
|
1438
|
+
export declare function embeddingSimilarOperator(deps: EmbeddingSimilarOperatorDeps): FilterOperatorDef;
|
|
1439
|
+
//#endregion
|
|
1440
|
+
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.d.ts
|
|
1441
|
+
interface InMemoryClusterStoreOptions {
|
|
1442
|
+
txScopeId?: string;
|
|
1443
|
+
}
|
|
1444
|
+
export declare class InMemoryClusterStore implements ClusterStore {
|
|
1445
|
+
#private;
|
|
1446
|
+
readonly txScopeId: string;
|
|
1447
|
+
constructor(opts?: InMemoryClusterStoreOptions);
|
|
1448
|
+
createCluster(input: NewCluster): Promise<ClusterRecord>;
|
|
1449
|
+
attachMembers(inputs: readonly NewMember[]): Promise<void>;
|
|
1450
|
+
listClusters(opts?: {
|
|
1451
|
+
includeRetired?: boolean;
|
|
1452
|
+
}): Promise<ClusterRecord[]>;
|
|
1453
|
+
getCluster(id: string): Promise<ClusterRecord | null>;
|
|
1454
|
+
listMembers(clusterId: string, opts?: {
|
|
1455
|
+
includeDeparted?: boolean;
|
|
1456
|
+
}): Promise<ClusterMemberRecord[]>;
|
|
1457
|
+
listAllMembers(): Promise<ClusterMemberRecord[]>;
|
|
1458
|
+
findClustersForMember(ref: MemberRef): Promise<ClusterMemberRecord[]>;
|
|
1459
|
+
claimedMembers(refs: readonly MemberRef[]): Promise<Set<string>>;
|
|
1460
|
+
markDeparted(refs: readonly MemberRef[], now?: Date): Promise<number>;
|
|
1461
|
+
retire(clusterId: string, now?: Date): Promise<void>;
|
|
1462
|
+
relabel(clusterId: string, label: string): Promise<ClusterRecord | null>;
|
|
1463
|
+
}
|
|
1464
|
+
//#endregion
|
|
1465
|
+
//#region ../../plugins/substrate-recall/src/in-memory-embeddings-store.d.ts
|
|
1466
|
+
interface InMemoryEmbeddingsStoreOptions {
|
|
1467
|
+
/**
|
|
1468
|
+
* Optional resolver for the dataset holding a (namespace, type)'s row
|
|
1469
|
+
* bodies. Without it, results carry a `{ id }` stub as their row.
|
|
1470
|
+
*/
|
|
1471
|
+
datasetFor?: (rowNamespace: string, rowType: string) => Dataset | undefined;
|
|
1472
|
+
/** Override for tests. Defaults to incrementing instance counter. */
|
|
1473
|
+
txScopeId?: string;
|
|
1474
|
+
}
|
|
1475
|
+
export declare class InMemoryEmbeddingsStore implements EmbeddingsStore {
|
|
1476
|
+
#private;
|
|
1477
|
+
readonly txScopeId: string;
|
|
1478
|
+
constructor(opts?: InMemoryEmbeddingsStoreOptions);
|
|
1479
|
+
upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
|
|
1480
|
+
delete(input: EmbeddingsStoreDeleteInput): Promise<void>;
|
|
1481
|
+
knnSearch(input: EmbeddingsStoreKnnInput): Promise<EmbeddingsStoreKnnResult[]>;
|
|
1482
|
+
boostSalience(targets: ReadonlyArray<SalienceTarget>, halfLifeHours: number, bump?: number): Promise<ReadonlyArray<SalienceTarget>>;
|
|
1483
|
+
fetchEmbeddings(ids: ReadonlyArray<{
|
|
1484
|
+
namespace: string;
|
|
1485
|
+
type: string;
|
|
1486
|
+
id: string;
|
|
1487
|
+
}>, model: string): Promise<ReadonlyArray<EmbeddingsStoreFetchResult>>;
|
|
1488
|
+
selectCandidates(selector: CandidateSelector, model: string): Promise<Candidate[]>;
|
|
1489
|
+
}
|
|
1490
|
+
//#endregion
|
|
1491
|
+
//#region ../../plugins/substrate-recall/src/in-memory-lexical-store.d.ts
|
|
1492
|
+
/**
|
|
1493
|
+
* Resolves a row body, mirroring pg's join against `rows`.
|
|
1494
|
+
*
|
|
1495
|
+
* Signature matches `InMemoryEmbeddingsStoreOptions.datasetFor` deliberately:
|
|
1496
|
+
* a host wires both from the same registry lookup.
|
|
1497
|
+
*/
|
|
1498
|
+
interface InMemoryLexicalStoreOptions {
|
|
1499
|
+
datasetFor?: (namespace: string) => Dataset | undefined;
|
|
1500
|
+
}
|
|
1501
|
+
export declare class InMemoryLexicalStore implements LexicalStore {
|
|
1502
|
+
#private;
|
|
1503
|
+
constructor(opts?: InMemoryLexicalStoreOptions);
|
|
1504
|
+
upsert(input: LexicalStoreUpsertInput): Promise<void>;
|
|
1505
|
+
delete(input: LexicalStoreDeleteInput): Promise<void>;
|
|
1506
|
+
countIndexed(namespace: string, type: string): Promise<number>;
|
|
1507
|
+
search(input: LexicalStoreSearchInput): Promise<LexicalStoreSearchResult[]>;
|
|
1508
|
+
}
|
|
1509
|
+
//#endregion
|
|
1510
|
+
//#region ../../plugins/substrate-recall/src/l2normalize.d.ts
|
|
1511
|
+
/**
|
|
1512
|
+
* L2 normalisation for the crystallise input path.
|
|
1513
|
+
*
|
|
1514
|
+
* A byte-for-byte copy of `l2normalize` in `@mnmal-ai/holomap-clusterer`'s
|
|
1515
|
+
* `npm/test/fixture.ts`, because the corpus regression gate measures against
|
|
1516
|
+
* figures that function produced; the clustering pipeline is sensitive enough
|
|
1517
|
+
* that an equivalent-in-the-limit function is not the same function.
|
|
1518
|
+
*
|
|
1519
|
+
* `sum` accumulates in f64 over the f32 values and only re-rounds on store.
|
|
1520
|
+
* Do NOT "simplify" this to operate on the pre-cast values: it looks like a
|
|
1521
|
+
* cleanup and changes the results.
|
|
1522
|
+
*/
|
|
1523
|
+
export declare function l2normalize(vectors: readonly Float32Array[]): Float32Array[];
|
|
1524
|
+
//#endregion
|
|
1525
|
+
//#region ../../plugins/substrate-recall/src/mcp-tool.d.ts
|
|
1526
|
+
interface RecallMcpToolConfig {
|
|
1527
|
+
embedder?: Embedder;
|
|
1528
|
+
store: EmbeddingsStore;
|
|
1529
|
+
/**
|
|
1530
|
+
* Lexical arm store. Absent → this surface runs
|
|
1531
|
+
* dense-only and the response's `arms.lexicalWhy` says `no-store`.
|
|
1532
|
+
*/
|
|
1533
|
+
lexicalStore?: LexicalStore;
|
|
1534
|
+
/**
|
|
1535
|
+
* Which types declare a `lexical.source`. Forwarded to the core so it can
|
|
1536
|
+
* answer `lexicalWhy: 'no-source'` instead of reporting `ran` for an arm
|
|
1537
|
+
* that cannot match. Absent → the core cannot say, and runs the arm.
|
|
1538
|
+
*/
|
|
1539
|
+
lexicalSources?: {
|
|
1540
|
+
get(namespace: string, type: string): unknown;
|
|
1541
|
+
size(): number;
|
|
1542
|
+
};
|
|
1543
|
+
/**
|
|
1544
|
+
* Which types declare an `embed.source`. Forwarded to the core so it can
|
|
1545
|
+
* answer `denseWhy: 'no-source'` for a type with no vectors instead of
|
|
1546
|
+
* reporting `dense: true` over a knn that cannot match. Absent → the core
|
|
1547
|
+
* cannot say, and runs the arm.
|
|
1548
|
+
*/
|
|
1549
|
+
embedSources?: {
|
|
1550
|
+
get(namespace: string, type: string): unknown;
|
|
1551
|
+
size(): number;
|
|
1552
|
+
};
|
|
1553
|
+
/**
|
|
1554
|
+
* Salience blend config. Threaded into `handleRecallRequest`
|
|
1555
|
+
* so MCP recall ranks + boosts identically to the HTTP route.
|
|
1556
|
+
* When omitted, falls back to `DEFAULT_SALIENCE_CONFIG` (weight=0 → off).
|
|
1557
|
+
*/
|
|
1558
|
+
salience?: SalienceConfig;
|
|
1559
|
+
/** Spread-target resolver. When omitted, spreading is skipped. */
|
|
1560
|
+
spreadResolver?: SpreadResolver;
|
|
1561
|
+
/** Expansion resolver. When omitted, `expand:true` is a no-op. */
|
|
1562
|
+
expansionResolver?: ExpansionResolver;
|
|
1563
|
+
/**
|
|
1564
|
+
* Plugin-level kill-switch for expansion. When false, `expand:true` in the
|
|
1565
|
+
* MCP input is ignored. Defaults to `true`.
|
|
1566
|
+
*/
|
|
1567
|
+
expansionEnabled?: boolean;
|
|
1568
|
+
/** Optional stats counter — records expansion failures for `/substrate/v1/embed-stats`. */
|
|
1569
|
+
stats?: EmbedStats;
|
|
1570
|
+
/**
|
|
1571
|
+
* Unbound read-policy check. This MCP surface has no identity
|
|
1572
|
+
* channel, so the handler binds `null` — fail-closed under a gated read
|
|
1573
|
+
* posture, unchanged under open postures. Absent → open.
|
|
1574
|
+
*/
|
|
1575
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
1576
|
+
/** Unbound write-policy check — bound to a null identity here. */
|
|
1577
|
+
checkWritePolicy?: WritePolicyCheck;
|
|
1578
|
+
}
|
|
1579
|
+
export declare function recallMcpTool(cfg: RecallMcpToolConfig): McpToolRegistration;
|
|
1580
|
+
//#endregion
|
|
1581
|
+
//#region ../../plugins/substrate-recall/src/migrations/0001_embeddings.d.ts
|
|
1582
|
+
interface EmbeddingsMigrationOptions {
|
|
1583
|
+
/** Vector dimension. Defaults to 768. */
|
|
1584
|
+
dimension?: number;
|
|
1585
|
+
/** HNSW `m` build param. Defaults to pgvector default (16). */
|
|
1586
|
+
hnswM?: number;
|
|
1587
|
+
/** HNSW `ef_construction` build param. Defaults to pgvector default (64). */
|
|
1588
|
+
hnswEfConstruction?: number;
|
|
1589
|
+
}
|
|
1590
|
+
export declare function embeddingsMigration(opts?: EmbeddingsMigrationOptions): KernelMigration;
|
|
1591
|
+
//#endregion
|
|
1592
|
+
//#region ../../plugins/substrate-recall/src/migrations/0002_recall_clusters.d.ts
|
|
1593
|
+
interface RecallClustersMigrationOptions {
|
|
1594
|
+
/**
|
|
1595
|
+
* Centroid dimension. Must match `embeddings.embedding`: centroids
|
|
1596
|
+
* live in the ORIGINAL embedding space, never reduced space.
|
|
1597
|
+
*/
|
|
1598
|
+
dimension?: number;
|
|
1599
|
+
}
|
|
1600
|
+
export declare function recallClustersMigration(opts?: RecallClustersMigrationOptions): KernelMigration;
|
|
1601
|
+
//#endregion
|
|
1602
|
+
//#region ../../plugins/substrate-recall/src/migrations/0003_lexical.d.ts
|
|
1603
|
+
interface LexicalMigrationOptions {
|
|
1604
|
+
/**
|
|
1605
|
+
* Postgres text search configuration used to build the generated tsvector.
|
|
1606
|
+
* Defaults to `english`. Must name a configuration that exists on the
|
|
1607
|
+
* server; an unknown name fails the migration rather than degrading.
|
|
1608
|
+
*/
|
|
1609
|
+
textSearchConfig?: string;
|
|
1610
|
+
}
|
|
1611
|
+
export declare function lexicalMigration(opts?: LexicalMigrationOptions): KernelMigration;
|
|
1612
|
+
//#endregion
|
|
1613
|
+
//#region ../../plugins/substrate-recall/src/params.d.ts
|
|
1614
|
+
/**
|
|
1615
|
+
* Unknown-parameter refusal, shared by every recall surface. Each surface keeps
|
|
1616
|
+
* its own allowed set (the frame spells the search text `to`, the route
|
|
1617
|
+
* `query`); the refusal logic is shared.
|
|
1618
|
+
*
|
|
1619
|
+
* Refused rather than dropped: a silently ignored flag is indistinguishable
|
|
1620
|
+
* from one that ran and did nothing.
|
|
1621
|
+
*/
|
|
1622
|
+
/** Keys present on `obj` that are not in `allowed`, in the caller's order. */
|
|
1623
|
+
export declare function unknownParams(obj: Record<string, unknown>, allowed: ReadonlyArray<string>): string[];
|
|
1624
|
+
/**
|
|
1625
|
+
* The refusal text. Names both the offending keys and the allowed set, the
|
|
1626
|
+
* latter generated from the array the parser uses.
|
|
1627
|
+
*/
|
|
1628
|
+
export declare function unknownParamMessage(unknown: ReadonlyArray<string>, allowed: ReadonlyArray<string>): string;
|
|
1629
|
+
/** `null` when every key is known, otherwise the refusal text. */
|
|
1630
|
+
export declare function checkParams(obj: Record<string, unknown>, allowed: ReadonlyArray<string>): string | null;
|
|
1631
|
+
//#endregion
|
|
1632
|
+
//#region ../../plugins/substrate-recall/src/pg-cluster-store.d.ts
|
|
1633
|
+
interface PgClusterStoreOptions {
|
|
1634
|
+
sql: Sql;
|
|
1635
|
+
}
|
|
1636
|
+
export declare class PgClusterStore implements ClusterStore {
|
|
1637
|
+
#private;
|
|
1638
|
+
readonly txScopeId: string;
|
|
1639
|
+
constructor(opts: PgClusterStoreOptions);
|
|
1640
|
+
/**
|
|
1641
|
+
* Runs `fn` holding a session advisory lock that excludes formation in every
|
|
1642
|
+
* process on this database; throws `crystallise_in_flight` if it is held.
|
|
1643
|
+
*
|
|
1644
|
+
* Lock and unlock share one reserved connection, since through the pool the
|
|
1645
|
+
* unlock can land on another session and fail to release.
|
|
1646
|
+
*/
|
|
1647
|
+
withFormationLock<T>(fn: () => Promise<T>): Promise<T>;
|
|
1648
|
+
createCluster(input: NewCluster): Promise<ClusterRecord>;
|
|
1649
|
+
attachMembers(inputs: readonly NewMember[]): Promise<void>;
|
|
1650
|
+
listClusters(opts?: {
|
|
1651
|
+
includeRetired?: boolean;
|
|
1652
|
+
}): Promise<ClusterRecord[]>;
|
|
1653
|
+
getCluster(id: string): Promise<ClusterRecord | null>;
|
|
1654
|
+
listMembers(clusterId: string, opts?: {
|
|
1655
|
+
includeDeparted?: boolean;
|
|
1656
|
+
}): Promise<ClusterMemberRecord[]>;
|
|
1657
|
+
listAllMembers(): Promise<ClusterMemberRecord[]>;
|
|
1658
|
+
findClustersForMember(ref: MemberRef): Promise<ClusterMemberRecord[]>;
|
|
1659
|
+
claimedMembers(refs: readonly MemberRef[]): Promise<Set<string>>;
|
|
1660
|
+
markDeparted(refs: readonly MemberRef[], now?: Date): Promise<number>;
|
|
1661
|
+
retire(clusterId: string, now?: Date): Promise<void>;
|
|
1662
|
+
relabel(clusterId: string, label: string): Promise<ClusterRecord | null>;
|
|
1663
|
+
}
|
|
1664
|
+
//#endregion
|
|
1665
|
+
//#region ../../plugins/substrate-recall/src/pg-embeddings-store.d.ts
|
|
1666
|
+
interface PgEmbeddingsStoreOptions {
|
|
1667
|
+
sql: Sql;
|
|
1668
|
+
}
|
|
1669
|
+
export declare class PgEmbeddingsStore implements EmbeddingsStore {
|
|
1670
|
+
#private;
|
|
1671
|
+
readonly txScopeId: string;
|
|
1672
|
+
constructor(opts: PgEmbeddingsStoreOptions);
|
|
1673
|
+
upsert(input: EmbeddingsStoreUpsertInput): Promise<void>;
|
|
1674
|
+
delete(input: EmbeddingsStoreDeleteInput): Promise<void>;
|
|
1675
|
+
knnSearch(input: EmbeddingsStoreKnnInput): Promise<EmbeddingsStoreKnnResult[]>;
|
|
1676
|
+
boostSalience(targets: ReadonlyArray<SalienceTarget>, halfLifeHours: number, bump?: number): Promise<ReadonlyArray<SalienceTarget>>;
|
|
1677
|
+
fetchEmbeddings(ids: ReadonlyArray<{
|
|
1678
|
+
namespace: string;
|
|
1679
|
+
type: string;
|
|
1680
|
+
id: string;
|
|
1681
|
+
}>, model: string): Promise<ReadonlyArray<EmbeddingsStoreFetchResult>>;
|
|
1682
|
+
selectCandidates(selector: CandidateSelector, model: string): Promise<Candidate[]>;
|
|
1683
|
+
/**
|
|
1684
|
+
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
1685
|
+
* Needed because `embeddings` has no FK to `rows` and no
|
|
1686
|
+
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
1687
|
+
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
1688
|
+
*
|
|
1689
|
+
* @returns number of orphaned embeddings deleted.
|
|
1690
|
+
*/
|
|
1691
|
+
purgeOrphans(): Promise<number>;
|
|
1692
|
+
}
|
|
1693
|
+
//#endregion
|
|
1694
|
+
//#region ../../plugins/substrate-recall/src/pg-lexical-store.d.ts
|
|
1695
|
+
interface PgLexicalStoreOptions {
|
|
1696
|
+
/**
|
|
1697
|
+
* Text search configuration used to parse the QUERY. Must match the one
|
|
1698
|
+
* migration 0006 baked into the generated `tsv` column, or queries silently
|
|
1699
|
+
* match nothing.
|
|
1700
|
+
*/
|
|
1701
|
+
textSearchConfig?: string;
|
|
1702
|
+
}
|
|
1703
|
+
export declare class PgLexicalStore implements LexicalStore {
|
|
1704
|
+
#private;
|
|
1705
|
+
constructor(sql: ReturnType<typeof postgres>, opts?: PgLexicalStoreOptions);
|
|
1706
|
+
upsert(input: LexicalStoreUpsertInput): Promise<void>;
|
|
1707
|
+
delete(input: LexicalStoreDeleteInput): Promise<void>;
|
|
1708
|
+
/**
|
|
1709
|
+
* Deletes every lexical row whose backing row is missing or soft-deleted,
|
|
1710
|
+
* as `PgEmbeddingsStore.purgeOrphans` does for vectors: `lexical` has no FK
|
|
1711
|
+
* to `rows`, so a drop, purge or tombstone would otherwise leave the text.
|
|
1712
|
+
*
|
|
1713
|
+
* @returns number of orphaned lexical rows deleted.
|
|
1714
|
+
*/
|
|
1715
|
+
purgeOrphans(): Promise<number>;
|
|
1716
|
+
countIndexed(namespace: string, type: string): Promise<number>;
|
|
1717
|
+
/**
|
|
1718
|
+
* Returns up to `k` live rows ranked by `ts_rank`. Matches ALL query terms
|
|
1719
|
+
* first; only if that finds nothing, retries matching ANY term.
|
|
1720
|
+
*
|
|
1721
|
+
* Two passes rather than always-OR: nothing downstream filters lexical hits
|
|
1722
|
+
* by threshold, so an OR pass that always ran would admit rows matching only
|
|
1723
|
+
* a common term. The fallback fires only where the strict pass found nothing.
|
|
1724
|
+
* Both passes apply `rowFilter`, so the fallback fires when no MATCHING row
|
|
1725
|
+
* carries every term.
|
|
1726
|
+
*/
|
|
1727
|
+
search(input: LexicalStoreSearchInput): Promise<LexicalStoreSearchResult[]>;
|
|
1728
|
+
}
|
|
1729
|
+
//#endregion
|
|
1730
|
+
//#region ../../plugins/substrate-recall/src/plugin.d.ts
|
|
1731
|
+
interface RecallPluginConfig {
|
|
1732
|
+
embedder?: Embedder;
|
|
1733
|
+
/**
|
|
1734
|
+
* Timeout for embedding a recall or `embeddingSimilar` query, in ms. Write
|
|
1735
|
+
* and reindex embeds keep the embedder's own timeout. Default 30000.
|
|
1736
|
+
*/
|
|
1737
|
+
queryEmbedTimeoutMs?: number;
|
|
1738
|
+
store: EmbeddingsStore;
|
|
1739
|
+
/**
|
|
1740
|
+
* Store for the lexical arm. Absent → types declaring `lexical.source`
|
|
1741
|
+
* contribute nothing, the same way an absent `embedder` disables the dense
|
|
1742
|
+
* arm. Independent of `embedder` in both directions on purpose: lexical
|
|
1743
|
+
* recall needs no vectors.
|
|
1744
|
+
*/
|
|
1745
|
+
lexicalStore?: LexicalStore;
|
|
1746
|
+
/**
|
|
1747
|
+
* Build params for the lexical table (text search configuration). The
|
|
1748
|
+
* migration applies whether or not `lexicalStore` is wired — same posture as
|
|
1749
|
+
* the embeddings migration under an absent embedder: the schema is present
|
|
1750
|
+
* so an operator turning the arm on later needs no separate migration pass.
|
|
1751
|
+
*/
|
|
1752
|
+
lexicalMigration?: LexicalMigrationOptions;
|
|
1753
|
+
/** Optional pre-built stats counter; a fresh one is created when omitted. */
|
|
1754
|
+
stats?: EmbedStats;
|
|
1755
|
+
/**
|
|
1756
|
+
* Migration build params (vector dimension + HNSW m / ef_construction).
|
|
1757
|
+
* When omitted, the migration falls back to its built-in defaults
|
|
1758
|
+
* (dim 768; m 16 and ef_construction 64, the pgvector defaults). When `embedder` is supplied, the caller is expected
|
|
1759
|
+
* to set `dimension` to `embedder.dimension` so fresh installs match.
|
|
1760
|
+
*/
|
|
1761
|
+
migration?: EmbeddingsMigrationOptions;
|
|
1762
|
+
/**
|
|
1763
|
+
* Optional dataset resolver. When supplied, each `embed.source(row, ctx)`
|
|
1764
|
+
* receives an `EmbedSourceContext` whose `read` is backed by this lookup,
|
|
1765
|
+
* letting source functions inline related-entity context via crossRef.
|
|
1766
|
+
* When omitted, source receives `undefined` for ctx.
|
|
1767
|
+
*/
|
|
1768
|
+
resolveDataset?: (namespace: string) => Dataset | null;
|
|
1769
|
+
/**
|
|
1770
|
+
* Salience blend config. When omitted, the route falls back to
|
|
1771
|
+
* `DEFAULT_SALIENCE_CONFIG` (weight=0, 7-day half-life).
|
|
1772
|
+
*/
|
|
1773
|
+
salience?: SalienceConfig;
|
|
1774
|
+
/**
|
|
1775
|
+
* EdgesStore used by activation spreading to resolve backward (inbound)
|
|
1776
|
+
* edges. When omitted, the spread path only walks forward (X → row X
|
|
1777
|
+
* references).
|
|
1778
|
+
*/
|
|
1779
|
+
edges?: EdgesStore | null;
|
|
1780
|
+
/**
|
|
1781
|
+
* Bounds on crossRef walking when `input.expand:true`. When omitted, falls
|
|
1782
|
+
* back to plugin defaults.
|
|
1783
|
+
*/
|
|
1784
|
+
expansion?: {
|
|
1785
|
+
enabled: boolean;
|
|
1786
|
+
maxNeighboursPerSeed: number;
|
|
1787
|
+
maxTotalNeighbours: number;
|
|
1788
|
+
};
|
|
1789
|
+
/**
|
|
1790
|
+
* Read-policy probe. Hosts wire this to `engine.evaluateReadPolicy`
|
|
1791
|
+
* (late-bound — the engine exists only after compose) so the recall
|
|
1792
|
+
* surfaces honour the same composed policies that gate `engine.query`.
|
|
1793
|
+
* When omitted, recall stays open.
|
|
1794
|
+
*/
|
|
1795
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
1796
|
+
/**
|
|
1797
|
+
* Write-policy probe for the mutating surfaces. Hosts wire this to
|
|
1798
|
+
* `engine.evaluateWritePolicy`. Unwired → those surfaces refuse to write.
|
|
1799
|
+
*/
|
|
1800
|
+
checkWritePolicy?: WritePolicyCheck;
|
|
1801
|
+
/**
|
|
1802
|
+
* Cluster persistence. When omitted, crystallisation composes as
|
|
1803
|
+
* unavailable: migration 0005 still applies, the route answers 503, and
|
|
1804
|
+
* recall/Cluster is not published.
|
|
1805
|
+
*/
|
|
1806
|
+
clusters?: ClusterStore;
|
|
1807
|
+
/** Clustering params + accretion threshold. Falls back to defaults. */
|
|
1808
|
+
crystallise?: CrystalliseConfig;
|
|
1809
|
+
/**
|
|
1810
|
+
* Probe backing the membership decay sweep: of these source rows, which are
|
|
1811
|
+
* gone? Answered as `memberKey` strings in one batched call — see
|
|
1812
|
+
* `CrystalliseCoreDeps.rowsMissing` for why the answer names the missing
|
|
1813
|
+
* rows rather than the surviving ones. Wired by the binary from the dataset
|
|
1814
|
+
* registry; when absent the sweep does not run (an embedded host with no
|
|
1815
|
+
* dataset resolver must not conclude every row has vanished).
|
|
1816
|
+
*/
|
|
1817
|
+
rowsMissing?: (refs: readonly MemberRef[]) => Promise<Set<string>>;
|
|
1818
|
+
/**
|
|
1819
|
+
* Seam for the clustering pass itself, threaded to the route and the frame
|
|
1820
|
+
* surface. Omitted = the real `runClusterer` (worker_threads + wasm).
|
|
1821
|
+
*/
|
|
1822
|
+
runClusterer?: (vectors: readonly Float32Array[], params: ClusterRunParams) => Promise<number[]>;
|
|
1823
|
+
}
|
|
1824
|
+
export declare function createRecallPlugin(cfg: RecallPluginConfig): KernelPlugin;
|
|
1825
|
+
//#endregion
|
|
1826
|
+
//#region ../../plugins/substrate-recall/src/reindex-core.d.ts
|
|
1827
|
+
/** Rows per call when the caller does not say. */
|
|
1828
|
+
export declare const REINDEX_DEFAULT_LIMIT = 100;
|
|
1829
|
+
/**
|
|
1830
|
+
* Hard ceiling per call. Each row costs one embedder round-trip, so an
|
|
1831
|
+
* unbounded page is an unbounded request; the cursor is how a caller reaches
|
|
1832
|
+
* the rest of the corpus.
|
|
1833
|
+
*/
|
|
1834
|
+
export declare const REINDEX_MAX_LIMIT = 1000;
|
|
1835
|
+
interface ReindexInput {
|
|
1836
|
+
namespace: string;
|
|
1837
|
+
type: string;
|
|
1838
|
+
/** Opaque cursor from a prior call's `nextCursor`. */
|
|
1839
|
+
cursor?: string;
|
|
1840
|
+
limit?: number;
|
|
1841
|
+
}
|
|
1842
|
+
interface ReindexFailure {
|
|
1843
|
+
id: string;
|
|
1844
|
+
error: string;
|
|
1845
|
+
}
|
|
1846
|
+
interface ReindexResult {
|
|
1847
|
+
namespace: string;
|
|
1848
|
+
type: string;
|
|
1849
|
+
/** Rows read from the dataset this page. */
|
|
1850
|
+
scanned: number;
|
|
1851
|
+
embedded: number;
|
|
1852
|
+
/** Source returned the null sentinel; any existing vector was removed. */
|
|
1853
|
+
unindexable: number;
|
|
1854
|
+
failed: number;
|
|
1855
|
+
/**
|
|
1856
|
+
* One entry per failed row. Always emitted (`[]` when clean) so an empty
|
|
1857
|
+
* list means "nothing failed" rather than "this server does not say".
|
|
1858
|
+
*/
|
|
1859
|
+
failures: ReindexFailure[];
|
|
1860
|
+
hasMore: boolean;
|
|
1861
|
+
nextCursor?: string;
|
|
1862
|
+
}
|
|
1863
|
+
interface ReindexCoreDeps {
|
|
1864
|
+
registry: EmbedSourceRegistry;
|
|
1865
|
+
/**
|
|
1866
|
+
* Row source. Absent → the core answers `unavailable` rather than reporting
|
|
1867
|
+
* a clean run over zero rows: an embedded host with no dataset resolver must
|
|
1868
|
+
* not be able to conclude that a type has nothing to reindex.
|
|
1869
|
+
*/
|
|
1870
|
+
resolveDataset?: (ns: string) => Dataset | null;
|
|
1871
|
+
}
|
|
1872
|
+
/** Throws `{ code: 'bad_request' }` on any shape violation. */
|
|
1873
|
+
export declare function validateReindexInput(raw: unknown): ReindexInput;
|
|
1874
|
+
export declare function handleReindex(input: ReindexInput, deps: ReindexCoreDeps): Promise<ReindexResult>;
|
|
1875
|
+
//#endregion
|
|
1876
|
+
//#region ../../plugins/substrate-recall/src/reindex-route.d.ts
|
|
1877
|
+
type ReindexRouteConfig = ReindexCoreDeps;
|
|
1878
|
+
export declare function reindexRouteHandler(cfg: ReindexRouteConfig): HttpHandler;
|
|
1879
|
+
//#endregion
|
|
1880
|
+
//#region ../../plugins/substrate-recall/src/rrf.d.ts
|
|
1881
|
+
/**
|
|
1882
|
+
* Reciprocal Rank Fusion over an arbitrary number of retrieval arms.
|
|
1883
|
+
*
|
|
1884
|
+
* Ranks, not scores: the arms' scores are on incomparable scales (cosine,
|
|
1885
|
+
* `ts_rank`, term counts), and a rank is the one thing every arm agrees on.
|
|
1886
|
+
*
|
|
1887
|
+
* Union, not intersection: a row only one arm found still enters the pool,
|
|
1888
|
+
* so the lexical arm can surface rows the dense arm missed.
|
|
1889
|
+
*
|
|
1890
|
+
* A missing arm contributes no term, not a penalty term, so enabling an arm
|
|
1891
|
+
* cannot demote rows it says nothing about.
|
|
1892
|
+
*/
|
|
1893
|
+
/**
|
|
1894
|
+
* The RRF constant from Cormack et al. It damps the
|
|
1895
|
+
* difference between adjacent top ranks, so a row needs agreement across arms
|
|
1896
|
+
* rather than a narrow win within one to move.
|
|
1897
|
+
*/
|
|
1898
|
+
export declare const RRF_K = 60;
|
|
1899
|
+
/**
|
|
1900
|
+
* Fuse ordered arms into a score per id.
|
|
1901
|
+
*
|
|
1902
|
+
* Each arm is an ordered list of row ids, best first. Ranks are 1-based and
|
|
1903
|
+
* counted PER ARM — a row leading the second arm scores as a leader, not as
|
|
1904
|
+
* though it were buried behind the first arm's length.
|
|
1905
|
+
*
|
|
1906
|
+
* Duplicates within one arm are ignored rather than summed: a store returning
|
|
1907
|
+
* the same row twice is reporting one hit, and double-counting would let a
|
|
1908
|
+
* buggy arm outvote a correct one.
|
|
1909
|
+
*/
|
|
1910
|
+
export declare function fuseByRank(arms: ReadonlyArray<ReadonlyArray<string>>, k?: number): Map<string, number>;
|
|
1911
|
+
/** One arm's contribution to a fused row, for diagnostics. */
|
|
1912
|
+
interface ArmRank {
|
|
1913
|
+
/** 1-based rank within that arm, or null when the arm did not return it. */
|
|
1914
|
+
dense: number | null;
|
|
1915
|
+
lexical: number | null;
|
|
1916
|
+
}
|
|
1917
|
+
/**
|
|
1918
|
+
* Per-arm ranks for one id, for the opt-in diagnostic envelope.
|
|
1919
|
+
*
|
|
1920
|
+
* `null` means the arm did not return the row; ranks are 1-based and count
|
|
1921
|
+
* each id once, matching the rank `fuseByRank` scored.
|
|
1922
|
+
*/
|
|
1923
|
+
export declare function armRanks(id: string, dense: ReadonlyArray<string>, lexical: ReadonlyArray<string>): ArmRank;
|
|
1924
|
+
//#endregion
|
|
1925
|
+
//#region ../../plugins/substrate-recall/src/salience-boost-core.d.ts
|
|
1926
|
+
/** Per-request target cap — bounds the batched store write. */
|
|
1927
|
+
export declare const SALIENCE_BOOST_MAX_TARGETS = 500;
|
|
1928
|
+
/** What a boost actually did, as opposed to what it was asked to do. */
|
|
1929
|
+
interface SalienceBoostResult {
|
|
1930
|
+
ok: true;
|
|
1931
|
+
/** Targets the caller named. */
|
|
1932
|
+
requested: number;
|
|
1933
|
+
/**
|
|
1934
|
+
* Targets that matched a stored embedding and were reinforced — the field a
|
|
1935
|
+
* caller can assert on, since `requested` counts intent only.
|
|
1936
|
+
*/
|
|
1937
|
+
applied: number;
|
|
1938
|
+
/**
|
|
1939
|
+
* The targets that matched no embedding row, echoed back so a stale or
|
|
1940
|
+
* wrong id is diagnosable rather than a silent no-op.
|
|
1941
|
+
*/
|
|
1942
|
+
missing: SalienceTarget[];
|
|
1943
|
+
}
|
|
1944
|
+
//#endregion
|
|
1945
|
+
//#region ../../plugins/substrate-recall/src/salience-boost-route.d.ts
|
|
1946
|
+
interface SalienceBoostRouteConfig {
|
|
1947
|
+
store: EmbeddingsStore;
|
|
1948
|
+
/** Supplies the boost decay half-life. Defaults to DEFAULT_SALIENCE_CONFIG. */
|
|
1949
|
+
salience?: SalienceConfig;
|
|
1950
|
+
/** Unbound read-policy check — the route binds `req.identity`. */
|
|
1951
|
+
checkReadPolicy?: ReadPolicyCheck;
|
|
1952
|
+
/** Unbound write-policy check — the route binds `req.identity`. Absent → refused. */
|
|
1953
|
+
checkWritePolicy?: WritePolicyCheck;
|
|
1954
|
+
/** Embedding model whose stored bodies row filters are judged on. */
|
|
1955
|
+
model?: string;
|
|
1956
|
+
}
|
|
1957
|
+
/**
|
|
1958
|
+
* KernelRouteClaim handler factory for `POST /substrate/v1/salience/boost`.
|
|
1959
|
+
* Registered by `createRecallPlugin` alongside the recall route. Thin
|
|
1960
|
+
* transport adapter over `handleSalienceBoost` — JSON-parse + bind identity,
|
|
1961
|
+
* then map coded throws (bad_request → 400, policy_denied → 403) to envelopes.
|
|
1962
|
+
*/
|
|
1963
|
+
export declare function salienceBoostRouteHandler(cfg: SalienceBoostRouteConfig): HttpHandler;
|
|
1964
|
+
//#endregion
|
|
1965
|
+
//#region ../../plugins/substrate-recall/src/index.d.ts
|
|
1966
|
+
/**
|
|
1967
|
+
* @mnmal-ai/substrate-recall — the recall substrate plugin.
|
|
1968
|
+
*
|
|
1969
|
+
* Owns the embedder, the dense (pgvector) and lexical recall arms, and
|
|
1970
|
+
* crystallisation. Provides, among others:
|
|
1971
|
+
* - SubstrateMigrations: embeddings, lexical, recall clusters
|
|
1972
|
+
* - FilterOperator: embeddingSimilar (where-clause recall)
|
|
1973
|
+
* - KernelRoutes: POST /substrate/v1/recall, GET /substrate/v1/embed-stats, reindex routes
|
|
1974
|
+
* - McpTool: recall
|
|
1975
|
+
* - Primitives: EmbeddingsStore, LexicalStore, embedAndStore for plugin authors
|
|
1976
|
+
*
|
|
1977
|
+
* `createRecallPlugin` is the factory for direct programmatic callers.
|
|
1978
|
+
*/
|
|
1979
|
+
export declare const RECALL_VERSION: string;
|
|
1980
|
+
//#endregion
|
|
1981
|
+
export type { ArmRank, Candidate, CandidateSelector, ClusterMemberRecord, ClusterProvenance, ClusterRecord, ClusterRunParams, ClusterStore, ClusterSurfaceDeps, CrystalliseConfig, CrystalliseCoreDeps, CrystalliseGuard, CrystalliseInput, CrystalliseOutput, CrystalliseRouteConfig, EmbedAndStoreOptions, EmbedBinding, EmbedOutcome, EmbedSourceEntry, EmbedStats, EmbedStatsComposition, EmbedStatsResponse, EmbedStatsSnapshot, EmbeddedTypeRef, Embedder, EmbeddingsFetchRouteConfig, EmbeddingsMigrationOptions, EmbeddingsStore, EmbeddingsStoreDeleteInput, EmbeddingsStoreFetchResult, EmbeddingsStoreKnnInput, EmbeddingsStoreKnnResult, EmbeddingsStoreUpsertInput, InMemoryClusterStoreOptions, InMemoryEmbeddingsStoreOptions, InMemoryLexicalStoreOptions, LexicalClassifyOutcome, LexicalCoverageDeps, LexicalCoverageInput, LexicalCoverageResult, LexicalMigrationOptions, LexicalPostWriteBinding, LexicalStore, LexicalStoreDeleteInput, LexicalStoreSearchInput, LexicalStoreSearchResult, LexicalStoreUpsertInput, LexicalTypeCoverage, MemberRef, NewCluster, NewMember, PgClusterStoreOptions, PgEmbeddingsStoreOptions, PgLexicalStoreOptions, ReadPolicyCheck, RecallClustersMigrationOptions, RecallInput, RecallMcpToolConfig, RecallOutput, RecallPluginConfig, RecallRouteConfig, RecallRow, ReindexCoreDeps, ReindexFailure, ReindexInput, ReindexResult, ReindexRouteConfig, SalienceBoostResult, SalienceBoostRouteConfig };
|