@mnmal-ai/substrate 0.0.0-stage → 1.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +119 -2
- package/dist/agent-token-BIboWOU0.js +200 -0
- package/dist/client.d.ts +782 -0
- package/dist/client.js +7 -0
- package/dist/cluster-worker.js +35 -0
- package/dist/content-type-C7T28xNb.js +25 -0
- package/dist/dataset-BXkus75W.js +1527 -0
- package/dist/env-utils-B7SGmoeJ.js +21 -0
- package/dist/errors-D4vKuwz8.d.ts +130 -0
- package/dist/fake-DUVtKVSE.js +36 -0
- package/dist/index-D7mSOFlM.d.ts +69 -0
- package/dist/pack.d.ts +192 -0
- package/dist/pack.js +5 -0
- package/dist/plugin-j1jt1l3n.js +3844 -0
- package/dist/plugin-t2g2wX0b.d.ts +1568 -0
- package/dist/proposals-bootstrap.js +37 -0
- package/dist/proposals.config.schema.json +20 -0
- package/dist/proposals.d.ts +118 -0
- package/dist/proposals.default-config.json +4 -0
- package/dist/proposals.js +4 -0
- package/dist/recall-bootstrap.js +451 -0
- package/dist/recall.config.schema.json +232 -0
- package/dist/recall.d.ts +1981 -0
- package/dist/recall.default-config.json +4 -0
- package/dist/recall.js +5 -0
- package/dist/repl-C52wyfMs.js +2213 -0
- package/dist/row-ByLFE2Wx.d.ts +719 -0
- package/dist/row-DK7lPZVe.js +732 -0
- package/dist/runtime-metrics-DwpDJ8vz.d.ts +159 -0
- package/dist/schema-g4FQLr0V.d.ts +433 -0
- package/dist/sdk.d.ts +694 -0
- package/dist/sdk.js +112 -0
- package/dist/src-C4DzaIQZ.js +837 -0
- package/dist/src-CrEgiMJQ.js +1015 -0
- package/dist/src-CzxvmhFn.js +2327 -0
- package/dist/src-D88LvCX8.js +1797 -0
- package/dist/src-D91yij6C.js +13523 -0
- package/dist/src-DuopV8u7.js +2408 -0
- package/dist/substrate-admin.js +808 -0
- package/dist/substrate.js +5477 -0
- package/dist/testing.d.ts +1240 -0
- package/dist/testing.js +702 -0
- package/dist/types-8c0gtnd2.d.ts +35 -0
- package/dist/ws-C72iVS5T.d.ts +143 -0
- package/package.json +117 -3
|
@@ -0,0 +1,1015 @@
|
|
|
1
|
+
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
+
import { B as ROW_FILTER_FIELD, V as matchesRowFilter, W as inFlightError, Y as unresolvedRow, Z as cosineSimilarity, et as memberKey } from "./plugin-j1jt1l3n.js";
|
|
3
|
+
import { n as txScopeIdFor } from "./src-CzxvmhFn.js";
|
|
4
|
+
|
|
5
|
+
//#region ../../plugins/substrate-recall/package.json
|
|
6
|
+
var version = "1.0.0-alpha.0";
|
|
7
|
+
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region ../../plugins/substrate-recall/src/in-memory-cluster-store.ts
|
|
10
|
+
let nextTxScopeId$1 = 1;
|
|
11
|
+
var InMemoryClusterStore = class {
|
|
12
|
+
txScopeId;
|
|
13
|
+
#clusters = /* @__PURE__ */ new Map();
|
|
14
|
+
#members = [];
|
|
15
|
+
#nextShortId = 1;
|
|
16
|
+
constructor(opts = {}) {
|
|
17
|
+
this.txScopeId = opts.txScopeId ?? `memory-clusters-${nextTxScopeId$1++}`;
|
|
18
|
+
}
|
|
19
|
+
async createCluster(input) {
|
|
20
|
+
const record = {
|
|
21
|
+
id: input.id,
|
|
22
|
+
shortId: this.#nextShortId++,
|
|
23
|
+
centroid: new Float32Array(input.centroid),
|
|
24
|
+
memberCount: 0,
|
|
25
|
+
departedCount: 0,
|
|
26
|
+
cohesion: input.cohesion,
|
|
27
|
+
label: input.label,
|
|
28
|
+
labelledBy: input.labelledBy,
|
|
29
|
+
provenance: input.provenance,
|
|
30
|
+
formedAt: input.now ?? /* @__PURE__ */ new Date(),
|
|
31
|
+
lastAccretedAt: null,
|
|
32
|
+
retiredAt: null
|
|
33
|
+
};
|
|
34
|
+
this.#clusters.set(record.id, record);
|
|
35
|
+
return this.#copyCluster(record);
|
|
36
|
+
}
|
|
37
|
+
async attachMembers(inputs) {
|
|
38
|
+
const batchNow = /* @__PURE__ */ new Date();
|
|
39
|
+
const stamps = /* @__PURE__ */ new Map();
|
|
40
|
+
for (const input of inputs) {
|
|
41
|
+
if (this.#members.find((m) => m.clusterId === input.clusterId && m.memberNs === input.ref.namespace && m.memberType === input.ref.type && m.memberId === input.ref.id) !== void 0) continue;
|
|
42
|
+
this.#members.push({
|
|
43
|
+
id: input.id,
|
|
44
|
+
shortId: this.#nextShortId++,
|
|
45
|
+
clusterId: input.clusterId,
|
|
46
|
+
memberNs: input.ref.namespace,
|
|
47
|
+
memberType: input.ref.type,
|
|
48
|
+
memberId: input.ref.id,
|
|
49
|
+
origin: input.origin,
|
|
50
|
+
similarity: input.similarity,
|
|
51
|
+
attachedAt: input.now ?? batchNow,
|
|
52
|
+
departedAt: null
|
|
53
|
+
});
|
|
54
|
+
stamps.set(input.clusterId, input.now ?? batchNow);
|
|
55
|
+
}
|
|
56
|
+
for (const [clusterId, stamp] of stamps) {
|
|
57
|
+
const cluster = this.#clusters.get(clusterId);
|
|
58
|
+
if (cluster === void 0) continue;
|
|
59
|
+
this.#recount(clusterId);
|
|
60
|
+
cluster.lastAccretedAt = stamp;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
async listClusters(opts = {}) {
|
|
64
|
+
return Array.from(this.#clusters.values()).filter((c) => opts.includeRetired === true || c.retiredAt === null).sort((a, b) => a.formedAt.getTime() - b.formedAt.getTime() || a.shortId - b.shortId).map((c) => this.#copyCluster(c));
|
|
65
|
+
}
|
|
66
|
+
async getCluster(id) {
|
|
67
|
+
const found = this.#clusters.get(id);
|
|
68
|
+
return found === void 0 ? null : this.#copyCluster(found);
|
|
69
|
+
}
|
|
70
|
+
async listMembers(clusterId, opts = {}) {
|
|
71
|
+
return this.#members.filter((m) => m.clusterId === clusterId).filter((m) => opts.includeDeparted === true || m.departedAt === null).map((m) => ({ ...m }));
|
|
72
|
+
}
|
|
73
|
+
async listAllMembers() {
|
|
74
|
+
return this.#members.filter((m) => m.departedAt === null).map((m) => ({ ...m }));
|
|
75
|
+
}
|
|
76
|
+
async findClustersForMember(ref) {
|
|
77
|
+
return this.#members.filter((m) => m.memberNs === ref.namespace && m.memberType === ref.type && m.memberId === ref.id).map((m) => ({ ...m }));
|
|
78
|
+
}
|
|
79
|
+
async claimedMembers(refs) {
|
|
80
|
+
const wanted = new Set(refs.map(memberKey));
|
|
81
|
+
const out = /* @__PURE__ */ new Set();
|
|
82
|
+
for (const m of this.#members) {
|
|
83
|
+
if (m.departedAt !== null) continue;
|
|
84
|
+
const key = memberKey({
|
|
85
|
+
namespace: m.memberNs,
|
|
86
|
+
type: m.memberType,
|
|
87
|
+
id: m.memberId
|
|
88
|
+
});
|
|
89
|
+
if (wanted.has(key)) out.add(key);
|
|
90
|
+
}
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
93
|
+
async markDeparted(refs, now) {
|
|
94
|
+
const wanted = new Set(refs.map(memberKey));
|
|
95
|
+
const stamp = now ?? /* @__PURE__ */ new Date();
|
|
96
|
+
const touched = /* @__PURE__ */ new Set();
|
|
97
|
+
let count = 0;
|
|
98
|
+
for (const m of this.#members) {
|
|
99
|
+
if (m.departedAt !== null) continue;
|
|
100
|
+
const key = memberKey({
|
|
101
|
+
namespace: m.memberNs,
|
|
102
|
+
type: m.memberType,
|
|
103
|
+
id: m.memberId
|
|
104
|
+
});
|
|
105
|
+
if (!wanted.has(key)) continue;
|
|
106
|
+
m.departedAt = stamp;
|
|
107
|
+
touched.add(m.clusterId);
|
|
108
|
+
count++;
|
|
109
|
+
}
|
|
110
|
+
for (const clusterId of touched) this.#recount(clusterId);
|
|
111
|
+
return count;
|
|
112
|
+
}
|
|
113
|
+
async retire(clusterId, now) {
|
|
114
|
+
const cluster = this.#clusters.get(clusterId);
|
|
115
|
+
if (cluster === void 0 || cluster.retiredAt !== null) return;
|
|
116
|
+
cluster.retiredAt = now ?? /* @__PURE__ */ new Date();
|
|
117
|
+
}
|
|
118
|
+
async relabel(clusterId, label) {
|
|
119
|
+
const cluster = this.#clusters.get(clusterId);
|
|
120
|
+
if (cluster === void 0) return null;
|
|
121
|
+
cluster.label = label;
|
|
122
|
+
cluster.labelledBy = "agent";
|
|
123
|
+
return this.#copyCluster(cluster);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Copy a cluster for return, including `centroid` and `provenance.params`,
|
|
127
|
+
* so a caller mutating the result cannot corrupt the stored cluster.
|
|
128
|
+
*/
|
|
129
|
+
#copyCluster(record) {
|
|
130
|
+
return {
|
|
131
|
+
...record,
|
|
132
|
+
centroid: new Float32Array(record.centroid),
|
|
133
|
+
provenance: {
|
|
134
|
+
...record.provenance,
|
|
135
|
+
params: { ...record.provenance.params }
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/** Re-derive both counters from the membership rows — never increment blind. */
|
|
140
|
+
#recount(clusterId) {
|
|
141
|
+
const cluster = this.#clusters.get(clusterId);
|
|
142
|
+
if (cluster === void 0) return;
|
|
143
|
+
const mine = this.#members.filter((m) => m.clusterId === clusterId);
|
|
144
|
+
cluster.memberCount = mine.filter((m) => m.departedAt === null).length;
|
|
145
|
+
cluster.departedCount = mine.length - cluster.memberCount;
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
//#endregion
|
|
150
|
+
//#region ../../plugins/substrate-recall/src/in-memory-embeddings-store.ts
|
|
151
|
+
/** A row with no dataset wired to read it from. */
|
|
152
|
+
const UNRESOLVED$1 = Symbol("unresolved");
|
|
153
|
+
let nextTxScopeId = 1;
|
|
154
|
+
function key(ns, type, id) {
|
|
155
|
+
return `${ns}/${type}/${id}`;
|
|
156
|
+
}
|
|
157
|
+
var InMemoryEmbeddingsStore = class {
|
|
158
|
+
txScopeId;
|
|
159
|
+
#entries = /* @__PURE__ */ new Map();
|
|
160
|
+
#datasetFor;
|
|
161
|
+
constructor(opts = {}) {
|
|
162
|
+
this.txScopeId = opts.txScopeId ?? `mem-embeddings:${nextTxScopeId++}`;
|
|
163
|
+
this.#datasetFor = opts.datasetFor;
|
|
164
|
+
}
|
|
165
|
+
async upsert(input) {
|
|
166
|
+
const k = key(input.rowNamespace, input.rowType, input.rowId);
|
|
167
|
+
const existing = this.#entries.get(k);
|
|
168
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
169
|
+
if (existing !== void 0 && existing.updatedAt > now) return;
|
|
170
|
+
this.#entries.set(k, {
|
|
171
|
+
rowNamespace: input.rowNamespace,
|
|
172
|
+
rowType: input.rowType,
|
|
173
|
+
rowId: input.rowId,
|
|
174
|
+
vector: input.vector,
|
|
175
|
+
model: input.model,
|
|
176
|
+
updatedAt: now,
|
|
177
|
+
salience: existing?.salience ?? 0,
|
|
178
|
+
salienceUpdatedAt: existing?.salienceUpdatedAt ?? now
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
async delete(input) {
|
|
182
|
+
this.#entries.delete(key(input.rowNamespace, input.rowType, input.rowId));
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Materialize a row body via `datasetFor`, mirroring pg's join against
|
|
186
|
+
* `rows`. Returns UNRESOLVED when no dataset is wired, and `null`
|
|
187
|
+
* when the row is missing or deleted so the caller can exclude it.
|
|
188
|
+
*/
|
|
189
|
+
async #readRow(namespace, type, id) {
|
|
190
|
+
const dataset = this.#datasetFor?.(namespace, type);
|
|
191
|
+
if (dataset === void 0) return UNRESOLVED$1;
|
|
192
|
+
const found = (await dataset.read(type, { id })).rows[0];
|
|
193
|
+
return found === void 0 ? null : found;
|
|
194
|
+
}
|
|
195
|
+
async knnSearch(input) {
|
|
196
|
+
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
197
|
+
const candidates = [];
|
|
198
|
+
for (const entry of this.#entries.values()) {
|
|
199
|
+
if (entry.rowNamespace !== input.namespace) continue;
|
|
200
|
+
if (entry.rowType !== input.type) continue;
|
|
201
|
+
if (entry.model !== input.model) continue;
|
|
202
|
+
const score = cosineSimilarity(entry.vector, input.vector);
|
|
203
|
+
if (score < threshold) continue;
|
|
204
|
+
candidates.push({
|
|
205
|
+
entry,
|
|
206
|
+
score
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
candidates.sort((a, b) => b.score - a.score);
|
|
210
|
+
const limit = Math.max(1, Math.floor(input.k));
|
|
211
|
+
const results = [];
|
|
212
|
+
for (const { entry, score } of candidates) {
|
|
213
|
+
if (results.length >= limit) break;
|
|
214
|
+
const read = await this.#readRow(entry.rowNamespace, entry.rowType, entry.rowId);
|
|
215
|
+
if (read === null) continue;
|
|
216
|
+
if (input.rowFilter !== void 0 && (read === UNRESOLVED$1 || !matchesRowFilter(read, input.rowFilter))) continue;
|
|
217
|
+
const row = read === UNRESOLVED$1 ? unresolvedRow({ id: entry.rowId }) : read;
|
|
218
|
+
results.push({
|
|
219
|
+
row,
|
|
220
|
+
score,
|
|
221
|
+
salience: entry.salience,
|
|
222
|
+
...input.returnVector === true ? { vector: Array.from(entry.vector) } : {}
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
return results;
|
|
226
|
+
}
|
|
227
|
+
async boostSalience(targets, halfLifeHours, bump = 1) {
|
|
228
|
+
if (targets.length === 0) return [];
|
|
229
|
+
const now = /* @__PURE__ */ new Date();
|
|
230
|
+
const halfLifeMs = halfLifeHours * 3600 * 1e3;
|
|
231
|
+
const applied = [];
|
|
232
|
+
for (const t of targets) {
|
|
233
|
+
const entry = this.#entries.get(key(t.namespace, t.type, t.id));
|
|
234
|
+
if (entry === void 0) continue;
|
|
235
|
+
const elapsedMs = Math.max(0, now.getTime() - entry.salienceUpdatedAt.getTime());
|
|
236
|
+
const decay = Math.exp(-Math.LN2 * elapsedMs / halfLifeMs);
|
|
237
|
+
entry.salience = entry.salience * decay + bump;
|
|
238
|
+
entry.salienceUpdatedAt = now;
|
|
239
|
+
applied.push({
|
|
240
|
+
namespace: t.namespace,
|
|
241
|
+
type: t.type,
|
|
242
|
+
id: t.id
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
return applied;
|
|
246
|
+
}
|
|
247
|
+
async fetchEmbeddings(ids, model) {
|
|
248
|
+
if (ids.length === 0) return [];
|
|
249
|
+
const out = [];
|
|
250
|
+
for (const ref of ids) {
|
|
251
|
+
const entry = this.#entries.get(key(ref.namespace, ref.type, ref.id));
|
|
252
|
+
if (entry === void 0) continue;
|
|
253
|
+
if (entry.model !== model) continue;
|
|
254
|
+
const dataset = this.#datasetFor?.(entry.rowNamespace, entry.rowType);
|
|
255
|
+
let row = unresolvedRow({ id: entry.rowId });
|
|
256
|
+
if (dataset !== void 0) {
|
|
257
|
+
const found = (await dataset.read(entry.rowType, { id: entry.rowId })).rows[0];
|
|
258
|
+
if (found === void 0) continue;
|
|
259
|
+
row = found;
|
|
260
|
+
}
|
|
261
|
+
out.push({
|
|
262
|
+
namespace: entry.rowNamespace,
|
|
263
|
+
type: entry.rowType,
|
|
264
|
+
id: entry.rowId,
|
|
265
|
+
row,
|
|
266
|
+
vector: entry.vector.slice(),
|
|
267
|
+
salience: entry.salience
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
return out;
|
|
271
|
+
}
|
|
272
|
+
async selectCandidates(selector, model) {
|
|
273
|
+
const inScope = (ns, type) => selector.scopes.some((s) => s.namespace === ns && s.type === type);
|
|
274
|
+
const cmp = (x, y) => x < y ? -1 : x > y ? 1 : 0;
|
|
275
|
+
const entries = Array.from(this.#entries.values()).filter((e) => e.model === model && inScope(e.rowNamespace, e.rowType)).sort((a, b) => cmp(a.rowNamespace, b.rowNamespace) || cmp(a.rowType, b.rowType) || cmp(a.rowId, b.rowId));
|
|
276
|
+
const out = [];
|
|
277
|
+
for (const e of entries) {
|
|
278
|
+
if (out.length >= selector.limit) break;
|
|
279
|
+
const read = await this.#readRow(e.rowNamespace, e.rowType, e.rowId);
|
|
280
|
+
if (read === null) continue;
|
|
281
|
+
const row = read === UNRESOLVED$1 ? { id: e.rowId } : read;
|
|
282
|
+
if (selector.excludeTextPrefixes.some((ex) => {
|
|
283
|
+
const value = row[ex.field];
|
|
284
|
+
if (value === void 0 || value === null) return false;
|
|
285
|
+
const t = typeof value;
|
|
286
|
+
if (t !== "string" && t !== "number" && t !== "boolean") return false;
|
|
287
|
+
return String(value).startsWith(ex.prefix);
|
|
288
|
+
})) continue;
|
|
289
|
+
out.push({
|
|
290
|
+
namespace: e.rowNamespace,
|
|
291
|
+
type: e.rowType,
|
|
292
|
+
id: e.rowId,
|
|
293
|
+
vector: new Float32Array(e.vector),
|
|
294
|
+
row
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
return out;
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
//#endregion
|
|
302
|
+
//#region ../../plugins/substrate-recall/src/in-memory-lexical-store.ts
|
|
303
|
+
/** No row resolver is wired, so the row body is unknown and cannot satisfy a filter. */
|
|
304
|
+
const UNRESOLVED = Symbol("unresolved");
|
|
305
|
+
/** Lowercase, split on anything that is not a letter or digit, drop empties. */
|
|
306
|
+
function tokenise(text) {
|
|
307
|
+
return text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((t) => t.length > 0);
|
|
308
|
+
}
|
|
309
|
+
var InMemoryLexicalStore = class {
|
|
310
|
+
#byKey = /* @__PURE__ */ new Map();
|
|
311
|
+
#opts;
|
|
312
|
+
constructor(opts = {}) {
|
|
313
|
+
this.#opts = opts;
|
|
314
|
+
}
|
|
315
|
+
#key(ns, type, id) {
|
|
316
|
+
return `${ns}/${type}/${id}`;
|
|
317
|
+
}
|
|
318
|
+
async upsert(input) {
|
|
319
|
+
const k = this.#key(input.rowNamespace, input.rowType, input.rowId);
|
|
320
|
+
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
321
|
+
if ((this.#byKey.get(k)?.updatedAt ?? now) > now) return;
|
|
322
|
+
this.#byKey.set(k, {
|
|
323
|
+
rowNamespace: input.rowNamespace,
|
|
324
|
+
rowType: input.rowType,
|
|
325
|
+
rowId: input.rowId,
|
|
326
|
+
tokens: new Set(tokenise(input.text)),
|
|
327
|
+
updatedAt: now
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
async delete(input) {
|
|
331
|
+
this.#byKey.delete(this.#key(input.rowNamespace, input.rowType, input.rowId));
|
|
332
|
+
}
|
|
333
|
+
async countIndexed(namespace, type) {
|
|
334
|
+
let n = 0;
|
|
335
|
+
for (const entry of this.#byKey.values()) if (entry.rowNamespace === namespace && entry.rowType === type) n++;
|
|
336
|
+
return n;
|
|
337
|
+
}
|
|
338
|
+
async search(input) {
|
|
339
|
+
const terms = [...new Set(tokenise(input.query))];
|
|
340
|
+
if (terms.length === 0) return [];
|
|
341
|
+
const scored = [];
|
|
342
|
+
for (const entry of this.#byKey.values()) {
|
|
343
|
+
if (entry.rowNamespace !== input.namespace) continue;
|
|
344
|
+
if (entry.rowType !== input.type) continue;
|
|
345
|
+
const score = terms.reduce((n, t) => n + (entry.tokens.has(t) ? 1 : 0), 0);
|
|
346
|
+
if (score === 0) continue;
|
|
347
|
+
scored.push({
|
|
348
|
+
entry,
|
|
349
|
+
score
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
const full = scored.filter((s) => s.score === terms.length);
|
|
353
|
+
const strict = await this.#take(full, input.k, input.rowFilter);
|
|
354
|
+
if (strict.length > 0 || full.length === scored.length) return strict;
|
|
355
|
+
return this.#take(scored, input.k, input.rowFilter);
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* The best `k` of `pool` whose rows can be read and satisfy `filter`. Rows
|
|
359
|
+
* are read in rank order and only until k are kept, so the filter applies
|
|
360
|
+
* before the cut.
|
|
361
|
+
*/
|
|
362
|
+
async #take(pool, k, filter) {
|
|
363
|
+
const ranked = [...pool].sort((a, b) => b.score - a.score || a.entry.rowId.localeCompare(b.entry.rowId));
|
|
364
|
+
const out = [];
|
|
365
|
+
for (const { entry, score } of ranked) {
|
|
366
|
+
if (out.length >= k) break;
|
|
367
|
+
const row = await this.#readRow(entry.rowNamespace, entry.rowType, entry.rowId);
|
|
368
|
+
if (row === null) continue;
|
|
369
|
+
if (filter !== void 0 && (row === UNRESOLVED || !matchesRowFilter(row, filter))) continue;
|
|
370
|
+
out.push({
|
|
371
|
+
rowNamespace: entry.rowNamespace,
|
|
372
|
+
rowType: entry.rowType,
|
|
373
|
+
rowId: entry.rowId,
|
|
374
|
+
row: row === UNRESOLVED ? unresolvedRow({}) : row,
|
|
375
|
+
score
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
return out;
|
|
379
|
+
}
|
|
380
|
+
/** The row, `null` when it is gone, or UNRESOLVED when no resolver is wired. */
|
|
381
|
+
async #readRow(ns, type, id) {
|
|
382
|
+
const dataset = this.#opts.datasetFor?.(ns);
|
|
383
|
+
if (dataset === void 0 || dataset === null) return UNRESOLVED;
|
|
384
|
+
return (await dataset.read(type, { id })).rows[0] ?? null;
|
|
385
|
+
}
|
|
386
|
+
};
|
|
387
|
+
|
|
388
|
+
//#endregion
|
|
389
|
+
//#region ../../plugins/substrate-recall/src/pg-vector-text.ts
|
|
390
|
+
/**
|
|
391
|
+
* pgvector wire helpers, shared by PgEmbeddingsStore and PgClusterStore.
|
|
392
|
+
*
|
|
393
|
+
* pgvector parses a text literal `[v1,v2,...,vN]`; Float32Array is not a
|
|
394
|
+
* recognised postgres.js parameter, so callers convert before interpolation
|
|
395
|
+
* and cast inside SQL.
|
|
396
|
+
*/
|
|
397
|
+
function vectorLiteral(v) {
|
|
398
|
+
return `[${Array.from(v).join(",")}]`;
|
|
399
|
+
}
|
|
400
|
+
function parseVectorText(text) {
|
|
401
|
+
const parts = text.replace(/^\[|\]$/g, "").split(",");
|
|
402
|
+
const out = new Float32Array(parts.length);
|
|
403
|
+
for (let i = 0; i < parts.length; i++) out[i] = Number(parts[i]);
|
|
404
|
+
return out;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
//#endregion
|
|
408
|
+
//#region ../../plugins/substrate-recall/src/pg-cluster-store.ts
|
|
409
|
+
/**
|
|
410
|
+
* PgClusterStore — postgres implementation of `ClusterStore`.
|
|
411
|
+
*
|
|
412
|
+
* Sibling of `PgEmbeddingsStore`: same handle type, same `txScopeIdFor`,
|
|
413
|
+
* same pgvector text-literal wire convention.
|
|
414
|
+
*
|
|
415
|
+
* Counters are always RE-DERIVED from the membership rows inside the same
|
|
416
|
+
* transaction as the write that touched them, never incremented in place:
|
|
417
|
+
* `departedCount` and `memberCount` are changed by different code paths, and
|
|
418
|
+
* an increment is only correct if every such path increments too.
|
|
419
|
+
*/
|
|
420
|
+
function toCluster(r) {
|
|
421
|
+
return {
|
|
422
|
+
id: r.id,
|
|
423
|
+
shortId: Number(r.short_id),
|
|
424
|
+
centroid: parseVectorText(r.centroid),
|
|
425
|
+
memberCount: r.member_count,
|
|
426
|
+
departedCount: r.departed_count,
|
|
427
|
+
cohesion: r.cohesion,
|
|
428
|
+
label: r.label,
|
|
429
|
+
labelledBy: r.labelled_by === "agent" ? "agent" : "medoid",
|
|
430
|
+
provenance: JSON.parse(r.provenance),
|
|
431
|
+
formedAt: r.formed_at,
|
|
432
|
+
lastAccretedAt: r.last_accreted_at,
|
|
433
|
+
retiredAt: r.retired_at
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
function toMember(r) {
|
|
437
|
+
return {
|
|
438
|
+
id: r.id,
|
|
439
|
+
shortId: Number(r.short_id),
|
|
440
|
+
clusterId: r.cluster_id,
|
|
441
|
+
memberNs: r.member_ns,
|
|
442
|
+
memberType: r.member_type,
|
|
443
|
+
memberId: r.member_id,
|
|
444
|
+
origin: r.origin === "accretion" ? "accretion" : "formation",
|
|
445
|
+
similarity: r.similarity,
|
|
446
|
+
attachedAt: r.attached_at,
|
|
447
|
+
departedAt: r.departed_at
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
/** `centroid::text` — pgvector has no postgres.js codec, so it arrives as text. */
|
|
451
|
+
const CLUSTER_COLUMNS = "id, short_id, centroid::text AS centroid, member_count, departed_count, cohesion, label, labelled_by, provenance, formed_at, last_accreted_at, retired_at";
|
|
452
|
+
/**
|
|
453
|
+
* Re-derive `member_count` / `departed_count` for `clusterIds` from the
|
|
454
|
+
* membership rows, inside the caller's transaction, in one set-based UPDATE.
|
|
455
|
+
*
|
|
456
|
+
* `stamps` optionally sets a per-cluster `last_accreted_at`; a cluster with
|
|
457
|
+
* no entry keeps its existing value.
|
|
458
|
+
*/
|
|
459
|
+
async function recountClusters(tx, clusterIds, stamps) {
|
|
460
|
+
if (clusterIds.length === 0) return;
|
|
461
|
+
await tx`
|
|
462
|
+
UPDATE recall_clusters c SET
|
|
463
|
+
member_count = counts.live,
|
|
464
|
+
departed_count = counts.departed,
|
|
465
|
+
last_accreted_at = COALESCE(want.stamp::timestamptz, c.last_accreted_at)
|
|
466
|
+
FROM (
|
|
467
|
+
SELECT cluster_id,
|
|
468
|
+
count(*) FILTER (WHERE departed_at IS NULL) AS live,
|
|
469
|
+
count(*) FILTER (WHERE departed_at IS NOT NULL) AS departed
|
|
470
|
+
FROM recall_cluster_members
|
|
471
|
+
WHERE cluster_id = ANY(${clusterIds}::text[])
|
|
472
|
+
GROUP BY cluster_id
|
|
473
|
+
) counts
|
|
474
|
+
JOIN unnest(${clusterIds}::text[], ${clusterIds.map((id) => stamps?.get(id)?.toISOString() ?? null)}::text[]) AS want(cluster_id, stamp)
|
|
475
|
+
ON want.cluster_id = counts.cluster_id
|
|
476
|
+
WHERE c.id = counts.cluster_id
|
|
477
|
+
`;
|
|
478
|
+
}
|
|
479
|
+
/** Distinct from the migrator's `substrate-migrations` key; hashed the same way. */
|
|
480
|
+
const FORMATION_LOCK_KEY = "substrate-recall-crystallise";
|
|
481
|
+
var PgClusterStore = class {
|
|
482
|
+
#sql;
|
|
483
|
+
txScopeId;
|
|
484
|
+
constructor(opts) {
|
|
485
|
+
this.#sql = opts.sql;
|
|
486
|
+
this.txScopeId = txScopeIdFor(opts.sql);
|
|
487
|
+
}
|
|
488
|
+
/**
|
|
489
|
+
* Runs `fn` holding a session advisory lock that excludes formation in every
|
|
490
|
+
* process on this database; throws `crystallise_in_flight` if it is held.
|
|
491
|
+
*
|
|
492
|
+
* Lock and unlock share one reserved connection, since through the pool the
|
|
493
|
+
* unlock can land on another session and fail to release.
|
|
494
|
+
*/
|
|
495
|
+
async withFormationLock(fn) {
|
|
496
|
+
const conn = await this.#sql.reserve();
|
|
497
|
+
try {
|
|
498
|
+
const [row] = await conn`
|
|
499
|
+
SELECT pg_try_advisory_lock(hashtext(${FORMATION_LOCK_KEY})) AS acquired
|
|
500
|
+
`;
|
|
501
|
+
if (row?.acquired !== true) throw inFlightError();
|
|
502
|
+
try {
|
|
503
|
+
return await fn();
|
|
504
|
+
} finally {
|
|
505
|
+
await conn`SELECT pg_advisory_unlock(hashtext(${FORMATION_LOCK_KEY}))`;
|
|
506
|
+
}
|
|
507
|
+
} finally {
|
|
508
|
+
conn.release();
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
async createCluster(input) {
|
|
512
|
+
const row = (await this.#sql.unsafe(`INSERT INTO recall_clusters
|
|
513
|
+
(id, centroid, cohesion, label, labelled_by, provenance, formed_at)
|
|
514
|
+
VALUES ($1, $2::vector, $3, $4, $5, $6, $7)
|
|
515
|
+
RETURNING ${CLUSTER_COLUMNS}`, [
|
|
516
|
+
input.id,
|
|
517
|
+
vectorLiteral(input.centroid),
|
|
518
|
+
input.cohesion,
|
|
519
|
+
input.label,
|
|
520
|
+
input.labelledBy,
|
|
521
|
+
JSON.stringify(input.provenance),
|
|
522
|
+
input.now ?? /* @__PURE__ */ new Date()
|
|
523
|
+
]))[0];
|
|
524
|
+
if (row === void 0) throw new Error(`PgClusterStore.createCluster: insert returned no row for '${input.id}'`);
|
|
525
|
+
return toCluster(row);
|
|
526
|
+
}
|
|
527
|
+
async attachMembers(inputs) {
|
|
528
|
+
if (inputs.length === 0) return;
|
|
529
|
+
const sql = this.#sql;
|
|
530
|
+
const batchNow = /* @__PURE__ */ new Date();
|
|
531
|
+
await sql.begin(async (tx) => {
|
|
532
|
+
const stamps = /* @__PURE__ */ new Map();
|
|
533
|
+
for (const input of inputs) if ((await tx`
|
|
534
|
+
INSERT INTO recall_cluster_members
|
|
535
|
+
(id, cluster_id, member_ns, member_type, member_id, origin, similarity, attached_at)
|
|
536
|
+
VALUES (
|
|
537
|
+
${input.id}, ${input.clusterId}, ${input.ref.namespace}, ${input.ref.type},
|
|
538
|
+
${input.ref.id}, ${input.origin}, ${input.similarity}, ${input.now ?? batchNow}
|
|
539
|
+
)
|
|
540
|
+
ON CONFLICT (cluster_id, member_ns, member_type, member_id) DO NOTHING
|
|
541
|
+
RETURNING id
|
|
542
|
+
`).length > 0) stamps.set(input.clusterId, input.now ?? batchNow);
|
|
543
|
+
await recountClusters(tx, Array.from(stamps.keys()), stamps);
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
async listClusters(opts = {}) {
|
|
547
|
+
return (await this.#sql.unsafe(`SELECT ${CLUSTER_COLUMNS} FROM recall_clusters
|
|
548
|
+
${opts.includeRetired === true ? "" : "WHERE retired_at IS NULL"}
|
|
549
|
+
ORDER BY formed_at ASC, short_id ASC`)).map(toCluster);
|
|
550
|
+
}
|
|
551
|
+
async getCluster(id) {
|
|
552
|
+
const row = (await this.#sql.unsafe(`SELECT ${CLUSTER_COLUMNS} FROM recall_clusters WHERE id = $1`, [id]))[0];
|
|
553
|
+
return row === void 0 ? null : toCluster(row);
|
|
554
|
+
}
|
|
555
|
+
async listMembers(clusterId, opts = {}) {
|
|
556
|
+
const sql = this.#sql;
|
|
557
|
+
return (await sql`
|
|
558
|
+
SELECT * FROM recall_cluster_members
|
|
559
|
+
WHERE cluster_id = ${clusterId}
|
|
560
|
+
${opts.includeDeparted === true ? sql`` : sql`AND departed_at IS NULL`}
|
|
561
|
+
ORDER BY short_id ASC
|
|
562
|
+
`).map(toMember);
|
|
563
|
+
}
|
|
564
|
+
async listAllMembers() {
|
|
565
|
+
return (await this.#sql`
|
|
566
|
+
SELECT * FROM recall_cluster_members
|
|
567
|
+
WHERE departed_at IS NULL
|
|
568
|
+
ORDER BY short_id ASC
|
|
569
|
+
`).map(toMember);
|
|
570
|
+
}
|
|
571
|
+
async findClustersForMember(ref) {
|
|
572
|
+
return (await this.#sql`
|
|
573
|
+
SELECT * FROM recall_cluster_members
|
|
574
|
+
WHERE member_ns = ${ref.namespace}
|
|
575
|
+
AND member_type = ${ref.type}
|
|
576
|
+
AND member_id = ${ref.id}
|
|
577
|
+
ORDER BY short_id ASC
|
|
578
|
+
`).map(toMember);
|
|
579
|
+
}
|
|
580
|
+
async claimedMembers(refs) {
|
|
581
|
+
if (refs.length === 0) return /* @__PURE__ */ new Set();
|
|
582
|
+
const rows = await this.#sql`
|
|
583
|
+
SELECT DISTINCT m.member_ns, m.member_type, m.member_id
|
|
584
|
+
FROM recall_cluster_members m
|
|
585
|
+
JOIN unnest(${refs.map((r) => r.namespace)}::text[], ${refs.map((r) => r.type)}::text[], ${refs.map((r) => r.id)}::text[])
|
|
586
|
+
AS want(ns, type, id)
|
|
587
|
+
ON want.ns = m.member_ns AND want.type = m.member_type AND want.id = m.member_id
|
|
588
|
+
WHERE m.departed_at IS NULL
|
|
589
|
+
`;
|
|
590
|
+
return new Set(rows.map((r) => memberKey({
|
|
591
|
+
namespace: r.member_ns,
|
|
592
|
+
type: r.member_type,
|
|
593
|
+
id: r.member_id
|
|
594
|
+
})));
|
|
595
|
+
}
|
|
596
|
+
async markDeparted(refs, now) {
|
|
597
|
+
if (refs.length === 0) return 0;
|
|
598
|
+
const sql = this.#sql;
|
|
599
|
+
const stamp = now ?? /* @__PURE__ */ new Date();
|
|
600
|
+
const namespaces = refs.map((r) => r.namespace);
|
|
601
|
+
const types = refs.map((r) => r.type);
|
|
602
|
+
const ids = refs.map((r) => r.id);
|
|
603
|
+
return sql.begin(async (tx) => {
|
|
604
|
+
const updated = await tx`
|
|
605
|
+
UPDATE recall_cluster_members m
|
|
606
|
+
SET departed_at = ${stamp}
|
|
607
|
+
FROM unnest(${namespaces}::text[], ${types}::text[], ${ids}::text[]) AS want(ns, type, id)
|
|
608
|
+
WHERE want.ns = m.member_ns
|
|
609
|
+
AND want.type = m.member_type
|
|
610
|
+
AND want.id = m.member_id
|
|
611
|
+
AND m.departed_at IS NULL
|
|
612
|
+
RETURNING m.cluster_id
|
|
613
|
+
`;
|
|
614
|
+
if (updated.length === 0) return 0;
|
|
615
|
+
await recountClusters(tx, Array.from(new Set(updated.map((r) => r.cluster_id))));
|
|
616
|
+
return updated.length;
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
async retire(clusterId, now) {
|
|
620
|
+
await this.#sql`
|
|
621
|
+
UPDATE recall_clusters
|
|
622
|
+
SET retired_at = ${now ?? /* @__PURE__ */ new Date()}
|
|
623
|
+
WHERE id = ${clusterId} AND retired_at IS NULL
|
|
624
|
+
`;
|
|
625
|
+
}
|
|
626
|
+
async relabel(clusterId, label) {
|
|
627
|
+
const row = (await this.#sql.unsafe(`UPDATE recall_clusters SET label = $2, labelled_by = 'agent'
|
|
628
|
+
WHERE id = $1
|
|
629
|
+
RETURNING ${CLUSTER_COLUMNS}`, [clusterId, label]))[0];
|
|
630
|
+
return row === void 0 ? null : toCluster(row);
|
|
631
|
+
}
|
|
632
|
+
};
|
|
633
|
+
|
|
634
|
+
//#endregion
|
|
635
|
+
//#region ../../plugins/substrate-recall/src/pg-row-filter.ts
|
|
636
|
+
/** The row body, as the dataset read path returns it. */
|
|
637
|
+
const ROW_BODY_SQL = "r.body";
|
|
638
|
+
/**
|
|
639
|
+
* The row filter as `AND …` clauses, for appending to a WHERE. Every
|
|
640
|
+
* comparison is as text against the body, the same semantics as
|
|
641
|
+
* `matchesRowFilter`. Field names are re-checked
|
|
642
|
+
* against ROW_FILTER_FIELD and values are bound, so nothing caller-supplied is
|
|
643
|
+
* spliced into SQL.
|
|
644
|
+
*/
|
|
645
|
+
function rowFilterSql(tx, filter) {
|
|
646
|
+
if (filter === void 0) return tx``;
|
|
647
|
+
let out = tx``;
|
|
648
|
+
for (const [field, pred] of Object.entries(filter)) {
|
|
649
|
+
if (!ROW_FILTER_FIELD.test(field)) throw new Error(`row filter field '${field}' is not a plain identifier`);
|
|
650
|
+
const text = tx`(r.body ->> ${field})`;
|
|
651
|
+
const isNull = tx`(r.body ->> ${field}) IS NULL`;
|
|
652
|
+
const values = pred.in ?? [pred.eq ?? null];
|
|
653
|
+
const strings = values.filter((v) => v !== null).map((v) => String(v));
|
|
654
|
+
const wantsNull = values.some((v) => v === null);
|
|
655
|
+
const clause = strings.length === 0 ? isNull : wantsNull ? tx`(${text} = ANY(${strings}::text[]) OR ${isNull})` : tx`${text} = ANY(${strings}::text[])`;
|
|
656
|
+
out = tx`${out} AND ${clause}`;
|
|
657
|
+
}
|
|
658
|
+
return out;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
//#endregion
|
|
662
|
+
//#region ../../plugins/substrate-recall/src/pg-embeddings-store.ts
|
|
663
|
+
/**
|
|
664
|
+
* PgEmbeddingsStore — PostgreSQL+pgvector implementation of `EmbeddingsStore`.
|
|
665
|
+
*
|
|
666
|
+
* One table (`embeddings`) keyed by (rowNamespace, rowType, rowId). The
|
|
667
|
+
* model identifier is stored with the vector; knnSearch filters on it after
|
|
668
|
+
* the vector scan.
|
|
669
|
+
*
|
|
670
|
+
* Vectors cross the wire as pgvector text literals (`[v1,...,vN]`), bound as
|
|
671
|
+
* parameters and cast in SQL.
|
|
672
|
+
*
|
|
673
|
+
* The read path sets `hnsw.iterative_scan = 'strict_order'` and
|
|
674
|
+
* `hnsw.ef_search` with SET LOCAL inside a transaction. Without iterative
|
|
675
|
+
* scan, a WHERE on namespace/type makes the planner fall back to seqscan +
|
|
676
|
+
* sort and bypass the HNSW index; model stays out of the WHERE for the same
|
|
677
|
+
* reason.
|
|
678
|
+
*/
|
|
679
|
+
const DEFAULT_HNSW_EF_SEARCH = 200;
|
|
680
|
+
function clampK(k) {
|
|
681
|
+
if (!Number.isFinite(k) || k < 1) return 1;
|
|
682
|
+
if (k > 100) return 100;
|
|
683
|
+
return Math.floor(k);
|
|
684
|
+
}
|
|
685
|
+
var PgEmbeddingsStore = class {
|
|
686
|
+
#sql;
|
|
687
|
+
txScopeId;
|
|
688
|
+
constructor(opts) {
|
|
689
|
+
this.#sql = opts.sql;
|
|
690
|
+
this.txScopeId = txScopeIdFor(opts.sql);
|
|
691
|
+
}
|
|
692
|
+
async upsert(input) {
|
|
693
|
+
const sql = this.#sql;
|
|
694
|
+
const literal = vectorLiteral(input.vector);
|
|
695
|
+
try {
|
|
696
|
+
await sql`
|
|
697
|
+
INSERT INTO embeddings (row_namespace, row_type, row_id, embedding, model, updated_at)
|
|
698
|
+
VALUES (
|
|
699
|
+
${input.rowNamespace},
|
|
700
|
+
${input.rowType},
|
|
701
|
+
${input.rowId},
|
|
702
|
+
${literal}::vector,
|
|
703
|
+
${input.model},
|
|
704
|
+
${input.now ?? /* @__PURE__ */ new Date()}
|
|
705
|
+
)
|
|
706
|
+
ON CONFLICT (row_namespace, row_type, row_id) DO UPDATE SET
|
|
707
|
+
embedding = EXCLUDED.embedding,
|
|
708
|
+
model = EXCLUDED.model,
|
|
709
|
+
updated_at = EXCLUDED.updated_at
|
|
710
|
+
WHERE embeddings.updated_at <= EXCLUDED.updated_at
|
|
711
|
+
`;
|
|
712
|
+
} catch (err) {
|
|
713
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
714
|
+
if (/expected \d+ dimensions/i.test(msg)) throw new Error(`PgEmbeddingsStore.upsert: embedding dimension mismatch for ${input.rowNamespace}/${input.rowType}/${input.rowId} (${input.vector.length} dims) — ${msg}. The embedder's output dimension disagrees with the embeddings.embedding column; align the embedder config with the table, or re-embed via POST /substrate/v1/embed/reindex.`);
|
|
715
|
+
throw err;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
async delete(input) {
|
|
719
|
+
await this.#sql`
|
|
720
|
+
DELETE FROM embeddings
|
|
721
|
+
WHERE row_namespace = ${input.rowNamespace}
|
|
722
|
+
AND row_type = ${input.rowType}
|
|
723
|
+
AND row_id = ${input.rowId}
|
|
724
|
+
`;
|
|
725
|
+
}
|
|
726
|
+
async knnSearch(input) {
|
|
727
|
+
const sql = this.#sql;
|
|
728
|
+
const literal = vectorLiteral(input.vector);
|
|
729
|
+
const k = clampK(input.k);
|
|
730
|
+
const threshold = typeof input.threshold === "number" ? input.threshold : 0;
|
|
731
|
+
const wantVector = input.returnVector === true;
|
|
732
|
+
return (await sql.begin(async (tx) => {
|
|
733
|
+
await tx`SET LOCAL hnsw.iterative_scan = 'strict_order'`;
|
|
734
|
+
await tx.unsafe(`SET LOCAL hnsw.ef_search = ${DEFAULT_HNSW_EF_SEARCH}`);
|
|
735
|
+
return tx`
|
|
736
|
+
SELECT
|
|
737
|
+
${tx.unsafe(ROW_BODY_SQL)} AS row,
|
|
738
|
+
1 - (e.embedding <=> ${literal}::vector) AS score,
|
|
739
|
+
e.salience AS salience
|
|
740
|
+
${wantVector ? tx`, e.embedding::text AS embedding` : tx``}
|
|
741
|
+
FROM embeddings e
|
|
742
|
+
JOIN rows r
|
|
743
|
+
ON r.namespace = e.row_namespace
|
|
744
|
+
AND r.type = e.row_type
|
|
745
|
+
AND r.id = e.row_id
|
|
746
|
+
WHERE e.row_namespace = ${input.namespace}
|
|
747
|
+
AND e.row_type = ${input.type}
|
|
748
|
+
AND e.model = ${input.model}
|
|
749
|
+
AND r.deleted_at IS NULL
|
|
750
|
+
AND 1 - (e.embedding <=> ${literal}::vector) >= ${threshold}
|
|
751
|
+
${rowFilterSql(tx, input.rowFilter)}
|
|
752
|
+
ORDER BY e.embedding <=> ${literal}::vector
|
|
753
|
+
LIMIT ${k}
|
|
754
|
+
`;
|
|
755
|
+
})).map((r) => ({
|
|
756
|
+
row: r.row,
|
|
757
|
+
score: r.score,
|
|
758
|
+
salience: r.salience,
|
|
759
|
+
...wantVector && typeof r.embedding === "string" ? { vector: Array.from(parseVectorText(r.embedding)) } : {}
|
|
760
|
+
}));
|
|
761
|
+
}
|
|
762
|
+
async boostSalience(targets, halfLifeHours, bump = 1) {
|
|
763
|
+
if (targets.length === 0) return [];
|
|
764
|
+
const sql = this.#sql;
|
|
765
|
+
return (await sql`
|
|
766
|
+
UPDATE embeddings AS e
|
|
767
|
+
SET salience = e.salience
|
|
768
|
+
* exp(
|
|
769
|
+
-ln(2) * GREATEST(0, EXTRACT(EPOCH FROM (now() - e.salience_updated_at)))
|
|
770
|
+
/ ${halfLifeHours * 3600}
|
|
771
|
+
)
|
|
772
|
+
+ ${bump},
|
|
773
|
+
salience_updated_at = now()
|
|
774
|
+
FROM (VALUES ${sql(targets.map((t) => [
|
|
775
|
+
t.namespace,
|
|
776
|
+
t.type,
|
|
777
|
+
t.id
|
|
778
|
+
]))}) AS t(ns, ty, id)
|
|
779
|
+
WHERE e.row_namespace = t.ns
|
|
780
|
+
AND e.row_type = t.ty
|
|
781
|
+
AND e.row_id = t.id
|
|
782
|
+
RETURNING e.row_namespace AS ns, e.row_type AS ty, e.row_id AS id
|
|
783
|
+
`).map((r) => ({
|
|
784
|
+
namespace: r.ns,
|
|
785
|
+
type: r.ty,
|
|
786
|
+
id: r.id
|
|
787
|
+
}));
|
|
788
|
+
}
|
|
789
|
+
async fetchEmbeddings(ids, model) {
|
|
790
|
+
if (ids.length === 0) return [];
|
|
791
|
+
const sql = this.#sql;
|
|
792
|
+
return (await sql`
|
|
793
|
+
SELECT e.row_namespace, e.row_type, e.row_id, e.embedding::text AS embedding,
|
|
794
|
+
${sql.unsafe(ROW_BODY_SQL)} AS body, e.salience
|
|
795
|
+
FROM (VALUES ${sql(ids.map((id) => [
|
|
796
|
+
id.namespace,
|
|
797
|
+
id.type,
|
|
798
|
+
id.id
|
|
799
|
+
]))}) AS t(ns, ty, id)
|
|
800
|
+
JOIN embeddings e
|
|
801
|
+
ON e.row_namespace = t.ns AND e.row_type = t.ty AND e.row_id = t.id
|
|
802
|
+
JOIN rows r
|
|
803
|
+
ON r.namespace = e.row_namespace AND r.type = e.row_type AND r.id = e.row_id
|
|
804
|
+
WHERE e.model = ${model}
|
|
805
|
+
AND r.deleted_at IS NULL
|
|
806
|
+
`).map((r) => ({
|
|
807
|
+
namespace: r.row_namespace,
|
|
808
|
+
type: r.row_type,
|
|
809
|
+
id: r.row_id,
|
|
810
|
+
row: r.body,
|
|
811
|
+
vector: parseVectorText(r.embedding),
|
|
812
|
+
salience: r.salience
|
|
813
|
+
}));
|
|
814
|
+
}
|
|
815
|
+
async selectCandidates(selector, model) {
|
|
816
|
+
const sql = this.#sql;
|
|
817
|
+
const namespaces = selector.scopes.map((s) => s.namespace);
|
|
818
|
+
const types = selector.scopes.map((s) => s.type);
|
|
819
|
+
const exclusionFields = selector.excludeTextPrefixes.map((e) => e.field);
|
|
820
|
+
const exclusionPrefixes = selector.excludeTextPrefixes.map((e) => e.prefix);
|
|
821
|
+
return (await sql`
|
|
822
|
+
SELECT e.row_namespace, e.row_type, e.row_id, e.embedding::text AS embedding,
|
|
823
|
+
${sql.unsafe(ROW_BODY_SQL)} AS body
|
|
824
|
+
FROM embeddings e
|
|
825
|
+
JOIN rows r
|
|
826
|
+
ON r.namespace = e.row_namespace
|
|
827
|
+
AND r.type = e.row_type
|
|
828
|
+
AND r.id = e.row_id
|
|
829
|
+
JOIN unnest(${namespaces}::text[], ${types}::text[]) AS scope(ns, type)
|
|
830
|
+
ON scope.ns = e.row_namespace AND scope.type = e.row_type
|
|
831
|
+
WHERE e.model = ${model}
|
|
832
|
+
AND r.deleted_at IS NULL
|
|
833
|
+
AND NOT EXISTS (
|
|
834
|
+
SELECT 1
|
|
835
|
+
FROM unnest(${exclusionFields}::text[], ${exclusionPrefixes}::text[]) AS ex(field, prefix)
|
|
836
|
+
-- starts_with(), not LIKE: a caller-supplied prefix must be matched
|
|
837
|
+
-- literally. LIKE treats '_' and '%' in the prefix as wildcards, so
|
|
838
|
+
-- a prefix like 'perf_seed' would also exclude 'perfXseed…' — the
|
|
839
|
+
-- in-memory side (String(value).startsWith(...)) never had that
|
|
840
|
+
-- hazard, so the two implementations silently disagreed. Restricted
|
|
841
|
+
-- to scalar JSON values (jsonb_typeof) to match the in-memory side,
|
|
842
|
+
-- which coerces only string/number/boolean via String(value) — a
|
|
843
|
+
-- composite (object/array) has no agreed-on text rendering between
|
|
844
|
+
-- pg's jsonb ->> and JS's String(), so both sides now treat a
|
|
845
|
+
-- composite as never-excluded rather than risk disagreeing.
|
|
846
|
+
WHERE jsonb_typeof(${sql.unsafe(ROW_BODY_SQL)} -> ex.field) IN ('string', 'number', 'boolean')
|
|
847
|
+
AND starts_with(${sql.unsafe(ROW_BODY_SQL)} ->> ex.field, ex.prefix)
|
|
848
|
+
)
|
|
849
|
+
ORDER BY e.row_namespace COLLATE "C" ASC, e.row_type COLLATE "C" ASC, e.row_id COLLATE "C" ASC
|
|
850
|
+
LIMIT ${selector.limit}
|
|
851
|
+
`).map((r) => ({
|
|
852
|
+
namespace: r.row_namespace,
|
|
853
|
+
type: r.row_type,
|
|
854
|
+
id: r.row_id,
|
|
855
|
+
vector: parseVectorText(r.embedding),
|
|
856
|
+
row: r.body
|
|
857
|
+
}));
|
|
858
|
+
}
|
|
859
|
+
/**
|
|
860
|
+
* Deletes every embedding whose backing row is missing or soft-deleted.
|
|
861
|
+
* Needed because `embeddings` has no FK to `rows` and no
|
|
862
|
+
* dataset delete path touches it; tombstones are terminal, so soft-deleted
|
|
863
|
+
* rows are safe to collect immediately. Run as a sweep stage by the binary.
|
|
864
|
+
*
|
|
865
|
+
* @returns number of orphaned embeddings deleted.
|
|
866
|
+
*/
|
|
867
|
+
async purgeOrphans() {
|
|
868
|
+
return (await this.#sql`
|
|
869
|
+
DELETE FROM embeddings e
|
|
870
|
+
WHERE NOT EXISTS (
|
|
871
|
+
SELECT 1 FROM rows r
|
|
872
|
+
WHERE r.namespace = e.row_namespace
|
|
873
|
+
AND r.type = e.row_type
|
|
874
|
+
AND r.id = e.row_id
|
|
875
|
+
AND r.deleted_at IS NULL
|
|
876
|
+
)
|
|
877
|
+
`).count;
|
|
878
|
+
}
|
|
879
|
+
};
|
|
880
|
+
|
|
881
|
+
//#endregion
|
|
882
|
+
//#region ../../plugins/substrate-recall/src/pg-lexical-store.ts
|
|
883
|
+
/** Hard ceiling on `k`, mirroring the embeddings store's cap. */
|
|
884
|
+
const MAX_K = 100;
|
|
885
|
+
var PgLexicalStore = class {
|
|
886
|
+
#sql;
|
|
887
|
+
#tsConfig;
|
|
888
|
+
constructor(sql, opts = {}) {
|
|
889
|
+
this.#sql = sql;
|
|
890
|
+
this.#tsConfig = opts.textSearchConfig ?? "english";
|
|
891
|
+
}
|
|
892
|
+
async upsert(input) {
|
|
893
|
+
await this.#sql`
|
|
894
|
+
INSERT INTO lexical (row_namespace, row_type, row_id, content, updated_at)
|
|
895
|
+
VALUES (${input.rowNamespace}, ${input.rowType}, ${input.rowId}, ${input.text}, ${input.now ?? /* @__PURE__ */ new Date()})
|
|
896
|
+
ON CONFLICT (row_namespace, row_type, row_id)
|
|
897
|
+
DO UPDATE SET content = EXCLUDED.content, updated_at = EXCLUDED.updated_at
|
|
898
|
+
WHERE lexical.updated_at <= EXCLUDED.updated_at
|
|
899
|
+
`;
|
|
900
|
+
}
|
|
901
|
+
async delete(input) {
|
|
902
|
+
await this.#sql`
|
|
903
|
+
DELETE FROM lexical
|
|
904
|
+
WHERE row_namespace = ${input.rowNamespace}
|
|
905
|
+
AND row_type = ${input.rowType}
|
|
906
|
+
AND row_id = ${input.rowId}
|
|
907
|
+
`;
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
910
|
+
* Deletes every lexical row whose backing row is missing or soft-deleted,
|
|
911
|
+
* as `PgEmbeddingsStore.purgeOrphans` does for vectors: `lexical` has no FK
|
|
912
|
+
* to `rows`, so a drop, purge or tombstone would otherwise leave the text.
|
|
913
|
+
*
|
|
914
|
+
* @returns number of orphaned lexical rows deleted.
|
|
915
|
+
*/
|
|
916
|
+
async purgeOrphans() {
|
|
917
|
+
return (await this.#sql`
|
|
918
|
+
DELETE FROM lexical l
|
|
919
|
+
WHERE NOT EXISTS (
|
|
920
|
+
SELECT 1 FROM rows r
|
|
921
|
+
WHERE r.namespace = l.row_namespace
|
|
922
|
+
AND r.type = l.row_type
|
|
923
|
+
AND r.id = l.row_id
|
|
924
|
+
AND r.deleted_at IS NULL
|
|
925
|
+
)
|
|
926
|
+
`).count;
|
|
927
|
+
}
|
|
928
|
+
async countIndexed(namespace, type) {
|
|
929
|
+
const rows = await this.#sql`
|
|
930
|
+
SELECT COUNT(*) AS n
|
|
931
|
+
FROM lexical l
|
|
932
|
+
JOIN rows r
|
|
933
|
+
ON r.namespace = l.row_namespace
|
|
934
|
+
AND r.type = l.row_type
|
|
935
|
+
AND r.id = l.row_id
|
|
936
|
+
WHERE l.row_namespace = ${namespace}
|
|
937
|
+
AND l.row_type = ${type}
|
|
938
|
+
AND r.deleted_at IS NULL
|
|
939
|
+
`;
|
|
940
|
+
return Number(rows[0]?.n ?? 0);
|
|
941
|
+
}
|
|
942
|
+
/**
|
|
943
|
+
* Returns up to `k` live rows ranked by `ts_rank`. Matches ALL query terms
|
|
944
|
+
* first; only if that finds nothing, retries matching ANY term.
|
|
945
|
+
*
|
|
946
|
+
* Two passes rather than always-OR: nothing downstream filters lexical hits
|
|
947
|
+
* by threshold, so an OR pass that always ran would admit rows matching only
|
|
948
|
+
* a common term. The fallback fires only where the strict pass found nothing.
|
|
949
|
+
* Both passes apply `rowFilter`, so the fallback fires when no MATCHING row
|
|
950
|
+
* carries every term.
|
|
951
|
+
*/
|
|
952
|
+
async search(input) {
|
|
953
|
+
const k = Math.min(Math.max(1, Math.trunc(input.k)), MAX_K);
|
|
954
|
+
const strict = await this.#run(input, k, false);
|
|
955
|
+
if (strict.length > 0) return strict;
|
|
956
|
+
return this.#run(input, k, true);
|
|
957
|
+
}
|
|
958
|
+
async #run(input, k, degrade) {
|
|
959
|
+
const query = degrade ? this.#sql`(
|
|
960
|
+
SELECT CASE
|
|
961
|
+
WHEN position('!' in t) > 0 THEN NULL
|
|
962
|
+
ELSE replace(t, ' & ', ' | ')::tsquery
|
|
963
|
+
END AS query
|
|
964
|
+
FROM (SELECT websearch_to_tsquery(${this.#tsConfig}, ${input.query})::text AS t) s
|
|
965
|
+
)` : this.#sql`websearch_to_tsquery(${this.#tsConfig}, ${input.query})`;
|
|
966
|
+
return (await this.#sql`
|
|
967
|
+
SELECT
|
|
968
|
+
l.row_namespace,
|
|
969
|
+
l.row_type,
|
|
970
|
+
l.row_id,
|
|
971
|
+
${this.#sql.unsafe(ROW_BODY_SQL)} AS body,
|
|
972
|
+
ts_rank(l.tsv, q.query) AS score
|
|
973
|
+
FROM lexical l
|
|
974
|
+
CROSS JOIN ${query} AS q(query)
|
|
975
|
+
JOIN rows r
|
|
976
|
+
ON r.namespace = l.row_namespace
|
|
977
|
+
AND r.type = l.row_type
|
|
978
|
+
AND r.id = l.row_id
|
|
979
|
+
WHERE l.row_namespace = ${input.namespace}
|
|
980
|
+
AND l.row_type = ${input.type}
|
|
981
|
+
AND r.deleted_at IS NULL
|
|
982
|
+
AND q.query IS NOT NULL
|
|
983
|
+
AND l.tsv @@ q.query
|
|
984
|
+
${rowFilterSql(this.#sql, input.rowFilter)}
|
|
985
|
+
ORDER BY score DESC, l.row_id ASC
|
|
986
|
+
LIMIT ${k}
|
|
987
|
+
`).map((r) => ({
|
|
988
|
+
rowNamespace: r.row_namespace,
|
|
989
|
+
rowType: r.row_type,
|
|
990
|
+
rowId: r.row_id,
|
|
991
|
+
row: r.body,
|
|
992
|
+
score: r.score
|
|
993
|
+
}));
|
|
994
|
+
}
|
|
995
|
+
};
|
|
996
|
+
|
|
997
|
+
//#endregion
|
|
998
|
+
//#region ../../plugins/substrate-recall/src/index.ts
|
|
999
|
+
/**
|
|
1000
|
+
* @mnmal-ai/substrate-recall — the recall substrate plugin.
|
|
1001
|
+
*
|
|
1002
|
+
* Owns the embedder, the dense (pgvector) and lexical recall arms, and
|
|
1003
|
+
* crystallisation. Provides, among others:
|
|
1004
|
+
* - SubstrateMigrations: embeddings, lexical, recall clusters
|
|
1005
|
+
* - FilterOperator: embeddingSimilar (where-clause recall)
|
|
1006
|
+
* - KernelRoutes: POST /substrate/v1/recall, GET /substrate/v1/embed-stats, reindex routes
|
|
1007
|
+
* - McpTool: recall
|
|
1008
|
+
* - Primitives: EmbeddingsStore, LexicalStore, embedAndStore for plugin authors
|
|
1009
|
+
*
|
|
1010
|
+
* `createRecallPlugin` is the factory for direct programmatic callers.
|
|
1011
|
+
*/
|
|
1012
|
+
const RECALL_VERSION = version;
|
|
1013
|
+
|
|
1014
|
+
//#endregion
|
|
1015
|
+
export { InMemoryLexicalStore as a, PgClusterStore as i, PgLexicalStore as n, InMemoryEmbeddingsStore as o, PgEmbeddingsStore as r, InMemoryClusterStore as s, RECALL_VERSION as t };
|