@dzhechkov/harness-core 0.3.76 → 0.3.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agentdb-index.d.ts +84 -0
- package/dist/agentdb-index.d.ts.map +1 -1
- package/dist/agentdb-index.js +235 -0
- package/dist/agentdb-index.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/operations.d.ts.map +1 -1
- package/dist/operations.js +17 -0
- package/dist/operations.js.map +1 -1
- package/dist/patterns.d.ts +66 -4
- package/dist/patterns.d.ts.map +1 -1
- package/dist/patterns.js +0 -0
- package/dist/patterns.js.map +1 -1
- package/dist/vector-tier.d.ts +378 -0
- package/dist/vector-tier.d.ts.map +1 -0
- package/dist/vector-tier.js +1154 -0
- package/dist/vector-tier.js.map +1 -0
- package/package.json +12 -6
- package/src/agentdb-index.ts +300 -0
- package/src/index.ts +48 -3
- package/src/operations.ts +17 -0
- package/src/patterns.ts +0 -0
- package/src/vector-tier.ts +1458 -0
|
@@ -0,0 +1,1154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Tier — the SEMANTIC half of the dz self-learning pattern store
|
|
3
|
+
* (`features/dz-rvf-vector-bridge`, ADR-001 "Option A extended").
|
|
4
|
+
*
|
|
5
|
+
* One PORT ({@link VectorEngine}) with two adapters behind it:
|
|
6
|
+
*
|
|
7
|
+
* - **AgentdbVectorEngine** (default) — reuses the `.dz/agentdb.db` ReasoningBank store the
|
|
8
|
+
* consolidate Option-C mirror already writes (`agentdb-index.ts`), so semantic recall reads
|
|
9
|
+
* the vectors that exist today. Zero new dependencies: `agentdb`/`better-sqlite3` are
|
|
10
|
+
* dynamically resolved from the PROJECT, never imported at module top level.
|
|
11
|
+
* - **RvfVectorEngine** (opt-in, `memory.vector.engine = "rvf"`) — the portable single-file
|
|
12
|
+
* VECTOR form (`.dz/memory/patterns.rvf` + `.idmap.json`/`.manifest.json` sidecars) via a
|
|
13
|
+
* lazily imported `@ruvector/rvf`. Never a `dependencies` entry (at most a documentation-only
|
|
14
|
+
* `peerDependenciesMeta`).
|
|
15
|
+
*
|
|
16
|
+
* HONEST-ERROR CONTRACT (load-bearing, Invariant I-1): every function in this module returns an
|
|
17
|
+
* honest `{ …, error?: string }` receipt and NEVER throws or hangs — engine absence, a failed
|
|
18
|
+
* embed, a locked DB, or a timeout all degrade to today's exact lexical behavior. The lexical
|
|
19
|
+
* store (`patterns.sqlite`/`patterns.json`) is the SOURCE OF TRUTH; the vector tier is a
|
|
20
|
+
* best-effort MIRROR that can be rebuilt from it at any time (`dz consolidate` backfill), and a
|
|
21
|
+
* vector hit whose lexical twin is gone is DROPPED, never resurrected (Invariant V-1).
|
|
22
|
+
*
|
|
23
|
+
* Constraint 7 (QR-10, field-diagnosis discipline): when lexical and vector counts diverge,
|
|
24
|
+
* **rule out local DB corruption before blaming the mirror** — check `dz doctor`'s store health
|
|
25
|
+
* first; the mirror's own divergence line (`dz vector status`) reports BOTH counts plus the
|
|
26
|
+
* `dz consolidate` backfill hint, and is informational, never an error.
|
|
27
|
+
*
|
|
28
|
+
* Both engine calls (read `search` AND write `upsert`/`listIds`) are wall-time bounded
|
|
29
|
+
* ({@link DEFAULT_VECTOR_TIMEOUT_MS}); a write-side timeout (e.g. a first-run embedding-model
|
|
30
|
+
* download) lands the batch in `.dz/mirror-pending.json` and never blocks `dz teach` (NC1).
|
|
31
|
+
*
|
|
32
|
+
* @packageDocumentation
|
|
33
|
+
*/
|
|
34
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, renameSync, appendFileSync, copyFileSync } from 'node:fs';
|
|
35
|
+
import { basename, dirname, join } from 'node:path';
|
|
36
|
+
import { pathToFileURL } from 'node:url';
|
|
37
|
+
import { createRequire } from 'node:module';
|
|
38
|
+
import { isNoiseInsight } from '@dzhechkov/memory';
|
|
39
|
+
import { recallPatterns, recordToPattern, patternRecordId, patternIdentityOf, dreamRecordId, loadStoreRecords, removePatternsByIds, snapshotStore, } from './patterns.js';
|
|
40
|
+
import { indexPatternsToAgentdb, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb, } from './agentdb-index.js';
|
|
41
|
+
/** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
|
|
42
|
+
export const DEFAULT_VECTOR_TIMEOUT_MS = 10_000;
|
|
43
|
+
/** The pinned local embedding model + dimension (agentdb's `EmbeddingService`; the RVF manifest space). */
|
|
44
|
+
const LOCAL_EMBED_MODEL = 'Xenova/all-MiniLM-L6-v2';
|
|
45
|
+
const LOCAL_EMBED_DIM = 384;
|
|
46
|
+
/** Default cosine cutoff for near-duplicate clustering (`--threshold` / config overrides). */
|
|
47
|
+
export const DEFAULT_HARMONIZE_THRESHOLD = 0.92;
|
|
48
|
+
/* ------------------------------------------------------------------ */
|
|
49
|
+
/* Timeout wrapper (both legs — NC1/QR-1) */
|
|
50
|
+
/* ------------------------------------------------------------------ */
|
|
51
|
+
/**
|
|
52
|
+
* Bound `promise` to `ms` wall-clock milliseconds. On timeout, resolve with `onTimeout()`
|
|
53
|
+
* instead — the underlying operation keeps running detached (its eventual write is later
|
|
54
|
+
* deduplicated by dzId), but the CALLER's latency is bounded. A rejection also resolves via
|
|
55
|
+
* `onTimeout()` (honest-error contract: this wrapper never throws). The timer is cleared /
|
|
56
|
+
* unref'd so a fast path never keeps the process alive.
|
|
57
|
+
*/
|
|
58
|
+
export async function withVectorTimeout(promise, ms, onTimeout) {
|
|
59
|
+
let timer;
|
|
60
|
+
try {
|
|
61
|
+
return await Promise.race([
|
|
62
|
+
promise.catch(() => onTimeout()),
|
|
63
|
+
new Promise((resolvePromise) => {
|
|
64
|
+
timer = setTimeout(() => resolvePromise(onTimeout()), ms);
|
|
65
|
+
timer.unref?.();
|
|
66
|
+
}),
|
|
67
|
+
]);
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
if (timer !== undefined)
|
|
71
|
+
clearTimeout(timer);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Invoke an engine call so that BOTH a synchronous throw and an async rejection surface as an
|
|
76
|
+
* honest `onError(message)` value (never as the timeout fallback — a throw and a timeout are
|
|
77
|
+
* different diagnoses in the field). Pairs with {@link withVectorTimeout} at every call site.
|
|
78
|
+
*/
|
|
79
|
+
function safeEngineCall(fn, onError) {
|
|
80
|
+
const msg = (err) => (err instanceof Error ? err.message : String(err));
|
|
81
|
+
try {
|
|
82
|
+
return fn().then((v) => v, (err) => onError(msg(err)));
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
return Promise.resolve(onError(msg(err)));
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/* ------------------------------------------------------------------ */
|
|
89
|
+
/* Noise gate + ACL mappers (I-6, V-3) */
|
|
90
|
+
/* ------------------------------------------------------------------ */
|
|
91
|
+
const TOOL_TELEMETRY_RE = /^Tool \S+ invoked during session$/;
|
|
92
|
+
/** Text that must never be embedded: bare-approval echoes + tool telemetry (V-3 / ADR-002). */
|
|
93
|
+
export function isVectorNoise(text) {
|
|
94
|
+
return isNoiseInsight(text) || TOOL_TELEMETRY_RE.test(text);
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* ACL: taught {@link PatternRecord} → {@link VectorEntry}. Returns `undefined` for noise (the
|
|
98
|
+
* ingest gate — I-6). Score is the record's REAL reward, never a fabricated 1.0.
|
|
99
|
+
*/
|
|
100
|
+
export function patternVectorEntry(p, source = 'dz-teach') {
|
|
101
|
+
if (isVectorNoise(p.pattern))
|
|
102
|
+
return undefined;
|
|
103
|
+
const dzId = patternRecordId(p);
|
|
104
|
+
return {
|
|
105
|
+
dzId,
|
|
106
|
+
text: p.pattern,
|
|
107
|
+
score: p.reward,
|
|
108
|
+
taskType: 'dz-teach',
|
|
109
|
+
tags: ['dz-teach', p.type],
|
|
110
|
+
metadata: { dzId, source, ts: p.ts, domain: p.domain },
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* ACL: harvested {@link DreamPattern} → {@link VectorEntry}. Byte-compatible with the
|
|
115
|
+
* pre-feature consolidate mirror rows (same task_type `dz-learning`, same tags, `dreamId`
|
|
116
|
+
* preserved in metadata — NFR-7); `dzId` is additive.
|
|
117
|
+
*/
|
|
118
|
+
export function dreamVectorEntry(d) {
|
|
119
|
+
if (isVectorNoise(d.insight))
|
|
120
|
+
return undefined;
|
|
121
|
+
const dzId = dreamRecordId(d);
|
|
122
|
+
return {
|
|
123
|
+
dzId,
|
|
124
|
+
text: d.insight,
|
|
125
|
+
score: d.score,
|
|
126
|
+
taskType: 'dz-learning',
|
|
127
|
+
tags: ['dz-consolidate', d.outcome],
|
|
128
|
+
metadata: { source: 'dz-consolidate', skillId: d.skillId, sessionFile: d.sessionFile, ts: d.timestamp, dreamId: dzId, dzId },
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
/** ACL: stored {@link MemoryRecord} → {@link VectorEntry} (the consolidate-backfill mapper). */
|
|
132
|
+
export function memoryRecordVectorEntry(r) {
|
|
133
|
+
if (isVectorNoise(r.text))
|
|
134
|
+
return undefined;
|
|
135
|
+
return {
|
|
136
|
+
dzId: r.id,
|
|
137
|
+
text: r.text,
|
|
138
|
+
score: r.score,
|
|
139
|
+
taskType: r.id.startsWith('dream:') ? 'dz-learning' : 'dz-teach',
|
|
140
|
+
tags: ['dz-backfill', r.outcome],
|
|
141
|
+
metadata: { dzId: r.id, source: r.metadata?.['source'] ?? 'dz-backfill', ts: r.timestamp, skillId: r.skillId },
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
/* ------------------------------------------------------------------ */
|
|
145
|
+
/* Config + engine resolution cascade (05 §2.1) */
|
|
146
|
+
/* ------------------------------------------------------------------ */
|
|
147
|
+
/** Read `memory.vector.engine` from `.dz/config.json`. Absent/corrupt ⇒ `auto` (never throws). */
|
|
148
|
+
export function readVectorEngineMode(projectRoot) {
|
|
149
|
+
try {
|
|
150
|
+
const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8'));
|
|
151
|
+
const mode = cfg.memory?.vector?.engine;
|
|
152
|
+
return mode === 'off' || mode === 'agentdb' || mode === 'rvf' || mode === 'auto' ? mode : 'auto';
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
return 'auto';
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Read `memory.vector.harmonizeThreshold` from `.dz/config.json`. Absent/corrupt/out-of-range ⇒
|
|
160
|
+
* {@link DEFAULT_HARMONIZE_THRESHOLD} (never throws). `--threshold` overrides this at the call site.
|
|
161
|
+
*/
|
|
162
|
+
export function readHarmonizeThreshold(projectRoot) {
|
|
163
|
+
try {
|
|
164
|
+
const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8'));
|
|
165
|
+
const t = cfg.memory?.vector?.harmonizeThreshold;
|
|
166
|
+
return typeof t === 'number' && t > 0 && t <= 1 ? t : DEFAULT_HARMONIZE_THRESHOLD;
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return DEFAULT_HARMONIZE_THRESHOLD;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Should `dz teach` attempt the best-effort vector mirror at all? True when the project opted
|
|
174
|
+
* into the agentdb memory backend (`memory.backend === 'agentdb'`, the same gate consolidate
|
|
175
|
+
* uses — D3) or explicitly configured a vector engine. A fresh, unconfigured project returns
|
|
176
|
+
* `false`, so its `dz teach` output stays byte-identical to the pre-feature baseline (AC-1).
|
|
177
|
+
*/
|
|
178
|
+
export function vectorMirrorEnabled(projectRoot) {
|
|
179
|
+
try {
|
|
180
|
+
const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8'));
|
|
181
|
+
if (cfg.memory?.backend === 'agentdb')
|
|
182
|
+
return true;
|
|
183
|
+
const engine = cfg.memory?.vector?.engine;
|
|
184
|
+
return engine === 'agentdb' || engine === 'rvf';
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Engine selection cascade: config mode → `require.resolve` probe (NO module load — a
|
|
192
|
+
* lexical-only project pays zero engine cost, NFR-5) → adapter or an honest reason.
|
|
193
|
+
* Never throws. `auto` prefers agentdb (it reads the vectors consolidate already wrote),
|
|
194
|
+
* falling through to rvf.
|
|
195
|
+
*/
|
|
196
|
+
/**
|
|
197
|
+
* Is `pkgName` installed for this PROJECT? A pure filesystem probe: walk `node_modules` up the
|
|
198
|
+
* directory tree from `projectRoot` (the npm resolution chain) — deliberately NOT
|
|
199
|
+
* `require.resolve`, which also consults process-global paths (`NODE_PATH`/global folders) and
|
|
200
|
+
* would make a lexical-only project's engine availability depend on the HOST process instead
|
|
201
|
+
* of the project (the I-1 determinism leak). No module is loaded (NFR-5).
|
|
202
|
+
*/
|
|
203
|
+
function isPackageInstalled(projectRoot, pkgName) {
|
|
204
|
+
let dir = projectRoot;
|
|
205
|
+
for (;;) {
|
|
206
|
+
if (existsSync(join(dir, 'node_modules', pkgName, 'package.json')))
|
|
207
|
+
return true;
|
|
208
|
+
const parent = dirname(dir);
|
|
209
|
+
if (parent === dir)
|
|
210
|
+
return false;
|
|
211
|
+
dir = parent;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
export function resolveVectorEngine(projectRoot) {
|
|
215
|
+
const mode = readVectorEngineMode(projectRoot);
|
|
216
|
+
if (mode === 'off')
|
|
217
|
+
return { reason: 'vector tier disabled (memory.vector.engine = "off")' };
|
|
218
|
+
const canResolve = (id) => isPackageInstalled(projectRoot, id);
|
|
219
|
+
if (mode === 'agentdb' || mode === 'auto') {
|
|
220
|
+
if (canResolve('agentdb') && canResolve('better-sqlite3'))
|
|
221
|
+
return { engine: agentdbVectorEngine(projectRoot) };
|
|
222
|
+
if (mode === 'agentdb') {
|
|
223
|
+
return { reason: 'agentdb/better-sqlite3 not installed in project (run: dz setup --memory agentdb)' };
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (mode === 'rvf' || mode === 'auto') {
|
|
227
|
+
if (canResolve('@ruvector/rvf'))
|
|
228
|
+
return { engine: rvfVectorEngine(projectRoot) };
|
|
229
|
+
if (mode === 'rvf') {
|
|
230
|
+
return { reason: '@ruvector/rvf not installed in project (npm i @ruvector/rvf) — vector tier inactive' };
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return { reason: 'no vector engine available: agentdb/better-sqlite3 not installed in project (run: dz setup --memory agentdb)' };
|
|
234
|
+
}
|
|
235
|
+
function pendingPath(projectRoot) {
|
|
236
|
+
return join(projectRoot, '.dz', 'mirror-pending.json');
|
|
237
|
+
}
|
|
238
|
+
function toPending(e) {
|
|
239
|
+
return { ...e, insight: e.text };
|
|
240
|
+
}
|
|
241
|
+
/** Legacy queue rows (pre-port `DreamPattern[]`) are converted on read — nothing is lost. */
|
|
242
|
+
function readVectorPending(projectRoot) {
|
|
243
|
+
try {
|
|
244
|
+
const arr = JSON.parse(readFileSync(pendingPath(projectRoot), 'utf-8'));
|
|
245
|
+
if (!Array.isArray(arr))
|
|
246
|
+
return [];
|
|
247
|
+
const out = [];
|
|
248
|
+
for (const item of arr) {
|
|
249
|
+
if (typeof item !== 'object' || item === null)
|
|
250
|
+
continue;
|
|
251
|
+
const rec = item;
|
|
252
|
+
if (typeof rec['dzId'] === 'string' && typeof rec['text'] === 'string') {
|
|
253
|
+
out.push(toPending(rec));
|
|
254
|
+
}
|
|
255
|
+
else if (typeof rec['insight'] === 'string' && typeof rec['timestamp'] === 'string') {
|
|
256
|
+
const entry = dreamVectorEntry(rec);
|
|
257
|
+
if (entry !== undefined)
|
|
258
|
+
out.push(toPending(entry));
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return out;
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
return [];
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
function writeVectorPending(projectRoot, entries) {
|
|
268
|
+
try {
|
|
269
|
+
if (entries.length === 0) {
|
|
270
|
+
if (existsSync(pendingPath(projectRoot)))
|
|
271
|
+
rmSync(pendingPath(projectRoot));
|
|
272
|
+
}
|
|
273
|
+
else {
|
|
274
|
+
const path = pendingPath(projectRoot);
|
|
275
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
276
|
+
// Atomic write: a concurrent teach/consolidate must never observe a torn file. Write to a
|
|
277
|
+
// temp sibling then rename() (atomic on POSIX). Recovery guarantee: even if a racing writer
|
|
278
|
+
// clobbers the queue, backfillVectorMirror re-derives the missing set from the lexical store,
|
|
279
|
+
// so a dropped entry is recovered on the next consolidate — no permanent loss.
|
|
280
|
+
const tmp = `${path}.tmp`;
|
|
281
|
+
writeFileSync(tmp, JSON.stringify(entries, null, 2));
|
|
282
|
+
renameSync(tmp, path);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
catch { /* best-effort */ }
|
|
286
|
+
}
|
|
287
|
+
/** Honest failure note next to the session telemetry (the detached SessionEnd path is silent). */
|
|
288
|
+
function logMirrorNote(projectRoot, error, pending) {
|
|
289
|
+
try {
|
|
290
|
+
appendFileSync(join(projectRoot, '.dz', 'sessions.jsonl'), JSON.stringify({ event: 'mirror', ts: new Date().toISOString(), error, pending }) + '\n');
|
|
291
|
+
}
|
|
292
|
+
catch { /* best-effort */ }
|
|
293
|
+
}
|
|
294
|
+
function pickEngine(projectRoot, opts) {
|
|
295
|
+
if (opts.engine === null)
|
|
296
|
+
return { reason: 'vector engine disabled (injected)' };
|
|
297
|
+
if (opts.engine !== undefined)
|
|
298
|
+
return { engine: opts.engine };
|
|
299
|
+
return resolveVectorEngine(projectRoot);
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Mirror prepared {@link VectorEntry}s into the vector store — **the single write seam** that
|
|
303
|
+
* teach, `teach --from-json`, consolidate, and the backfill all route through (QR-6). The
|
|
304
|
+
* lexical write is ALWAYS already durable before this runs (I-3). Semantics:
|
|
305
|
+
*
|
|
306
|
+
* 1. noise-gate the entries (I-6), merge with the pending queue (dedup by dzId),
|
|
307
|
+
* 2. nothing to do ⇒ `{mirrored:0}` with NO error and no queue file,
|
|
308
|
+
* 3. engine absent ⇒ park the batch in the queue + honest reason (heals on the next consolidate),
|
|
309
|
+
* 4. dedup against `engine.listIds()` (I-5 idempotency — a re-mirror adds 0 rows),
|
|
310
|
+
* 5. time-bounded `engine.upsert` (NC1); failure/timeout ⇒ queue + `sessions.jsonl` note.
|
|
311
|
+
*
|
|
312
|
+
* NEVER throws; the caller's exit code is unaffected by any outcome here (I-1).
|
|
313
|
+
*/
|
|
314
|
+
export async function mirrorEntriesToVector(projectRoot, entries, opts = {}) {
|
|
315
|
+
try {
|
|
316
|
+
let skipped = 0;
|
|
317
|
+
const gated = [];
|
|
318
|
+
for (const e of entries) {
|
|
319
|
+
if (isVectorNoise(e.text))
|
|
320
|
+
skipped += 1;
|
|
321
|
+
else
|
|
322
|
+
gated.push(e);
|
|
323
|
+
}
|
|
324
|
+
const byId = new Map();
|
|
325
|
+
for (const e of [...readVectorPending(projectRoot), ...gated.map(toPending)]) {
|
|
326
|
+
if (!byId.has(e.dzId))
|
|
327
|
+
byId.set(e.dzId, e);
|
|
328
|
+
}
|
|
329
|
+
const batch = [...byId.values()];
|
|
330
|
+
if (batch.length === 0)
|
|
331
|
+
return { mirrored: 0, skipped, queued: 0 };
|
|
332
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
333
|
+
if (resolved.engine === undefined) {
|
|
334
|
+
writeVectorPending(projectRoot, batch);
|
|
335
|
+
const error = resolved.reason ?? 'no vector engine available';
|
|
336
|
+
logMirrorNote(projectRoot, error, batch.length);
|
|
337
|
+
return { mirrored: 0, skipped, queued: batch.length, error };
|
|
338
|
+
}
|
|
339
|
+
const engine = resolved.engine;
|
|
340
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
341
|
+
// I-5 idempotency: skip what the store already holds (best-effort, time-bounded).
|
|
342
|
+
let toSend = batch;
|
|
343
|
+
const listed = await withVectorTimeout(safeEngineCall(() => engine.listIds(), (m) => ({ ids: [], error: `vector listIds failed: ${m}` })), timeoutMs, () => ({ ids: [], error: 'vector listIds timed out' }));
|
|
344
|
+
if (listed.error === undefined) {
|
|
345
|
+
const have = new Set(listed.ids);
|
|
346
|
+
const before = toSend.length;
|
|
347
|
+
toSend = toSend.filter((e) => !have.has(e.dzId));
|
|
348
|
+
skipped += before - toSend.length;
|
|
349
|
+
}
|
|
350
|
+
if (toSend.length === 0) {
|
|
351
|
+
writeVectorPending(projectRoot, []);
|
|
352
|
+
return { mirrored: 0, skipped, queued: 0, engine: engine.kind };
|
|
353
|
+
}
|
|
354
|
+
const up = await withVectorTimeout(safeEngineCall(() => engine.upsert(toSend.map(({ insight: _insight, ...entry }) => entry)), (m) => ({ indexed: 0, error: `vector mirror failed: ${m}` })), timeoutMs, () => ({ indexed: 0, error: `vector mirror timed out after ${timeoutMs}ms (batch queued for the next consolidate)` }));
|
|
355
|
+
if (up.error !== undefined) {
|
|
356
|
+
writeVectorPending(projectRoot, toSend);
|
|
357
|
+
logMirrorNote(projectRoot, up.error, toSend.length);
|
|
358
|
+
return { mirrored: up.indexed, skipped, queued: toSend.length, engine: engine.kind, error: up.error };
|
|
359
|
+
}
|
|
360
|
+
writeVectorPending(projectRoot, []);
|
|
361
|
+
return { mirrored: up.indexed, skipped, queued: 0, engine: engine.kind };
|
|
362
|
+
}
|
|
363
|
+
catch (err) {
|
|
364
|
+
// Belt-and-braces: the mirror must NEVER take the caller down (I-1/I-3).
|
|
365
|
+
return { mirrored: 0, skipped: 0, queued: 0, error: `mirror failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
/** Convenience seam for taught patterns: ACL-map + delegate to {@link mirrorEntriesToVector}. */
|
|
369
|
+
export async function mirrorPatternsToVector(projectRoot, patterns, source = 'dz-teach', opts = {}) {
|
|
370
|
+
const entries = [];
|
|
371
|
+
let gatedOut = 0;
|
|
372
|
+
for (const p of patterns) {
|
|
373
|
+
const e = patternVectorEntry(p, source);
|
|
374
|
+
if (e !== undefined)
|
|
375
|
+
entries.push(e);
|
|
376
|
+
else
|
|
377
|
+
gatedOut += 1; // noise never maps (I-6) — reported honestly as skipped
|
|
378
|
+
}
|
|
379
|
+
const receipt = await mirrorEntriesToVector(projectRoot, entries, opts);
|
|
380
|
+
return gatedOut === 0 ? receipt : { ...receipt, skipped: receipt.skipped + gatedOut };
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Eventual consistency (FR-2): diff `lexical dzIds ∖ engine.listIds()` and mirror the missing
|
|
384
|
+
* set (bounded batch) + drain the pending queue. Run by `dz consolidate` after the watermark
|
|
385
|
+
* write, so a teach-time mirror failure heals on the next consolidate (AC-3). Engine absent ⇒
|
|
386
|
+
* silent no-op (the absent tier is a state, not an error).
|
|
387
|
+
*/
|
|
388
|
+
export async function backfillVectorMirror(projectRoot, opts = {}) {
|
|
389
|
+
try {
|
|
390
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
391
|
+
if (resolved.engine === undefined) {
|
|
392
|
+
return { mirrored: 0, skipped: 0, queued: readVectorPending(projectRoot).length };
|
|
393
|
+
}
|
|
394
|
+
const engine = resolved.engine;
|
|
395
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
396
|
+
const listed = await withVectorTimeout(safeEngineCall(() => engine.listIds(), (m) => ({ ids: [], error: `vector listIds failed: ${m}` })), timeoutMs, () => ({ ids: [], error: 'vector listIds timed out' }));
|
|
397
|
+
if (listed.error !== undefined) {
|
|
398
|
+
return { mirrored: 0, skipped: 0, queued: readVectorPending(projectRoot).length, engine: engine.kind, error: listed.error };
|
|
399
|
+
}
|
|
400
|
+
const have = new Set(listed.ids);
|
|
401
|
+
const limit = opts.batchLimit ?? 200;
|
|
402
|
+
const missing = [];
|
|
403
|
+
for (const r of loadStoreRecords(projectRoot)) {
|
|
404
|
+
if (have.has(r.id))
|
|
405
|
+
continue;
|
|
406
|
+
const e = memoryRecordVectorEntry(r);
|
|
407
|
+
if (e === undefined)
|
|
408
|
+
continue;
|
|
409
|
+
missing.push(e);
|
|
410
|
+
if (missing.length >= limit)
|
|
411
|
+
break;
|
|
412
|
+
}
|
|
413
|
+
// The seam drains the pending queue too (it merges + dedups internally).
|
|
414
|
+
return mirrorEntriesToVector(projectRoot, missing, { engine, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}) });
|
|
415
|
+
}
|
|
416
|
+
catch (err) {
|
|
417
|
+
return { mirrored: 0, skipped: 0, queued: 0, error: `backfill failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
const RRF_K = 60;
|
|
421
|
+
/**
|
|
422
|
+
* Reciprocal Rank Fusion merge: `score(p) = Σ 1/(60 + rank)` over the lists containing `p`
|
|
423
|
+
* (semantic ranks weighted by `semanticWeight`). Dedup by id; `backend: 'both'` when a pattern
|
|
424
|
+
* appears in both lists. DETERMINISTIC (AC-6): ties break on id, so fixed inputs always yield
|
|
425
|
+
* the same ordering. Pure — no I/O.
|
|
426
|
+
*/
|
|
427
|
+
export function mergeHybridHits(lexical, semantic, opts) {
|
|
428
|
+
const weight = opts.semanticWeight ?? 1;
|
|
429
|
+
const acc = new Map();
|
|
430
|
+
lexical.forEach((h, rank) => {
|
|
431
|
+
const cur = acc.get(h.id) ?? { pattern: h.pattern, sem: false, score: 0 };
|
|
432
|
+
cur.lex = h.backend;
|
|
433
|
+
cur.score += 1 / (RRF_K + rank + 1);
|
|
434
|
+
acc.set(h.id, cur);
|
|
435
|
+
});
|
|
436
|
+
semantic.forEach((h, rank) => {
|
|
437
|
+
const cur = acc.get(h.id) ?? { pattern: h.pattern, sem: false, score: 0 };
|
|
438
|
+
cur.sem = true;
|
|
439
|
+
cur.score += weight / (RRF_K + rank + 1);
|
|
440
|
+
acc.set(h.id, cur);
|
|
441
|
+
});
|
|
442
|
+
return [...acc.entries()]
|
|
443
|
+
.sort((a, b) => b[1].score - a[1].score || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
|
|
444
|
+
.slice(0, opts.limit)
|
|
445
|
+
.map(([, v]) => ({
|
|
446
|
+
pattern: v.pattern,
|
|
447
|
+
backend: v.lex !== undefined && v.sem ? 'both' : v.lex ?? 'vector',
|
|
448
|
+
score: v.score,
|
|
449
|
+
}));
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Hybrid recall (FR-3): lexical `recallPatterns` FIRST (always, sync, UNCHANGED — AC-5), then a
|
|
453
|
+
* time-bounded semantic leg merged via RRF. Degradation contract (I-1): with no engine — or on
|
|
454
|
+
* any engine error/timeout — the returned hits are CONTENT-IDENTICAL to plain `recallPatterns`
|
|
455
|
+
* output, with the honest `vectorReason`/`vectorError` alongside. A vector hit whose dzId no
|
|
456
|
+
* longer resolves in the lexical store is DROPPED (V-1 — pruned patterns never resurrect, QR-4).
|
|
457
|
+
*/
|
|
458
|
+
export async function recallHybrid(projectRoot, query, opts = {}) {
|
|
459
|
+
// Config-surface note (QE P3, benign by design): recall resolves the engine directly, while teach
|
|
460
|
+
// only mirrors when the memory backend is agentdb (or an engine is explicit). In the window where
|
|
461
|
+
// the agentdb deps are INSTALLED but `memory.backend` hasn't been switched, the semantic leg reads a
|
|
462
|
+
// store teach never populated → empty/foreign hits. That degrades honestly (orphan dzIds are dropped
|
|
463
|
+
// against the lexical store, V-1) and lexical results are always returned, so it never misleads — it
|
|
464
|
+
// only spends a bounded, cached read. Not gated on purpose: a read-only recall must not depend on the
|
|
465
|
+
// write-side backend flag.
|
|
466
|
+
const limit = opts.limit ?? 10;
|
|
467
|
+
const mode = opts.mode ?? 'hybrid';
|
|
468
|
+
const lexical = recallPatterns(projectRoot, query, limit);
|
|
469
|
+
const lexicalBackend = lexical[0]?.backend === 'sqlite' ? 'sqlite' : 'json';
|
|
470
|
+
const lexicalOnly = (extra) => ({
|
|
471
|
+
hits: lexical.map((h, rank) => ({ pattern: h.pattern, backend: h.backend, score: 1 / (RRF_K + rank + 1) })),
|
|
472
|
+
lexicalBackend,
|
|
473
|
+
vectorEngine: 'none',
|
|
474
|
+
...extra,
|
|
475
|
+
});
|
|
476
|
+
if (mode === 'lexical')
|
|
477
|
+
return lexicalOnly({});
|
|
478
|
+
let resolved;
|
|
479
|
+
try {
|
|
480
|
+
resolved = pickEngine(projectRoot, opts);
|
|
481
|
+
}
|
|
482
|
+
catch (err) {
|
|
483
|
+
return lexicalOnly({ vectorReason: err instanceof Error ? err.message : String(err) });
|
|
484
|
+
}
|
|
485
|
+
if (resolved.engine === undefined) {
|
|
486
|
+
return lexicalOnly(resolved.reason !== undefined ? { vectorReason: resolved.reason } : {});
|
|
487
|
+
}
|
|
488
|
+
const engine = resolved.engine;
|
|
489
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
490
|
+
const sr = await withVectorTimeout(safeEngineCall(() => engine.search(query, limit * 2), (m) => ({ hits: [], error: `vector search failed: ${m}` })), timeoutMs, () => ({ hits: [], error: `vector search timed out after ${timeoutMs}ms` }));
|
|
491
|
+
if (sr.error !== undefined) {
|
|
492
|
+
return { ...lexicalOnly({}), vectorEngine: engine.kind, vectorError: sr.error };
|
|
493
|
+
}
|
|
494
|
+
// Resolve dzId → the FULL lexical record (source of truth). Orphans are dropped (V-1/QR-4).
|
|
495
|
+
let records;
|
|
496
|
+
try {
|
|
497
|
+
records = loadStoreRecords(projectRoot);
|
|
498
|
+
}
|
|
499
|
+
catch {
|
|
500
|
+
records = [];
|
|
501
|
+
}
|
|
502
|
+
const idToPattern = new Map();
|
|
503
|
+
const identityToId = new Map();
|
|
504
|
+
for (const r of records) {
|
|
505
|
+
const p = recordToPattern(r);
|
|
506
|
+
idToPattern.set(r.id, p);
|
|
507
|
+
identityToId.set(patternIdentityOf(p), r.id);
|
|
508
|
+
}
|
|
509
|
+
const semantic = [];
|
|
510
|
+
const seen = new Set();
|
|
511
|
+
for (const h of sr.hits) {
|
|
512
|
+
if (seen.has(h.dzId))
|
|
513
|
+
continue;
|
|
514
|
+
const p = idToPattern.get(h.dzId);
|
|
515
|
+
if (p === undefined)
|
|
516
|
+
continue; // vector-only orphan — the store pruned/expired it; NEVER resurrect
|
|
517
|
+
seen.add(h.dzId);
|
|
518
|
+
semantic.push({ id: h.dzId, pattern: p, backend: 'vector' });
|
|
519
|
+
}
|
|
520
|
+
const lex = lexical.map((h) => ({
|
|
521
|
+
id: identityToId.get(patternIdentityOf(h.pattern)) ?? patternRecordId(h.pattern),
|
|
522
|
+
pattern: h.pattern,
|
|
523
|
+
backend: h.backend,
|
|
524
|
+
}));
|
|
525
|
+
const hits = mergeHybridHits(lex, semantic, { limit, semanticWeight: mode === 'semantic' ? 2 : 1 });
|
|
526
|
+
return { hits, lexicalBackend, vectorEngine: engine.kind };
|
|
527
|
+
}
|
|
528
|
+
/* ------------------------------------------------------------------ */
|
|
529
|
+
/* Status (dz vector status / dz doctor divergence line) */
|
|
530
|
+
/* ------------------------------------------------------------------ */
|
|
531
|
+
/** Field observability: engine availability + mirrored-vs-lexical counts + queue size. */
|
|
532
|
+
export async function vectorTierStatus(projectRoot, opts = {}) {
|
|
533
|
+
const mode = readVectorEngineMode(projectRoot);
|
|
534
|
+
let records;
|
|
535
|
+
try {
|
|
536
|
+
records = loadStoreRecords(projectRoot);
|
|
537
|
+
}
|
|
538
|
+
catch {
|
|
539
|
+
records = [];
|
|
540
|
+
}
|
|
541
|
+
const lexicalMirrorable = records.filter((r) => !isVectorNoise(r.text)).length;
|
|
542
|
+
const pending = readVectorPending(projectRoot).length;
|
|
543
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
544
|
+
if (resolved.engine === undefined) {
|
|
545
|
+
return {
|
|
546
|
+
mode,
|
|
547
|
+
available: false,
|
|
548
|
+
...(resolved.reason !== undefined ? { reason: resolved.reason } : {}),
|
|
549
|
+
lexicalTotal: records.length,
|
|
550
|
+
lexicalMirrorable,
|
|
551
|
+
pending,
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
const engine = resolved.engine;
|
|
555
|
+
const listed = await withVectorTimeout(safeEngineCall(() => engine.listIds(), (m) => ({ ids: [], error: `vector listIds failed: ${m}` })), opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS, () => ({ ids: [], error: 'vector listIds timed out' }));
|
|
556
|
+
return {
|
|
557
|
+
mode,
|
|
558
|
+
kind: engine.kind,
|
|
559
|
+
available: true,
|
|
560
|
+
...(listed.error !== undefined ? { reason: listed.error } : {}),
|
|
561
|
+
lexicalTotal: records.length,
|
|
562
|
+
lexicalMirrorable,
|
|
563
|
+
mirrored: listed.error === undefined ? listed.ids.length : undefined,
|
|
564
|
+
pending,
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
/* ------------------------------------------------------------------ */
|
|
568
|
+
/* Harmonize — SEMANTIC dedup of the lexical store (05 §2.1) */
|
|
569
|
+
/* ------------------------------------------------------------------ */
|
|
570
|
+
/** Bounded, honest embed of one text — a throw/timeout surfaces as `{ error }`, never propagates. */
|
|
571
|
+
async function boundedEmbed(embed, text, timeoutMs) {
|
|
572
|
+
return withVectorTimeout(safeEngineCall(() => embed(text), (m) => ({ error: m })), timeoutMs, () => ({ error: 'embed timed out' }));
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Deterministic keeper INDEX within a near-dup cluster (a TOTAL order over fixed inputs — NFR-7):
|
|
576
|
+
* (1) highest reward → (2) longer / more-specific text → (3) newer `ts` → (4) `dzId` (stable
|
|
577
|
+
* final tiebreak). Pure — no I/O. The keeper survives; the other members are the drop set.
|
|
578
|
+
*/
|
|
579
|
+
export function selectClusterKeeper(members) {
|
|
580
|
+
let best = 0;
|
|
581
|
+
for (let i = 1; i < members.length; i += 1) {
|
|
582
|
+
if (isBetterKeeper(members[i], members[best]))
|
|
583
|
+
best = i;
|
|
584
|
+
}
|
|
585
|
+
return best;
|
|
586
|
+
}
|
|
587
|
+
function isBetterKeeper(a, b) {
|
|
588
|
+
if (a.reward !== b.reward)
|
|
589
|
+
return a.reward > b.reward; // (1) highest reward
|
|
590
|
+
if (a.text.length !== b.text.length)
|
|
591
|
+
return a.text.length > b.text.length; // (2) longer / more specific
|
|
592
|
+
if (a.ts !== b.ts)
|
|
593
|
+
return a.ts > b.ts; // (3) newer
|
|
594
|
+
return a.dzId < b.dzId; // (4) stable, deterministic final tiebreak
|
|
595
|
+
}
|
|
596
|
+
/** Connected components over undirected `edges` (union-find) — transitive clusters (A~B,B~C ⇒ {A,B,C}). */
|
|
597
|
+
function connectedComponents(n, edges) {
|
|
598
|
+
const parent = Array.from({ length: n }, (_, i) => i);
|
|
599
|
+
const find = (x) => {
|
|
600
|
+
let r = x;
|
|
601
|
+
while (parent[r] !== r)
|
|
602
|
+
r = parent[r];
|
|
603
|
+
while (parent[x] !== r) {
|
|
604
|
+
const next = parent[x];
|
|
605
|
+
parent[x] = r;
|
|
606
|
+
x = next;
|
|
607
|
+
}
|
|
608
|
+
return r;
|
|
609
|
+
};
|
|
610
|
+
for (const [a, b] of edges) {
|
|
611
|
+
const ra = find(a);
|
|
612
|
+
const rb = find(b);
|
|
613
|
+
if (ra !== rb)
|
|
614
|
+
parent[ra] = rb;
|
|
615
|
+
}
|
|
616
|
+
const groups = new Map();
|
|
617
|
+
for (let i = 0; i < n; i += 1) {
|
|
618
|
+
const r = find(i);
|
|
619
|
+
const g = groups.get(r);
|
|
620
|
+
if (g === undefined)
|
|
621
|
+
groups.set(r, [i]);
|
|
622
|
+
else
|
|
623
|
+
g.push(i);
|
|
624
|
+
}
|
|
625
|
+
return [...groups.values()];
|
|
626
|
+
}
|
|
627
|
+
/** Build a {@link HarmonizeCluster} from a component's item indices + keeper's cosine to each drop. */
|
|
628
|
+
function buildCluster(items, indices, cosToKeeper) {
|
|
629
|
+
const members = indices.map((i) => items[i]);
|
|
630
|
+
const keeperIdx = indices[selectClusterKeeper(members)];
|
|
631
|
+
const keeper = items[keeperIdx];
|
|
632
|
+
const drops = indices
|
|
633
|
+
.filter((i) => i !== keeperIdx)
|
|
634
|
+
.map((i) => ({ dzId: items[i].dzId, text: items[i].text, reward: items[i].reward, cos: cosToKeeper(i, keeperIdx) }));
|
|
635
|
+
return { keep: { dzId: keeper.dzId, text: keeper.text, reward: keeper.reward, ts: keeper.ts }, drops };
|
|
636
|
+
}
|
|
637
|
+
/** Semantic clusters: pairwise cosine ≥ θ (i<j) ⇒ union-find edge; components of size ≥ 2 are clusters. */
|
|
638
|
+
function semanticClusters(items, vecs, threshold) {
|
|
639
|
+
const n = items.length;
|
|
640
|
+
const edges = [];
|
|
641
|
+
for (let i = 0; i < n; i += 1) {
|
|
642
|
+
for (let j = i + 1; j < n; j += 1) {
|
|
643
|
+
if (cosineSimilarity(vecs[i], vecs[j]) >= threshold)
|
|
644
|
+
edges.push([i, j]);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
const clusters = [];
|
|
648
|
+
for (const comp of connectedComponents(n, edges)) {
|
|
649
|
+
if (comp.length < 2)
|
|
650
|
+
continue;
|
|
651
|
+
clusters.push(buildCluster(items, comp, (d, k) => cosineSimilarity(vecs[d], vecs[k])));
|
|
652
|
+
}
|
|
653
|
+
return clusters;
|
|
654
|
+
}
|
|
655
|
+
/** Exact-text fallback: group by RAW pattern text (the identity `teach --from-json` dedups on); cos = 1.0. */
|
|
656
|
+
function exactClusters(items) {
|
|
657
|
+
const byText = new Map();
|
|
658
|
+
items.forEach((it, i) => {
|
|
659
|
+
const g = byText.get(it.text);
|
|
660
|
+
if (g === undefined)
|
|
661
|
+
byText.set(it.text, [i]);
|
|
662
|
+
else
|
|
663
|
+
g.push(i);
|
|
664
|
+
});
|
|
665
|
+
const clusters = [];
|
|
666
|
+
for (const indices of byText.values()) {
|
|
667
|
+
if (indices.length < 2)
|
|
668
|
+
continue;
|
|
669
|
+
clusters.push(buildCluster(items, indices, () => 1.0));
|
|
670
|
+
}
|
|
671
|
+
return clusters;
|
|
672
|
+
}
|
|
673
|
+
/** Honest note next to the session telemetry (mirrors {@link logMirrorNote}). */
|
|
674
|
+
function logHarmonizeNote(projectRoot, info) {
|
|
675
|
+
try {
|
|
676
|
+
appendFileSync(join(projectRoot, '.dz', 'sessions.jsonl'), JSON.stringify({ event: 'harmonize', ts: new Date().toISOString(), ...info }) + '\n');
|
|
677
|
+
}
|
|
678
|
+
catch { /* best-effort */ }
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* SEMANTIC dedup of the learned-pattern store (`dz vector harmonize` / `dz teach --harmonize`) —
|
|
682
|
+
* **NON-DESTRUCTIVE by contract**. Finds near-duplicate PAIRS via pairwise cosine over the embedder
|
|
683
|
+
* both adapters share (θ default {@link DEFAULT_HARMONIZE_THRESHOLD}), union-finds them into clusters,
|
|
684
|
+
* and within each cluster KEEPs the highest-signal member ({@link selectClusterKeeper}), dropping the
|
|
685
|
+
* rest. Modes:
|
|
686
|
+
*
|
|
687
|
+
* - **dry-run (default)**: previews the clusters and returns — writes NOTHING (the store is
|
|
688
|
+
* byte-identical after).
|
|
689
|
+
* - **`--apply`**: writes a restorable backup FIRST (`.dz/memory/patterns.pre-harmonize.json`); a
|
|
690
|
+
* failed backup ABORTS the drop (no partial mutation). Then removes the non-keepers from BOTH
|
|
691
|
+
* lexical tiers via {@link removePatternsByIds}. A UNIQUE (singleton) pattern is NEVER a drop.
|
|
692
|
+
*
|
|
693
|
+
* Degrades honestly: with no engine/embedder it falls back to EXACT-text dedup + a `fellBackToExact`
|
|
694
|
+
* note, exits without throwing (dry-run still writes nothing). Reversal: `dz teach --from-json <backup>`.
|
|
695
|
+
*/
|
|
696
|
+
export async function harmonizeVectorStore(projectRoot, opts = {}) {
|
|
697
|
+
const apply = opts.apply === true;
|
|
698
|
+
const threshold = opts.threshold !== undefined && opts.threshold > 0 && opts.threshold <= 1
|
|
699
|
+
? opts.threshold
|
|
700
|
+
: readHarmonizeThreshold(projectRoot);
|
|
701
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
702
|
+
// 1. LOAD the pool from the lexical source of truth (id = dzId).
|
|
703
|
+
let records;
|
|
704
|
+
try {
|
|
705
|
+
records = loadStoreRecords(projectRoot);
|
|
706
|
+
}
|
|
707
|
+
catch {
|
|
708
|
+
records = [];
|
|
709
|
+
}
|
|
710
|
+
const items = records.map((r) => ({
|
|
711
|
+
dzId: r.id,
|
|
712
|
+
text: r.text,
|
|
713
|
+
reward: r.score,
|
|
714
|
+
ts: r.timestamp,
|
|
715
|
+
taskType: r.id.startsWith('dream:') ? 'dz-learning' : 'dz-teach',
|
|
716
|
+
}));
|
|
717
|
+
// 2. GATE: an embedder ⇒ SEMANTIC clustering; absence/failure ⇒ EXACT-text fallback (D4).
|
|
718
|
+
let embed;
|
|
719
|
+
let engineKind = 'none';
|
|
720
|
+
let fellBackToExact = false;
|
|
721
|
+
if (opts.embed === null) {
|
|
722
|
+
fellBackToExact = true;
|
|
723
|
+
}
|
|
724
|
+
else if (opts.embed !== undefined) {
|
|
725
|
+
embed = opts.embed;
|
|
726
|
+
engineKind = 'agentdb';
|
|
727
|
+
}
|
|
728
|
+
else {
|
|
729
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
730
|
+
if (resolved.engine === undefined) {
|
|
731
|
+
fellBackToExact = true;
|
|
732
|
+
}
|
|
733
|
+
else {
|
|
734
|
+
engineKind = resolved.engine.kind;
|
|
735
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
736
|
+
if ('error' in emb)
|
|
737
|
+
fellBackToExact = true;
|
|
738
|
+
else
|
|
739
|
+
embed = (t) => emb.embed(t);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
// 3. CLUSTER (nothing to cluster ⇒ no clusters, everything unique).
|
|
743
|
+
let clusters;
|
|
744
|
+
if (items.length >= 2 && !fellBackToExact && embed !== undefined) {
|
|
745
|
+
const vecs = [];
|
|
746
|
+
let ok = true;
|
|
747
|
+
for (const it of items) {
|
|
748
|
+
const v = await boundedEmbed(embed, `${it.taskType}: ${it.text}`, timeoutMs);
|
|
749
|
+
if (!(v instanceof Float32Array)) {
|
|
750
|
+
ok = false;
|
|
751
|
+
break;
|
|
752
|
+
}
|
|
753
|
+
vecs.push(v);
|
|
754
|
+
}
|
|
755
|
+
if (ok)
|
|
756
|
+
clusters = semanticClusters(items, vecs, threshold);
|
|
757
|
+
else
|
|
758
|
+
fellBackToExact = true; // embed failed/timed out — fall back to exact
|
|
759
|
+
}
|
|
760
|
+
if (clusters === undefined) {
|
|
761
|
+
fellBackToExact = fellBackToExact || embed === undefined;
|
|
762
|
+
clusters = items.length >= 2 ? exactClusters(items) : [];
|
|
763
|
+
}
|
|
764
|
+
// 4. TOTALS (a unique = a singleton; never a member of a drop set).
|
|
765
|
+
const dropDzIds = new Set();
|
|
766
|
+
for (const c of clusters)
|
|
767
|
+
for (const d of c.drops)
|
|
768
|
+
dropDzIds.add(d.dzId);
|
|
769
|
+
const kept = clusters.length;
|
|
770
|
+
const dropped = dropDzIds.size;
|
|
771
|
+
const unique = items.length - kept - dropped;
|
|
772
|
+
const base = {
|
|
773
|
+
mode: apply ? 'apply' : 'dry-run',
|
|
774
|
+
engine: engineKind,
|
|
775
|
+
fellBackToExact,
|
|
776
|
+
threshold,
|
|
777
|
+
clusters,
|
|
778
|
+
kept,
|
|
779
|
+
dropped,
|
|
780
|
+
unique,
|
|
781
|
+
};
|
|
782
|
+
// 5a. DRY-RUN (default): return — ZERO writes (the store is byte-identical after).
|
|
783
|
+
if (!apply)
|
|
784
|
+
return base;
|
|
785
|
+
// 5b. --apply: BACKUP FIRST, then drop the non-keepers. Nothing to drop ⇒ no backup, no mutation.
|
|
786
|
+
if (dropped === 0) {
|
|
787
|
+
logHarmonizeNote(projectRoot, { dropped: 0, kept, engine: engineKind });
|
|
788
|
+
return base;
|
|
789
|
+
}
|
|
790
|
+
const backupPath = join(projectRoot, '.dz', 'memory', 'patterns.pre-harmonize.json');
|
|
791
|
+
const snap = snapshotStore(projectRoot, backupPath);
|
|
792
|
+
if (snap.error !== undefined) {
|
|
793
|
+
// Backup write failed ⇒ ABORT the drop (no partial mutation — the store is untouched).
|
|
794
|
+
return { ...base, error: `backup failed — drop aborted: ${snap.error}` };
|
|
795
|
+
}
|
|
796
|
+
const removal = removePatternsByIds(projectRoot, dropDzIds);
|
|
797
|
+
logHarmonizeNote(projectRoot, { dropped: removal.removed, kept, engine: engineKind, error: removal.error });
|
|
798
|
+
return { ...base, backupPath, ...(removal.error !== undefined ? { error: removal.error } : {}) };
|
|
799
|
+
}
|
|
800
|
+
/* ------------------------------------------------------------------ */
|
|
801
|
+
/* Import — RVF checkpoint ingest, UPSERT-BY-dzId (05 §2.2) */
|
|
802
|
+
/* ------------------------------------------------------------------ */
|
|
803
|
+
/**
|
|
804
|
+
* Ingest an external `.rvf` checkpoint's vectors into THIS project's vector store, **UPSERT-BY-dzId,
|
|
805
|
+
* NON-DESTRUCTIVE** (`dz vector import <file.rvf>`). The `.idmap.json` sidecar is the dzId authority
|
|
806
|
+
* (the shipped `@ruvector/rvf` SDK exposes no vector read-out — see rUv `rvf-backend-blocker.md`), so
|
|
807
|
+
* for each checkpoint dzId that exists in the LOCAL lexical store the vector is reproduced by
|
|
808
|
+
* re-embedding the local text (D7 — under the manifest guard the same model over the same text yields
|
|
809
|
+
* the checkpoint's vector) and upserted by dzId via {@link VectorEngine.importVectors}. dzIds absent
|
|
810
|
+
* locally are ORPHANS — skipped + counted (their text must be imported first via `dz teach --from-json`).
|
|
811
|
+
*
|
|
812
|
+
* Non-destructive: only the imported dzIds are inserted/replaced; re-importing the same file adds 0
|
|
813
|
+
* duplicates and deletes nothing. A model/dim manifest mismatch is REFUSED (no cross-space merge). All
|
|
814
|
+
* failure modes return an honest `{ error }`, never a throw.
|
|
815
|
+
*/
|
|
816
|
+
export async function importRvfCheckpoint(projectRoot, source, opts = {}) {
|
|
817
|
+
const fail = (error, engine = 'none') => ({ imported: 0, skippedOrphans: 0, engine, source, error });
|
|
818
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS;
|
|
819
|
+
// 1. Source dzIds: the `.idmap.json` sidecar (dzId authority) — or injected rows (tests).
|
|
820
|
+
const injected = new Map();
|
|
821
|
+
let sourceDzIds;
|
|
822
|
+
if (opts.sourceRows !== undefined) {
|
|
823
|
+
for (const r of opts.sourceRows)
|
|
824
|
+
injected.set(r.dzId, r.vector);
|
|
825
|
+
sourceDzIds = [...injected.keys()];
|
|
826
|
+
}
|
|
827
|
+
else {
|
|
828
|
+
if (!existsSync(source))
|
|
829
|
+
return fail(`no such file: ${source}`);
|
|
830
|
+
const idmapPath = `${source}.idmap.json`;
|
|
831
|
+
if (!existsSync(idmapPath)) {
|
|
832
|
+
return fail(`missing sidecar ${basename(idmapPath)} — export writes it next to the .rvf (re-run: dz vector export)`);
|
|
833
|
+
}
|
|
834
|
+
let idmap;
|
|
835
|
+
try {
|
|
836
|
+
const parsed = JSON.parse(readFileSync(idmapPath, 'utf-8'));
|
|
837
|
+
idmap = typeof parsed === 'object' && parsed !== null && typeof parsed.slots === 'object' ? parsed : { version: 1, slots: {} };
|
|
838
|
+
}
|
|
839
|
+
catch {
|
|
840
|
+
return fail(`unreadable idmap sidecar: ${basename(idmapPath)}`);
|
|
841
|
+
}
|
|
842
|
+
// Manifest guard (R-i1): refuse a foreign embedding model/dim — no silent cross-space merge.
|
|
843
|
+
const manifestPath = `${source}.manifest.json`;
|
|
844
|
+
if (existsSync(manifestPath)) {
|
|
845
|
+
try {
|
|
846
|
+
const m = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
847
|
+
if ((typeof m.model === 'string' && m.model !== LOCAL_EMBED_MODEL) || (typeof m.dim === 'number' && m.dim !== LOCAL_EMBED_DIM)) {
|
|
848
|
+
return fail(`manifest mismatch: checkpoint (${String(m.model)}/${String(m.dim)}) ≠ local (${LOCAL_EMBED_MODEL}/${LOCAL_EMBED_DIM}) — refusing a cross-embedding-space merge`);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
catch { /* unreadable manifest — tolerate; the idmap is the authority */ }
|
|
852
|
+
}
|
|
853
|
+
sourceDzIds = [...new Set(Object.values(idmap.slots))];
|
|
854
|
+
}
|
|
855
|
+
// 2. Resolve the TARGET engine (agentdb default, or rvf if configured).
|
|
856
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
857
|
+
if (resolved.engine === undefined)
|
|
858
|
+
return fail(resolved.reason ?? 'no vector engine available');
|
|
859
|
+
const engine = resolved.engine;
|
|
860
|
+
if (engine.importVectors === undefined) {
|
|
861
|
+
return { imported: 0, skippedOrphans: 0, engine: engine.kind, source, error: `the ${engine.kind} engine cannot import precomputed vectors` };
|
|
862
|
+
}
|
|
863
|
+
// 3. ORPHAN GATE against the lexical source of truth.
|
|
864
|
+
let records;
|
|
865
|
+
try {
|
|
866
|
+
records = loadStoreRecords(projectRoot);
|
|
867
|
+
}
|
|
868
|
+
catch {
|
|
869
|
+
records = [];
|
|
870
|
+
}
|
|
871
|
+
const byId = new Map();
|
|
872
|
+
for (const r of records)
|
|
873
|
+
byId.set(r.id, r);
|
|
874
|
+
let skippedOrphans = 0;
|
|
875
|
+
const kept = [];
|
|
876
|
+
for (const dzId of sourceDzIds) {
|
|
877
|
+
const rec = byId.get(dzId);
|
|
878
|
+
if (rec === undefined)
|
|
879
|
+
skippedOrphans += 1;
|
|
880
|
+
else
|
|
881
|
+
kept.push({ dzId, rec });
|
|
882
|
+
}
|
|
883
|
+
if (kept.length === 0)
|
|
884
|
+
return { imported: 0, skippedOrphans, engine: engine.kind, source };
|
|
885
|
+
// 4. VECTOR per kept dzId: injected verbatim vector, else RE-EMBED the local text (D7).
|
|
886
|
+
let embed = opts.embed;
|
|
887
|
+
if (embed === undefined && kept.some((k) => !injected.has(k.dzId))) {
|
|
888
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
889
|
+
if ('error' in emb)
|
|
890
|
+
return { imported: 0, skippedOrphans, engine: engine.kind, source, error: emb.error };
|
|
891
|
+
embed = (t) => emb.embed(t);
|
|
892
|
+
}
|
|
893
|
+
const rows = [];
|
|
894
|
+
for (const { dzId, rec } of kept) {
|
|
895
|
+
const taskType = dzId.startsWith('dream:') ? 'dz-learning' : 'dz-teach';
|
|
896
|
+
let vector = injected.get(dzId);
|
|
897
|
+
if (vector === undefined) {
|
|
898
|
+
const v = await boundedEmbed(embed, `${taskType}: ${rec.text}`, timeoutMs);
|
|
899
|
+
if (!(v instanceof Float32Array)) {
|
|
900
|
+
return { imported: 0, skippedOrphans, engine: engine.kind, source, error: `embed failed: ${v.error}` };
|
|
901
|
+
}
|
|
902
|
+
vector = v;
|
|
903
|
+
}
|
|
904
|
+
rows.push({ dzId, vector, text: rec.text, taskType, score: rec.score, metadata: { dzId } });
|
|
905
|
+
}
|
|
906
|
+
// 5. UPSERT-BY-dzId (re-import of the same dzIds REPLACEs in place — 0 new rows, nothing deleted).
|
|
907
|
+
const up = await engine.importVectors(rows);
|
|
908
|
+
if (up.error !== undefined)
|
|
909
|
+
return { imported: up.imported, skippedOrphans, engine: engine.kind, source, error: up.error };
|
|
910
|
+
return { imported: up.imported, skippedOrphans, engine: engine.kind, source };
|
|
911
|
+
}
|
|
912
|
+
/* ------------------------------------------------------------------ */
|
|
913
|
+
/* Adapter A (default): AgentdbVectorEngine */
|
|
914
|
+
/* ------------------------------------------------------------------ */
|
|
915
|
+
/**
|
|
916
|
+
* Option A: the `.dz/agentdb.db` ReasoningBank store. `upsert` delegates to the very same
|
|
917
|
+
* {@link indexPatternsToAgentdb} rows the consolidate Option-C mirror writes today (schema
|
|
918
|
+
* unchanged — the `agentdb-memory` MCP skill keeps reading them, NFR-7); `search`/`listIds`
|
|
919
|
+
* are the new READONLY halves in `agentdb-index.ts`.
|
|
920
|
+
*/
|
|
921
|
+
function agentdbVectorEngine(projectRoot) {
|
|
922
|
+
return {
|
|
923
|
+
kind: 'agentdb',
|
|
924
|
+
async upsert(entries) {
|
|
925
|
+
const r = await indexPatternsToAgentdb(projectRoot, entries.map((e) => ({
|
|
926
|
+
taskType: e.taskType,
|
|
927
|
+
text: e.text,
|
|
928
|
+
score: e.score,
|
|
929
|
+
...(e.tags !== undefined ? { tags: e.tags } : {}),
|
|
930
|
+
...(e.metadata !== undefined ? { metadata: e.metadata } : {}),
|
|
931
|
+
})));
|
|
932
|
+
return { indexed: r.indexed, ...(r.error !== undefined ? { error: r.error } : {}) };
|
|
933
|
+
},
|
|
934
|
+
async search(query, limit) {
|
|
935
|
+
const r = await searchAgentdbPatterns(projectRoot, query, { limit });
|
|
936
|
+
const hits = [];
|
|
937
|
+
for (const h of r.hits) {
|
|
938
|
+
if (h.dzId !== undefined)
|
|
939
|
+
hits.push({ dzId: h.dzId, similarity: h.similarity, text: h.text });
|
|
940
|
+
}
|
|
941
|
+
return { hits, ...(r.error !== undefined ? { error: r.error } : {}) };
|
|
942
|
+
},
|
|
943
|
+
async listIds() {
|
|
944
|
+
return listAgentdbDzIds(projectRoot);
|
|
945
|
+
},
|
|
946
|
+
async importVectors(rows) {
|
|
947
|
+
return importVectorsToAgentdb(projectRoot, rows.map((r) => ({
|
|
948
|
+
dzId: r.dzId,
|
|
949
|
+
vector: r.vector,
|
|
950
|
+
text: r.text,
|
|
951
|
+
taskType: r.taskType,
|
|
952
|
+
score: r.score,
|
|
953
|
+
...(r.metadata !== undefined ? { metadata: r.metadata } : {}),
|
|
954
|
+
})));
|
|
955
|
+
},
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
function rvfBase(projectRoot) {
|
|
959
|
+
return join(projectRoot, '.dz', 'memory', 'patterns.rvf');
|
|
960
|
+
}
|
|
961
|
+
function readRvfIdmap(projectRoot) {
|
|
962
|
+
try {
|
|
963
|
+
const parsed = JSON.parse(readFileSync(`${rvfBase(projectRoot)}.idmap.json`, 'utf-8'));
|
|
964
|
+
return typeof parsed === 'object' && parsed !== null && typeof parsed.slots === 'object' ? parsed : { version: 1, slots: {} };
|
|
965
|
+
}
|
|
966
|
+
catch {
|
|
967
|
+
return { version: 1, slots: {} };
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
function writeRvfSidecars(projectRoot, idmap) {
|
|
971
|
+
const base = rvfBase(projectRoot);
|
|
972
|
+
mkdirSync(dirname(base), { recursive: true });
|
|
973
|
+
writeFileSync(`${base}.idmap.json`, JSON.stringify(idmap, null, 2));
|
|
974
|
+
writeFileSync(`${base}.manifest.json`, JSON.stringify({ model: 'Xenova/all-MiniLM-L6-v2', dim: 384, engine: '@ruvector/rvf', version: 1 }, null, 2));
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* Open a `@ruvector/rvf` store, pinned to the REAL published SDK surface (grounded in
|
|
978
|
+
* ruvector/npm/packages/rvf/src/index.ts + a live linux-x64 smoke against @ruvector/rvf@0.2.3):
|
|
979
|
+
* the canonical class is `RvfDatabase` with `create(path, { dimensions })` → `ingestBatch([{id,
|
|
980
|
+
* vector}])` → `query(vector, k)` returning `[{ id, distance }]` → `close()`. A few tolerant
|
|
981
|
+
* fallbacks (add/insert, search) keep older/alt shapes working; anything unrecognized returns an
|
|
982
|
+
* HONEST error (the D8 no-go evidence), never a throw. NOTE: RVF stores the vector under the `id`
|
|
983
|
+
* we pass (= the dzId), so no slot↔id mapping is needed — the query result's `id` IS the dzId.
|
|
984
|
+
*/
|
|
985
|
+
export async function openRvfStore(mod, path, dimensions) {
|
|
986
|
+
try {
|
|
987
|
+
const dflt = mod['default'];
|
|
988
|
+
const cls = (mod['RvfDatabase'] ?? dflt?.['RvfDatabase'] ?? mod['RvfStore'] ?? mod['Store'] ?? dflt?.['RvfStore'] ?? dflt ?? mod);
|
|
989
|
+
let db;
|
|
990
|
+
if (typeof cls.create === 'function')
|
|
991
|
+
db = (await cls.create(path, { dimensions, dimension: dimensions }));
|
|
992
|
+
else if (typeof cls.open === 'function')
|
|
993
|
+
db = (await cls.open(path, { dimensions, dimension: dimensions }));
|
|
994
|
+
else if (typeof cls === 'function')
|
|
995
|
+
db = new cls(path, { dimensions });
|
|
996
|
+
if (db === undefined)
|
|
997
|
+
return { error: 'unsupported @ruvector/rvf API (no RvfDatabase.create/open/constructor) — record a D8 no-go' };
|
|
998
|
+
const ingestBatch = (db['ingestBatch'] ?? db['ingest'] ?? db['add'] ?? db['insert']);
|
|
999
|
+
const query = (db['query'] ?? db['search']);
|
|
1000
|
+
if (typeof ingestBatch !== 'function' || typeof query !== 'function') {
|
|
1001
|
+
return { error: 'unsupported @ruvector/rvf store surface (no ingestBatch/ingest + query/search) — record a D8 no-go' };
|
|
1002
|
+
}
|
|
1003
|
+
const close = db['close'];
|
|
1004
|
+
const exp = db['exportCheckpoint'] ?? db['export_checkpoint'] ?? db['checkpoint'];
|
|
1005
|
+
return {
|
|
1006
|
+
ingest: (id, vec) => ingestBatch.call(db, [{ id, vector: vec }]),
|
|
1007
|
+
query: (vec, k) => query.call(db, vec, k),
|
|
1008
|
+
close: typeof close === 'function' ? close.bind(db) : undefined,
|
|
1009
|
+
exportCheckpoint: typeof exp === 'function' ? exp.bind(db) : undefined,
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
catch (err) {
|
|
1013
|
+
return { error: `@ruvector/rvf store open failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
async function loadRvfModule(projectRoot) {
|
|
1017
|
+
try {
|
|
1018
|
+
const req = createRequire(join(projectRoot, 'package.json'));
|
|
1019
|
+
const mod = (await import(pathToFileURL(req.resolve('@ruvector/rvf')).href));
|
|
1020
|
+
return { ok: true, mod };
|
|
1021
|
+
}
|
|
1022
|
+
catch (err) {
|
|
1023
|
+
return { ok: false, error: `@ruvector/rvf failed to load: ${err instanceof Error ? err.message : String(err)}` };
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Option B: the portable single-file VECTOR form (`.rvf`, magic `0x52564653`) with
|
|
1028
|
+
* `.idmap.json` (slot ↔ dzId) and `.manifest.json` (model/dim — Constraint 5 staleness
|
|
1029
|
+
* detection) sidecars. Embeddings come from agentdb's `EmbeddingService` when resolvable —
|
|
1030
|
+
* with NEITHER embedder the engine degrades gracefully with an honest reason (05 §3.6).
|
|
1031
|
+
*/
|
|
1032
|
+
function rvfVectorEngine(projectRoot) {
|
|
1033
|
+
const noEmbedder = 'rvf engine present but no embedder — install agentdb (dz setup --memory agentdb)';
|
|
1034
|
+
return {
|
|
1035
|
+
kind: 'rvf',
|
|
1036
|
+
async upsert(entries) {
|
|
1037
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
1038
|
+
if ('error' in emb)
|
|
1039
|
+
return { indexed: 0, error: noEmbedder };
|
|
1040
|
+
const loaded = await loadRvfModule(projectRoot);
|
|
1041
|
+
if (!loaded.ok)
|
|
1042
|
+
return { indexed: 0, error: loaded.error };
|
|
1043
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
|
|
1044
|
+
if ('error' in store)
|
|
1045
|
+
return { indexed: 0, error: store.error };
|
|
1046
|
+
try {
|
|
1047
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1048
|
+
let indexed = 0;
|
|
1049
|
+
for (const e of entries) {
|
|
1050
|
+
const vec = await emb.embed(`${e.taskType}: ${e.text}`);
|
|
1051
|
+
await store.ingest(e.dzId, vec); // RVF stores the vector UNDER id = dzId (no slot mapping)
|
|
1052
|
+
idmap.slots[e.dzId] = e.dzId; // sidecar keeps the dzId set for listIds/observability
|
|
1053
|
+
indexed += 1;
|
|
1054
|
+
}
|
|
1055
|
+
await store.close?.();
|
|
1056
|
+
writeRvfSidecars(projectRoot, idmap);
|
|
1057
|
+
return { indexed };
|
|
1058
|
+
}
|
|
1059
|
+
catch (err) {
|
|
1060
|
+
return { indexed: 0, error: `rvf upsert failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
1061
|
+
}
|
|
1062
|
+
},
|
|
1063
|
+
async search(query, limit) {
|
|
1064
|
+
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
1065
|
+
if ('error' in emb)
|
|
1066
|
+
return { hits: [], error: noEmbedder };
|
|
1067
|
+
const loaded = await loadRvfModule(projectRoot);
|
|
1068
|
+
if (!loaded.ok)
|
|
1069
|
+
return { hits: [], error: loaded.error };
|
|
1070
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
|
|
1071
|
+
if ('error' in store)
|
|
1072
|
+
return { hits: [], error: store.error };
|
|
1073
|
+
try {
|
|
1074
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1075
|
+
const raw = await store.query(await emb.embed(query), limit);
|
|
1076
|
+
await store.close?.();
|
|
1077
|
+
const hits = [];
|
|
1078
|
+
if (Array.isArray(raw)) {
|
|
1079
|
+
for (const item of raw) {
|
|
1080
|
+
const id = Array.isArray(item) ? item[0] : item['id'] ?? item['slot'] ?? item['label'];
|
|
1081
|
+
const distance = Array.isArray(item) ? item[1] : item['distance'] ?? item['score'] ?? item['similarity'];
|
|
1082
|
+
// RVF returns the id we ingested (= dzId); the sidecar is a safety join for alt shapes.
|
|
1083
|
+
const dzId = idmap.slots[String(id)] ?? (typeof id === 'string' ? id : undefined);
|
|
1084
|
+
// distance: lower = closer → negate so higher = better (RRF ranks by position regardless).
|
|
1085
|
+
if (dzId !== undefined)
|
|
1086
|
+
hits.push({ dzId, similarity: typeof distance === 'number' ? -distance : 0 });
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
return { hits };
|
|
1090
|
+
}
|
|
1091
|
+
catch (err) {
|
|
1092
|
+
return { hits: [], error: `rvf search failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
1093
|
+
}
|
|
1094
|
+
},
|
|
1095
|
+
async listIds() {
|
|
1096
|
+
// Sidecar-only read — no SDK load needed for observability/dedup.
|
|
1097
|
+
return { ids: [...new Set(Object.values(readRvfIdmap(projectRoot).slots))] };
|
|
1098
|
+
},
|
|
1099
|
+
async importVectors(rows) {
|
|
1100
|
+
const loaded = await loadRvfModule(projectRoot);
|
|
1101
|
+
if (!loaded.ok)
|
|
1102
|
+
return { imported: 0, error: loaded.error };
|
|
1103
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
|
|
1104
|
+
if ('error' in store)
|
|
1105
|
+
return { imported: 0, error: store.error };
|
|
1106
|
+
try {
|
|
1107
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1108
|
+
let imported = 0;
|
|
1109
|
+
for (const r of rows) {
|
|
1110
|
+
await store.ingest(r.dzId, r.vector); // RVF ingest is upsert-by-id (id = dzId) — no duplicates
|
|
1111
|
+
idmap.slots[r.dzId] = r.dzId;
|
|
1112
|
+
imported += 1;
|
|
1113
|
+
}
|
|
1114
|
+
await store.close?.();
|
|
1115
|
+
writeRvfSidecars(projectRoot, idmap);
|
|
1116
|
+
return { imported };
|
|
1117
|
+
}
|
|
1118
|
+
catch (err) {
|
|
1119
|
+
return { imported: 0, error: `rvf import failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
1120
|
+
}
|
|
1121
|
+
},
|
|
1122
|
+
async exportCheckpoint(dest) {
|
|
1123
|
+
try {
|
|
1124
|
+
const base = rvfBase(projectRoot);
|
|
1125
|
+
if (!existsSync(base))
|
|
1126
|
+
return { error: `no ${base} yet — teach/consolidate with the rvf engine first` };
|
|
1127
|
+
const loaded = await loadRvfModule(projectRoot);
|
|
1128
|
+
let exported = false;
|
|
1129
|
+
if (loaded.ok) {
|
|
1130
|
+
const store = await openRvfStore(loaded.mod, base, 384);
|
|
1131
|
+
if (!('error' in store) && store.exportCheckpoint !== undefined) {
|
|
1132
|
+
await store.exportCheckpoint(dest);
|
|
1133
|
+
await store.close?.();
|
|
1134
|
+
exported = true;
|
|
1135
|
+
}
|
|
1136
|
+
else if (!('error' in store)) {
|
|
1137
|
+
await store.close?.();
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
if (!exported)
|
|
1141
|
+
copyFileSync(base, dest); // append-only format — a file copy IS a checkpoint
|
|
1142
|
+
for (const sidecar of ['.idmap.json', '.manifest.json']) {
|
|
1143
|
+
if (existsSync(`${base}${sidecar}`))
|
|
1144
|
+
copyFileSync(`${base}${sidecar}`, `${dest}${sidecar}`);
|
|
1145
|
+
}
|
|
1146
|
+
return {};
|
|
1147
|
+
}
|
|
1148
|
+
catch (err) {
|
|
1149
|
+
return { error: `rvf export failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
1150
|
+
}
|
|
1151
|
+
},
|
|
1152
|
+
};
|
|
1153
|
+
}
|
|
1154
|
+
//# sourceMappingURL=vector-tier.js.map
|