@dzhechkov/harness-core 0.3.86 → 0.3.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/brain.ts CHANGED
@@ -17,11 +17,13 @@
17
17
 
18
18
  import { join, dirname } from 'node:path';
19
19
  import { homedir } from 'node:os';
20
- import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
20
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, readdirSync, type Dirent } from 'node:fs';
21
21
  import { pathToFileURL } from 'node:url';
22
22
  import { createRequire } from 'node:module';
23
+ import { spawn } from 'node:child_process';
23
24
  import { putBookKnowledge, queryBookKnowledge, bookKbPath, type BookKU, type BookKUHit } from './book-kb.js';
24
- import { indexPatternsToAgentdb, type AgentdbRow } from './agentdb-index.js';
25
+ import { indexPatternsToAgentdb, searchAgentdbPatterns, reindexAgentdbRows, type AgentdbRow } from './agentdb-index.js';
26
+ import { resolveEmbedModel, type EmbedModelConfig } from './embedding-config.js';
25
27
 
26
28
  // ─────────────────────────────────────── Home & paths ───────────────────────────────────────
27
29
 
@@ -783,6 +785,8 @@ const RERANK_PRIOR_BONUS = 0.5;
783
785
 
784
786
  /** Over-fetch cap for the rerank pass — bounds work while leaving room for a `limit*3` window. */
785
787
  const RERANK_OVERFETCH_CAP = 200;
788
+ const GROUND_VECTOR_TIMEOUT_MS = 1_000;
789
+ const GROUND_VECTOR_SIMILARITY_FLOOR = 0.35;
786
790
 
787
791
  /**
788
792
  * A DETERMINISTIC lexical reranker (ADR-001 §11 P3 / G3) — lifts precision on the top-K without a
@@ -855,6 +859,180 @@ export async function queryBrain(opts: {
855
859
  return { hits: rerankHits(opts.query, res.hits, { limit: effLimit }) };
856
860
  }
857
861
 
862
+ async function bounded<T>(promise: Promise<T>, ms: number, fallback: () => T): Promise<T> {
863
+ let timer: ReturnType<typeof setTimeout> | undefined;
864
+ try {
865
+ return await Promise.race([
866
+ promise.catch(() => fallback()),
867
+ new Promise<T>((resolvePromise) => {
868
+ timer = setTimeout(() => resolvePromise(fallback()), ms);
869
+ timer.unref?.();
870
+ }),
871
+ ]);
872
+ } finally {
873
+ if (timer !== undefined) clearTimeout(timer);
874
+ }
875
+ }
876
+
877
+ const backgroundWarmups = new Set<string>();
878
+
879
+ function isFakeAgentdbForTests(depsRoot: string): boolean {
880
+ try {
881
+ const req = createRequire(join(depsRoot, 'package.json'));
882
+ const pkg = JSON.parse(readFileSync(req.resolve('agentdb/package.json'), 'utf-8')) as { version?: unknown };
883
+ return typeof pkg.version === 'string' && pkg.version.includes('test');
884
+ } catch {
885
+ return false;
886
+ }
887
+ }
888
+
889
+ function hasOnnxFile(dir: string, depth = 0, seen: { count: number } = { count: 0 }): boolean {
890
+ if (depth > 8 || seen.count > 2_000 || !existsSync(dir)) return false;
891
+ let entries: Dirent[];
892
+ try {
893
+ entries = readdirSync(dir, { withFileTypes: true });
894
+ } catch {
895
+ return false;
896
+ }
897
+ for (const entry of entries) {
898
+ seen.count += 1;
899
+ const p = join(dir, entry.name);
900
+ if (entry.isFile() && entry.name.endsWith('.onnx')) return true;
901
+ if (entry.isDirectory() && hasOnnxFile(p, depth + 1, seen)) return true;
902
+ if (seen.count > 2_000) return false;
903
+ }
904
+ return false;
905
+ }
906
+
907
+ function cacheRoots(): string[] {
908
+ const roots: string[] = [];
909
+ const transformers = process.env['TRANSFORMERS_CACHE'];
910
+ if (transformers !== undefined && transformers !== '') roots.push(transformers);
911
+ const hfHome = process.env['HF_HOME'];
912
+ if (hfHome !== undefined && hfHome !== '') roots.push(join(hfHome, 'hub'));
913
+ const xdg = process.env['XDG_CACHE_HOME'];
914
+ roots.push(join(xdg !== undefined && xdg !== '' ? xdg : join(homedir(), '.cache'), 'huggingface', 'hub'));
915
+ return [...new Set(roots)];
916
+ }
917
+
918
+ function isEmbedModelCached(depsRoot: string, model: EmbedModelConfig): boolean {
919
+ if (isFakeAgentdbForTests(depsRoot)) return true;
920
+ const modelDir = `models--${model.model.replace(/\//g, '--')}`;
921
+ for (const root of cacheRoots()) {
922
+ if (hasOnnxFile(join(root, modelDir)) || hasOnnxFile(root.endsWith(modelDir) ? root : join(root, 'hub', modelDir))) return true;
923
+ }
924
+ return false;
925
+ }
926
+
927
+ function warmEmbedModelInBackground(depsRoot: string, model: EmbedModelConfig): void {
928
+ const key = `${depsRoot}\0${model.model}`;
929
+ if (backgroundWarmups.has(key)) return;
930
+ backgroundWarmups.add(key);
931
+ setImmediate(() => {
932
+ const script = `
933
+ import { createRequire } from 'node:module';
934
+ import { dirname, join } from 'node:path';
935
+ import { pathToFileURL } from 'node:url';
936
+ const root = ${JSON.stringify(depsRoot)};
937
+ const model = ${JSON.stringify(model.model)};
938
+ const dim = ${JSON.stringify(model.dim)};
939
+ const req = createRequire(join(root, 'package.json'));
940
+ const agentdbDir = dirname(req.resolve('agentdb'));
941
+ const mod = await import(pathToFileURL(join(agentdbDir, 'controllers', 'EmbeddingService.js')).href);
942
+ const emb = new mod.EmbeddingService({ model, dimension: dim, provider: 'transformers' });
943
+ await emb.initialize();
944
+ await emb.embed('warm embedding model cache');
945
+ `;
946
+ try {
947
+ const nodeBin = existsSync(process.execPath) ? process.execPath : (process.argv[0] ?? 'node');
948
+ const child = spawn(nodeBin, ['--input-type=module', '-e', script], {
949
+ cwd: depsRoot,
950
+ detached: true,
951
+ stdio: ['ignore', 'ignore', 'ignore'],
952
+ env: { ...process.env, DZ_EMBED_MODEL: model.model },
953
+ });
954
+ child.on('error', () => undefined);
955
+ child.unref();
956
+ } catch {
957
+ /* warmup is advisory; the foreground grounding path must never block or throw */
958
+ }
959
+ });
960
+ }
961
+
962
+ export async function searchBrainVectors(opts: {
963
+ query: string;
964
+ brainHome?: string;
965
+ depsRoot?: string;
966
+ source?: string;
967
+ limit?: number;
968
+ timeoutMs?: number;
969
+ }): Promise<{ hits: Array<BookKUHit & { similarity: number }>; error?: string }> {
970
+ const home = opts.brainHome ?? brainHome();
971
+ const depsRoot = opts.depsRoot ?? process.cwd();
972
+ const limit = Math.max(1, opts.limit ?? DEFAULT_QUERY_LIMIT);
973
+ const model = resolveEmbedModel(depsRoot);
974
+ if ('error' in model) return { hits: [], error: model.error };
975
+ if (!isEmbedModelCached(depsRoot, model)) {
976
+ // eslint-disable-next-line no-console
977
+ console.warn(`[dz brain] SKIP vector leg: embedding model cache is cold for ${model.model}; warming in background`);
978
+ warmEmbedModelInBackground(depsRoot, model);
979
+ return { hits: [], error: 'brain vector search skipped: embedding model cache cold' };
980
+ }
981
+ const run = async (): Promise<{ hits: Array<BookKUHit & { similarity: number }>; error?: string }> => {
982
+ const vec = await searchAgentdbPatterns(depsRoot, opts.query, {
983
+ dbPath: brainAgentdbPath(home),
984
+ taskTypes: ['book-knowledge'],
985
+ limit: limit * 3,
986
+ reindexHint: 'dz brain reindex',
987
+ });
988
+ if (vec.error !== undefined) return { hits: [], error: vec.error };
989
+ if (vec.hits.length === 0) return { hits: [] };
990
+ const read = readBookKus({ storePath: brainBooksPath(home), depsRoot, ...(opts.source !== undefined ? { source: opts.source } : {}) });
991
+ if (read.error !== undefined) return { hits: [], error: read.error };
992
+ const byKu = new Map(read.kus.map((ku) => [ku.kuId, ku]));
993
+ const hits: Array<BookKUHit & { similarity: number }> = [];
994
+ const seen = new Set<string>();
995
+ for (const h of vec.hits) {
996
+ if (h.dzId === undefined || seen.has(h.dzId)) continue;
997
+ const ku = byKu.get(h.dzId);
998
+ if (ku === undefined) continue;
999
+ seen.add(h.dzId);
1000
+ hits.push({
1001
+ book: ku.book,
1002
+ kuId: ku.kuId,
1003
+ type: ku.type,
1004
+ name: ku.name,
1005
+ problem: ku.problem,
1006
+ content: ku.content,
1007
+ ...(ku.chapter !== undefined ? { chapter: ku.chapter } : {}),
1008
+ ...(ku.pages !== undefined ? { pages: ku.pages } : {}),
1009
+ similarity: h.similarity,
1010
+ });
1011
+ if (hits.length >= limit) break;
1012
+ }
1013
+ return { hits };
1014
+ };
1015
+ return bounded(run(), opts.timeoutMs ?? GROUND_VECTOR_TIMEOUT_MS, () => ({ hits: [], error: 'brain vector search timed out' }));
1016
+ }
1017
+
1018
+ export async function reindexBrainVectors(opts: {
1019
+ brainHome?: string;
1020
+ depsRoot?: string;
1021
+ }): Promise<{ reembedded: number; model?: string; version?: number; backupPath?: string; error?: string }> {
1022
+ const home = opts.brainHome ?? brainHome();
1023
+ const depsRoot = opts.depsRoot ?? process.cwd();
1024
+ const read = readBookKus({ storePath: brainBooksPath(home), depsRoot });
1025
+ if (read.error !== undefined) return { reembedded: 0, error: read.error };
1026
+ const rows: AgentdbRow[] = read.kus.map((ku) => ({
1027
+ taskType: 'book-knowledge',
1028
+ text: `${ku.name}\n${ku.problem}\n${ku.content}`,
1029
+ score: 1.0,
1030
+ tags: ['book-knowledge', ku.book, ku.type],
1031
+ metadata: { source: 'book-kb', book: ku.book, kuId: ku.kuId, dzId: ku.kuId, corpusVersion: ku.corpusVersion },
1032
+ }));
1033
+ return reindexAgentdbRows(depsRoot, rows, { dbPath: brainAgentdbPath(home), taskTypes: ['book-knowledge'] });
1034
+ }
1035
+
858
1036
  // ──────────────────────────────────────── Grounding ─────────────────────────────────────────
859
1037
 
860
1038
  /**
@@ -951,8 +1129,22 @@ export async function groundPrompt(opts: {
951
1129
  const res = await queryBrain(query);
952
1130
 
953
1131
  // Gate 2: any lexical hit? On error or zero hits, stay silent (no noise, no block).
954
- if (res.error !== undefined) return silent;
955
- if (res.hits.length === 0) return silent;
1132
+ const lexicalHits = res.error === undefined ? res.hits : [];
1133
+ const semantic = await searchBrainVectors({
1134
+ query: terms.join(' '),
1135
+ limit: opts.k ?? 5,
1136
+ ...(opts.brainHome !== undefined ? { brainHome: opts.brainHome } : {}),
1137
+ ...(opts.depsRoot !== undefined ? { depsRoot: opts.depsRoot } : {}),
1138
+ ...(opts.source !== undefined ? { source: opts.source } : {}),
1139
+ timeoutMs: GROUND_VECTOR_TIMEOUT_MS,
1140
+ });
1141
+ const semanticHits = semantic.hits.filter((h) => h.similarity >= GROUND_VECTOR_SIMILARITY_FLOOR);
1142
+ if (lexicalHits.length === 0 && semanticHits.length === 0) return silent;
1143
+
1144
+ const mergedByKu = new Map<string, BookKUHit>();
1145
+ for (const h of lexicalHits) mergedByKu.set(h.kuId, h);
1146
+ for (const h of semanticHits) if (!mergedByKu.has(h.kuId)) mergedByKu.set(h.kuId, h);
1147
+ const merged = rerankHits(terms.join(' '), [...mergedByKu.values()], { limit: opts.k ?? 5 });
956
1148
 
957
1149
  // Gate 3 — COVERAGE (the OR balance): OR recall alone over-fires when a single common term
958
1150
  // ("today", "data") incidentally matches a KU. Require the retrieved hits to actually cover the
@@ -961,20 +1153,20 @@ export async function groundPrompt(opts: {
961
1153
  // off-topic prompt whose only matching word is incidental ("weather today" → only "today"
962
1154
  // matches) stays silent, while a real one ("репликацию single multi leader" → 4 covered) grounds.
963
1155
  if (terms.length >= 2) {
964
- const hay = res.hits.map((h) => `${h.name} ${h.problem} ${h.content}`.toLowerCase()).join(' ');
1156
+ const hay = lexicalHits.map((h) => `${h.name} ${h.problem} ${h.content}`.toLowerCase()).join(' ');
965
1157
  const covered = terms.filter((t) => hay.includes(t.toLowerCase())).length;
966
- if (covered < 2) return silent;
1158
+ if (covered < 2 && semanticHits.length === 0) return silent;
967
1159
  }
968
1160
 
969
1161
  // Build the GROUNDING DIRECTIVE block (§7.1): directive line + numbered citations.
970
1162
  const lines: string[] = [GROUNDING_DIRECTIVE, ''];
971
- res.hits.forEach((h, i) => {
1163
+ merged.forEach((h, i) => {
972
1164
  const ch = h.chapter !== undefined && h.chapter !== '' ? ` гл.${h.chapter}` : '';
973
1165
  const pg = h.pages !== undefined && h.pages.length > 0 ? ` с.${h.pages.join('–')}` : '';
974
1166
  const body = snippet(h.problem !== '' ? h.problem : h.content);
975
1167
  lines.push(`[K${i + 1}] ${h.book}${ch}${pg} — ${h.name}: ${body}`);
976
1168
  });
977
- return { emitted: true, block: lines.join('\n'), hitCount: res.hits.length };
1169
+ return { emitted: true, block: lines.join('\n'), hitCount: merged.length };
978
1170
  } catch {
979
1171
  // Grounding is advisory — a bug here must never surface to the user or block the prompt.
980
1172
  return silent;
@@ -0,0 +1,117 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync } from 'node:fs';
2
+ import { dirname, join } from 'node:path';
3
+
4
+ export type EmbedModelSource = 'env' | 'config' | 'default';
5
+
6
+ export interface EmbedModelConfig {
7
+ readonly model: string;
8
+ readonly dim: 384;
9
+ readonly source: EmbedModelSource;
10
+ }
11
+
12
+ export interface EmbedManifest {
13
+ readonly model: string;
14
+ readonly dim: 384;
15
+ readonly version: number;
16
+ readonly engine?: string;
17
+ }
18
+
19
+ export const DEFAULT_EMBED_MODEL = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2';
20
+ export const LEGACY_EMBED_MODEL = 'Xenova/all-MiniLM-L6-v2';
21
+ export const DEFAULT_EMBED_DIM = 384;
22
+
23
+ export const KNOWN_EMBED_DIMS: Readonly<Record<string, 384>> = {
24
+ [DEFAULT_EMBED_MODEL]: DEFAULT_EMBED_DIM,
25
+ [LEGACY_EMBED_MODEL]: DEFAULT_EMBED_DIM,
26
+ // e5 models require asymmetric "query:" / "passage:" prefixes at call sites to reach full quality.
27
+ 'Xenova/multilingual-e5-small': DEFAULT_EMBED_DIM,
28
+ 'Xenova/paraphrase-multilingual-mpnet-base-v2': DEFAULT_EMBED_DIM,
29
+ };
30
+
31
+ export function resolveEmbedModel(projectRoot: string): EmbedModelConfig | { error: string } {
32
+ const env = process.env['DZ_EMBED_MODEL'];
33
+ if (env !== undefined && env.trim() !== '') return modelConfig(env.trim(), 'env');
34
+ const cfgPath = join(projectRoot, '.dz', 'config.json');
35
+ if (existsSync(cfgPath)) {
36
+ try {
37
+ const cfg = JSON.parse(readFileSync(cfgPath, 'utf-8')) as Record<string, unknown>;
38
+ const memory = cfg['memory'] as Record<string, unknown> | undefined;
39
+ const agentdb = memory?.['agentdb'] as Record<string, unknown> | undefined;
40
+ const embed = memory?.['embed'] as Record<string, unknown> | undefined;
41
+ const configured = agentdb?.['embeddingModel'] ?? embed?.['model'];
42
+ if (typeof configured === 'string' && configured.trim() !== '') return modelConfig(configured.trim(), 'config');
43
+ } catch {
44
+ /* corrupt config falls back to the default, matching the existing config-read discipline */
45
+ }
46
+ }
47
+ return modelConfig(DEFAULT_EMBED_MODEL, 'default');
48
+ }
49
+
50
+ function modelConfig(model: string, source: EmbedModelSource): EmbedModelConfig | { error: string } {
51
+ const dim = KNOWN_EMBED_DIMS[model];
52
+ if (dim === undefined) {
53
+ return { error: `unsupported embedding model '${model}' (known 384-dim models: ${Object.keys(KNOWN_EMBED_DIMS).join(', ')})` };
54
+ }
55
+ return { model, dim, source };
56
+ }
57
+
58
+ export function embedManifestPath(storePath: string): string {
59
+ return `${storePath}.embed-manifest.json`;
60
+ }
61
+
62
+ export function readEmbedManifest(storePath: string): EmbedManifest | undefined {
63
+ return readManifestFile(embedManifestPath(storePath));
64
+ }
65
+
66
+ function readManifestFile(p: string): EmbedManifest | undefined {
67
+ if (!existsSync(p)) return undefined;
68
+ try {
69
+ const m = JSON.parse(readFileSync(p, 'utf-8')) as Partial<EmbedManifest>;
70
+ if (typeof m.model !== 'string' || m.model === '') return undefined;
71
+ if (m.dim !== DEFAULT_EMBED_DIM) return undefined;
72
+ return { model: m.model, dim: DEFAULT_EMBED_DIM, version: typeof m.version === 'number' ? m.version : 1, ...(typeof m.engine === 'string' ? { engine: m.engine } : {}) };
73
+ } catch {
74
+ return undefined;
75
+ }
76
+ }
77
+
78
+ function readStoreManifest(storePath: string): EmbedManifest | undefined {
79
+ return readEmbedManifest(storePath) ?? readManifestFile(`${storePath}.manifest.json`);
80
+ }
81
+
82
+ export function writeEmbedManifest(storePath: string, manifest: EmbedManifest): void {
83
+ const p = embedManifestPath(storePath);
84
+ mkdirSync(dirname(p), { recursive: true });
85
+ writeFileSync(p, `${JSON.stringify(manifest, null, 2)}\n`);
86
+ }
87
+
88
+ export function legacyEmbedManifest(): EmbedManifest {
89
+ return { model: LEGACY_EMBED_MODEL, dim: DEFAULT_EMBED_DIM, version: 1 };
90
+ }
91
+
92
+ export function currentEmbedManifest(configured: EmbedModelConfig, version = 1, engine?: string): EmbedManifest {
93
+ return { model: configured.model, dim: configured.dim, version, ...(engine !== undefined ? { engine } : {}) };
94
+ }
95
+
96
+ export function guardEmbedSpace(args: {
97
+ storePath: string;
98
+ configured: EmbedModelConfig;
99
+ hasRows: boolean;
100
+ reindexHint: string;
101
+ }): { ok: true; manifest: EmbedManifest } | { ok: false; error: string; manifest: EmbedManifest } {
102
+ const manifest = readStoreManifest(args.storePath)
103
+ ?? (args.hasRows ? legacyEmbedManifest() : currentEmbedManifest(args.configured));
104
+ if (manifest.model !== args.configured.model || manifest.dim !== args.configured.dim) {
105
+ return {
106
+ ok: false,
107
+ manifest,
108
+ error: `embedding model mismatch: index built with ${manifest.model}/${manifest.dim}, configured ${args.configured.model}/${args.configured.dim}; run ${args.reindexHint}`,
109
+ };
110
+ }
111
+ return { ok: true, manifest };
112
+ }
113
+
114
+ export function snapshotEmbedManifest(storePath: string, backupPath: string): void {
115
+ const p = embedManifestPath(storePath);
116
+ if (existsSync(p)) copyFileSync(p, backupPath);
117
+ }
package/src/index.ts CHANGED
@@ -47,6 +47,7 @@ export {
47
47
  mergeHybridHits,
48
48
  recallHybrid,
49
49
  vectorTierStatus,
50
+ reindexVectorStore,
50
51
  harmonizeVectorStore,
51
52
  selectClusterKeeper,
52
53
  importRvfCheckpoint,
@@ -72,12 +73,15 @@ export type {
72
73
  HarmonizeOptions,
73
74
  ImportReport,
74
75
  ImportOptions,
76
+ ReindexVectorReport,
75
77
  } from './vector-tier.js';
76
78
  export { runSetup, generateHooksConfig, generateAgentdbWriter, writerVersionOf, AGENTDB_WRITER_VERSION } from './setup.js';
77
79
  export { statuslineData, readFeatureAdrState, writeFeatureAdrState, featureAdrStatePath } from './statusline.js';
78
80
  export type { StatuslineData, FeatureAdrState, WriteFeatureAdrStateInput } from './statusline.js';
79
- export { indexPatternsToAgentdb, resolveAgentdbPath, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb } from './agentdb-index.js';
81
+ export { indexPatternsToAgentdb, resolveAgentdbPath, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb, reindexAgentdbRows } from './agentdb-index.js';
80
82
  export type { AgentdbSearchHit, AgentdbSearchResult, AgentdbImportRow } from './agentdb-index.js';
83
+ export { DEFAULT_EMBED_MODEL, LEGACY_EMBED_MODEL, DEFAULT_EMBED_DIM, KNOWN_EMBED_DIMS, resolveEmbedModel, readEmbedManifest, writeEmbedManifest, embedManifestPath, legacyEmbedManifest } from './embedding-config.js';
84
+ export type { EmbedModelConfig, EmbedModelSource, EmbedManifest } from './embedding-config.js';
81
85
  export { putBookKnowledge, queryBookKnowledge, bookKbPath } from './book-kb.js';
82
86
  export type { BookKU, BookKUHit } from './book-kb.js';
83
87
  export {
@@ -91,6 +95,8 @@ export {
91
95
  promoteProjectToBrain,
92
96
  updateBrainSource,
93
97
  queryBrain,
98
+ searchBrainVectors,
99
+ reindexBrainVectors,
94
100
  rerankHits,
95
101
  groundPrompt,
96
102
  buildPrimer,
package/src/setup.ts CHANGED
@@ -271,7 +271,7 @@ function generateDzConfig(target: string, preset: string | undefined, backend: M
271
271
  mcpServer: 'agentdb',
272
272
  // Both the hook writer and the MCP server read/write THIS path (env AGENTDB_PATH).
273
273
  storePath: '.dz/agentdb.db',
274
- embeddingModel: 'Xenova/all-MiniLM-L6-v2',
274
+ embeddingModel: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2',
275
275
  sessionHookWrites: true,
276
276
  } : undefined,
277
277
  },
@@ -59,7 +59,9 @@ import {
59
59
  resolveAgentdbEmbedder,
60
60
  cosineSimilarity,
61
61
  importVectorsToAgentdb,
62
+ reindexAgentdbRows,
62
63
  } from './agentdb-index.js';
64
+ import { currentEmbedManifest, guardEmbedSpace, DEFAULT_EMBED_DIM, resolveEmbedModel, type EmbedModelConfig } from './embedding-config.js';
63
65
 
64
66
  /* ------------------------------------------------------------------ */
65
67
  /* Types (04_domain_model §3.4 / §4.1) */
@@ -165,6 +167,7 @@ export interface VectorTierStatus {
165
167
  readonly kind?: VectorEngineKind | undefined;
166
168
  readonly available: boolean;
167
169
  readonly reason?: string | undefined;
170
+ readonly embeddingModel?: string | undefined;
168
171
  readonly lexicalTotal: number;
169
172
  readonly lexicalMirrorable: number;
170
173
  readonly mirrored?: number | undefined;
@@ -174,10 +177,6 @@ export interface VectorTierStatus {
174
177
  /** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
175
178
  export const DEFAULT_VECTOR_TIMEOUT_MS = 10_000;
176
179
 
177
- /** The pinned local embedding model + dimension (agentdb's `EmbeddingService`; the RVF manifest space). */
178
- const LOCAL_EMBED_MODEL = 'Xenova/all-MiniLM-L6-v2';
179
- const LOCAL_EMBED_DIM = 384;
180
-
181
180
  /** Default cosine cutoff for near-duplicate clustering (`--threshold` / config overrides). */
182
181
  export const DEFAULT_HARMONIZE_THRESHOLD = 0.92;
183
182
 
@@ -255,6 +254,14 @@ export interface ImportOptions extends VectorServiceOptions {
255
254
  readonly embed?: ((text: string) => Promise<Float32Array>) | undefined;
256
255
  }
257
256
 
257
+ export interface ReindexVectorReport {
258
+ readonly reembedded: number;
259
+ readonly model?: string;
260
+ readonly version?: number;
261
+ readonly backupPath?: string;
262
+ readonly error?: string;
263
+ }
264
+
258
265
  /* ------------------------------------------------------------------ */
259
266
  /* Timeout wrapper (both legs — NC1/QR-1) */
260
267
  /* ------------------------------------------------------------------ */
@@ -817,6 +824,7 @@ export async function vectorTierStatus(
817
824
  opts: VectorServiceOptions = {},
818
825
  ): Promise<VectorTierStatus> {
819
826
  const mode = readVectorEngineMode(projectRoot);
827
+ const model = resolveEmbedModel(projectRoot);
820
828
  let records: MemoryRecord[];
821
829
  try {
822
830
  records = loadStoreRecords(projectRoot);
@@ -831,6 +839,7 @@ export async function vectorTierStatus(
831
839
  mode,
832
840
  available: false,
833
841
  ...(resolved.reason !== undefined ? { reason: resolved.reason } : {}),
842
+ ...(!('error' in model) ? { embeddingModel: model.model } : {}),
834
843
  lexicalTotal: records.length,
835
844
  lexicalMirrorable,
836
845
  pending,
@@ -847,6 +856,7 @@ export async function vectorTierStatus(
847
856
  kind: engine.kind,
848
857
  available: true,
849
858
  ...(listed.error !== undefined ? { reason: listed.error } : {}),
859
+ ...(!('error' in model) ? { embeddingModel: model.model } : {}),
850
860
  lexicalTotal: records.length,
851
861
  lexicalMirrorable,
852
862
  mirrored: listed.error === undefined ? listed.ids.length : undefined,
@@ -854,6 +864,31 @@ export async function vectorTierStatus(
854
864
  };
855
865
  }
856
866
 
867
+ export async function reindexVectorStore(projectRoot: string, opts: VectorServiceOptions = {}): Promise<ReindexVectorReport> {
868
+ const resolved = pickEngine(projectRoot, opts);
869
+ if (resolved.engine === undefined) return { reembedded: 0, error: resolved.reason ?? 'no vector engine available' };
870
+ if (resolved.engine.kind !== 'agentdb') {
871
+ return { reembedded: 0, error: 'dz vector reindex currently rewrites the agentdb learned-pattern mirror; switch memory.vector.engine to agentdb/auto' };
872
+ }
873
+ let records: MemoryRecord[];
874
+ try {
875
+ records = loadStoreRecords(projectRoot);
876
+ } catch {
877
+ records = [];
878
+ }
879
+ const rows = records
880
+ .map(memoryRecordVectorEntry)
881
+ .filter((r): r is VectorEntry => r !== undefined)
882
+ .map((r) => ({
883
+ taskType: r.taskType,
884
+ text: r.text,
885
+ score: r.score,
886
+ ...(r.tags !== undefined ? { tags: r.tags } : {}),
887
+ ...(r.metadata !== undefined ? { metadata: r.metadata } : {}),
888
+ }));
889
+ return reindexAgentdbRows(projectRoot, rows);
890
+ }
891
+
857
892
  /* ------------------------------------------------------------------ */
858
893
  /* Harmonize — SEMANTIC dedup of the lexical store (05 §2.1) */
859
894
  /* ------------------------------------------------------------------ */
@@ -1139,8 +1174,15 @@ export async function importRvfCheckpoint(projectRoot: string, source: string, o
1139
1174
  if (existsSync(manifestPath)) {
1140
1175
  try {
1141
1176
  const m = JSON.parse(readFileSync(manifestPath, 'utf-8')) as { model?: unknown; dim?: unknown };
1142
- if ((typeof m.model === 'string' && m.model !== LOCAL_EMBED_MODEL) || (typeof m.dim === 'number' && m.dim !== LOCAL_EMBED_DIM)) {
1143
- return fail(`manifest mismatch: checkpoint (${String(m.model)}/${String(m.dim)}) local (${LOCAL_EMBED_MODEL}/${LOCAL_EMBED_DIM}) — refusing a cross-embedding-space merge`);
1177
+ const configured = resolveEmbedModel(projectRoot);
1178
+ if ('error' in configured) return fail(configured.error);
1179
+ const manifest = {
1180
+ model: typeof m.model === 'string' ? m.model : configured.model,
1181
+ dim: typeof m.dim === 'number' ? m.dim : configured.dim,
1182
+ version: 1,
1183
+ };
1184
+ if (manifest.model !== configured.model || manifest.dim !== configured.dim) {
1185
+ return fail(`manifest mismatch: checkpoint (${String(manifest.model)}/${String(manifest.dim)}) ≠ local (${configured.model}/${configured.dim}) — run dz vector reindex; refusing a cross-embedding-space merge`);
1144
1186
  }
1145
1187
  } catch { /* unreadable manifest — tolerate; the idmap is the authority */ }
1146
1188
  }
@@ -1276,12 +1318,26 @@ function readRvfIdmap(projectRoot: string): RvfIdmap {
1276
1318
  }
1277
1319
  }
1278
1320
 
1279
- function writeRvfSidecars(projectRoot: string, idmap: RvfIdmap): void {
1321
+ function guardRvfEmbedSpace(projectRoot: string, idmap: RvfIdmap): { ok: true; configured: EmbedModelConfig; version: number } | { ok: false; error: string } {
1322
+ const configured = resolveEmbedModel(projectRoot);
1323
+ if ('error' in configured) return { ok: false, error: configured.error };
1324
+ const guard = guardEmbedSpace({
1325
+ storePath: rvfBase(projectRoot),
1326
+ configured,
1327
+ hasRows: Object.keys(idmap.slots).length > 0,
1328
+ reindexHint: 'dz vector reindex',
1329
+ });
1330
+ if (!guard.ok) return { ok: false, error: guard.error };
1331
+ return { ok: true, configured, version: guard.manifest.version };
1332
+ }
1333
+
1334
+ function writeRvfSidecars(projectRoot: string, idmap: RvfIdmap, configured: EmbedModelConfig, version: number): void {
1280
1335
  const base = rvfBase(projectRoot);
1281
1336
  mkdirSync(dirname(base), { recursive: true });
1282
1337
  writeFileSync(`${base}.idmap.json`, JSON.stringify(idmap, null, 2));
1338
+ const manifest = currentEmbedManifest(configured, version, '@ruvector/rvf');
1283
1339
  writeFileSync(`${base}.manifest.json`, JSON.stringify(
1284
- { model: 'Xenova/all-MiniLM-L6-v2', dim: 384, engine: '@ruvector/rvf', version: 1 },
1340
+ manifest,
1285
1341
  null,
1286
1342
  2,
1287
1343
  ));
@@ -1356,14 +1412,16 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
1356
1412
  return {
1357
1413
  kind: 'rvf',
1358
1414
  async upsert(entries) {
1415
+ const idmap = readRvfIdmap(projectRoot);
1416
+ const guard = guardRvfEmbedSpace(projectRoot, idmap);
1417
+ if (!guard.ok) return { indexed: 0, error: guard.error };
1359
1418
  const emb = await resolveAgentdbEmbedder(projectRoot);
1360
1419
  if ('error' in emb) return { indexed: 0, error: noEmbedder };
1361
1420
  const loaded = await loadRvfModule(projectRoot);
1362
1421
  if (!loaded.ok) return { indexed: 0, error: loaded.error };
1363
- const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
1422
+ const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), DEFAULT_EMBED_DIM);
1364
1423
  if ('error' in store) return { indexed: 0, error: store.error };
1365
1424
  try {
1366
- const idmap = readRvfIdmap(projectRoot);
1367
1425
  let indexed = 0;
1368
1426
  for (const e of entries) {
1369
1427
  const vec = await emb.embed(`${e.taskType}: ${e.text}`);
@@ -1372,7 +1430,7 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
1372
1430
  indexed += 1;
1373
1431
  }
1374
1432
  await store.close?.();
1375
- writeRvfSidecars(projectRoot, idmap);
1433
+ writeRvfSidecars(projectRoot, idmap, guard.configured, guard.version);
1376
1434
  return { indexed };
1377
1435
  } catch (err) {
1378
1436
  return { indexed: 0, error: `rvf upsert failed: ${err instanceof Error ? err.message : String(err)}` };
@@ -1381,12 +1439,14 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
1381
1439
  async search(query, limit) {
1382
1440
  const emb = await resolveAgentdbEmbedder(projectRoot);
1383
1441
  if ('error' in emb) return { hits: [], error: noEmbedder };
1442
+ const idmap = readRvfIdmap(projectRoot);
1443
+ const guard = guardRvfEmbedSpace(projectRoot, idmap);
1444
+ if (!guard.ok) return { hits: [], error: guard.error };
1384
1445
  const loaded = await loadRvfModule(projectRoot);
1385
1446
  if (!loaded.ok) return { hits: [], error: loaded.error };
1386
- const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
1447
+ const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), DEFAULT_EMBED_DIM);
1387
1448
  if ('error' in store) return { hits: [], error: store.error };
1388
1449
  try {
1389
- const idmap = readRvfIdmap(projectRoot);
1390
1450
  const raw = await store.query(await emb.embed(query), limit);
1391
1451
  await store.close?.();
1392
1452
  const hits: VectorHit[] = [];
@@ -1410,12 +1470,14 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
1410
1470
  return { ids: [...new Set(Object.values(readRvfIdmap(projectRoot).slots))] };
1411
1471
  },
1412
1472
  async importVectors(rows) {
1473
+ const idmap = readRvfIdmap(projectRoot);
1474
+ const guard = guardRvfEmbedSpace(projectRoot, idmap);
1475
+ if (!guard.ok) return { imported: 0, error: guard.error };
1413
1476
  const loaded = await loadRvfModule(projectRoot);
1414
1477
  if (!loaded.ok) return { imported: 0, error: loaded.error };
1415
- const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), 384);
1478
+ const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), DEFAULT_EMBED_DIM);
1416
1479
  if ('error' in store) return { imported: 0, error: store.error };
1417
1480
  try {
1418
- const idmap = readRvfIdmap(projectRoot);
1419
1481
  let imported = 0;
1420
1482
  for (const r of rows) {
1421
1483
  await store.ingest(r.dzId, r.vector); // RVF ingest is upsert-by-id (id = dzId) — no duplicates
@@ -1423,7 +1485,7 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
1423
1485
  imported += 1;
1424
1486
  }
1425
1487
  await store.close?.();
1426
- writeRvfSidecars(projectRoot, idmap);
1488
+ writeRvfSidecars(projectRoot, idmap, guard.configured, guard.version);
1427
1489
  return { imported };
1428
1490
  } catch (err) {
1429
1491
  return { imported: 0, error: `rvf import failed: ${err instanceof Error ? err.message : String(err)}` };