@dzhechkov/harness-core 0.3.86 → 0.3.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agentdb-index.d.ts +16 -2
- package/dist/agentdb-index.d.ts.map +1 -1
- package/dist/agentdb-index.js +129 -22
- package/dist/agentdb-index.js.map +1 -1
- package/dist/brain.d.ts +23 -0
- package/dist/brain.d.ts.map +1 -1
- package/dist/brain.js +205 -9
- package/dist/brain.js.map +1 -1
- package/dist/embedding-config.d.ts +39 -0
- package/dist/embedding-config.d.ts.map +1 -0
- package/dist/embedding-config.js +93 -0
- package/dist/embedding-config.js.map +1 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/setup.js +1 -1
- package/dist/setup.js.map +1 -1
- package/dist/vector-tier.d.ts +9 -0
- package/dist/vector-tier.d.ts.map +1 -1
- package/dist/vector-tier.js +75 -16
- package/dist/vector-tier.js.map +1 -1
- package/package.json +4 -4
- package/src/agentdb-index.ts +132 -23
- package/src/brain.ts +200 -8
- package/src/embedding-config.ts +117 -0
- package/src/index.ts +7 -1
- package/src/setup.ts +1 -1
- package/src/vector-tier.ts +78 -16
package/src/brain.ts
CHANGED
|
@@ -17,11 +17,13 @@
|
|
|
17
17
|
|
|
18
18
|
import { join, dirname } from 'node:path';
|
|
19
19
|
import { homedir } from 'node:os';
|
|
20
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
|
|
20
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, readdirSync, type Dirent } from 'node:fs';
|
|
21
21
|
import { pathToFileURL } from 'node:url';
|
|
22
22
|
import { createRequire } from 'node:module';
|
|
23
|
+
import { spawn } from 'node:child_process';
|
|
23
24
|
import { putBookKnowledge, queryBookKnowledge, bookKbPath, type BookKU, type BookKUHit } from './book-kb.js';
|
|
24
|
-
import { indexPatternsToAgentdb, type AgentdbRow } from './agentdb-index.js';
|
|
25
|
+
import { indexPatternsToAgentdb, searchAgentdbPatterns, reindexAgentdbRows, type AgentdbRow } from './agentdb-index.js';
|
|
26
|
+
import { resolveEmbedModel, type EmbedModelConfig } from './embedding-config.js';
|
|
25
27
|
|
|
26
28
|
// ─────────────────────────────────────── Home & paths ───────────────────────────────────────
|
|
27
29
|
|
|
@@ -783,6 +785,8 @@ const RERANK_PRIOR_BONUS = 0.5;
|
|
|
783
785
|
|
|
784
786
|
/** Over-fetch cap for the rerank pass — bounds work while leaving room for a `limit*3` window. */
|
|
785
787
|
const RERANK_OVERFETCH_CAP = 200;
|
|
788
|
+
const GROUND_VECTOR_TIMEOUT_MS = 1_000;
|
|
789
|
+
const GROUND_VECTOR_SIMILARITY_FLOOR = 0.35;
|
|
786
790
|
|
|
787
791
|
/**
|
|
788
792
|
* A DETERMINISTIC lexical reranker (ADR-001 §11 P3 / G3) — lifts precision on the top-K without a
|
|
@@ -855,6 +859,180 @@ export async function queryBrain(opts: {
|
|
|
855
859
|
return { hits: rerankHits(opts.query, res.hits, { limit: effLimit }) };
|
|
856
860
|
}
|
|
857
861
|
|
|
862
|
+
async function bounded<T>(promise: Promise<T>, ms: number, fallback: () => T): Promise<T> {
|
|
863
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
864
|
+
try {
|
|
865
|
+
return await Promise.race([
|
|
866
|
+
promise.catch(() => fallback()),
|
|
867
|
+
new Promise<T>((resolvePromise) => {
|
|
868
|
+
timer = setTimeout(() => resolvePromise(fallback()), ms);
|
|
869
|
+
timer.unref?.();
|
|
870
|
+
}),
|
|
871
|
+
]);
|
|
872
|
+
} finally {
|
|
873
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
const backgroundWarmups = new Set<string>();
|
|
878
|
+
|
|
879
|
+
function isFakeAgentdbForTests(depsRoot: string): boolean {
|
|
880
|
+
try {
|
|
881
|
+
const req = createRequire(join(depsRoot, 'package.json'));
|
|
882
|
+
const pkg = JSON.parse(readFileSync(req.resolve('agentdb/package.json'), 'utf-8')) as { version?: unknown };
|
|
883
|
+
return typeof pkg.version === 'string' && pkg.version.includes('test');
|
|
884
|
+
} catch {
|
|
885
|
+
return false;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
function hasOnnxFile(dir: string, depth = 0, seen: { count: number } = { count: 0 }): boolean {
|
|
890
|
+
if (depth > 8 || seen.count > 2_000 || !existsSync(dir)) return false;
|
|
891
|
+
let entries: Dirent[];
|
|
892
|
+
try {
|
|
893
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
894
|
+
} catch {
|
|
895
|
+
return false;
|
|
896
|
+
}
|
|
897
|
+
for (const entry of entries) {
|
|
898
|
+
seen.count += 1;
|
|
899
|
+
const p = join(dir, entry.name);
|
|
900
|
+
if (entry.isFile() && entry.name.endsWith('.onnx')) return true;
|
|
901
|
+
if (entry.isDirectory() && hasOnnxFile(p, depth + 1, seen)) return true;
|
|
902
|
+
if (seen.count > 2_000) return false;
|
|
903
|
+
}
|
|
904
|
+
return false;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
function cacheRoots(): string[] {
|
|
908
|
+
const roots: string[] = [];
|
|
909
|
+
const transformers = process.env['TRANSFORMERS_CACHE'];
|
|
910
|
+
if (transformers !== undefined && transformers !== '') roots.push(transformers);
|
|
911
|
+
const hfHome = process.env['HF_HOME'];
|
|
912
|
+
if (hfHome !== undefined && hfHome !== '') roots.push(join(hfHome, 'hub'));
|
|
913
|
+
const xdg = process.env['XDG_CACHE_HOME'];
|
|
914
|
+
roots.push(join(xdg !== undefined && xdg !== '' ? xdg : join(homedir(), '.cache'), 'huggingface', 'hub'));
|
|
915
|
+
return [...new Set(roots)];
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
function isEmbedModelCached(depsRoot: string, model: EmbedModelConfig): boolean {
|
|
919
|
+
if (isFakeAgentdbForTests(depsRoot)) return true;
|
|
920
|
+
const modelDir = `models--${model.model.replace(/\//g, '--')}`;
|
|
921
|
+
for (const root of cacheRoots()) {
|
|
922
|
+
if (hasOnnxFile(join(root, modelDir)) || hasOnnxFile(root.endsWith(modelDir) ? root : join(root, 'hub', modelDir))) return true;
|
|
923
|
+
}
|
|
924
|
+
return false;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
function warmEmbedModelInBackground(depsRoot: string, model: EmbedModelConfig): void {
|
|
928
|
+
const key = `${depsRoot}\0${model.model}`;
|
|
929
|
+
if (backgroundWarmups.has(key)) return;
|
|
930
|
+
backgroundWarmups.add(key);
|
|
931
|
+
setImmediate(() => {
|
|
932
|
+
const script = `
|
|
933
|
+
import { createRequire } from 'node:module';
|
|
934
|
+
import { dirname, join } from 'node:path';
|
|
935
|
+
import { pathToFileURL } from 'node:url';
|
|
936
|
+
const root = ${JSON.stringify(depsRoot)};
|
|
937
|
+
const model = ${JSON.stringify(model.model)};
|
|
938
|
+
const dim = ${JSON.stringify(model.dim)};
|
|
939
|
+
const req = createRequire(join(root, 'package.json'));
|
|
940
|
+
const agentdbDir = dirname(req.resolve('agentdb'));
|
|
941
|
+
const mod = await import(pathToFileURL(join(agentdbDir, 'controllers', 'EmbeddingService.js')).href);
|
|
942
|
+
const emb = new mod.EmbeddingService({ model, dimension: dim, provider: 'transformers' });
|
|
943
|
+
await emb.initialize();
|
|
944
|
+
await emb.embed('warm embedding model cache');
|
|
945
|
+
`;
|
|
946
|
+
try {
|
|
947
|
+
const nodeBin = existsSync(process.execPath) ? process.execPath : (process.argv[0] ?? 'node');
|
|
948
|
+
const child = spawn(nodeBin, ['--input-type=module', '-e', script], {
|
|
949
|
+
cwd: depsRoot,
|
|
950
|
+
detached: true,
|
|
951
|
+
stdio: ['ignore', 'ignore', 'ignore'],
|
|
952
|
+
env: { ...process.env, DZ_EMBED_MODEL: model.model },
|
|
953
|
+
});
|
|
954
|
+
child.on('error', () => undefined);
|
|
955
|
+
child.unref();
|
|
956
|
+
} catch {
|
|
957
|
+
/* warmup is advisory; the foreground grounding path must never block or throw */
|
|
958
|
+
}
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
export async function searchBrainVectors(opts: {
|
|
963
|
+
query: string;
|
|
964
|
+
brainHome?: string;
|
|
965
|
+
depsRoot?: string;
|
|
966
|
+
source?: string;
|
|
967
|
+
limit?: number;
|
|
968
|
+
timeoutMs?: number;
|
|
969
|
+
}): Promise<{ hits: Array<BookKUHit & { similarity: number }>; error?: string }> {
|
|
970
|
+
const home = opts.brainHome ?? brainHome();
|
|
971
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
972
|
+
const limit = Math.max(1, opts.limit ?? DEFAULT_QUERY_LIMIT);
|
|
973
|
+
const model = resolveEmbedModel(depsRoot);
|
|
974
|
+
if ('error' in model) return { hits: [], error: model.error };
|
|
975
|
+
if (!isEmbedModelCached(depsRoot, model)) {
|
|
976
|
+
// eslint-disable-next-line no-console
|
|
977
|
+
console.warn(`[dz brain] SKIP vector leg: embedding model cache is cold for ${model.model}; warming in background`);
|
|
978
|
+
warmEmbedModelInBackground(depsRoot, model);
|
|
979
|
+
return { hits: [], error: 'brain vector search skipped: embedding model cache cold' };
|
|
980
|
+
}
|
|
981
|
+
const run = async (): Promise<{ hits: Array<BookKUHit & { similarity: number }>; error?: string }> => {
|
|
982
|
+
const vec = await searchAgentdbPatterns(depsRoot, opts.query, {
|
|
983
|
+
dbPath: brainAgentdbPath(home),
|
|
984
|
+
taskTypes: ['book-knowledge'],
|
|
985
|
+
limit: limit * 3,
|
|
986
|
+
reindexHint: 'dz brain reindex',
|
|
987
|
+
});
|
|
988
|
+
if (vec.error !== undefined) return { hits: [], error: vec.error };
|
|
989
|
+
if (vec.hits.length === 0) return { hits: [] };
|
|
990
|
+
const read = readBookKus({ storePath: brainBooksPath(home), depsRoot, ...(opts.source !== undefined ? { source: opts.source } : {}) });
|
|
991
|
+
if (read.error !== undefined) return { hits: [], error: read.error };
|
|
992
|
+
const byKu = new Map(read.kus.map((ku) => [ku.kuId, ku]));
|
|
993
|
+
const hits: Array<BookKUHit & { similarity: number }> = [];
|
|
994
|
+
const seen = new Set<string>();
|
|
995
|
+
for (const h of vec.hits) {
|
|
996
|
+
if (h.dzId === undefined || seen.has(h.dzId)) continue;
|
|
997
|
+
const ku = byKu.get(h.dzId);
|
|
998
|
+
if (ku === undefined) continue;
|
|
999
|
+
seen.add(h.dzId);
|
|
1000
|
+
hits.push({
|
|
1001
|
+
book: ku.book,
|
|
1002
|
+
kuId: ku.kuId,
|
|
1003
|
+
type: ku.type,
|
|
1004
|
+
name: ku.name,
|
|
1005
|
+
problem: ku.problem,
|
|
1006
|
+
content: ku.content,
|
|
1007
|
+
...(ku.chapter !== undefined ? { chapter: ku.chapter } : {}),
|
|
1008
|
+
...(ku.pages !== undefined ? { pages: ku.pages } : {}),
|
|
1009
|
+
similarity: h.similarity,
|
|
1010
|
+
});
|
|
1011
|
+
if (hits.length >= limit) break;
|
|
1012
|
+
}
|
|
1013
|
+
return { hits };
|
|
1014
|
+
};
|
|
1015
|
+
return bounded(run(), opts.timeoutMs ?? GROUND_VECTOR_TIMEOUT_MS, () => ({ hits: [], error: 'brain vector search timed out' }));
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
export async function reindexBrainVectors(opts: {
|
|
1019
|
+
brainHome?: string;
|
|
1020
|
+
depsRoot?: string;
|
|
1021
|
+
}): Promise<{ reembedded: number; model?: string; version?: number; backupPath?: string; error?: string }> {
|
|
1022
|
+
const home = opts.brainHome ?? brainHome();
|
|
1023
|
+
const depsRoot = opts.depsRoot ?? process.cwd();
|
|
1024
|
+
const read = readBookKus({ storePath: brainBooksPath(home), depsRoot });
|
|
1025
|
+
if (read.error !== undefined) return { reembedded: 0, error: read.error };
|
|
1026
|
+
const rows: AgentdbRow[] = read.kus.map((ku) => ({
|
|
1027
|
+
taskType: 'book-knowledge',
|
|
1028
|
+
text: `${ku.name}\n${ku.problem}\n${ku.content}`,
|
|
1029
|
+
score: 1.0,
|
|
1030
|
+
tags: ['book-knowledge', ku.book, ku.type],
|
|
1031
|
+
metadata: { source: 'book-kb', book: ku.book, kuId: ku.kuId, dzId: ku.kuId, corpusVersion: ku.corpusVersion },
|
|
1032
|
+
}));
|
|
1033
|
+
return reindexAgentdbRows(depsRoot, rows, { dbPath: brainAgentdbPath(home), taskTypes: ['book-knowledge'] });
|
|
1034
|
+
}
|
|
1035
|
+
|
|
858
1036
|
// ──────────────────────────────────────── Grounding ─────────────────────────────────────────
|
|
859
1037
|
|
|
860
1038
|
/**
|
|
@@ -951,8 +1129,22 @@ export async function groundPrompt(opts: {
|
|
|
951
1129
|
const res = await queryBrain(query);
|
|
952
1130
|
|
|
953
1131
|
// Gate 2: any lexical hit? On error or zero hits, stay silent (no noise, no block).
|
|
954
|
-
|
|
955
|
-
|
|
1132
|
+
const lexicalHits = res.error === undefined ? res.hits : [];
|
|
1133
|
+
const semantic = await searchBrainVectors({
|
|
1134
|
+
query: terms.join(' '),
|
|
1135
|
+
limit: opts.k ?? 5,
|
|
1136
|
+
...(opts.brainHome !== undefined ? { brainHome: opts.brainHome } : {}),
|
|
1137
|
+
...(opts.depsRoot !== undefined ? { depsRoot: opts.depsRoot } : {}),
|
|
1138
|
+
...(opts.source !== undefined ? { source: opts.source } : {}),
|
|
1139
|
+
timeoutMs: GROUND_VECTOR_TIMEOUT_MS,
|
|
1140
|
+
});
|
|
1141
|
+
const semanticHits = semantic.hits.filter((h) => h.similarity >= GROUND_VECTOR_SIMILARITY_FLOOR);
|
|
1142
|
+
if (lexicalHits.length === 0 && semanticHits.length === 0) return silent;
|
|
1143
|
+
|
|
1144
|
+
const mergedByKu = new Map<string, BookKUHit>();
|
|
1145
|
+
for (const h of lexicalHits) mergedByKu.set(h.kuId, h);
|
|
1146
|
+
for (const h of semanticHits) if (!mergedByKu.has(h.kuId)) mergedByKu.set(h.kuId, h);
|
|
1147
|
+
const merged = rerankHits(terms.join(' '), [...mergedByKu.values()], { limit: opts.k ?? 5 });
|
|
956
1148
|
|
|
957
1149
|
// Gate 3 — COVERAGE (the OR balance): OR recall alone over-fires when a single common term
|
|
958
1150
|
// ("today", "data") incidentally matches a KU. Require the retrieved hits to actually cover the
|
|
@@ -961,20 +1153,20 @@ export async function groundPrompt(opts: {
|
|
|
961
1153
|
// off-topic prompt whose only matching word is incidental ("weather today" → only "today"
|
|
962
1154
|
// matches) stays silent, while a real one ("репликацию single multi leader" → 4 covered) grounds.
|
|
963
1155
|
if (terms.length >= 2) {
|
|
964
|
-
const hay =
|
|
1156
|
+
const hay = lexicalHits.map((h) => `${h.name} ${h.problem} ${h.content}`.toLowerCase()).join(' ');
|
|
965
1157
|
const covered = terms.filter((t) => hay.includes(t.toLowerCase())).length;
|
|
966
|
-
if (covered < 2) return silent;
|
|
1158
|
+
if (covered < 2 && semanticHits.length === 0) return silent;
|
|
967
1159
|
}
|
|
968
1160
|
|
|
969
1161
|
// Build the GROUNDING DIRECTIVE block (§7.1): directive line + numbered citations.
|
|
970
1162
|
const lines: string[] = [GROUNDING_DIRECTIVE, ''];
|
|
971
|
-
|
|
1163
|
+
merged.forEach((h, i) => {
|
|
972
1164
|
const ch = h.chapter !== undefined && h.chapter !== '' ? ` гл.${h.chapter}` : '';
|
|
973
1165
|
const pg = h.pages !== undefined && h.pages.length > 0 ? ` с.${h.pages.join('–')}` : '';
|
|
974
1166
|
const body = snippet(h.problem !== '' ? h.problem : h.content);
|
|
975
1167
|
lines.push(`[K${i + 1}] ${h.book}${ch}${pg} — ${h.name}: ${body}`);
|
|
976
1168
|
});
|
|
977
|
-
return { emitted: true, block: lines.join('\n'), hitCount:
|
|
1169
|
+
return { emitted: true, block: lines.join('\n'), hitCount: merged.length };
|
|
978
1170
|
} catch {
|
|
979
1171
|
// Grounding is advisory — a bug here must never surface to the user or block the prompt.
|
|
980
1172
|
return silent;
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
export type EmbedModelSource = 'env' | 'config' | 'default';
|
|
5
|
+
|
|
6
|
+
export interface EmbedModelConfig {
|
|
7
|
+
readonly model: string;
|
|
8
|
+
readonly dim: 384;
|
|
9
|
+
readonly source: EmbedModelSource;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface EmbedManifest {
|
|
13
|
+
readonly model: string;
|
|
14
|
+
readonly dim: 384;
|
|
15
|
+
readonly version: number;
|
|
16
|
+
readonly engine?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export const DEFAULT_EMBED_MODEL = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2';
|
|
20
|
+
export const LEGACY_EMBED_MODEL = 'Xenova/all-MiniLM-L6-v2';
|
|
21
|
+
export const DEFAULT_EMBED_DIM = 384;
|
|
22
|
+
|
|
23
|
+
export const KNOWN_EMBED_DIMS: Readonly<Record<string, 384>> = {
|
|
24
|
+
[DEFAULT_EMBED_MODEL]: DEFAULT_EMBED_DIM,
|
|
25
|
+
[LEGACY_EMBED_MODEL]: DEFAULT_EMBED_DIM,
|
|
26
|
+
// e5 models require asymmetric "query:" / "passage:" prefixes at call sites to reach full quality.
|
|
27
|
+
'Xenova/multilingual-e5-small': DEFAULT_EMBED_DIM,
|
|
28
|
+
'Xenova/paraphrase-multilingual-mpnet-base-v2': DEFAULT_EMBED_DIM,
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export function resolveEmbedModel(projectRoot: string): EmbedModelConfig | { error: string } {
|
|
32
|
+
const env = process.env['DZ_EMBED_MODEL'];
|
|
33
|
+
if (env !== undefined && env.trim() !== '') return modelConfig(env.trim(), 'env');
|
|
34
|
+
const cfgPath = join(projectRoot, '.dz', 'config.json');
|
|
35
|
+
if (existsSync(cfgPath)) {
|
|
36
|
+
try {
|
|
37
|
+
const cfg = JSON.parse(readFileSync(cfgPath, 'utf-8')) as Record<string, unknown>;
|
|
38
|
+
const memory = cfg['memory'] as Record<string, unknown> | undefined;
|
|
39
|
+
const agentdb = memory?.['agentdb'] as Record<string, unknown> | undefined;
|
|
40
|
+
const embed = memory?.['embed'] as Record<string, unknown> | undefined;
|
|
41
|
+
const configured = agentdb?.['embeddingModel'] ?? embed?.['model'];
|
|
42
|
+
if (typeof configured === 'string' && configured.trim() !== '') return modelConfig(configured.trim(), 'config');
|
|
43
|
+
} catch {
|
|
44
|
+
/* corrupt config falls back to the default, matching the existing config-read discipline */
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return modelConfig(DEFAULT_EMBED_MODEL, 'default');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function modelConfig(model: string, source: EmbedModelSource): EmbedModelConfig | { error: string } {
|
|
51
|
+
const dim = KNOWN_EMBED_DIMS[model];
|
|
52
|
+
if (dim === undefined) {
|
|
53
|
+
return { error: `unsupported embedding model '${model}' (known 384-dim models: ${Object.keys(KNOWN_EMBED_DIMS).join(', ')})` };
|
|
54
|
+
}
|
|
55
|
+
return { model, dim, source };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function embedManifestPath(storePath: string): string {
|
|
59
|
+
return `${storePath}.embed-manifest.json`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function readEmbedManifest(storePath: string): EmbedManifest | undefined {
|
|
63
|
+
return readManifestFile(embedManifestPath(storePath));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function readManifestFile(p: string): EmbedManifest | undefined {
|
|
67
|
+
if (!existsSync(p)) return undefined;
|
|
68
|
+
try {
|
|
69
|
+
const m = JSON.parse(readFileSync(p, 'utf-8')) as Partial<EmbedManifest>;
|
|
70
|
+
if (typeof m.model !== 'string' || m.model === '') return undefined;
|
|
71
|
+
if (m.dim !== DEFAULT_EMBED_DIM) return undefined;
|
|
72
|
+
return { model: m.model, dim: DEFAULT_EMBED_DIM, version: typeof m.version === 'number' ? m.version : 1, ...(typeof m.engine === 'string' ? { engine: m.engine } : {}) };
|
|
73
|
+
} catch {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function readStoreManifest(storePath: string): EmbedManifest | undefined {
|
|
79
|
+
return readEmbedManifest(storePath) ?? readManifestFile(`${storePath}.manifest.json`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function writeEmbedManifest(storePath: string, manifest: EmbedManifest): void {
|
|
83
|
+
const p = embedManifestPath(storePath);
|
|
84
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
85
|
+
writeFileSync(p, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function legacyEmbedManifest(): EmbedManifest {
|
|
89
|
+
return { model: LEGACY_EMBED_MODEL, dim: DEFAULT_EMBED_DIM, version: 1 };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function currentEmbedManifest(configured: EmbedModelConfig, version = 1, engine?: string): EmbedManifest {
|
|
93
|
+
return { model: configured.model, dim: configured.dim, version, ...(engine !== undefined ? { engine } : {}) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function guardEmbedSpace(args: {
|
|
97
|
+
storePath: string;
|
|
98
|
+
configured: EmbedModelConfig;
|
|
99
|
+
hasRows: boolean;
|
|
100
|
+
reindexHint: string;
|
|
101
|
+
}): { ok: true; manifest: EmbedManifest } | { ok: false; error: string; manifest: EmbedManifest } {
|
|
102
|
+
const manifest = readStoreManifest(args.storePath)
|
|
103
|
+
?? (args.hasRows ? legacyEmbedManifest() : currentEmbedManifest(args.configured));
|
|
104
|
+
if (manifest.model !== args.configured.model || manifest.dim !== args.configured.dim) {
|
|
105
|
+
return {
|
|
106
|
+
ok: false,
|
|
107
|
+
manifest,
|
|
108
|
+
error: `embedding model mismatch: index built with ${manifest.model}/${manifest.dim}, configured ${args.configured.model}/${args.configured.dim}; run ${args.reindexHint}`,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
return { ok: true, manifest };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function snapshotEmbedManifest(storePath: string, backupPath: string): void {
|
|
115
|
+
const p = embedManifestPath(storePath);
|
|
116
|
+
if (existsSync(p)) copyFileSync(p, backupPath);
|
|
117
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -47,6 +47,7 @@ export {
|
|
|
47
47
|
mergeHybridHits,
|
|
48
48
|
recallHybrid,
|
|
49
49
|
vectorTierStatus,
|
|
50
|
+
reindexVectorStore,
|
|
50
51
|
harmonizeVectorStore,
|
|
51
52
|
selectClusterKeeper,
|
|
52
53
|
importRvfCheckpoint,
|
|
@@ -72,12 +73,15 @@ export type {
|
|
|
72
73
|
HarmonizeOptions,
|
|
73
74
|
ImportReport,
|
|
74
75
|
ImportOptions,
|
|
76
|
+
ReindexVectorReport,
|
|
75
77
|
} from './vector-tier.js';
|
|
76
78
|
export { runSetup, generateHooksConfig, generateAgentdbWriter, writerVersionOf, AGENTDB_WRITER_VERSION } from './setup.js';
|
|
77
79
|
export { statuslineData, readFeatureAdrState, writeFeatureAdrState, featureAdrStatePath } from './statusline.js';
|
|
78
80
|
export type { StatuslineData, FeatureAdrState, WriteFeatureAdrStateInput } from './statusline.js';
|
|
79
|
-
export { indexPatternsToAgentdb, resolveAgentdbPath, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb } from './agentdb-index.js';
|
|
81
|
+
export { indexPatternsToAgentdb, resolveAgentdbPath, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb, reindexAgentdbRows } from './agentdb-index.js';
|
|
80
82
|
export type { AgentdbSearchHit, AgentdbSearchResult, AgentdbImportRow } from './agentdb-index.js';
|
|
83
|
+
export { DEFAULT_EMBED_MODEL, LEGACY_EMBED_MODEL, DEFAULT_EMBED_DIM, KNOWN_EMBED_DIMS, resolveEmbedModel, readEmbedManifest, writeEmbedManifest, embedManifestPath, legacyEmbedManifest } from './embedding-config.js';
|
|
84
|
+
export type { EmbedModelConfig, EmbedModelSource, EmbedManifest } from './embedding-config.js';
|
|
81
85
|
export { putBookKnowledge, queryBookKnowledge, bookKbPath } from './book-kb.js';
|
|
82
86
|
export type { BookKU, BookKUHit } from './book-kb.js';
|
|
83
87
|
export {
|
|
@@ -91,6 +95,8 @@ export {
|
|
|
91
95
|
promoteProjectToBrain,
|
|
92
96
|
updateBrainSource,
|
|
93
97
|
queryBrain,
|
|
98
|
+
searchBrainVectors,
|
|
99
|
+
reindexBrainVectors,
|
|
94
100
|
rerankHits,
|
|
95
101
|
groundPrompt,
|
|
96
102
|
buildPrimer,
|
package/src/setup.ts
CHANGED
|
@@ -271,7 +271,7 @@ function generateDzConfig(target: string, preset: string | undefined, backend: M
|
|
|
271
271
|
mcpServer: 'agentdb',
|
|
272
272
|
// Both the hook writer and the MCP server read/write THIS path (env AGENTDB_PATH).
|
|
273
273
|
storePath: '.dz/agentdb.db',
|
|
274
|
-
embeddingModel: 'Xenova/
|
|
274
|
+
embeddingModel: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2',
|
|
275
275
|
sessionHookWrites: true,
|
|
276
276
|
} : undefined,
|
|
277
277
|
},
|
package/src/vector-tier.ts
CHANGED
|
@@ -59,7 +59,9 @@ import {
|
|
|
59
59
|
resolveAgentdbEmbedder,
|
|
60
60
|
cosineSimilarity,
|
|
61
61
|
importVectorsToAgentdb,
|
|
62
|
+
reindexAgentdbRows,
|
|
62
63
|
} from './agentdb-index.js';
|
|
64
|
+
import { currentEmbedManifest, guardEmbedSpace, DEFAULT_EMBED_DIM, resolveEmbedModel, type EmbedModelConfig } from './embedding-config.js';
|
|
63
65
|
|
|
64
66
|
/* ------------------------------------------------------------------ */
|
|
65
67
|
/* Types (04_domain_model §3.4 / §4.1) */
|
|
@@ -165,6 +167,7 @@ export interface VectorTierStatus {
|
|
|
165
167
|
readonly kind?: VectorEngineKind | undefined;
|
|
166
168
|
readonly available: boolean;
|
|
167
169
|
readonly reason?: string | undefined;
|
|
170
|
+
readonly embeddingModel?: string | undefined;
|
|
168
171
|
readonly lexicalTotal: number;
|
|
169
172
|
readonly lexicalMirrorable: number;
|
|
170
173
|
readonly mirrored?: number | undefined;
|
|
@@ -174,10 +177,6 @@ export interface VectorTierStatus {
|
|
|
174
177
|
/** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
|
|
175
178
|
export const DEFAULT_VECTOR_TIMEOUT_MS = 10_000;
|
|
176
179
|
|
|
177
|
-
/** The pinned local embedding model + dimension (agentdb's `EmbeddingService`; the RVF manifest space). */
|
|
178
|
-
const LOCAL_EMBED_MODEL = 'Xenova/all-MiniLM-L6-v2';
|
|
179
|
-
const LOCAL_EMBED_DIM = 384;
|
|
180
|
-
|
|
181
180
|
/** Default cosine cutoff for near-duplicate clustering (`--threshold` / config overrides). */
|
|
182
181
|
export const DEFAULT_HARMONIZE_THRESHOLD = 0.92;
|
|
183
182
|
|
|
@@ -255,6 +254,14 @@ export interface ImportOptions extends VectorServiceOptions {
|
|
|
255
254
|
readonly embed?: ((text: string) => Promise<Float32Array>) | undefined;
|
|
256
255
|
}
|
|
257
256
|
|
|
257
|
+
export interface ReindexVectorReport {
|
|
258
|
+
readonly reembedded: number;
|
|
259
|
+
readonly model?: string;
|
|
260
|
+
readonly version?: number;
|
|
261
|
+
readonly backupPath?: string;
|
|
262
|
+
readonly error?: string;
|
|
263
|
+
}
|
|
264
|
+
|
|
258
265
|
/* ------------------------------------------------------------------ */
|
|
259
266
|
/* Timeout wrapper (both legs — NC1/QR-1) */
|
|
260
267
|
/* ------------------------------------------------------------------ */
|
|
@@ -817,6 +824,7 @@ export async function vectorTierStatus(
|
|
|
817
824
|
opts: VectorServiceOptions = {},
|
|
818
825
|
): Promise<VectorTierStatus> {
|
|
819
826
|
const mode = readVectorEngineMode(projectRoot);
|
|
827
|
+
const model = resolveEmbedModel(projectRoot);
|
|
820
828
|
let records: MemoryRecord[];
|
|
821
829
|
try {
|
|
822
830
|
records = loadStoreRecords(projectRoot);
|
|
@@ -831,6 +839,7 @@ export async function vectorTierStatus(
|
|
|
831
839
|
mode,
|
|
832
840
|
available: false,
|
|
833
841
|
...(resolved.reason !== undefined ? { reason: resolved.reason } : {}),
|
|
842
|
+
...(!('error' in model) ? { embeddingModel: model.model } : {}),
|
|
834
843
|
lexicalTotal: records.length,
|
|
835
844
|
lexicalMirrorable,
|
|
836
845
|
pending,
|
|
@@ -847,6 +856,7 @@ export async function vectorTierStatus(
|
|
|
847
856
|
kind: engine.kind,
|
|
848
857
|
available: true,
|
|
849
858
|
...(listed.error !== undefined ? { reason: listed.error } : {}),
|
|
859
|
+
...(!('error' in model) ? { embeddingModel: model.model } : {}),
|
|
850
860
|
lexicalTotal: records.length,
|
|
851
861
|
lexicalMirrorable,
|
|
852
862
|
mirrored: listed.error === undefined ? listed.ids.length : undefined,
|
|
@@ -854,6 +864,31 @@ export async function vectorTierStatus(
|
|
|
854
864
|
};
|
|
855
865
|
}
|
|
856
866
|
|
|
867
|
+
export async function reindexVectorStore(projectRoot: string, opts: VectorServiceOptions = {}): Promise<ReindexVectorReport> {
|
|
868
|
+
const resolved = pickEngine(projectRoot, opts);
|
|
869
|
+
if (resolved.engine === undefined) return { reembedded: 0, error: resolved.reason ?? 'no vector engine available' };
|
|
870
|
+
if (resolved.engine.kind !== 'agentdb') {
|
|
871
|
+
return { reembedded: 0, error: 'dz vector reindex currently rewrites the agentdb learned-pattern mirror; switch memory.vector.engine to agentdb/auto' };
|
|
872
|
+
}
|
|
873
|
+
let records: MemoryRecord[];
|
|
874
|
+
try {
|
|
875
|
+
records = loadStoreRecords(projectRoot);
|
|
876
|
+
} catch {
|
|
877
|
+
records = [];
|
|
878
|
+
}
|
|
879
|
+
const rows = records
|
|
880
|
+
.map(memoryRecordVectorEntry)
|
|
881
|
+
.filter((r): r is VectorEntry => r !== undefined)
|
|
882
|
+
.map((r) => ({
|
|
883
|
+
taskType: r.taskType,
|
|
884
|
+
text: r.text,
|
|
885
|
+
score: r.score,
|
|
886
|
+
...(r.tags !== undefined ? { tags: r.tags } : {}),
|
|
887
|
+
...(r.metadata !== undefined ? { metadata: r.metadata } : {}),
|
|
888
|
+
}));
|
|
889
|
+
return reindexAgentdbRows(projectRoot, rows);
|
|
890
|
+
}
|
|
891
|
+
|
|
857
892
|
/* ------------------------------------------------------------------ */
|
|
858
893
|
/* Harmonize — SEMANTIC dedup of the lexical store (05 §2.1) */
|
|
859
894
|
/* ------------------------------------------------------------------ */
|
|
@@ -1139,8 +1174,15 @@ export async function importRvfCheckpoint(projectRoot: string, source: string, o
|
|
|
1139
1174
|
if (existsSync(manifestPath)) {
|
|
1140
1175
|
try {
|
|
1141
1176
|
const m = JSON.parse(readFileSync(manifestPath, 'utf-8')) as { model?: unknown; dim?: unknown };
|
|
1142
|
-
|
|
1143
|
-
|
|
1177
|
+
const configured = resolveEmbedModel(projectRoot);
|
|
1178
|
+
if ('error' in configured) return fail(configured.error);
|
|
1179
|
+
const manifest = {
|
|
1180
|
+
model: typeof m.model === 'string' ? m.model : configured.model,
|
|
1181
|
+
dim: typeof m.dim === 'number' ? m.dim : configured.dim,
|
|
1182
|
+
version: 1,
|
|
1183
|
+
};
|
|
1184
|
+
if (manifest.model !== configured.model || manifest.dim !== configured.dim) {
|
|
1185
|
+
return fail(`manifest mismatch: checkpoint (${String(manifest.model)}/${String(manifest.dim)}) ≠ local (${configured.model}/${configured.dim}) — run dz vector reindex; refusing a cross-embedding-space merge`);
|
|
1144
1186
|
}
|
|
1145
1187
|
} catch { /* unreadable manifest — tolerate; the idmap is the authority */ }
|
|
1146
1188
|
}
|
|
@@ -1276,12 +1318,26 @@ function readRvfIdmap(projectRoot: string): RvfIdmap {
|
|
|
1276
1318
|
}
|
|
1277
1319
|
}
|
|
1278
1320
|
|
|
1279
|
-
function
|
|
1321
|
+
function guardRvfEmbedSpace(projectRoot: string, idmap: RvfIdmap): { ok: true; configured: EmbedModelConfig; version: number } | { ok: false; error: string } {
|
|
1322
|
+
const configured = resolveEmbedModel(projectRoot);
|
|
1323
|
+
if ('error' in configured) return { ok: false, error: configured.error };
|
|
1324
|
+
const guard = guardEmbedSpace({
|
|
1325
|
+
storePath: rvfBase(projectRoot),
|
|
1326
|
+
configured,
|
|
1327
|
+
hasRows: Object.keys(idmap.slots).length > 0,
|
|
1328
|
+
reindexHint: 'dz vector reindex',
|
|
1329
|
+
});
|
|
1330
|
+
if (!guard.ok) return { ok: false, error: guard.error };
|
|
1331
|
+
return { ok: true, configured, version: guard.manifest.version };
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
function writeRvfSidecars(projectRoot: string, idmap: RvfIdmap, configured: EmbedModelConfig, version: number): void {
|
|
1280
1335
|
const base = rvfBase(projectRoot);
|
|
1281
1336
|
mkdirSync(dirname(base), { recursive: true });
|
|
1282
1337
|
writeFileSync(`${base}.idmap.json`, JSON.stringify(idmap, null, 2));
|
|
1338
|
+
const manifest = currentEmbedManifest(configured, version, '@ruvector/rvf');
|
|
1283
1339
|
writeFileSync(`${base}.manifest.json`, JSON.stringify(
|
|
1284
|
-
|
|
1340
|
+
manifest,
|
|
1285
1341
|
null,
|
|
1286
1342
|
2,
|
|
1287
1343
|
));
|
|
@@ -1356,14 +1412,16 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
|
|
|
1356
1412
|
return {
|
|
1357
1413
|
kind: 'rvf',
|
|
1358
1414
|
async upsert(entries) {
|
|
1415
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1416
|
+
const guard = guardRvfEmbedSpace(projectRoot, idmap);
|
|
1417
|
+
if (!guard.ok) return { indexed: 0, error: guard.error };
|
|
1359
1418
|
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
1360
1419
|
if ('error' in emb) return { indexed: 0, error: noEmbedder };
|
|
1361
1420
|
const loaded = await loadRvfModule(projectRoot);
|
|
1362
1421
|
if (!loaded.ok) return { indexed: 0, error: loaded.error };
|
|
1363
|
-
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot),
|
|
1422
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), DEFAULT_EMBED_DIM);
|
|
1364
1423
|
if ('error' in store) return { indexed: 0, error: store.error };
|
|
1365
1424
|
try {
|
|
1366
|
-
const idmap = readRvfIdmap(projectRoot);
|
|
1367
1425
|
let indexed = 0;
|
|
1368
1426
|
for (const e of entries) {
|
|
1369
1427
|
const vec = await emb.embed(`${e.taskType}: ${e.text}`);
|
|
@@ -1372,7 +1430,7 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
|
|
|
1372
1430
|
indexed += 1;
|
|
1373
1431
|
}
|
|
1374
1432
|
await store.close?.();
|
|
1375
|
-
writeRvfSidecars(projectRoot, idmap);
|
|
1433
|
+
writeRvfSidecars(projectRoot, idmap, guard.configured, guard.version);
|
|
1376
1434
|
return { indexed };
|
|
1377
1435
|
} catch (err) {
|
|
1378
1436
|
return { indexed: 0, error: `rvf upsert failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
@@ -1381,12 +1439,14 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
|
|
|
1381
1439
|
async search(query, limit) {
|
|
1382
1440
|
const emb = await resolveAgentdbEmbedder(projectRoot);
|
|
1383
1441
|
if ('error' in emb) return { hits: [], error: noEmbedder };
|
|
1442
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1443
|
+
const guard = guardRvfEmbedSpace(projectRoot, idmap);
|
|
1444
|
+
if (!guard.ok) return { hits: [], error: guard.error };
|
|
1384
1445
|
const loaded = await loadRvfModule(projectRoot);
|
|
1385
1446
|
if (!loaded.ok) return { hits: [], error: loaded.error };
|
|
1386
|
-
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot),
|
|
1447
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), DEFAULT_EMBED_DIM);
|
|
1387
1448
|
if ('error' in store) return { hits: [], error: store.error };
|
|
1388
1449
|
try {
|
|
1389
|
-
const idmap = readRvfIdmap(projectRoot);
|
|
1390
1450
|
const raw = await store.query(await emb.embed(query), limit);
|
|
1391
1451
|
await store.close?.();
|
|
1392
1452
|
const hits: VectorHit[] = [];
|
|
@@ -1410,12 +1470,14 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
|
|
|
1410
1470
|
return { ids: [...new Set(Object.values(readRvfIdmap(projectRoot).slots))] };
|
|
1411
1471
|
},
|
|
1412
1472
|
async importVectors(rows) {
|
|
1473
|
+
const idmap = readRvfIdmap(projectRoot);
|
|
1474
|
+
const guard = guardRvfEmbedSpace(projectRoot, idmap);
|
|
1475
|
+
if (!guard.ok) return { imported: 0, error: guard.error };
|
|
1413
1476
|
const loaded = await loadRvfModule(projectRoot);
|
|
1414
1477
|
if (!loaded.ok) return { imported: 0, error: loaded.error };
|
|
1415
|
-
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot),
|
|
1478
|
+
const store = await openRvfStore(loaded.mod, rvfBase(projectRoot), DEFAULT_EMBED_DIM);
|
|
1416
1479
|
if ('error' in store) return { imported: 0, error: store.error };
|
|
1417
1480
|
try {
|
|
1418
|
-
const idmap = readRvfIdmap(projectRoot);
|
|
1419
1481
|
let imported = 0;
|
|
1420
1482
|
for (const r of rows) {
|
|
1421
1483
|
await store.ingest(r.dzId, r.vector); // RVF ingest is upsert-by-id (id = dzId) — no duplicates
|
|
@@ -1423,7 +1485,7 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
|
|
|
1423
1485
|
imported += 1;
|
|
1424
1486
|
}
|
|
1425
1487
|
await store.close?.();
|
|
1426
|
-
writeRvfSidecars(projectRoot, idmap);
|
|
1488
|
+
writeRvfSidecars(projectRoot, idmap, guard.configured, guard.version);
|
|
1427
1489
|
return { imported };
|
|
1428
1490
|
} catch (err) {
|
|
1429
1491
|
return { imported: 0, error: `rvf import failed: ${err instanceof Error ? err.message : String(err)}` };
|