@wiati/dfe-agent 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +196 -0
- package/dist/agent.md +48 -0
- package/dist/bin/dfe-agent.d.ts +2 -0
- package/dist/bin/dfe-agent.d.ts.map +1 -0
- package/dist/bin/dfe-agent.js +19 -0
- package/dist/bin/dfe-agent.js.map +1 -0
- package/dist/cli.d.ts +25 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +105 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/install.d.ts +23 -0
- package/dist/commands/install.d.ts.map +1 -0
- package/dist/commands/install.js +74 -0
- package/dist/commands/install.js.map +1 -0
- package/dist/commands/query.d.ts +14 -0
- package/dist/commands/query.d.ts.map +1 -0
- package/dist/commands/query.js +18 -0
- package/dist/commands/query.js.map +1 -0
- package/dist/commands/status.d.ts +10 -0
- package/dist/commands/status.d.ts.map +1 -0
- package/dist/commands/status.js +58 -0
- package/dist/commands/status.js.map +1 -0
- package/dist/commands/update.d.ts +26 -0
- package/dist/commands/update.d.ts.map +1 -0
- package/dist/commands/update.js +144 -0
- package/dist/commands/update.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/query/cache.d.ts +38 -0
- package/dist/query/cache.d.ts.map +1 -0
- package/dist/query/cache.js +80 -0
- package/dist/query/cache.js.map +1 -0
- package/dist/query/constants.d.ts +20 -0
- package/dist/query/constants.d.ts.map +1 -0
- package/dist/query/constants.js +20 -0
- package/dist/query/constants.js.map +1 -0
- package/dist/query/contextBuilder.d.ts +40 -0
- package/dist/query/contextBuilder.d.ts.map +1 -0
- package/dist/query/contextBuilder.js +54 -0
- package/dist/query/contextBuilder.js.map +1 -0
- package/dist/query/embedder.d.ts +30 -0
- package/dist/query/embedder.d.ts.map +1 -0
- package/dist/query/embedder.js +69 -0
- package/dist/query/embedder.js.map +1 -0
- package/dist/query/ftsSearch.d.ts +28 -0
- package/dist/query/ftsSearch.d.ts.map +1 -0
- package/dist/query/ftsSearch.js +48 -0
- package/dist/query/ftsSearch.js.map +1 -0
- package/dist/query/hybrid.d.ts +31 -0
- package/dist/query/hybrid.d.ts.map +1 -0
- package/dist/query/hybrid.js +67 -0
- package/dist/query/hybrid.js.map +1 -0
- package/dist/query/index.d.ts +45 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +153 -0
- package/dist/query/index.js.map +1 -0
- package/dist/query/vectorSearch.d.ts +32 -0
- package/dist/query/vectorSearch.d.ts.map +1 -0
- package/dist/query/vectorSearch.js +54 -0
- package/dist/query/vectorSearch.js.map +1 -0
- package/dist/skill/dfe-fiscal/SKILL.md +140 -0
- package/package.json +59 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* embedder.ts — wrapper de @xenova/transformers com LRU cache em memoria.
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.1
|
|
5
|
+
*
|
|
6
|
+
* Decisao D6: modelo canonico e `paraphrase-multilingual-MiniLM-L12-v2`
|
|
7
|
+
* (mesmo do DFe-Agent Py via sentence-transformers). Drift entre Py e Node
|
|
8
|
+
* deve ser < 0.01 (gate D.7 validado por tests/integration/test_embedding_parity).
|
|
9
|
+
*
|
|
10
|
+
* LRU 128: cobre uso interativo; descarta entradas mais antigas.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
|
+
import { env, pipeline } from "@xenova/transformers";
|
|
14
|
+
const MODEL_NAME = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
15
|
+
const EMBEDDING_DIM = 384;
|
|
16
|
+
const LRU_SIZE = 128;
|
|
17
|
+
export const EMBEDDING_MODEL_NAME = MODEL_NAME;
|
|
18
|
+
export const EMBEDDING_DIMENSION = EMBEDDING_DIM;
|
|
19
|
+
const cache = new Map();
|
|
20
|
+
function hash(text) {
|
|
21
|
+
return createHash("sha256")
|
|
22
|
+
.update(text.trim().toLowerCase())
|
|
23
|
+
.digest("hex");
|
|
24
|
+
}
|
|
25
|
+
let pipelineSingleton = null;
|
|
26
|
+
async function getExtractor() {
|
|
27
|
+
if (pipelineSingleton)
|
|
28
|
+
return pipelineSingleton;
|
|
29
|
+
env.allowLocalModels = false;
|
|
30
|
+
env.useFS = false;
|
|
31
|
+
pipelineSingleton = await pipeline("feature-extraction", MODEL_NAME);
|
|
32
|
+
return pipelineSingleton;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Codifica `text` em vetor 384-d (cosine-normalized).
|
|
36
|
+
*
|
|
37
|
+
* Cache LRU: 2a chamada com mesmo texto retorna o mesmo Float32Array sem
|
|
38
|
+
* nova inferencia.
|
|
39
|
+
*
|
|
40
|
+
* @throws se o modelo nao conseguir carregar (problema conhecido: Node
|
|
41
|
+
* v22.21.1 com sharp/better-sqlite3 nativos; ver Sprint 13).
|
|
42
|
+
*/
|
|
43
|
+
export async function encode(text) {
|
|
44
|
+
const key = hash(text);
|
|
45
|
+
const cached = cache.get(key);
|
|
46
|
+
if (cached) {
|
|
47
|
+
// Move-to-end (LRU)
|
|
48
|
+
cache.delete(key);
|
|
49
|
+
cache.set(key, cached);
|
|
50
|
+
return cached;
|
|
51
|
+
}
|
|
52
|
+
const extractor = await getExtractor();
|
|
53
|
+
const out = await extractor(text, { pooling: "mean", normalize: true });
|
|
54
|
+
const vec = new Float32Array(out.data);
|
|
55
|
+
cache.set(key, vec);
|
|
56
|
+
if (cache.size > LRU_SIZE) {
|
|
57
|
+
const firstKey = cache.keys().next().value;
|
|
58
|
+
if (firstKey !== undefined)
|
|
59
|
+
cache.delete(firstKey);
|
|
60
|
+
}
|
|
61
|
+
return vec;
|
|
62
|
+
}
|
|
63
|
+
export function clearCache() {
|
|
64
|
+
cache.clear();
|
|
65
|
+
}
|
|
66
|
+
export function cacheSize() {
|
|
67
|
+
return cache.size;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/query/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAErD,MAAM,UAAU,GAAG,8CAA8C,CAAC;AAClE,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC;AAErB,MAAM,CAAC,MAAM,oBAAoB,GAAG,UAAU,CAAC;AAC/C,MAAM,CAAC,MAAM,mBAAmB,GAAG,aAAa,CAAC;AAOjD,MAAM,KAAK,GAA8B,IAAI,GAAG,EAAE,CAAC;AAEnD,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,UAAU,CAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SACjC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnB,CAAC;AAED,IAAI,iBAAiB,GAAe,IAAI,CAAC;AAEzC,KAAK,UAAU,YAAY;IACzB,IAAI,iBAAiB;QAAE,OAAO,iBAAiB,CAAC;IAChD,GAAG,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAC7B,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,iBAAiB,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC;IACrE,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAY;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,MAAM,EAAE,CAAC;QACX,oBAAoB;QACpB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,IAAyB,CAAC,CAAC;IAE5D,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACpB,IAAI,KAAK,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAA2B,CAAC;QACjE,IAAI,QAAQ,KAAK,SAAS;YAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,KAAK,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,KAAK,CAAC,IAAI,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ftsSearch.ts — busca lexical via FTS5 (BM25).
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.2
|
|
5
|
+
*
|
|
6
|
+
* Estrategia:
|
|
7
|
+
- saneamento da query (escape chars que quebram MATCH)
|
|
8
|
+
- MATCH na tabela fts_chunks (BM25 nativo)
|
|
9
|
+
- ORDER BY bm25(fts_chunks) LIMIT k
|
|
10
|
+
*
|
|
11
|
+
* BM25 score: menor = mais relevante (FTS5 inverte o sinal).
|
|
12
|
+
*/
|
|
13
|
+
import type { Database as BetterSqlite3Database } from "better-sqlite3";
|
|
14
|
+
export interface FtsHit {
|
|
15
|
+
chunk_id: number;
|
|
16
|
+
doc_id: number;
|
|
17
|
+
score: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Busca top-K chunks por BM25. Retorna hits ordenados do mais relevante
|
|
21
|
+
* (menor score BM25) ao menos relevante.
|
|
22
|
+
*
|
|
23
|
+
* @param handle Better-sqlite3 handle (read-only)
|
|
24
|
+
* @param query Query em linguagem natural
|
|
25
|
+
* @param k Numero de hits
|
|
26
|
+
*/
|
|
27
|
+
export declare function ftsSearch(handle: BetterSqlite3Database, query: string, k: number): FtsHit[];
|
|
28
|
+
//# sourceMappingURL=ftsSearch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ftsSearch.d.ts","sourceRoot":"","sources":["../../src/query/ftsSearch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,QAAQ,IAAI,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAExE,MAAM,WAAW,MAAM;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAaD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CACvB,MAAM,EAAE,qBAAqB,EAC7B,KAAK,EAAE,MAAM,EACb,CAAC,EAAE,MAAM,GACR,MAAM,EAAE,CAmBV"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ftsSearch.ts — busca lexical via FTS5 (BM25).
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.2
|
|
5
|
+
*
|
|
6
|
+
* Estrategia:
|
|
7
|
+
- saneamento da query (escape chars que quebram MATCH)
|
|
8
|
+
- MATCH na tabela fts_chunks (BM25 nativo)
|
|
9
|
+
- ORDER BY bm25(fts_chunks) LIMIT k
|
|
10
|
+
*
|
|
11
|
+
* BM25 score: menor = mais relevante (FTS5 inverte o sinal).
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Remove caracteres que quebram o MATCH do FTS5. Mantemos alfanumericos,
|
|
15
|
+
* espacos e aspas (para phrase queries).
|
|
16
|
+
*/
|
|
17
|
+
function sanitizeQuery(q) {
|
|
18
|
+
return q
|
|
19
|
+
.replace(/[^\w\s"]/g, " ")
|
|
20
|
+
.replace(/\s+/g, " ")
|
|
21
|
+
.trim();
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Busca top-K chunks por BM25. Retorna hits ordenados do mais relevante
|
|
25
|
+
* (menor score BM25) ao menos relevante.
|
|
26
|
+
*
|
|
27
|
+
* @param handle Better-sqlite3 handle (read-only)
|
|
28
|
+
* @param query Query em linguagem natural
|
|
29
|
+
* @param k Numero de hits
|
|
30
|
+
*/
|
|
31
|
+
export function ftsSearch(handle, query, k) {
|
|
32
|
+
const ftsQuery = sanitizeQuery(query);
|
|
33
|
+
if (!ftsQuery)
|
|
34
|
+
return [];
|
|
35
|
+
const rows = handle
|
|
36
|
+
.prepare(`SELECT chunk_id, doc_id, bm25(fts_chunks) AS score
|
|
37
|
+
FROM fts_chunks
|
|
38
|
+
WHERE fts_chunks MATCH ?
|
|
39
|
+
ORDER BY score
|
|
40
|
+
LIMIT ?`)
|
|
41
|
+
.all(ftsQuery, k);
|
|
42
|
+
return rows.map((r) => ({
|
|
43
|
+
chunk_id: r.chunk_id,
|
|
44
|
+
doc_id: r.doc_id,
|
|
45
|
+
score: r.score,
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=ftsSearch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ftsSearch.js","sourceRoot":"","sources":["../../src/query/ftsSearch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAUH;;;GAGG;AACH,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,CAAC;SACL,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CACvB,MAA6B,EAC7B,KAAa,EACb,CAAS;IAET,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IAEzB,MAAM,IAAI,GAAG,MAAM;SAChB,OAAO,CACN;;;;gBAIU,CACX;SACA,GAAG,CAAC,QAAQ,EAAE,CAAC,CAA+D,CAAC;IAElF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hybrid.ts — Reciprocal Rank Fusion (RRF) entre vector search e FTS5.
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.3
|
|
5
|
+
*
|
|
6
|
+
* Algoritmo RRF canonico (Cormack et al., 2009):
|
|
7
|
+
* rrf_score(d) = sum_modes (1 / (k0 + rank_m(d)))
|
|
8
|
+
* Onde k0 = 60 e rank_m(d) e' o rank do doc `d` no modo `m` (1-indexed).
|
|
9
|
+
*
|
|
10
|
+
* Documentos que aparecem em ambos os modos (vector + fts) recebem score
|
|
11
|
+
* somado, naturalmente subindo no ranking final.
|
|
12
|
+
*/
|
|
13
|
+
import type { VectorHit } from "./vectorSearch.js";
|
|
14
|
+
import type { FtsHit } from "./ftsSearch.js";
|
|
15
|
+
export interface HybridHit {
|
|
16
|
+
chunk_id: number;
|
|
17
|
+
doc_id: number;
|
|
18
|
+
score: number;
|
|
19
|
+
sources: Array<"vector" | "fts">;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Fonde resultados de vector + FTS via RRF.
|
|
23
|
+
*
|
|
24
|
+
* @param vectorHits Top-K do vector search (ordenado por distance)
|
|
25
|
+
* @param ftsHits Top-K do FTS5 (ordenado por bm25)
|
|
26
|
+
* @param k Tamanho final do ranking
|
|
27
|
+
* @returns Array de hits ordenados por score decrescente
|
|
28
|
+
*/
|
|
29
|
+
export declare function rrf(vectorHits: VectorHit[], ftsHits: FtsHit[], k: number, k0?: number): HybridHit[];
|
|
30
|
+
export declare const RRF_K0_CANONICAL = 60;
|
|
31
|
+
//# sourceMappingURL=hybrid.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/query/hybrid.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC;CAClC;AAID;;;;;;;GAOG;AACH,wBAAgB,GAAG,CACjB,UAAU,EAAE,SAAS,EAAE,EACvB,OAAO,EAAE,MAAM,EAAE,EACjB,CAAC,EAAE,MAAM,EACT,EAAE,GAAE,MAAe,GAClB,SAAS,EAAE,CA4Cb;AAED,eAAO,MAAM,gBAAgB,KAAS,CAAC"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hybrid.ts — Reciprocal Rank Fusion (RRF) entre vector search e FTS5.
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.3
|
|
5
|
+
*
|
|
6
|
+
* Algoritmo RRF canonico (Cormack et al., 2009):
|
|
7
|
+
* rrf_score(d) = sum_modes (1 / (k0 + rank_m(d)))
|
|
8
|
+
* Onde k0 = 60 e rank_m(d) e' o rank do doc `d` no modo `m` (1-indexed).
|
|
9
|
+
*
|
|
10
|
+
* Documentos que aparecem em ambos os modos (vector + fts) recebem score
|
|
11
|
+
* somado, naturalmente subindo no ranking final.
|
|
12
|
+
*/
|
|
13
|
+
const RRF_K0 = 60;
|
|
14
|
+
/**
|
|
15
|
+
* Fonde resultados de vector + FTS via RRF.
|
|
16
|
+
*
|
|
17
|
+
* @param vectorHits Top-K do vector search (ordenado por distance)
|
|
18
|
+
* @param ftsHits Top-K do FTS5 (ordenado por bm25)
|
|
19
|
+
* @param k Tamanho final do ranking
|
|
20
|
+
* @returns Array de hits ordenados por score decrescente
|
|
21
|
+
*/
|
|
22
|
+
export function rrf(vectorHits, ftsHits, k, k0 = RRF_K0) {
|
|
23
|
+
const scoreMap = new Map();
|
|
24
|
+
vectorHits.forEach((hit, i) => {
|
|
25
|
+
const existing = scoreMap.get(hit.chunk_id);
|
|
26
|
+
const rankScore = 1 / (k0 + i + 1);
|
|
27
|
+
if (existing) {
|
|
28
|
+
existing.score += rankScore;
|
|
29
|
+
existing.sources.add("vector");
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
scoreMap.set(hit.chunk_id, {
|
|
33
|
+
chunk_id: hit.chunk_id,
|
|
34
|
+
doc_id: hit.doc_id,
|
|
35
|
+
score: rankScore,
|
|
36
|
+
sources: new Set(["vector"]),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
ftsHits.forEach((hit, i) => {
|
|
41
|
+
const existing = scoreMap.get(hit.chunk_id);
|
|
42
|
+
const rankScore = 1 / (k0 + i + 1);
|
|
43
|
+
if (existing) {
|
|
44
|
+
existing.score += rankScore;
|
|
45
|
+
existing.sources.add("fts");
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
scoreMap.set(hit.chunk_id, {
|
|
49
|
+
chunk_id: hit.chunk_id,
|
|
50
|
+
doc_id: hit.doc_id,
|
|
51
|
+
score: rankScore,
|
|
52
|
+
sources: new Set(["fts"]),
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
return Array.from(scoreMap.values())
|
|
57
|
+
.sort((a, b) => b.score - a.score)
|
|
58
|
+
.slice(0, k)
|
|
59
|
+
.map((entry) => ({
|
|
60
|
+
chunk_id: entry.chunk_id,
|
|
61
|
+
doc_id: entry.doc_id,
|
|
62
|
+
score: entry.score,
|
|
63
|
+
sources: Array.from(entry.sources),
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
export const RRF_K0_CANONICAL = RRF_K0;
|
|
67
|
+
//# sourceMappingURL=hybrid.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/query/hybrid.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAYH,MAAM,MAAM,GAAG,EAAE,CAAC;AAElB;;;;;;;GAOG;AACH,MAAM,UAAU,GAAG,CACjB,UAAuB,EACvB,OAAiB,EACjB,CAAS,EACT,KAAa,MAAM;IAEnB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+F,CAAC;IAExH,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC;YAC5B,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACzB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,IAAI,GAAG,CAAmB,CAAC,QAAQ,CAAC,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACzB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC;YAC5B,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACzB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,IAAI,GAAG,CAAmB,CAAC,KAAK,CAAC,CAAC;aAC5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACf,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;KACnC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,MAAM,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* query/index.ts — query engine Node (port de src/query/query_engine.py).
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Fase E (5 tasks)
|
|
5
|
+
* @see src/query/query_engine.py — equivalente Py
|
|
6
|
+
*
|
|
7
|
+
* Modos opt-in (mesmo do CLI Py):
|
|
8
|
+
* semantic vector search com sqlite-vec (default)
|
|
9
|
+
* fts FTS5 BM25 lexical
|
|
10
|
+
* hybrid RRF k=60 entre vector + FTS5
|
|
11
|
+
*
|
|
12
|
+
* Constante NO_EVIDENCE_MESSAGE canonica — mesma string em
|
|
13
|
+
* src/query/context_builder.py do Py (gate dfe-rules.md #4).
|
|
14
|
+
* NUNCA duplique este literal.
|
|
15
|
+
*/
|
|
16
|
+
import { EMBEDDING_MODEL_NAME } from "./embedder.js";
|
|
17
|
+
import { hasSufficientEvidence } from "./contextBuilder.js";
|
|
18
|
+
import { NO_EVIDENCE_MESSAGE, SUPPORTED_MODES, DEFAULT_TOP_K, MIN_RELEVANCE_SCORE, type SearchMode } from "./constants.js";
|
|
19
|
+
export { NO_EVIDENCE_MESSAGE, SUPPORTED_MODES, DEFAULT_TOP_K };
|
|
20
|
+
export type { SearchMode };
|
|
21
|
+
export interface Source {
|
|
22
|
+
url: string;
|
|
23
|
+
title: string;
|
|
24
|
+
score: number;
|
|
25
|
+
}
|
|
26
|
+
export interface SearchResult {
|
|
27
|
+
answer: string;
|
|
28
|
+
sources: Source[];
|
|
29
|
+
}
|
|
30
|
+
export interface SearchOptions {
|
|
31
|
+
mode?: SearchMode;
|
|
32
|
+
topK?: number;
|
|
33
|
+
baseDirOverride?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Resolve path da base RAG. Default: $DFE_AGENT_BASE_DIR ou ~/.dfe-agent.
|
|
37
|
+
*/
|
|
38
|
+
export declare function resolveBaseDir(override?: string): string;
|
|
39
|
+
/**
|
|
40
|
+
* Busca principal. Retorna {answer, sources[]} no contrato canonico.
|
|
41
|
+
*/
|
|
42
|
+
export declare function search(question: string, opts?: SearchOptions): Promise<SearchResult>;
|
|
43
|
+
export { EMBEDDING_MODEL_NAME, MIN_RELEVANCE_SCORE };
|
|
44
|
+
export { hasSufficientEvidence };
|
|
45
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAMH,OAAO,EAAU,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAK7D,OAAO,EAEL,qBAAqB,EAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,aAAa,EACb,mBAAmB,EAGnB,KAAK,UAAU,EAChB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;AAC/D,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAIxD;AAmED;;GAEG;AACH,wBAAsB,MAAM,CAC1B,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,CAAC,CAmEvB;AAED,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,CAAC"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* query/index.ts — query engine Node (port de src/query/query_engine.py).
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Fase E (5 tasks)
|
|
5
|
+
* @see src/query/query_engine.py — equivalente Py
|
|
6
|
+
*
|
|
7
|
+
* Modos opt-in (mesmo do CLI Py):
|
|
8
|
+
* semantic vector search com sqlite-vec (default)
|
|
9
|
+
* fts FTS5 BM25 lexical
|
|
10
|
+
* hybrid RRF k=60 entre vector + FTS5
|
|
11
|
+
*
|
|
12
|
+
* Constante NO_EVIDENCE_MESSAGE canonica — mesma string em
|
|
13
|
+
* src/query/context_builder.py do Py (gate dfe-rules.md #4).
|
|
14
|
+
* NUNCA duplique este literal.
|
|
15
|
+
*/
|
|
16
|
+
import Database from "better-sqlite3";
|
|
17
|
+
import { homedir, tmpdir } from "node:os";
|
|
18
|
+
import { resolve } from "node:path";
|
|
19
|
+
import { encode, EMBEDDING_MODEL_NAME } from "./embedder.js";
|
|
20
|
+
import { vectorSearch } from "./vectorSearch.js";
|
|
21
|
+
import { ftsSearch } from "./ftsSearch.js";
|
|
22
|
+
import { rrf } from "./hybrid.js";
|
|
23
|
+
import { QueryCache } from "./cache.js";
|
|
24
|
+
import { buildContext, hasSufficientEvidence, } from "./contextBuilder.js";
|
|
25
|
+
import { NO_EVIDENCE_MESSAGE, SUPPORTED_MODES, DEFAULT_TOP_K, MIN_RELEVANCE_SCORE, RECENCY_WEIGHT, RECENCY_HALF_LIFE_DAYS, } from "./constants.js";
|
|
26
|
+
export { NO_EVIDENCE_MESSAGE, SUPPORTED_MODES, DEFAULT_TOP_K };
|
|
27
|
+
/**
|
|
28
|
+
* Resolve path da base RAG. Default: $DFE_AGENT_BASE_DIR ou ~/.dfe-agent.
|
|
29
|
+
*/
|
|
30
|
+
export function resolveBaseDir(override) {
|
|
31
|
+
return override
|
|
32
|
+
?? process.env.DFE_AGENT_BASE_DIR
|
|
33
|
+
?? resolve(process.env.HOME ?? homedir() ?? tmpdir(), ".dfe-agent");
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Recency boost (gate paridade com Py `src/query/query_engine.py:34-36`).
|
|
37
|
+
*
|
|
38
|
+
* Decaimento exponencial: doc recente (now) -> 1.0; doc com idade = half_life
|
|
39
|
+
* -> 0.5; doc 2x half_life -> 0.25; etc. Doc sem published_at -> 0.5 (neutro).
|
|
40
|
+
*/
|
|
41
|
+
function recencyScore(publishedAt, now = Date.now()) {
|
|
42
|
+
if (!publishedAt)
|
|
43
|
+
return 0.5;
|
|
44
|
+
const ts = Date.parse(publishedAt);
|
|
45
|
+
if (Number.isNaN(ts))
|
|
46
|
+
return 0.5;
|
|
47
|
+
const ageDays = (now - ts) / 86_400_000;
|
|
48
|
+
return Math.pow(0.5, ageDays / RECENCY_HALF_LIFE_DAYS);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Combina score (semantico/fts/rrf) com recencia via peso RECENCY_WEIGHT.
|
|
52
|
+
* Espelha Py `_apply_recency_and_topk` em query_engine.py:288-291.
|
|
53
|
+
*/
|
|
54
|
+
function applyRecency(score, publishedAt) {
|
|
55
|
+
const rec = recencyScore(publishedAt);
|
|
56
|
+
return (1 - RECENCY_WEIGHT) * score + RECENCY_WEIGHT * rec;
|
|
57
|
+
}
|
|
58
|
+
function hydrateChunks(handle, hits) {
|
|
59
|
+
if (hits.length === 0)
|
|
60
|
+
return [];
|
|
61
|
+
const ids = hits.map((h) => h.chunk_id);
|
|
62
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
63
|
+
const rows = handle
|
|
64
|
+
.prepare(`SELECT c.id AS chunk_id, c.doc_id, c.text, d.url, d.title, d.published_at
|
|
65
|
+
FROM chunks c
|
|
66
|
+
JOIN documents d ON c.doc_id = d.id
|
|
67
|
+
WHERE c.id IN (${placeholders})`)
|
|
68
|
+
.all(...ids);
|
|
69
|
+
const byId = new Map(rows.map((r) => [r.chunk_id, r]));
|
|
70
|
+
return hits
|
|
71
|
+
.map((h) => {
|
|
72
|
+
const r = byId.get(h.chunk_id);
|
|
73
|
+
if (!r)
|
|
74
|
+
return null;
|
|
75
|
+
return {
|
|
76
|
+
chunk_id: r.chunk_id,
|
|
77
|
+
doc_id: r.doc_id,
|
|
78
|
+
text: r.text,
|
|
79
|
+
url: r.url,
|
|
80
|
+
title: r.title,
|
|
81
|
+
published_at: r.published_at,
|
|
82
|
+
score: h.score,
|
|
83
|
+
};
|
|
84
|
+
})
|
|
85
|
+
.filter((c) => c !== null);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Busca principal. Retorna {answer, sources[]} no contrato canonico.
|
|
89
|
+
*/
|
|
90
|
+
export async function search(question, opts = {}) {
|
|
91
|
+
const mode = (opts.mode ?? "semantic");
|
|
92
|
+
const topK = opts.topK ?? DEFAULT_TOP_K;
|
|
93
|
+
if (!SUPPORTED_MODES.includes(mode)) {
|
|
94
|
+
throw new Error(`unsupported mode: ${mode}. Supported: ${SUPPORTED_MODES.join(", ")}`);
|
|
95
|
+
}
|
|
96
|
+
if (!question.trim()) {
|
|
97
|
+
return { answer: NO_EVIDENCE_MESSAGE, sources: [] };
|
|
98
|
+
}
|
|
99
|
+
const dbPath = resolve(resolveBaseDir(opts.baseDirOverride), "dfe.db");
|
|
100
|
+
const handle = new Database(dbPath, { readonly: true });
|
|
101
|
+
try {
|
|
102
|
+
// 1. Cache check
|
|
103
|
+
const cache = new QueryCache(handle);
|
|
104
|
+
let queryVec = null;
|
|
105
|
+
if (mode !== "fts") {
|
|
106
|
+
queryVec = cache.get(mode, question);
|
|
107
|
+
if (!queryVec) {
|
|
108
|
+
queryVec = await encode(question);
|
|
109
|
+
cache.set(mode, question, queryVec);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
// 2. Buscar chunks (score pre-recency)
|
|
113
|
+
let hits = [];
|
|
114
|
+
if (mode === "semantic") {
|
|
115
|
+
const vecHits = vectorSearch(handle, queryVec, topK);
|
|
116
|
+
// Converter distance (menor = melhor) em score (maior = melhor)
|
|
117
|
+
hits = vecHits.map((h) => ({ ...h, score: 1 - h.distance }));
|
|
118
|
+
}
|
|
119
|
+
else if (mode === "fts") {
|
|
120
|
+
// FTS5 retorna bm25 (menor = mais relevante); normalizamos para score
|
|
121
|
+
// crescente via 1/(1+bm25_normalized). Para paridade com Py, mantemos
|
|
122
|
+
// bm25 relativo (bm25 do chunk / -bm25_max).
|
|
123
|
+
const ftsHits = ftsSearch(handle, question, topK);
|
|
124
|
+
// FTS5 bm25() retorna score NEGATIVO (menor = melhor); invertemos.
|
|
125
|
+
// Exemplo: bm25 = -2.5 (bom) -> score = 2.5; bm25 = -0.5 (ruim) -> score = 0.5.
|
|
126
|
+
hits = ftsHits.map((h) => ({ ...h, score: -h.score }));
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
const vecHits = vectorSearch(handle, queryVec, topK);
|
|
130
|
+
const ftsHits = ftsSearch(handle, question, topK);
|
|
131
|
+
const hybridHits = rrf(vecHits, ftsHits, topK);
|
|
132
|
+
hits = hybridHits.map((h) => ({ ...h, score: h.score }));
|
|
133
|
+
}
|
|
134
|
+
// 3. Hidratar (precisa de published_at para recency)
|
|
135
|
+
const hydrated = hydrateChunks(handle, hits);
|
|
136
|
+
// 4. Aplicar recency boost (gate paridade Py query_engine.py:288-291)
|
|
137
|
+
for (const c of hydrated) {
|
|
138
|
+
c.score = applyRecency(c.score, c.published_at ?? null);
|
|
139
|
+
}
|
|
140
|
+
// 5. Re-ordenar por score final (recency-adjusted)
|
|
141
|
+
hydrated.sort((a, b) => b.score - a.score);
|
|
142
|
+
// 6. Cortar para top-K final
|
|
143
|
+
const topKFinal = hydrated.slice(0, topK);
|
|
144
|
+
// 7. Montar contexto
|
|
145
|
+
return buildContext(topKFinal);
|
|
146
|
+
}
|
|
147
|
+
finally {
|
|
148
|
+
handle.close();
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
export { EMBEDDING_MODEL_NAME, MIN_RELEVANCE_SCORE };
|
|
152
|
+
export { hasSufficientEvidence };
|
|
153
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,MAAM,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAkB,MAAM,mBAAmB,CAAC;AACjE,OAAO,EAAE,SAAS,EAAe,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EACL,YAAY,EACZ,qBAAqB,GAEtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,aAAa,EACb,mBAAmB,EACnB,cAAc,EACd,sBAAsB,GAEvB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC;AAoB/D;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,QAAiB;IAC9C,OAAO,QAAQ;WACV,OAAO,CAAC,GAAG,CAAC,kBAAkB;WAC9B,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,OAAO,EAAE,IAAI,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;AACxE,CAAC;AAWD;;;;;GAKG;AACH,SAAS,YAAY,CAAC,WAA0B,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;IACxE,IAAI,CAAC,WAAW;QAAE,OAAO,GAAG,CAAC;IAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAAE,OAAO,GAAG,CAAC;IACjC,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC;IACxC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,GAAG,sBAAsB,CAAC,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAa,EAAE,WAA0B;IAC7D,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACtC,OAAO,CAAC,CAAC,GAAG,cAAc,CAAC,GAAG,KAAK,GAAG,cAAc,GAAG,GAAG,CAAC;AAC7D,CAAC;AAED,SAAS,aAAa,CACpB,MAAyB,EACzB,IAAgE;IAEhE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,MAAM;SAChB,OAAO,CACN;;;yBAGmB,YAAY,GAAG,CACnC;SACA,GAAG,CAAC,GAAG,GAAG,CAAe,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI;SACR,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACpB,OAAO;YACL,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,KAAK,EAAE,CAAC,CAAC,KAAK;SACE,CAAC;IACrB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAsB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,QAAgB,EAChB,OAAsB,EAAE;IAExB,MAAM,IAAI,GAAe,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAe,CAAC;IACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,aAAa,CAAC;IAExC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,gBAAgB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACrB,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACtD,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAExD,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,QAAQ,GAAwB,IAAI,CAAC;QACzC,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAClC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,IAAI,IAAI,GAA+D,EAAE,CAAC;QAE1E,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACxB,MAAM,OAAO,GAAgB,YAAY,CAAC,MAAM,EAAE,QAAS,EAAE,IAAI,CAAC,CAAC;YACnE,gEAAgE;YAChE,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC/D,CAAC;aAAM,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YAC1B,sEAAsE;YACtE,sEAAsE;YACtE,6CAA6C;YAC7C,MAAM,OAAO,GAAa,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5D,mEAAmE;YACnE,gFAAgF;YAChF,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACzD,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,QAAS,EAAE,IAAI,CAAC,CAAC;YACtD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAC/C,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,qDAAqD;QACrD,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE7C,sEAAsE;QACtE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;QAC1D,CAAC;QAED,mDAAmD;QACnD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAE3C,6BAA6B;QAC7B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAE1C,qBAAqB;QACrB,OAAO,YAAY,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;AACH,CAAC;AAED,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vectorSearch.ts — busca semantica via sqlite-vec (vec_chunks).
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.1
|
|
5
|
+
*
|
|
6
|
+
* Estrategia:
|
|
7
|
+
- embedding da query (384-d, cosine-normalized)
|
|
8
|
+
- MATCH na tabela virtual vec_chunks (cosine distance)
|
|
9
|
+
- ORDER BY distance LIMIT k
|
|
10
|
+
- dedup por doc_id (mantem o chunk mais proximo por documento)
|
|
11
|
+
- boost temporal opcional (nao implementado aqui; ver E.5 contextBuilder)
|
|
12
|
+
*/
|
|
13
|
+
import type { Database as BetterSqlite3Database } from "better-sqlite3";
|
|
14
|
+
export interface VectorHit {
|
|
15
|
+
chunk_id: number;
|
|
16
|
+
doc_id: number;
|
|
17
|
+
distance: number;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Carrega extensao sqlite-vec no handle. Idempotente.
|
|
21
|
+
*/
|
|
22
|
+
export declare function loadVecExtension(handle: BetterSqlite3Database): void;
|
|
23
|
+
/**
|
|
24
|
+
* Busca top-K chunks via cosine distance. Retorna hits ordenados do mais
|
|
25
|
+
* similar (menor distance) ao menos similar.
|
|
26
|
+
*
|
|
27
|
+
* @param handle Better-sqlite3 handle aberto em modo readonly
|
|
28
|
+
* @param query_vec Vetor 384-d da query (cosine-normalized)
|
|
29
|
+
* @param k Numero de hits antes do dedup
|
|
30
|
+
*/
|
|
31
|
+
export declare function vectorSearch(handle: BetterSqlite3Database, query_vec: Float32Array, k: number): VectorHit[];
|
|
32
|
+
//# sourceMappingURL=vectorSearch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vectorSearch.d.ts","sourceRoot":"","sources":["../../src/query/vectorSearch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,QAAQ,IAAI,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAGxE,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,qBAAqB,GAAG,IAAI,CAEpE;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE,qBAAqB,EAC7B,SAAS,EAAE,YAAY,EACvB,CAAC,EAAE,MAAM,GACR,SAAS,EAAE,CA4Bb"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vectorSearch.ts — busca semantica via sqlite-vec (vec_chunks).
|
|
3
|
+
*
|
|
4
|
+
* @see PLAN_SPRINT14.md Task E.1
|
|
5
|
+
*
|
|
6
|
+
* Estrategia:
|
|
7
|
+
- embedding da query (384-d, cosine-normalized)
|
|
8
|
+
- MATCH na tabela virtual vec_chunks (cosine distance)
|
|
9
|
+
- ORDER BY distance LIMIT k
|
|
10
|
+
- dedup por doc_id (mantem o chunk mais proximo por documento)
|
|
11
|
+
- boost temporal opcional (nao implementado aqui; ver E.5 contextBuilder)
|
|
12
|
+
*/
|
|
13
|
+
import * as sqliteVec from "sqlite-vec";
|
|
14
|
+
/**
|
|
15
|
+
* Carrega extensao sqlite-vec no handle. Idempotente.
|
|
16
|
+
*/
|
|
17
|
+
export function loadVecExtension(handle) {
|
|
18
|
+
sqliteVec.load(handle);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Busca top-K chunks via cosine distance. Retorna hits ordenados do mais
|
|
22
|
+
* similar (menor distance) ao menos similar.
|
|
23
|
+
*
|
|
24
|
+
* @param handle Better-sqlite3 handle aberto em modo readonly
|
|
25
|
+
* @param query_vec Vetor 384-d da query (cosine-normalized)
|
|
26
|
+
* @param k Numero de hits antes do dedup
|
|
27
|
+
*/
|
|
28
|
+
export function vectorSearch(handle, query_vec, k) {
|
|
29
|
+
loadVecExtension(handle);
|
|
30
|
+
// sqlite-vec usa serializacao nativa; precisamos passar Buffer
|
|
31
|
+
const vecBuf = Buffer.from(query_vec.buffer);
|
|
32
|
+
const rows = handle
|
|
33
|
+
.prepare(`SELECT chunk_id, doc_id, distance
|
|
34
|
+
FROM vec_chunks
|
|
35
|
+
WHERE embedding MATCH ?
|
|
36
|
+
ORDER BY distance
|
|
37
|
+
LIMIT ?`)
|
|
38
|
+
.all(vecBuf, k);
|
|
39
|
+
// Dedup por doc_id (mantem o primeiro = menor distance)
|
|
40
|
+
const seen = new Set();
|
|
41
|
+
const deduped = [];
|
|
42
|
+
for (const r of rows) {
|
|
43
|
+
if (seen.has(r.doc_id))
|
|
44
|
+
continue;
|
|
45
|
+
seen.add(r.doc_id);
|
|
46
|
+
deduped.push({
|
|
47
|
+
chunk_id: r.chunk_id,
|
|
48
|
+
doc_id: r.doc_id,
|
|
49
|
+
distance: r.distance,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
return deduped;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=vectorSearch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vectorSearch.js","sourceRoot":"","sources":["../../src/query/vectorSearch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AAQxC;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAA6B;IAC5D,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAC1B,MAA6B,EAC7B,SAAuB,EACvB,CAAS;IAET,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAEzB,+DAA+D;IAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM;SAChB,OAAO,CACN;;;;gBAIU,CACX;SACA,GAAG,CAAC,MAAM,EAAE,CAAC,CAAkE,CAAC;IAEnF,wDAAwD;IACxD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC;YACX,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|