@jigyasudham/veto 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -4
- package/dist/cli.js +31 -0
- package/dist/cli.js.map +1 -1
- package/dist/memory/sync.js +2 -2
- package/dist/memory/sync.js.map +1 -1
- package/dist/transcripts/cache.d.ts +66 -0
- package/dist/transcripts/cache.d.ts.map +1 -0
- package/dist/transcripts/cache.js +0 -0
- package/dist/transcripts/cache.js.map +1 -0
- package/dist/transcripts/chunk.d.ts +26 -0
- package/dist/transcripts/chunk.d.ts.map +1 -0
- package/dist/transcripts/chunk.js +54 -0
- package/dist/transcripts/chunk.js.map +1 -0
- package/dist/transcripts/embed.d.ts +58 -0
- package/dist/transcripts/embed.d.ts.map +1 -0
- package/dist/transcripts/embed.js +303 -0
- package/dist/transcripts/embed.js.map +1 -0
- package/dist/transcripts/ingest.d.ts.map +1 -1
- package/dist/transcripts/ingest.js +9 -2
- package/dist/transcripts/ingest.js.map +1 -1
- package/dist/transcripts/manage.d.ts.map +1 -1
- package/dist/transcripts/manage.js +6 -0
- package/dist/transcripts/manage.js.map +1 -1
- package/dist/transcripts/recall.d.ts +7 -0
- package/dist/transcripts/recall.d.ts.map +1 -1
- package/dist/transcripts/recall.js +25 -3
- package/dist/transcripts/recall.js.map +1 -1
- package/dist/transcripts/schema.d.ts +11 -1
- package/dist/transcripts/schema.d.ts.map +1 -1
- package/dist/transcripts/schema.js +33 -1
- package/dist/transcripts/schema.js.map +1 -1
- package/dist/transcripts/search.d.ts +35 -0
- package/dist/transcripts/search.d.ts.map +1 -1
- package/dist/transcripts/search.js +232 -34
- package/dist/transcripts/search.js.map +1 -1
- package/dist/transcripts/vectors.d.ts +40 -0
- package/dist/transcripts/vectors.d.ts.map +1 -0
- package/dist/transcripts/vectors.js +203 -0
- package/dist/transcripts/vectors.js.map +1 -0
- package/package.json +2 -1
- package/server.json +2 -2
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// leftover file is harmless disk, a dangling row is not).
|
|
8
8
|
import { statSync, existsSync, rmSync } from 'node:fs';
|
|
9
9
|
import { getTranscriptsDb, transcriptsDbPath } from './store.js';
|
|
10
|
+
import { resetCaches } from './cache.js';
|
|
10
11
|
import { normalizeProjectDir } from '../memory/local.js';
|
|
11
12
|
import { buildTOC } from './toc.js';
|
|
12
13
|
import { buildFacts } from './pyramid.js';
|
|
@@ -44,8 +45,12 @@ function purgeArchiveRows(archives) {
|
|
|
44
45
|
// binding doesn't dedupe numbered parameters (SQLITE_RANGE on CI).
|
|
45
46
|
res.indexRows += db.prepare(`SELECT COUNT(*) n FROM search_docs WHERE archive_id = ?`).get(a.id).n;
|
|
46
47
|
res.indexRows += db.prepare(`SELECT COUNT(*) n FROM search_postings WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).get(a.id).n;
|
|
48
|
+
// Chunk vectors hang off the same doc ids, so they orphan the same way
|
|
49
|
+
// postings would if this were forgotten (C3 asserts zero orphans).
|
|
50
|
+
res.indexRows += db.prepare(`SELECT COUNT(*) n FROM search_vectors WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).get(a.id).n;
|
|
47
51
|
db.prepare(`DELETE FROM events WHERE archive_id = ?`).run(a.id);
|
|
48
52
|
db.prepare(`DELETE FROM search_postings WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).run(a.id);
|
|
53
|
+
db.prepare(`DELETE FROM search_vectors WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).run(a.id);
|
|
49
54
|
db.prepare(`DELETE FROM search_docs WHERE archive_id = ?`).run(a.id);
|
|
50
55
|
db.prepare(`DELETE FROM archives WHERE id = ?`).run(a.id);
|
|
51
56
|
const m = db.prepare(`DELETE FROM session_map WHERE source = ? AND source_session_id = ?`).run(a.source, a.source_session_id);
|
|
@@ -55,6 +60,7 @@ function purgeArchiveRows(archives) {
|
|
|
55
60
|
paths.push(a.archive_path);
|
|
56
61
|
}
|
|
57
62
|
db.exec('COMMIT');
|
|
63
|
+
resetCaches();
|
|
58
64
|
}
|
|
59
65
|
catch (e) {
|
|
60
66
|
db.exec('ROLLBACK');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manage.js","sourceRoot":"","sources":["../../src/transcripts/manage.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,kFAAkF;AAClF,4EAA4E;AAC5E,0DAA0D;AAE1D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAgB,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,UAAU,EAAqB,MAAM,cAAc,CAAC;AAe7D,SAAS,SAAS,CAAC,CAAa;IAC9B,OAAO;QACL,eAAe,EAAE,CAAC,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW;QACjF,MAAM,EAAE,CAAC,CAAC,mBAAmB,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC,aAAa;QACzF,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAgD,EAAE;IAC7E,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU;QAC1B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,+EAA+E,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC;QAC9I,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrF,OAAQ,IAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC/C,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACpE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2B,CAAC;IACjJ,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AAID,SAAS,gBAAgB,CAAC,QAAsB;IAC9C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACzF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,IAAK,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAC9G,oEAAoE;YACpE,mEAAmE;YACnE,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,yDAAyD,CAC1D,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,0GAA0G,CAC3G,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,OAAO,CACR,+FAA+F,CAChG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACZ,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC9H,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACvC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,CAAC,YAAY;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"manage.js","sourceRoot":"","sources":["../../src/transcripts/manage.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,kFAAkF;AAClF,4EAA4E;AAC5E,0DAA0D;AAE1D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAgB,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,UAAU,EAAqB,MAAM,cAAc,CAAC;AAe7D,SAAS,SAAS,CAAC,CAAa;IAC9B,OAAO;QACL,eAAe,EAAE,CAAC,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW;QACjF,MAAM,EAAE,CAAC,CAAC,mBAAmB,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC,aAAa;QACzF,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAgD,EAAE;IAC7E,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU;QAC1B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,+EAA+E,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC;QAC9I,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrF,OAAQ,IAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC/C,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACpE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2B,CAAC;IACjJ,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AAID,SAAS,gBAAgB,CAAC,QAAsB;IAC9C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACzF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,IAAK,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAC9G,oEAAoE;YACpE,mEAAmE;YACnE,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,yDAAyD,CAC1D,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,0GAA0G,CAC3G,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,uEAAuE;YACvE,mEAAmE;YACnE,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,yGAAyG,CAC1G,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,OAAO,CACR,+FAA+F,CAChG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACZ,EAAE,CAAC,OAAO,CACR,8FAA8F,CAC/F,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACZ,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC9H,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACvC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,CAAC,YAAY;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,WAAW,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC;IACV,CAAC;IACD,gEAAgE;IAChE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YAAC,IAAI,CAAC;gBAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,+BAA+B,CAAC,CAAC;QAAC,CAAC;IACrG,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACrE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2B,CAAC;IACpJ,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpD,0EAA0E;IAC1E,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACxH,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,UAAkB;IAC7C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAiB,CAAC;IACjI,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,EAAkB,CAAC;IAC5E,MAAM,GAAG,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACvC,kDAAkD;IAClD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC,GAAG,EAAE,CAAC;IACtD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAID,MAAM,UAAU,oBAAoB;IAClC,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,EAA8B,CAAC;IAC/H,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QAAC,OAAO,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,6BAA6B,CAAC,CAAC;IAC7F,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,IAAI,CAAC;IAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AAChD,CAAC"}
|
|
@@ -16,6 +16,13 @@ export type RecallQueryResult = {
|
|
|
16
16
|
sourceSessionId?: string;
|
|
17
17
|
archivesIndexed: number;
|
|
18
18
|
};
|
|
19
|
+
/** Which retrieval ran. `semantic` names the model that produced the
|
|
20
|
+
* vectors, or null when recall was keyword-only — so a caller can tell
|
|
21
|
+
* a degraded result from a poor one (condition A3). */
|
|
22
|
+
retrieval: {
|
|
23
|
+
lexical: true;
|
|
24
|
+
semantic: string | null;
|
|
25
|
+
};
|
|
19
26
|
hits: Array<Pick<SearchHit, 'sourceSessionId' | 'seq' | 'kind' | 'snippet' | 'score' | 'eventId' | 'archiveId'>>;
|
|
20
27
|
toc?: Segment[];
|
|
21
28
|
facts?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAWA,OAAO,
|
|
1
|
+
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAWA,OAAO,EAAsB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAGjE,OAAO,EAAY,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAElD,OAAO,EAA4B,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAoB1E,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC;IAClF;;4DAEwD;IACxD,SAAS,EAAE;QAAE,OAAO,EAAE,IAAI,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACtD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC;IACjH,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,KAAK,EAAE,gBAAgB,GAAG,iBAAiB,CAyDtE;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE,kFAAkF;AAClF,wBAAgB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,kBAAkB,CAuBzE"}
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
// The host AI is the reranker/navigator (free, no embeddings). Indexing is lazy:
|
|
9
9
|
// archives in scope are ensured-indexed at query time.
|
|
10
10
|
import { ensureIndexed } from './ingest.js';
|
|
11
|
-
import {
|
|
11
|
+
import { searchEventsHybrid } from './search.js';
|
|
12
|
+
import { embedArchive } from './vectors.js';
|
|
13
|
+
import { embeddingsAvailable, modelProvenance } from './embed.js';
|
|
12
14
|
import { buildTOC } from './toc.js';
|
|
13
15
|
import { buildFacts, renderFacts } from './pyramid.js';
|
|
14
16
|
import { expandEvent, expandRange } from './expand.js';
|
|
@@ -32,7 +34,24 @@ export function recallQuery(input) {
|
|
|
32
34
|
: [];
|
|
33
35
|
for (const a of scopeArchives)
|
|
34
36
|
ensureIndexed(a.id);
|
|
35
|
-
|
|
37
|
+
// Semantic vectors are built here, at the first query that needs them, and
|
|
38
|
+
// never at server boot (condition A2). embedArchive is a no-op when the
|
|
39
|
+
// model package is absent, so this whole block silently costs nothing and
|
|
40
|
+
// recall falls back to BM25 alone.
|
|
41
|
+
for (const a of scopeArchives)
|
|
42
|
+
embedArchive(a.id);
|
|
43
|
+
// Recorded in the response, not just inferred from result quality: the
|
|
44
|
+
// semantic layer degrades silently by design, so the only honest way for a
|
|
45
|
+
// caller to know it ran is to be told which model ran it.
|
|
46
|
+
let semanticModel = null;
|
|
47
|
+
try {
|
|
48
|
+
if (embeddingsAvailable()) {
|
|
49
|
+
const { model_id, revision } = modelProvenance();
|
|
50
|
+
semanticModel = `${model_id}@${revision}`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch { /* optional layer — never fail recall over provenance */ }
|
|
54
|
+
const hits = searchEventsHybrid(input.query, {
|
|
36
55
|
projectDir: input.projectDir,
|
|
37
56
|
sourceSessionId: input.sourceSessionId,
|
|
38
57
|
limit: input.limit ?? 8,
|
|
@@ -51,12 +70,15 @@ export function recallQuery(input) {
|
|
|
51
70
|
ok: true,
|
|
52
71
|
query: input.query,
|
|
53
72
|
scope: { projectDir: input.projectDir, sourceSessionId: input.sourceSessionId, archivesIndexed: scopeArchives.length },
|
|
73
|
+
retrieval: { lexical: true, semantic: semanticModel },
|
|
54
74
|
hits,
|
|
55
75
|
toc,
|
|
56
76
|
facts,
|
|
57
77
|
guidance: hits.length
|
|
58
78
|
? 'Reason over these hits + TOC, then call expand with {eventId} for one line, or {archiveId|sourceSessionId, fromSeq, toSeq} / {sourceSessionId, segmentIndex} for a range. Results are masked.'
|
|
59
|
-
:
|
|
79
|
+
: (semanticModel
|
|
80
|
+
? 'No matches, lexical or semantic. Try different identifiers/terms, or expand a TOC segment directly.'
|
|
81
|
+
: 'No lexical matches, and semantic recall is unavailable on this install. Try different identifiers/terms, or expand a TOC segment directly.'),
|
|
60
82
|
};
|
|
61
83
|
}
|
|
62
84
|
/** Phase 2 — expand a hit/segment/range to exact masked lines with provenance. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recall.js","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,iFAAiF;AACjF,8EAA8E;AAC9E,EAAE;AACF,iFAAiF;AACjF,uDAAuD;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"recall.js","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,iFAAiF;AACjF,8EAA8E;AAC9E,EAAE;AACF,iFAAiF;AACjF,uDAAuD;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAkB,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAgB,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAqB,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzD,MAAM,SAAS,GAAG,sHAAsH,CAAC;AAEzI,SAAS,kBAAkB,CAAC,UAAkB;IAC5C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAO,EAAE,CAAC,OAAO,CACf,2FAA2F,CAC5F,CAAC,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAgD,CAAC;AACxF,CAAC;AAED,SAAS,iBAAiB,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACnE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAQ,EAAE,CAAC,OAAO,CAChB,uFAAuF,CACxF,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2D,IAAI,IAAI,CAAC;AACnG,CAAC;AAwBD,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,KAAuB;IACjD,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe;QACzC,CAAC,CAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAiD;QAC7G,CAAC,CAAC,KAAK,CAAC,UAAU;YAChB,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC;YACtC,CAAC,CAAC,EAAE,CAAC;IAET,KAAK,MAAM,CAAC,IAAI,aAAa;QAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEnD,2EAA2E;IAC3E,wEAAwE;IACxE,0EAA0E;IAC1E,mCAAmC;IACnC,KAAK,MAAM,CAAC,IAAI,aAAa;QAAE,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAElD,uEAAuE;IACvE,2EAA2E;IAC3E,0DAA0D;IAC1D,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,IAAI,mBAAmB,EAAE,EAAE,CAAC;YAC1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,eAAe,EAAE,CAAC;YACjD,aAAa,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAC5C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,wDAAwD,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,KAAK,EAAE;QAC3C,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,CAAC;KACxB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAEhK,+EAA+E;IAC/E,IAAI,GAA0B,CAAC;IAC/B,IAAI,KAAyB,CAAC;IAC9B,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,aAAa,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnG,IAAI,aAAa,EAAE,CAAC;QAClB,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACjC,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI;QACR,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,eAAe,EAAE,aAAa,CAAC,MAAM,EAAE;QACtH,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE;QACrD,IAAI;QACJ,GAAG;QACH,KAAK;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM;YACnB,CAAC,CAAC,+LAA+L;YACjM,CAAC,CAAC,CAAC,aAAa;gBACd,CAAC,CAAC,qGAAqG;gBACvG,CAAC,CAAC,4IAA4I,CAAC;KACpJ,CAAC;AACJ,CAAC;AAaD,kFAAkF;AAClF,MAAM,UAAU,YAAY,CAAC,KAAwB;IACnD,MAAM,IAAI,GAAG,CAAC,CAAe,EAAsB,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;IAExF,IAAI,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAE3D,2DAA2D;IAC3D,IAAI,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;IAChC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QACxC,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,6CAA6C,EAAE,CAAC,CAAC;IAElG,IAAI,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACpD,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACpH,CAAC;IAED,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mDAAmD,EAAE,CAAC,CAAC;AAC1F,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const TRANSCRIPTS_SCHEMA_VERSION =
|
|
1
|
+
export declare const TRANSCRIPTS_SCHEMA_VERSION = 4;
|
|
2
2
|
export declare const EVENT_KINDS: readonly ["user_message", "assistant_message", "tool_call", "tool_result", "reasoning", "meta", "unknown"];
|
|
3
3
|
export type EventKind = (typeof EVENT_KINDS)[number];
|
|
4
4
|
export type Migration = {
|
|
@@ -22,6 +22,16 @@ export type ArchiveRow = {
|
|
|
22
22
|
indexed_through_seq: number;
|
|
23
23
|
captured_at: string;
|
|
24
24
|
updated_at: string;
|
|
25
|
+
/** 0 until chunk vectors exist; gates re-derivation independently of parsing. */
|
|
26
|
+
chunker_version: number;
|
|
27
|
+
/** `model_id@revision` that produced the stored vectors, or null. */
|
|
28
|
+
embed_model: string | null;
|
|
29
|
+
};
|
|
30
|
+
export type SearchVectorRow = {
|
|
31
|
+
doc_id: number;
|
|
32
|
+
chunk_index: number;
|
|
33
|
+
vec: Uint8Array;
|
|
34
|
+
norm: number;
|
|
25
35
|
};
|
|
26
36
|
export type SessionMapRow = {
|
|
27
37
|
source: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/transcripts/schema.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAG5C,eAAO,MAAM,WAAW,4GAEd,CAAC;AACX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD,MAAM,MAAM,SAAS,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAExD,eAAO,MAAM,UAAU,EAAE,SAAS,EA+
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/transcripts/schema.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAG5C,eAAO,MAAM,WAAW,4GAEd,CAAC;AACX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD,MAAM,MAAM,SAAS,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAExD,eAAO,MAAM,UAAU,EAAE,SAAS,EA+JjC,CAAC;AAIF,eAAO,MAAM,gBAAgB,EAAE,SAAS,EAAsE,CAAC;AAE/G,MAAM,MAAM,UAAU,GAAG;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,eAAe,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,UAAU,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,SAAS,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,SAAS,CAAC;IAChB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC"}
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// rather than editing an existing one, so the schema grows without churn and
|
|
8
8
|
// re-open is idempotent.
|
|
9
9
|
// Highest migration version defined below.
|
|
10
|
-
export const TRANSCRIPTS_SCHEMA_VERSION =
|
|
10
|
+
export const TRANSCRIPTS_SCHEMA_VERSION = 4;
|
|
11
11
|
// The common vocabulary every source normalizes into.
|
|
12
12
|
export const EVENT_KINDS = [
|
|
13
13
|
'user_message', 'assistant_message', 'tool_call', 'tool_result', 'reasoning', 'meta', 'unknown',
|
|
@@ -139,6 +139,38 @@ export const MIGRATIONS = [
|
|
|
139
139
|
CREATE INDEX IF NOT EXISTS idx_postings_doc ON search_postings(doc_id);
|
|
140
140
|
`,
|
|
141
141
|
},
|
|
142
|
+
{
|
|
143
|
+
// v4 — chunk vectors for the semantic layer (Phase B).
|
|
144
|
+
//
|
|
145
|
+
// One row per WINDOW, not per event: whole-event vectors fail the
|
|
146
|
+
// paraphrase gate to mean-pooling dilution (spec §11 amendment). Keyed by
|
|
147
|
+
// the same INTEGER doc_id the postings use — storing the 36-char event
|
|
148
|
+
// uuid here would repeat the mistake that made the lexical index 3x too
|
|
149
|
+
// big. Chunks are internal (C6): nothing outside search.ts sees them.
|
|
150
|
+
//
|
|
151
|
+
// No `scale` column: cosine similarity is invariant to a positive
|
|
152
|
+
// per-vector scalar, so the dequantization scale cancels entirely. The
|
|
153
|
+
// stored `norm` is the L2 norm of the int8 vector itself, precomputed so a
|
|
154
|
+
// query is one dot product and one divide.
|
|
155
|
+
//
|
|
156
|
+
// Vectors live in their own lifecycle: `archives.chunker_version` and
|
|
157
|
+
// `archives.embed_model` gate re-derivation independently of
|
|
158
|
+
// parser_version, so ingest never depends on the model being installed and
|
|
159
|
+
// installing it later backfills without re-parsing anything.
|
|
160
|
+
version: 4,
|
|
161
|
+
up: `
|
|
162
|
+
CREATE TABLE IF NOT EXISTS search_vectors (
|
|
163
|
+
doc_id INTEGER NOT NULL,
|
|
164
|
+
chunk_index INTEGER NOT NULL,
|
|
165
|
+
vec BLOB NOT NULL, -- int8, one byte per dimension
|
|
166
|
+
norm REAL NOT NULL, -- L2 norm of the int8 vector
|
|
167
|
+
PRIMARY KEY (doc_id, chunk_index)
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
ALTER TABLE archives ADD COLUMN chunker_version INTEGER NOT NULL DEFAULT 0;
|
|
171
|
+
ALTER TABLE archives ADD COLUMN embed_model TEXT;
|
|
172
|
+
`,
|
|
173
|
+
},
|
|
142
174
|
];
|
|
143
175
|
// Event kinds worth indexing for recall — spine + tool digests. Reasoning
|
|
144
176
|
// (verbose thinking) and meta/unknown are excluded.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/transcripts/schema.ts"],"names":[],"mappings":"AAAA,oFAAoF;AACpF,EAAE;AACF,6EAA6E;AAC7E,8EAA8E;AAC9E,2EAA2E;AAC3E,2EAA2E;AAC3E,6EAA6E;AAC7E,yBAAyB;AAEzB,2CAA2C;AAC3C,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAE5C,sDAAsD;AACtD,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,cAAc,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS;CACvF,CAAC;AAKX,MAAM,CAAC,MAAM,UAAU,GAAgB;IACrC;QACE,6EAA6E;QAC7E,uEAAuE;QACvE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAiCH;KACF;IACD;QACE,2EAA2E;QAC3E,uEAAuE;QACvE,8EAA8E;QAC9E,wEAAwE;QACxE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;KA2BH;KACF;IACD;QACE,uEAAuE;QACvE,mEAAmE;QACnE,wEAAwE;QACxE,uEAAuE;QACvE,mEAAmE;QACnE,EAAE;QACF,0EAA0E;QAC1E,yEAAyE;QACzE,0EAA0E;QAC1E,sEAAsE;QACtE,iEAAiE;QACjE,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAiCH;KACF;CACF,CAAC;AAEF,0EAA0E;AAC1E,oDAAoD;AACpD,MAAM,CAAC,MAAM,gBAAgB,GAAgB,CAAC,cAAc,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/transcripts/schema.ts"],"names":[],"mappings":"AAAA,oFAAoF;AACpF,EAAE;AACF,6EAA6E;AAC7E,8EAA8E;AAC9E,2EAA2E;AAC3E,2EAA2E;AAC3E,6EAA6E;AAC7E,yBAAyB;AAEzB,2CAA2C;AAC3C,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAE5C,sDAAsD;AACtD,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,cAAc,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS;CACvF,CAAC;AAKX,MAAM,CAAC,MAAM,UAAU,GAAgB;IACrC;QACE,6EAA6E;QAC7E,uEAAuE;QACvE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAiCH;KACF;IACD;QACE,2EAA2E;QAC3E,uEAAuE;QACvE,8EAA8E;QAC9E,wEAAwE;QACxE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;KA2BH;KACF;IACD;QACE,uEAAuE;QACvE,mEAAmE;QACnE,wEAAwE;QACxE,uEAAuE;QACvE,mEAAmE;QACnE,EAAE;QACF,0EAA0E;QAC1E,yEAAyE;QACzE,0EAA0E;QAC1E,sEAAsE;QACtE,iEAAiE;QACjE,EAAE;QACF,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAiCH;KACF;IACD;QACE,uDAAuD;QACvD,EAAE;QACF,kEAAkE;QAClE,0EAA0E;QAC1E,uEAAuE;QACvE,wEAAwE;QACxE,sEAAsE;QACtE,EAAE;QACF,kEAAkE;QAClE,uEAAuE;QACvE,2EAA2E;QAC3E,2CAA2C;QAC3C,EAAE;QACF,sEAAsE;QACtE,6DAA6D;QAC7D,2EAA2E;QAC3E,6DAA6D;QAC7D,OAAO,EAAE,CAAC;QACV,EAAE,EAAE;;;;;;;;;;;KAWH;KACF;CACF,CAAC;AAEF,0EAA0E;AAC1E,oDAAoD;AACpD,MAAM,CAAC,MAAM,gBAAgB,GAAgB,CAAC,cAAc,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC"}
|
|
@@ -19,4 +19,39 @@ export type SearchOptions = {
|
|
|
19
19
|
*/
|
|
20
20
|
export declare function buildSnippet(text: string, terms: Set<string>): string;
|
|
21
21
|
export declare function searchEvents(query: string, opts?: SearchOptions): SearchHit[];
|
|
22
|
+
/**
|
|
23
|
+
* One ranker's contribution for a 0-based rank.
|
|
24
|
+
*
|
|
25
|
+
* Exported so the property the fix turns on can be asserted directly rather
|
|
26
|
+
* than through retrieval, where it is not reliably constructible: with a
|
|
27
|
+
* bag-of-words embedding model, lexical overlap implies semantic similarity,
|
|
28
|
+
* so a document that matches a query's words but not its meaning barely
|
|
29
|
+
* exists. The arithmetic, however, is exact.
|
|
30
|
+
*/
|
|
31
|
+
export declare function rrfWeight(rank0: number): number;
|
|
32
|
+
/**
|
|
33
|
+
* Fuse ranked id lists by reciprocal rank. Pure, for testing and reuse.
|
|
34
|
+
*
|
|
35
|
+
* The property that matters: a top-ranked find from ONE ranker must be able
|
|
36
|
+
* to outrank a document both rankers place mid-list. That is the case the
|
|
37
|
+
* semantic layer exists to serve — a strict paraphrase is found by the
|
|
38
|
+
* semantic ranker alone — and it is exactly what k=60 at depth 50 broke.
|
|
39
|
+
*/
|
|
40
|
+
export declare function fuseByRank(lists: string[][]): {
|
|
41
|
+
id: string;
|
|
42
|
+
score: number;
|
|
43
|
+
}[];
|
|
44
|
+
/**
|
|
45
|
+
* BM25 and semantic recall, fused by reciprocal rank.
|
|
46
|
+
*
|
|
47
|
+
* RRF over raw score blending is deliberate: BM25 scores are unbounded and
|
|
48
|
+
* corpus-dependent while cosines sit in [-1, 1], so any weighted sum would be
|
|
49
|
+
* a hidden tuning knob that drifts with corpus size. Ranks have neither
|
|
50
|
+
* problem, and lexical survival is measurable (a keyword hit must not be
|
|
51
|
+
* pushed out of the fused list by semantic noise).
|
|
52
|
+
*
|
|
53
|
+
* Degrades silently to BM25 alone when no vectors are stored or the model is
|
|
54
|
+
* not installed (A2) — the caller cannot tell the difference except by score.
|
|
55
|
+
*/
|
|
56
|
+
export declare function searchEventsHybrid(query: string, opts?: SearchOptions): SearchHit[];
|
|
22
57
|
//# sourceMappingURL=search.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/transcripts/search.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/transcripts/search.ts"],"names":[],"mappings":"AAoBA,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAiBF;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAiCrE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,aAAkB,GAAG,SAAS,EAAE,CA2EjF;AAgDD;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,CAQ7E;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,aAAkB,GAAG,SAAS,EAAE,CA2FvF"}
|
|
@@ -12,6 +12,9 @@
|
|
|
12
12
|
import { getTranscriptsDb } from './store.js';
|
|
13
13
|
import { normalizeProjectDir } from '../memory/local.js';
|
|
14
14
|
import { tokenize } from './tokenize.js';
|
|
15
|
+
import { STRIDE_CHARS, WINDOW_CHARS } from './chunk.js';
|
|
16
|
+
import { searchVectors } from './vectors.js';
|
|
17
|
+
import { getDocCache, scopeMask } from './cache.js';
|
|
15
18
|
const K1 = 1.2;
|
|
16
19
|
const B = 0.75;
|
|
17
20
|
// Terms present in more than half the corpus carry ~no signal but force
|
|
@@ -74,7 +77,8 @@ export function searchEvents(query, opts = {}) {
|
|
|
74
77
|
const terms = [...new Set(tokenize(query))];
|
|
75
78
|
if (terms.length === 0)
|
|
76
79
|
return [];
|
|
77
|
-
const
|
|
80
|
+
const docs = getDocCache();
|
|
81
|
+
const stats = { n: docs.count, avgdl: docs.avgdl };
|
|
78
82
|
if (stats.n === 0)
|
|
79
83
|
return [];
|
|
80
84
|
// Resolve terms against the dictionary in one round trip.
|
|
@@ -88,47 +92,241 @@ export function searchEvents(query, opts = {}) {
|
|
|
88
92
|
let kept = withDf.filter(t => t.df / stats.n <= DF_CAP_RATIO);
|
|
89
93
|
if (kept.length === 0)
|
|
90
94
|
kept = withDf; // every term is that common — keep them all
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
}
|
|
97
|
-
if (opts.sourceSessionId) {
|
|
98
|
-
where.push('d.source_session_id = ?');
|
|
99
|
-
params.push(opts.sourceSessionId);
|
|
100
|
-
}
|
|
101
|
-
const rows = db.prepare(`SELECT p.term_id, p.tf, d.event_id, d.archive_id, d.source_session_id, d.project_dir, d.seq, d.kind, d.len
|
|
102
|
-
FROM search_postings p JOIN search_docs d ON d.id = p.doc_id
|
|
103
|
-
WHERE ${where.join(' AND ')}`).all(...params);
|
|
95
|
+
// Postings only — three narrow columns, no join. Joining search_docs per
|
|
96
|
+
// POSTING to fetch columns needed for the final handful of rows cost 7.1 ms
|
|
97
|
+
// per common term at 50 sessions; this costs 2.6 ms. Scope and length come
|
|
98
|
+
// from the cached metadata instead, and full rows are read for the top-K.
|
|
99
|
+
const rows = db.prepare(`SELECT term_id, doc_id, tf FROM search_postings WHERE term_id IN (${kept.map(() => '?').join(', ')})`).all(...kept.map(t => t.id));
|
|
104
100
|
if (rows.length === 0)
|
|
105
101
|
return [];
|
|
102
|
+
const mask = scopeMask(docs, {
|
|
103
|
+
projectDir: opts.projectDir ? normalizeProjectDir(opts.projectDir) : undefined,
|
|
104
|
+
sourceSessionId: opts.sourceSessionId,
|
|
105
|
+
});
|
|
106
106
|
const dfById = new Map(kept.map(t => [t.id, t.df]));
|
|
107
107
|
const acc = new Map();
|
|
108
108
|
for (const r of rows) {
|
|
109
|
+
if (mask && mask[r.doc_id] !== 1)
|
|
110
|
+
continue;
|
|
109
111
|
const df = dfById.get(r.term_id) ?? 1;
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
if (prev)
|
|
113
|
-
prev.score += inc;
|
|
114
|
-
else
|
|
115
|
-
acc.set(r.event_id, { doc: r, score: inc });
|
|
112
|
+
const inc = bm25(r.tf, df, stats.n, docs.len[r.doc_id] ?? 0, stats.avgdl);
|
|
113
|
+
acc.set(r.doc_id, (acc.get(r.doc_id) ?? 0) + inc);
|
|
116
114
|
}
|
|
115
|
+
if (acc.size === 0)
|
|
116
|
+
return [];
|
|
117
117
|
const limit = Math.min(Math.max(opts.limit ?? 10, 1), 50);
|
|
118
|
-
const top = [...acc.
|
|
119
|
-
//
|
|
120
|
-
const
|
|
121
|
-
const
|
|
122
|
-
const
|
|
118
|
+
const top = [...acc.entries()].sort((a, b) => b[1] - a[1]).slice(0, limit);
|
|
119
|
+
// Metadata and text for the returned rows only.
|
|
120
|
+
const docIds = top.map(([id]) => id);
|
|
121
|
+
const marks2 = docIds.map(() => '?').join(', ');
|
|
122
|
+
const metaRows = db.prepare(`SELECT d.id, d.event_id, d.archive_id, d.source_session_id, d.seq, d.kind, e.text
|
|
123
|
+
FROM search_docs d JOIN events e ON e.id = d.event_id
|
|
124
|
+
WHERE d.id IN (${marks2})`).all(...docIds);
|
|
125
|
+
const metaById = new Map(metaRows.map(r => [r.id, r]));
|
|
123
126
|
const termSet = new Set(kept.map(t => t.term));
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
127
|
+
const out = [];
|
|
128
|
+
for (const [docId, score] of top) {
|
|
129
|
+
const m = metaById.get(docId);
|
|
130
|
+
if (!m)
|
|
131
|
+
continue;
|
|
132
|
+
out.push({
|
|
133
|
+
eventId: m.event_id,
|
|
134
|
+
archiveId: m.archive_id,
|
|
135
|
+
sourceSessionId: m.source_session_id,
|
|
136
|
+
seq: m.seq,
|
|
137
|
+
kind: m.kind,
|
|
138
|
+
snippet: buildSnippet(m.text ?? '', termSet),
|
|
139
|
+
score,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
// --- Hybrid retrieval -------------------------------------------------------
|
|
145
|
+
// How deep each ranker is consulted before fusing. Deeper than the returned
|
|
146
|
+
// limit so a result ranked mid-list by one ranker can still be lifted by the
|
|
147
|
+
// other — the entire mechanism by which a paraphrase beats keyword overlap.
|
|
148
|
+
const CANDIDATE_DEPTH = 50;
|
|
149
|
+
/**
|
|
150
|
+
* Reciprocal-rank fusion constant.
|
|
151
|
+
*
|
|
152
|
+
* NOT 60. That value comes from the RRF paper's TREC runs over ~1000-document
|
|
153
|
+
* lists; at our candidate depth it is actively harmful. With k=60 and d=50 the
|
|
154
|
+
* whole ranking compresses into a 1.8x spread (rank 1 = 0.0164, rank 50 =
|
|
155
|
+
* 0.0091), which destroys rank information and lets agreement dominate
|
|
156
|
+
* quality: two mid-list agreements (0.011 + 0.011) outscore one top-ranked
|
|
157
|
+
* single-ranker find (0.0164).
|
|
158
|
+
*
|
|
159
|
+
* That is fatal for the case this layer exists to serve. A strict paraphrase
|
|
160
|
+
* shares no vocabulary with its target, so the target is found by the semantic
|
|
161
|
+
* ranker ALONE and earns exactly one contribution — while any keyword-adjacent
|
|
162
|
+
* event earns a comparable one for free. The semantic discovery loses to noise.
|
|
163
|
+
*
|
|
164
|
+
* Requirement: a rank-1 hit from a single ranker must be able to outrank a pair
|
|
165
|
+
* of mid-list agreements. 1/(k+1) > 2/(k+d/2) solves to k < d/2 - 2, i.e.
|
|
166
|
+
* k < 23 at d=50. 10 sits comfortably inside that bound and keeps the top of
|
|
167
|
+
* the list sharply separated (rank 1 = 0.091, rank 10 = 0.050, rank 50 = 0.017).
|
|
168
|
+
*/
|
|
169
|
+
const RRF_K = 10;
|
|
170
|
+
/**
|
|
171
|
+
* BM25 contributes only candidates it is actually confident about: anything
|
|
172
|
+
* below this fraction of its own top score is tail noise that would otherwise
|
|
173
|
+
* occupy fusion slots and crowd out semantic finds.
|
|
174
|
+
*
|
|
175
|
+
* Applied to BM25 only, deliberately. The two rankers have different score
|
|
176
|
+
* geometries: BM25 decays fast and spans orders of magnitude, so relative
|
|
177
|
+
* score is meaningful. Cosines sit in a narrow band — measured on the real
|
|
178
|
+
* corpus, off-topic queries reach 0.40 while on-topic start at 0.45 — so a
|
|
179
|
+
* relative floor there would be noise itself. Depth is the semantic side's
|
|
180
|
+
* only honest control.
|
|
181
|
+
*
|
|
182
|
+
* Only the tail is affected, so a strong keyword hit can never be dropped and
|
|
183
|
+
* lexical survival is preserved by construction.
|
|
184
|
+
*/
|
|
185
|
+
const BM25_RELATIVE_FLOOR = 0.25;
|
|
186
|
+
/**
|
|
187
|
+
* One ranker's contribution for a 0-based rank.
|
|
188
|
+
*
|
|
189
|
+
* Exported so the property the fix turns on can be asserted directly rather
|
|
190
|
+
* than through retrieval, where it is not reliably constructible: with a
|
|
191
|
+
* bag-of-words embedding model, lexical overlap implies semantic similarity,
|
|
192
|
+
* so a document that matches a query's words but not its meaning barely
|
|
193
|
+
* exists. The arithmetic, however, is exact.
|
|
194
|
+
*/
|
|
195
|
+
export function rrfWeight(rank0) {
|
|
196
|
+
return 1 / (RRF_K + rank0 + 1);
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Fuse ranked id lists by reciprocal rank. Pure, for testing and reuse.
|
|
200
|
+
*
|
|
201
|
+
* The property that matters: a top-ranked find from ONE ranker must be able
|
|
202
|
+
* to outrank a document both rankers place mid-list. That is the case the
|
|
203
|
+
* semantic layer exists to serve — a strict paraphrase is found by the
|
|
204
|
+
* semantic ranker alone — and it is exactly what k=60 at depth 50 broke.
|
|
205
|
+
*/
|
|
206
|
+
export function fuseByRank(lists) {
|
|
207
|
+
const acc = new Map();
|
|
208
|
+
for (const list of lists) {
|
|
209
|
+
list.forEach((id, i) => acc.set(id, (acc.get(id) ?? 0) + rrfWeight(i)));
|
|
210
|
+
}
|
|
211
|
+
return [...acc.entries()]
|
|
212
|
+
.map(([id, score]) => ({ id, score }))
|
|
213
|
+
.sort((a, b) => b.score - a.score);
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* BM25 and semantic recall, fused by reciprocal rank.
|
|
217
|
+
*
|
|
218
|
+
* RRF over raw score blending is deliberate: BM25 scores are unbounded and
|
|
219
|
+
* corpus-dependent while cosines sit in [-1, 1], so any weighted sum would be
|
|
220
|
+
* a hidden tuning knob that drifts with corpus size. Ranks have neither
|
|
221
|
+
* problem, and lexical survival is measurable (a keyword hit must not be
|
|
222
|
+
* pushed out of the fused list by semantic noise).
|
|
223
|
+
*
|
|
224
|
+
* Degrades silently to BM25 alone when no vectors are stored or the model is
|
|
225
|
+
* not installed (A2) — the caller cannot tell the difference except by score.
|
|
226
|
+
*/
|
|
227
|
+
export function searchEventsHybrid(query, opts = {}) {
|
|
228
|
+
const limit = Math.min(Math.max(opts.limit ?? 10, 1), 50);
|
|
229
|
+
const lexicalAll = searchEvents(query, { ...opts, limit: CANDIDATE_DEPTH });
|
|
230
|
+
const lexFloor = (lexicalAll[0]?.score ?? 0) * BM25_RELATIVE_FLOOR;
|
|
231
|
+
const lexical = lexicalAll.filter(h => h.score >= lexFloor);
|
|
232
|
+
// Cosine ranking has no notion of "no match" — it always returns its nearest
|
|
233
|
+
// neighbours, however far away they are. A similarity floor cannot fix that
|
|
234
|
+
// here: measured on the real corpus, off-topic queries reach 0.40 while true
|
|
235
|
+
// strict paraphrases go as low as 0.21, so the distributions overlap and any
|
|
236
|
+
// threshold that suppresses noise also cuts the recall this layer exists for.
|
|
237
|
+
//
|
|
238
|
+
// What DOES separate them is whether the query has any footing in the corpus
|
|
239
|
+
// at all. A paraphrase shares no words with its target but still speaks the
|
|
240
|
+
// corpus's language; `zzz-nonexistent-term` shares nothing with anything. So
|
|
241
|
+
// the semantic layer is allowed to generalize, never to extrapolate from a
|
|
242
|
+
// query the corpus has no word of — and "nothing matched" stays expressible.
|
|
243
|
+
// Unfiltered on the fallback paths: the relative floor exists to stop tail
|
|
244
|
+
// noise occupying FUSION slots. With no fusion happening it would just be
|
|
245
|
+
// hiding results the lexical ranker legitimately found.
|
|
246
|
+
if (!queryHasCorpusTerms(query))
|
|
247
|
+
return lexicalAll.slice(0, limit);
|
|
248
|
+
let semantic = [];
|
|
249
|
+
try {
|
|
250
|
+
semantic = searchVectors(query, {
|
|
251
|
+
projectDir: opts.projectDir ? normalizeProjectDir(opts.projectDir) : undefined,
|
|
252
|
+
sourceSessionId: opts.sourceSessionId,
|
|
253
|
+
limit: CANDIDATE_DEPTH,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
semantic = []; // a broken model must not take the lexical path down with it
|
|
258
|
+
}
|
|
259
|
+
if (semantic.length === 0)
|
|
260
|
+
return lexicalAll.slice(0, limit);
|
|
261
|
+
const db = getTranscriptsDb();
|
|
262
|
+
// Resolve the semantic hits' doc ids to event metadata in one round trip.
|
|
263
|
+
const docIds = semantic.map(s => s.docId);
|
|
264
|
+
const docRows = db.prepare(`SELECT id, event_id, archive_id, source_session_id, seq, kind
|
|
265
|
+
FROM search_docs WHERE id IN (${docIds.map(() => '?').join(', ')})`).all(...docIds);
|
|
266
|
+
const docById = new Map(docRows.map(r => [r.id, r]));
|
|
267
|
+
const fused = new Map();
|
|
268
|
+
lexical.forEach((hit, i) => {
|
|
269
|
+
fused.set(hit.eventId, { hit, rrf: rrfWeight(i) });
|
|
270
|
+
});
|
|
271
|
+
semantic.forEach((s, i) => {
|
|
272
|
+
const doc = docById.get(s.docId);
|
|
273
|
+
if (!doc)
|
|
274
|
+
return;
|
|
275
|
+
const prev = fused.get(doc.event_id);
|
|
276
|
+
const inc = rrfWeight(i);
|
|
277
|
+
if (prev) {
|
|
278
|
+
prev.rrf += inc;
|
|
279
|
+
prev.chunkIndex = s.chunkIndex;
|
|
280
|
+
}
|
|
281
|
+
else
|
|
282
|
+
fused.set(doc.event_id, { doc, chunkIndex: s.chunkIndex, rrf: inc });
|
|
283
|
+
});
|
|
284
|
+
const ranked = [...fused.entries()]
|
|
285
|
+
.sort((a, b) => b[1].rrf - a[1].rrf)
|
|
286
|
+
.slice(0, limit);
|
|
287
|
+
// Text is needed only for entries that arrived semantically — lexical hits
|
|
288
|
+
// already carry a snippet built from their matched terms.
|
|
289
|
+
const needText = ranked.filter(([, e]) => !e.hit).map(([eventId]) => eventId);
|
|
290
|
+
const textById = new Map();
|
|
291
|
+
if (needText.length > 0) {
|
|
292
|
+
const rows = db.prepare(`SELECT id, text FROM events WHERE id IN (${needText.map(() => '?').join(', ')})`).all(...needText);
|
|
293
|
+
for (const r of rows)
|
|
294
|
+
textById.set(r.id, r.text ?? '');
|
|
295
|
+
}
|
|
296
|
+
const terms = new Set(tokenize(query));
|
|
297
|
+
return ranked.map(([eventId, e]) => {
|
|
298
|
+
if (e.hit)
|
|
299
|
+
return { ...e.hit, score: e.rrf };
|
|
300
|
+
const doc = e.doc;
|
|
301
|
+
const text = textById.get(eventId) ?? '';
|
|
302
|
+
return {
|
|
303
|
+
eventId,
|
|
304
|
+
archiveId: doc.archive_id,
|
|
305
|
+
sourceSessionId: doc.source_session_id,
|
|
306
|
+
seq: doc.seq,
|
|
307
|
+
kind: doc.kind,
|
|
308
|
+
// Aim the excerpt at the window that actually matched, so a long event
|
|
309
|
+
// does not open on an unrelated first paragraph (C6's opportunity).
|
|
310
|
+
snippet: buildSnippet(windowAround(text, e.chunkIndex ?? 0), terms),
|
|
311
|
+
score: e.rrf,
|
|
312
|
+
};
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
/** True when at least one query token appears in the corpus dictionary. */
|
|
316
|
+
function queryHasCorpusTerms(query) {
|
|
317
|
+
const terms = [...new Set(tokenize(query))];
|
|
318
|
+
if (terms.length === 0)
|
|
319
|
+
return false;
|
|
320
|
+
const db = getTranscriptsDb();
|
|
321
|
+
const hit = db.prepare(`SELECT 1 AS present FROM search_terms WHERE term IN (${terms.map(() => '?').join(', ')}) LIMIT 1`).get(...terms);
|
|
322
|
+
return hit !== undefined;
|
|
323
|
+
}
|
|
324
|
+
/** The slice of `text` the winning chunk covered, widened by one stride each
|
|
325
|
+
* way so the snippet has context to work with. */
|
|
326
|
+
function windowAround(text, chunkIndex) {
|
|
327
|
+
if (text.length <= WINDOW_CHARS)
|
|
328
|
+
return text;
|
|
329
|
+
const start = Math.max(0, chunkIndex * STRIDE_CHARS - STRIDE_CHARS);
|
|
330
|
+
return text.slice(start, start + WINDOW_CHARS + STRIDE_CHARS * 2);
|
|
133
331
|
}
|
|
134
332
|
//# sourceMappingURL=search.js.map
|