@jigyasudham/veto 3.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +630 -589
- package/dist/agents/llm-runner.js +57 -57
- package/dist/cli/statusline.js +2 -2
- package/dist/cli.js +57 -26
- package/dist/cli.js.map +1 -1
- package/dist/memory/decisions.js +1 -1
- package/dist/memory/local.js +144 -144
- package/dist/memory/schema.js +228 -228
- package/dist/memory/sync.js +25 -25
- package/dist/router/implicit-outcomes.js +4 -4
- package/dist/router/learning-updater.js +45 -45
- package/dist/server.js +1 -1
- package/dist/tools/compact.js +3 -3
- package/dist/transcripts/archive.js +5 -5
- package/dist/transcripts/cache.d.ts +66 -0
- package/dist/transcripts/cache.d.ts.map +1 -0
- package/dist/transcripts/cache.js +0 -0
- package/dist/transcripts/cache.js.map +1 -0
- package/dist/transcripts/chunk.d.ts +26 -0
- package/dist/transcripts/chunk.d.ts.map +1 -0
- package/dist/transcripts/chunk.js +54 -0
- package/dist/transcripts/chunk.js.map +1 -0
- package/dist/transcripts/embed.d.ts +58 -0
- package/dist/transcripts/embed.d.ts.map +1 -0
- package/dist/transcripts/embed.js +303 -0
- package/dist/transcripts/embed.js.map +1 -0
- package/dist/transcripts/ingest.d.ts.map +1 -1
- package/dist/transcripts/ingest.js +14 -7
- package/dist/transcripts/ingest.js.map +1 -1
- package/dist/transcripts/manage.d.ts.map +1 -1
- package/dist/transcripts/manage.js +6 -0
- package/dist/transcripts/manage.js.map +1 -1
- package/dist/transcripts/mapping.js +5 -5
- package/dist/transcripts/pyramid.js +2 -2
- package/dist/transcripts/recall.d.ts +7 -0
- package/dist/transcripts/recall.d.ts.map +1 -1
- package/dist/transcripts/recall.js +25 -3
- package/dist/transcripts/recall.js.map +1 -1
- package/dist/transcripts/schema.d.ts +11 -1
- package/dist/transcripts/schema.d.ts.map +1 -1
- package/dist/transcripts/schema.js +126 -94
- package/dist/transcripts/schema.js.map +1 -1
- package/dist/transcripts/search.d.ts +35 -0
- package/dist/transcripts/search.d.ts.map +1 -1
- package/dist/transcripts/search.js +232 -34
- package/dist/transcripts/search.js.map +1 -1
- package/dist/transcripts/vectors.d.ts +40 -0
- package/dist/transcripts/vectors.d.ts.map +1 -0
- package/dist/transcripts/vectors.js +203 -0
- package/dist/transcripts/vectors.js.map +1 -0
- package/package.json +70 -61
- package/server.json +23 -23
- package/dist/skills/development/skill-api-design.d.ts +0 -15
- package/dist/skills/development/skill-api-design.d.ts.map +0 -1
- package/dist/skills/development/skill-api-design.js +0 -313
- package/dist/skills/development/skill-api-design.js.map +0 -1
- package/dist/skills/development/skill-auth.d.ts +0 -15
- package/dist/skills/development/skill-auth.d.ts.map +0 -1
- package/dist/skills/development/skill-auth.js +0 -255
- package/dist/skills/development/skill-auth.js.map +0 -1
- package/dist/skills/development/skill-ci-cd.d.ts +0 -2
- package/dist/skills/development/skill-ci-cd.d.ts.map +0 -1
- package/dist/skills/development/skill-ci-cd.js +0 -2
- package/dist/skills/development/skill-ci-cd.js.map +0 -1
- package/dist/skills/development/skill-crud.d.ts +0 -15
- package/dist/skills/development/skill-crud.d.ts.map +0 -1
- package/dist/skills/development/skill-crud.js +0 -193
- package/dist/skills/development/skill-crud.js.map +0 -1
- package/dist/skills/development/skill-db-schema.d.ts +0 -2
- package/dist/skills/development/skill-db-schema.d.ts.map +0 -1
- package/dist/skills/development/skill-db-schema.js +0 -2
- package/dist/skills/development/skill-db-schema.js.map +0 -1
- package/dist/skills/development/skill-docker.d.ts +0 -2
- package/dist/skills/development/skill-docker.d.ts.map +0 -1
- package/dist/skills/development/skill-docker.js +0 -2
- package/dist/skills/development/skill-docker.js.map +0 -1
- package/dist/skills/development/skill-env-setup.d.ts +0 -2
- package/dist/skills/development/skill-env-setup.d.ts.map +0 -1
- package/dist/skills/development/skill-env-setup.js +0 -2
- package/dist/skills/development/skill-env-setup.js.map +0 -1
- package/dist/skills/development/skill-scaffold.d.ts +0 -15
- package/dist/skills/development/skill-scaffold.d.ts.map +0 -1
- package/dist/skills/development/skill-scaffold.js +0 -299
- package/dist/skills/development/skill-scaffold.js.map +0 -1
- package/dist/skills/intelligence/skill-complexity-score.d.ts +0 -15
- package/dist/skills/intelligence/skill-complexity-score.d.ts.map +0 -1
- package/dist/skills/intelligence/skill-complexity-score.js +0 -66
- package/dist/skills/intelligence/skill-complexity-score.js.map +0 -1
- package/dist/skills/intelligence/skill-cost-track.d.ts +0 -14
- package/dist/skills/intelligence/skill-cost-track.d.ts.map +0 -1
- package/dist/skills/intelligence/skill-cost-track.js +0 -36
- package/dist/skills/intelligence/skill-cost-track.js.map +0 -1
- package/dist/skills/intelligence/skill-learning-loop.d.ts +0 -15
- package/dist/skills/intelligence/skill-learning-loop.d.ts.map +0 -1
- package/dist/skills/intelligence/skill-learning-loop.js +0 -66
- package/dist/skills/intelligence/skill-learning-loop.js.map +0 -1
- package/dist/skills/intelligence/skill-pattern-detect.d.ts +0 -14
- package/dist/skills/intelligence/skill-pattern-detect.d.ts.map +0 -1
- package/dist/skills/intelligence/skill-pattern-detect.js +0 -35
- package/dist/skills/intelligence/skill-pattern-detect.js.map +0 -1
- package/dist/skills/intelligence/skill-rate-watch.d.ts +0 -15
- package/dist/skills/intelligence/skill-rate-watch.d.ts.map +0 -1
- package/dist/skills/intelligence/skill-rate-watch.js +0 -58
- package/dist/skills/intelligence/skill-rate-watch.js.map +0 -1
- package/dist/skills/memory/skill-context-compress.d.ts +0 -15
- package/dist/skills/memory/skill-context-compress.d.ts.map +0 -1
- package/dist/skills/memory/skill-context-compress.js +0 -82
- package/dist/skills/memory/skill-context-compress.js.map +0 -1
- package/dist/skills/memory/skill-cross-sync.d.ts +0 -15
- package/dist/skills/memory/skill-cross-sync.d.ts.map +0 -1
- package/dist/skills/memory/skill-cross-sync.js +0 -88
- package/dist/skills/memory/skill-cross-sync.js.map +0 -1
- package/dist/skills/memory/skill-decision-log.d.ts +0 -15
- package/dist/skills/memory/skill-decision-log.d.ts.map +0 -1
- package/dist/skills/memory/skill-decision-log.js +0 -103
- package/dist/skills/memory/skill-decision-log.js.map +0 -1
- package/dist/skills/memory/skill-session-restore.d.ts +0 -15
- package/dist/skills/memory/skill-session-restore.d.ts.map +0 -1
- package/dist/skills/memory/skill-session-restore.js +0 -39
- package/dist/skills/memory/skill-session-restore.js.map +0 -1
- package/dist/skills/memory/skill-session-save.d.ts +0 -15
- package/dist/skills/memory/skill-session-save.d.ts.map +0 -1
- package/dist/skills/memory/skill-session-save.js +0 -74
- package/dist/skills/memory/skill-session-save.js.map +0 -1
- package/dist/skills/quality/skill-accessibility.d.ts +0 -2
- package/dist/skills/quality/skill-accessibility.d.ts.map +0 -1
- package/dist/skills/quality/skill-accessibility.js +0 -2
- package/dist/skills/quality/skill-accessibility.js.map +0 -1
- package/dist/skills/quality/skill-code-review.d.ts +0 -15
- package/dist/skills/quality/skill-code-review.d.ts.map +0 -1
- package/dist/skills/quality/skill-code-review.js +0 -60
- package/dist/skills/quality/skill-code-review.js.map +0 -1
- package/dist/skills/quality/skill-docs-gen.d.ts +0 -2
- package/dist/skills/quality/skill-docs-gen.d.ts.map +0 -1
- package/dist/skills/quality/skill-docs-gen.js +0 -2
- package/dist/skills/quality/skill-docs-gen.js.map +0 -1
- package/dist/skills/quality/skill-perf-audit.d.ts +0 -2
- package/dist/skills/quality/skill-perf-audit.d.ts.map +0 -1
- package/dist/skills/quality/skill-perf-audit.js +0 -2
- package/dist/skills/quality/skill-perf-audit.js.map +0 -1
- package/dist/skills/quality/skill-security-scan.d.ts +0 -15
- package/dist/skills/quality/skill-security-scan.d.ts.map +0 -1
- package/dist/skills/quality/skill-security-scan.js +0 -67
- package/dist/skills/quality/skill-security-scan.js.map +0 -1
- package/dist/skills/quality/skill-test-suite.d.ts +0 -15
- package/dist/skills/quality/skill-test-suite.d.ts.map +0 -1
- package/dist/skills/quality/skill-test-suite.js +0 -274
- package/dist/skills/quality/skill-test-suite.js.map +0 -1
- package/dist/skills/workflow/skill-deploy.d.ts +0 -2
- package/dist/skills/workflow/skill-deploy.d.ts.map +0 -1
- package/dist/skills/workflow/skill-deploy.js +0 -2
- package/dist/skills/workflow/skill-deploy.js.map +0 -1
- package/dist/skills/workflow/skill-git-workflow.d.ts +0 -2
- package/dist/skills/workflow/skill-git-workflow.d.ts.map +0 -1
- package/dist/skills/workflow/skill-git-workflow.js +0 -2
- package/dist/skills/workflow/skill-git-workflow.js.map +0 -1
- package/dist/skills/workflow/skill-rollback.d.ts +0 -2
- package/dist/skills/workflow/skill-rollback.d.ts.map +0 -1
- package/dist/skills/workflow/skill-rollback.js +0 -2
- package/dist/skills/workflow/skill-rollback.js.map +0 -1
- package/dist/skills/workflow/skill-task-breakdown.d.ts +0 -2
- package/dist/skills/workflow/skill-task-breakdown.d.ts.map +0 -1
- package/dist/skills/workflow/skill-task-breakdown.js +0 -2
- package/dist/skills/workflow/skill-task-breakdown.js.map +0 -1
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
// Pure-JS Model2Vec inference — the semantic half of transcript recall.
|
|
2
|
+
//
|
|
3
|
+
// The model is frozen DATA, not an API: a quantized matrix plus a vocabulary.
|
|
4
|
+
// So this is a reimplementation against a pinned contract, not a translation
|
|
5
|
+
// of upstream code. The contract lives in the model package's `model.json`
|
|
6
|
+
// and every assumption below is asserted at load time, because the failure
|
|
7
|
+
// mode we care about is not a crash — it is plausible-looking wrong vectors.
|
|
8
|
+
//
|
|
9
|
+
// Correctness is pinned by tests/transcripts/embed.test.ts, which replays the
|
|
10
|
+
// golden vectors shipped alongside the weights (fixed text -> expected
|
|
11
|
+
// output, generated by the reference Python implementation in
|
|
12
|
+
// scripts/repack-model.py). If this file drifts, those fail.
|
|
13
|
+
//
|
|
14
|
+
// Zero dependencies, and nothing here runs at import: the model is resolved
|
|
15
|
+
// on first use and never at server boot (condition A2), so a missing or
|
|
16
|
+
// corrupt copy degrades search to BM25 instead of taking Veto down.
|
|
17
|
+
import { readFileSync } from 'node:fs';
|
|
18
|
+
import { createRequire } from 'node:module';
|
|
19
|
+
import { dirname, join } from 'node:path';
|
|
20
|
+
const _require = createRequire(import.meta.url);
|
|
21
|
+
/** Dim of the shipped table. Cross-checked against the header on load. */
|
|
22
|
+
const EXPECTED_FORMAT = 'veto-model/1';
|
|
23
|
+
// model2vec normalizes with this epsilon; matching it keeps golden vectors
|
|
24
|
+
// bit-comparable for zero-norm inputs instead of producing NaN.
|
|
25
|
+
const NORM_EPSILON = 1e-32;
|
|
26
|
+
let _model;
|
|
27
|
+
/** Package root holding the payload. Env override exists for tests and for
|
|
28
|
+
* local work before the model package is published. */
|
|
29
|
+
function modelDir() {
|
|
30
|
+
if (process.env.VETO_MODEL_DIR)
|
|
31
|
+
return join(process.env.VETO_MODEL_DIR, 'model');
|
|
32
|
+
// The package's exports map points at the header; the payload sits beside it.
|
|
33
|
+
return dirname(_require.resolve('@jigyasudham/veto-model/model.json'));
|
|
34
|
+
}
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
// BertNormalizer
|
|
37
|
+
//
|
|
38
|
+
// Order is upstream's and matters: clean -> CJK spacing -> accents -> case.
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// Upstream treats tab/newline/CR as text, not control chars, and strips the
|
|
41
|
+
// rest of the C categories along with NUL and the replacement char.
|
|
42
|
+
const CONTROL = /\p{C}/u;
|
|
43
|
+
const PUNCT = /[!-/:-@[-`{-~]|\p{P}/u;
|
|
44
|
+
function isControl(ch) {
|
|
45
|
+
if (ch === '\t' || ch === '\n' || ch === '\r')
|
|
46
|
+
return false;
|
|
47
|
+
return CONTROL.test(ch);
|
|
48
|
+
}
|
|
49
|
+
function isWhitespace(ch) {
|
|
50
|
+
return ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r' || /\s/u.test(ch);
|
|
51
|
+
}
|
|
52
|
+
// The CJK blocks BERT spaces out so every ideograph becomes its own word.
|
|
53
|
+
// Hiragana/katakana are deliberately absent upstream — they tokenize as runs.
|
|
54
|
+
function isChinese(cp) {
|
|
55
|
+
return ((cp >= 0x4e00 && cp <= 0x9fff) ||
|
|
56
|
+
(cp >= 0x3400 && cp <= 0x4dbf) ||
|
|
57
|
+
(cp >= 0x20000 && cp <= 0x2a6df) ||
|
|
58
|
+
(cp >= 0x2a700 && cp <= 0x2b73f) ||
|
|
59
|
+
(cp >= 0x2b740 && cp <= 0x2b81f) ||
|
|
60
|
+
(cp >= 0x2b820 && cp <= 0x2ceaf) ||
|
|
61
|
+
(cp >= 0xf900 && cp <= 0xfaff) ||
|
|
62
|
+
(cp >= 0x2f800 && cp <= 0x2fa1f));
|
|
63
|
+
}
|
|
64
|
+
function normalize(text, m) {
|
|
65
|
+
let out = text;
|
|
66
|
+
if (m.cleanText) {
|
|
67
|
+
let buf = '';
|
|
68
|
+
for (const ch of out) {
|
|
69
|
+
// NUL and U+FFFD as escapes, matching upstream's filter. Literals
|
|
70
|
+
// here embed raw control bytes and make tools treat this file as binary.
|
|
71
|
+
if (ch === '\u0000' || ch === '\uFFFD' || isControl(ch))
|
|
72
|
+
continue;
|
|
73
|
+
buf += isWhitespace(ch) ? ' ' : ch;
|
|
74
|
+
}
|
|
75
|
+
out = buf;
|
|
76
|
+
}
|
|
77
|
+
if (m.handleChineseChars) {
|
|
78
|
+
let buf = '';
|
|
79
|
+
for (const ch of out) {
|
|
80
|
+
const cp = ch.codePointAt(0);
|
|
81
|
+
buf += isChinese(cp) ? ` ${ch} ` : ch;
|
|
82
|
+
}
|
|
83
|
+
out = buf;
|
|
84
|
+
}
|
|
85
|
+
// `strip_accents: null` in tokenizer.json does NOT mean "off" — upstream
|
|
86
|
+
// resolves it to the value of `lowercase`. Reading the null literally is
|
|
87
|
+
// the silent-divergence bug this whole file is most likely to have; it only
|
|
88
|
+
// shows on accented text, so the golden set carries "café naïve résumé".
|
|
89
|
+
if (m.stripAccents)
|
|
90
|
+
out = out.normalize('NFD').replace(/\p{Mn}/gu, '');
|
|
91
|
+
if (m.lowercase)
|
|
92
|
+
out = out.toLowerCase();
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
// BertPreTokenizer — split on whitespace, then isolate punctuation as its own
|
|
97
|
+
// word. "Isolated" means the punctuation is KEPT as a token, not discarded.
|
|
98
|
+
// ---------------------------------------------------------------------------
|
|
99
|
+
function preTokenize(text) {
|
|
100
|
+
const words = [];
|
|
101
|
+
for (const chunk of text.split(/\s+/u)) {
|
|
102
|
+
if (!chunk)
|
|
103
|
+
continue;
|
|
104
|
+
let cur = '';
|
|
105
|
+
for (const ch of chunk) {
|
|
106
|
+
if (PUNCT.test(ch)) {
|
|
107
|
+
if (cur) {
|
|
108
|
+
words.push(cur);
|
|
109
|
+
cur = '';
|
|
110
|
+
}
|
|
111
|
+
words.push(ch);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
cur += ch;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (cur)
|
|
118
|
+
words.push(cur);
|
|
119
|
+
}
|
|
120
|
+
return words;
|
|
121
|
+
}
|
|
122
|
+
// ---------------------------------------------------------------------------
|
|
123
|
+
// WordPiece — greedy longest-match-first, continuation pieces carry "##".
|
|
124
|
+
// Operates on code points, not UTF-16 units, so astral characters (emoji)
|
|
125
|
+
// are matched and length-capped as single characters like upstream does.
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
function wordPiece(word, m, out) {
|
|
128
|
+
const chars = Array.from(word);
|
|
129
|
+
if (chars.length > m.maxWordChars) {
|
|
130
|
+
if (m.unkId !== undefined)
|
|
131
|
+
out.push(m.unkId);
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
const pieces = [];
|
|
135
|
+
let start = 0;
|
|
136
|
+
while (start < chars.length) {
|
|
137
|
+
let end = chars.length;
|
|
138
|
+
let id;
|
|
139
|
+
while (start < end) {
|
|
140
|
+
const sub = (start > 0 ? m.prefix : '') + chars.slice(start, end).join('');
|
|
141
|
+
id = m.vocab.get(sub);
|
|
142
|
+
if (id !== undefined)
|
|
143
|
+
break;
|
|
144
|
+
end -= 1;
|
|
145
|
+
}
|
|
146
|
+
// No prefix of the remainder is in the vocabulary: upstream discards every
|
|
147
|
+
// piece found so far and emits a single UNK for the whole word.
|
|
148
|
+
if (id === undefined) {
|
|
149
|
+
if (m.unkId !== undefined)
|
|
150
|
+
out.push(m.unkId);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
pieces.push(id);
|
|
154
|
+
start = end;
|
|
155
|
+
}
|
|
156
|
+
for (const p of pieces)
|
|
157
|
+
out.push(p);
|
|
158
|
+
}
|
|
159
|
+
/** Token ids for `text`, with no special tokens added (the shipped contract). */
|
|
160
|
+
export function tokenizeForEmbedding(text, model = load()) {
|
|
161
|
+
const ids = [];
|
|
162
|
+
for (const word of preTokenize(normalize(text, model)))
|
|
163
|
+
wordPiece(word, model, ids);
|
|
164
|
+
// Upstream drops UNK entirely rather than pooling it (model2vec model.py:166,
|
|
165
|
+
// "necessary for word-level models"), so an out-of-vocab word contributes
|
|
166
|
+
// nothing instead of dragging every vector toward a shared UNK row. Filtering
|
|
167
|
+
// here rather than skipping at emit time matches upstream for the odd case of
|
|
168
|
+
// a literal "[UNK]" in the input, which WordPiece would otherwise keep.
|
|
169
|
+
return model.unkId === undefined ? ids : ids.filter((id) => id !== model.unkId);
|
|
170
|
+
}
|
|
171
|
+
// ---------------------------------------------------------------------------
|
|
172
|
+
// Load
|
|
173
|
+
// ---------------------------------------------------------------------------
|
|
174
|
+
function load() {
|
|
175
|
+
if (_model)
|
|
176
|
+
return _model;
|
|
177
|
+
const dir = modelDir();
|
|
178
|
+
const header = JSON.parse(readFileSync(join(dir, 'model.json'), 'utf8'));
|
|
179
|
+
if (header.format !== EXPECTED_FORMAT) {
|
|
180
|
+
throw new Error(`veto-model: unsupported payload format ${header.format} (expected ${EXPECTED_FORMAT})`);
|
|
181
|
+
}
|
|
182
|
+
// model2vec multiplies token rows by an optional per-token weight vector
|
|
183
|
+
// before pooling. potion-base-8M ships none, and this implementation pools
|
|
184
|
+
// unweighted — so a model that DOES ship them must not load silently.
|
|
185
|
+
if (header.inference.per_token_weights != null) {
|
|
186
|
+
throw new Error('veto-model: payload declares per-token weights, which this implementation does not apply');
|
|
187
|
+
}
|
|
188
|
+
if (header.inference.add_special_tokens) {
|
|
189
|
+
throw new Error('veto-model: payload expects special tokens, which this implementation does not add');
|
|
190
|
+
}
|
|
191
|
+
if (header.inference.pooling !== 'mean' || !header.inference.normalize) {
|
|
192
|
+
throw new Error(`veto-model: unsupported inference contract (${header.inference.pooling}, normalize=${header.inference.normalize})`);
|
|
193
|
+
}
|
|
194
|
+
const { vocab_size: vocabSize, dim } = header;
|
|
195
|
+
const tableBuf = readFileSync(join(dir, 'embeddings.int8.bin'));
|
|
196
|
+
const scaleBuf = readFileSync(join(dir, 'scales.f32.bin'));
|
|
197
|
+
if (tableBuf.length !== vocabSize * dim) {
|
|
198
|
+
throw new Error(`veto-model: table is ${tableBuf.length} bytes, expected ${vocabSize * dim}`);
|
|
199
|
+
}
|
|
200
|
+
if (scaleBuf.length !== vocabSize * 4) {
|
|
201
|
+
throw new Error(`veto-model: scales are ${scaleBuf.length} bytes, expected ${vocabSize * 4}`);
|
|
202
|
+
}
|
|
203
|
+
const tok = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));
|
|
204
|
+
const norm = tok.normalizer ?? {};
|
|
205
|
+
const tm = tok.model ?? {};
|
|
206
|
+
if (tm.type !== 'WordPiece' || norm.type !== 'BertNormalizer' || tok.pre_tokenizer?.type !== 'BertPreTokenizer') {
|
|
207
|
+
throw new Error(`veto-model: unsupported tokenizer (${norm.type} / ${tok.pre_tokenizer?.type} / ${tm.type})`);
|
|
208
|
+
}
|
|
209
|
+
const vocab = new Map(Object.entries(tm.vocab ?? {}));
|
|
210
|
+
if (vocab.size !== vocabSize) {
|
|
211
|
+
throw new Error(`veto-model: vocab has ${vocab.size} entries, table has ${vocabSize} rows`);
|
|
212
|
+
}
|
|
213
|
+
const lowercase = norm.lowercase !== false;
|
|
214
|
+
const model = {
|
|
215
|
+
header,
|
|
216
|
+
dim,
|
|
217
|
+
table: new Int8Array(tableBuf.buffer, tableBuf.byteOffset, tableBuf.length),
|
|
218
|
+
// Little-endian float32; Node is LE on every supported platform, and a
|
|
219
|
+
// byteOffset-aligned view would be brittle, so copy through a DataView.
|
|
220
|
+
scales: readScales(scaleBuf, vocabSize),
|
|
221
|
+
vocab,
|
|
222
|
+
unkId: tm.unk_token ? vocab.get(tm.unk_token) : undefined,
|
|
223
|
+
prefix: tm.continuing_subword_prefix ?? '##',
|
|
224
|
+
maxWordChars: tm.max_input_chars_per_word ?? 100,
|
|
225
|
+
lowercase,
|
|
226
|
+
// Resolve null the way upstream does — see normalize().
|
|
227
|
+
stripAccents: norm.strip_accents == null ? lowercase : norm.strip_accents,
|
|
228
|
+
cleanText: norm.clean_text !== false,
|
|
229
|
+
handleChineseChars: norm.handle_chinese_chars !== false,
|
|
230
|
+
};
|
|
231
|
+
if (model.stripAccents !== header.tokenizer.strip_accents_effective) {
|
|
232
|
+
throw new Error('veto-model: resolved strip_accents disagrees with the payload header');
|
|
233
|
+
}
|
|
234
|
+
_model = model;
|
|
235
|
+
return model;
|
|
236
|
+
}
|
|
237
|
+
function readScales(buf, n) {
|
|
238
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.length);
|
|
239
|
+
const out = new Float32Array(n);
|
|
240
|
+
for (let i = 0; i < n; i++)
|
|
241
|
+
out[i] = view.getFloat32(i * 4, true);
|
|
242
|
+
return out;
|
|
243
|
+
}
|
|
244
|
+
// ---------------------------------------------------------------------------
|
|
245
|
+
// Embed
|
|
246
|
+
// ---------------------------------------------------------------------------
|
|
247
|
+
/**
|
|
248
|
+
* Embed `text` into a unit-length vector.
|
|
249
|
+
*
|
|
250
|
+
* Gather each token's row, dequantize it by that row's scale, mean-pool, then
|
|
251
|
+
* L2-normalize. Text with no tokens (empty or punctuation-free-of-vocab)
|
|
252
|
+
* yields the zero vector rather than NaN — upstream's behaviour, and the
|
|
253
|
+
* reason the epsilon is in the denominator.
|
|
254
|
+
*/
|
|
255
|
+
export function embed(text, model = load()) {
|
|
256
|
+
const ids = tokenizeForEmbedding(text, model);
|
|
257
|
+
const { dim, table, scales } = model;
|
|
258
|
+
const out = new Float32Array(dim);
|
|
259
|
+
if (ids.length === 0)
|
|
260
|
+
return out;
|
|
261
|
+
for (const id of ids) {
|
|
262
|
+
const base = id * dim;
|
|
263
|
+
const scale = scales[id];
|
|
264
|
+
for (let d = 0; d < dim; d++)
|
|
265
|
+
out[d] += table[base + d] * scale;
|
|
266
|
+
}
|
|
267
|
+
let norm = 0;
|
|
268
|
+
for (let d = 0; d < dim; d++) {
|
|
269
|
+
out[d] /= ids.length;
|
|
270
|
+
norm += out[d] * out[d];
|
|
271
|
+
}
|
|
272
|
+
norm = Math.sqrt(norm) + NORM_EPSILON;
|
|
273
|
+
for (let d = 0; d < dim; d++)
|
|
274
|
+
out[d] /= norm;
|
|
275
|
+
return out;
|
|
276
|
+
}
|
|
277
|
+
/** Cosine similarity of two unit vectors — a dot product, given the inputs. */
|
|
278
|
+
export function cosine(a, b) {
|
|
279
|
+
let s = 0;
|
|
280
|
+
for (let i = 0; i < a.length; i++)
|
|
281
|
+
s += a[i] * b[i];
|
|
282
|
+
return s;
|
|
283
|
+
}
|
|
284
|
+
/** True when the model package resolves and its payload passes validation. */
|
|
285
|
+
export function embeddingsAvailable() {
|
|
286
|
+
try {
|
|
287
|
+
load();
|
|
288
|
+
return true;
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
return false;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/** Upstream model id + revision of the loaded payload, for recall provenance. */
|
|
295
|
+
export function modelProvenance() {
|
|
296
|
+
const { upstream } = load().header;
|
|
297
|
+
return { model_id: upstream.model_id, revision: upstream.revision };
|
|
298
|
+
}
|
|
299
|
+
/** Drop the cached table. Tests only — production loads once per process. */
|
|
300
|
+
export function resetModelCache() {
|
|
301
|
+
_model = undefined;
|
|
302
|
+
}
|
|
303
|
+
//# sourceMappingURL=embed.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../src/transcripts/embed.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,EAAE;AACF,8EAA8E;AAC9E,6EAA6E;AAC7E,2EAA2E;AAC3E,2EAA2E;AAC3E,6EAA6E;AAC7E,EAAE;AACF,8EAA8E;AAC9E,uEAAuE;AACvE,8DAA8D;AAC9D,6DAA6D;AAC7D,EAAE;AACF,4EAA4E;AAC5E,wEAAwE;AACxE,oEAAoE;AAEpE,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEhD,0EAA0E;AAC1E,MAAM,eAAe,GAAG,cAAc,CAAC;AACvC,2EAA2E;AAC3E,gEAAgE;AAChE,MAAM,YAAY,GAAG,KAAK,CAAC;AAiC3B,IAAI,MAAyB,CAAC;AAE9B;wDACwD;AACxD,SAAS,QAAQ;IACf,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;QAAE,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;IACjF,8EAA8E;IAC9E,OAAO,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAE9E,4EAA4E;AAC5E,oEAAoE;AACpE,MAAM,OAAO,GAAG,QAAQ,CAAC;AACzB,MAAM,KAAK,GAAG,uBAAuB,CAAC;AAEtC,SAAS,SAAS,CAAC,EAAU;IAC3B,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC5D,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACnF,CAAC;AAED,0EAA0E;AAC1E,8EAA8E;AAC9E,SAAS,SAAS,CAAC,EAAU;IAC3B,OAAO,CACL,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC,CACjC,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,CAAQ;IACvC,IAAI,GAAG,GAAG,IAAI,CAAC;IAEf,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;QAChB,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,kEAAkE;YAClE,yEAAyE;YACzE,IAAI,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,QAAQ,IAAI,SAAS,CAAC,EAAE,CAAC;gBAAE,SAAS;YAClE,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACrC,CAAC;QACD,GAAG,GAAG,GAAG,CAAC;IACZ,CAAC;IAED,IAAI,CAAC,CAAC,kBAAkB,EAAE,CAAC;QACzB,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;YAC9B,GAAG,IAAI,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,CAAC;QACD,GAAG,GAAG,GAAG,CAAC;IACZ,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,4EAA4E;IAC5E,yEAAyE;IACzE,IAAI,CAAC,CAAC,YAAY;QAAE,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEvE,IAAI,CAAC,CAAC,SAAS;QAAE,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAEzC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8EAA8E;AAC9E,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AAE9E,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;gBACnB,IAAI,GAAG,EAAE,CAAC;oBAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAAC,GAAG,GAAG,EAAE,CAAC;gBAAC,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,GAAG,IAAI,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,0EAA0E;AAC1E,0EAA0E;AAC1E,yEAAyE;AACzE,8EAA8E;AAE9E,SAAS,SAAS,CAAC,IAAY,EAAE,CAAQ,EAAE,GAAa;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;QACvB,IAAI,EAAsB,CAAC;QAC3B,OAAO,KAAK,GAAG,GAAG,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3E,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,EAAE,KAAK,SAAS;gBAAE,MAAM;YAC5B,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;QACD,2EAA2E;QAC3E,gEAAgE;QAChE,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChB,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,KAAK,GAAG,IAAI,EAAE;IAC/D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAAE,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAEpF,8EAA8E;IAC9E,0EAA0E;IAC1E,8EAA8E;IAC9E,8EAA8E;IAC9E,wEAAwE;IACxE,OAAO,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC;AAClF,CAAC;AAED,8EAA8E;AAC9E,OAAO;AACP,8EAA8E;AAE9E,SAAS,IAAI;IACX,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAAW,CAAC;IAEnF,IAAI,MAAM,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,0CAA0C,MAAM,CAAC,MAAM,cAAc,eAAe,GAAG,CAAC,CAAC;IAC3G,CAAC;IACD,yEAAyE;IACzE,2EAA2E;IAC3E,sEAAsE;IACtE,IAAI,MAAM,CAAC,SAAS,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;IAC9G,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,kBAAkB,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,oFAAoF,CAAC,CAAC;IACxG,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,KAAK,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,+CAA+C,MAAM,CAAC,SAAS,CAAC,OAAO,eAAe,MAAM,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;IACvI,CAAC;IAED,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;IAC9C,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,GAAG,GAAG,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,MAAM,oBAAoB,SAAS,GAAG,GAAG,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,oBAAoB,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC,CAIvE,CAAC;IACF,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC;IAClC,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;IAC3B,IAAI,EAAE,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,GAAG,CAAC,aAAa,EAAE,IAAI,KAAK,kBAAkB,EAAE,CAAC;QAChH,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,IAAI,MAAM,GAAG,CAAC,aAAa,EAAE,IAAI,MAAM,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC;IAChH,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,CAAiB,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IACtE,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,CAAC,IAAI,uBAAuB,SAAS,OAAO,CAAC,CAAC;IAC9F,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;IAC3C,MAAM,KAAK,GAAU;QACnB,MAAM;QACN,GAAG;QACH,KAAK,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC;QAC3E,uEAAuE;QACvE,wEAAwE;QACxE,MAAM,EAAE,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC;QACvC,KAAK;QACL,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QACzD,MAAM,EAAE,EAAE,CAAC,yBAAyB,IAAI,IAAI;QAC5C,YAAY,EAAE,EAAE,CAAC,wBAAwB,IAAI,GAAG;QAChD,SAAS;QACT,wDAAwD;QACxD,YAAY,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa;QACzE,SAAS,EAAE,IAAI,CAAC,UAAU,KAAK,KAAK;QACpC,kBAAkB,EAAE,IAAI,CAAC,oBAAoB,KAAK,KAAK;KACxD,CAAC;IAEF,IAAI,KAAK,CAAC,YAAY,KAAK,MAAM,CAAC,SAAS,CAAC,uBAAuB,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IAED,MAAM,GAAG,KAAK,CAAC;IACf,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,CAAS;IACxC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClE,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;IAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAClE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,UAAU,KAAK,CAAC,IAAY,EAAE,KAAK,GAAG,IAAI,EAAE;IAChD,MAAM,GAAG,GAAG,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC9C,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IACrC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAEjC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAE,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,KAAK,CAAC;IACpE,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,CAAC,CAAC,CAAE,IAAI,GAAG,CAAC,MAAM,CAAC;QACtB,IAAI,IAAI,GAAG,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;IAC5B,CAAC;IACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;IAE9C,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,MAAM,CAAC,CAAe,EAAE,CAAe;IACrD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACtD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,mBAAmB;IACjC,IAAI,CAAC;QAAC,IAAI,EAAE,CAAC;QAAC,OAAO,IAAI,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAC;IAAC,CAAC;AACtD,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,eAAe;IAC7B,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC;IACnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC;AACtE,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe;IAC7B,MAAM,GAAG,SAAS,CAAC;AACrB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/transcripts/ingest.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/transcripts/ingest.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAqC,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AAO/E,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,MAAM,MAAM,YAAY,GACpB,SAAS,GAAG,iBAAiB,GAAG,WAAW,GAAG,iBAAiB,GAAG,kBAAkB,GAAG,OAAO,CAAC;AAEnG,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,YAAY,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAWF,oEAAoE;AACpE,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,CA0F7D;AAED,4FAA4F;AAC5F,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,CAE7D;AAED,0DAA0D;AAC1D,wBAAgB,aAAa,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,SAAW,GAAG,YAAY,CAOtF;AAED,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,QAAQ,EAAE,CAGvD"}
|
|
@@ -10,6 +10,7 @@ import { readFileSync, existsSync } from 'node:fs';
|
|
|
10
10
|
import { gunzipSync } from 'node:zlib';
|
|
11
11
|
import { randomUUID } from 'node:crypto';
|
|
12
12
|
import { getTranscriptsDb } from './store.js';
|
|
13
|
+
import { resetCaches } from './cache.js';
|
|
13
14
|
import { parseClaudeTranscript } from './adapters/claude.js';
|
|
14
15
|
import { tokenize } from './tokenize.js';
|
|
15
16
|
import { SEARCHABLE_KINDS } from './schema.js';
|
|
@@ -41,13 +42,13 @@ export function ingestArchive(archiveId) {
|
|
|
41
42
|
if (sessionIds.size > 0 && !sessionIds.has(row.source_session_id)) {
|
|
42
43
|
return { status: 'session_mismatch', reason: 'archive session id not present in transcript' };
|
|
43
44
|
}
|
|
44
|
-
const ins = db.prepare(`INSERT INTO events (id, archive_id, source_session_id, seq, line_index, block_index, kind,
|
|
45
|
-
source_type, role, tool_name, text, secret_count, event_uuid, parent_uuid, is_sidechain,
|
|
46
|
-
ts_source, ts_utc, raw_offset, raw_length)
|
|
45
|
+
const ins = db.prepare(`INSERT INTO events (id, archive_id, source_session_id, seq, line_index, block_index, kind,
|
|
46
|
+
source_type, role, tool_name, text, secret_count, event_uuid, parent_uuid, is_sidechain,
|
|
47
|
+
ts_source, ts_utc, raw_offset, raw_length)
|
|
47
48
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
48
|
-
const insDoc = db.prepare(`INSERT INTO search_docs (event_id, archive_id, source_session_id, project_dir, seq, kind, len)
|
|
49
|
+
const insDoc = db.prepare(`INSERT INTO search_docs (event_id, archive_id, source_session_id, project_dir, seq, kind, len)
|
|
49
50
|
VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING id`);
|
|
50
|
-
const insTerm = db.prepare(`INSERT INTO search_terms (term) VALUES (?)
|
|
51
|
+
const insTerm = db.prepare(`INSERT INTO search_terms (term) VALUES (?)
|
|
51
52
|
ON CONFLICT(term) DO UPDATE SET term = term RETURNING id`);
|
|
52
53
|
const insPosting = db.prepare(`INSERT INTO search_postings (term_id, doc_id, tf) VALUES (?, ?, ?)`);
|
|
53
54
|
// Term ids are cached per ingest run — a session reuses its vocabulary
|
|
@@ -65,6 +66,9 @@ export function ingestArchive(archiveId) {
|
|
|
65
66
|
try {
|
|
66
67
|
db.prepare(`DELETE FROM events WHERE archive_id = ?`).run(archiveId);
|
|
67
68
|
db.prepare(`DELETE FROM search_postings WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).run(archiveId);
|
|
69
|
+
// Doc ids are reassigned by this re-derive, so stale vectors would not
|
|
70
|
+
// just orphan — they would silently re-attach to the wrong events.
|
|
71
|
+
db.prepare(`DELETE FROM search_vectors WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).run(archiveId);
|
|
68
72
|
db.prepare(`DELETE FROM search_docs WHERE archive_id = ?`).run(archiveId);
|
|
69
73
|
for (const e of events) {
|
|
70
74
|
const eventId = randomUUID();
|
|
@@ -80,9 +84,12 @@ export function ingestArchive(archiveId) {
|
|
|
80
84
|
insPosting.run(termId(term), docId, count);
|
|
81
85
|
}
|
|
82
86
|
}
|
|
83
|
-
|
|
84
|
-
|
|
87
|
+
// Vectors were just dropped with their docs; clearing the marker lets
|
|
88
|
+
// the lazy embed pass rebuild them without re-parsing the transcript.
|
|
89
|
+
db.prepare(`UPDATE archives SET indexed_through_seq = ?, parser_version = ?, chunker_version = 0, embed_model = NULL
|
|
90
|
+
WHERE id = ?`).run(events.length, PARSER_VERSION, archiveId);
|
|
85
91
|
db.exec('COMMIT');
|
|
92
|
+
resetCaches();
|
|
86
93
|
}
|
|
87
94
|
catch (e) {
|
|
88
95
|
db.exec('ROLLBACK');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/transcripts/ingest.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,EAAE;AACF,2EAA2E;AAC3E,gFAAgF;AAChF,8EAA8E;AAC9E,gFAAgF;AAChF,kFAAkF;AAClF,mDAAmD;AAEnD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAkC,MAAM,aAAa,CAAC;AAE/E,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAC;AAErD,iFAAiF;AACjF,8EAA8E;AAC9E,wEAAwE;AACxE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAYhC,SAAS,cAAc,CAAC,SAAiB;IACvC,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAQ,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,SAAS,CAA4B,IAAI,IAAI,CAAC;AAC9G,CAAC;AAED,SAAS,aAAa,CAAC,GAAe;IACpC,OAAO,GAAG,CAAC,cAAc,KAAK,cAAc,CAAC;AAC/C,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAC/F,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QAExE,MAAM,GAAG,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;QACvD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAE3E,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,EAAE,8CAA8C,EAAE,CAAC;QAChG,CAAC;QAED,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CACpB;;;wEAGkE,CACnE,CAAC;QACF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB;iDAC2C,CAC5C,CAAC;QACF,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB;gEAC0D,CAC3D,CAAC;QACF,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,oEAAoE,CACrE,CAAC;QACF,uEAAuE;QACvE,iDAAiD;QACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,MAAM,MAAM,GAAG,CAAC,IAAY,EAAU,EAAE;YACtC,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,EAAE,GAAG,MAAM,CAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAA6B,CAAC,EAAE,CAAC,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACrE,EAAE,CAAC,OAAO,CACR,+FAA+F,CAChG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACjB,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC1E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CACL,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EACnF,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACzG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,SAAS,CAC9C,CAAC;gBACF,mEAAmE;gBACnE,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAChC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;oBACrC,KAAK,MAAM,CAAC,IAAI,MAAM;wBAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxD,MAAM,KAAK,GAAG,MAAM,CAAE,MAAM,CAAC,GAAG,CAC9B,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,iBAAiB,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAC7D,CAAC,EAAE,CAAC,CAAC;oBAClC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;wBAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YACD,EAAE,CAAC,OAAO,
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/transcripts/ingest.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,EAAE;AACF,2EAA2E;AAC3E,gFAAgF;AAChF,8EAA8E;AAC9E,gFAAgF;AAChF,kFAAkF;AAClF,mDAAmD;AAEnD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAkC,MAAM,aAAa,CAAC;AAE/E,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAC;AAErD,iFAAiF;AACjF,8EAA8E;AAC9E,wEAAwE;AACxE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAYhC,SAAS,cAAc,CAAC,SAAiB;IACvC,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAQ,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,CAAC,SAAS,CAA4B,IAAI,IAAI,CAAC;AAC9G,CAAC;AAED,SAAS,aAAa,CAAC,GAAe;IACpC,OAAO,GAAG,CAAC,cAAc,KAAK,cAAc,CAAC;AAC/C,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;QACzC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAC/F,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;QAExE,MAAM,GAAG,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;QACvD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAE3E,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,MAAM,EAAE,8CAA8C,EAAE,CAAC;QAChG,CAAC;QAED,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CACpB;;;wEAGkE,CACnE,CAAC;QACF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB;iDAC2C,CAC5C,CAAC;QACF,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB;gEAC0D,CAC3D,CAAC;QACF,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,oEAAoE,CACrE,CAAC;QACF,uEAAuE;QACvE,iDAAiD;QACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,MAAM,MAAM,GAAG,CAAC,IAAY,EAAU,EAAE;YACtC,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,EAAE,GAAG,MAAM,CAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAA6B,CAAC,EAAE,CAAC,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACrE,EAAE,CAAC,OAAO,CACR,+FAA+F,CAChG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACjB,uEAAuE;YACvE,mEAAmE;YACnE,EAAE,CAAC,OAAO,CACR,8FAA8F,CAC/F,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACjB,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC1E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;gBAC7B,GAAG,CAAC,GAAG,CACL,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EACnF,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACzG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,SAAS,CAC9C,CAAC;gBACF,mEAAmE;gBACnE,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBACtD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAChC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;oBACrC,KAAK,MAAM,CAAC,IAAI,MAAM;wBAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxD,MAAM,KAAK,GAAG,MAAM,CAAE,MAAM,CAAC,GAAG,CAC9B,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,iBAAiB,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAC7D,CAAC,EAAE,CAAC,CAAC;oBAClC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;wBAAE,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,EAAE,CAAC,OAAO,CACR;uBACe,CAChB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;YAChD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClB,WAAW,EAAE,CAAC;QAChB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACpB,MAAM,CAAC,CAAC;QACV,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC;IACvE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACjF,CAAC;AACH,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,OAAO,aAAa,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,aAAa,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACtE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAI,EAAE,CAAC,OAAO,CACrB,oEAAoE,CACrE,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAAgC,IAAI,IAAI,CAAC;IACtE,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACzC,OAAO,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,SAAiB;IACzC,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAO,EAAE,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC,GAAG,CAAC,SAAS,CAAe,CAAC;AAC3G,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manage.d.ts","sourceRoot":"","sources":["../../src/transcripts/manage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"manage.d.ts","sourceRoot":"","sources":["../../src/transcripts/manage.ts"],"names":[],"mappings":"AAYA,OAAO,EAAY,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAClD,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAG7D,MAAM,MAAM,cAAc,GAAG;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAUF,wBAAgB,YAAY,CAAC,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,cAAc,EAAE,CAOjG;AAED,MAAM,MAAM,aAAa,GAAG;IAAE,OAAO,EAAE,cAAc,CAAC;IAAC,GAAG,EAAE,OAAO,EAAE,CAAC;IAAC,KAAK,EAAE,YAAY,CAAA;CAAE,GAAG,IAAI,CAAC;AAEpG,wBAAgB,WAAW,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,SAAW,GAAG,aAAa,CAKrF;AAED,MAAM,MAAM,WAAW,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAkDnH,wBAAgB,YAAY,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,SAAW,GAAG,WAAW,CAUpF;AAED,wBAAgB,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,CAI5D;AAED,wBAAgB,QAAQ,IAAI,WAAW,CAQtC;AAED,MAAM,MAAM,SAAS,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAExG,wBAAgB,oBAAoB,IAAI,SAAS,CAMhD;AAED,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAI1C"}
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// leftover file is harmless disk, a dangling row is not).
|
|
8
8
|
import { statSync, existsSync, rmSync } from 'node:fs';
|
|
9
9
|
import { getTranscriptsDb, transcriptsDbPath } from './store.js';
|
|
10
|
+
import { resetCaches } from './cache.js';
|
|
10
11
|
import { normalizeProjectDir } from '../memory/local.js';
|
|
11
12
|
import { buildTOC } from './toc.js';
|
|
12
13
|
import { buildFacts } from './pyramid.js';
|
|
@@ -44,8 +45,12 @@ function purgeArchiveRows(archives) {
|
|
|
44
45
|
// binding doesn't dedupe numbered parameters (SQLITE_RANGE on CI).
|
|
45
46
|
res.indexRows += db.prepare(`SELECT COUNT(*) n FROM search_docs WHERE archive_id = ?`).get(a.id).n;
|
|
46
47
|
res.indexRows += db.prepare(`SELECT COUNT(*) n FROM search_postings WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).get(a.id).n;
|
|
48
|
+
// Chunk vectors hang off the same doc ids, so they orphan the same way
|
|
49
|
+
// postings would if this were forgotten (C3 asserts zero orphans).
|
|
50
|
+
res.indexRows += db.prepare(`SELECT COUNT(*) n FROM search_vectors WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).get(a.id).n;
|
|
47
51
|
db.prepare(`DELETE FROM events WHERE archive_id = ?`).run(a.id);
|
|
48
52
|
db.prepare(`DELETE FROM search_postings WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).run(a.id);
|
|
53
|
+
db.prepare(`DELETE FROM search_vectors WHERE doc_id IN (SELECT id FROM search_docs WHERE archive_id = ?)`).run(a.id);
|
|
49
54
|
db.prepare(`DELETE FROM search_docs WHERE archive_id = ?`).run(a.id);
|
|
50
55
|
db.prepare(`DELETE FROM archives WHERE id = ?`).run(a.id);
|
|
51
56
|
const m = db.prepare(`DELETE FROM session_map WHERE source = ? AND source_session_id = ?`).run(a.source, a.source_session_id);
|
|
@@ -55,6 +60,7 @@ function purgeArchiveRows(archives) {
|
|
|
55
60
|
paths.push(a.archive_path);
|
|
56
61
|
}
|
|
57
62
|
db.exec('COMMIT');
|
|
63
|
+
resetCaches();
|
|
58
64
|
}
|
|
59
65
|
catch (e) {
|
|
60
66
|
db.exec('ROLLBACK');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manage.js","sourceRoot":"","sources":["../../src/transcripts/manage.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,kFAAkF;AAClF,4EAA4E;AAC5E,0DAA0D;AAE1D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAgB,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,UAAU,EAAqB,MAAM,cAAc,CAAC;AAe7D,SAAS,SAAS,CAAC,CAAa;IAC9B,OAAO;QACL,eAAe,EAAE,CAAC,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW;QACjF,MAAM,EAAE,CAAC,CAAC,mBAAmB,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC,aAAa;QACzF,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAgD,EAAE;IAC7E,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU;QAC1B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,+EAA+E,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC;QAC9I,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrF,OAAQ,IAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC/C,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACpE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2B,CAAC;IACjJ,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AAID,SAAS,gBAAgB,CAAC,QAAsB;IAC9C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACzF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,IAAK,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAC9G,oEAAoE;YACpE,mEAAmE;YACnE,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,yDAAyD,CAC1D,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,0GAA0G,CAC3G,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,OAAO,CACR,+FAA+F,CAChG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACZ,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC9H,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACvC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,CAAC,YAAY;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"manage.js","sourceRoot":"","sources":["../../src/transcripts/manage.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,kFAAkF;AAClF,4EAA4E;AAC5E,0DAA0D;AAE1D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAgB,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,UAAU,EAAqB,MAAM,cAAc,CAAC;AAe7D,SAAS,SAAS,CAAC,CAAa;IAC9B,OAAO;QACL,eAAe,EAAE,CAAC,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW;QACjF,MAAM,EAAE,CAAC,CAAC,mBAAmB,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,CAAC,aAAa;QACzF,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAgD,EAAE;IAC7E,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU;QAC1B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,+EAA+E,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,CAAC;QAC9I,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrF,OAAQ,IAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC/C,CAAC;AAID,MAAM,UAAU,WAAW,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACpE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2B,CAAC;IACjJ,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AAID,SAAS,gBAAgB,CAAC,QAAsB;IAC9C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IACzF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,IAAK,EAAE,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAC9G,oEAAoE;YACpE,mEAAmE;YACnE,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,yDAAyD,CAC1D,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,0GAA0G,CAC3G,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,uEAAuE;YACvE,mEAAmE;YACnE,GAAG,CAAC,SAAS,IAAK,EAAE,CAAC,OAAO,CAC1B,yGAAyG,CAC1G,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAmB,CAAC,CAAC,CAAC;YAChC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,OAAO,CACR,+FAA+F,CAChG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACZ,EAAE,CAAC,OAAO,CACR,8FAA8F,CAC/F,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACZ,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC9H,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;YACvC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,CAAC,YAAY;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjD,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,WAAW,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC;IACV,CAAC;IACD,gEAAgE;IAChE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YAAC,IAAI,CAAC;gBAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,+BAA+B,CAAC,CAAC;QAAC,CAAC;IACrG,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACrE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,mEAAmE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2B,CAAC;IACpJ,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpD,0EAA0E;IAC1E,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACxH,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,UAAkB;IAC7C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAiB,CAAC;IACjI,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,QAAQ;IACtB,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,EAAkB,CAAC;IAC5E,MAAM,GAAG,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACvC,kDAAkD;IAClD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC,GAAG,EAAE,CAAC;IACtD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9D,OAAO,GAAG,CAAC;AACb,CAAC;AAID,MAAM,UAAU,oBAAoB;IAClC,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,EAA8B,CAAC;IAC/H,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QAAC,OAAO,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAC,CAAC,IAAI,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,6BAA6B,CAAC,CAAC;IAC7F,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,IAAI,CAAC;IAC9B,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC1D,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AAChD,CAAC"}
|
|
@@ -17,11 +17,11 @@ export function recordSessionMapping(m) {
|
|
|
17
17
|
const db = getTranscriptsDb();
|
|
18
18
|
const now = new Date().toISOString();
|
|
19
19
|
const proj = m.projectDir ? normalizeProjectDir(m.projectDir) : null;
|
|
20
|
-
db.prepare(`INSERT INTO session_map (source, source_session_id, transcript_path, project_dir, last_seen_at)
|
|
21
|
-
VALUES (?, ?, ?, ?, ?)
|
|
22
|
-
ON CONFLICT(source, source_session_id) DO UPDATE SET
|
|
23
|
-
transcript_path = excluded.transcript_path,
|
|
24
|
-
project_dir = excluded.project_dir,
|
|
20
|
+
db.prepare(`INSERT INTO session_map (source, source_session_id, transcript_path, project_dir, last_seen_at)
|
|
21
|
+
VALUES (?, ?, ?, ?, ?)
|
|
22
|
+
ON CONFLICT(source, source_session_id) DO UPDATE SET
|
|
23
|
+
transcript_path = excluded.transcript_path,
|
|
24
|
+
project_dir = excluded.project_dir,
|
|
25
25
|
last_seen_at = excluded.last_seen_at`).run(m.source ?? 'claude', m.sourceSessionId, m.transcriptPath, proj, now);
|
|
26
26
|
}
|
|
27
27
|
/** Look up the mapping for an exact session id. */
|
|
@@ -15,8 +15,8 @@ export const ERROR_RE = /\b(error|errno|failed|failure|exception|traceback|not f
|
|
|
15
15
|
/** L2 — the conversation spine (user + assistant text, in order). */
|
|
16
16
|
export function buildSpine(archiveId) {
|
|
17
17
|
const db = getTranscriptsDb();
|
|
18
|
-
const rows = db.prepare(`SELECT seq, role, kind, ts_utc, text FROM events
|
|
19
|
-
WHERE archive_id = ? AND kind IN ('user_message','assistant_message')
|
|
18
|
+
const rows = db.prepare(`SELECT seq, role, kind, ts_utc, text FROM events
|
|
19
|
+
WHERE archive_id = ? AND kind IN ('user_message','assistant_message')
|
|
20
20
|
ORDER BY seq`).all(archiveId);
|
|
21
21
|
return rows.map(r => ({
|
|
22
22
|
seq: r.seq,
|
|
@@ -16,6 +16,13 @@ export type RecallQueryResult = {
|
|
|
16
16
|
sourceSessionId?: string;
|
|
17
17
|
archivesIndexed: number;
|
|
18
18
|
};
|
|
19
|
+
/** Which retrieval ran. `semantic` names the model that produced the
|
|
20
|
+
* vectors, or null when recall was keyword-only — so a caller can tell
|
|
21
|
+
* a degraded result from a poor one (condition A3). */
|
|
22
|
+
retrieval: {
|
|
23
|
+
lexical: true;
|
|
24
|
+
semantic: string | null;
|
|
25
|
+
};
|
|
19
26
|
hits: Array<Pick<SearchHit, 'sourceSessionId' | 'seq' | 'kind' | 'snippet' | 'score' | 'eventId' | 'archiveId'>>;
|
|
20
27
|
toc?: Segment[];
|
|
21
28
|
facts?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAWA,OAAO,
|
|
1
|
+
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAWA,OAAO,EAAsB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAGjE,OAAO,EAAY,KAAK,OAAO,EAAE,MAAM,UAAU,CAAC;AAElD,OAAO,EAA4B,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAoB1E,MAAM,MAAM,gBAAgB,GAAG;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC;IAClF;;4DAEwD;IACxD,SAAS,EAAE;QAAE,OAAO,EAAE,IAAI,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACtD,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC;IACjH,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,iFAAiF;AACjF,wBAAgB,WAAW,CAAC,KAAK,EAAE,gBAAgB,GAAG,iBAAiB,CAyDtE;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE,kFAAkF;AAClF,wBAAgB,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,kBAAkB,CAuBzE"}
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
// The host AI is the reranker/navigator (free, no embeddings). Indexing is lazy:
|
|
9
9
|
// archives in scope are ensured-indexed at query time.
|
|
10
10
|
import { ensureIndexed } from './ingest.js';
|
|
11
|
-
import {
|
|
11
|
+
import { searchEventsHybrid } from './search.js';
|
|
12
|
+
import { embedArchive } from './vectors.js';
|
|
13
|
+
import { embeddingsAvailable, modelProvenance } from './embed.js';
|
|
12
14
|
import { buildTOC } from './toc.js';
|
|
13
15
|
import { buildFacts, renderFacts } from './pyramid.js';
|
|
14
16
|
import { expandEvent, expandRange } from './expand.js';
|
|
@@ -32,7 +34,24 @@ export function recallQuery(input) {
|
|
|
32
34
|
: [];
|
|
33
35
|
for (const a of scopeArchives)
|
|
34
36
|
ensureIndexed(a.id);
|
|
35
|
-
|
|
37
|
+
// Semantic vectors are built here, at the first query that needs them, and
|
|
38
|
+
// never at server boot (condition A2). embedArchive is a no-op when the
|
|
39
|
+
// model package is absent, so this whole block silently costs nothing and
|
|
40
|
+
// recall falls back to BM25 alone.
|
|
41
|
+
for (const a of scopeArchives)
|
|
42
|
+
embedArchive(a.id);
|
|
43
|
+
// Recorded in the response, not just inferred from result quality: the
|
|
44
|
+
// semantic layer degrades silently by design, so the only honest way for a
|
|
45
|
+
// caller to know it ran is to be told which model ran it.
|
|
46
|
+
let semanticModel = null;
|
|
47
|
+
try {
|
|
48
|
+
if (embeddingsAvailable()) {
|
|
49
|
+
const { model_id, revision } = modelProvenance();
|
|
50
|
+
semanticModel = `${model_id}@${revision}`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch { /* optional layer — never fail recall over provenance */ }
|
|
54
|
+
const hits = searchEventsHybrid(input.query, {
|
|
36
55
|
projectDir: input.projectDir,
|
|
37
56
|
sourceSessionId: input.sourceSessionId,
|
|
38
57
|
limit: input.limit ?? 8,
|
|
@@ -51,12 +70,15 @@ export function recallQuery(input) {
|
|
|
51
70
|
ok: true,
|
|
52
71
|
query: input.query,
|
|
53
72
|
scope: { projectDir: input.projectDir, sourceSessionId: input.sourceSessionId, archivesIndexed: scopeArchives.length },
|
|
73
|
+
retrieval: { lexical: true, semantic: semanticModel },
|
|
54
74
|
hits,
|
|
55
75
|
toc,
|
|
56
76
|
facts,
|
|
57
77
|
guidance: hits.length
|
|
58
78
|
? 'Reason over these hits + TOC, then call expand with {eventId} for one line, or {archiveId|sourceSessionId, fromSeq, toSeq} / {sourceSessionId, segmentIndex} for a range. Results are masked.'
|
|
59
|
-
:
|
|
79
|
+
: (semanticModel
|
|
80
|
+
? 'No matches, lexical or semantic. Try different identifiers/terms, or expand a TOC segment directly.'
|
|
81
|
+
: 'No lexical matches, and semantic recall is unavailable on this install. Try different identifiers/terms, or expand a TOC segment directly.'),
|
|
60
82
|
};
|
|
61
83
|
}
|
|
62
84
|
/** Phase 2 — expand a hit/segment/range to exact masked lines with provenance. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recall.js","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,iFAAiF;AACjF,8EAA8E;AAC9E,EAAE;AACF,iFAAiF;AACjF,uDAAuD;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"recall.js","sourceRoot":"","sources":["../../src/transcripts/recall.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,iFAAiF;AACjF,8EAA8E;AAC9E,EAAE;AACF,iFAAiF;AACjF,uDAAuD;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAkB,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAgB,MAAM,UAAU,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAqB,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzD,MAAM,SAAS,GAAG,sHAAsH,CAAC;AAEzI,SAAS,kBAAkB,CAAC,UAAkB;IAC5C,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAO,EAAE,CAAC,OAAO,CACf,2FAA2F,CAC5F,CAAC,GAAG,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAgD,CAAC;AACxF,CAAC;AAED,SAAS,iBAAiB,CAAC,eAAuB,EAAE,MAAM,GAAG,QAAQ;IACnE,MAAM,EAAE,GAAG,gBAAgB,EAAE,CAAC;IAC9B,OAAQ,EAAE,CAAC,OAAO,CAChB,uFAAuF,CACxF,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAA2D,IAAI,IAAI,CAAC;AACnG,CAAC;AAwBD,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,KAAuB;IACjD,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe;QACzC,CAAC,CAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAiD;QAC7G,CAAC,CAAC,KAAK,CAAC,UAAU;YAChB,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC;YACtC,CAAC,CAAC,EAAE,CAAC;IAET,KAAK,MAAM,CAAC,IAAI,aAAa;QAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEnD,2EAA2E;IAC3E,wEAAwE;IACxE,0EAA0E;IAC1E,mCAAmC;IACnC,KAAK,MAAM,CAAC,IAAI,aAAa;QAAE,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAElD,uEAAuE;IACvE,2EAA2E;IAC3E,0DAA0D;IAC1D,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,IAAI,mBAAmB,EAAE,EAAE,CAAC;YAC1B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,eAAe,EAAE,CAAC;YACjD,aAAa,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAC5C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,wDAAwD,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,KAAK,EAAE;QAC3C,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,CAAC;KACxB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAEhK,+EAA+E;IAC/E,IAAI,GAA0B,CAAC;IAC/B,IAAI,KAAyB,CAAC;IAC9B,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,aAAa,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnG,IAAI,aAAa,EAAE,CAAC;QAClB,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QACjC,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,OAAO;QACL,EAAE,EAAE,IAAI;QACR,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,EAAE,eAAe,EAAE,aAAa,CAAC,MAAM,EAAE;QACtH,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE;QACrD,IAAI;QACJ,GAAG;QACH,KAAK;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM;YACnB,CAAC,CAAC,+LAA+L;YACjM,CAAC,CAAC,CAAC,aAAa;gBACd,CAAC,CAAC,qGAAqG;gBACvG,CAAC,CAAC,4IAA4I,CAAC;KACpJ,CAAC;AACJ,CAAC;AAaD,kFAAkF;AAClF,MAAM,UAAU,YAAY,CAAC,KAAwB;IACnD,MAAM,IAAI,GAAG,CAAC,CAAe,EAAsB,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC;IAExF,IAAI,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAE3D,2DAA2D;IAC3D,IAAI,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;IAChC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QACxC,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;IAC3D,CAAC;IACD,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,6CAA6C,EAAE,CAAC,CAAC;IAElG,IAAI,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACpD,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACpH,CAAC;IAED,OAAO,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mDAAmD,EAAE,CAAC,CAAC;AAC1F,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const TRANSCRIPTS_SCHEMA_VERSION =
|
|
1
|
+
export declare const TRANSCRIPTS_SCHEMA_VERSION = 4;
|
|
2
2
|
export declare const EVENT_KINDS: readonly ["user_message", "assistant_message", "tool_call", "tool_result", "reasoning", "meta", "unknown"];
|
|
3
3
|
export type EventKind = (typeof EVENT_KINDS)[number];
|
|
4
4
|
export type Migration = {
|
|
@@ -22,6 +22,16 @@ export type ArchiveRow = {
|
|
|
22
22
|
indexed_through_seq: number;
|
|
23
23
|
captured_at: string;
|
|
24
24
|
updated_at: string;
|
|
25
|
+
/** 0 until chunk vectors exist; gates re-derivation independently of parsing. */
|
|
26
|
+
chunker_version: number;
|
|
27
|
+
/** `model_id@revision` that produced the stored vectors, or null. */
|
|
28
|
+
embed_model: string | null;
|
|
29
|
+
};
|
|
30
|
+
export type SearchVectorRow = {
|
|
31
|
+
doc_id: number;
|
|
32
|
+
chunk_index: number;
|
|
33
|
+
vec: Uint8Array;
|
|
34
|
+
norm: number;
|
|
25
35
|
};
|
|
26
36
|
export type SessionMapRow = {
|
|
27
37
|
source: string;
|