daftari 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/README.md +74 -25
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +85 -2
- package/dist/index.js.map +1 -1
- package/dist/search/bm25.d.ts +1 -17
- package/dist/search/bm25.d.ts.map +1 -1
- package/dist/search/bm25.js +43 -65
- package/dist/search/bm25.js.map +1 -1
- package/dist/search/embedding-provider.d.ts +8 -0
- package/dist/search/embedding-provider.d.ts.map +1 -0
- package/dist/search/embedding-provider.js +26 -0
- package/dist/search/embedding-provider.js.map +1 -0
- package/dist/search/hybrid.d.ts.map +1 -1
- package/dist/search/hybrid.js +106 -34
- package/dist/search/hybrid.js.map +1 -1
- package/dist/search/index-state.d.ts +10 -0
- package/dist/search/index-state.d.ts.map +1 -1
- package/dist/search/index-state.js +58 -3
- package/dist/search/index-state.js.map +1 -1
- package/dist/search/providers/local-minilm.d.ts +7 -0
- package/dist/search/providers/local-minilm.d.ts.map +1 -0
- package/dist/search/providers/local-minilm.js +114 -0
- package/dist/search/providers/local-minilm.js.map +1 -0
- package/dist/search/providers/openai-3-small.d.ts +5 -0
- package/dist/search/providers/openai-3-small.d.ts.map +1 -0
- package/dist/search/providers/openai-3-small.js +174 -0
- package/dist/search/providers/openai-3-small.js.map +1 -0
- package/dist/search/reindex.d.ts.map +1 -1
- package/dist/search/reindex.js +63 -13
- package/dist/search/reindex.js.map +1 -1
- package/dist/search/self-write.d.ts +4 -0
- package/dist/search/self-write.d.ts.map +1 -0
- package/dist/search/self-write.js +62 -0
- package/dist/search/self-write.js.map +1 -0
- package/dist/search/vector.d.ts +10 -1
- package/dist/search/vector.d.ts.map +1 -1
- package/dist/search/vector.js +102 -59
- package/dist/search/vector.js.map +1 -1
- package/dist/search/watcher.d.ts +18 -0
- package/dist/search/watcher.d.ts.map +1 -0
- package/dist/search/watcher.js +300 -0
- package/dist/search/watcher.js.map +1 -0
- package/dist/storage/index-db.d.ts +6 -4
- package/dist/storage/index-db.d.ts.map +1 -1
- package/dist/storage/index-db.js +262 -39
- package/dist/storage/index-db.js.map +1 -1
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +11 -3
- package/dist/tools/search.js.map +1 -1
- package/dist/tools/write.d.ts.map +1 -1
- package/dist/tools/write.js +9 -0
- package/dist/tools/write.js.map +1 -1
- package/dist/utils/config.d.ts +5 -0
- package/dist/utils/config.d.ts.map +1 -1
- package/dist/utils/config.js +53 -0
- package/dist/utils/config.js.map +1 -1
- package/package.json +4 -2
package/dist/search/hybrid.js
CHANGED
|
@@ -1,19 +1,35 @@
|
|
|
1
|
-
// Hybrid search: combine
|
|
1
|
+
// Hybrid search: combine FTS5 lexical ranking with sqlite-vec semantic
|
|
2
|
+
// ranking.
|
|
2
3
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
4
|
+
// Both halves are now SQL-native:
|
|
5
|
+
// - The lexical half runs an FTS5 MATCH query over `documents_fts` and
|
|
6
|
+
// reads SQLite's built-in BM25 score.
|
|
7
|
+
// - The vector half runs a KNN query over the sqlite-vec `embeddings_vec`
|
|
8
|
+
// virtual table, joining back to `chunks` to map content hashes onto
|
|
9
|
+
// document paths.
|
|
10
|
+
//
|
|
11
|
+
// Each ranker still produces raw scores on its own scale, so both are
|
|
12
|
+
// min-normalised to [0, 1] (divide by the top score) before being mixed by
|
|
13
|
+
// weight. Default weighting is an even 0.5 / 0.5 split.
|
|
6
14
|
//
|
|
7
15
|
// Vector ranking is best-effort. If the query cannot be embedded (model
|
|
8
16
|
// unavailable) or the index holds no embeddings, the search degrades to
|
|
9
17
|
// lexical-only and reports vectorUsed: false rather than failing.
|
|
10
18
|
import { computeDecay } from "../curation/decay.js";
|
|
11
19
|
import { ok } from "../frontmatter/types.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
20
|
+
import { embeddingToBlob, getAllDocuments, getChunksForPath, getDocument, } from "../storage/index-db.js";
|
|
21
|
+
import { buildMatchQuery, tokenize } from "./bm25.js";
|
|
22
|
+
import { embedQuery, getProvider, meanEmbedding } from "./vector.js";
|
|
15
23
|
export const DEFAULT_WEIGHTS = { bm25: 0.5, vector: 0.5 };
|
|
16
24
|
const SNIPPET_RADIUS = 140;
|
|
25
|
+
// How many KNN neighbours to ask sqlite-vec for. The vec table is per-chunk,
|
|
26
|
+
// not per-document, so this is the chunk fan-out we will then collapse to
|
|
27
|
+
// best-per-document. A multiple of the user-facing limit keeps the hybrid
|
|
28
|
+
// fusion honest — if we only fetched `limit` chunks we'd risk every one
|
|
29
|
+
// belonging to the same document and starving the rest of the candidate set.
|
|
30
|
+
// 64 is empirically generous for typical limit ≤ 10; bump if vault chunk
|
|
31
|
+
// counts grow into the millions.
|
|
32
|
+
const VEC_KNN_K = 64;
|
|
17
33
|
// Pulls a readable excerpt from a document body, centred on the earliest
|
|
18
34
|
// occurrence of any query term. Falls back to the document head when no term
|
|
19
35
|
// is found (e.g. a purely semantic match).
|
|
@@ -53,29 +69,73 @@ function normalize(scores) {
|
|
|
53
69
|
return new Map([...scores].map(([k]) => [k, 0]));
|
|
54
70
|
return new Map([...scores].map(([k, v]) => [k, v / max]));
|
|
55
71
|
}
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
|
|
72
|
+
// Runs an FTS5 MATCH against `documents_fts` and returns a path → score
|
|
73
|
+
// map. The FTS5 `bm25()` function is INVERSE (smaller = better, can be
|
|
74
|
+
// negative for strong hits), so we flip the sign to `larger = better` and
|
|
75
|
+
// then normalise the largest to 1.0 in the caller. A null query (no usable
|
|
76
|
+
// tokens after sanitization) returns an empty map.
|
|
77
|
+
function ftsRanking(db, query) {
|
|
78
|
+
if (query === null)
|
|
79
|
+
return new Map();
|
|
80
|
+
const rows = db
|
|
81
|
+
.prepare(`SELECT d.path AS path, -bm25(documents_fts) AS score
|
|
82
|
+
FROM documents_fts
|
|
83
|
+
JOIN documents AS d ON d.rowid = documents_fts.rowid
|
|
84
|
+
WHERE documents_fts MATCH ?
|
|
85
|
+
ORDER BY bm25(documents_fts)`)
|
|
86
|
+
.all(query);
|
|
87
|
+
const result = new Map();
|
|
88
|
+
for (const r of rows) {
|
|
89
|
+
// Some rows may produce a negative flipped score if FTS5 returned a
|
|
90
|
+
// positive bm25 (rare with prefix matches); shift to ensure the
|
|
91
|
+
// normalize step sees only non-negative values.
|
|
92
|
+
if (r.score > 0)
|
|
93
|
+
result.set(r.path, r.score);
|
|
94
|
+
}
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
// Runs a KNN query against the sqlite-vec `embeddings_vec` mirror, joins
|
|
98
|
+
// against `chunks` to map content hashes onto document paths, and returns a
|
|
99
|
+
// path → best-similarity map. sqlite-vec returns a cosine *distance*
|
|
100
|
+
// (smaller = closer), so similarity is `1 - distance` clamped to [0, 1].
|
|
101
|
+
// We keep each document's best-matching chunk.
|
|
102
|
+
function vecRanking(db, queryEmbedding, modelId) {
|
|
103
|
+
const queryBlob = embeddingToBlob(queryEmbedding);
|
|
104
|
+
const rows = db
|
|
105
|
+
.prepare(`SELECT c.path AS path, v.distance AS distance
|
|
106
|
+
FROM embeddings_vec AS v
|
|
107
|
+
JOIN chunks AS c ON c.content_hash = v.content_hash
|
|
108
|
+
WHERE v.embedding MATCH ?
|
|
109
|
+
AND v.model = ?
|
|
110
|
+
AND v.k = ?
|
|
111
|
+
ORDER BY v.distance`)
|
|
112
|
+
.all(queryBlob, modelId, VEC_KNN_K);
|
|
113
|
+
const result = new Map();
|
|
114
|
+
for (const r of rows) {
|
|
115
|
+
const sim = Math.max(0, 1 - r.distance);
|
|
116
|
+
const prev = result.get(r.path) ?? -Infinity;
|
|
117
|
+
if (sim > prev)
|
|
118
|
+
result.set(r.path, sim);
|
|
119
|
+
}
|
|
120
|
+
return result;
|
|
121
|
+
}
|
|
122
|
+
// Core ranker shared by query search and related-document search.
|
|
123
|
+
// `matchQuery` is the FTS5 MATCH string (already prefix-OR'd, or null);
|
|
124
|
+
// `queryEmbedding` (when present) drives sqlite-vec KNN against every
|
|
125
|
+
// indexed chunk, keeping each document's best-matching chunk.
|
|
126
|
+
// `queryTokensForSnippet` is used purely to centre snippets on the first
|
|
127
|
+
// matching term — it doesn't drive ranking.
|
|
128
|
+
function rankDocuments(db, matchQuery, queryEmbedding, queryTokensForSnippet, opts) {
|
|
60
129
|
const documents = getAllDocuments(db);
|
|
61
130
|
const byPath = new Map(documents.map((d) => [d.path, d]));
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
for (const hit of searchBm25(bm25Model, queryTokens)) {
|
|
65
|
-
bm25Raw.set(hit.path, hit.score);
|
|
66
|
-
}
|
|
67
|
-
const vectorRaw = new Map();
|
|
131
|
+
const bm25Raw = ftsRanking(db, matchQuery);
|
|
132
|
+
let vectorRaw = new Map();
|
|
68
133
|
let vectorUsed = false;
|
|
69
134
|
if (queryEmbedding) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
135
|
+
const provider = getProvider();
|
|
136
|
+
vectorRaw = vecRanking(db, queryEmbedding, provider.id);
|
|
137
|
+
if (vectorRaw.size > 0)
|
|
73
138
|
vectorUsed = true;
|
|
74
|
-
const sim = cosineSimilarity(queryEmbedding, chunk.embedding);
|
|
75
|
-
const prev = vectorRaw.get(chunk.path) ?? -Infinity;
|
|
76
|
-
if (sim > prev)
|
|
77
|
-
vectorRaw.set(chunk.path, sim);
|
|
78
|
-
}
|
|
79
139
|
}
|
|
80
140
|
const bm25Norm = normalize(bm25Raw);
|
|
81
141
|
const vectorNorm = normalize(vectorRaw);
|
|
@@ -102,7 +162,7 @@ function rankDocuments(db, queryTokens, queryEmbedding, opts) {
|
|
|
102
162
|
score,
|
|
103
163
|
bm25Score,
|
|
104
164
|
vectorScore,
|
|
105
|
-
snippet: makeSnippet(doc.content,
|
|
165
|
+
snippet: makeSnippet(doc.content, queryTokensForSnippet),
|
|
106
166
|
decay: computeDecay({
|
|
107
167
|
status: doc.status,
|
|
108
168
|
confidence: doc.confidence,
|
|
@@ -120,10 +180,11 @@ function rankDocuments(db, queryTokens, queryEmbedding, opts) {
|
|
|
120
180
|
export async function hybridSearch(db, query, options = {}) {
|
|
121
181
|
const weights = options.weights ?? DEFAULT_WEIGHTS;
|
|
122
182
|
const limit = options.limit ?? 10;
|
|
123
|
-
const
|
|
183
|
+
const matchQuery = buildMatchQuery(query);
|
|
184
|
+
const snippetTokens = tokenize(query);
|
|
124
185
|
const embedResult = await embedQuery(query);
|
|
125
186
|
const queryEmbedding = embedResult.ok ? embedResult.value : null;
|
|
126
|
-
const { hits, vectorUsed } = rankDocuments(db,
|
|
187
|
+
const { hits, vectorUsed } = rankDocuments(db, matchQuery, queryEmbedding, snippetTokens, {
|
|
127
188
|
weights,
|
|
128
189
|
limit,
|
|
129
190
|
excludePath: undefined,
|
|
@@ -137,10 +198,10 @@ export async function hybridSearch(db, query, options = {}) {
|
|
|
137
198
|
});
|
|
138
199
|
}
|
|
139
200
|
// Finds documents related to an already-indexed document. The source document
|
|
140
|
-
// itself is the query: its
|
|
141
|
-
// of its chunk embeddings drives semantic
|
|
142
|
-
//
|
|
143
|
-
// stored in the index.
|
|
201
|
+
// itself is the query: its tokens drive an FTS5 MATCH for lexical
|
|
202
|
+
// similarity, and the mean of its chunk embeddings drives semantic
|
|
203
|
+
// similarity via sqlite-vec. The source is excluded from its own results.
|
|
204
|
+
// Needs no embedding model — it reuses vectors already stored in the index.
|
|
144
205
|
export function relatedSearch(db, path, options = {}) {
|
|
145
206
|
const weights = options.weights ?? DEFAULT_WEIGHTS;
|
|
146
207
|
const limit = options.limit ?? 10;
|
|
@@ -151,11 +212,22 @@ export function relatedSearch(db, path, options = {}) {
|
|
|
151
212
|
error: new Error(`document not indexed: ${path} (try vault_reindex)`),
|
|
152
213
|
};
|
|
153
214
|
}
|
|
154
|
-
const
|
|
215
|
+
const provider = getProvider();
|
|
216
|
+
const chunkVectors = getChunksForPath(db, path, provider.id, provider.dim)
|
|
155
217
|
.map((c) => c.embedding)
|
|
156
218
|
.filter((e) => e !== null);
|
|
157
219
|
const queryEmbedding = meanEmbedding(chunkVectors);
|
|
158
|
-
|
|
220
|
+
// Build the FTS5 match string from the source document's stored token
|
|
221
|
+
// list (title + tags + body, tokenized at index time). Cap the token
|
|
222
|
+
// count: a long document's full token list produces a MATCH string that
|
|
223
|
+
// is mostly noise and forces FTS5 to do enormous work. The most
|
|
224
|
+
// informative terms are typically the rarer ones, but since we don't
|
|
225
|
+
// have IDF readily available here we use a simple truncate to the first
|
|
226
|
+
// N unique tokens — title + early body — which is the same heuristic the
|
|
227
|
+
// hand-rolled BM25 implicitly used.
|
|
228
|
+
const sourceTokens = [...new Set(doc.tokens)].slice(0, 64);
|
|
229
|
+
const matchQuery = sourceTokens.length === 0 ? null : sourceTokens.map((t) => `${t}*`).join(" OR ");
|
|
230
|
+
const { hits, vectorUsed } = rankDocuments(db, matchQuery, queryEmbedding, doc.tokens, {
|
|
159
231
|
weights,
|
|
160
232
|
limit,
|
|
161
233
|
excludePath: path,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/search/hybrid.ts"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/search/hybrid.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,WAAW;AACX,EAAE;AACF,kCAAkC;AAClC,yEAAyE;AACzE,0CAA0C;AAC1C,4EAA4E;AAC5E,yEAAyE;AACzE,sBAAsB;AACtB,EAAE;AACF,sEAAsE;AACtE,2EAA2E;AAC3E,wDAAwD;AACxD,EAAE;AACF,wEAAwE;AACxE,wEAAwE;AACxE,kEAAkE;AAElE,OAAO,EAAE,YAAY,EAAmB,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EACL,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,WAAW,GAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAOrE,MAAM,CAAC,MAAM,eAAe,GAAkB,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAsBzE,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,6EAA6E;AAC7E,0EAA0E;AAC1E,0EAA0E;AAC1E,wEAAwE;AACxE,6EAA6E;AAC7E,yEAAyE;AACzE,iCAAiC;AACjC,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB,yEAAyE;AACzE,6EAA6E;AAC7E,2CAA2C;AAC3C,SAAS,WAAW,CAAC,OAAe,EAAE,WAAqB;IACzD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC;YAAE,KAAK,GAAG,EAAE,CAAC;IAC5D,CAAC;IAED,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC,MAAM,GAAG,cAAc,GAAG,CAAC;YAC1C,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,GAAG;YAC9C,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,cAAc,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,GAAG,cAAc,CAAC,CAAC;IAC/D,IAAI,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IACvC,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM;QAAE,OAAO,GAAG,GAAG,OAAO,GAAG,CAAC;IACpD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,6EAA6E;AAC7E,wCAAwC;AACxC,SAAS,SAAS,CAAC,MAA2B;IAC5C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,IAAI,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,CAAC,CAAC;IACtD,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,wEAAwE;AACxE,uEAAuE;AACvE,0EAA0E;AAC1E,2EAA2E;AAC3E,mDAAmD;AACnD,SAAS,UAAU,CAAC,EAAW,EAAE,KAAoB;IACnD,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN;;;;qCAI+B,CAChC;SACA,GAAG,CAAC,KAAK,CAAsC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,oEAAoE;QACpE,gEAAgE;QAChE,gDAAgD;QAChD,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,4EAA4E;AAC5E,qEAAqE;AACrE,yEAAyE;AACzE,+CAA+C;AAC/C,SAAS,UAAU,CACjB,EAAW,EACX,cAA4B,EAC5B,OAAe;IAEf,MAAM,SAAS,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN;;;;;;4BAMsB,CACvB;SACA,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAyC,CAAC;IAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC7C,IAAI,GAAG,GAAG,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAQD,kEAAkE;AAClE,wEAAwE;AACxE,sEAAsE;AACtE,8DAA8D;AAC9D,yEAAyE;AACzE,4CAA4C;AAC5C,SAAS,aAAa,CACpB,EAAW,EACX,UAAyB,EACzB,cAAmC,EACnC,qBAA+B,EAC/B,IAAiB;IAEjB,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,MAAM,OAAO,GAAG,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAE3C,IAAI,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;QAC/B,SAAS,GAAG,UAAU,CAAC,EAAE,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC;YAAE,UAAU,GAAG,IAAI,CAAC;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IAExC,yEAAyE;IACzE,MAAM,OAAO,GAAkB,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAElF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE/E,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,IAAI,KAAK,IAAI,CAAC,WAAW;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;QACtE,IAAI,KAAK,IAAI,CAAC;YAAE,SAAS;QACzB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI;YACJ,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK;YACL,SAAS;YACT,WAAW;YACX,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,qBAAqB,CAAC;YACxD,KAAK,EAAE,YAAY,CAAC;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,OAAO;gBACrB,aAAa,EAAE,GAAG,CAAC,YAAY;aAChC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;AACzD,CAAC;AAOD,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAW,EACX,KAAa,EACb,UAA+B,EAAE;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEtC,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE;QACxF,OAAO;QACP,KAAK;QACL,WAAW,EAAE,SAAS;KACvB,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;QACR,KAAK;QACL,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,UAAU;QACV,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACtD,IAAI;KACL,CAAC,CAAC;AACL,CAAC;AAUD,8EAA8E;AAC9E,kEAAkE;AAClE,mEAAmE;AACnE,0EAA0E;AAC1E,4EAA4E;AAC5E,MAAM,UAAU,aAAa,CAC3B,EAAW,EACX,IAAY,EACZ,UAA+B,EAAE;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAElC,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,IAAI,KAAK,CAAC,yBAAyB,IAAI,sBAAsB,CAAC;SACtE,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,YAAY,GAAG,gBAAgB,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC;SACvE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;SACvB,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAEnD,sEAAsE;IACtE,qEAAqE;IACrE,wEAAwE;IACxE,gEAAgE;IAChE,qEAAqE;IACrE,wEAAwE;IACxE,yEAAyE;IACzE,oCAAoC;IACpC,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3D,MAAM,UAAU,GACd,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEnF,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE;QACrF,OAAO;QACP,KAAK;QACL,WAAW,EAAE,IAAI;KAClB,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;QACR,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,UAAU;QACV,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACtD,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export type IndexStatus = "ready" | "indexing" | "error";
|
|
2
|
+
export type ModelStatus = "cold" | "warming" | "ready" | "error";
|
|
2
3
|
export interface IndexSnapshot {
|
|
3
4
|
status: IndexStatus;
|
|
4
5
|
done: number;
|
|
@@ -6,12 +7,21 @@ export interface IndexSnapshot {
|
|
|
6
7
|
error: string | null;
|
|
7
8
|
startedAt: string | null;
|
|
8
9
|
finishedAt: string | null;
|
|
10
|
+
modelStatus: ModelStatus;
|
|
11
|
+
modelError: string | null;
|
|
9
12
|
}
|
|
10
13
|
export declare function getIndexStatus(): IndexSnapshot;
|
|
11
14
|
export declare function markIndexing(): void;
|
|
12
15
|
export declare function setIndexProgress(done: number, total: number): void;
|
|
13
16
|
export declare function markIndexReady(): void;
|
|
14
17
|
export declare function markIndexError(message: string): void;
|
|
18
|
+
export declare function markModelWarming(): void;
|
|
19
|
+
export declare function markModelReady(): void;
|
|
20
|
+
export declare function markModelError(message: string): void;
|
|
15
21
|
export declare function resetIndexState(): void;
|
|
22
|
+
export declare function markPathIndexing(relPath: string): void;
|
|
23
|
+
export declare function markPathReady(relPath: string): void;
|
|
24
|
+
export declare function isPathIndexing(relPath: string): boolean;
|
|
25
|
+
export declare function getInflightPaths(): string[];
|
|
16
26
|
export declare function indexingBusyMessage(snapshot: IndexSnapshot): string;
|
|
17
27
|
//# sourceMappingURL=index-state.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-state.d.ts","sourceRoot":"","sources":["../../src/search/index-state.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index-state.d.ts","sourceRoot":"","sources":["../../src/search/index-state.ts"],"names":[],"mappings":"AAiBA,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,UAAU,GAAG,OAAO,CAAC;AAOzD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,WAAW,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAiBD,wBAAgB,cAAc,IAAI,aAAa,CAE9C;AAED,wBAAgB,YAAY,IAAI,IAAI,CAUnC;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAGlE;AAED,wBAAgB,cAAc,IAAI,IAAI,CAOrC;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAOpD;AAMD,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC;AAED,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAEpD;AAID,wBAAgB,eAAe,IAAI,IAAI,CAGtC;AAYD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAEtD;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAEnD;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAEvD;AAED,wBAAgB,gBAAgB,IAAI,MAAM,EAAE,CAE3C;AAQD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,GAAG,MAAM,CAWnE"}
|
|
@@ -6,6 +6,14 @@
|
|
|
6
6
|
// they consult this module to decide whether to serve or to reply "still
|
|
7
7
|
// indexing — N/M chunks". A single in-process snapshot is enough because the
|
|
8
8
|
// server is one Node process per vault.
|
|
9
|
+
//
|
|
10
|
+
// Embedding model load is tracked separately from indexing because the two
|
|
11
|
+
// are orthogonal: a fully-cached reindex (every chunk hash already has an
|
|
12
|
+
// embedding row) never loads the model at all, while a search may need to
|
|
13
|
+
// load the model even when the index is "ready". The `modelStatus` lets a
|
|
14
|
+
// tool report "embeddings warming" instead of misleading a client with a
|
|
15
|
+
// generic indexing message when the slow operation is actually the model
|
|
16
|
+
// load.
|
|
9
17
|
function freshState() {
|
|
10
18
|
return {
|
|
11
19
|
status: "ready",
|
|
@@ -14,6 +22,8 @@ function freshState() {
|
|
|
14
22
|
error: null,
|
|
15
23
|
startedAt: null,
|
|
16
24
|
finishedAt: null,
|
|
25
|
+
modelStatus: "cold",
|
|
26
|
+
modelError: null,
|
|
17
27
|
};
|
|
18
28
|
}
|
|
19
29
|
let state = freshState();
|
|
@@ -22,6 +32,7 @@ export function getIndexStatus() {
|
|
|
22
32
|
}
|
|
23
33
|
export function markIndexing() {
|
|
24
34
|
state = {
|
|
35
|
+
...state,
|
|
25
36
|
status: "indexing",
|
|
26
37
|
done: 0,
|
|
27
38
|
total: 0,
|
|
@@ -51,17 +62,61 @@ export function markIndexError(message) {
|
|
|
51
62
|
finishedAt: new Date().toISOString(),
|
|
52
63
|
};
|
|
53
64
|
}
|
|
65
|
+
// Model lifecycle transitions. Called by vector.ts as the memoised
|
|
66
|
+
// extractor promise progresses; the warm-up entry point and the lazy
|
|
67
|
+
// first-embed path both flow through the same getExtractor() so callers
|
|
68
|
+
// here do not need to special-case the trigger.
|
|
69
|
+
export function markModelWarming() {
|
|
70
|
+
state = { ...state, modelStatus: "warming", modelError: null };
|
|
71
|
+
}
|
|
72
|
+
export function markModelReady() {
|
|
73
|
+
state = { ...state, modelStatus: "ready", modelError: null };
|
|
74
|
+
}
|
|
75
|
+
export function markModelError(message) {
|
|
76
|
+
state = { ...state, modelStatus: "error", modelError: message };
|
|
77
|
+
}
|
|
54
78
|
// Tests load tools without running main(); resetting the singleton between
|
|
55
79
|
// suites keeps cross-test pollution out of the state machine.
|
|
56
80
|
export function resetIndexState() {
|
|
57
81
|
state = freshState();
|
|
82
|
+
inflightPaths.clear();
|
|
83
|
+
}
|
|
84
|
+
// Per-path "currently indexing this one file" tracker. The global `status`
|
|
85
|
+
// above describes whole-vault reindex passes (startup / vault_reindex). The
|
|
86
|
+
// fs.watch path drives many concurrent per-file re-indexes that must not
|
|
87
|
+
// block unrelated writes or searches; their in-flight membership lives here.
|
|
88
|
+
// Membership is advisory: searches that read a slightly-stale row for a few
|
|
89
|
+
// hundred ms while a per-file index is mid-write is acceptable. The
|
|
90
|
+
// per-path *serialisation* between an external edit and a Daftari write is
|
|
91
|
+
// the file-level write lock in src/access/locks.ts.
|
|
92
|
+
const inflightPaths = new Set();
|
|
93
|
+
export function markPathIndexing(relPath) {
|
|
94
|
+
inflightPaths.add(relPath);
|
|
95
|
+
}
|
|
96
|
+
export function markPathReady(relPath) {
|
|
97
|
+
inflightPaths.delete(relPath);
|
|
98
|
+
}
|
|
99
|
+
export function isPathIndexing(relPath) {
|
|
100
|
+
return inflightPaths.has(relPath);
|
|
101
|
+
}
|
|
102
|
+
export function getInflightPaths() {
|
|
103
|
+
return [...inflightPaths];
|
|
58
104
|
}
|
|
59
105
|
// Formatted message tools return to clients while indexing is in progress.
|
|
60
106
|
// One place so the phrasing is consistent across vault_search, vault_write,
|
|
61
|
-
// vault_reindex, etc.
|
|
107
|
+
// vault_reindex, etc. When the model is warming and the index is otherwise
|
|
108
|
+
// ready, the message says so explicitly — the slow operation is the model
|
|
109
|
+
// load, not an indexing pass, and a client that retries blindly against an
|
|
110
|
+
// "indexing" message is missing useful context.
|
|
62
111
|
export function indexingBusyMessage(snapshot) {
|
|
63
|
-
if (snapshot.
|
|
64
|
-
|
|
112
|
+
if (snapshot.status === "indexing") {
|
|
113
|
+
if (snapshot.total > 0) {
|
|
114
|
+
return `vault is still indexing (${snapshot.done}/${snapshot.total} chunks) — try again shortly`;
|
|
115
|
+
}
|
|
116
|
+
return `vault is still indexing — try again shortly`;
|
|
117
|
+
}
|
|
118
|
+
if (snapshot.modelStatus === "warming") {
|
|
119
|
+
return `embedding model is warming — try again shortly`;
|
|
65
120
|
}
|
|
66
121
|
return `vault is still indexing — try again shortly`;
|
|
67
122
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-state.js","sourceRoot":"","sources":["../../src/search/index-state.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAC9E,yEAAyE;AACzE,6EAA6E;AAC7E,wCAAwC;
|
|
1
|
+
{"version":3,"file":"index-state.js","sourceRoot":"","sources":["../../src/search/index-state.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAC9E,yEAAyE;AACzE,6EAA6E;AAC7E,wCAAwC;AACxC,EAAE;AACF,2EAA2E;AAC3E,0EAA0E;AAC1E,0EAA0E;AAC1E,0EAA0E;AAC1E,yEAAyE;AACzE,yEAAyE;AACzE,QAAQ;AAsBR,SAAS,UAAU;IACjB,OAAO;QACL,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI;QAChB,WAAW,EAAE,MAAM;QACnB,UAAU,EAAE,IAAI;KACjB,CAAC;AACJ,CAAC;AAED,IAAI,KAAK,GAAkB,UAAU,EAAE,CAAC;AAExC,MAAM,UAAU,cAAc;IAC5B,OAAO,EAAE,GAAG,KAAK,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,KAAK,GAAG;QACN,GAAG,KAAK;QACR,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,IAAI;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,KAAa;IAC1D,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO;IACxC,KAAK,GAAG,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,KAAK,GAAG;QACN,GAAG,KAAK;QACR,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,KAAK,GAAG;QACN,GAAG,KAAK;QACR,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,qEAAqE;AACrE,wEAAwE;AACxE,gDAAgD;AAChD,MAAM,UAAU,gBAAgB;IAC9B,KAAK,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,KAAK,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,KAAK,GAAG,EAAE,GAAG,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AAClE,CAAC;AAED,2EAA2E;AAC3E,8DAA8D;AAC9D,MAAM,UAAU,eAAe;IAC7B,KAAK,GAAG,UAAU,EAAE,CAAC;IACrB,aAAa,CAAC,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,2EAA2E;AAC3E,4EAA4E;AAC5E,yEAAyE;AACzE,6EAA6E;AAC7E,4EAA4E;AAC5E,oEAAoE;AACpE,2EAA2E;AAC3E,oDAAoD;AACpD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;AAExC,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,OAAO,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,CAAC,GAAG,aAAa,CAAC,CAAC;AAC5B,CAAC;AAED,2EAA2E;AAC3E,4EAA4E;AAC5E,2EAA2E;AAC3E,0EAA0E;AAC1E,2EAA2E;AAC3E,gDAAgD;AAChD,MAAM,UAAU,mBAAmB,CAAC,QAAuB;IACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,4BAA4B,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,8BAA8B,CAAC;QACnG,CAAC;QACD,OAAO,6CAA6C,CAAC;IACvD,CAAC;IACD,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACvC,OAAO,gDAAgD,CAAC;IAC1D,CAAC;IACD,OAAO,6CAA6C,CAAC;AACvD,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from "../embedding-provider.js";
|
|
2
|
+
export declare const LOCAL_MINILM_ID = "local-minilm";
|
|
3
|
+
export declare const LOCAL_MINILM_DIM = 384;
|
|
4
|
+
export declare function isLocalMinilmLoaded(): boolean;
|
|
5
|
+
export declare function resetLocalMinilmForTests(): void;
|
|
6
|
+
export declare const localMinilmProvider: EmbeddingProvider;
|
|
7
|
+
//# sourceMappingURL=local-minilm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-minilm.d.ts","sourceRoot":"","sources":["../../../src/search/providers/local-minilm.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAGlE,eAAO,MAAM,eAAe,iBAAiB,CAAC;AAC9C,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAwDpC,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAKD,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C;AA+CD,eAAO,MAAM,mBAAmB,EAAE,iBAKjC,CAAC"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// local-minilm — sentence-transformers/all-MiniLM-L6-v2 run locally via
|
|
2
|
+
// @huggingface/transformers. 384 dims, fully local, no network at query time
|
|
3
|
+
// once the model is cached. The default embedding provider.
|
|
4
|
+
//
|
|
5
|
+
// The model loads lazily and is memoised for the process; the first call pays
|
|
6
|
+
// a ~500ms-to-multi-second cold start (model file download on first run,
|
|
7
|
+
// CPU init thereafter). A warm-up entry point exists so the server can pay
|
|
8
|
+
// that cost in the background after startup rather than on the first user
|
|
9
|
+
// search. Failures (no network on first run, model download blocked) come
|
|
10
|
+
// back as Result.err so the caller can degrade to BM25-only — embedding is
|
|
11
|
+
// never load-bearing for the server staying up.
|
|
12
|
+
import { err, ok } from "../../frontmatter/types.js";
|
|
13
|
+
import { markModelError, markModelReady, markModelWarming } from "../index-state.js";
|
|
14
|
+
export const LOCAL_MINILM_ID = "local-minilm";
|
|
15
|
+
export const LOCAL_MINILM_DIM = 384;
|
|
16
|
+
const HF_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
17
|
+
// Texts are embedded in fixed-size sub-batches rather than one call. The model
|
|
18
|
+
// pads every batch to its longest sequence and allocates activation tensors
|
|
19
|
+
// proportional to the batch size, so an unbounded batch makes peak memory
|
|
20
|
+
// scale with the whole vault — a few hundred documents is enough to exhaust
|
|
21
|
+
// RAM and stall in a GC death spiral. A small fixed batch keeps peak memory
|
|
22
|
+
// flat regardless of vault size.
|
|
23
|
+
//
|
|
24
|
+
// 8 was measured as the sweet spot: on CPU inference larger batches were both
|
|
25
|
+
// heavier (more activation memory) and slower (more compute wasted padding
|
|
26
|
+
// short chunks up to the batch's longest sequence), not faster.
|
|
27
|
+
const EMBED_BATCH_SIZE = 8;
|
|
28
|
+
let extractorPromise = null;
|
|
29
|
+
async function getExtractor() {
|
|
30
|
+
if (!extractorPromise) {
|
|
31
|
+
// The model is being loaded for the first time. Surface that in the
|
|
32
|
+
// process-wide IndexState so tools can tell a "warming embeddings" pause
|
|
33
|
+
// apart from a real indexing pass. The state is best-effort signal — we
|
|
34
|
+
// never let a state-machine wobble break embedding.
|
|
35
|
+
markModelWarming();
|
|
36
|
+
extractorPromise = import("@huggingface/transformers").then(({ pipeline }) => pipeline("feature-extraction", HF_MODEL)).then((extractor) => {
|
|
37
|
+
markModelReady();
|
|
38
|
+
return extractor;
|
|
39
|
+
}, (e) => {
|
|
40
|
+
// Surface the failure but reset the cached promise so the next call
|
|
41
|
+
// can try again — useful for the no-network-on-first-run case where
|
|
42
|
+
// a later retry might succeed. Without the reset a single transient
|
|
43
|
+
// failure would poison the process for its whole lifetime.
|
|
44
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
45
|
+
markModelError(reason);
|
|
46
|
+
extractorPromise = null;
|
|
47
|
+
throw e;
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return extractorPromise;
|
|
51
|
+
}
|
|
52
|
+
// Returns true once the model has been loaded into memory (the memoised
|
|
53
|
+
// promise has resolved). Used by tests and by lazy-load coverage to assert
|
|
54
|
+
// that startup paths do not invoke the model when they should not.
|
|
55
|
+
export function isLocalMinilmLoaded() {
|
|
56
|
+
return extractorPromise !== null;
|
|
57
|
+
}
|
|
58
|
+
// Test-only: clear the memoised extractor so a fresh import is forced on
|
|
59
|
+
// the next call. Production code must not invoke this — the model load is
|
|
60
|
+
// expensive and the whole point of the memo is that it survives the process.
|
|
61
|
+
export function resetLocalMinilmForTests() {
|
|
62
|
+
extractorPromise = null;
|
|
63
|
+
}
|
|
64
|
+
async function warm() {
|
|
65
|
+
try {
|
|
66
|
+
await getExtractor();
|
|
67
|
+
return ok(undefined);
|
|
68
|
+
}
|
|
69
|
+
catch (e) {
|
|
70
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
71
|
+
return err(new Error(`embedding model warm-up failed: ${reason}`));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async function embed(texts, onProgress) {
|
|
75
|
+
if (texts.length === 0)
|
|
76
|
+
return ok([]);
|
|
77
|
+
try {
|
|
78
|
+
const extractor = await getExtractor();
|
|
79
|
+
const vectors = [];
|
|
80
|
+
for (let start = 0; start < texts.length; start += EMBED_BATCH_SIZE) {
|
|
81
|
+
const batch = texts.slice(start, start + EMBED_BATCH_SIZE);
|
|
82
|
+
const output = await extractor(batch, {
|
|
83
|
+
pooling: "mean",
|
|
84
|
+
normalize: true,
|
|
85
|
+
});
|
|
86
|
+
const dim = output.dims[output.dims.length - 1] ?? LOCAL_MINILM_DIM;
|
|
87
|
+
for (let i = 0; i < batch.length; i++) {
|
|
88
|
+
vectors.push(output.data.slice(i * dim, (i + 1) * dim));
|
|
89
|
+
}
|
|
90
|
+
// Progress is a best-effort side channel: a failing reporter (e.g. a
|
|
91
|
+
// closed stderr pipe) must never abort embedding the vault.
|
|
92
|
+
if (onProgress) {
|
|
93
|
+
try {
|
|
94
|
+
onProgress(vectors.length, texts.length);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// ignore — progress reporting is not load-bearing
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return ok(vectors);
|
|
102
|
+
}
|
|
103
|
+
catch (e) {
|
|
104
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
105
|
+
return err(new Error(`embedding failed: ${reason}`));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
export const localMinilmProvider = {
|
|
109
|
+
id: LOCAL_MINILM_ID,
|
|
110
|
+
dim: LOCAL_MINILM_DIM,
|
|
111
|
+
warm,
|
|
112
|
+
embed,
|
|
113
|
+
};
|
|
114
|
+
//# sourceMappingURL=local-minilm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-minilm.js","sourceRoot":"","sources":["../../../src/search/providers/local-minilm.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,6EAA6E;AAC7E,4DAA4D;AAC5D,EAAE;AACF,8EAA8E;AAC9E,yEAAyE;AACzE,2EAA2E;AAC3E,0EAA0E;AAC1E,0EAA0E;AAC1E,2EAA2E;AAC3E,gDAAgD;AAEhD,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,4BAA4B,CAAC;AAElE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAErF,MAAM,CAAC,MAAM,eAAe,GAAG,cAAc,CAAC;AAC9C,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AACpC,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAE3C,+EAA+E;AAC/E,4EAA4E;AAC5E,0EAA0E;AAC1E,4EAA4E;AAC5E,4EAA4E;AAC5E,iCAAiC;AACjC,EAAE;AACF,8EAA8E;AAC9E,2EAA2E;AAC3E,gEAAgE;AAChE,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAO3B,IAAI,gBAAgB,GAA8B,IAAI,CAAC;AAEvD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,oEAAoE;QACpE,yEAAyE;QACzE,wEAAwE;QACxE,oDAAoD;QACpD,gBAAgB,EAAE,CAAC;QACnB,gBAAgB,GACd,MAAM,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CACxD,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAE3C,CAAC,IAAI,CACJ,CAAC,SAAS,EAAE,EAAE;YACZ,cAAc,EAAE,CAAC;YACjB,OAAO,SAAS,CAAC;QACnB,CAAC,EACD,CAAC,CAAC,EAAE,EAAE;YACJ,oEAAoE;YACpE,oEAAoE;YACpE,oEAAoE;YACpE,2DAA2D;YAC3D,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC1D,cAAc,CAAC,MAAM,CAAC,CAAC;YACvB,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,CAAC,CAAC;QACV,CAAC,CACF,CAAC;IACJ,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,wEAAwE;AACxE,2EAA2E;AAC3E,mEAAmE;AACnE,MAAM,UAAU,mBAAmB;IACjC,OAAO,gBAAgB,KAAK,IAAI,CAAC;AACnC,CAAC;AAED,yEAAyE;AACzE,0EAA0E;AAC1E,6EAA6E;AAC7E,MAAM,UAAU,wBAAwB;IACtC,gBAAgB,GAAG,IAAI,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,IAAI,CAAC;QACH,MAAM,YAAY,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,mCAAmC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,KAAK,CAClB,KAAe,EACf,UAAkD;IAElD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;QACvC,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,gBAAgB,EAAE,CAAC;YACpE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,gBAAgB,CAAC,CAAC;YAC3D,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE;gBACpC,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,gBAAgB,CAAC;YACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YAC1D,CAAC;YACD,qEAAqE;YACrE,4DAA4D;YAC5D,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBAAC,MAAM,CAAC;oBACP,kDAAkD;gBACpD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAsB;IACpD,EAAE,EAAE,eAAe;IACnB,GAAG,EAAE,gBAAgB;IACrB,IAAI;IACJ,KAAK;CACN,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from "../embedding-provider.js";
|
|
2
|
+
export declare const OPENAI_3_SMALL_ID = "openai-3-small";
|
|
3
|
+
export declare const OPENAI_3_SMALL_DIM = 1536;
|
|
4
|
+
export declare function makeOpenAi3SmallProvider(apiKey: string): EmbeddingProvider;
|
|
5
|
+
//# sourceMappingURL=openai-3-small.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-3-small.d.ts","sourceRoot":"","sources":["../../../src/search/providers/openai-3-small.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAElE,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAClD,eAAO,MAAM,kBAAkB,OAAO,CAAC;AAoIvC,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,iBAAiB,CAgC1E"}
|