nexum-code 0.18.7 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/dist/{app-aqWxJw2S.mjs → app-CYLTD0b3.mjs} +19994 -15221
- package/dist/{dispatch-DQ6ufFQt.mjs → dispatch-DnZRjRNm.mjs} +21929 -9483
- package/dist/{dispatch-ByfuGE0h.mjs → dispatch-cUxbpzTE.mjs} +1 -1
- package/dist/{markit-XGWU21rO.mjs → esm-C0toZ4e0.mjs} +19 -2344
- package/dist/main.mjs +1 -1
- package/dist/markdown-BwvhOvCA.mjs +25943 -0
- package/dist/markit-LwaDGgYt.mjs +2334 -0
- package/dist/{src-pqi1HIGh.mjs → src-C6YAs3fd.mjs} +1 -1
- package/dist/{src-BPh-7A8j.mjs → src-C8mW2wSe.mjs} +779 -276
- package/dist/{text-input-dialog-CAaXBGxG.mjs → text-input-dialog-BbyJmVbH.mjs} +1 -1
- package/dist/{text-input-dialog-BIhpfWk_.mjs → text-input-dialog-CAxwn6Ze.mjs} +930 -74
- package/docs/architecture.md +80 -0
- package/docs/configuration.md +204 -0
- package/docs/line.md +159 -0
- package/docs/mcp.md +141 -0
- package/docs/plugins.md +198 -0
- package/docs/sessions.md +78 -0
- package/docs/skills.md +76 -0
- package/package.json +9 -8
|
@@ -4,14 +4,14 @@ import { dirname as __cjsShimDirname } from 'node:path';
|
|
|
4
4
|
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
5
|
const __dirname = __cjsShimDirname(__filename);
|
|
6
6
|
import { createHash, randomBytes } from "node:crypto";
|
|
7
|
-
import
|
|
7
|
+
import Kt, { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
8
|
+
import path, { basename, join, posix, resolve, win32 } from "node:path";
|
|
9
|
+
import ro, { mkdir, open, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
|
|
8
10
|
import I from "fs";
|
|
9
11
|
import Qr, { EventEmitter } from "events";
|
|
10
|
-
import js, { dirname, parse } from "path";
|
|
12
|
+
import js, { dirname as dirname$1, parse } from "path";
|
|
11
13
|
import { Buffer as Buffer$1 } from "buffer";
|
|
12
14
|
import Cs from "node:stream";
|
|
13
|
-
import Kt, { createReadStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
|
|
14
|
-
import path, { basename, join, posix, win32 } from "node:path";
|
|
15
15
|
import { DatabaseSync } from "node:sqlite";
|
|
16
16
|
import { EventEmitter as EventEmitter$1 } from "node:events";
|
|
17
17
|
import { StringDecoder } from "node:string_decoder";
|
|
@@ -181,6 +181,93 @@ const parse$1 = function(p) {
|
|
|
181
181
|
};
|
|
182
182
|
};
|
|
183
183
|
//#endregion
|
|
184
|
+
//#region ../../packages/knowledge/src/exact-search.ts
|
|
185
|
+
const CONTEXT_LENGTH = 160;
|
|
186
|
+
const segmenter = new Intl.Segmenter();
|
|
187
|
+
function normalizeText(text) {
|
|
188
|
+
return text.normalize("NFKC").replaceAll(/\s+/gu, " ").trim();
|
|
189
|
+
}
|
|
190
|
+
function exactTextPattern(terms) {
|
|
191
|
+
const unique = [...new Set(terms.map(normalizeText).filter(Boolean))];
|
|
192
|
+
if (unique.length === 0) return void 0;
|
|
193
|
+
unique.sort((left, right) => right.length - left.length);
|
|
194
|
+
const patterns = unique.map((term) => {
|
|
195
|
+
const escaped = term.replaceAll(/[.*+?^$(){}|[\]\\]/g, "\\$&");
|
|
196
|
+
const before = /^[a-z0-9_]/i.test(term) ? "(?<![a-z0-9_])" : "";
|
|
197
|
+
const after = /[a-z0-9_]$/i.test(term) ? "(?![a-z0-9_])" : "";
|
|
198
|
+
return before + escaped + after;
|
|
199
|
+
});
|
|
200
|
+
return new RegExp(patterns.join("|"), "giu");
|
|
201
|
+
}
|
|
202
|
+
/** Translate an offset within a normalized line back to the original grapheme boundary. */
|
|
203
|
+
function sourceOffset(line, offset, end) {
|
|
204
|
+
if (line.text === normalizeText(line.text)) return line.sourceOffset + offset;
|
|
205
|
+
let consumed = 0;
|
|
206
|
+
let previousSpace = true;
|
|
207
|
+
for (const { segment, index } of segmenter.segment(line.text)) {
|
|
208
|
+
let normalized = segment.normalize("NFKC").replaceAll(/\s+/gu, " ");
|
|
209
|
+
if (previousSpace && normalized.startsWith(" ")) normalized = normalized.slice(1);
|
|
210
|
+
const next = consumed + normalized.length;
|
|
211
|
+
if (end ? offset <= next : offset < next) return line.sourceOffset + index + (end ? segment.length : 0);
|
|
212
|
+
if (normalized.length > 0) previousSpace = normalized.endsWith(" ");
|
|
213
|
+
consumed = next;
|
|
214
|
+
}
|
|
215
|
+
return line.sourceOffset + line.text.length;
|
|
216
|
+
}
|
|
217
|
+
function* indexLines(text) {
|
|
218
|
+
let number = 1;
|
|
219
|
+
let normalizedOffset = 0;
|
|
220
|
+
for (const match of text.matchAll(/[^\r\n]+|\r\n|\r|\n/g)) {
|
|
221
|
+
const line = match[0];
|
|
222
|
+
if (line.startsWith("\r") || line === "\n") {
|
|
223
|
+
number++;
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
const normalized = normalizeText(line);
|
|
227
|
+
if (normalized.length === 0) continue;
|
|
228
|
+
yield {
|
|
229
|
+
text: line,
|
|
230
|
+
number,
|
|
231
|
+
sourceOffset: match.index,
|
|
232
|
+
normalizedOffset
|
|
233
|
+
};
|
|
234
|
+
normalizedOffset += normalized.length + 1;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
function* findExactTextEvidence(text, pattern) {
|
|
238
|
+
const matches = normalizeText(text).matchAll(pattern);
|
|
239
|
+
let current = matches.next();
|
|
240
|
+
if (current.done) return;
|
|
241
|
+
const lines = indexLines(text);
|
|
242
|
+
const firstLine = lines.next();
|
|
243
|
+
if (firstLine.done) return;
|
|
244
|
+
let line = firstLine.value;
|
|
245
|
+
let next = lines.next();
|
|
246
|
+
let reportedLine = 0;
|
|
247
|
+
for (; !current.done; current = matches.next()) {
|
|
248
|
+
const match = current.value;
|
|
249
|
+
const end = match.index + match[0].length;
|
|
250
|
+
while (!next.done && next.value.normalizedOffset <= match.index) {
|
|
251
|
+
line = next.value;
|
|
252
|
+
next = lines.next();
|
|
253
|
+
}
|
|
254
|
+
const first = line;
|
|
255
|
+
if (first.number === reportedLine) continue;
|
|
256
|
+
while (!next.done && next.value.normalizedOffset < end) {
|
|
257
|
+
line = next.value;
|
|
258
|
+
next = lines.next();
|
|
259
|
+
}
|
|
260
|
+
const startOffset = sourceOffset(first, match.index - first.normalizedOffset, false);
|
|
261
|
+
const endOffset = sourceOffset(line, end - line.normalizedOffset, true);
|
|
262
|
+
reportedLine = first.number;
|
|
263
|
+
yield {
|
|
264
|
+
content: text.slice(Math.max(first.sourceOffset, startOffset - CONTEXT_LENGTH), Math.min(line.sourceOffset + line.text.length, endOffset + CONTEXT_LENGTH)),
|
|
265
|
+
matchedText: text.slice(startOffset, endOffset),
|
|
266
|
+
textLine: first.number
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
//#endregion
|
|
184
271
|
//#region ../../packages/knowledge/src/store.ts
|
|
185
272
|
const ENTITY_TYPES$1 = new Set([
|
|
186
273
|
"person",
|
|
@@ -840,18 +927,59 @@ var KnowledgeStore = class {
|
|
|
840
927
|
entityId: asString(row["entity_id"])
|
|
841
928
|
}));
|
|
842
929
|
}
|
|
930
|
+
async searchExact(terms, options = {}) {
|
|
931
|
+
const pattern = exactTextPattern(terms);
|
|
932
|
+
const limit = Math.min(100, Math.floor(options.limit ?? 20));
|
|
933
|
+
if (pattern === void 0 || !(limit > 0)) return [];
|
|
934
|
+
await this.init();
|
|
935
|
+
if (this.db === void 0) return [];
|
|
936
|
+
const scope = options.documentId === void 0 ? [] : [options.documentId];
|
|
937
|
+
const documents = this.db.prepare(`SELECT d.id, d.source_id, s.name AS source_name, d.content
|
|
938
|
+
FROM knowledge_documents d JOIN knowledge_sources s ON s.id = d.source_id
|
|
939
|
+
WHERE d.status = 'completed' ${scope.length === 0 ? "" : "AND d.id = ?"}
|
|
940
|
+
ORDER BY d.rowid DESC`).iterate(...scope);
|
|
941
|
+
const chunks = this.db.prepare("SELECT raw_content, content FROM knowledge_chunks WHERE document_id = ? ORDER BY rank");
|
|
942
|
+
const results = [];
|
|
943
|
+
for (const document of documents) {
|
|
944
|
+
const provenance = {
|
|
945
|
+
documentId: document.id,
|
|
946
|
+
sourceId: document.source_id,
|
|
947
|
+
sourceName: document.source_name
|
|
948
|
+
};
|
|
949
|
+
if (document.content?.trim()) for (const evidence of findExactTextEvidence(document.content, pattern)) {
|
|
950
|
+
results.push({
|
|
951
|
+
...provenance,
|
|
952
|
+
...evidence
|
|
953
|
+
});
|
|
954
|
+
if (results.length >= limit) return results;
|
|
955
|
+
}
|
|
956
|
+
else for (const chunk of chunks.iterate(document.id)) {
|
|
957
|
+
const text = asString(chunk["raw_content"]) || asString(chunk["content"]);
|
|
958
|
+
for (const evidence of findExactTextEvidence(text, pattern)) {
|
|
959
|
+
results.push({
|
|
960
|
+
...provenance,
|
|
961
|
+
...evidence,
|
|
962
|
+
textLine: null
|
|
963
|
+
});
|
|
964
|
+
if (results.length >= limit) return results;
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
return results;
|
|
969
|
+
}
|
|
843
970
|
/** FTS hits ranked by bm25 relevance — score = -bm25 (larger is better). */
|
|
844
|
-
async ftsSearchChunks(query, limit = 50) {
|
|
971
|
+
async ftsSearchChunks(query, limit = 50, documentId) {
|
|
845
972
|
await this.init();
|
|
846
973
|
if (this.db === void 0) return [];
|
|
847
974
|
const ftsQuery = buildFtsQuery$1(query);
|
|
848
975
|
if (ftsQuery === void 0) return [];
|
|
976
|
+
const scope = documentId === void 0 ? [] : [documentId];
|
|
849
977
|
return this.db.prepare(`SELECT c.*, bm25(knowledge_chunks_fts) AS bm
|
|
850
978
|
FROM knowledge_chunks c
|
|
851
979
|
JOIN knowledge_chunks_fts f ON c.rowid = f.rowid
|
|
852
|
-
WHERE f.knowledge_chunks_fts MATCH ?
|
|
980
|
+
WHERE f.knowledge_chunks_fts MATCH ? ${scope.length === 0 ? "" : "AND c.document_id = ?"}
|
|
853
981
|
ORDER BY bm ASC
|
|
854
|
-
LIMIT ?`).all(ftsQuery, limit).map((row) => ({
|
|
982
|
+
LIMIT ?`).all(ftsQuery, ...scope, limit).map((row) => ({
|
|
855
983
|
chunk: rowToChunk(row),
|
|
856
984
|
score: -Number(row["bm"] ?? 0)
|
|
857
985
|
}));
|
|
@@ -1143,7 +1271,7 @@ function computeRelevanceScore(memo, query, usageCount = 0, currentProjectDir, p
|
|
|
1143
1271
|
const ti = intent?.temporalBias ?? .3;
|
|
1144
1272
|
const fi = intent?.factualBias ?? .6;
|
|
1145
1273
|
const factors = {
|
|
1146
|
-
keywordOverlap:
|
|
1274
|
+
keywordOverlap: computeKeywordOverlap(memo, query),
|
|
1147
1275
|
recency: computeRecency(memo.recordedAt),
|
|
1148
1276
|
usageBoost: Math.min(.3, usageCount * .1),
|
|
1149
1277
|
projectBoost: computeProjectBoost(memo.projectDir, currentProjectDir),
|
|
@@ -1341,7 +1469,21 @@ function extractKeywords(text) {
|
|
|
1341
1469
|
return [...new Set(tokens)];
|
|
1342
1470
|
}
|
|
1343
1471
|
function computeKeywordSimilarity(memo, query) {
|
|
1344
|
-
|
|
1472
|
+
return jaccardSimilarity(keywordText(memo), query);
|
|
1473
|
+
}
|
|
1474
|
+
/**
|
|
1475
|
+
* Ranking variant: the distilled conclusion and the note are separate
|
|
1476
|
+
* retrieval surfaces. Score each field on its own and keep the best match —
|
|
1477
|
+
* a short conclusion would otherwise be diluted by the long base text.
|
|
1478
|
+
*/
|
|
1479
|
+
function computeKeywordOverlap(memo, query) {
|
|
1480
|
+
return Math.max(jaccardSimilarity(keywordText(memo), query), jaccardSimilarity(memo.distilled ?? "", query), jaccardSimilarity(memo.note ?? "", query));
|
|
1481
|
+
}
|
|
1482
|
+
function keywordText(memo) {
|
|
1483
|
+
return `${memo.userNeed} ${memo.approach} ${memo.whatFailed} ${memo.whatWorked}`;
|
|
1484
|
+
}
|
|
1485
|
+
function jaccardSimilarity(memoText, query) {
|
|
1486
|
+
const memoWords = extractKeywords(memoText);
|
|
1345
1487
|
const queryWords = extractKeywords(query);
|
|
1346
1488
|
if (memoWords.length === 0 || queryWords.length === 0) return 0;
|
|
1347
1489
|
const intersection = memoWords.filter((w) => queryWords.includes(w)).length;
|
|
@@ -1394,6 +1536,10 @@ function createMemoryMemo(partial) {
|
|
|
1394
1536
|
outcome: partial.outcome,
|
|
1395
1537
|
whatFailed: partial.whatFailed,
|
|
1396
1538
|
whatWorked: partial.whatWorked,
|
|
1539
|
+
distilled: partial.distilled,
|
|
1540
|
+
origin: partial.origin ?? "session",
|
|
1541
|
+
distilledFrom: partial.distilledFrom,
|
|
1542
|
+
distilledAt: partial.distilledAt,
|
|
1397
1543
|
extractionSource: partial.extractionSource,
|
|
1398
1544
|
recordedAt: partial.recordedAt ?? Date.now(),
|
|
1399
1545
|
projectDir: partial.projectDir ?? "",
|
|
@@ -1415,6 +1561,10 @@ function toSummary(memo) {
|
|
|
1415
1561
|
outcome: memo.outcome,
|
|
1416
1562
|
whatFailed: memo.whatFailed,
|
|
1417
1563
|
whatWorked: memo.whatWorked,
|
|
1564
|
+
distilled: memo.distilled,
|
|
1565
|
+
origin: memo.origin,
|
|
1566
|
+
distilledFrom: memo.distilledFrom,
|
|
1567
|
+
distilledAt: memo.distilledAt,
|
|
1418
1568
|
extractionSource: memo.extractionSource,
|
|
1419
1569
|
recordedAt: memo.recordedAt,
|
|
1420
1570
|
projectDir: memo.projectDir,
|
|
@@ -2850,7 +3000,7 @@ var vn = (s) => {
|
|
|
2850
3000
|
if (o === l) c = !1;
|
|
2851
3001
|
else {
|
|
2852
3002
|
let d = e.get(o);
|
|
2853
|
-
c = d !== void 0 ? d : n(dirname(o), l, a + 1);
|
|
3003
|
+
c = d !== void 0 ? d : n(dirname$1(o), l, a + 1);
|
|
2854
3004
|
}
|
|
2855
3005
|
return e.set(o, c), c;
|
|
2856
3006
|
};
|
|
@@ -4456,7 +4606,7 @@ function buildEmbeddingText(memo) {
|
|
|
4456
4606
|
*/
|
|
4457
4607
|
const engineCache = /* @__PURE__ */ new Map();
|
|
4458
4608
|
/**
|
|
4459
|
-
* Create an embedding engine backed by
|
|
4609
|
+
* Create an embedding engine backed by the in-process WASM pipeline.
|
|
4460
4610
|
* Lazily loads the model on first use so startup is not blocked.
|
|
4461
4611
|
* Engines are cached by cacheDir so repeated calls with the same cacheDir
|
|
4462
4612
|
* return the same instance, sharing model state and download progress.
|
|
@@ -4464,21 +4614,22 @@ const engineCache = /* @__PURE__ */ new Map();
|
|
|
4464
4614
|
* Defaults to "local_cache" (CWD-relative) if not provided — prefer
|
|
4465
4615
|
* passing an explicit path so the cache doesn't duplicate across CWDs.
|
|
4466
4616
|
*/
|
|
4467
|
-
function
|
|
4468
|
-
const key = cacheDir
|
|
4617
|
+
function createEmbeddingEngine(cacheDir) {
|
|
4618
|
+
const key = cacheDir === void 0 ? "" : resolve(cacheDir);
|
|
4469
4619
|
const cached = engineCache.get(key);
|
|
4470
4620
|
if (cached !== void 0) return cached;
|
|
4471
|
-
const engine =
|
|
4621
|
+
const engine = createEmbeddingEngineImpl(cacheDir);
|
|
4472
4622
|
engineCache.set(key, engine);
|
|
4473
4623
|
return engine;
|
|
4474
4624
|
}
|
|
4475
4625
|
/** Stable identity of the model used by every engine created here. */
|
|
4476
4626
|
const EMBEDDING_MODEL_NAME = "bge-small-zh-v1.5";
|
|
4477
|
-
function
|
|
4627
|
+
function createEmbeddingEngineImpl(cacheDir) {
|
|
4478
4628
|
let embedder = null;
|
|
4479
4629
|
let initPromise = null;
|
|
4480
4630
|
let loadFailed = false;
|
|
4481
4631
|
let lastError;
|
|
4632
|
+
let progressListener;
|
|
4482
4633
|
return {
|
|
4483
4634
|
modelName: EMBEDDING_MODEL_NAME,
|
|
4484
4635
|
get available() {
|
|
@@ -4487,6 +4638,17 @@ function createFastEmbedEngineImpl(cacheDir) {
|
|
|
4487
4638
|
get lastError() {
|
|
4488
4639
|
return lastError;
|
|
4489
4640
|
},
|
|
4641
|
+
setProgressListener(listener) {
|
|
4642
|
+
progressListener = listener;
|
|
4643
|
+
},
|
|
4644
|
+
unload() {
|
|
4645
|
+
if (embedder === null) return true;
|
|
4646
|
+
embedder = null;
|
|
4647
|
+
initPromise = null;
|
|
4648
|
+
loadFailed = false;
|
|
4649
|
+
lastError = void 0;
|
|
4650
|
+
return true;
|
|
4651
|
+
},
|
|
4490
4652
|
async embedBatch(texts) {
|
|
4491
4653
|
if (!this.available) return null;
|
|
4492
4654
|
if (texts.length === 0) return [];
|
|
@@ -4523,12 +4685,12 @@ function createFastEmbedEngineImpl(cacheDir) {
|
|
|
4523
4685
|
if (loadFailed || initPromise === null) {
|
|
4524
4686
|
loadFailed = false;
|
|
4525
4687
|
lastError = void 0;
|
|
4526
|
-
initPromise =
|
|
4688
|
+
initPromise = loadEmbeddingRuntime(cacheDir, (progress) => progressListener?.(progress));
|
|
4527
4689
|
}
|
|
4528
4690
|
embedder = await initPromise;
|
|
4529
4691
|
if (embedder === null) {
|
|
4530
4692
|
initPromise = null;
|
|
4531
|
-
lastError = "
|
|
4693
|
+
lastError = "embedding runtime returned an empty session";
|
|
4532
4694
|
return false;
|
|
4533
4695
|
}
|
|
4534
4696
|
loadFailed = false;
|
|
@@ -4540,49 +4702,156 @@ function createFastEmbedEngineImpl(cacheDir) {
|
|
|
4540
4702
|
lastError = error instanceof Error ? error.message : String(error);
|
|
4541
4703
|
return false;
|
|
4542
4704
|
}
|
|
4543
|
-
},
|
|
4544
|
-
unload() {
|
|
4545
|
-
if (embedder === null) return true;
|
|
4546
|
-
embedder = null;
|
|
4547
|
-
initPromise = null;
|
|
4548
|
-
loadFailed = false;
|
|
4549
|
-
lastError = void 0;
|
|
4550
|
-
return true;
|
|
4551
4705
|
}
|
|
4552
4706
|
};
|
|
4553
4707
|
}
|
|
4554
|
-
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4708
|
+
/** Maximum tokens per text; matches the model's position-embedding limit. */
|
|
4709
|
+
const MAX_SEQ_LEN = 512;
|
|
4710
|
+
/** Texts per inference batch: 16 × 512 × 512 floats ≈ 16MB of activations. */
|
|
4711
|
+
const EMBED_BATCH_SIZE = 16;
|
|
4712
|
+
/** Threads for the WASM backend. One keeps resolution deterministic in packaged
|
|
4713
|
+
* builds; `NEXUM_CODE_EMBEDDING_THREADS=4` is the supported opt-in. */
|
|
4714
|
+
const EMBEDDING_THREADS_ENV = "NEXUM_CODE_EMBEDDING_THREADS";
|
|
4715
|
+
/**
|
|
4716
|
+
* Token-level truncation that keeps the sentence boundary tokens.
|
|
4717
|
+
*
|
|
4718
|
+
* The tokenizer package does not apply the truncation policy itself and the
|
|
4719
|
+
* model package ships no truncation config, so this is the only place it can
|
|
4720
|
+
* happen. Keeping the head (which starts with `[CLS]`) and re-appending the
|
|
4721
|
+
* trailing special token (`[SEP]`) mirrors what a HuggingFace pipeline would
|
|
4722
|
+
* produce; slicing the tail off blindly would feed the encoder a sentence that
|
|
4723
|
+
* never closes and drift from the reference vectors on long inputs.
|
|
4724
|
+
*/
|
|
4725
|
+
function truncateEncoded(ids, types) {
|
|
4726
|
+
if (ids.length <= MAX_SEQ_LEN) return {
|
|
4727
|
+
ids: [...ids],
|
|
4728
|
+
types: [...types]
|
|
4729
|
+
};
|
|
4730
|
+
const headSize = MAX_SEQ_LEN - 1;
|
|
4731
|
+
const head = ids.slice(0, headSize);
|
|
4732
|
+
const headTypes = types.slice(0, headSize);
|
|
4733
|
+
return {
|
|
4734
|
+
ids: [...head, ids.at(-1)],
|
|
4735
|
+
types: [...headTypes, types.at(-1)]
|
|
4736
|
+
};
|
|
4737
|
+
}
|
|
4738
|
+
/**
|
|
4739
|
+
* Builds the embedding runtime in-process on `onnxruntime-web` (WASM) and the
|
|
4740
|
+
* pure-JS tokenizer. No native binaries, no install scripts: the whole stack is
|
|
4741
|
+
* JavaScript plus one WebAssembly artifact resolved from the package.
|
|
4742
|
+
*
|
|
4743
|
+
* The numeric pipeline is load-bearing and must stay compatible with the
|
|
4744
|
+
* reference embeddings of this model: token ids come from `tokenizer.json`, and
|
|
4745
|
+
* the sentence vector is the **CLS token** hidden state (`last_hidden_state[0]`)
|
|
4746
|
+
* followed by L2 normalization. Mean pooling produces visibly different vectors
|
|
4747
|
+
* (cosine ≈ 0.9 against the reference), so pooling is not a stylistic choice.
|
|
4748
|
+
*/
|
|
4749
|
+
async function createWasmRuntime(modelDir) {
|
|
4750
|
+
const ort = await import(["onnxruntime-web", "wasm"].join("/"));
|
|
4751
|
+
const { Tokenizer } = await import("@huggingface/tokenizers");
|
|
4752
|
+
const requestedThreads = Number(process.env[EMBEDDING_THREADS_ENV]);
|
|
4753
|
+
ort.env.wasm.numThreads = Number.isFinite(requestedThreads) && requestedThreads > 0 ? requestedThreads : 1;
|
|
4754
|
+
ort.env.logLevel = "error";
|
|
4755
|
+
const tokenizer = new Tokenizer(JSON.parse(readFileSync(join(modelDir, "tokenizer.json"), "utf8")), JSON.parse(readFileSync(join(modelDir, "tokenizer_config.json"), "utf8")));
|
|
4756
|
+
const session = await ort.InferenceSession.create(readFileSync(join(modelDir, "model_optimized.onnx")), { executionProviders: ["wasm"] });
|
|
4757
|
+
if (session === null || session === void 0) return null;
|
|
4758
|
+
return { async *embed(textStrings) {
|
|
4759
|
+
if (textStrings.length === 0) return;
|
|
4760
|
+
const encoded = textStrings.map((text, index) => {
|
|
4761
|
+
const encoding = tokenizer.encode(text, { return_token_type_ids: true });
|
|
4762
|
+
const types = encoding.token_type_ids ?? encoding.ids.map(() => 0);
|
|
4763
|
+
const truncated = truncateEncoded(encoding.ids, types);
|
|
4764
|
+
return {
|
|
4765
|
+
index,
|
|
4766
|
+
ids: truncated.ids,
|
|
4767
|
+
types: truncated.types
|
|
4768
|
+
};
|
|
4769
|
+
});
|
|
4770
|
+
const results = Array.from({ length: textStrings.length });
|
|
4771
|
+
const buckets = encoded.toSorted((a, b) => a.ids.length - b.ids.length);
|
|
4772
|
+
for (let start = 0; start < buckets.length; start += EMBED_BATCH_SIZE) {
|
|
4773
|
+
const group = buckets.slice(start, start + EMBED_BATCH_SIZE);
|
|
4774
|
+
const rows = group.length;
|
|
4775
|
+
const width = Math.max(...group.map((entry) => entry.ids.length));
|
|
4776
|
+
const ids = new BigInt64Array(rows * width);
|
|
4777
|
+
const mask = new BigInt64Array(rows * width);
|
|
4778
|
+
const types = new BigInt64Array(rows * width);
|
|
4779
|
+
group.forEach((entry, row) => {
|
|
4780
|
+
for (let column = 0; column < width; column++) {
|
|
4781
|
+
const offset = row * width + column;
|
|
4782
|
+
ids[offset] = BigInt(column < entry.ids.length ? entry.ids[column] : 0);
|
|
4783
|
+
mask[offset] = column < entry.ids.length ? 1n : 0n;
|
|
4784
|
+
types[offset] = BigInt(column < entry.types.length ? entry.types[column] : 0);
|
|
4785
|
+
}
|
|
4786
|
+
});
|
|
4787
|
+
const output = await session.run({
|
|
4788
|
+
input_ids: new ort.Tensor("int64", ids, [rows, width]),
|
|
4789
|
+
attention_mask: new ort.Tensor("int64", mask, [rows, width]),
|
|
4790
|
+
token_type_ids: new ort.Tensor("int64", types, [rows, width])
|
|
4791
|
+
});
|
|
4792
|
+
const hidden = output["last_hidden_state"] ?? output[Object.keys(output)[0]];
|
|
4793
|
+
const hiddenSize = hidden.dims.at(-1);
|
|
4794
|
+
const data = hidden.data;
|
|
4795
|
+
group.forEach((entry, row) => {
|
|
4796
|
+
const rowStart = row * width * hiddenSize;
|
|
4797
|
+
const vector = Array.from({ length: hiddenSize }, (_, index) => data[rowStart + index]);
|
|
4798
|
+
let norm = 0;
|
|
4799
|
+
for (const value of vector) norm += value * value;
|
|
4800
|
+
norm = Math.sqrt(norm) || 1;
|
|
4801
|
+
for (let i = 0; i < vector.length; i += 1) vector[i] = vector[i] / norm;
|
|
4802
|
+
results[entry.index] = vector;
|
|
4803
|
+
});
|
|
4804
|
+
}
|
|
4805
|
+
for (let start = 0; start < results.length; start += EMBED_BATCH_SIZE) yield results.slice(start, start + EMBED_BATCH_SIZE);
|
|
4806
|
+
} };
|
|
4807
|
+
}
|
|
4808
|
+
async function loadEmbeddingRuntime(cacheDir, emit) {
|
|
4809
|
+
if (cacheDir === void 0) {
|
|
4810
|
+
emit?.({ phase: "load" });
|
|
4811
|
+
return createWasmRuntime(join("local_cache", BGESMALLZH_CACHE_NAME));
|
|
4572
4812
|
}
|
|
4813
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
4814
|
+
await ensureEmbeddingModelFiles(cacheDir, {
|
|
4815
|
+
onProgress: (received, total) => {
|
|
4816
|
+
emit?.(total === void 0 ? {
|
|
4817
|
+
phase: "download",
|
|
4818
|
+
received
|
|
4819
|
+
} : {
|
|
4820
|
+
phase: "download",
|
|
4821
|
+
received,
|
|
4822
|
+
total
|
|
4823
|
+
});
|
|
4824
|
+
},
|
|
4825
|
+
onPhase: (phase) => {
|
|
4826
|
+
if (phase === "verify") emit?.({ phase: "verify" });
|
|
4827
|
+
}
|
|
4828
|
+
});
|
|
4829
|
+
emit?.({ phase: "load" });
|
|
4830
|
+
return createWasmRuntime(join(cacheDir, BGESMALLZH_CACHE_NAME));
|
|
4573
4831
|
}
|
|
4574
4832
|
const BGESMALLZH_CACHE_NAME = "fast-bge-small-zh-v1.5";
|
|
4833
|
+
/** Files the runtime needs before the model can be considered installed. */
|
|
4834
|
+
const EMBEDDING_REQUIRED_FILES = [
|
|
4835
|
+
"model_optimized.onnx",
|
|
4836
|
+
"tokenizer.json",
|
|
4837
|
+
"tokenizer_config.json"
|
|
4838
|
+
];
|
|
4575
4839
|
/** Escape hatch for air-gapped installs and tests: points at an archive file. */
|
|
4576
4840
|
const EMBEDDING_MODEL_URL_ENV = "NEXUM_CODE_EMBEDDING_MODEL_URL";
|
|
4577
|
-
|
|
4578
|
-
|
|
4841
|
+
/** Default stall guards: a trickling connection must not spin a spinner forever. */
|
|
4842
|
+
const DEFAULT_IDLE_TIMEOUT_MS = 6e4;
|
|
4843
|
+
const DEFAULT_TOTAL_TIMEOUT_MS = 15 * 6e4;
|
|
4844
|
+
/** Progress callback throttle: frequent enough to move, rare enough to be free. */
|
|
4845
|
+
const PROGRESS_THROTTLE_MS = 100;
|
|
4846
|
+
function modelDirPath(cacheDir) {
|
|
4847
|
+
return join(cacheDir, BGESMALLZH_CACHE_NAME);
|
|
4579
4848
|
}
|
|
4580
4849
|
function sha256File(filePath) {
|
|
4581
4850
|
return readFile(filePath).then((data) => createHash("sha256").update(data).digest("hex"));
|
|
4582
4851
|
}
|
|
4583
4852
|
/**
|
|
4584
4853
|
* Stage the embedding model files when they are missing: download the pinned
|
|
4585
|
-
* archive (HTTP status verified
|
|
4854
|
+
* archive (HTTP status verified, unlike a naive downloader), extract,
|
|
4586
4855
|
* and verify the weights payload against the pinned sha256.
|
|
4587
4856
|
*
|
|
4588
4857
|
* A partially extracted tree or a hash mismatch is removed so the caller keeps
|
|
@@ -4592,62 +4861,149 @@ function sha256File(filePath) {
|
|
|
4592
4861
|
async function ensureEmbeddingModelFiles(cacheDir, options) {
|
|
4593
4862
|
if (hasEmbeddingModelCache(cacheDir)) return;
|
|
4594
4863
|
mkdirSync(cacheDir, { recursive: true });
|
|
4595
|
-
const modelDir =
|
|
4864
|
+
const modelDir = modelDirPath(cacheDir);
|
|
4596
4865
|
const stagingPath = join(cacheDir, `${BGESMALLZH_CACHE_NAME}.tar.gz.part`);
|
|
4597
4866
|
const url = options?.sourceUrl ?? process.env[EMBEDDING_MODEL_URL_ENV] ?? "https://github.com/LIUTod/nexum-code/releases/download/embedding-models-v1/fast-bge-small-zh-v1.5.tar.gz";
|
|
4867
|
+
const idleTimeoutMs = options?.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
|
|
4868
|
+
const totalTimeoutMs = options?.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS;
|
|
4869
|
+
const controller = new AbortController();
|
|
4870
|
+
const onExternalAbort = () => {
|
|
4871
|
+
controller.abort();
|
|
4872
|
+
};
|
|
4873
|
+
if (options?.signal !== void 0) if (options.signal.aborted) controller.abort();
|
|
4874
|
+
else options.signal.addEventListener("abort", onExternalAbort, { once: true });
|
|
4875
|
+
let idleTimer;
|
|
4876
|
+
const armIdleTimer = () => {
|
|
4877
|
+
if (idleTimer !== void 0) clearTimeout(idleTimer);
|
|
4878
|
+
idleTimer = setTimeout(() => {
|
|
4879
|
+
controller.abort();
|
|
4880
|
+
}, idleTimeoutMs);
|
|
4881
|
+
};
|
|
4882
|
+
const totalTimer = setTimeout(() => {
|
|
4883
|
+
controller.abort();
|
|
4884
|
+
}, totalTimeoutMs);
|
|
4885
|
+
let received = 0;
|
|
4886
|
+
let announcedTotal;
|
|
4887
|
+
let downloadCompleted = false;
|
|
4888
|
+
const announceProgress = () => {
|
|
4889
|
+
options?.onProgress?.(received, announcedTotal);
|
|
4890
|
+
};
|
|
4598
4891
|
try {
|
|
4599
4892
|
let response;
|
|
4600
4893
|
try {
|
|
4601
|
-
|
|
4894
|
+
armIdleTimer();
|
|
4895
|
+
response = await fetch(url, { signal: controller.signal });
|
|
4602
4896
|
} catch (error) {
|
|
4603
4897
|
const message = error instanceof Error ? error.message : String(error);
|
|
4604
4898
|
throw new Error(`fetch failed for ${url}: ${message}`, { cause: error });
|
|
4605
4899
|
}
|
|
4606
4900
|
if (!response.ok) throw new Error(`model source returned HTTP ${response.status} for ${url}`);
|
|
4607
|
-
|
|
4901
|
+
const contentLength = Number(response.headers.get("content-length"));
|
|
4902
|
+
announcedTotal = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : void 0;
|
|
4903
|
+
announceProgress();
|
|
4904
|
+
const body = response.body;
|
|
4905
|
+
if (body === null) throw new Error(`model source returned an empty body for ${url}`);
|
|
4906
|
+
const reader = body.getReader();
|
|
4907
|
+
const abortRejection = new Promise((_, reject) => {
|
|
4908
|
+
if (controller.signal.aborted) {
|
|
4909
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
4910
|
+
return;
|
|
4911
|
+
}
|
|
4912
|
+
controller.signal.addEventListener("abort", () => {
|
|
4913
|
+
reject(/* @__PURE__ */ new Error("aborted"));
|
|
4914
|
+
}, { once: true });
|
|
4915
|
+
});
|
|
4916
|
+
const abortAwareRead = () => Promise.race([reader.read(), abortRejection]);
|
|
4917
|
+
let handle;
|
|
4918
|
+
try {
|
|
4919
|
+
handle = await open(stagingPath, "w");
|
|
4920
|
+
} catch (error) {
|
|
4921
|
+
await reader.cancel().catch(() => void 0);
|
|
4922
|
+
throw error;
|
|
4923
|
+
}
|
|
4924
|
+
let lastProgressAt = 0;
|
|
4925
|
+
try {
|
|
4926
|
+
await (async () => {
|
|
4927
|
+
for (;;) {
|
|
4928
|
+
let chunk;
|
|
4929
|
+
try {
|
|
4930
|
+
chunk = await abortAwareRead();
|
|
4931
|
+
} catch (error) {
|
|
4932
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4933
|
+
throw new Error(`${message} (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
|
|
4934
|
+
}
|
|
4935
|
+
if (chunk.done || chunk.value === void 0) break;
|
|
4936
|
+
const bytes = chunk.value;
|
|
4937
|
+
armIdleTimer();
|
|
4938
|
+
await handle.write(bytes);
|
|
4939
|
+
received += bytes.byteLength;
|
|
4940
|
+
const now = Date.now();
|
|
4941
|
+
if (now - lastProgressAt >= PROGRESS_THROTTLE_MS) {
|
|
4942
|
+
lastProgressAt = now;
|
|
4943
|
+
announceProgress();
|
|
4944
|
+
}
|
|
4945
|
+
}
|
|
4946
|
+
announceProgress();
|
|
4947
|
+
downloadCompleted = true;
|
|
4948
|
+
})();
|
|
4949
|
+
} finally {
|
|
4950
|
+
if (idleTimer !== void 0) clearTimeout(idleTimer);
|
|
4951
|
+
await handle.close();
|
|
4952
|
+
await reader.cancel().catch(() => void 0);
|
|
4953
|
+
}
|
|
4954
|
+
options?.onPhase?.("verify");
|
|
4608
4955
|
await So({
|
|
4609
4956
|
file: stagingPath,
|
|
4610
4957
|
cwd: cacheDir
|
|
4611
4958
|
});
|
|
4612
|
-
const
|
|
4613
|
-
|
|
4614
|
-
|
|
4959
|
+
for (const fileName of EMBEDDING_REQUIRED_FILES) {
|
|
4960
|
+
const target = join(modelDir, fileName);
|
|
4961
|
+
if (!existsSync(target) || statSync(target).size === 0) throw new Error(`model package is missing ${fileName} after extraction: ${url}`);
|
|
4962
|
+
}
|
|
4963
|
+
const sha256 = await sha256File(join(modelDir, "model_optimized.onnx"));
|
|
4615
4964
|
const expected = options?.expectedOnnxSha256 ?? "1294ea4b6331115a353d81f96b85e8c8d7fdcc284453d5b2fab5b016230aad38";
|
|
4616
4965
|
if (sha256 !== expected) throw new Error(`model weights sha256 mismatch (${sha256.slice(0, 12)}… != ${expected.slice(0, 12)}…): ${url}`);
|
|
4617
4966
|
} catch (error) {
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4967
|
+
try {
|
|
4968
|
+
rmSync(modelDir, {
|
|
4969
|
+
recursive: true,
|
|
4970
|
+
force: true
|
|
4971
|
+
});
|
|
4972
|
+
} catch {}
|
|
4973
|
+
if (!downloadCompleted && controller.signal.aborted && !(options?.signal?.aborted ?? false)) throw new Error(`model download timed out (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
|
|
4974
|
+
throw error instanceof Error ? error : new Error(String(error), { cause: error });
|
|
4623
4975
|
} finally {
|
|
4976
|
+
clearTimeout(totalTimer);
|
|
4977
|
+
if (idleTimer !== void 0) clearTimeout(idleTimer);
|
|
4978
|
+
if (options?.signal !== void 0) options.signal.removeEventListener("abort", onExternalAbort);
|
|
4624
4979
|
rmSync(stagingPath, { force: true });
|
|
4625
4980
|
}
|
|
4626
4981
|
}
|
|
4627
|
-
const
|
|
4628
|
-
const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib/iu;
|
|
4629
|
-
const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}/iu;
|
|
4982
|
+
const INSTALL_FAILURE = /Cannot find (?:package|module) '(?:onnxruntime-web(?:\/wasm)?|@huggingface\/tokenizers)'|ERR_MODULE_NOT_FOUND|Failed to resolve module specifier/iu;
|
|
4983
|
+
const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib|after extraction|sha256 mismatch/iu;
|
|
4984
|
+
const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}|timed out|empty body/iu;
|
|
4630
4985
|
/**
|
|
4631
|
-
* Classify a load failure so callers can say something true about it:
|
|
4632
|
-
*
|
|
4633
|
-
*
|
|
4986
|
+
* Classify a load failure so callers can say something true about it: an
|
|
4987
|
+
* incomplete install, a corrupt cache, a network problem and anything else need
|
|
4988
|
+
* four different answers, and they used to be reported as one network problem.
|
|
4634
4989
|
*/
|
|
4635
4990
|
function classifyEmbeddingFailure(message) {
|
|
4636
4991
|
if (message === void 0) return "other";
|
|
4637
|
-
if (
|
|
4992
|
+
if (INSTALL_FAILURE.test(message)) return "install";
|
|
4638
4993
|
if (ARCHIVE_FAILURE.test(message)) return "archive";
|
|
4639
4994
|
if (NETWORK_FAILURE.test(message)) return "network";
|
|
4640
4995
|
return "other";
|
|
4641
4996
|
}
|
|
4642
4997
|
/**
|
|
4643
|
-
* Loads
|
|
4644
|
-
*
|
|
4645
|
-
* failed".
|
|
4646
|
-
*
|
|
4998
|
+
* Loads both runtime packages — no model download — so callers can tell "this
|
|
4999
|
+
* installation cannot run the local model at all" apart from "the download
|
|
5000
|
+
* failed". The runtime is JavaScript plus WebAssembly, so a failure here is an
|
|
5001
|
+
* incomplete installation rather than an unsupported platform.
|
|
4647
5002
|
*/
|
|
4648
5003
|
async function probeLocalEmbeddingSupport() {
|
|
4649
5004
|
try {
|
|
4650
|
-
await import("
|
|
5005
|
+
await import(["onnxruntime-web", "wasm"].join("/"));
|
|
5006
|
+
await import("@huggingface/tokenizers");
|
|
4651
5007
|
return { supported: true };
|
|
4652
5008
|
} catch (error) {
|
|
4653
5009
|
return {
|
|
@@ -4658,15 +5014,11 @@ async function probeLocalEmbeddingSupport() {
|
|
|
4658
5014
|
}
|
|
4659
5015
|
/**
|
|
4660
5016
|
* Remove every trace of a previous BGESmallZH download: the extracted model
|
|
4661
|
-
* directory
|
|
5017
|
+
* directory and the legacy staged archive name.
|
|
4662
5018
|
*
|
|
4663
|
-
* The
|
|
4664
|
-
*
|
|
4665
|
-
*
|
|
4666
|
-
* error) reject and leave whatever was written behind. So an archive sitting
|
|
4667
|
-
* next to a failed attempt is usually a partial leftover, and keeping it makes
|
|
4668
|
-
* every retry extract the same junk locally: no network traffic, a few
|
|
4669
|
-
* milliseconds, the same failure forever.
|
|
5019
|
+
* The legacy `<name>.tar.gz` file is still swept because older versions staged
|
|
5020
|
+
* the archive under that name; leaving it behind would waste 55MB per doomed
|
|
5021
|
+
* attempt. Our own downloader stages `.tar.gz.part` and always removes it.
|
|
4670
5022
|
*
|
|
4671
5023
|
* Never throws: a file locked by another process (on Windows a mapped ONNX file
|
|
4672
5024
|
* stays locked) must not replace the download error the caller is reporting.
|
|
@@ -4684,48 +5036,19 @@ function clearEmbeddingModelCache(cacheDir) {
|
|
|
4684
5036
|
}
|
|
4685
5037
|
}
|
|
4686
5038
|
/**
|
|
4687
|
-
* True when the
|
|
4688
|
-
*
|
|
4689
|
-
*
|
|
4690
|
-
*
|
|
5039
|
+
* True when every file the runtime needs is already on disk. Loading then never
|
|
5040
|
+
* downloads, so callers can safely trigger a local load without risking a
|
|
5041
|
+
* surprise mid-query download. All required files are checked (not just the
|
|
5042
|
+
* weights): a partially restored cache must count as missing so the setup flow
|
|
5043
|
+
* repairs it instead of failing at load time.
|
|
4691
5044
|
*/
|
|
4692
5045
|
function hasEmbeddingModelCache(cacheDir) {
|
|
4693
|
-
const modelDir =
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
/**
|
|
4697
|
-
* Small config/tokenizer files that fastembed expects alongside model.onnx.
|
|
4698
|
-
* If these are missing (e.g. GCS download partially failed), fastembed throws.
|
|
4699
|
-
* We download them from HuggingFace so the model can load — this covers the
|
|
4700
|
-
* case where the GCS tarball was incomplete but the HF repo is reachable.
|
|
4701
|
-
*/
|
|
4702
|
-
const FASTEMBED_SIDECARS = [
|
|
4703
|
-
"config.json",
|
|
4704
|
-
"tokenizer.json",
|
|
4705
|
-
"tokenizer_config.json",
|
|
4706
|
-
"special_tokens_map.json"
|
|
4707
|
-
];
|
|
4708
|
-
const FASTEMBED_HF_REPOS = { "fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5" };
|
|
4709
|
-
async function ensureFastembedModelSidecars(model, cacheDir) {
|
|
4710
|
-
const repo = FASTEMBED_HF_REPOS[model];
|
|
4711
|
-
if (repo === void 0) return;
|
|
4712
|
-
const modelDir = join(cacheDir ?? "local_cache", model);
|
|
4713
|
-
mkdirSync(modelDir, { recursive: true });
|
|
4714
|
-
for (const fileName of FASTEMBED_SIDECARS) {
|
|
5046
|
+
const modelDir = modelDirPath(cacheDir);
|
|
5047
|
+
if (!existsSync(modelDir)) return false;
|
|
5048
|
+
return EMBEDDING_REQUIRED_FILES.every((fileName) => {
|
|
4715
5049
|
const target = join(modelDir, fileName);
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
await access(target);
|
|
4719
|
-
continue;
|
|
4720
|
-
} catch {}
|
|
4721
|
-
const hfUrl = `https://huggingface.co/${repo}/resolve/main/${fileName}`;
|
|
4722
|
-
try {
|
|
4723
|
-
const response = await fetch(hfUrl);
|
|
4724
|
-
if (!response.ok) continue;
|
|
4725
|
-
const { writeFile } = await import("node:fs/promises");
|
|
4726
|
-
await writeFile(target, Buffer.from(await response.arrayBuffer()));
|
|
4727
|
-
} catch {}
|
|
4728
|
-
}
|
|
5050
|
+
return existsSync(target) && statSync(target).size > 0;
|
|
5051
|
+
});
|
|
4729
5052
|
}
|
|
4730
5053
|
//#endregion
|
|
4731
5054
|
//#region ../../packages/memory/src/store.ts
|
|
@@ -4869,7 +5192,19 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
4869
5192
|
JOIN memos_fts f ON m.rowid = f.rowid
|
|
4870
5193
|
WHERE f.memos_fts MATCH ? AND (m.project_dir = ? OR m.project_dir = '')
|
|
4871
5194
|
ORDER BY m.recorded_at DESC LIMIT ?`);
|
|
4872
|
-
|
|
5195
|
+
const rows = projectDir === void 0 ? stmt.all(ftsQuery, limit) : stmt.all(ftsQuery, projectDir, limit);
|
|
5196
|
+
const like = likePattern(query);
|
|
5197
|
+
const remaining = Math.max(0, limit - rows.length);
|
|
5198
|
+
const extraRows = remaining === 0 ? [] : projectDir === void 0 ? this.db.prepare("SELECT * FROM memos WHERE distilled LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\' ORDER BY recorded_at DESC LIMIT ?").all(like, like, remaining) : this.db.prepare("SELECT * FROM memos WHERE (distilled LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\') AND (project_dir = ? OR project_dir = '') ORDER BY recorded_at DESC LIMIT ?").all(like, like, projectDir, remaining);
|
|
5199
|
+
const seen = new Set(rows.map((row) => String(row["id"])));
|
|
5200
|
+
const merged = rows.map(rowToMemo);
|
|
5201
|
+
for (const row of extraRows) {
|
|
5202
|
+
const id = String(row["id"]);
|
|
5203
|
+
if (seen.has(id)) continue;
|
|
5204
|
+
seen.add(id);
|
|
5205
|
+
merged.push(rowToMemo(row));
|
|
5206
|
+
}
|
|
5207
|
+
return merged;
|
|
4873
5208
|
}
|
|
4874
5209
|
/** List memos with optional full-text search and pagination. */
|
|
4875
5210
|
async list(options) {
|
|
@@ -5011,7 +5346,12 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5011
5346
|
extraction_source TEXT NOT NULL CHECK(extraction_source IN ('compaction', 'exit', 'manual')),
|
|
5012
5347
|
recorded_at INTEGER NOT NULL,
|
|
5013
5348
|
project_dir TEXT NOT NULL DEFAULT '',
|
|
5014
|
-
tags TEXT NOT NULL DEFAULT '[]'
|
|
5349
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
5350
|
+
note TEXT,
|
|
5351
|
+
distilled TEXT NOT NULL DEFAULT '',
|
|
5352
|
+
origin TEXT NOT NULL DEFAULT 'session',
|
|
5353
|
+
distilled_from TEXT,
|
|
5354
|
+
distilled_at INTEGER
|
|
5015
5355
|
);
|
|
5016
5356
|
|
|
5017
5357
|
CREATE INDEX IF NOT EXISTS idx_memos_project_dir ON memos(project_dir);
|
|
@@ -5041,6 +5381,14 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5041
5381
|
const info = this.db.prepare("PRAGMA table_info(memos)").all();
|
|
5042
5382
|
if (!info.some((col) => col.name === "project_dir")) this.db.exec("ALTER TABLE memos ADD COLUMN project_dir TEXT NOT NULL DEFAULT ''");
|
|
5043
5383
|
if (!info.some((col) => col.name === "tags")) this.db.exec("ALTER TABLE memos ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'");
|
|
5384
|
+
if (!info.some((col) => col.name === "note")) this.db.exec("ALTER TABLE memos ADD COLUMN note TEXT");
|
|
5385
|
+
if (!info.some((col) => col.name === "distilled")) this.db.exec("ALTER TABLE memos ADD COLUMN distilled TEXT NOT NULL DEFAULT ''");
|
|
5386
|
+
if (!info.some((col) => col.name === "origin")) {
|
|
5387
|
+
this.db.exec("ALTER TABLE memos ADD COLUMN origin TEXT NOT NULL DEFAULT 'session'");
|
|
5388
|
+
this.db.exec("UPDATE memos SET origin = 'dream' WHERE distilled <> ''");
|
|
5389
|
+
}
|
|
5390
|
+
if (!info.some((col) => col.name === "distilled_from")) this.db.exec("ALTER TABLE memos ADD COLUMN distilled_from TEXT");
|
|
5391
|
+
if (!info.some((col) => col.name === "distilled_at")) this.db.exec("ALTER TABLE memos ADD COLUMN distilled_at INTEGER");
|
|
5044
5392
|
this.db.exec(`
|
|
5045
5393
|
CREATE INDEX IF NOT EXISTS idx_memos_project_dir ON memos(project_dir);
|
|
5046
5394
|
CREATE INDEX IF NOT EXISTS idx_memory_embeddings_created_at ON memory_embeddings(created_at DESC);
|
|
@@ -5084,15 +5432,16 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5084
5432
|
if (this.db === void 0 || memos.length === 0) return;
|
|
5085
5433
|
const insert = this.db.prepare(`INSERT OR IGNORE INTO memos (
|
|
5086
5434
|
id, source_session_id, source_session_title, user_need, approach,
|
|
5087
|
-
outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags
|
|
5088
|
-
|
|
5435
|
+
outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags,
|
|
5436
|
+
note, distilled, origin, distilled_from, distilled_at
|
|
5437
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
5089
5438
|
RETURNING rowid`);
|
|
5090
5439
|
const insertFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
|
|
5091
5440
|
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
5092
5441
|
this.db.exec("BEGIN TRANSACTION");
|
|
5093
5442
|
try {
|
|
5094
5443
|
for (const memo of memos) {
|
|
5095
|
-
const row = insert.get(memo.id, memo.sourceSessionId, memo.sourceSessionTitle ?? null, memo.userNeed, memo.approach, memo.outcome, memo.whatFailed, memo.whatWorked, memo.extractionSource, memo.recordedAt, memo.projectDir ?? "", JSON.stringify(memo.tags ?? []));
|
|
5444
|
+
const row = insert.get(memo.id, memo.sourceSessionId, memo.sourceSessionTitle ?? null, memo.userNeed, memo.approach, memo.outcome, memo.whatFailed, memo.whatWorked, memo.extractionSource, memo.recordedAt, memo.projectDir ?? "", JSON.stringify(memo.tags ?? []), memo.note ?? null, memo.distilled ?? "", memo.origin ?? "session", stringifyDistilledFrom(memo.distilledFrom), memo.distilledAt ?? null);
|
|
5096
5445
|
if (row === void 0) continue;
|
|
5097
5446
|
insertFts.run(row.rowid, toFtsText(memo.userNeed), toFtsText(memo.approach), toFtsText(memo.whatFailed), toFtsText(memo.whatWorked), toFtsText(memo.sourceSessionTitle ?? ""));
|
|
5098
5447
|
}
|
|
@@ -5107,14 +5456,15 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5107
5456
|
if (this.db === void 0) return;
|
|
5108
5457
|
const insert = this.db.prepare(`INSERT INTO memos (
|
|
5109
5458
|
id, source_session_id, source_session_title, user_need, approach,
|
|
5110
|
-
outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags
|
|
5111
|
-
|
|
5459
|
+
outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags,
|
|
5460
|
+
note, distilled, origin, distilled_from, distilled_at
|
|
5461
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
5112
5462
|
RETURNING rowid`);
|
|
5113
5463
|
const insertFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
|
|
5114
5464
|
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
5115
5465
|
this.db.exec("BEGIN TRANSACTION");
|
|
5116
5466
|
try {
|
|
5117
|
-
const row = insert.get(entry.id, entry.sourceSessionId, entry.sourceSessionTitle ?? null, entry.userNeed, entry.approach, entry.outcome, entry.whatFailed, entry.whatWorked, entry.extractionSource, entry.recordedAt, entry.projectDir ?? "", JSON.stringify(entry.tags ?? []));
|
|
5467
|
+
const row = insert.get(entry.id, entry.sourceSessionId, entry.sourceSessionTitle ?? null, entry.userNeed, entry.approach, entry.outcome, entry.whatFailed, entry.whatWorked, entry.extractionSource, entry.recordedAt, entry.projectDir ?? "", JSON.stringify(entry.tags ?? []), entry.note ?? null, entry.distilled ?? "", entry.origin ?? "session", stringifyDistilledFrom(entry.distilledFrom), entry.distilledAt ?? null);
|
|
5118
5468
|
insertFts.run(row.rowid, toFtsText(entry.userNeed), toFtsText(entry.approach), toFtsText(entry.whatFailed), toFtsText(entry.whatWorked), toFtsText(entry.sourceSessionTitle ?? ""));
|
|
5119
5469
|
this.db.exec("COMMIT");
|
|
5120
5470
|
this.scheduleEmbedding(entry);
|
|
@@ -5150,7 +5500,12 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5150
5500
|
extraction_source = ?,
|
|
5151
5501
|
recorded_at = ?,
|
|
5152
5502
|
project_dir = ?,
|
|
5153
|
-
tags =
|
|
5503
|
+
tags = ?,
|
|
5504
|
+
note = ?,
|
|
5505
|
+
distilled = ?,
|
|
5506
|
+
origin = ?,
|
|
5507
|
+
distilled_from = ?,
|
|
5508
|
+
distilled_at = ?
|
|
5154
5509
|
WHERE id = ?
|
|
5155
5510
|
RETURNING rowid`);
|
|
5156
5511
|
const updateFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
|
|
@@ -5164,7 +5519,7 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5164
5519
|
this.db.exec("ROLLBACK");
|
|
5165
5520
|
return false;
|
|
5166
5521
|
}
|
|
5167
|
-
const row = update.get(updated.sourceSessionId, updated.sourceSessionTitle ?? null, updated.userNeed, updated.approach, updated.outcome, updated.whatFailed, updated.whatWorked, updated.extractionSource, updated.recordedAt, updated.projectDir ?? "", JSON.stringify(updated.tags ?? []), id);
|
|
5522
|
+
const row = update.get(updated.sourceSessionId, updated.sourceSessionTitle ?? null, updated.userNeed, updated.approach, updated.outcome, updated.whatFailed, updated.whatWorked, updated.extractionSource, updated.recordedAt, updated.projectDir ?? "", JSON.stringify(updated.tags ?? []), updated.note ?? null, updated.distilled ?? "", updated.origin ?? "session", stringifyDistilledFrom(updated.distilledFrom), updated.distilledAt ?? null, id);
|
|
5168
5523
|
if (row === void 0) {
|
|
5169
5524
|
this.db.exec("ROLLBACK");
|
|
5170
5525
|
return false;
|
|
@@ -5283,6 +5638,36 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5283
5638
|
*/
|
|
5284
5639
|
scheduleEmbedding(memo) {
|
|
5285
5640
|
this.embeddingQueue.add(memo.id);
|
|
5641
|
+
this.armEmbeddingFlush();
|
|
5642
|
+
}
|
|
5643
|
+
/**
|
|
5644
|
+
* Queue every memo that has no vector yet (newest first, at most `limit`) and
|
|
5645
|
+
* arm a flush. Memos written while embeddings could not be stored — or before
|
|
5646
|
+
* an engine existed — would otherwise never get one: `scheduleEmbedding` only
|
|
5647
|
+
* runs for a write. Safe to call repeatedly; already-embedded memos are
|
|
5648
|
+
* skipped by the SQL.
|
|
5649
|
+
*
|
|
5650
|
+
* Loading the engine can download the model, which must stay the user's
|
|
5651
|
+
* decision, so an engine that is not loaded yet is left alone unless the
|
|
5652
|
+
* caller passes `loadEngine` (it knows the model is already on disk).
|
|
5653
|
+
*/
|
|
5654
|
+
async backfillMissingEmbeddings(options = {}) {
|
|
5655
|
+
const engine = this.embeddingEngine;
|
|
5656
|
+
if (engine === void 0) return 0;
|
|
5657
|
+
if (!engine.available && options.loadEngine !== true) return 0;
|
|
5658
|
+
await this.init();
|
|
5659
|
+
if (this.db === void 0) return 0;
|
|
5660
|
+
const limit = Math.max(0, Math.floor(options.limit ?? 500));
|
|
5661
|
+
const rows = this.db.prepare(`SELECT m.id AS id FROM memos m
|
|
5662
|
+
WHERE NOT EXISTS (SELECT 1 FROM memory_embeddings e WHERE e.memory_id = m.id)
|
|
5663
|
+
ORDER BY m.recorded_at DESC
|
|
5664
|
+
LIMIT ?`).all(limit);
|
|
5665
|
+
for (const row of rows) this.embeddingQueue.add(row.id);
|
|
5666
|
+
if (rows.length > 0) this.armEmbeddingFlush();
|
|
5667
|
+
return rows.length;
|
|
5668
|
+
}
|
|
5669
|
+
/** Debounced flush of the queue, loading the engine first when it is not ready yet. */
|
|
5670
|
+
armEmbeddingFlush() {
|
|
5286
5671
|
if (this.embeddingEngine === void 0) return;
|
|
5287
5672
|
if (!this.embeddingEngine.available) {
|
|
5288
5673
|
this.embeddingEngine.ensureReady().then((ready) => {
|
|
@@ -5307,46 +5692,66 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5307
5692
|
this.embeddingFlushing = true;
|
|
5308
5693
|
const ids = [...this.embeddingQueue];
|
|
5309
5694
|
this.embeddingQueue.clear();
|
|
5310
|
-
|
|
5311
|
-
|
|
5695
|
+
let owed = ids;
|
|
5696
|
+
let failed = false;
|
|
5697
|
+
const requeueOwed = () => {
|
|
5698
|
+
for (const id of owed) this.embeddingQueue.add(id);
|
|
5312
5699
|
};
|
|
5313
5700
|
try {
|
|
5314
5701
|
await this.init();
|
|
5315
5702
|
if (this.db === void 0) return;
|
|
5316
5703
|
const pending = [];
|
|
5317
5704
|
for (const id of ids) {
|
|
5318
|
-
if (this.db.prepare("SELECT
|
|
5705
|
+
if (this.db.prepare("SELECT 1 FROM memory_embeddings WHERE memory_id = ?").get(id) !== void 0) continue;
|
|
5319
5706
|
const memo = await this.get(id);
|
|
5320
5707
|
if (memo !== void 0) pending.push({
|
|
5321
5708
|
id,
|
|
5322
5709
|
text: buildEmbeddingText(memo)
|
|
5323
5710
|
});
|
|
5324
5711
|
}
|
|
5325
|
-
|
|
5326
|
-
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
this.
|
|
5342
|
-
|
|
5712
|
+
owed = pending.map((entry) => entry.id);
|
|
5713
|
+
for (let start = 0; start < pending.length; start += 1) {
|
|
5714
|
+
const db = this.db;
|
|
5715
|
+
if (db === void 0) {
|
|
5716
|
+
failed = true;
|
|
5717
|
+
requeueOwed();
|
|
5718
|
+
return;
|
|
5719
|
+
}
|
|
5720
|
+
const chunk = pending.slice(start, start + 1);
|
|
5721
|
+
const vectors = await this.tryEmbedBatch(chunk.map((entry) => entry.text));
|
|
5722
|
+
if (vectors === null || vectors.length !== chunk.length) {
|
|
5723
|
+
this.markEmbeddingFailure(/* @__PURE__ */ new Error(vectors === null ? "embedBatch returned null" : "embedding count mismatch"));
|
|
5724
|
+
failed = true;
|
|
5725
|
+
requeueOwed();
|
|
5726
|
+
return;
|
|
5727
|
+
}
|
|
5728
|
+
this.clearEmbeddingFailure();
|
|
5729
|
+
const insert = db.prepare("INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding_json, model, created_at) VALUES (?, ?, ?, ?)");
|
|
5730
|
+
const now = Date.now();
|
|
5731
|
+
db.exec("BEGIN TRANSACTION");
|
|
5732
|
+
try {
|
|
5733
|
+
for (const [index, entry] of chunk.entries()) insert.run(entry.id, JSON.stringify([...vectors[index]]), EMBEDDING_MODEL_NAME, now);
|
|
5734
|
+
db.exec("COMMIT");
|
|
5735
|
+
} catch (error) {
|
|
5736
|
+
db.exec("ROLLBACK");
|
|
5737
|
+
this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
|
|
5738
|
+
failed = true;
|
|
5739
|
+
requeueOwed();
|
|
5740
|
+
return;
|
|
5741
|
+
}
|
|
5742
|
+
owed = pending.slice(start + chunk.length).map((entry) => entry.id);
|
|
5743
|
+
if (owed.length > 0) await new Promise((resolve) => {
|
|
5744
|
+
setImmediate(resolve);
|
|
5745
|
+
});
|
|
5343
5746
|
}
|
|
5344
5747
|
} catch (error) {
|
|
5345
5748
|
this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
|
|
5346
|
-
|
|
5749
|
+
failed = true;
|
|
5750
|
+
requeueOwed();
|
|
5347
5751
|
} finally {
|
|
5348
5752
|
this.embeddingFlushing = false;
|
|
5349
5753
|
this.embeddingTimer = void 0;
|
|
5754
|
+
if (!failed && this.embeddingQueue.size > 0) this.armEmbeddingFlush();
|
|
5350
5755
|
}
|
|
5351
5756
|
}
|
|
5352
5757
|
async tryEmbedBatch(texts) {
|
|
@@ -5412,6 +5817,9 @@ var MemoryMemoStore = class MemoryMemoStore {
|
|
|
5412
5817
|
function rowToMemo(row) {
|
|
5413
5818
|
const sourceSessionTitle = row["source_session_title"];
|
|
5414
5819
|
const projectDir = row["project_dir"];
|
|
5820
|
+
const distilled = row["distilled"];
|
|
5821
|
+
const note = row["note"];
|
|
5822
|
+
const distilledAt = row["distilled_at"];
|
|
5415
5823
|
return {
|
|
5416
5824
|
id: String(row["id"]),
|
|
5417
5825
|
sourceSessionId: String(row["source_session_id"]),
|
|
@@ -5421,12 +5829,36 @@ function rowToMemo(row) {
|
|
|
5421
5829
|
outcome: String(row["outcome"]),
|
|
5422
5830
|
whatFailed: String(row["what_failed"]),
|
|
5423
5831
|
whatWorked: String(row["what_worked"]),
|
|
5832
|
+
distilled: typeof distilled === "string" && distilled.trim().length > 0 ? distilled : void 0,
|
|
5833
|
+
origin: parseOrigin(row["origin"]),
|
|
5834
|
+
distilledFrom: parseDistilledFrom(row["distilled_from"]),
|
|
5835
|
+
distilledAt: typeof distilledAt === "number" && Number.isFinite(distilledAt) ? distilledAt : void 0,
|
|
5424
5836
|
extractionSource: row["extraction_source"],
|
|
5425
5837
|
recordedAt: Number(row["recorded_at"]),
|
|
5426
5838
|
projectDir: typeof projectDir === "string" ? projectDir : "",
|
|
5427
|
-
tags: parseTags(row["tags"])
|
|
5839
|
+
tags: parseTags(row["tags"]),
|
|
5840
|
+
note: typeof note === "string" && note.length > 0 ? note : void 0
|
|
5428
5841
|
};
|
|
5429
5842
|
}
|
|
5843
|
+
function parseOrigin(value) {
|
|
5844
|
+
return value === "dream" || value === "user" || value === "session" ? value : "session";
|
|
5845
|
+
}
|
|
5846
|
+
/** Column value for `distilled_from`: undefined when there is nothing to store. */
|
|
5847
|
+
function stringifyDistilledFrom(sources) {
|
|
5848
|
+
return sources === void 0 || sources.length === 0 ? null : JSON.stringify(sources);
|
|
5849
|
+
}
|
|
5850
|
+
/** Parse the `distilled_from` JSON column, dropping entries that fail validation. */
|
|
5851
|
+
function parseDistilledFrom(value) {
|
|
5852
|
+
if (typeof value !== "string" || value.length === 0) return void 0;
|
|
5853
|
+
try {
|
|
5854
|
+
const parsed = JSON.parse(value);
|
|
5855
|
+
if (!Array.isArray(parsed)) return void 0;
|
|
5856
|
+
const sources = parsed.filter((item) => typeof item === "object" && item !== null && typeof item.id === "string" && typeof item.title === "string" && typeof item.at === "number" && Number.isFinite(item.at));
|
|
5857
|
+
return sources.length > 0 ? sources.slice(0, 100) : void 0;
|
|
5858
|
+
} catch {
|
|
5859
|
+
return;
|
|
5860
|
+
}
|
|
5861
|
+
}
|
|
5430
5862
|
function parseTags(value) {
|
|
5431
5863
|
if (typeof value !== "string" || value.length === 0) return void 0;
|
|
5432
5864
|
try {
|
|
@@ -5438,12 +5870,17 @@ function parseTags(value) {
|
|
|
5438
5870
|
return;
|
|
5439
5871
|
}
|
|
5440
5872
|
}
|
|
5873
|
+
function likePattern(value) {
|
|
5874
|
+
return `%${value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_")}%`;
|
|
5875
|
+
}
|
|
5441
5876
|
function memoMatchesSearch(memo, search) {
|
|
5442
5877
|
return [
|
|
5443
5878
|
memo.userNeed,
|
|
5444
5879
|
memo.approach,
|
|
5445
5880
|
memo.whatFailed,
|
|
5446
5881
|
memo.whatWorked,
|
|
5882
|
+
memo.distilled ?? "",
|
|
5883
|
+
memo.note ?? "",
|
|
5447
5884
|
memo.sourceSessionTitle ?? "",
|
|
5448
5885
|
...memo.tags ?? []
|
|
5449
5886
|
].join(" ").toLowerCase().replaceAll(/\s+/g, "").includes(search.replaceAll(/\s+/g, ""));
|
|
@@ -5617,6 +6054,7 @@ async function buildConsolidationPlan(store, options) {
|
|
|
5617
6054
|
async function applyConsolidation(store, plan) {
|
|
5618
6055
|
let deleted = 0;
|
|
5619
6056
|
let created = 0;
|
|
6057
|
+
let skipped = 0;
|
|
5620
6058
|
if (plan.resolved.length > 0) {
|
|
5621
6059
|
const worked = plan.resolved.filter((m) => m.whatWorked).map((m) => m.whatWorked).join("; ");
|
|
5622
6060
|
const failed = plan.resolved.filter((m) => m.whatFailed).map((m) => m.whatFailed).join("; ");
|
|
@@ -5655,9 +6093,22 @@ async function applyConsolidation(store, plan) {
|
|
|
5655
6093
|
created++;
|
|
5656
6094
|
}
|
|
5657
6095
|
}
|
|
6096
|
+
const mergeableGroups = [];
|
|
5658
6097
|
for (const group of plan.duplicateGroups) {
|
|
6098
|
+
const distilledText = (group.merged.distilled ?? "").trim();
|
|
6099
|
+
if (distilledText.length === 0 && group.memos.some(isDistilledRule)) {
|
|
6100
|
+
skipped++;
|
|
6101
|
+
continue;
|
|
6102
|
+
}
|
|
6103
|
+
mergeableGroups.push({
|
|
6104
|
+
group,
|
|
6105
|
+
distilledText
|
|
6106
|
+
});
|
|
6107
|
+
}
|
|
6108
|
+
for (const { group, distilledText } of mergeableGroups) {
|
|
5659
6109
|
const newest = group.memos.reduce((a, b) => a.recordedAt > b.recordedAt ? a : b);
|
|
5660
6110
|
const mergedTags = normalizeTags(group.memos.flatMap((m) => m.tags ?? []));
|
|
6111
|
+
const mergedNotes = [...new Set(group.memos.map((memo) => memo.note?.trim() ?? "").filter((note) => note.length > 0))];
|
|
5661
6112
|
const merged = createMemoryMemo({
|
|
5662
6113
|
sourceSessionId: newest.sourceSessionId,
|
|
5663
6114
|
sourceSessionTitle: newest.sourceSessionTitle,
|
|
@@ -5666,14 +6117,19 @@ async function applyConsolidation(store, plan) {
|
|
|
5666
6117
|
outcome: group.merged.outcome,
|
|
5667
6118
|
whatFailed: group.merged.whatFailed,
|
|
5668
6119
|
whatWorked: group.merged.whatWorked,
|
|
6120
|
+
note: mergedNotes.length > 0 ? mergedNotes.join("; ") : void 0,
|
|
5669
6121
|
tags: group.merged.tags ?? mergedTags,
|
|
6122
|
+
distilled: distilledText.length > 0 ? distilledText : void 0,
|
|
5670
6123
|
projectDir: newest.projectDir,
|
|
5671
|
-
extractionSource: "compaction"
|
|
6124
|
+
extractionSource: "compaction",
|
|
6125
|
+
origin: "dream",
|
|
6126
|
+
distilledAt: distilledText.length > 0 ? Date.now() : void 0,
|
|
6127
|
+
distilledFrom: collectLineage(group.memos)
|
|
5672
6128
|
});
|
|
5673
6129
|
await store.append(merged);
|
|
5674
6130
|
created++;
|
|
5675
6131
|
}
|
|
5676
|
-
for (const group of
|
|
6132
|
+
for (const { group } of mergeableGroups) for (const memo of group.memos) {
|
|
5677
6133
|
await store.delete(memo.id);
|
|
5678
6134
|
deleted++;
|
|
5679
6135
|
}
|
|
@@ -5687,9 +6143,31 @@ async function applyConsolidation(store, plan) {
|
|
|
5687
6143
|
}
|
|
5688
6144
|
return {
|
|
5689
6145
|
deleted,
|
|
5690
|
-
created
|
|
6146
|
+
created,
|
|
6147
|
+
skipped
|
|
5691
6148
|
};
|
|
5692
6149
|
}
|
|
6150
|
+
/**
|
|
6151
|
+
* Build the lineage snapshot for a merged memo: every source memo in the group
|
|
6152
|
+
* plus each source's own lineage, deduped by id so a rule distilled from other
|
|
6153
|
+
* rules keeps its whole ancestry. Sorted oldest-first and capped at 100.
|
|
6154
|
+
*/
|
|
6155
|
+
function collectLineage(memos) {
|
|
6156
|
+
const seen = /* @__PURE__ */ new Map();
|
|
6157
|
+
for (const memo of memos) {
|
|
6158
|
+
if (!seen.has(memo.id)) seen.set(memo.id, {
|
|
6159
|
+
id: memo.id,
|
|
6160
|
+
title: clip(memo.userNeed, 80),
|
|
6161
|
+
at: memo.recordedAt
|
|
6162
|
+
});
|
|
6163
|
+
for (const source of memo.distilledFrom ?? []) if (!seen.has(source.id)) seen.set(source.id, source);
|
|
6164
|
+
}
|
|
6165
|
+
return [...seen.values()].toSorted((a, b) => a.at - b.at).slice(-100);
|
|
6166
|
+
}
|
|
6167
|
+
function clip(text, maxLength) {
|
|
6168
|
+
const chars = Array.from(text);
|
|
6169
|
+
return chars.length <= maxLength ? text : chars.slice(0, maxLength).join("");
|
|
6170
|
+
}
|
|
5693
6171
|
function findDuplicateGroups(memos) {
|
|
5694
6172
|
const groups = [];
|
|
5695
6173
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -5768,7 +6246,8 @@ function buildDuplicateGroup(cluster) {
|
|
|
5768
6246
|
approach: `合并 ${cluster.length} 条相关记录。最新方案: ${newest.approach}`,
|
|
5769
6247
|
outcome: bestOutcome,
|
|
5770
6248
|
whatFailed: failures.size > 0 ? [...failures].join("; ") : "none",
|
|
5771
|
-
whatWorked: successes.size > 0 ? [...successes].join("; ") : "none"
|
|
6249
|
+
whatWorked: successes.size > 0 ? [...successes].join("; ") : "none",
|
|
6250
|
+
distilled: ""
|
|
5772
6251
|
},
|
|
5773
6252
|
reason: `发现 ${cluster.length} 条相似记录(关键词重叠 > ${Math.round(SIMILARITY_THRESHOLD * 100)}%)`
|
|
5774
6253
|
};
|
|
@@ -5831,12 +6310,19 @@ function isOutcomeCompleted(outcome) {
|
|
|
5831
6310
|
const lower = outcome.toLowerCase();
|
|
5832
6311
|
return lower.includes("完成") || lower.includes("done") || lower.includes("completed") || lower.includes("成功") || lower.includes("success");
|
|
5833
6312
|
}
|
|
6313
|
+
/**
|
|
6314
|
+
* A distilled rule is not a stale experience: pruning it by age would delete
|
|
6315
|
+
* the conclusion /dream exists to accumulate.
|
|
6316
|
+
*/
|
|
6317
|
+
function isDistilledRule(memo) {
|
|
6318
|
+
return memo.origin === "dream" || memo.origin === "user" || (memo.distilled ?? "").trim().length > 0;
|
|
6319
|
+
}
|
|
5834
6320
|
function findResolved(memos) {
|
|
5835
|
-
return memos.filter((m) => isOutcomeCompleted(m.outcome) && Date.now() - m.recordedAt > 10080 * 60 * 1e3);
|
|
6321
|
+
return memos.filter((m) => !isDistilledRule(m) && isOutcomeCompleted(m.outcome) && Date.now() - m.recordedAt > 10080 * 60 * 1e3);
|
|
5836
6322
|
}
|
|
5837
6323
|
function findStale(memos, staleDays) {
|
|
5838
6324
|
const threshold = Date.now() - staleDays * 24 * 60 * 60 * 1e3;
|
|
5839
|
-
return memos.filter((m) => m.recordedAt < threshold && !isOutcomeCompleted(m.outcome) && !m.outcome.includes("blocked"));
|
|
6325
|
+
return memos.filter((m) => !isDistilledRule(m) && m.recordedAt < threshold && !isOutcomeCompleted(m.outcome) && !m.outcome.includes("blocked"));
|
|
5840
6326
|
}
|
|
5841
6327
|
//#endregion
|
|
5842
6328
|
//#region ../../packages/memory/src/dream.ts
|
|
@@ -5918,10 +6404,16 @@ var DreamTracker = class {
|
|
|
5918
6404
|
shouldSuggest() {
|
|
5919
6405
|
return (Date.now() - new Date(this.state.lastDreamAt).getTime()) / (1e3 * 60 * 60) >= MIN_HOURS_BETWEEN_DREAMS && this.state.sessionsSinceLastDream >= MIN_SESSIONS_BETWEEN_DREAMS;
|
|
5920
6406
|
}
|
|
5921
|
-
/**
|
|
6407
|
+
/**
|
|
6408
|
+
* The suggestion injected as a system reminder when conditions are met. It is
|
|
6409
|
+
* read by the model, which relays it to the user in the user's language, so it
|
|
6410
|
+
* is English like every other reminder the engine injects.
|
|
6411
|
+
*/
|
|
5922
6412
|
getSuggestionMessage() {
|
|
5923
6413
|
const hoursSince = (Date.now() - new Date(this.state.lastDreamAt).getTime()) / (1e3 * 60 * 60);
|
|
5924
|
-
|
|
6414
|
+
const days = Math.floor(hoursSince / 24);
|
|
6415
|
+
const sessions = this.state.sessionsSinceLastDream;
|
|
6416
|
+
return `It has been ${days} ${days === 1 ? "day" : "days"} and ${sessions} ${sessions === 1 ? "session" : "sessions"} since the last memory consolidation. Consider suggesting that the user run /dream to merge duplicate memories, clear stale entries and resolve contradictions.`;
|
|
5925
6417
|
}
|
|
5926
6418
|
async persist() {
|
|
5927
6419
|
const data = {
|
|
@@ -5981,7 +6473,7 @@ function entryFor(homeDir) {
|
|
|
5981
6473
|
await store.init();
|
|
5982
6474
|
const cacheDir = join$1(homeDir, "cache", "fastembed");
|
|
5983
6475
|
mkdirSync(cacheDir, { recursive: true });
|
|
5984
|
-
store.setEmbeddingEngine(
|
|
6476
|
+
store.setEmbeddingEngine(createEmbeddingEngine(cacheDir));
|
|
5985
6477
|
} catch (error) {
|
|
5986
6478
|
entries.delete(homeDir);
|
|
5987
6479
|
store.close();
|
|
@@ -5999,7 +6491,7 @@ function entryFor(homeDir) {
|
|
|
5999
6491
|
*
|
|
6000
6492
|
* The map key is the raw homeDir string so the engine cache key
|
|
6001
6493
|
* join(homeDir, 'cache', 'fastembed') stays byte-identical with every other
|
|
6002
|
-
*
|
|
6494
|
+
* createEmbeddingEngine call site: same string → same engine instance.
|
|
6003
6495
|
*/
|
|
6004
6496
|
function sharedKnowledgeStore(homeDir) {
|
|
6005
6497
|
return entryFor(homeDir).store;
|
|
@@ -6013,12 +6505,13 @@ function sharedKnowledgeStoreReady(homeDir) {
|
|
|
6013
6505
|
const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
6014
6506
|
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
|
6015
6507
|
/**
|
|
6016
|
-
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the
|
|
6017
|
-
*
|
|
6508
|
+
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the WASM model used here)
|
|
6509
|
+
* has a 512-token sequence limit; chunks longer than that get truncated
|
|
6018
6510
|
* at the embedding layer, hurting retrieval quality. We split at ~480 to leave
|
|
6019
6511
|
* headroom for the heading line that gets prepended at embed time.
|
|
6020
6512
|
*/
|
|
6021
6513
|
const MAX_CHUNK_TOKENS = 480;
|
|
6514
|
+
const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
|
6022
6515
|
/**
|
|
6023
6516
|
* Split a markdown document into sections using heading_strict strategy:
|
|
6024
6517
|
* each section begins at a heading and contains all body content until the
|
|
@@ -6094,55 +6587,33 @@ function stripMarkdown(text) {
|
|
|
6094
6587
|
* embedding model's token cap (same MAX_CHUNK_TOKENS as markdown).
|
|
6095
6588
|
*/
|
|
6096
6589
|
function chunkText(content) {
|
|
6097
|
-
const MAX_PARA_TOKENS = MAX_CHUNK_TOKENS;
|
|
6098
|
-
const MAX_PARA_CHARS = MAX_PARA_TOKENS * 4;
|
|
6099
6590
|
const paragraphs = content.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
6100
6591
|
const sections = [];
|
|
6101
6592
|
let rank = 0;
|
|
6102
6593
|
for (const para of paragraphs) {
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
|
|
6110
|
-
|
|
6111
|
-
continue;
|
|
6112
|
-
}
|
|
6113
|
-
const sentences = para.match(/[^.!?。!?\n]+[.!?。!?\n]+|[^.!?。!?\n]+$/g) ?? [para];
|
|
6114
|
-
let buffer = "";
|
|
6115
|
-
const flushBuffer = () => {
|
|
6116
|
-
const text = buffer.trim();
|
|
6117
|
-
if (text.length === 0) return;
|
|
6118
|
-
sections.push({
|
|
6119
|
-
heading: null,
|
|
6120
|
-
headingLevel: null,
|
|
6121
|
-
content: text,
|
|
6122
|
-
rawContent: text,
|
|
6123
|
-
rank: rank++
|
|
6124
|
-
});
|
|
6125
|
-
buffer = "";
|
|
6126
|
-
};
|
|
6127
|
-
for (const sentence of sentences) {
|
|
6128
|
-
const candidate = (buffer + " " + sentence).trim();
|
|
6129
|
-
if (buffer.length > 0 && candidate.length > MAX_PARA_CHARS) flushBuffer();
|
|
6130
|
-
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
6131
|
-
}
|
|
6132
|
-
flushBuffer();
|
|
6594
|
+
const parts = estimateTokens(para) <= MAX_CHUNK_TOKENS ? [para] : splitParagraph(para);
|
|
6595
|
+
for (const text of parts) sections.push({
|
|
6596
|
+
heading: null,
|
|
6597
|
+
headingLevel: null,
|
|
6598
|
+
content: text,
|
|
6599
|
+
rawContent: text,
|
|
6600
|
+
rank: rank++
|
|
6601
|
+
});
|
|
6133
6602
|
}
|
|
6134
6603
|
return sections;
|
|
6135
6604
|
}
|
|
6136
|
-
/**
|
|
6605
|
+
/** Approximate one token per CJK character and one per four other characters. */
|
|
6137
6606
|
function estimateTokens(text) {
|
|
6138
|
-
|
|
6607
|
+
let quarterTokens = 0;
|
|
6608
|
+
for (const char of text) quarterTokens += CJK_CHAR.test(char) ? 4 : char.length;
|
|
6609
|
+
return Math.ceil(quarterTokens / 4);
|
|
6139
6610
|
}
|
|
6140
6611
|
/**
|
|
6141
6612
|
* Split a section that exceeds MAX_CHUNK_TOKENS into smaller sub-sections.
|
|
6142
6613
|
* Strategy (mirrors SAG's `splitLargeSection`):
|
|
6143
6614
|
* 1. Split into paragraphs by blank lines.
|
|
6144
6615
|
* 2. Greedily pack paragraphs into chunks under the token cap.
|
|
6145
|
-
* 3. A paragraph that alone exceeds the cap is split at sentence boundaries;
|
|
6616
|
+
* 3. A paragraph that alone exceeds the cap is split at line or sentence boundaries;
|
|
6146
6617
|
* a sentence that itself exceeds the cap is split by character limit.
|
|
6147
6618
|
*
|
|
6148
6619
|
* Sections under the cap are returned unchanged (single-element array).
|
|
@@ -6152,7 +6623,7 @@ function estimateTokens(text) {
|
|
|
6152
6623
|
function splitLargeSection(section) {
|
|
6153
6624
|
if (estimateTokens(section.content) <= MAX_CHUNK_TOKENS) return [section];
|
|
6154
6625
|
const paragraphs = section.rawContent.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
6155
|
-
if (paragraphs.length <= 1) return
|
|
6626
|
+
if (paragraphs.length <= 1) return splitSectionParagraph(section, section.rawContent);
|
|
6156
6627
|
const result = [];
|
|
6157
6628
|
let buffer = [];
|
|
6158
6629
|
let bufferTokens = 0;
|
|
@@ -6178,7 +6649,7 @@ function splitLargeSection(section) {
|
|
|
6178
6649
|
const paraTokens = estimateTokens(para);
|
|
6179
6650
|
if (paraTokens > MAX_CHUNK_TOKENS) {
|
|
6180
6651
|
flushBuffer();
|
|
6181
|
-
for (const frag of
|
|
6652
|
+
for (const frag of splitSectionParagraph(section, para)) result.push(frag);
|
|
6182
6653
|
continue;
|
|
6183
6654
|
}
|
|
6184
6655
|
if (buffer.length > 0 && bufferTokens + paraTokens > MAX_CHUNK_TOKENS) flushBuffer();
|
|
@@ -6188,73 +6659,58 @@ function splitLargeSection(section) {
|
|
|
6188
6659
|
flushBuffer();
|
|
6189
6660
|
return result.length > 0 ? result : [section];
|
|
6190
6661
|
}
|
|
6191
|
-
|
|
6192
|
-
|
|
6193
|
-
|
|
6194
|
-
|
|
6195
|
-
|
|
6196
|
-
|
|
6662
|
+
function splitSectionParagraph(section, text) {
|
|
6663
|
+
return splitParagraph(text).map((raw) => ({
|
|
6664
|
+
...section,
|
|
6665
|
+
content: stripMarkdown(raw),
|
|
6666
|
+
rawContent: raw,
|
|
6667
|
+
rank: 0
|
|
6668
|
+
}));
|
|
6669
|
+
}
|
|
6670
|
+
/** Keep lines intact where possible, including Markdown table rows. */
|
|
6671
|
+
function splitParagraph(text) {
|
|
6672
|
+
const segments = (text.match(/[^\n]*\n|[^\n]+$/g) ?? [text]).flatMap((line) => estimateTokens(line) <= MAX_CHUNK_TOKENS ? [line] : line.match(/[^。!?!?.]*[。!?!?.]+|[^。!?!?.]+$/g) ?? [line]);
|
|
6197
6673
|
const result = [];
|
|
6198
6674
|
let buffer = "";
|
|
6199
6675
|
let bufferTokens = 0;
|
|
6200
6676
|
const flush = () => {
|
|
6201
6677
|
const trimmed = buffer.trim();
|
|
6202
|
-
if (trimmed.length
|
|
6203
|
-
buffer = "";
|
|
6204
|
-
bufferTokens = 0;
|
|
6205
|
-
return;
|
|
6206
|
-
}
|
|
6207
|
-
result.push({
|
|
6208
|
-
heading: section.heading,
|
|
6209
|
-
headingLevel: section.headingLevel,
|
|
6210
|
-
content: stripMarkdown(trimmed),
|
|
6211
|
-
rawContent: trimmed,
|
|
6212
|
-
rank: 0
|
|
6213
|
-
});
|
|
6678
|
+
if (trimmed.length > 0) result.push(trimmed);
|
|
6214
6679
|
buffer = "";
|
|
6215
6680
|
bufferTokens = 0;
|
|
6216
6681
|
};
|
|
6217
|
-
for (const
|
|
6218
|
-
const
|
|
6219
|
-
if (
|
|
6682
|
+
for (const segment of segments) {
|
|
6683
|
+
const tokens = estimateTokens(segment);
|
|
6684
|
+
if (tokens > MAX_CHUNK_TOKENS) {
|
|
6220
6685
|
flush();
|
|
6221
|
-
for (const
|
|
6222
|
-
heading: section.heading,
|
|
6223
|
-
headingLevel: section.headingLevel,
|
|
6224
|
-
content: stripMarkdown(frag),
|
|
6225
|
-
rawContent: frag,
|
|
6226
|
-
rank: 0
|
|
6227
|
-
});
|
|
6686
|
+
for (const part of splitByCharLimit(segment)) result.push(part);
|
|
6228
6687
|
continue;
|
|
6229
6688
|
}
|
|
6230
|
-
if (buffer.length > 0 && bufferTokens +
|
|
6231
|
-
buffer
|
|
6232
|
-
bufferTokens +=
|
|
6689
|
+
if (buffer.length > 0 && bufferTokens + tokens > MAX_CHUNK_TOKENS) flush();
|
|
6690
|
+
buffer += segment;
|
|
6691
|
+
bufferTokens += tokens;
|
|
6233
6692
|
}
|
|
6234
6693
|
flush();
|
|
6235
|
-
return result.length > 0 ? result : [
|
|
6236
|
-
heading: section.heading,
|
|
6237
|
-
headingLevel: section.headingLevel,
|
|
6238
|
-
content: section.content,
|
|
6239
|
-
rawContent: section.rawContent,
|
|
6240
|
-
rank: 0
|
|
6241
|
-
}];
|
|
6242
|
-
}
|
|
6243
|
-
/** Split text into sentences using CJK + ASCII punctuation. */
|
|
6244
|
-
function splitSentences(text) {
|
|
6245
|
-
return text.match(/[^。!?!?.\n]+[。!?!?.\n]+|[^。!?!?.\n]+$/g) ?? [text];
|
|
6694
|
+
return result.length > 0 ? result : [text];
|
|
6246
6695
|
}
|
|
6247
6696
|
/** Hard character cut for pathological single sentences longer than the cap. */
|
|
6248
6697
|
function splitByCharLimit(text) {
|
|
6249
|
-
const maxChars = MAX_CHUNK_TOKENS * 4;
|
|
6250
6698
|
const chunks = [];
|
|
6251
|
-
let
|
|
6252
|
-
|
|
6253
|
-
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
|
|
6257
|
-
|
|
6699
|
+
let start = 0;
|
|
6700
|
+
let offset = 0;
|
|
6701
|
+
let quarterTokens = 0;
|
|
6702
|
+
for (const char of text) {
|
|
6703
|
+
const cost = CJK_CHAR.test(char) ? 4 : char.length;
|
|
6704
|
+
if (quarterTokens + cost > MAX_CHUNK_TOKENS * 4) {
|
|
6705
|
+
chunks.push(text.slice(start, offset));
|
|
6706
|
+
start = offset;
|
|
6707
|
+
quarterTokens = 0;
|
|
6708
|
+
}
|
|
6709
|
+
quarterTokens += cost;
|
|
6710
|
+
offset += char.length;
|
|
6711
|
+
}
|
|
6712
|
+
chunks.push(text.slice(start));
|
|
6713
|
+
return chunks.map((chunk) => chunk.trim()).filter((chunk) => chunk.length > 0);
|
|
6258
6714
|
}
|
|
6259
6715
|
//#endregion
|
|
6260
6716
|
//#region ../../packages/knowledge/src/extractor.ts
|
|
@@ -6549,14 +7005,52 @@ async function embedOrNulls(engine, texts, skipEmbedding) {
|
|
|
6549
7005
|
}
|
|
6550
7006
|
/** LLM concurrency for extraction — callers and comments below refer to this constant. */
|
|
6551
7007
|
const LLM_CONCURRENCY = 20;
|
|
6552
|
-
const
|
|
7008
|
+
const TEXT_EXTENSIONS = new Set([
|
|
6553
7009
|
".md",
|
|
6554
7010
|
".markdown",
|
|
6555
7011
|
".txt"
|
|
6556
7012
|
]);
|
|
6557
|
-
|
|
6558
|
-
|
|
6559
|
-
|
|
7013
|
+
const DOCUMENT_EXTENSIONS = new Set([
|
|
7014
|
+
".pdf",
|
|
7015
|
+
".docx",
|
|
7016
|
+
".xlsx"
|
|
7017
|
+
]);
|
|
7018
|
+
const SUPPORTED_EXTENSIONS = new Set([...TEXT_EXTENSIONS, ...DOCUMENT_EXTENSIONS]);
|
|
7019
|
+
const MAX_DOCUMENT_BYTES = 50 * 1024 * 1024;
|
|
7020
|
+
function stripImagePlaceholders(markdown) {
|
|
7021
|
+
return markdown.replaceAll(/<!--\s*image:[^>]*-->/g, "").replaceAll(/\n{3,}/g, "\n\n").trim();
|
|
7022
|
+
}
|
|
7023
|
+
async function readSource(filePath) {
|
|
7024
|
+
const ext = getFileExtension(filePath);
|
|
7025
|
+
if (!TEXT_EXTENSIONS.has(ext)) {
|
|
7026
|
+
const source = await stat(filePath);
|
|
7027
|
+
if (!source.isFile()) throw new Error(`来源不是文件: ${basename$1(filePath)}`);
|
|
7028
|
+
const { size } = source;
|
|
7029
|
+
if (size > MAX_DOCUMENT_BYTES) throw new Error(`文档超过 ${MAX_DOCUMENT_BYTES / 1024 / 1024}MB 上限,已跳过: ${basename$1(filePath)}`);
|
|
7030
|
+
}
|
|
7031
|
+
const bytes = await readFile(filePath);
|
|
7032
|
+
return {
|
|
7033
|
+
bytes,
|
|
7034
|
+
ext,
|
|
7035
|
+
contentHash: TEXT_EXTENSIONS.has(ext) ? createHash("sha256").update(bytes.toString("utf8")).digest("hex") : createHash("sha256").update(bytes).digest("hex")
|
|
7036
|
+
};
|
|
7037
|
+
}
|
|
7038
|
+
async function convertSource(filePath, bytes, ext, onConverting) {
|
|
7039
|
+
onConverting?.(basename$1(filePath));
|
|
7040
|
+
const { convertDocumentToMarkdown, DocumentMarkdownError } = await import("./markdown-BwvhOvCA.mjs");
|
|
7041
|
+
try {
|
|
7042
|
+
const content = stripImagePlaceholders((await convertDocumentToMarkdown({
|
|
7043
|
+
bytes,
|
|
7044
|
+
filename: basename$1(filePath)
|
|
7045
|
+
}, { useCache: false })).markdown);
|
|
7046
|
+
if (content.length > 0) return content;
|
|
7047
|
+
} catch (error) {
|
|
7048
|
+
if (!(error instanceof DocumentMarkdownError) || error.code !== "empty_output") {
|
|
7049
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
7050
|
+
throw new Error(`文档转换失败: ${message}`, { cause: error });
|
|
7051
|
+
}
|
|
7052
|
+
}
|
|
7053
|
+
throw new Error(ext === ".pdf" ? "文档未提取到文字(可能是扫描版 PDF),当前转换器不包含 OCR" : "文档未提取到可索引的文字,文件可能已损坏或为空");
|
|
6560
7054
|
}
|
|
6561
7055
|
function getFileExtension(filePath) {
|
|
6562
7056
|
const lastDot = filePath.lastIndexOf(".");
|
|
@@ -6606,10 +7100,10 @@ async function ensureEmbeddingModelMatches(store, engine) {
|
|
|
6606
7100
|
if (recorded !== engine.modelName) throw new Error(`知识库由模型 ${recorded} 嵌入,当前模型为 ${engine.modelName},请先在 /knowledge 中重新嵌入知识库`);
|
|
6607
7101
|
}
|
|
6608
7102
|
/**
|
|
6609
|
-
* Ingest a
|
|
7103
|
+
* Ingest a file into the knowledge base.
|
|
6610
7104
|
*
|
|
6611
7105
|
* Steps:
|
|
6612
|
-
* 1.
|
|
7106
|
+
* 1. Fingerprint the source by path and content, keeping the previous revision until commit.
|
|
6613
7107
|
* 2. Read file → chunkMarkdown (heading_strict).
|
|
6614
7108
|
* 3. Embed chunks ("{heading}\n{content}") → store knowledge_chunks.
|
|
6615
7109
|
* 4. For each chunk (concurrency LLM_CONCURRENCY): LLM extract 1 event + N entities.
|
|
@@ -6624,6 +7118,10 @@ async function ensureEmbeddingModelMatches(store, engine) {
|
|
|
6624
7118
|
* Reports progress via callback.
|
|
6625
7119
|
*/
|
|
6626
7120
|
async function ingestFile(store, llm, filePath, onProgress) {
|
|
7121
|
+
if (!isSupportedFile(filePath)) {
|
|
7122
|
+
const ext = getFileExtension(filePath);
|
|
7123
|
+
throw new Error(`不支持的文件类型: ${ext === "" ? "(无扩展名)" : ext}`);
|
|
7124
|
+
}
|
|
6627
7125
|
const engine = store.getEmbeddingEngine();
|
|
6628
7126
|
const skipEmbed = engine === void 0 || !engine.available;
|
|
6629
7127
|
if (skipEmbed) onProgress?.({
|
|
@@ -6635,12 +7133,10 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
6635
7133
|
stage: "chunking",
|
|
6636
7134
|
message: `读取并切分文件: ${basename$1(filePath)}`
|
|
6637
7135
|
});
|
|
6638
|
-
const content = await readFileContent(filePath);
|
|
6639
|
-
const contentHash = createHash("sha256").update(content).digest("hex");
|
|
6640
7136
|
const existing = await store.findSourceByFilePath(filePath);
|
|
6641
|
-
|
|
7137
|
+
const { bytes, ext, contentHash } = await readSource(filePath);
|
|
6642
7138
|
if (existing !== void 0) {
|
|
6643
|
-
if (existing.contentHash === contentHash || existing.contentHash === null) {
|
|
7139
|
+
if (existing.contentHash === contentHash || existing.contentHash === null && ext !== ".xlsx") {
|
|
6644
7140
|
if (existing.contentHash === null) await store.updateSourceContentHash(existing.id, contentHash);
|
|
6645
7141
|
onProgress?.({
|
|
6646
7142
|
stage: "completed",
|
|
@@ -6656,21 +7152,27 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
6656
7152
|
}
|
|
6657
7153
|
onProgress?.({
|
|
6658
7154
|
stage: "chunking",
|
|
6659
|
-
message:
|
|
7155
|
+
message: `准备更新知识文件: ${existing.name}`
|
|
6660
7156
|
});
|
|
6661
|
-
await store.deleteSource(existing.id);
|
|
6662
|
-
replacedOldVersion = true;
|
|
6663
7157
|
}
|
|
6664
|
-
const
|
|
7158
|
+
const content = TEXT_EXTENSIONS.has(ext) ? bytes.toString("utf8") : await convertSource(filePath, bytes, ext, (file) => {
|
|
7159
|
+
onProgress?.({
|
|
7160
|
+
stage: "converting",
|
|
7161
|
+
message: file
|
|
7162
|
+
});
|
|
7163
|
+
});
|
|
7164
|
+
const sections = chunkContent(content, ext);
|
|
7165
|
+
if (sections.length === 0) {
|
|
7166
|
+
onProgress?.({
|
|
7167
|
+
stage: "error",
|
|
7168
|
+
message: "文件无有效内容"
|
|
7169
|
+
});
|
|
7170
|
+
throw new Error("文件无有效内容可摄入");
|
|
7171
|
+
}
|
|
6665
7172
|
store.beginTransaction();
|
|
6666
7173
|
try {
|
|
6667
|
-
if (
|
|
6668
|
-
|
|
6669
|
-
stage: "error",
|
|
6670
|
-
message: "文件无有效内容"
|
|
6671
|
-
});
|
|
6672
|
-
throw new Error("文件无有效内容可摄入");
|
|
6673
|
-
}
|
|
7174
|
+
if ((await store.findSourceByFilePath(filePath))?.id !== existing?.id) throw new Error(`知识文件索引已被其他操作更新,请重试: ${basename$1(filePath)}`);
|
|
7175
|
+
if (existing !== void 0) await store.deleteSource(existing.id);
|
|
6674
7176
|
const fileName = basename$1(filePath);
|
|
6675
7177
|
const source = await store.createSource({
|
|
6676
7178
|
name: fileName,
|
|
@@ -6695,14 +7197,10 @@ async function ingestFile(store, llm, filePath, onProgress) {
|
|
|
6695
7197
|
chunkCount: chunks.length,
|
|
6696
7198
|
eventCount: events.length,
|
|
6697
7199
|
entityCount: uniqueEntities.length,
|
|
6698
|
-
outcome:
|
|
7200
|
+
outcome: existing === void 0 ? "created" : "updated"
|
|
6699
7201
|
};
|
|
6700
7202
|
} catch (error) {
|
|
6701
7203
|
store.rollbackTransaction();
|
|
6702
|
-
if (replacedOldVersion) {
|
|
6703
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
6704
|
-
throw new Error(`旧版本已删除、新版本摄入失败,请重新摄入该文件: ${filePath}(${message})`, { cause: error });
|
|
6705
|
-
}
|
|
6706
7204
|
throw error;
|
|
6707
7205
|
}
|
|
6708
7206
|
}
|
|
@@ -6865,6 +7363,7 @@ async function collectSupportedFiles(dirPath) {
|
|
|
6865
7363
|
const files = [];
|
|
6866
7364
|
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
6867
7365
|
for (const entry of entries) {
|
|
7366
|
+
if (entry.name.startsWith(".") || entry.name.startsWith("~$")) continue;
|
|
6868
7367
|
const fullPath = join$1(dirPath, entry.name);
|
|
6869
7368
|
if (entry.isDirectory()) files.push(...await collectSupportedFiles(fullPath));
|
|
6870
7369
|
else if (entry.isFile() && isSupportedFile(fullPath)) files.push(fullPath);
|
|
@@ -6873,7 +7372,7 @@ async function collectSupportedFiles(dirPath) {
|
|
|
6873
7372
|
}
|
|
6874
7373
|
async function ingestDirectory(store, llm, dirPath, onProgress) {
|
|
6875
7374
|
const files = await collectSupportedFiles(dirPath);
|
|
6876
|
-
if (files.length === 0) throw new Error(`目录下没有支持的文件 (.
|
|
7375
|
+
if (files.length === 0) throw new Error(`目录下没有支持的文件 (${[...SUPPORTED_EXTENSIONS].join(", ")}): ${dirPath}`);
|
|
6877
7376
|
const errors = [];
|
|
6878
7377
|
let succeeded = 0;
|
|
6879
7378
|
let created = 0;
|
|
@@ -6885,6 +7384,10 @@ async function ingestDirectory(store, llm, dirPath, onProgress) {
|
|
|
6885
7384
|
for (let i = 0; i < files.length; i++) {
|
|
6886
7385
|
const filePath = files[i];
|
|
6887
7386
|
const fileProgress = (progress) => {
|
|
7387
|
+
if (progress.stage === "converting") {
|
|
7388
|
+
onProgress?.(progress);
|
|
7389
|
+
return;
|
|
7390
|
+
}
|
|
6888
7391
|
const prefix = `批量摄入 ${i + 1}/${files.length}: ${basename$1(filePath)}`;
|
|
6889
7392
|
onProgress?.({
|
|
6890
7393
|
...progress,
|
|
@@ -7259,4 +7762,4 @@ async function chunkVectorFallback(store, queryVec, topK) {
|
|
|
7259
7762
|
return results;
|
|
7260
7763
|
}
|
|
7261
7764
|
//#endregion
|
|
7262
|
-
export { classifyEmbeddingFailure as A, buildProjectTagCloud as B, buildConsolidationPlan as C, stripMemoryTags as D, parseMemoryMemos as E, So as F, isAbsolute$1 as G, KnowledgeStore as H, createMemoryMemo as I, parse$1 as J, join$1 as K, toSummary as L,
|
|
7765
|
+
export { classifyEmbeddingFailure as A, buildProjectTagCloud as B, buildConsolidationPlan as C, stripMemoryTags as D, parseMemoryMemos as E, So as F, isAbsolute$1 as G, KnowledgeStore as H, createMemoryMemo as I, parse$1 as J, join$1 as K, toSummary as L, createEmbeddingEngine as M, hasEmbeddingModelCache as N, MemoryMemoStore as O, probeLocalEmbeddingSupport as P, generateTags as R, applyConsolidation as S, buildExitExtractionPrompt as T, basename$1 as U, rankMemos as V, dirname$2 as W, resolve$1 as X, relative$1 as Y, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, clearEmbeddingModelCache as j, sharedMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, normalize as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, EXIT_EXTRACTION_SYSTEM_PROMPT as w, DreamTracker as x, sharedKnowledgeStore as y, normalizeTags as z };
|