nexum-code 0.18.7 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,14 +4,14 @@ import { dirname as __cjsShimDirname } from 'node:path';
4
4
  const __filename = __cjsShimFileURLToPath(import.meta.url);
5
5
  const __dirname = __cjsShimDirname(__filename);
6
6
  import { createHash, randomBytes } from "node:crypto";
7
- import ro, { mkdir, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
7
+ import Kt, { createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs";
8
+ import path, { basename, join, posix, resolve, win32 } from "node:path";
9
+ import ro, { mkdir, open, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from "node:fs/promises";
8
10
  import I from "fs";
9
11
  import Qr, { EventEmitter } from "events";
10
- import js, { dirname, parse } from "path";
12
+ import js, { dirname as dirname$1, parse } from "path";
11
13
  import { Buffer as Buffer$1 } from "buffer";
12
14
  import Cs from "node:stream";
13
- import Kt, { createReadStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
14
- import path, { basename, join, posix, win32 } from "node:path";
15
15
  import { DatabaseSync } from "node:sqlite";
16
16
  import { EventEmitter as EventEmitter$1 } from "node:events";
17
17
  import { StringDecoder } from "node:string_decoder";
@@ -181,6 +181,93 @@ const parse$1 = function(p) {
181
181
  };
182
182
  };
183
183
  //#endregion
184
+ //#region ../../packages/knowledge/src/exact-search.ts
185
+ const CONTEXT_LENGTH = 160;
186
+ const segmenter = new Intl.Segmenter();
187
+ function normalizeText(text) {
188
+ return text.normalize("NFKC").replaceAll(/\s+/gu, " ").trim();
189
+ }
190
+ function exactTextPattern(terms) {
191
+ const unique = [...new Set(terms.map(normalizeText).filter(Boolean))];
192
+ if (unique.length === 0) return void 0;
193
+ unique.sort((left, right) => right.length - left.length);
194
+ const patterns = unique.map((term) => {
195
+ const escaped = term.replaceAll(/[.*+?^$(){}|[\]\\]/g, "\\$&");
196
+ const before = /^[a-z0-9_]/i.test(term) ? "(?<![a-z0-9_])" : "";
197
+ const after = /[a-z0-9_]$/i.test(term) ? "(?![a-z0-9_])" : "";
198
+ return before + escaped + after;
199
+ });
200
+ return new RegExp(patterns.join("|"), "giu");
201
+ }
202
+ /** Translate an offset within a normalized line back to the original grapheme boundary. */
203
+ function sourceOffset(line, offset, end) {
204
+ if (line.text === normalizeText(line.text)) return line.sourceOffset + offset;
205
+ let consumed = 0;
206
+ let previousSpace = true;
207
+ for (const { segment, index } of segmenter.segment(line.text)) {
208
+ let normalized = segment.normalize("NFKC").replaceAll(/\s+/gu, " ");
209
+ if (previousSpace && normalized.startsWith(" ")) normalized = normalized.slice(1);
210
+ const next = consumed + normalized.length;
211
+ if (end ? offset <= next : offset < next) return line.sourceOffset + index + (end ? segment.length : 0);
212
+ if (normalized.length > 0) previousSpace = normalized.endsWith(" ");
213
+ consumed = next;
214
+ }
215
+ return line.sourceOffset + line.text.length;
216
+ }
217
+ function* indexLines(text) {
218
+ let number = 1;
219
+ let normalizedOffset = 0;
220
+ for (const match of text.matchAll(/[^\r\n]+|\r\n|\r|\n/g)) {
221
+ const line = match[0];
222
+ if (line.startsWith("\r") || line === "\n") {
223
+ number++;
224
+ continue;
225
+ }
226
+ const normalized = normalizeText(line);
227
+ if (normalized.length === 0) continue;
228
+ yield {
229
+ text: line,
230
+ number,
231
+ sourceOffset: match.index,
232
+ normalizedOffset
233
+ };
234
+ normalizedOffset += normalized.length + 1;
235
+ }
236
+ }
237
+ function* findExactTextEvidence(text, pattern) {
238
+ const matches = normalizeText(text).matchAll(pattern);
239
+ let current = matches.next();
240
+ if (current.done) return;
241
+ const lines = indexLines(text);
242
+ const firstLine = lines.next();
243
+ if (firstLine.done) return;
244
+ let line = firstLine.value;
245
+ let next = lines.next();
246
+ let reportedLine = 0;
247
+ for (; !current.done; current = matches.next()) {
248
+ const match = current.value;
249
+ const end = match.index + match[0].length;
250
+ while (!next.done && next.value.normalizedOffset <= match.index) {
251
+ line = next.value;
252
+ next = lines.next();
253
+ }
254
+ const first = line;
255
+ if (first.number === reportedLine) continue;
256
+ while (!next.done && next.value.normalizedOffset < end) {
257
+ line = next.value;
258
+ next = lines.next();
259
+ }
260
+ const startOffset = sourceOffset(first, match.index - first.normalizedOffset, false);
261
+ const endOffset = sourceOffset(line, end - line.normalizedOffset, true);
262
+ reportedLine = first.number;
263
+ yield {
264
+ content: text.slice(Math.max(first.sourceOffset, startOffset - CONTEXT_LENGTH), Math.min(line.sourceOffset + line.text.length, endOffset + CONTEXT_LENGTH)),
265
+ matchedText: text.slice(startOffset, endOffset),
266
+ textLine: first.number
267
+ };
268
+ }
269
+ }
270
+ //#endregion
184
271
  //#region ../../packages/knowledge/src/store.ts
185
272
  const ENTITY_TYPES$1 = new Set([
186
273
  "person",
@@ -840,18 +927,59 @@ var KnowledgeStore = class {
840
927
  entityId: asString(row["entity_id"])
841
928
  }));
842
929
  }
930
+ async searchExact(terms, options = {}) {
931
+ const pattern = exactTextPattern(terms);
932
+ const limit = Math.min(100, Math.floor(options.limit ?? 20));
933
+ if (pattern === void 0 || !(limit > 0)) return [];
934
+ await this.init();
935
+ if (this.db === void 0) return [];
936
+ const scope = options.documentId === void 0 ? [] : [options.documentId];
937
+ const documents = this.db.prepare(`SELECT d.id, d.source_id, s.name AS source_name, d.content
938
+ FROM knowledge_documents d JOIN knowledge_sources s ON s.id = d.source_id
939
+ WHERE d.status = 'completed' ${scope.length === 0 ? "" : "AND d.id = ?"}
940
+ ORDER BY d.rowid DESC`).iterate(...scope);
941
+ const chunks = this.db.prepare("SELECT raw_content, content FROM knowledge_chunks WHERE document_id = ? ORDER BY rank");
942
+ const results = [];
943
+ for (const document of documents) {
944
+ const provenance = {
945
+ documentId: document.id,
946
+ sourceId: document.source_id,
947
+ sourceName: document.source_name
948
+ };
949
+ if (document.content?.trim()) for (const evidence of findExactTextEvidence(document.content, pattern)) {
950
+ results.push({
951
+ ...provenance,
952
+ ...evidence
953
+ });
954
+ if (results.length >= limit) return results;
955
+ }
956
+ else for (const chunk of chunks.iterate(document.id)) {
957
+ const text = asString(chunk["raw_content"]) || asString(chunk["content"]);
958
+ for (const evidence of findExactTextEvidence(text, pattern)) {
959
+ results.push({
960
+ ...provenance,
961
+ ...evidence,
962
+ textLine: null
963
+ });
964
+ if (results.length >= limit) return results;
965
+ }
966
+ }
967
+ }
968
+ return results;
969
+ }
843
970
  /** FTS hits ranked by bm25 relevance — score = -bm25 (larger is better). */
844
- async ftsSearchChunks(query, limit = 50) {
971
+ async ftsSearchChunks(query, limit = 50, documentId) {
845
972
  await this.init();
846
973
  if (this.db === void 0) return [];
847
974
  const ftsQuery = buildFtsQuery$1(query);
848
975
  if (ftsQuery === void 0) return [];
976
+ const scope = documentId === void 0 ? [] : [documentId];
849
977
  return this.db.prepare(`SELECT c.*, bm25(knowledge_chunks_fts) AS bm
850
978
  FROM knowledge_chunks c
851
979
  JOIN knowledge_chunks_fts f ON c.rowid = f.rowid
852
- WHERE f.knowledge_chunks_fts MATCH ?
980
+ WHERE f.knowledge_chunks_fts MATCH ? ${scope.length === 0 ? "" : "AND c.document_id = ?"}
853
981
  ORDER BY bm ASC
854
- LIMIT ?`).all(ftsQuery, limit).map((row) => ({
982
+ LIMIT ?`).all(ftsQuery, ...scope, limit).map((row) => ({
855
983
  chunk: rowToChunk(row),
856
984
  score: -Number(row["bm"] ?? 0)
857
985
  }));
@@ -1143,7 +1271,7 @@ function computeRelevanceScore(memo, query, usageCount = 0, currentProjectDir, p
1143
1271
  const ti = intent?.temporalBias ?? .3;
1144
1272
  const fi = intent?.factualBias ?? .6;
1145
1273
  const factors = {
1146
- keywordOverlap: computeKeywordSimilarity(memo, query),
1274
+ keywordOverlap: computeKeywordOverlap(memo, query),
1147
1275
  recency: computeRecency(memo.recordedAt),
1148
1276
  usageBoost: Math.min(.3, usageCount * .1),
1149
1277
  projectBoost: computeProjectBoost(memo.projectDir, currentProjectDir),
@@ -1341,7 +1469,21 @@ function extractKeywords(text) {
1341
1469
  return [...new Set(tokens)];
1342
1470
  }
1343
1471
  function computeKeywordSimilarity(memo, query) {
1344
- const memoWords = extractKeywords(`${memo.userNeed} ${memo.approach} ${memo.whatFailed} ${memo.whatWorked}`);
1472
+ return jaccardSimilarity(keywordText(memo), query);
1473
+ }
1474
+ /**
1475
+ * Ranking variant: the distilled conclusion and the note are separate
1476
+ * retrieval surfaces. Score each field on its own and keep the best match —
1477
+ * a short conclusion would otherwise be diluted by the long base text.
1478
+ */
1479
+ function computeKeywordOverlap(memo, query) {
1480
+ return Math.max(jaccardSimilarity(keywordText(memo), query), jaccardSimilarity(memo.distilled ?? "", query), jaccardSimilarity(memo.note ?? "", query));
1481
+ }
1482
+ function keywordText(memo) {
1483
+ return `${memo.userNeed} ${memo.approach} ${memo.whatFailed} ${memo.whatWorked}`;
1484
+ }
1485
+ function jaccardSimilarity(memoText, query) {
1486
+ const memoWords = extractKeywords(memoText);
1345
1487
  const queryWords = extractKeywords(query);
1346
1488
  if (memoWords.length === 0 || queryWords.length === 0) return 0;
1347
1489
  const intersection = memoWords.filter((w) => queryWords.includes(w)).length;
@@ -1394,6 +1536,10 @@ function createMemoryMemo(partial) {
1394
1536
  outcome: partial.outcome,
1395
1537
  whatFailed: partial.whatFailed,
1396
1538
  whatWorked: partial.whatWorked,
1539
+ distilled: partial.distilled,
1540
+ origin: partial.origin ?? "session",
1541
+ distilledFrom: partial.distilledFrom,
1542
+ distilledAt: partial.distilledAt,
1397
1543
  extractionSource: partial.extractionSource,
1398
1544
  recordedAt: partial.recordedAt ?? Date.now(),
1399
1545
  projectDir: partial.projectDir ?? "",
@@ -1415,6 +1561,10 @@ function toSummary(memo) {
1415
1561
  outcome: memo.outcome,
1416
1562
  whatFailed: memo.whatFailed,
1417
1563
  whatWorked: memo.whatWorked,
1564
+ distilled: memo.distilled,
1565
+ origin: memo.origin,
1566
+ distilledFrom: memo.distilledFrom,
1567
+ distilledAt: memo.distilledAt,
1418
1568
  extractionSource: memo.extractionSource,
1419
1569
  recordedAt: memo.recordedAt,
1420
1570
  projectDir: memo.projectDir,
@@ -2850,7 +3000,7 @@ var vn = (s) => {
2850
3000
  if (o === l) c = !1;
2851
3001
  else {
2852
3002
  let d = e.get(o);
2853
- c = d !== void 0 ? d : n(dirname(o), l, a + 1);
3003
+ c = d !== void 0 ? d : n(dirname$1(o), l, a + 1);
2854
3004
  }
2855
3005
  return e.set(o, c), c;
2856
3006
  };
@@ -4456,7 +4606,7 @@ function buildEmbeddingText(memo) {
4456
4606
  */
4457
4607
  const engineCache = /* @__PURE__ */ new Map();
4458
4608
  /**
4459
- * Create an embedding engine backed by fastembed.
4609
+ * Create an embedding engine backed by the in-process WASM pipeline.
4460
4610
  * Lazily loads the model on first use so startup is not blocked.
4461
4611
  * Engines are cached by cacheDir so repeated calls with the same cacheDir
4462
4612
  * return the same instance, sharing model state and download progress.
@@ -4464,21 +4614,22 @@ const engineCache = /* @__PURE__ */ new Map();
4464
4614
  * Defaults to "local_cache" (CWD-relative) if not provided — prefer
4465
4615
  * passing an explicit path so the cache doesn't duplicate across CWDs.
4466
4616
  */
4467
- function createFastEmbedEngine(cacheDir) {
4468
- const key = cacheDir ?? "";
4617
+ function createEmbeddingEngine(cacheDir) {
4618
+ const key = cacheDir === void 0 ? "" : resolve(cacheDir);
4469
4619
  const cached = engineCache.get(key);
4470
4620
  if (cached !== void 0) return cached;
4471
- const engine = createFastEmbedEngineImpl(cacheDir);
4621
+ const engine = createEmbeddingEngineImpl(cacheDir);
4472
4622
  engineCache.set(key, engine);
4473
4623
  return engine;
4474
4624
  }
4475
4625
  /** Stable identity of the model used by every engine created here. */
4476
4626
  const EMBEDDING_MODEL_NAME = "bge-small-zh-v1.5";
4477
- function createFastEmbedEngineImpl(cacheDir) {
4627
+ function createEmbeddingEngineImpl(cacheDir) {
4478
4628
  let embedder = null;
4479
4629
  let initPromise = null;
4480
4630
  let loadFailed = false;
4481
4631
  let lastError;
4632
+ let progressListener;
4482
4633
  return {
4483
4634
  modelName: EMBEDDING_MODEL_NAME,
4484
4635
  get available() {
@@ -4487,6 +4638,17 @@ function createFastEmbedEngineImpl(cacheDir) {
4487
4638
  get lastError() {
4488
4639
  return lastError;
4489
4640
  },
4641
+ setProgressListener(listener) {
4642
+ progressListener = listener;
4643
+ },
4644
+ unload() {
4645
+ if (embedder === null) return true;
4646
+ embedder = null;
4647
+ initPromise = null;
4648
+ loadFailed = false;
4649
+ lastError = void 0;
4650
+ return true;
4651
+ },
4490
4652
  async embedBatch(texts) {
4491
4653
  if (!this.available) return null;
4492
4654
  if (texts.length === 0) return [];
@@ -4523,12 +4685,12 @@ function createFastEmbedEngineImpl(cacheDir) {
4523
4685
  if (loadFailed || initPromise === null) {
4524
4686
  loadFailed = false;
4525
4687
  lastError = void 0;
4526
- initPromise = loadEmbedder(cacheDir);
4688
+ initPromise = loadEmbeddingRuntime(cacheDir, (progress) => progressListener?.(progress));
4527
4689
  }
4528
4690
  embedder = await initPromise;
4529
4691
  if (embedder === null) {
4530
4692
  initPromise = null;
4531
- lastError = "fastembed returned an empty model";
4693
+ lastError = "embedding runtime returned an empty session";
4532
4694
  return false;
4533
4695
  }
4534
4696
  loadFailed = false;
@@ -4540,49 +4702,156 @@ function createFastEmbedEngineImpl(cacheDir) {
4540
4702
  lastError = error instanceof Error ? error.message : String(error);
4541
4703
  return false;
4542
4704
  }
4543
- },
4544
- unload() {
4545
- if (embedder === null) return true;
4546
- embedder = null;
4547
- initPromise = null;
4548
- loadFailed = false;
4549
- lastError = void 0;
4550
- return true;
4551
4705
  }
4552
4706
  };
4553
4707
  }
4554
- async function loadEmbedder(cacheDir) {
4555
- const { FlagEmbedding, EmbeddingModel } = await import("fastembed");
4556
- if (cacheDir !== void 0) {
4557
- mkdirSync(cacheDir, { recursive: true });
4558
- await ensureEmbeddingModelFiles(cacheDir);
4559
- }
4560
- const model = EmbeddingModel.BGESmallZH;
4561
- const initOpts = cacheDir !== void 0 ? {
4562
- model,
4563
- cacheDir
4564
- } : { model };
4565
- try {
4566
- return await FlagEmbedding.init(initOpts);
4567
- } catch (initError) {
4568
- const msg = initError instanceof Error ? initError.message : "";
4569
- if (!/Config file not found|Tokenizer file not found|Tokens map file not found/iu.test(msg)) throw initError;
4570
- await ensureFastembedModelSidecars(model, cacheDir);
4571
- return FlagEmbedding.init(initOpts);
4708
+ /** Maximum tokens per text; matches the model's position-embedding limit. */
4709
+ const MAX_SEQ_LEN = 512;
4710
+ /** Texts per inference batch: 16 × 512 × 512 floats ≈ 16MB of activations. */
4711
+ const EMBED_BATCH_SIZE = 16;
4712
+ /** Threads for the WASM backend. One keeps resolution deterministic in packaged
4713
+ * builds; `NEXUM_CODE_EMBEDDING_THREADS=4` is the supported opt-in. */
4714
+ const EMBEDDING_THREADS_ENV = "NEXUM_CODE_EMBEDDING_THREADS";
4715
+ /**
4716
+ * Token-level truncation that keeps the sentence boundary tokens.
4717
+ *
4718
+ * The tokenizer package does not apply the truncation policy itself and the
4719
+ * model package ships no truncation config, so this is the only place it can
4720
+ * happen. Keeping the head (which starts with `[CLS]`) and re-appending the
4721
+ * trailing special token (`[SEP]`) mirrors what a HuggingFace pipeline would
4722
+ * produce; slicing the tail off blindly would feed the encoder a sentence that
4723
+ * never closes and drift from the reference vectors on long inputs.
4724
+ */
4725
+ function truncateEncoded(ids, types) {
4726
+ if (ids.length <= MAX_SEQ_LEN) return {
4727
+ ids: [...ids],
4728
+ types: [...types]
4729
+ };
4730
+ const headSize = MAX_SEQ_LEN - 1;
4731
+ const head = ids.slice(0, headSize);
4732
+ const headTypes = types.slice(0, headSize);
4733
+ return {
4734
+ ids: [...head, ids.at(-1)],
4735
+ types: [...headTypes, types.at(-1)]
4736
+ };
4737
+ }
4738
+ /**
4739
+ * Builds the embedding runtime in-process on `onnxruntime-web` (WASM) and the
4740
+ * pure-JS tokenizer. No native binaries, no install scripts: the whole stack is
4741
+ * JavaScript plus one WebAssembly artifact resolved from the package.
4742
+ *
4743
+ * The numeric pipeline is load-bearing and must stay compatible with the
4744
+ * reference embeddings of this model: token ids come from `tokenizer.json`, and
4745
+ * the sentence vector is the **CLS token** hidden state (`last_hidden_state[0]`)
4746
+ * followed by L2 normalization. Mean pooling produces visibly different vectors
4747
+ * (cosine ≈ 0.9 against the reference), so pooling is not a stylistic choice.
4748
+ */
4749
+ async function createWasmRuntime(modelDir) {
4750
+ const ort = await import(["onnxruntime-web", "wasm"].join("/"));
4751
+ const { Tokenizer } = await import("@huggingface/tokenizers");
4752
+ const requestedThreads = Number(process.env[EMBEDDING_THREADS_ENV]);
4753
+ ort.env.wasm.numThreads = Number.isFinite(requestedThreads) && requestedThreads > 0 ? requestedThreads : 1;
4754
+ ort.env.logLevel = "error";
4755
+ const tokenizer = new Tokenizer(JSON.parse(readFileSync(join(modelDir, "tokenizer.json"), "utf8")), JSON.parse(readFileSync(join(modelDir, "tokenizer_config.json"), "utf8")));
4756
+ const session = await ort.InferenceSession.create(readFileSync(join(modelDir, "model_optimized.onnx")), { executionProviders: ["wasm"] });
4757
+ if (session === null || session === void 0) return null;
4758
+ return { async *embed(textStrings) {
4759
+ if (textStrings.length === 0) return;
4760
+ const encoded = textStrings.map((text, index) => {
4761
+ const encoding = tokenizer.encode(text, { return_token_type_ids: true });
4762
+ const types = encoding.token_type_ids ?? encoding.ids.map(() => 0);
4763
+ const truncated = truncateEncoded(encoding.ids, types);
4764
+ return {
4765
+ index,
4766
+ ids: truncated.ids,
4767
+ types: truncated.types
4768
+ };
4769
+ });
4770
+ const results = Array.from({ length: textStrings.length });
4771
+ const buckets = encoded.toSorted((a, b) => a.ids.length - b.ids.length);
4772
+ for (let start = 0; start < buckets.length; start += EMBED_BATCH_SIZE) {
4773
+ const group = buckets.slice(start, start + EMBED_BATCH_SIZE);
4774
+ const rows = group.length;
4775
+ const width = Math.max(...group.map((entry) => entry.ids.length));
4776
+ const ids = new BigInt64Array(rows * width);
4777
+ const mask = new BigInt64Array(rows * width);
4778
+ const types = new BigInt64Array(rows * width);
4779
+ group.forEach((entry, row) => {
4780
+ for (let column = 0; column < width; column++) {
4781
+ const offset = row * width + column;
4782
+ ids[offset] = BigInt(column < entry.ids.length ? entry.ids[column] : 0);
4783
+ mask[offset] = column < entry.ids.length ? 1n : 0n;
4784
+ types[offset] = BigInt(column < entry.types.length ? entry.types[column] : 0);
4785
+ }
4786
+ });
4787
+ const output = await session.run({
4788
+ input_ids: new ort.Tensor("int64", ids, [rows, width]),
4789
+ attention_mask: new ort.Tensor("int64", mask, [rows, width]),
4790
+ token_type_ids: new ort.Tensor("int64", types, [rows, width])
4791
+ });
4792
+ const hidden = output["last_hidden_state"] ?? output[Object.keys(output)[0]];
4793
+ const hiddenSize = hidden.dims.at(-1);
4794
+ const data = hidden.data;
4795
+ group.forEach((entry, row) => {
4796
+ const rowStart = row * width * hiddenSize;
4797
+ const vector = Array.from({ length: hiddenSize }, (_, index) => data[rowStart + index]);
4798
+ let norm = 0;
4799
+ for (const value of vector) norm += value * value;
4800
+ norm = Math.sqrt(norm) || 1;
4801
+ for (let i = 0; i < vector.length; i += 1) vector[i] = vector[i] / norm;
4802
+ results[entry.index] = vector;
4803
+ });
4804
+ }
4805
+ for (let start = 0; start < results.length; start += EMBED_BATCH_SIZE) yield results.slice(start, start + EMBED_BATCH_SIZE);
4806
+ } };
4807
+ }
4808
+ async function loadEmbeddingRuntime(cacheDir, emit) {
4809
+ if (cacheDir === void 0) {
4810
+ emit?.({ phase: "load" });
4811
+ return createWasmRuntime(join("local_cache", BGESMALLZH_CACHE_NAME));
4572
4812
  }
4813
+ mkdirSync(cacheDir, { recursive: true });
4814
+ await ensureEmbeddingModelFiles(cacheDir, {
4815
+ onProgress: (received, total) => {
4816
+ emit?.(total === void 0 ? {
4817
+ phase: "download",
4818
+ received
4819
+ } : {
4820
+ phase: "download",
4821
+ received,
4822
+ total
4823
+ });
4824
+ },
4825
+ onPhase: (phase) => {
4826
+ if (phase === "verify") emit?.({ phase: "verify" });
4827
+ }
4828
+ });
4829
+ emit?.({ phase: "load" });
4830
+ return createWasmRuntime(join(cacheDir, BGESMALLZH_CACHE_NAME));
4573
4831
  }
4574
4832
  const BGESMALLZH_CACHE_NAME = "fast-bge-small-zh-v1.5";
4833
+ /** Files the runtime needs before the model can be considered installed. */
4834
+ const EMBEDDING_REQUIRED_FILES = [
4835
+ "model_optimized.onnx",
4836
+ "tokenizer.json",
4837
+ "tokenizer_config.json"
4838
+ ];
4575
4839
  /** Escape hatch for air-gapped installs and tests: points at an archive file. */
4576
4840
  const EMBEDDING_MODEL_URL_ENV = "NEXUM_CODE_EMBEDDING_MODEL_URL";
4577
- function modelWeightsPath(cacheDir) {
4578
- return join(cacheDir, BGESMALLZH_CACHE_NAME, "model_optimized.onnx");
4841
+ /** Default stall guards: a trickling connection must not spin a spinner forever. */
4842
+ const DEFAULT_IDLE_TIMEOUT_MS = 6e4;
4843
+ const DEFAULT_TOTAL_TIMEOUT_MS = 15 * 6e4;
4844
+ /** Progress callback throttle: frequent enough to move, rare enough to be free. */
4845
+ const PROGRESS_THROTTLE_MS = 100;
4846
+ function modelDirPath(cacheDir) {
4847
+ return join(cacheDir, BGESMALLZH_CACHE_NAME);
4579
4848
  }
4580
4849
  function sha256File(filePath) {
4581
4850
  return readFile(filePath).then((data) => createHash("sha256").update(data).digest("hex"));
4582
4851
  }
4583
4852
  /**
4584
4853
  * Stage the embedding model files when they are missing: download the pinned
4585
- * archive (HTTP status verified — unlike fastembed's own downloader), extract,
4854
+ * archive (HTTP status verified, unlike a naive downloader), extract,
4586
4855
  * and verify the weights payload against the pinned sha256.
4587
4856
  *
4588
4857
  * A partially extracted tree or a hash mismatch is removed so the caller keeps
@@ -4592,62 +4861,149 @@ function sha256File(filePath) {
4592
4861
  async function ensureEmbeddingModelFiles(cacheDir, options) {
4593
4862
  if (hasEmbeddingModelCache(cacheDir)) return;
4594
4863
  mkdirSync(cacheDir, { recursive: true });
4595
- const modelDir = join(cacheDir, BGESMALLZH_CACHE_NAME);
4864
+ const modelDir = modelDirPath(cacheDir);
4596
4865
  const stagingPath = join(cacheDir, `${BGESMALLZH_CACHE_NAME}.tar.gz.part`);
4597
4866
  const url = options?.sourceUrl ?? process.env[EMBEDDING_MODEL_URL_ENV] ?? "https://github.com/LIUTod/nexum-code/releases/download/embedding-models-v1/fast-bge-small-zh-v1.5.tar.gz";
4867
+ const idleTimeoutMs = options?.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
4868
+ const totalTimeoutMs = options?.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS;
4869
+ const controller = new AbortController();
4870
+ const onExternalAbort = () => {
4871
+ controller.abort();
4872
+ };
4873
+ if (options?.signal !== void 0) if (options.signal.aborted) controller.abort();
4874
+ else options.signal.addEventListener("abort", onExternalAbort, { once: true });
4875
+ let idleTimer;
4876
+ const armIdleTimer = () => {
4877
+ if (idleTimer !== void 0) clearTimeout(idleTimer);
4878
+ idleTimer = setTimeout(() => {
4879
+ controller.abort();
4880
+ }, idleTimeoutMs);
4881
+ };
4882
+ const totalTimer = setTimeout(() => {
4883
+ controller.abort();
4884
+ }, totalTimeoutMs);
4885
+ let received = 0;
4886
+ let announcedTotal;
4887
+ let downloadCompleted = false;
4888
+ const announceProgress = () => {
4889
+ options?.onProgress?.(received, announcedTotal);
4890
+ };
4598
4891
  try {
4599
4892
  let response;
4600
4893
  try {
4601
- response = await fetch(url);
4894
+ armIdleTimer();
4895
+ response = await fetch(url, { signal: controller.signal });
4602
4896
  } catch (error) {
4603
4897
  const message = error instanceof Error ? error.message : String(error);
4604
4898
  throw new Error(`fetch failed for ${url}: ${message}`, { cause: error });
4605
4899
  }
4606
4900
  if (!response.ok) throw new Error(`model source returned HTTP ${response.status} for ${url}`);
4607
- await writeFile(stagingPath, Buffer.from(await response.arrayBuffer()));
4901
+ const contentLength = Number(response.headers.get("content-length"));
4902
+ announcedTotal = Number.isFinite(contentLength) && contentLength > 0 ? contentLength : void 0;
4903
+ announceProgress();
4904
+ const body = response.body;
4905
+ if (body === null) throw new Error(`model source returned an empty body for ${url}`);
4906
+ const reader = body.getReader();
4907
+ const abortRejection = new Promise((_, reject) => {
4908
+ if (controller.signal.aborted) {
4909
+ reject(/* @__PURE__ */ new Error("aborted"));
4910
+ return;
4911
+ }
4912
+ controller.signal.addEventListener("abort", () => {
4913
+ reject(/* @__PURE__ */ new Error("aborted"));
4914
+ }, { once: true });
4915
+ });
4916
+ const abortAwareRead = () => Promise.race([reader.read(), abortRejection]);
4917
+ let handle;
4918
+ try {
4919
+ handle = await open(stagingPath, "w");
4920
+ } catch (error) {
4921
+ await reader.cancel().catch(() => void 0);
4922
+ throw error;
4923
+ }
4924
+ let lastProgressAt = 0;
4925
+ try {
4926
+ await (async () => {
4927
+ for (;;) {
4928
+ let chunk;
4929
+ try {
4930
+ chunk = await abortAwareRead();
4931
+ } catch (error) {
4932
+ const message = error instanceof Error ? error.message : String(error);
4933
+ throw new Error(`${message} (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
4934
+ }
4935
+ if (chunk.done || chunk.value === void 0) break;
4936
+ const bytes = chunk.value;
4937
+ armIdleTimer();
4938
+ await handle.write(bytes);
4939
+ received += bytes.byteLength;
4940
+ const now = Date.now();
4941
+ if (now - lastProgressAt >= PROGRESS_THROTTLE_MS) {
4942
+ lastProgressAt = now;
4943
+ announceProgress();
4944
+ }
4945
+ }
4946
+ announceProgress();
4947
+ downloadCompleted = true;
4948
+ })();
4949
+ } finally {
4950
+ if (idleTimer !== void 0) clearTimeout(idleTimer);
4951
+ await handle.close();
4952
+ await reader.cancel().catch(() => void 0);
4953
+ }
4954
+ options?.onPhase?.("verify");
4608
4955
  await So({
4609
4956
  file: stagingPath,
4610
4957
  cwd: cacheDir
4611
4958
  });
4612
- const weightsPath = modelWeightsPath(cacheDir);
4613
- if (!existsSync(weightsPath) || statSync(weightsPath).size === 0) throw new Error(`model package did not contain weights after extraction: ${url}`);
4614
- const sha256 = await sha256File(weightsPath);
4959
+ for (const fileName of EMBEDDING_REQUIRED_FILES) {
4960
+ const target = join(modelDir, fileName);
4961
+ if (!existsSync(target) || statSync(target).size === 0) throw new Error(`model package is missing ${fileName} after extraction: ${url}`);
4962
+ }
4963
+ const sha256 = await sha256File(join(modelDir, "model_optimized.onnx"));
4615
4964
  const expected = options?.expectedOnnxSha256 ?? "1294ea4b6331115a353d81f96b85e8c8d7fdcc284453d5b2fab5b016230aad38";
4616
4965
  if (sha256 !== expected) throw new Error(`model weights sha256 mismatch (${sha256.slice(0, 12)}… != ${expected.slice(0, 12)}…): ${url}`);
4617
4966
  } catch (error) {
4618
- rmSync(modelDir, {
4619
- recursive: true,
4620
- force: true
4621
- });
4622
- throw error instanceof Error ? error : new Error(String(error));
4967
+ try {
4968
+ rmSync(modelDir, {
4969
+ recursive: true,
4970
+ force: true
4971
+ });
4972
+ } catch {}
4973
+ if (!downloadCompleted && controller.signal.aborted && !(options?.signal?.aborted ?? false)) throw new Error(`model download timed out (received ${received} of ${announcedTotal ?? "?"} bytes): ${url}`, { cause: error });
4974
+ throw error instanceof Error ? error : new Error(String(error), { cause: error });
4623
4975
  } finally {
4976
+ clearTimeout(totalTimer);
4977
+ if (idleTimer !== void 0) clearTimeout(idleTimer);
4978
+ if (options?.signal !== void 0) options.signal.removeEventListener("abort", onExternalAbort);
4624
4979
  rmSync(stagingPath, { force: true });
4625
4980
  }
4626
4981
  }
4627
- const PLATFORM_FAILURE = /@anush008\/tokenizers|Failed to load native binding|Unsupported (?:OS|platform|architecture)|ERR_DLOPEN_FAILED|invalid ELF header|Exec format error|is not a valid Win32 application|GLIBC_\d+[^"\n]*not found|Error loading shared library|cannot open shared object file/iu;
4628
- const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib/iu;
4629
- const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}/iu;
4982
+ const INSTALL_FAILURE = /Cannot find (?:package|module) '(?:onnxruntime-web(?:\/wasm)?|@huggingface\/tokenizers)'|ERR_MODULE_NOT_FOUND|Failed to resolve module specifier/iu;
4983
+ const ARCHIVE_FAILURE = /\bTAR_[A-Z_]+|incorrect header check|unexpected end of file|invalid tar|zlib|after extraction|sha256 mismatch/iu;
4984
+ const NETWORK_FAILURE = /\b(?:ENOTFOUND|EAI_AGAIN|ETIMEDOUT|ECONNRESET|ECONNREFUSED|EHOSTUNREACH|socket hang up|fetch failed)\b|network error|HTTP \d{3}|timed out|empty body/iu;
4630
4985
  /**
4631
- * Classify a load failure so callers can say something true about it: the four
4632
- * causes need four different answers (unsupported platform, corrupt cache,
4633
- * network, unknown), and they used to be reported as one network problem.
4986
+ * Classify a load failure so callers can say something true about it: an
4987
+ * incomplete install, a corrupt cache, a network problem and anything else need
4988
+ * four different answers, and they used to be reported as one network problem.
4634
4989
  */
4635
4990
  function classifyEmbeddingFailure(message) {
4636
4991
  if (message === void 0) return "other";
4637
- if (PLATFORM_FAILURE.test(message)) return "platform";
4992
+ if (INSTALL_FAILURE.test(message)) return "install";
4638
4993
  if (ARCHIVE_FAILURE.test(message)) return "archive";
4639
4994
  if (NETWORK_FAILURE.test(message)) return "network";
4640
4995
  return "other";
4641
4996
  }
4642
4997
  /**
4643
- * Loads only the module entry point — no model download — so callers can tell
4644
- * "this machine cannot run the local model at all" apart from "the download
4645
- * failed". fastembed imports its native tokenizer binding at module scope, which
4646
- * is exactly the step a platform without a published binary cannot pass.
4998
+ * Loads both runtime packages — no model download — so callers can tell "this
4999
+ * installation cannot run the local model at all" apart from "the download
5000
+ * failed". The runtime is JavaScript plus WebAssembly, so a failure here is an
5001
+ * incomplete installation rather than an unsupported platform.
4647
5002
  */
4648
5003
  async function probeLocalEmbeddingSupport() {
4649
5004
  try {
4650
- await import("fastembed");
5005
+ await import(["onnxruntime-web", "wasm"].join("/"));
5006
+ await import("@huggingface/tokenizers");
4651
5007
  return { supported: true };
4652
5008
  } catch (error) {
4653
5009
  return {
@@ -4658,15 +5014,11 @@ async function probeLocalEmbeddingSupport() {
4658
5014
  }
4659
5015
  /**
4660
5016
  * Remove every trace of a previous BGESmallZH download: the extracted model
4661
- * directory *and* the staged archive it came from.
5017
+ * directory and the legacy staged archive name.
4662
5018
  *
4663
- * The archive has to go with it. fastembed skips the download whenever
4664
- * `<cacheDir>/<name>.tar.gz` exists, and it only deletes that file after a
4665
- * *successful* extract — its download error paths (response error, file stream
4666
- * error) reject and leave whatever was written behind. So an archive sitting
4667
- * next to a failed attempt is usually a partial leftover, and keeping it makes
4668
- * every retry extract the same junk locally: no network traffic, a few
4669
- * milliseconds, the same failure forever.
5019
+ * The legacy `<name>.tar.gz` file is still swept because older versions staged
5020
+ * the archive under that name; leaving it behind would waste 55MB per doomed
5021
+ * attempt. Our own downloader stages `.tar.gz.part` and always removes it.
4670
5022
  *
4671
5023
  * Never throws: a file locked by another process (on Windows a mapped ONNX file
4672
5024
  * stays locked) must not replace the download error the caller is reporting.
@@ -4684,48 +5036,19 @@ function clearEmbeddingModelCache(cacheDir) {
4684
5036
  }
4685
5037
  }
4686
5038
  /**
4687
- * True when the extracted model weights are already on disk. Loading then
4688
- * never downloads (fastembed skips the fetch whenever the model dir exists),
4689
- * so callers can safely trigger a local load without risking a surprise
4690
- * mid-query download.
5039
+ * True when every file the runtime needs is already on disk. Loading then never
5040
+ * downloads, so callers can safely trigger a local load without risking a
5041
+ * surprise mid-query download. All required files are checked (not just the
5042
+ * weights): a partially restored cache must count as missing so the setup flow
5043
+ * repairs it instead of failing at load time.
4691
5044
  */
4692
5045
  function hasEmbeddingModelCache(cacheDir) {
4693
- const modelDir = join(cacheDir, BGESMALLZH_CACHE_NAME);
4694
- return existsSync(modelDir) && existsSync(join(modelDir, "model_optimized.onnx"));
4695
- }
4696
- /**
4697
- * Small config/tokenizer files that fastembed expects alongside model.onnx.
4698
- * If these are missing (e.g. GCS download partially failed), fastembed throws.
4699
- * We download them from HuggingFace so the model can load — this covers the
4700
- * case where the GCS tarball was incomplete but the HF repo is reachable.
4701
- */
4702
- const FASTEMBED_SIDECARS = [
4703
- "config.json",
4704
- "tokenizer.json",
4705
- "tokenizer_config.json",
4706
- "special_tokens_map.json"
4707
- ];
4708
- const FASTEMBED_HF_REPOS = { "fast-bge-small-zh-v1.5": "BAAI/bge-small-zh-v1.5" };
4709
- async function ensureFastembedModelSidecars(model, cacheDir) {
4710
- const repo = FASTEMBED_HF_REPOS[model];
4711
- if (repo === void 0) return;
4712
- const modelDir = join(cacheDir ?? "local_cache", model);
4713
- mkdirSync(modelDir, { recursive: true });
4714
- for (const fileName of FASTEMBED_SIDECARS) {
5046
+ const modelDir = modelDirPath(cacheDir);
5047
+ if (!existsSync(modelDir)) return false;
5048
+ return EMBEDDING_REQUIRED_FILES.every((fileName) => {
4715
5049
  const target = join(modelDir, fileName);
4716
- try {
4717
- const { access } = await import("node:fs/promises");
4718
- await access(target);
4719
- continue;
4720
- } catch {}
4721
- const hfUrl = `https://huggingface.co/${repo}/resolve/main/${fileName}`;
4722
- try {
4723
- const response = await fetch(hfUrl);
4724
- if (!response.ok) continue;
4725
- const { writeFile } = await import("node:fs/promises");
4726
- await writeFile(target, Buffer.from(await response.arrayBuffer()));
4727
- } catch {}
4728
- }
5050
+ return existsSync(target) && statSync(target).size > 0;
5051
+ });
4729
5052
  }
4730
5053
  //#endregion
4731
5054
  //#region ../../packages/memory/src/store.ts
@@ -4869,7 +5192,19 @@ var MemoryMemoStore = class MemoryMemoStore {
4869
5192
  JOIN memos_fts f ON m.rowid = f.rowid
4870
5193
  WHERE f.memos_fts MATCH ? AND (m.project_dir = ? OR m.project_dir = '')
4871
5194
  ORDER BY m.recorded_at DESC LIMIT ?`);
4872
- return (projectDir === void 0 ? stmt.all(ftsQuery, limit) : stmt.all(ftsQuery, projectDir, limit)).map(rowToMemo);
5195
+ const rows = projectDir === void 0 ? stmt.all(ftsQuery, limit) : stmt.all(ftsQuery, projectDir, limit);
5196
+ const like = likePattern(query);
5197
+ const remaining = Math.max(0, limit - rows.length);
5198
+ const extraRows = remaining === 0 ? [] : projectDir === void 0 ? this.db.prepare("SELECT * FROM memos WHERE distilled LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\' ORDER BY recorded_at DESC LIMIT ?").all(like, like, remaining) : this.db.prepare("SELECT * FROM memos WHERE (distilled LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\') AND (project_dir = ? OR project_dir = '') ORDER BY recorded_at DESC LIMIT ?").all(like, like, projectDir, remaining);
5199
+ const seen = new Set(rows.map((row) => String(row["id"])));
5200
+ const merged = rows.map(rowToMemo);
5201
+ for (const row of extraRows) {
5202
+ const id = String(row["id"]);
5203
+ if (seen.has(id)) continue;
5204
+ seen.add(id);
5205
+ merged.push(rowToMemo(row));
5206
+ }
5207
+ return merged;
4873
5208
  }
4874
5209
  /** List memos with optional full-text search and pagination. */
4875
5210
  async list(options) {
@@ -5011,7 +5346,12 @@ var MemoryMemoStore = class MemoryMemoStore {
5011
5346
  extraction_source TEXT NOT NULL CHECK(extraction_source IN ('compaction', 'exit', 'manual')),
5012
5347
  recorded_at INTEGER NOT NULL,
5013
5348
  project_dir TEXT NOT NULL DEFAULT '',
5014
- tags TEXT NOT NULL DEFAULT '[]'
5349
+ tags TEXT NOT NULL DEFAULT '[]',
5350
+ note TEXT,
5351
+ distilled TEXT NOT NULL DEFAULT '',
5352
+ origin TEXT NOT NULL DEFAULT 'session',
5353
+ distilled_from TEXT,
5354
+ distilled_at INTEGER
5015
5355
  );
5016
5356
 
5017
5357
  CREATE INDEX IF NOT EXISTS idx_memos_project_dir ON memos(project_dir);
@@ -5041,6 +5381,14 @@ var MemoryMemoStore = class MemoryMemoStore {
5041
5381
  const info = this.db.prepare("PRAGMA table_info(memos)").all();
5042
5382
  if (!info.some((col) => col.name === "project_dir")) this.db.exec("ALTER TABLE memos ADD COLUMN project_dir TEXT NOT NULL DEFAULT ''");
5043
5383
  if (!info.some((col) => col.name === "tags")) this.db.exec("ALTER TABLE memos ADD COLUMN tags TEXT NOT NULL DEFAULT '[]'");
5384
+ if (!info.some((col) => col.name === "note")) this.db.exec("ALTER TABLE memos ADD COLUMN note TEXT");
5385
+ if (!info.some((col) => col.name === "distilled")) this.db.exec("ALTER TABLE memos ADD COLUMN distilled TEXT NOT NULL DEFAULT ''");
5386
+ if (!info.some((col) => col.name === "origin")) {
5387
+ this.db.exec("ALTER TABLE memos ADD COLUMN origin TEXT NOT NULL DEFAULT 'session'");
5388
+ this.db.exec("UPDATE memos SET origin = 'dream' WHERE distilled <> ''");
5389
+ }
5390
+ if (!info.some((col) => col.name === "distilled_from")) this.db.exec("ALTER TABLE memos ADD COLUMN distilled_from TEXT");
5391
+ if (!info.some((col) => col.name === "distilled_at")) this.db.exec("ALTER TABLE memos ADD COLUMN distilled_at INTEGER");
5044
5392
  this.db.exec(`
5045
5393
  CREATE INDEX IF NOT EXISTS idx_memos_project_dir ON memos(project_dir);
5046
5394
  CREATE INDEX IF NOT EXISTS idx_memory_embeddings_created_at ON memory_embeddings(created_at DESC);
@@ -5084,15 +5432,16 @@ var MemoryMemoStore = class MemoryMemoStore {
5084
5432
  if (this.db === void 0 || memos.length === 0) return;
5085
5433
  const insert = this.db.prepare(`INSERT OR IGNORE INTO memos (
5086
5434
  id, source_session_id, source_session_title, user_need, approach,
5087
- outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags
5088
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5435
+ outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags,
5436
+ note, distilled, origin, distilled_from, distilled_at
5437
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5089
5438
  RETURNING rowid`);
5090
5439
  const insertFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
5091
5440
  VALUES (?, ?, ?, ?, ?, ?)`);
5092
5441
  this.db.exec("BEGIN TRANSACTION");
5093
5442
  try {
5094
5443
  for (const memo of memos) {
5095
- const row = insert.get(memo.id, memo.sourceSessionId, memo.sourceSessionTitle ?? null, memo.userNeed, memo.approach, memo.outcome, memo.whatFailed, memo.whatWorked, memo.extractionSource, memo.recordedAt, memo.projectDir ?? "", JSON.stringify(memo.tags ?? []));
5444
+ const row = insert.get(memo.id, memo.sourceSessionId, memo.sourceSessionTitle ?? null, memo.userNeed, memo.approach, memo.outcome, memo.whatFailed, memo.whatWorked, memo.extractionSource, memo.recordedAt, memo.projectDir ?? "", JSON.stringify(memo.tags ?? []), memo.note ?? null, memo.distilled ?? "", memo.origin ?? "session", stringifyDistilledFrom(memo.distilledFrom), memo.distilledAt ?? null);
5096
5445
  if (row === void 0) continue;
5097
5446
  insertFts.run(row.rowid, toFtsText(memo.userNeed), toFtsText(memo.approach), toFtsText(memo.whatFailed), toFtsText(memo.whatWorked), toFtsText(memo.sourceSessionTitle ?? ""));
5098
5447
  }
@@ -5107,14 +5456,15 @@ var MemoryMemoStore = class MemoryMemoStore {
5107
5456
  if (this.db === void 0) return;
5108
5457
  const insert = this.db.prepare(`INSERT INTO memos (
5109
5458
  id, source_session_id, source_session_title, user_need, approach,
5110
- outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags
5111
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5459
+ outcome, what_failed, what_worked, extraction_source, recorded_at, project_dir, tags,
5460
+ note, distilled, origin, distilled_from, distilled_at
5461
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
5112
5462
  RETURNING rowid`);
5113
5463
  const insertFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
5114
5464
  VALUES (?, ?, ?, ?, ?, ?)`);
5115
5465
  this.db.exec("BEGIN TRANSACTION");
5116
5466
  try {
5117
- const row = insert.get(entry.id, entry.sourceSessionId, entry.sourceSessionTitle ?? null, entry.userNeed, entry.approach, entry.outcome, entry.whatFailed, entry.whatWorked, entry.extractionSource, entry.recordedAt, entry.projectDir ?? "", JSON.stringify(entry.tags ?? []));
5467
+ const row = insert.get(entry.id, entry.sourceSessionId, entry.sourceSessionTitle ?? null, entry.userNeed, entry.approach, entry.outcome, entry.whatFailed, entry.whatWorked, entry.extractionSource, entry.recordedAt, entry.projectDir ?? "", JSON.stringify(entry.tags ?? []), entry.note ?? null, entry.distilled ?? "", entry.origin ?? "session", stringifyDistilledFrom(entry.distilledFrom), entry.distilledAt ?? null);
5118
5468
  insertFts.run(row.rowid, toFtsText(entry.userNeed), toFtsText(entry.approach), toFtsText(entry.whatFailed), toFtsText(entry.whatWorked), toFtsText(entry.sourceSessionTitle ?? ""));
5119
5469
  this.db.exec("COMMIT");
5120
5470
  this.scheduleEmbedding(entry);
@@ -5150,7 +5500,12 @@ var MemoryMemoStore = class MemoryMemoStore {
5150
5500
  extraction_source = ?,
5151
5501
  recorded_at = ?,
5152
5502
  project_dir = ?,
5153
- tags = ?
5503
+ tags = ?,
5504
+ note = ?,
5505
+ distilled = ?,
5506
+ origin = ?,
5507
+ distilled_from = ?,
5508
+ distilled_at = ?
5154
5509
  WHERE id = ?
5155
5510
  RETURNING rowid`);
5156
5511
  const updateFts = this.db.prepare(`INSERT INTO memos_fts(rowid, user_need, approach, what_failed, what_worked, source_session_title)
@@ -5164,7 +5519,7 @@ var MemoryMemoStore = class MemoryMemoStore {
5164
5519
  this.db.exec("ROLLBACK");
5165
5520
  return false;
5166
5521
  }
5167
- const row = update.get(updated.sourceSessionId, updated.sourceSessionTitle ?? null, updated.userNeed, updated.approach, updated.outcome, updated.whatFailed, updated.whatWorked, updated.extractionSource, updated.recordedAt, updated.projectDir ?? "", JSON.stringify(updated.tags ?? []), id);
5522
+ const row = update.get(updated.sourceSessionId, updated.sourceSessionTitle ?? null, updated.userNeed, updated.approach, updated.outcome, updated.whatFailed, updated.whatWorked, updated.extractionSource, updated.recordedAt, updated.projectDir ?? "", JSON.stringify(updated.tags ?? []), updated.note ?? null, updated.distilled ?? "", updated.origin ?? "session", stringifyDistilledFrom(updated.distilledFrom), updated.distilledAt ?? null, id);
5168
5523
  if (row === void 0) {
5169
5524
  this.db.exec("ROLLBACK");
5170
5525
  return false;
@@ -5283,6 +5638,36 @@ var MemoryMemoStore = class MemoryMemoStore {
5283
5638
  */
5284
5639
  scheduleEmbedding(memo) {
5285
5640
  this.embeddingQueue.add(memo.id);
5641
+ this.armEmbeddingFlush();
5642
+ }
5643
+ /**
5644
+ * Queue every memo that has no vector yet (newest first, at most `limit`) and
5645
+ * arm a flush. Memos written while embeddings could not be stored — or before
5646
+ * an engine existed — would otherwise never get one: `scheduleEmbedding` only
5647
+ * runs for a write. Safe to call repeatedly; already-embedded memos are
5648
+ * skipped by the SQL.
5649
+ *
5650
+ * Loading the engine can download the model, which must stay the user's
5651
+ * decision, so an engine that is not loaded yet is left alone unless the
5652
+ * caller passes `loadEngine` (it knows the model is already on disk).
5653
+ */
5654
+ async backfillMissingEmbeddings(options = {}) {
5655
+ const engine = this.embeddingEngine;
5656
+ if (engine === void 0) return 0;
5657
+ if (!engine.available && options.loadEngine !== true) return 0;
5658
+ await this.init();
5659
+ if (this.db === void 0) return 0;
5660
+ const limit = Math.max(0, Math.floor(options.limit ?? 500));
5661
+ const rows = this.db.prepare(`SELECT m.id AS id FROM memos m
5662
+ WHERE NOT EXISTS (SELECT 1 FROM memory_embeddings e WHERE e.memory_id = m.id)
5663
+ ORDER BY m.recorded_at DESC
5664
+ LIMIT ?`).all(limit);
5665
+ for (const row of rows) this.embeddingQueue.add(row.id);
5666
+ if (rows.length > 0) this.armEmbeddingFlush();
5667
+ return rows.length;
5668
+ }
5669
+ /** Debounced flush of the queue, loading the engine first when it is not ready yet. */
5670
+ armEmbeddingFlush() {
5286
5671
  if (this.embeddingEngine === void 0) return;
5287
5672
  if (!this.embeddingEngine.available) {
5288
5673
  this.embeddingEngine.ensureReady().then((ready) => {
@@ -5307,46 +5692,66 @@ var MemoryMemoStore = class MemoryMemoStore {
5307
5692
  this.embeddingFlushing = true;
5308
5693
  const ids = [...this.embeddingQueue];
5309
5694
  this.embeddingQueue.clear();
5310
- const reenqueue = () => {
5311
- for (const id of ids) this.embeddingQueue.add(id);
5695
+ let owed = ids;
5696
+ let failed = false;
5697
+ const requeueOwed = () => {
5698
+ for (const id of owed) this.embeddingQueue.add(id);
5312
5699
  };
5313
5700
  try {
5314
5701
  await this.init();
5315
5702
  if (this.db === void 0) return;
5316
5703
  const pending = [];
5317
5704
  for (const id of ids) {
5318
- if (this.db.prepare("SELECT id FROM memory_embeddings WHERE memory_id = ?").get(id) !== void 0) continue;
5705
+ if (this.db.prepare("SELECT 1 FROM memory_embeddings WHERE memory_id = ?").get(id) !== void 0) continue;
5319
5706
  const memo = await this.get(id);
5320
5707
  if (memo !== void 0) pending.push({
5321
5708
  id,
5322
5709
  text: buildEmbeddingText(memo)
5323
5710
  });
5324
5711
  }
5325
- if (pending.length === 0) return;
5326
- const vectors = await this.tryEmbedBatch(pending.map((p) => p.text));
5327
- if (vectors === null || vectors.length !== pending.length) {
5328
- this.markEmbeddingFailure(/* @__PURE__ */ new Error(vectors === null ? "embedBatch returned null" : "embedding count mismatch"));
5329
- reenqueue();
5330
- return;
5331
- }
5332
- this.clearEmbeddingFailure();
5333
- const insert = this.db.prepare("INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding_json, model, created_at) VALUES (?, ?, ?, ?)");
5334
- const now = Date.now();
5335
- this.db.exec("BEGIN TRANSACTION");
5336
- try {
5337
- for (let i = 0; i < pending.length; i++) insert.run(pending[i].id, JSON.stringify([...vectors[i]]), EMBEDDING_MODEL_NAME, now);
5338
- this.db.exec("COMMIT");
5339
- } catch (error) {
5340
- this.db.exec("ROLLBACK");
5341
- this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
5342
- reenqueue();
5712
+ owed = pending.map((entry) => entry.id);
5713
+ for (let start = 0; start < pending.length; start += 1) {
5714
+ const db = this.db;
5715
+ if (db === void 0) {
5716
+ failed = true;
5717
+ requeueOwed();
5718
+ return;
5719
+ }
5720
+ const chunk = pending.slice(start, start + 1);
5721
+ const vectors = await this.tryEmbedBatch(chunk.map((entry) => entry.text));
5722
+ if (vectors === null || vectors.length !== chunk.length) {
5723
+ this.markEmbeddingFailure(/* @__PURE__ */ new Error(vectors === null ? "embedBatch returned null" : "embedding count mismatch"));
5724
+ failed = true;
5725
+ requeueOwed();
5726
+ return;
5727
+ }
5728
+ this.clearEmbeddingFailure();
5729
+ const insert = db.prepare("INSERT OR REPLACE INTO memory_embeddings (memory_id, embedding_json, model, created_at) VALUES (?, ?, ?, ?)");
5730
+ const now = Date.now();
5731
+ db.exec("BEGIN TRANSACTION");
5732
+ try {
5733
+ for (const [index, entry] of chunk.entries()) insert.run(entry.id, JSON.stringify([...vectors[index]]), EMBEDDING_MODEL_NAME, now);
5734
+ db.exec("COMMIT");
5735
+ } catch (error) {
5736
+ db.exec("ROLLBACK");
5737
+ this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
5738
+ failed = true;
5739
+ requeueOwed();
5740
+ return;
5741
+ }
5742
+ owed = pending.slice(start + chunk.length).map((entry) => entry.id);
5743
+ if (owed.length > 0) await new Promise((resolve) => {
5744
+ setImmediate(resolve);
5745
+ });
5343
5746
  }
5344
5747
  } catch (error) {
5345
5748
  this.markEmbeddingFailure(error instanceof Error ? error : new Error(String(error)));
5346
- reenqueue();
5749
+ failed = true;
5750
+ requeueOwed();
5347
5751
  } finally {
5348
5752
  this.embeddingFlushing = false;
5349
5753
  this.embeddingTimer = void 0;
5754
+ if (!failed && this.embeddingQueue.size > 0) this.armEmbeddingFlush();
5350
5755
  }
5351
5756
  }
5352
5757
  async tryEmbedBatch(texts) {
@@ -5412,6 +5817,9 @@ var MemoryMemoStore = class MemoryMemoStore {
5412
5817
  function rowToMemo(row) {
5413
5818
  const sourceSessionTitle = row["source_session_title"];
5414
5819
  const projectDir = row["project_dir"];
5820
+ const distilled = row["distilled"];
5821
+ const note = row["note"];
5822
+ const distilledAt = row["distilled_at"];
5415
5823
  return {
5416
5824
  id: String(row["id"]),
5417
5825
  sourceSessionId: String(row["source_session_id"]),
@@ -5421,12 +5829,36 @@ function rowToMemo(row) {
5421
5829
  outcome: String(row["outcome"]),
5422
5830
  whatFailed: String(row["what_failed"]),
5423
5831
  whatWorked: String(row["what_worked"]),
5832
+ distilled: typeof distilled === "string" && distilled.trim().length > 0 ? distilled : void 0,
5833
+ origin: parseOrigin(row["origin"]),
5834
+ distilledFrom: parseDistilledFrom(row["distilled_from"]),
5835
+ distilledAt: typeof distilledAt === "number" && Number.isFinite(distilledAt) ? distilledAt : void 0,
5424
5836
  extractionSource: row["extraction_source"],
5425
5837
  recordedAt: Number(row["recorded_at"]),
5426
5838
  projectDir: typeof projectDir === "string" ? projectDir : "",
5427
- tags: parseTags(row["tags"])
5839
+ tags: parseTags(row["tags"]),
5840
+ note: typeof note === "string" && note.length > 0 ? note : void 0
5428
5841
  };
5429
5842
  }
5843
+ function parseOrigin(value) {
5844
+ return value === "dream" || value === "user" || value === "session" ? value : "session";
5845
+ }
5846
+ /** Column value for `distilled_from`: undefined when there is nothing to store. */
5847
+ function stringifyDistilledFrom(sources) {
5848
+ return sources === void 0 || sources.length === 0 ? null : JSON.stringify(sources);
5849
+ }
5850
+ /** Parse the `distilled_from` JSON column, dropping entries that fail validation. */
5851
+ function parseDistilledFrom(value) {
5852
+ if (typeof value !== "string" || value.length === 0) return void 0;
5853
+ try {
5854
+ const parsed = JSON.parse(value);
5855
+ if (!Array.isArray(parsed)) return void 0;
5856
+ const sources = parsed.filter((item) => typeof item === "object" && item !== null && typeof item.id === "string" && typeof item.title === "string" && typeof item.at === "number" && Number.isFinite(item.at));
5857
+ return sources.length > 0 ? sources.slice(0, 100) : void 0;
5858
+ } catch {
5859
+ return;
5860
+ }
5861
+ }
5430
5862
  function parseTags(value) {
5431
5863
  if (typeof value !== "string" || value.length === 0) return void 0;
5432
5864
  try {
@@ -5438,12 +5870,17 @@ function parseTags(value) {
5438
5870
  return;
5439
5871
  }
5440
5872
  }
5873
+ function likePattern(value) {
5874
+ return `%${value.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_")}%`;
5875
+ }
5441
5876
  function memoMatchesSearch(memo, search) {
5442
5877
  return [
5443
5878
  memo.userNeed,
5444
5879
  memo.approach,
5445
5880
  memo.whatFailed,
5446
5881
  memo.whatWorked,
5882
+ memo.distilled ?? "",
5883
+ memo.note ?? "",
5447
5884
  memo.sourceSessionTitle ?? "",
5448
5885
  ...memo.tags ?? []
5449
5886
  ].join(" ").toLowerCase().replaceAll(/\s+/g, "").includes(search.replaceAll(/\s+/g, ""));
@@ -5617,6 +6054,7 @@ async function buildConsolidationPlan(store, options) {
5617
6054
  async function applyConsolidation(store, plan) {
5618
6055
  let deleted = 0;
5619
6056
  let created = 0;
6057
+ let skipped = 0;
5620
6058
  if (plan.resolved.length > 0) {
5621
6059
  const worked = plan.resolved.filter((m) => m.whatWorked).map((m) => m.whatWorked).join("; ");
5622
6060
  const failed = plan.resolved.filter((m) => m.whatFailed).map((m) => m.whatFailed).join("; ");
@@ -5655,9 +6093,22 @@ async function applyConsolidation(store, plan) {
5655
6093
  created++;
5656
6094
  }
5657
6095
  }
6096
+ const mergeableGroups = [];
5658
6097
  for (const group of plan.duplicateGroups) {
6098
+ const distilledText = (group.merged.distilled ?? "").trim();
6099
+ if (distilledText.length === 0 && group.memos.some(isDistilledRule)) {
6100
+ skipped++;
6101
+ continue;
6102
+ }
6103
+ mergeableGroups.push({
6104
+ group,
6105
+ distilledText
6106
+ });
6107
+ }
6108
+ for (const { group, distilledText } of mergeableGroups) {
5659
6109
  const newest = group.memos.reduce((a, b) => a.recordedAt > b.recordedAt ? a : b);
5660
6110
  const mergedTags = normalizeTags(group.memos.flatMap((m) => m.tags ?? []));
6111
+ const mergedNotes = [...new Set(group.memos.map((memo) => memo.note?.trim() ?? "").filter((note) => note.length > 0))];
5661
6112
  const merged = createMemoryMemo({
5662
6113
  sourceSessionId: newest.sourceSessionId,
5663
6114
  sourceSessionTitle: newest.sourceSessionTitle,
@@ -5666,14 +6117,19 @@ async function applyConsolidation(store, plan) {
5666
6117
  outcome: group.merged.outcome,
5667
6118
  whatFailed: group.merged.whatFailed,
5668
6119
  whatWorked: group.merged.whatWorked,
6120
+ note: mergedNotes.length > 0 ? mergedNotes.join("; ") : void 0,
5669
6121
  tags: group.merged.tags ?? mergedTags,
6122
+ distilled: distilledText.length > 0 ? distilledText : void 0,
5670
6123
  projectDir: newest.projectDir,
5671
- extractionSource: "compaction"
6124
+ extractionSource: "compaction",
6125
+ origin: "dream",
6126
+ distilledAt: distilledText.length > 0 ? Date.now() : void 0,
6127
+ distilledFrom: collectLineage(group.memos)
5672
6128
  });
5673
6129
  await store.append(merged);
5674
6130
  created++;
5675
6131
  }
5676
- for (const group of plan.duplicateGroups) for (const memo of group.memos) {
6132
+ for (const { group } of mergeableGroups) for (const memo of group.memos) {
5677
6133
  await store.delete(memo.id);
5678
6134
  deleted++;
5679
6135
  }
@@ -5687,9 +6143,31 @@ async function applyConsolidation(store, plan) {
5687
6143
  }
5688
6144
  return {
5689
6145
  deleted,
5690
- created
6146
+ created,
6147
+ skipped
5691
6148
  };
5692
6149
  }
6150
+ /**
6151
+ * Build the lineage snapshot for a merged memo: every source memo in the group
6152
+ * plus each source's own lineage, deduped by id so a rule distilled from other
6153
+ * rules keeps its whole ancestry. Sorted oldest-first and capped at 100.
6154
+ */
6155
+ function collectLineage(memos) {
6156
+ const seen = /* @__PURE__ */ new Map();
6157
+ for (const memo of memos) {
6158
+ if (!seen.has(memo.id)) seen.set(memo.id, {
6159
+ id: memo.id,
6160
+ title: clip(memo.userNeed, 80),
6161
+ at: memo.recordedAt
6162
+ });
6163
+ for (const source of memo.distilledFrom ?? []) if (!seen.has(source.id)) seen.set(source.id, source);
6164
+ }
6165
+ return [...seen.values()].toSorted((a, b) => a.at - b.at).slice(-100);
6166
+ }
6167
+ function clip(text, maxLength) {
6168
+ const chars = Array.from(text);
6169
+ return chars.length <= maxLength ? text : chars.slice(0, maxLength).join("");
6170
+ }
5693
6171
  function findDuplicateGroups(memos) {
5694
6172
  const groups = [];
5695
6173
  const used = /* @__PURE__ */ new Set();
@@ -5768,7 +6246,8 @@ function buildDuplicateGroup(cluster) {
5768
6246
  approach: `合并 ${cluster.length} 条相关记录。最新方案: ${newest.approach}`,
5769
6247
  outcome: bestOutcome,
5770
6248
  whatFailed: failures.size > 0 ? [...failures].join("; ") : "none",
5771
- whatWorked: successes.size > 0 ? [...successes].join("; ") : "none"
6249
+ whatWorked: successes.size > 0 ? [...successes].join("; ") : "none",
6250
+ distilled: ""
5772
6251
  },
5773
6252
  reason: `发现 ${cluster.length} 条相似记录(关键词重叠 > ${Math.round(SIMILARITY_THRESHOLD * 100)}%)`
5774
6253
  };
@@ -5831,12 +6310,19 @@ function isOutcomeCompleted(outcome) {
5831
6310
  const lower = outcome.toLowerCase();
5832
6311
  return lower.includes("完成") || lower.includes("done") || lower.includes("completed") || lower.includes("成功") || lower.includes("success");
5833
6312
  }
6313
+ /**
6314
+ * A distilled rule is not a stale experience: pruning it by age would delete
6315
+ * the conclusion /dream exists to accumulate.
6316
+ */
6317
+ function isDistilledRule(memo) {
6318
+ return memo.origin === "dream" || memo.origin === "user" || (memo.distilled ?? "").trim().length > 0;
6319
+ }
5834
6320
  function findResolved(memos) {
5835
- return memos.filter((m) => isOutcomeCompleted(m.outcome) && Date.now() - m.recordedAt > 10080 * 60 * 1e3);
6321
+ return memos.filter((m) => !isDistilledRule(m) && isOutcomeCompleted(m.outcome) && Date.now() - m.recordedAt > 10080 * 60 * 1e3);
5836
6322
  }
5837
6323
  function findStale(memos, staleDays) {
5838
6324
  const threshold = Date.now() - staleDays * 24 * 60 * 60 * 1e3;
5839
- return memos.filter((m) => m.recordedAt < threshold && !isOutcomeCompleted(m.outcome) && !m.outcome.includes("blocked"));
6325
+ return memos.filter((m) => !isDistilledRule(m) && m.recordedAt < threshold && !isOutcomeCompleted(m.outcome) && !m.outcome.includes("blocked"));
5840
6326
  }
5841
6327
  //#endregion
5842
6328
  //#region ../../packages/memory/src/dream.ts
@@ -5918,10 +6404,16 @@ var DreamTracker = class {
5918
6404
  shouldSuggest() {
5919
6405
  return (Date.now() - new Date(this.state.lastDreamAt).getTime()) / (1e3 * 60 * 60) >= MIN_HOURS_BETWEEN_DREAMS && this.state.sessionsSinceLastDream >= MIN_SESSIONS_BETWEEN_DREAMS;
5920
6406
  }
5921
- /** Get a human-readable suggestion message when conditions are met. */
6407
+ /**
6408
+ * The suggestion injected as a system reminder when conditions are met. It is
6409
+ * read by the model, which relays it to the user in the user's language, so it
6410
+ * is English like every other reminder the engine injects.
6411
+ */
5922
6412
  getSuggestionMessage() {
5923
6413
  const hoursSince = (Date.now() - new Date(this.state.lastDreamAt).getTime()) / (1e3 * 60 * 60);
5924
- return `距离上次记忆整理已过去 ${Math.floor(hoursSince / 24)} 天、${this.state.sessionsSinceLastDream} 个会话。建议运行 /dream 来合并重复记忆、清理过期条目、解决矛盾信息。`;
6414
+ const days = Math.floor(hoursSince / 24);
6415
+ const sessions = this.state.sessionsSinceLastDream;
6416
+ return `It has been ${days} ${days === 1 ? "day" : "days"} and ${sessions} ${sessions === 1 ? "session" : "sessions"} since the last memory consolidation. Consider suggesting that the user run /dream to merge duplicate memories, clear stale entries and resolve contradictions.`;
5925
6417
  }
5926
6418
  async persist() {
5927
6419
  const data = {
@@ -5981,7 +6473,7 @@ function entryFor(homeDir) {
5981
6473
  await store.init();
5982
6474
  const cacheDir = join$1(homeDir, "cache", "fastembed");
5983
6475
  mkdirSync(cacheDir, { recursive: true });
5984
- store.setEmbeddingEngine(createFastEmbedEngine(cacheDir));
6476
+ store.setEmbeddingEngine(createEmbeddingEngine(cacheDir));
5985
6477
  } catch (error) {
5986
6478
  entries.delete(homeDir);
5987
6479
  store.close();
@@ -5999,7 +6491,7 @@ function entryFor(homeDir) {
5999
6491
  *
6000
6492
  * The map key is the raw homeDir string so the engine cache key
6001
6493
  * join(homeDir, 'cache', 'fastembed') stays byte-identical with every other
6002
- * createFastEmbedEngine call site: same string → same engine instance.
6494
+ * createEmbeddingEngine call site: same string → same engine instance.
6003
6495
  */
6004
6496
  function sharedKnowledgeStore(homeDir) {
6005
6497
  return entryFor(homeDir).store;
@@ -6013,12 +6505,13 @@ function sharedKnowledgeStoreReady(homeDir) {
6013
6505
  const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
6014
6506
  const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
6015
6507
  /**
6016
- * Soft cap on chunk token count. BGE-small-zh-v1.5 (the fastembed model used
6017
- * here) has a 512-token sequence limit; chunks longer than that get truncated
6508
+ * Soft cap on chunk token count. BGE-small-zh-v1.5 (the WASM model used here)
6509
+ * has a 512-token sequence limit; chunks longer than that get truncated
6018
6510
  * at the embedding layer, hurting retrieval quality. We split at ~480 to leave
6019
6511
  * headroom for the heading line that gets prepended at embed time.
6020
6512
  */
6021
6513
  const MAX_CHUNK_TOKENS = 480;
6514
+ const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
6022
6515
  /**
6023
6516
  * Split a markdown document into sections using heading_strict strategy:
6024
6517
  * each section begins at a heading and contains all body content until the
@@ -6094,55 +6587,33 @@ function stripMarkdown(text) {
6094
6587
  * embedding model's token cap (same MAX_CHUNK_TOKENS as markdown).
6095
6588
  */
6096
6589
  function chunkText(content) {
6097
- const MAX_PARA_TOKENS = MAX_CHUNK_TOKENS;
6098
- const MAX_PARA_CHARS = MAX_PARA_TOKENS * 4;
6099
6590
  const paragraphs = content.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
6100
6591
  const sections = [];
6101
6592
  let rank = 0;
6102
6593
  for (const para of paragraphs) {
6103
- if (estimateTokens(para) <= MAX_PARA_TOKENS) {
6104
- sections.push({
6105
- heading: null,
6106
- headingLevel: null,
6107
- content: para,
6108
- rawContent: para,
6109
- rank: rank++
6110
- });
6111
- continue;
6112
- }
6113
- const sentences = para.match(/[^.!?。!?\n]+[.!?。!?\n]+|[^.!?。!?\n]+$/g) ?? [para];
6114
- let buffer = "";
6115
- const flushBuffer = () => {
6116
- const text = buffer.trim();
6117
- if (text.length === 0) return;
6118
- sections.push({
6119
- heading: null,
6120
- headingLevel: null,
6121
- content: text,
6122
- rawContent: text,
6123
- rank: rank++
6124
- });
6125
- buffer = "";
6126
- };
6127
- for (const sentence of sentences) {
6128
- const candidate = (buffer + " " + sentence).trim();
6129
- if (buffer.length > 0 && candidate.length > MAX_PARA_CHARS) flushBuffer();
6130
- buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
6131
- }
6132
- flushBuffer();
6594
+ const parts = estimateTokens(para) <= MAX_CHUNK_TOKENS ? [para] : splitParagraph(para);
6595
+ for (const text of parts) sections.push({
6596
+ heading: null,
6597
+ headingLevel: null,
6598
+ content: text,
6599
+ rawContent: text,
6600
+ rank: rank++
6601
+ });
6133
6602
  }
6134
6603
  return sections;
6135
6604
  }
6136
- /** Rough token estimate — chars / 4 (English-leaning; CJK underestimates). */
6605
+ /** Approximate one token per CJK character and one per four other characters. */
6137
6606
  function estimateTokens(text) {
6138
- return Math.ceil(text.length / 4);
6607
+ let quarterTokens = 0;
6608
+ for (const char of text) quarterTokens += CJK_CHAR.test(char) ? 4 : char.length;
6609
+ return Math.ceil(quarterTokens / 4);
6139
6610
  }
6140
6611
  /**
6141
6612
  * Split a section that exceeds MAX_CHUNK_TOKENS into smaller sub-sections.
6142
6613
  * Strategy (mirrors SAG's `splitLargeSection`):
6143
6614
  * 1. Split into paragraphs by blank lines.
6144
6615
  * 2. Greedily pack paragraphs into chunks under the token cap.
6145
- * 3. A paragraph that alone exceeds the cap is split at sentence boundaries;
6616
+ * 3. A paragraph that alone exceeds the cap is split at line or sentence boundaries;
6146
6617
  * a sentence that itself exceeds the cap is split by character limit.
6147
6618
  *
6148
6619
  * Sections under the cap are returned unchanged (single-element array).
@@ -6152,7 +6623,7 @@ function estimateTokens(text) {
6152
6623
  function splitLargeSection(section) {
6153
6624
  if (estimateTokens(section.content) <= MAX_CHUNK_TOKENS) return [section];
6154
6625
  const paragraphs = section.rawContent.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 0);
6155
- if (paragraphs.length <= 1) return splitBySentences(section, section.rawContent);
6626
+ if (paragraphs.length <= 1) return splitSectionParagraph(section, section.rawContent);
6156
6627
  const result = [];
6157
6628
  let buffer = [];
6158
6629
  let bufferTokens = 0;
@@ -6178,7 +6649,7 @@ function splitLargeSection(section) {
6178
6649
  const paraTokens = estimateTokens(para);
6179
6650
  if (paraTokens > MAX_CHUNK_TOKENS) {
6180
6651
  flushBuffer();
6181
- for (const frag of splitBySentences(section, para)) result.push(frag);
6652
+ for (const frag of splitSectionParagraph(section, para)) result.push(frag);
6182
6653
  continue;
6183
6654
  }
6184
6655
  if (buffer.length > 0 && bufferTokens + paraTokens > MAX_CHUNK_TOKENS) flushBuffer();
@@ -6188,73 +6659,58 @@ function splitLargeSection(section) {
6188
6659
  flushBuffer();
6189
6660
  return result.length > 0 ? result : [section];
6190
6661
  }
6191
- /**
6192
- * Split a long text into sub-sections at sentence boundaries.
6193
- * Falls back to a hard character cut when a single sentence exceeds the cap.
6194
- */
6195
- function splitBySentences(section, text) {
6196
- const sentences = splitSentences(text);
6662
+ function splitSectionParagraph(section, text) {
6663
+ return splitParagraph(text).map((raw) => ({
6664
+ ...section,
6665
+ content: stripMarkdown(raw),
6666
+ rawContent: raw,
6667
+ rank: 0
6668
+ }));
6669
+ }
6670
+ /** Keep lines intact where possible, including Markdown table rows. */
6671
+ function splitParagraph(text) {
6672
+ const segments = (text.match(/[^\n]*\n|[^\n]+$/g) ?? [text]).flatMap((line) => estimateTokens(line) <= MAX_CHUNK_TOKENS ? [line] : line.match(/[^。!?!?.]*[。!?!?.]+|[^。!?!?.]+$/g) ?? [line]);
6197
6673
  const result = [];
6198
6674
  let buffer = "";
6199
6675
  let bufferTokens = 0;
6200
6676
  const flush = () => {
6201
6677
  const trimmed = buffer.trim();
6202
- if (trimmed.length === 0) {
6203
- buffer = "";
6204
- bufferTokens = 0;
6205
- return;
6206
- }
6207
- result.push({
6208
- heading: section.heading,
6209
- headingLevel: section.headingLevel,
6210
- content: stripMarkdown(trimmed),
6211
- rawContent: trimmed,
6212
- rank: 0
6213
- });
6678
+ if (trimmed.length > 0) result.push(trimmed);
6214
6679
  buffer = "";
6215
6680
  bufferTokens = 0;
6216
6681
  };
6217
- for (const sentence of sentences) {
6218
- const sentTokens = estimateTokens(sentence);
6219
- if (sentTokens > MAX_CHUNK_TOKENS) {
6682
+ for (const segment of segments) {
6683
+ const tokens = estimateTokens(segment);
6684
+ if (tokens > MAX_CHUNK_TOKENS) {
6220
6685
  flush();
6221
- for (const frag of splitByCharLimit(sentence)) result.push({
6222
- heading: section.heading,
6223
- headingLevel: section.headingLevel,
6224
- content: stripMarkdown(frag),
6225
- rawContent: frag,
6226
- rank: 0
6227
- });
6686
+ for (const part of splitByCharLimit(segment)) result.push(part);
6228
6687
  continue;
6229
6688
  }
6230
- if (buffer.length > 0 && bufferTokens + sentTokens > MAX_CHUNK_TOKENS) flush();
6231
- buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
6232
- bufferTokens += sentTokens;
6689
+ if (buffer.length > 0 && bufferTokens + tokens > MAX_CHUNK_TOKENS) flush();
6690
+ buffer += segment;
6691
+ bufferTokens += tokens;
6233
6692
  }
6234
6693
  flush();
6235
- return result.length > 0 ? result : [{
6236
- heading: section.heading,
6237
- headingLevel: section.headingLevel,
6238
- content: section.content,
6239
- rawContent: section.rawContent,
6240
- rank: 0
6241
- }];
6242
- }
6243
- /** Split text into sentences using CJK + ASCII punctuation. */
6244
- function splitSentences(text) {
6245
- return text.match(/[^。!?!?.\n]+[。!?!?.\n]+|[^。!?!?.\n]+$/g) ?? [text];
6694
+ return result.length > 0 ? result : [text];
6246
6695
  }
6247
6696
  /** Hard character cut for pathological single sentences longer than the cap. */
6248
6697
  function splitByCharLimit(text) {
6249
- const maxChars = MAX_CHUNK_TOKENS * 4;
6250
6698
  const chunks = [];
6251
- let remaining = text.trim();
6252
- while (remaining.length > maxChars) {
6253
- chunks.push(remaining.slice(0, maxChars));
6254
- remaining = remaining.slice(maxChars).trimStart();
6255
- }
6256
- if (remaining.length > 0) chunks.push(remaining);
6257
- return chunks;
6699
+ let start = 0;
6700
+ let offset = 0;
6701
+ let quarterTokens = 0;
6702
+ for (const char of text) {
6703
+ const cost = CJK_CHAR.test(char) ? 4 : char.length;
6704
+ if (quarterTokens + cost > MAX_CHUNK_TOKENS * 4) {
6705
+ chunks.push(text.slice(start, offset));
6706
+ start = offset;
6707
+ quarterTokens = 0;
6708
+ }
6709
+ quarterTokens += cost;
6710
+ offset += char.length;
6711
+ }
6712
+ chunks.push(text.slice(start));
6713
+ return chunks.map((chunk) => chunk.trim()).filter((chunk) => chunk.length > 0);
6258
6714
  }
6259
6715
  //#endregion
6260
6716
  //#region ../../packages/knowledge/src/extractor.ts
@@ -6549,14 +7005,52 @@ async function embedOrNulls(engine, texts, skipEmbedding) {
6549
7005
  }
6550
7006
  /** LLM concurrency for extraction — callers and comments below refer to this constant. */
6551
7007
  const LLM_CONCURRENCY = 20;
6552
- const SUPPORTED_EXTENSIONS = new Set([
7008
+ const TEXT_EXTENSIONS = new Set([
6553
7009
  ".md",
6554
7010
  ".markdown",
6555
7011
  ".txt"
6556
7012
  ]);
6557
- /** Read a file and return its content. */
6558
- async function readFileContent(filePath) {
6559
- return readFile(filePath, "utf-8");
7013
+ const DOCUMENT_EXTENSIONS = new Set([
7014
+ ".pdf",
7015
+ ".docx",
7016
+ ".xlsx"
7017
+ ]);
7018
+ const SUPPORTED_EXTENSIONS = new Set([...TEXT_EXTENSIONS, ...DOCUMENT_EXTENSIONS]);
7019
+ const MAX_DOCUMENT_BYTES = 50 * 1024 * 1024;
7020
+ function stripImagePlaceholders(markdown) {
7021
+ return markdown.replaceAll(/<!--\s*image:[^>]*-->/g, "").replaceAll(/\n{3,}/g, "\n\n").trim();
7022
+ }
7023
+ async function readSource(filePath) {
7024
+ const ext = getFileExtension(filePath);
7025
+ if (!TEXT_EXTENSIONS.has(ext)) {
7026
+ const source = await stat(filePath);
7027
+ if (!source.isFile()) throw new Error(`来源不是文件: ${basename$1(filePath)}`);
7028
+ const { size } = source;
7029
+ if (size > MAX_DOCUMENT_BYTES) throw new Error(`文档超过 ${MAX_DOCUMENT_BYTES / 1024 / 1024}MB 上限,已跳过: ${basename$1(filePath)}`);
7030
+ }
7031
+ const bytes = await readFile(filePath);
7032
+ return {
7033
+ bytes,
7034
+ ext,
7035
+ contentHash: TEXT_EXTENSIONS.has(ext) ? createHash("sha256").update(bytes.toString("utf8")).digest("hex") : createHash("sha256").update(bytes).digest("hex")
7036
+ };
7037
+ }
7038
+ async function convertSource(filePath, bytes, ext, onConverting) {
7039
+ onConverting?.(basename$1(filePath));
7040
+ const { convertDocumentToMarkdown, DocumentMarkdownError } = await import("./markdown-BwvhOvCA.mjs");
7041
+ try {
7042
+ const content = stripImagePlaceholders((await convertDocumentToMarkdown({
7043
+ bytes,
7044
+ filename: basename$1(filePath)
7045
+ }, { useCache: false })).markdown);
7046
+ if (content.length > 0) return content;
7047
+ } catch (error) {
7048
+ if (!(error instanceof DocumentMarkdownError) || error.code !== "empty_output") {
7049
+ const message = error instanceof Error ? error.message : String(error);
7050
+ throw new Error(`文档转换失败: ${message}`, { cause: error });
7051
+ }
7052
+ }
7053
+ throw new Error(ext === ".pdf" ? "文档未提取到文字(可能是扫描版 PDF),当前转换器不包含 OCR" : "文档未提取到可索引的文字,文件可能已损坏或为空");
6560
7054
  }
6561
7055
  function getFileExtension(filePath) {
6562
7056
  const lastDot = filePath.lastIndexOf(".");
@@ -6606,10 +7100,10 @@ async function ensureEmbeddingModelMatches(store, engine) {
6606
7100
  if (recorded !== engine.modelName) throw new Error(`知识库由模型 ${recorded} 嵌入,当前模型为 ${engine.modelName},请先在 /knowledge 中重新嵌入知识库`);
6607
7101
  }
6608
7102
  /**
6609
- * Ingest a markdown file into the knowledge base.
7103
+ * Ingest a file into the knowledge base.
6610
7104
  *
6611
7105
  * Steps:
6612
- * 1. Dedupe by file_path — if a source already exists for this file, error.
7106
+ * 1. Fingerprint the source by path and content, keeping the previous revision until commit.
6613
7107
  * 2. Read file → chunkMarkdown (heading_strict).
6614
7108
  * 3. Embed chunks ("{heading}\n{content}") → store knowledge_chunks.
6615
7109
  * 4. For each chunk (concurrency LLM_CONCURRENCY): LLM extract 1 event + N entities.
@@ -6624,6 +7118,10 @@ async function ensureEmbeddingModelMatches(store, engine) {
6624
7118
  * Reports progress via callback.
6625
7119
  */
6626
7120
  async function ingestFile(store, llm, filePath, onProgress) {
7121
+ if (!isSupportedFile(filePath)) {
7122
+ const ext = getFileExtension(filePath);
7123
+ throw new Error(`不支持的文件类型: ${ext === "" ? "(无扩展名)" : ext}`);
7124
+ }
6627
7125
  const engine = store.getEmbeddingEngine();
6628
7126
  const skipEmbed = engine === void 0 || !engine.available;
6629
7127
  if (skipEmbed) onProgress?.({
@@ -6635,12 +7133,10 @@ async function ingestFile(store, llm, filePath, onProgress) {
6635
7133
  stage: "chunking",
6636
7134
  message: `读取并切分文件: ${basename$1(filePath)}`
6637
7135
  });
6638
- const content = await readFileContent(filePath);
6639
- const contentHash = createHash("sha256").update(content).digest("hex");
6640
7136
  const existing = await store.findSourceByFilePath(filePath);
6641
- let replacedOldVersion = false;
7137
+ const { bytes, ext, contentHash } = await readSource(filePath);
6642
7138
  if (existing !== void 0) {
6643
- if (existing.contentHash === contentHash || existing.contentHash === null) {
7139
+ if (existing.contentHash === contentHash || existing.contentHash === null && ext !== ".xlsx") {
6644
7140
  if (existing.contentHash === null) await store.updateSourceContentHash(existing.id, contentHash);
6645
7141
  onProgress?.({
6646
7142
  stage: "completed",
@@ -6656,21 +7152,27 @@ async function ingestFile(store, llm, filePath, onProgress) {
6656
7152
  }
6657
7153
  onProgress?.({
6658
7154
  stage: "chunking",
6659
- message: `检测到内容变更,替换旧版本: ${existing.name}`
7155
+ message: `准备更新知识文件: ${existing.name}`
6660
7156
  });
6661
- await store.deleteSource(existing.id);
6662
- replacedOldVersion = true;
6663
7157
  }
6664
- const sections = chunkContent(content, getFileExtension(filePath));
7158
+ const content = TEXT_EXTENSIONS.has(ext) ? bytes.toString("utf8") : await convertSource(filePath, bytes, ext, (file) => {
7159
+ onProgress?.({
7160
+ stage: "converting",
7161
+ message: file
7162
+ });
7163
+ });
7164
+ const sections = chunkContent(content, ext);
7165
+ if (sections.length === 0) {
7166
+ onProgress?.({
7167
+ stage: "error",
7168
+ message: "文件无有效内容"
7169
+ });
7170
+ throw new Error("文件无有效内容可摄入");
7171
+ }
6665
7172
  store.beginTransaction();
6666
7173
  try {
6667
- if (sections.length === 0) {
6668
- onProgress?.({
6669
- stage: "error",
6670
- message: "文件无有效内容"
6671
- });
6672
- throw new Error("文件无有效内容可摄入");
6673
- }
7174
+ if ((await store.findSourceByFilePath(filePath))?.id !== existing?.id) throw new Error(`知识文件索引已被其他操作更新,请重试: ${basename$1(filePath)}`);
7175
+ if (existing !== void 0) await store.deleteSource(existing.id);
6674
7176
  const fileName = basename$1(filePath);
6675
7177
  const source = await store.createSource({
6676
7178
  name: fileName,
@@ -6695,14 +7197,10 @@ async function ingestFile(store, llm, filePath, onProgress) {
6695
7197
  chunkCount: chunks.length,
6696
7198
  eventCount: events.length,
6697
7199
  entityCount: uniqueEntities.length,
6698
- outcome: replacedOldVersion ? "updated" : "created"
7200
+ outcome: existing === void 0 ? "created" : "updated"
6699
7201
  };
6700
7202
  } catch (error) {
6701
7203
  store.rollbackTransaction();
6702
- if (replacedOldVersion) {
6703
- const message = error instanceof Error ? error.message : String(error);
6704
- throw new Error(`旧版本已删除、新版本摄入失败,请重新摄入该文件: ${filePath}(${message})`, { cause: error });
6705
- }
6706
7204
  throw error;
6707
7205
  }
6708
7206
  }
@@ -6865,6 +7363,7 @@ async function collectSupportedFiles(dirPath) {
6865
7363
  const files = [];
6866
7364
  const entries = await readdir(dirPath, { withFileTypes: true });
6867
7365
  for (const entry of entries) {
7366
+ if (entry.name.startsWith(".") || entry.name.startsWith("~$")) continue;
6868
7367
  const fullPath = join$1(dirPath, entry.name);
6869
7368
  if (entry.isDirectory()) files.push(...await collectSupportedFiles(fullPath));
6870
7369
  else if (entry.isFile() && isSupportedFile(fullPath)) files.push(fullPath);
@@ -6873,7 +7372,7 @@ async function collectSupportedFiles(dirPath) {
6873
7372
  }
6874
7373
  async function ingestDirectory(store, llm, dirPath, onProgress) {
6875
7374
  const files = await collectSupportedFiles(dirPath);
6876
- if (files.length === 0) throw new Error(`目录下没有支持的文件 (.md, .markdown, .txt): ${dirPath}`);
7375
+ if (files.length === 0) throw new Error(`目录下没有支持的文件 (${[...SUPPORTED_EXTENSIONS].join(", ")}): ${dirPath}`);
6877
7376
  const errors = [];
6878
7377
  let succeeded = 0;
6879
7378
  let created = 0;
@@ -6885,6 +7384,10 @@ async function ingestDirectory(store, llm, dirPath, onProgress) {
6885
7384
  for (let i = 0; i < files.length; i++) {
6886
7385
  const filePath = files[i];
6887
7386
  const fileProgress = (progress) => {
7387
+ if (progress.stage === "converting") {
7388
+ onProgress?.(progress);
7389
+ return;
7390
+ }
6888
7391
  const prefix = `批量摄入 ${i + 1}/${files.length}: ${basename$1(filePath)}`;
6889
7392
  onProgress?.({
6890
7393
  ...progress,
@@ -7259,4 +7762,4 @@ async function chunkVectorFallback(store, queryVec, topK) {
7259
7762
  return results;
7260
7763
  }
7261
7764
  //#endregion
7262
- export { classifyEmbeddingFailure as A, buildProjectTagCloud as B, buildConsolidationPlan as C, stripMemoryTags as D, parseMemoryMemos as E, So as F, isAbsolute$1 as G, KnowledgeStore as H, createMemoryMemo as I, parse$1 as J, join$1 as K, toSummary as L, createFastEmbedEngine as M, hasEmbeddingModelCache as N, MemoryMemoStore as O, probeLocalEmbeddingSupport as P, generateTags as R, applyConsolidation as S, buildExitExtractionPrompt as T, basename$1 as U, rankMemos as V, dirname$2 as W, resolve$1 as X, relative$1 as Y, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, clearEmbeddingModelCache as j, sharedMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, normalize as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, EXIT_EXTRACTION_SYSTEM_PROMPT as w, DreamTracker as x, sharedKnowledgeStore as y, normalizeTags as z };
7765
+ export { classifyEmbeddingFailure as A, buildProjectTagCloud as B, buildConsolidationPlan as C, stripMemoryTags as D, parseMemoryMemos as E, So as F, isAbsolute$1 as G, KnowledgeStore as H, createMemoryMemo as I, parse$1 as J, join$1 as K, toSummary as L, createEmbeddingEngine as M, hasEmbeddingModelCache as N, MemoryMemoStore as O, probeLocalEmbeddingSupport as P, generateTags as R, applyConsolidation as S, buildExitExtractionPrompt as T, basename$1 as U, rankMemos as V, dirname$2 as W, resolve$1 as X, relative$1 as Y, splitLargeSection as _, isSupportedFile as a, sharedKnowledgeStoreReady as b, buildExtractionUserPrompt as c, extractQueryEntities as d, parseExtractionResponse as f, estimateTokens as g, chunkText as h, ingestFile as i, clearEmbeddingModelCache as j, sharedMemoStore as k, extractEventFromChunk as l, chunkMarkdown as m, multiSearchWithTrace as n, ENTITY_TYPES as o, rerankEventsWithLlm as p, normalize as q, ingestDirectory as r, EXTRACTION_SYSTEM_PROMPT as s, multiSearch as t, extractJsonFromText as u, stripMarkdown as v, EXIT_EXTRACTION_SYSTEM_PROMPT as w, DreamTracker as x, sharedKnowledgeStore as y, normalizeTags as z };