chatlab-cli 0.36.2 → 0.37.1

package/dist/{chunk-TJ3NOBVI.mjs → chunk-JWJREWVN.mjs}

@@ -54,6 +54,7 @@ import {
   getChatLabTempScopeDir,
   getLocalProfileByModelId,
   getLocalProfilesForLocale,
+  hasUnmatchedFallbackOnlyMessage,
   isKeylessSemanticIndexApiBaseUrl,
   isSemanticIndexConfigured,
   isSemanticVoid,
@@ -68,32 +69,36 @@ import {
   semanticSearch,
   vecTableName,
   vecTableSchema
-} from "./chunk-VREVDZ4Q.mjs";
+} from "./chunk-GKCIRKOH.mjs";
 import {
   CHATLAB_FORMAT_VERSION,
   DESENSITIZE_RULES_SCHEMA_VERSION,
   MEANINGFUL_POS_TAGS,
-  POS_TAG_DEFINITIONS,
   StaticPathProvider,
   appLogger,
+  batchSegmentChineseWithStats,
+  batchSegmentWithFrequency,
   buildExcludeKeywordsConditions,
-  cleanText,
+  clearJiebaInstance,
+  collectPosTagStats,
   detectConflictsInMessages,
   getCollidingPlatformIds,
   getCollidingPlatformIdsFromMessages,
+  getJieba,
   isStopword,
-  isValidWord,
+  isSystemMessageContent,
   loadConfig,
   mergeMembers2 as mergeMembers,
   migrateFromElectronIfNeeded,
   normalizePlatformId,
   readJsonFile,
   resolveApiKey,
+  segment,
   snapshotPathProvider,
   withFileLock,
   writeConfigField,
   writeJsonFileAtomically
-} from "./chunk-TRGR3KKJ.mjs";
+} from "./chunk-OTYNMDDL.mjs";
 import {
   __export
 } from "./chunk-VSOWNKPA.mjs";
@@ -559,220 +564,6 @@ function expandHome(filePath) {
   return filePath;
 }
 
-// ../../packages/node-runtime/src/nlp/segmenter.ts
-import * as fs3 from "fs";
-import * as path3 from "path";
-import { createRequire } from "module";
-var _nlpDir = null;
-var jiebaInstances = /* @__PURE__ */ new Map();
-var require2 = createRequire(import.meta.url);
-function initNlpDir(nlpDir) {
-  _nlpDir = nlpDir;
-}
-function getNlpDir() {
-  return _nlpDir;
-}
-function existsDictOnDisk(dictId) {
-  if (!_nlpDir) return false;
-  return fs3.existsSync(path3.join(_nlpDir, `${dictId}.dict`));
-}
-function tryLoadDictFromDisk(dictId) {
-  if (!_nlpDir) return null;
-  const dictPath = path3.join(_nlpDir, `${dictId}.dict`);
-  if (!fs3.existsSync(dictPath)) return null;
-  try {
-    return fs3.readFileSync(dictPath);
-  } catch {
-    return null;
-  }
-}
-function getJieba(dictType = "default") {
-  const effectiveType = dictType === "default" ? "zh-CN" : dictType;
-  const cached = jiebaInstances.get(effectiveType);
-  if (cached) {
-    if (cached.fromDiskDict === existsDictOnDisk(effectiveType)) return cached.instance;
-    jiebaInstances.delete(effectiveType);
-    console.error(`[NLP] jieba cache invalidated (dict ${cached.fromDiskDict ? "removed" : "added"}): ${effectiveType}`);
-  }
-  try {
-    const { Jieba } = require2("@node-rs/jieba");
-    let instance;
-    const diskDict = tryLoadDictFromDisk(effectiveType);
-    if (diskDict) {
-      instance = Jieba.withDict(diskDict);
-      console.error(`[NLP] jieba dict loaded: ${effectiveType} (${diskDict.length} bytes)`);
-    } else {
-      instance = new Jieba();
-      console.warn(`[NLP] jieba dict missing: ${effectiveType}, fallback to built-in tokenizer`);
-    }
-    jiebaInstances.set(effectiveType, { instance, fromDiskDict: !!diskDict });
-    return instance;
-  } catch (error) {
-    console.error(`[NLP] Failed to load jieba module (dict=${effectiveType}):`, error);
-    throw new Error(`jieba \u6A21\u5757\u52A0\u8F7D\u5931\u8D25 (${effectiveType})`);
-  }
-}
-function clearJiebaInstance(dictType) {
-  jiebaInstances.delete(dictType);
-  console.error(`[NLP] jieba instance cleared: ${dictType}`);
-}
-function segmentChinese(text, options = {}) {
-  const { posFilterMode = "meaningful", customPosTags, dictType = "default" } = options;
-  const cleaned = cleanText(text);
-  if (!cleaned) return [];
-  try {
-    const jieba = getJieba(dictType);
-    if (posFilterMode === "all") return jieba.cut(cleaned, false);
-    const tagged = jieba.tag(cleaned);
-    const allowedTags = posFilterMode === "custom" && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS;
-    return tagged.filter((item) => allowedTags.has(item.tag)).map((item) => item.word);
-  } catch (error) {
-    console.error("[NLP] Chinese segmentation failed:", error);
-    try {
-      return getJieba("default").cut(cleanText(text), false);
-    } catch {
-      return cleaned.split("");
-    }
-  }
-}
-function segmentEnglish(text) {
-  const cleaned = cleanText(text);
-  if (!cleaned) return [];
-  try {
-    const segmenter = new Intl.Segmenter("en", { granularity: "word" });
-    return [...segmenter.segment(cleaned)].filter((s) => s.isWordLike).map((s) => s.segment.toLowerCase());
-  } catch {
-    return cleaned.toLowerCase().split(/\s+/).filter((w) => w.length > 0);
-  }
-}
-function segmentJapanese(text) {
-  const cleaned = cleanText(text);
-  if (!cleaned) return [];
-  try {
-    const segmenter = new Intl.Segmenter("ja", { granularity: "word" });
-    return [...segmenter.segment(cleaned)].filter((s) => s.isWordLike).map((s) => s.segment);
-  } catch {
-    return cleaned.split("").filter((ch) => ch.trim().length > 0);
-  }
-}
-function segment(text, locale, options = {}) {
-  const {
-    minLength,
-    posFilterMode = "meaningful",
-    customPosTags,
-    enableStopwords = true,
-    dictType = "default"
-  } = options;
-  const isChinese = locale.startsWith("zh");
-  const isJapanese = locale === "ja-JP";
-  const defaultMinLength = isChinese || isJapanese ? 2 : 3;
-  const effectiveMinLength = minLength ?? defaultMinLength;
-  let words;
-  if (isChinese) {
-    words = segmentChinese(text, { posFilterMode, customPosTags, dictType });
-  } else if (isJapanese) {
-    words = segmentJapanese(text);
-  } else {
-    words = segmentEnglish(text);
-  }
-  return words.filter((word) => isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword));
-}
-function batchSegmentWithFrequency(texts, locale, options = {}) {
-  const {
-    minLength,
-    minCount = 2,
-    topN = 100,
-    posFilterMode,
-    customPosTags,
-    enableStopwords,
-    dictType,
-    excludeWords
-  } = options;
-  const wordFrequency = /* @__PURE__ */ new Map();
-  const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null;
-  for (const text of texts) {
-    const words = segment(text, locale, { minLength, posFilterMode, customPosTags, enableStopwords, dictType });
-    for (const word of words) {
-      if (excludeSet && excludeSet.has(word.toLowerCase())) continue;
-      wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1);
-    }
-  }
-  const filtered = /* @__PURE__ */ new Map();
-  let totalWords = 0;
-  for (const [word, count] of wordFrequency) {
-    if (count >= minCount) {
-      filtered.set(word, count);
-      totalWords += count;
-    }
-  }
-  const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN);
-  return { words: new Map(sorted), uniqueWords: filtered.size, totalWords };
-}
-function batchSegmentChineseWithStats(texts, locale, options = {}) {
-  const {
-    minLength,
-    minCount = 2,
-    topN = 100,
-    posFilterMode = "meaningful",
-    customPosTags,
-    enableStopwords = true,
-    dictType = "default",
-    excludeWords
-  } = options;
-  const effectiveMinLength = minLength ?? 2;
-  const allowedTags = posFilterMode === "custom" && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS;
-  const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null;
-  const wordFrequency = /* @__PURE__ */ new Map();
-  const posTagStats = /* @__PURE__ */ new Map();
-  try {
-    const jieba = getJieba(dictType);
-    for (const text of texts) {
-      const cleaned = cleanText(text);
-      if (!cleaned) continue;
-      for (const { tag, word } of jieba.tag(cleaned)) {
-        if (!isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword)) continue;
-        posTagStats.set(tag, (posTagStats.get(tag) || 0) + 1);
-        if (!allowedTags.has(tag)) continue;
-        if (excludeSet && excludeSet.has(word.toLowerCase())) continue;
-        wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1);
-      }
-    }
-  } catch (error) {
-    console.error("[NLP] Chinese single-pass segmentation failed:", error);
-  }
-  const filtered = /* @__PURE__ */ new Map();
-  let totalWords = 0;
-  for (const [word, count] of wordFrequency) {
-    if (count >= minCount) {
-      filtered.set(word, count);
-      totalWords += count;
-    }
-  }
-  const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN);
-  return { words: new Map(sorted), uniqueWords: filtered.size, totalWords, posTagStats };
-}
-function collectPosTagStats(texts, minWordLength = 2, enableStopwords = true, dictType = "default") {
-  const posStats = /* @__PURE__ */ new Map();
-  try {
-    const jieba = getJieba(dictType);
-    for (const text of texts) {
-      const cleaned = cleanText(text);
-      if (!cleaned) continue;
-      const tagged = jieba.tag(cleaned);
-      for (const item of tagged) {
-        if (!isValidWord(item.word, "zh-CN", minWordLength, enableStopwords, isStopword)) continue;
-        posStats.set(item.tag, (posStats.get(item.tag) || 0) + 1);
-      }
-    }
-  } catch (error) {
-    console.error("[NLP] Failed to collect POS stats:", error);
-  }
-  return posStats;
-}
-function getPosTagDefinitions() {
-  return POS_TAG_DEFINITIONS;
-}
-
 // ../../packages/node-runtime/src/jieba-nlp-provider.ts
 function createJiebaNlpProvider(dictType = "default") {
   return {
@@ -806,7 +597,7 @@ function buildMessageQuery(timeFilter, memberId, excludeKeywords) {
     params.push(...exclude.params);
   }
   conditions.push("COALESCE(m.account_name, '') != '\u7CFB\u7EDF\u6D88\u606F'");
-  conditions.push("msg.type = 0");
+  conditions.push(`msg.type IN (${0 /* TEXT */}, ${5 /* EMOJI */})`);
   conditions.push("msg.content IS NOT NULL");
   conditions.push("TRIM(msg.content) != ''");
   return {
@@ -834,7 +625,11 @@ function computeWordFrequency(db, params) {
   if (messages.length === 0) {
     return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 };
   }
-  const texts = messages.map((m) => m.content);
+  const analyzableMessages = messages.filter((message) => !isSystemMessageContent(message.content));
+  if (analyzableMessages.length === 0) {
+    return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 };
+  }
+  const texts = analyzableMessages.map((m) => m.content);
   const segmentOptions = {
     minLength: minWordLength,
     minCount,
@@ -868,7 +663,7 @@ function computeWordFrequency(db, params) {
   return {
     words,
     totalWords: result.totalWords,
-    totalMessages: messages.length,
+    totalMessages: analyzableMessages.length,
     uniqueWords: result.uniqueWords,
     posTagStats
   };
@@ -878,8 +673,8 @@ function segmentText(text, locale, minLength) {
 }
 
 // ../../packages/node-runtime/src/nlp/dict-manager.ts
-import * as fs4 from "fs";
-import * as path4 from "path";
+import * as fs3 from "fs";
+import * as path3 from "path";
 import { createHash } from "crypto";
 var DICT_DOWNLOAD_URL_BASE = "https://chatlab.fun/assets/nlp";
 var DICT_SHA256 = {
@@ -891,19 +686,19 @@ var AVAILABLE_DICTS = [
   { id: "zh-TW", label: "\u7E41\u9AD4\u4E2D\u6587", locale: "zh-TW" }
 ];
 function getDictFilePath(nlpDir, dictId) {
-  return path4.join(nlpDir, `${dictId}.dict`);
+  return path3.join(nlpDir, `${dictId}.dict`);
 }
 function isDictDownloaded(nlpDir, dictId) {
-  return fs4.existsSync(getDictFilePath(nlpDir, dictId));
+  return fs3.existsSync(getDictFilePath(nlpDir, dictId));
 }
 function getDictList(nlpDir) {
   return AVAILABLE_DICTS.map((d) => {
     const filePath = getDictFilePath(nlpDir, d.id);
-    const downloaded = fs4.existsSync(filePath);
+    const downloaded = fs3.existsSync(filePath);
     let fileSize;
     if (downloaded) {
       try {
-        fileSize = fs4.statSync(filePath).size;
+        fileSize = fs3.statSync(filePath).size;
       } catch {
       }
     }
@@ -912,9 +707,9 @@ function getDictList(nlpDir) {
 }
 function loadDictBuffer(nlpDir, dictId) {
   const filePath = getDictFilePath(nlpDir, dictId);
-  if (!fs4.existsSync(filePath)) return null;
+  if (!fs3.existsSync(filePath)) return null;
   try {
-    return fs4.readFileSync(filePath);
+    return fs3.readFileSync(filePath);
   } catch (error) {
     console.error(`[NLP DictManager] Failed to read dict file: ${filePath}`, error);
     return null;
@@ -924,7 +719,7 @@ async function downloadDict(nlpDir, dictId, onProgress) {
   if (!AVAILABLE_DICTS.find((d) => d.id === dictId)) {
     return { success: false, error: `Unknown dict: ${dictId}` };
   }
-  fs4.mkdirSync(nlpDir, { recursive: true });
+  fs3.mkdirSync(nlpDir, { recursive: true });
   const url = `${DICT_DOWNLOAD_URL_BASE}/${dictId}.dict`;
   const filePath = getDictFilePath(nlpDir, dictId);
   const tmpPath = filePath + ".tmp";
@@ -965,16 +760,16 @@ async function downloadDict(nlpDir, dictId, onProgress) {
     if (actualSha256 !== expectedSha256) {
       return { success: false, error: "Dictionary integrity check failed (SHA256 mismatch)" };
     }
-    fs4.writeFileSync(tmpPath, buffer);
-    if (fs4.existsSync(filePath)) fs4.unlinkSync(filePath);
-    fs4.renameSync(tmpPath, filePath);
+    fs3.writeFileSync(tmpPath, buffer);
+    if (fs3.existsSync(filePath)) fs3.unlinkSync(filePath);
+    fs3.renameSync(tmpPath, filePath);
     clearJiebaInstance(dictId);
-    console.log(`[NLP DictManager] Dict downloaded: ${dictId} (${fs4.statSync(filePath).size} bytes)`);
+    console.log(`[NLP DictManager] Dict downloaded: ${dictId} (${fs3.statSync(filePath).size} bytes)`);
     return { success: true };
   } catch (error) {
-    if (fs4.existsSync(tmpPath)) {
+    if (fs3.existsSync(tmpPath)) {
       try {
-        fs4.unlinkSync(tmpPath);
+        fs3.unlinkSync(tmpPath);
       } catch {
       }
     }
@@ -985,9 +780,9 @@ async function downloadDict(nlpDir, dictId, onProgress) {
 }
 function deleteDict(nlpDir, dictId) {
   const filePath = getDictFilePath(nlpDir, dictId);
-  if (!fs4.existsSync(filePath)) return { success: true };
+  if (!fs3.existsSync(filePath)) return { success: true };
   try {
-    fs4.unlinkSync(filePath);
+    fs3.unlinkSync(filePath);
     clearJiebaInstance(dictId);
     console.log(`[NLP DictManager] Dict deleted: ${dictId}`);
     return { success: true };
@@ -997,8 +792,20 @@ function deleteDict(nlpDir, dictId) {
     return { success: false, error: msg };
   }
 }
-async function ensureDefaultDict(nlpDir) {
+async function ensureDefaultDict(nlpDir, bundledDictDir) {
   if (isDictDownloaded(nlpDir, "zh-CN")) return;
+  if (bundledDictDir) {
+    const bundledPath = getDictFilePath(bundledDictDir, "zh-CN");
+    if (fs3.existsSync(bundledPath)) {
+      fs3.mkdirSync(nlpDir, { recursive: true });
+      const filePath = getDictFilePath(nlpDir, "zh-CN");
+      const tmpPath = filePath + ".tmp";
+      fs3.copyFileSync(bundledPath, tmpPath);
+      fs3.renameSync(tmpPath, filePath);
+      console.log("[NLP DictManager] Installed bundled zh-CN dictionary");
+      return;
+    }
+  }
   console.log("[NLP DictManager] zh-CN dict not found, starting background download...");
   const result = await downloadDict(nlpDir, "zh-CN");
   if (result.success) {
@@ -1009,8 +816,8 @@ async function ensureDefaultDict(nlpDir) {
 }
 
 // ../../packages/node-runtime/src/preferences.ts
-import * as fs5 from "fs";
-import * as path5 from "path";
+import * as fs4 from "fs";
+import * as path4 from "path";
 var DEFAULTS = {
   pinnedSessionIds: [],
   assistantUpgradeSkippedVersions: {},
@@ -1051,13 +858,13 @@ var PreferencesManager = class {
   filePath;
   cache = null;
   constructor(systemDir) {
-    this.filePath = path5.join(systemDir, "preferences.json");
+    this.filePath = path4.join(systemDir, "preferences.json");
   }
   load() {
     if (this.cache) return this.cache;
     try {
-      if (fs5.existsSync(this.filePath)) {
-        const raw = fs5.readFileSync(this.filePath, "utf-8");
+      if (fs4.existsSync(this.filePath)) {
+        const raw = fs4.readFileSync(this.filePath, "utf-8");
         const parsed = JSON.parse(raw);
         const desensitizeMigration = this.migrateLegacyDesensitizeRules(parsed, raw);
         const ownerMigration = this.migrateLegacyOwnerExclusions(desensitizeMigration.preferences);
@@ -1117,6 +924,7 @@ var PreferencesManager = class {
   normalizeAiGlobalSettings(partial) {
     const normalized = { ...partial ?? {} };
     delete normalized.contextCompression;
+    delete normalized.allowProactiveMemory;
     return { ...DEFAULTS.aiGlobalSettings, ...normalized };
   }
   normalizeAiPreprocessConfig(partial) {
@@ -1199,21 +1007,21 @@ var PreferencesManager = class {
     };
   }
   backupLegacyPreferences(rawContent) {
-    const backupDir = path5.join(path5.dirname(this.filePath), "backups");
-    if (!fs5.existsSync(backupDir)) {
-      fs5.mkdirSync(backupDir, { recursive: true });
+    const backupDir = path4.join(path4.dirname(this.filePath), "backups");
+    if (!fs4.existsSync(backupDir)) {
+      fs4.mkdirSync(backupDir, { recursive: true });
     }
     const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-    const backupPath = path5.join(backupDir, `preferences-pre-desensitize-groups-${timestamp}.json`);
-    fs5.writeFileSync(backupPath, rawContent, "utf-8");
+    const backupPath = path4.join(backupDir, `preferences-pre-desensitize-groups-${timestamp}.json`);
+    fs4.writeFileSync(backupPath, rawContent, "utf-8");
     console.info(`[Preferences] Backed up legacy desensitize rules to ${backupPath}`);
   }
   writePreferences(preferences) {
-    const dir = path5.dirname(this.filePath);
-    if (!fs5.existsSync(dir)) {
-      fs5.mkdirSync(dir, { recursive: true });
+    const dir = path4.dirname(this.filePath);
+    if (!fs4.existsSync(dir)) {
+      fs4.mkdirSync(dir, { recursive: true });
     }
-    fs5.writeFileSync(this.filePath, JSON.stringify(preferences, null, 2), "utf-8");
+    fs4.writeFileSync(this.filePath, JSON.stringify(preferences, null, 2), "utf-8");
   }
   /**
    * Deep merge: arrays are replaced (not concatenated), objects are merged.
@@ -1261,6 +1069,31 @@ function createMessageIdNormalizer(scopes) {
     return `__chatlab_message_scope__${scopeIndexes.get(scope) ?? 0}__${encodeURIComponent(id)}`;
   };
 }
+function createFallbackOccurrenceTracker() {
+  const maximumOccurrences = /* @__PURE__ */ new Map();
+  const sourceOccurrences = /* @__PURE__ */ new Map();
+  return (message, sourceIndex) => {
+    if (message.platformMessageId) return false;
+    const fallbackKey = generateFallbackMessageKey({
+      timestamp: message.timestamp,
+      senderPlatformId: message.senderPlatformId,
+      type: message.type,
+      content: message.content ?? null,
+      replyToMessageId: message.replyToMessageId
+    });
+    let occurrences = sourceOccurrences.get(sourceIndex);
+    if (!occurrences) {
+      occurrences = /* @__PURE__ */ new Map();
+      sourceOccurrences.set(sourceIndex, occurrences);
+    }
+    const occurrence = (occurrences.get(fallbackKey) ?? 0) + 1;
+    occurrences.set(fallbackKey, occurrence);
+    const previousMaximum = maximumOccurrences.get(fallbackKey) ?? 0;
+    if (occurrence <= previousMaximum) return true;
+    maximumOccurrences.set(fallbackKey, occurrence);
+    return false;
+  };
+}
 function createMessageScopeResolver(contexts) {
   const contextBySource = /* @__PURE__ */ new Map();
   const parents = /* @__PURE__ */ new Map();
@@ -1339,22 +1172,21 @@ function checkConflictsFromSources(dataSources) {
   const normalizeMessageId = createMessageIdNormalizer(
     preparedSources.map(({ sourceIndex }) => scopeResolver.resolve(sourceIndex))
   );
-  const dedupState = createMessageDedupState();
+  const dedupState = createMessageDedupState([], [], [], { preserveFallbackMultiplicity: true });
+  const isCoveredFallbackOccurrence = createFallbackOccurrenceTracker();
   let totalMessages = 0;
   for (const { msg, sourceIndex, platform } of allMessages) {
     const messageScope = scopeResolver.resolve(sourceIndex);
     const senderPlatformId = normalizePlatformId(msg.senderPlatformId, platform, collidingIds);
-    const duplicate = registerMessageAndCheckDuplicate(
-      {
-        platformMessageId: normalizeMessageId(msg.platformMessageId, messageScope),
-        timestamp: msg.timestamp,
-        senderPlatformId,
-        type: msg.type,
-        content: msg.content ?? null,
-        replyToMessageId: normalizeMessageId(msg.replyToMessageId, messageScope)
-      },
-      dedupState
-    );
+    const dedupMessage = {
+      platformMessageId: normalizeMessageId(msg.platformMessageId, messageScope),
+      timestamp: msg.timestamp,
+      senderPlatformId,
+      type: msg.type,
+      content: msg.content ?? null,
+      replyToMessageId: normalizeMessageId(msg.replyToMessageId, messageScope)
+    };
+    const duplicate = isCoveredFallbackOccurrence(dedupMessage, sourceIndex) || registerMessageAndCheckDuplicate(dedupMessage, dedupState);
     if (!duplicate) totalMessages++;
   }
   return { ...result, totalMessages };
@@ -1398,7 +1230,8 @@ function buildMergedOutput(dataSources, outputName) {
   const normalizeMessageId = createMessageIdNormalizer(
     metas.map(({ sourceIndex }) => scopeResolver.resolve(sourceIndex))
   );
-  const dedupState = createMessageDedupState();
+  const dedupState = createMessageDedupState([], [], [], { preserveFallbackMultiplicity: true });
+  const isCoveredFallbackOccurrence = createFallbackOccurrenceTracker();
   const fallbackOnlyMessageIndexes = /* @__PURE__ */ new Map();
   const mergedMessages = [];
   for (const { source, meta, sourceIndex } of metas) {
@@ -1418,15 +1251,17 @@ function buildMergedOutput(dataSources, outputName) {
           replyToMessageId
         };
         const fallbackKey = generateFallbackMessageKey(dedupMessage);
+        if (isCoveredFallbackOccurrence(dedupMessage, sourceIndex)) continue;
         const shouldBackfillId = Boolean(
-          platformMessageId && !dedupState.platformMessageIds.has(platformMessageId) && dedupState.fallbackOnlyKeys.has(fallbackKey)
+          platformMessageId && !dedupState.platformMessageIds.has(platformMessageId) && hasUnmatchedFallbackOnlyMessage(dedupState, fallbackKey)
         );
         if (registerMessageAndCheckDuplicate(dedupMessage, dedupState)) {
           if (shouldBackfillId) {
-            const retainedIndex2 = fallbackOnlyMessageIndexes.get(fallbackKey);
+            const retainedIndexes = fallbackOnlyMessageIndexes.get(fallbackKey);
+            const retainedIndex2 = retainedIndexes?.shift();
             if (retainedIndex2 !== void 0) {
               mergedMessages[retainedIndex2].platformMessageId = platformMessageId;
-              fallbackOnlyMessageIndexes.delete(fallbackKey);
+              if (retainedIndexes?.length === 0) fallbackOnlyMessageIndexes.delete(fallbackKey);
             }
           }
           continue;
@@ -1442,7 +1277,11 @@ function buildMergedOutput(dataSources, outputName) {
           content: msg.content,
           replyToMessageId
         });
-        if (!platformMessageId) fallbackOnlyMessageIndexes.set(fallbackKey, retainedIndex);
+        if (!platformMessageId) {
+          const retainedIndexes = fallbackOnlyMessageIndexes.get(fallbackKey) ?? [];
+          retainedIndexes.push(retainedIndex);
+          fallbackOnlyMessageIndexes.set(fallbackKey, retainedIndexes);
+        }
       }
     });
   }
@@ -1495,7 +1334,7 @@ function* serializeChatLabToJsonl(data) {
 }
 
 // ../../packages/node-runtime/src/services/analytics.ts
-import * as path6 from "path";
+import * as path5 from "path";
 import { randomUUID as randomUUID2 } from "crypto";
 var DEFAULT_DATA = {
   lastReportDate: null,
@@ -1607,7 +1446,7 @@ var AnalyticsService = class {
   // Daily-active deduplication is intentionally process-local unless concurrent runtimes become supported.
   dailyActivePromise = null;
   constructor(systemDir, options) {
-    this.dataPath = path6.join(systemDir, "analytics.json");
+    this.dataPath = path5.join(systemDir, "analytics.json");
     this.options = {
       ...options,
       umami: options.umami ?? getUmamiConfigFromEnv()
@@ -1769,8 +1608,8 @@ var AnalyticsService = class {
 };
 
 // ../../packages/node-runtime/src/services/demo-import.ts
-import * as fs6 from "fs";
-import * as path7 from "path";
+import * as fs5 from "fs";
+import * as path6 from "path";
 var DEMO_BASE_URL = "https://chatlab.fun/assets/demo";
 var DEMO_FILES = ["demo-group.json", "demo-private-B-wukong.json"];
 async function importDemoSessions(options) {
@@ -1802,8 +1641,8 @@ async function importDemoSessions(options) {
     }
     const rebased = rebaseChatLabDemoDocuments(downloaded, options.now?.() ?? /* @__PURE__ */ new Date(), options.targetTimeZone);
     const localPaths = rebased.documents.map((document, index) => {
-      const localPath = path7.join(createdTempDir, DEMO_FILES[index]);
-      fs6.writeFileSync(localPath, document, "utf8");
+      const localPath = path6.join(createdTempDir, DEMO_FILES[index]);
+      fs5.writeFileSync(localPath, document, "utf8");
       return localPath;
     });
     appLogger.info("demo-import", "Demo timeline prepared", {
@@ -1914,7 +1753,7 @@ __export(semantic_index_exports, {
 });
 
 // ../../packages/node-runtime/src/semantic-index/worker-client.ts
-import path8 from "path";
+import path7 from "path";
 
 // ../../packages/node-runtime/src/semantic-index/worker-thread-transport.ts
 import { Worker } from "worker_threads";
@@ -2296,7 +2135,7 @@ function createSemanticIndexWorkerClient(options) {
 }
 function createSemanticIndexWorkerRuntimeClient(options) {
   const configStore = new SemanticIndexConfigStore(
-    path8.join(options.pathProvider.getAiDataDir(), SEMANTIC_INDEX_CONFIG_FILE)
+    path7.join(options.pathProvider.getAiDataDir(), SEMANTIC_INDEX_CONFIG_FILE)
   );
   return createSemanticIndexWorkerClient({
     configStore,
@@ -2351,14 +2190,6 @@ export {
   NodePathProvider,
   getDefaultNodeUserDataDir,
   getSystemLogsDir,
-  initNlpDir,
-  getNlpDir,
-  getJieba,
-  clearJiebaInstance,
-  segment,
-  batchSegmentWithFrequency,
-  collectPosTagStats,
-  getPosTagDefinitions,
   createJiebaNlpProvider,
   computeWordFrequency,
   segmentText,
@@ -2377,4 +2208,4 @@ export {
   createSemanticIndexWorkerRuntimeClient,
   semantic_index_exports
 };
-//# sourceMappingURL=chunk-TJ3NOBVI.mjs.map
+//# sourceMappingURL=chunk-JWJREWVN.mjs.map