chatlab-cli 0.34.2 → 0.35.0

package/dist/{chunk-2XCNTG4O.mjs → chunk-DC7IUDNV.mjs}

@@ -40,46 +40,49 @@ import {
   composeChunkId,
   computeChunkerConfigHash,
   computeDbPathHash,
-  createChatDbFtsSearcher,
   createChatDbMessageRangeReader,
   createChatDbMessageSource,
   createChatLabTempDir,
   createEmbedder,
   createLocalEmbeddingRuntimeManager,
+  createMessageDedupState,
   createSemanticIndexService,
   defaultSemanticIndexConfig,
   deriveParentId,
   estimateTokens,
-  extractFtsKeywords,
   formatEvidenceMessages,
+  generateFallbackMessageKey,
   getChatLabTempScopeDir,
   getLocalProfileByModelId,
   getLocalProfilesForLocale,
-  hybridSearch,
   isKeylessSemanticIndexApiBaseUrl,
   isSemanticIndexConfigured,
   isSemanticVoid,
   parseParentBounds,
   persistSemanticIndexConfig,
   rebaseChatLabDemoDocuments,
-  reciprocalRankFusion,
+  registerMessageAndCheckDuplicate,
   removeChatLabTempDir,
   resolveModelId,
   resolveSemanticIndexApiKeySet,
   runWarmup,
+  semanticSearch,
   vecTableName,
   vecTableSchema
-} from "./chunk-4LW6443F.mjs";
+} from "./chunk-4GWWKHWA.mjs";
 import {
   CHATLAB_FORMAT_VERSION,
   MEANINGFUL_POS_TAGS,
+  POS_TAG_DEFINITIONS,
   StaticPathProvider,
   appLogger,
+  buildExcludeKeywordsConditions,
+  cleanText,
   detectConflictsInMessages,
-  generateMessageKey,
   getCollidingPlatformIds,
-  getJieba,
+  getCollidingPlatformIdsFromMessages,
   isStopword,
+  isValidWord,
   loadConfig,
   mergeMembers2 as mergeMembers,
   migrateFromElectronIfNeeded,
@@ -90,7 +93,7 @@ import {
   withFileLock,
   writeConfigField,
   writeJsonFileAtomically
-} from "./chunk-INBJ7SND.mjs";
+} from "./chunk-22YJOBA3.mjs";
 import {
   __export
 } from "./chunk-VSOWNKPA.mjs";
@@ -556,6 +559,220 @@ function expandHome(filePath) {
   return filePath;
 }
 
+// ../../packages/node-runtime/src/nlp/segmenter.ts
+import * as fs3 from "fs";
+import * as path3 from "path";
+import { createRequire } from "module";
+var _nlpDir = null;
+var jiebaInstances = /* @__PURE__ */ new Map();
+var require2 = createRequire(import.meta.url);
+function initNlpDir(nlpDir) {
+  _nlpDir = nlpDir;
+}
+function getNlpDir() {
+  return _nlpDir;
+}
+function existsDictOnDisk(dictId) {
+  if (!_nlpDir) return false;
+  return fs3.existsSync(path3.join(_nlpDir, `${dictId}.dict`));
+}
+function tryLoadDictFromDisk(dictId) {
+  if (!_nlpDir) return null;
+  const dictPath = path3.join(_nlpDir, `${dictId}.dict`);
+  if (!fs3.existsSync(dictPath)) return null;
+  try {
+    return fs3.readFileSync(dictPath);
+  } catch {
+    return null;
+  }
+}
+function getJieba(dictType = "default") {
+  const effectiveType = dictType === "default" ? "zh-CN" : dictType;
+  const cached = jiebaInstances.get(effectiveType);
+  if (cached) {
+    if (cached.fromDiskDict === existsDictOnDisk(effectiveType)) return cached.instance;
+    jiebaInstances.delete(effectiveType);
+    console.error(`[NLP] jieba cache invalidated (dict ${cached.fromDiskDict ? "removed" : "added"}): ${effectiveType}`);
+  }
+  try {
+    const { Jieba } = require2("@node-rs/jieba");
+    let instance;
+    const diskDict = tryLoadDictFromDisk(effectiveType);
+    if (diskDict) {
+      instance = Jieba.withDict(diskDict);
+      console.error(`[NLP] jieba dict loaded: ${effectiveType} (${diskDict.length} bytes)`);
+    } else {
+      instance = new Jieba();
+      console.warn(`[NLP] jieba dict missing: ${effectiveType}, fallback to built-in tokenizer`);
+    }
+    jiebaInstances.set(effectiveType, { instance, fromDiskDict: !!diskDict });
+    return instance;
+  } catch (error) {
+    console.error(`[NLP] Failed to load jieba module (dict=${effectiveType}):`, error);
+    throw new Error(`jieba \u6A21\u5757\u52A0\u8F7D\u5931\u8D25 (${effectiveType})`);
+  }
+}
+function clearJiebaInstance(dictType) {
+  jiebaInstances.delete(dictType);
+  console.error(`[NLP] jieba instance cleared: ${dictType}`);
+}
+function segmentChinese(text, options = {}) {
+  const { posFilterMode = "meaningful", customPosTags, dictType = "default" } = options;
+  const cleaned = cleanText(text);
+  if (!cleaned) return [];
+  try {
+    const jieba = getJieba(dictType);
+    if (posFilterMode === "all") return jieba.cut(cleaned, false);
+    const tagged = jieba.tag(cleaned);
+    const allowedTags = posFilterMode === "custom" && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS;
+    return tagged.filter((item) => allowedTags.has(item.tag)).map((item) => item.word);
+  } catch (error) {
+    console.error("[NLP] Chinese segmentation failed:", error);
+    try {
+      return getJieba("default").cut(cleanText(text), false);
+    } catch {
+      return cleaned.split("");
+    }
+  }
+}
+function segmentEnglish(text) {
+  const cleaned = cleanText(text);
+  if (!cleaned) return [];
+  try {
+    const segmenter = new Intl.Segmenter("en", { granularity: "word" });
+    return [...segmenter.segment(cleaned)].filter((s) => s.isWordLike).map((s) => s.segment.toLowerCase());
+  } catch {
+    return cleaned.toLowerCase().split(/\s+/).filter((w) => w.length > 0);
+  }
+}
+function segmentJapanese(text) {
+  const cleaned = cleanText(text);
+  if (!cleaned) return [];
+  try {
+    const segmenter = new Intl.Segmenter("ja", { granularity: "word" });
+    return [...segmenter.segment(cleaned)].filter((s) => s.isWordLike).map((s) => s.segment);
+  } catch {
+    return cleaned.split("").filter((ch) => ch.trim().length > 0);
+  }
+}
+function segment(text, locale, options = {}) {
+  const {
+    minLength,
+    posFilterMode = "meaningful",
+    customPosTags,
+    enableStopwords = true,
+    dictType = "default"
+  } = options;
+  const isChinese = locale.startsWith("zh");
+  const isJapanese = locale === "ja-JP";
+  const defaultMinLength = isChinese || isJapanese ? 2 : 3;
+  const effectiveMinLength = minLength ?? defaultMinLength;
+  let words;
+  if (isChinese) {
+    words = segmentChinese(text, { posFilterMode, customPosTags, dictType });
+  } else if (isJapanese) {
+    words = segmentJapanese(text);
+  } else {
+    words = segmentEnglish(text);
+  }
+  return words.filter((word) => isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword));
+}
+function batchSegmentWithFrequency(texts, locale, options = {}) {
+  const {
+    minLength,
+    minCount = 2,
+    topN = 100,
+    posFilterMode,
+    customPosTags,
+    enableStopwords,
+    dictType,
+    excludeWords
+  } = options;
+  const wordFrequency = /* @__PURE__ */ new Map();
+  const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null;
+  for (const text of texts) {
+    const words = segment(text, locale, { minLength, posFilterMode, customPosTags, enableStopwords, dictType });
+    for (const word of words) {
+      if (excludeSet && excludeSet.has(word.toLowerCase())) continue;
+      wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1);
+    }
+  }
+  const filtered = /* @__PURE__ */ new Map();
+  let totalWords = 0;
+  for (const [word, count] of wordFrequency) {
+    if (count >= minCount) {
+      filtered.set(word, count);
+      totalWords += count;
+    }
+  }
+  const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN);
+  return { words: new Map(sorted), uniqueWords: filtered.size, totalWords };
+}
+function batchSegmentChineseWithStats(texts, locale, options = {}) {
+  const {
+    minLength,
+    minCount = 2,
+    topN = 100,
+    posFilterMode = "meaningful",
+    customPosTags,
+    enableStopwords = true,
+    dictType = "default",
+    excludeWords
+  } = options;
+  const effectiveMinLength = minLength ?? 2;
+  const allowedTags = posFilterMode === "custom" && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS;
+  const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null;
+  const wordFrequency = /* @__PURE__ */ new Map();
+  const posTagStats = /* @__PURE__ */ new Map();
+  try {
+    const jieba = getJieba(dictType);
+    for (const text of texts) {
+      const cleaned = cleanText(text);
+      if (!cleaned) continue;
+      for (const { tag, word } of jieba.tag(cleaned)) {
+        if (!isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword)) continue;
+        posTagStats.set(tag, (posTagStats.get(tag) || 0) + 1);
+        if (!allowedTags.has(tag)) continue;
+        if (excludeSet && excludeSet.has(word.toLowerCase())) continue;
+        wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1);
+      }
+    }
+  } catch (error) {
+    console.error("[NLP] Chinese single-pass segmentation failed:", error);
+  }
+  const filtered = /* @__PURE__ */ new Map();
+  let totalWords = 0;
+  for (const [word, count] of wordFrequency) {
+    if (count >= minCount) {
+      filtered.set(word, count);
+      totalWords += count;
+    }
+  }
+  const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN);
+  return { words: new Map(sorted), uniqueWords: filtered.size, totalWords, posTagStats };
+}
+function collectPosTagStats(texts, minWordLength = 2, enableStopwords = true, dictType = "default") {
+  const posStats = /* @__PURE__ */ new Map();
+  try {
+    const jieba = getJieba(dictType);
+    for (const text of texts) {
+      const cleaned = cleanText(text);
+      if (!cleaned) continue;
+      const tagged = jieba.tag(cleaned);
+      for (const item of tagged) {
+        if (!isValidWord(item.word, "zh-CN", minWordLength, enableStopwords, isStopword)) continue;
+        posStats.set(item.tag, (posStats.get(item.tag) || 0) + 1);
+      }
+    }
+  } catch (error) {
+    console.error("[NLP] Failed to collect POS stats:", error);
+  }
+  return posStats;
+}
+function getPosTagDefinitions() {
+  return POS_TAG_DEFINITIONS;
+}
+
 // ../../packages/node-runtime/src/jieba-nlp-provider.ts
 function createJiebaNlpProvider(dictType = "default") {
   return {
@@ -567,9 +784,233 @@ function createJiebaNlpProvider(dictType = "default") {
   };
 }
 
+// ../../packages/node-runtime/src/nlp/word-frequency.ts
+function buildMessageQuery(timeFilter, memberId, excludeKeywords) {
+  const conditions = [];
+  const params = [];
+  if (timeFilter?.startTs !== void 0) {
+    conditions.push("msg.ts >= ?");
+    params.push(timeFilter.startTs);
+  }
+  if (timeFilter?.endTs !== void 0) {
+    conditions.push("msg.ts <= ?");
+    params.push(timeFilter.endTs);
+  }
+  if (memberId !== void 0 && memberId !== null) {
+    conditions.push("msg.sender_id = ?");
+    params.push(memberId);
+  }
+  if (excludeKeywords && excludeKeywords.length > 0) {
+    const exclude = buildExcludeKeywordsConditions(excludeKeywords);
+    conditions.push(...exclude.conditions);
+    params.push(...exclude.params);
+  }
+  conditions.push("COALESCE(m.account_name, '') != '\u7CFB\u7EDF\u6D88\u606F'");
+  conditions.push("msg.type = 0");
+  conditions.push("msg.content IS NOT NULL");
+  conditions.push("TRIM(msg.content) != ''");
+  return {
+    clause: ` WHERE ${conditions.join(" AND ")}`,
+    params
+  };
+}
+function computeWordFrequency(db, params) {
+  const {
+    locale,
+    timeFilter,
+    memberId,
+    topN = 100,
+    minWordLength,
+    minCount = 2,
+    posFilterMode = "meaningful",
+    customPosTags,
+    enableStopwords = true,
+    dictType = "default",
+    excludeWords,
+    excludeKeywords
+  } = params;
+  const { clause, params: filterParams } = buildMessageQuery(timeFilter, memberId, excludeKeywords);
+  const messages = db.prepare(`SELECT msg.content FROM message msg JOIN member m ON msg.sender_id = m.id${clause}`).all(...filterParams);
+  if (messages.length === 0) {
+    return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 };
+  }
+  const texts = messages.map((m) => m.content);
+  const segmentOptions = {
+    minLength: minWordLength,
+    minCount,
+    topN,
+    posFilterMode,
+    customPosTags,
+    enableStopwords,
+    dictType,
+    excludeWords
+  };
+  let posTagStats;
+  let result;
+  if (locale.startsWith("zh") && posFilterMode !== "all") {
+    const combined = batchSegmentChineseWithStats(texts, locale, segmentOptions);
+    result = { words: combined.words, uniqueWords: combined.uniqueWords, totalWords: combined.totalWords };
+    posTagStats = [...combined.posTagStats.entries()].map(([tag, count]) => ({ tag, count }));
+  } else {
+    if (locale.startsWith("zh")) {
+      const posStatsMap = collectPosTagStats(texts, minWordLength ?? 2, enableStopwords, dictType);
+      posTagStats = [...posStatsMap.entries()].map(([tag, count]) => ({ tag, count }));
+    }
+    result = batchSegmentWithFrequency(texts, locale, segmentOptions);
+  }
+  let topNTotalWords = 0;
+  for (const count of result.words.values()) topNTotalWords += count;
+  const words = [...result.words.entries()].map(([word, count]) => ({
+    word,
+    count,
+    percentage: topNTotalWords > 0 ? Math.round(count / topNTotalWords * 1e4) / 100 : 0
+  }));
+  return {
+    words,
+    totalWords: result.totalWords,
+    totalMessages: messages.length,
+    uniqueWords: result.uniqueWords,
+    posTagStats
+  };
+}
+function segmentText(text, locale, minLength) {
+  return segment(text, locale, { minLength });
+}
+
+// ../../packages/node-runtime/src/nlp/dict-manager.ts
+import * as fs4 from "fs";
+import * as path4 from "path";
+import { createHash } from "crypto";
+var DICT_DOWNLOAD_URL_BASE = "https://chatlab.fun/assets/nlp";
+var DICT_SHA256 = {
+  "zh-CN": "139519822fe8ab9e10d9d07e68ea0451045380aedaf54ecc51e2a28c6b42a13f",
+  "zh-TW": "a63ec7e388f16f1b486dcd948a9f1a3b492be5d9b6bdab786a95e59966786dfd"
+};
+var AVAILABLE_DICTS = [
+  { id: "zh-CN", label: "\u7B80\u4F53\u4E2D\u6587", locale: "zh-CN" },
+  { id: "zh-TW", label: "\u7E41\u9AD4\u4E2D\u6587", locale: "zh-TW" }
+];
+function getDictFilePath(nlpDir, dictId) {
+  return path4.join(nlpDir, `${dictId}.dict`);
+}
+function isDictDownloaded(nlpDir, dictId) {
+  return fs4.existsSync(getDictFilePath(nlpDir, dictId));
+}
+function getDictList(nlpDir) {
+  return AVAILABLE_DICTS.map((d) => {
+    const filePath = getDictFilePath(nlpDir, d.id);
+    const downloaded = fs4.existsSync(filePath);
+    let fileSize;
+    if (downloaded) {
+      try {
+        fileSize = fs4.statSync(filePath).size;
+      } catch {
+      }
+    }
+    return { ...d, downloaded, fileSize };
+  });
+}
+function loadDictBuffer(nlpDir, dictId) {
+  const filePath = getDictFilePath(nlpDir, dictId);
+  if (!fs4.existsSync(filePath)) return null;
+  try {
+    return fs4.readFileSync(filePath);
+  } catch (error) {
+    console.error(`[NLP DictManager] Failed to read dict file: ${filePath}`, error);
+    return null;
+  }
+}
+async function downloadDict(nlpDir, dictId, onProgress) {
+  if (!AVAILABLE_DICTS.find((d) => d.id === dictId)) {
+    return { success: false, error: `Unknown dict: ${dictId}` };
+  }
+  fs4.mkdirSync(nlpDir, { recursive: true });
+  const url = `${DICT_DOWNLOAD_URL_BASE}/${dictId}.dict`;
+  const filePath = getDictFilePath(nlpDir, dictId);
+  const tmpPath = filePath + ".tmp";
+  try {
+    const response = await fetch(url, { signal: AbortSignal.timeout(12e4) });
+    if (!response.ok) throw new Error(`Download failed: HTTP ${response.status}`);
+    const total = Number(response.headers.get("content-length") || 0);
+    let buffer;
+    if (!response.body) {
+      buffer = Buffer.from(await response.arrayBuffer());
+    } else {
+      const reader = response.body.getReader();
+      const chunks = [];
+      let loaded = 0;
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        const chunk = Buffer.from(value);
+        chunks.push(chunk);
+        loaded += chunk.length;
+        if (total > 0 && onProgress) onProgress(Math.round(loaded / total * 100));
+      }
+      buffer = Buffer.concat(chunks);
+    }
+    const MIN_DICT_SIZE = 1e6;
+    if (buffer.length < MIN_DICT_SIZE) {
+      return { success: false, error: `Downloaded file is invalid (${buffer.length} bytes)` };
+    }
+    const head = buffer.subarray(0, 50).toString("utf-8").trim();
+    if (head.startsWith("<!") || head.startsWith("<html")) {
+      return { success: false, error: "Downloaded file is HTML, not a dictionary file" };
+    }
+    const expectedSha256 = DICT_SHA256[dictId];
+    if (!expectedSha256) {
+      return { success: false, error: `Missing SHA256 checksum for dict: ${dictId}` };
+    }
+    const actualSha256 = createHash("sha256").update(buffer).digest("hex");
+    if (actualSha256 !== expectedSha256) {
+      return { success: false, error: "Dictionary integrity check failed (SHA256 mismatch)" };
+    }
+    fs4.writeFileSync(tmpPath, buffer);
+    if (fs4.existsSync(filePath)) fs4.unlinkSync(filePath);
+    fs4.renameSync(tmpPath, filePath);
+    clearJiebaInstance(dictId);
+    console.log(`[NLP DictManager] Dict downloaded: ${dictId} (${fs4.statSync(filePath).size} bytes)`);
+    return { success: true };
+  } catch (error) {
+    if (fs4.existsSync(tmpPath)) {
+      try {
+        fs4.unlinkSync(tmpPath);
+      } catch {
+      }
+    }
+    const msg = error instanceof Error ? error.message : String(error);
+    console.error(`[NLP DictManager] Download failed for ${dictId}:`, msg);
+    return { success: false, error: msg };
+  }
+}
+function deleteDict(nlpDir, dictId) {
+  const filePath = getDictFilePath(nlpDir, dictId);
+  if (!fs4.existsSync(filePath)) return { success: true };
+  try {
+    fs4.unlinkSync(filePath);
+    clearJiebaInstance(dictId);
+    console.log(`[NLP DictManager] Dict deleted: ${dictId}`);
+    return { success: true };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    console.error(`[NLP DictManager] Delete failed for ${dictId}:`, msg);
+    return { success: false, error: msg };
+  }
+}
+async function ensureDefaultDict(nlpDir) {
+  if (isDictDownloaded(nlpDir, "zh-CN")) return;
+  console.log("[NLP DictManager] zh-CN dict not found, starting background download...");
+  const result = await downloadDict(nlpDir, "zh-CN");
+  if (result.success) {
+    console.log("[NLP DictManager] zh-CN dict auto-downloaded successfully");
+  } else {
+    console.warn("[NLP DictManager] zh-CN dict auto-download failed:", result.error);
+  }
+}
+
 // ../../packages/node-runtime/src/preferences.ts
-import * as fs3 from "fs";
-import * as path3 from "path";
+import * as fs5 from "fs";
+import * as path5 from "path";
 var DEFAULTS = {
   pinnedSessionIds: [],
   assistantUpgradeSkippedVersions: {},
@@ -592,13 +1033,7 @@ var DEFAULTS = {
     enableAutoSkill: true,
     chartAutoMode: "suggest",
     searchContextBefore: 2,
-    searchContextAfter: 2,
-    contextCompression: {
-      enabled: true,
-      tokenThresholdPercent: 75,
-      bufferSizePercent: 20,
-      maxToolResultPercent: 50
-    }
+    searchContextAfter: 2
   },
   customKeywordTemplates: [],
   deletedPresetTemplateIds: [],
@@ -616,13 +1051,13 @@ var PreferencesManager = class {
   filePath;
   cache = null;
   constructor(systemDir) {
-    this.filePath = path3.join(systemDir, "preferences.json");
+    this.filePath = path5.join(systemDir, "preferences.json");
   }
   load() {
     if (this.cache) return this.cache;
     try {
-      if (fs3.existsSync(this.filePath)) {
-        const raw = fs3.readFileSync(this.filePath, "utf-8");
+      if (fs5.existsSync(this.filePath)) {
+        const raw = fs5.readFileSync(this.filePath, "utf-8");
         const parsed = JSON.parse(raw);
         const migrated = this.migrateLegacyDesensitizeRules(parsed, raw);
         this.cache = this.mergeDefaults(migrated.preferences);
@@ -668,14 +1103,7 @@ var PreferencesManager = class {
       pinnedSessionIds: partial.pinnedSessionIds ?? DEFAULTS.pinnedSessionIds,
       assistantUpgradeSkippedVersions: this.normalizePositiveIntegerMap(partial.assistantUpgradeSkippedVersions),
       aiPreprocessConfig: partial.aiPreprocessConfig ? this.normalizeAiPreprocessConfig(partial.aiPreprocessConfig) : { ...DEFAULTS.aiPreprocessConfig },
-      aiGlobalSettings: partial.aiGlobalSettings ? {
-        ...DEFAULTS.aiGlobalSettings,
-        ...partial.aiGlobalSettings,
-        contextCompression: {
-          ...DEFAULTS.aiGlobalSettings.contextCompression,
-          ...partial.aiGlobalSettings.contextCompression ?? {}
-        }
-      } : { ...DEFAULTS.aiGlobalSettings },
+      aiGlobalSettings: this.normalizeAiGlobalSettings(partial.aiGlobalSettings),
       customKeywordTemplates: partial.customKeywordTemplates ?? DEFAULTS.customKeywordTemplates,
       deletedPresetTemplateIds: partial.deletedPresetTemplateIds ?? DEFAULTS.deletedPresetTemplateIds,
       wordFilter: partial.wordFilter ? { ...DEFAULTS.wordFilter, ...partial.wordFilter } : { ...DEFAULTS.wordFilter },
@@ -685,6 +1113,11 @@ var PreferencesManager = class {
       ownerPromptDismissedSessionIds: partial.ownerPromptDismissedSessionIds ?? DEFAULTS.ownerPromptDismissedSessionIds
     };
   }
+  normalizeAiGlobalSettings(partial) {
+    const normalized = { ...partial ?? {} };
+    delete normalized.contextCompression;
+    return { ...DEFAULTS.aiGlobalSettings, ...normalized };
+  }
   normalizeAiPreprocessConfig(partial) {
     const defaults = DEFAULTS.aiPreprocessConfig;
     const rules = Array.isArray(partial?.desensitizeRules) ? partial.desensitizeRules : [];
@@ -748,21 +1181,21 @@ var PreferencesManager = class {
     };
   }
   backupLegacyPreferences(rawContent) {
-    const backupDir = path3.join(path3.dirname(this.filePath), "backups");
-    if (!fs3.existsSync(backupDir)) {
-      fs3.mkdirSync(backupDir, { recursive: true });
+    const backupDir = path5.join(path5.dirname(this.filePath), "backups");
+    if (!fs5.existsSync(backupDir)) {
+      fs5.mkdirSync(backupDir, { recursive: true });
     }
     const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-    const backupPath = path3.join(backupDir, `preferences-pre-desensitize-groups-${timestamp}.json`);
-    fs3.writeFileSync(backupPath, rawContent, "utf-8");
+    const backupPath = path5.join(backupDir, `preferences-pre-desensitize-groups-${timestamp}.json`);
+    fs5.writeFileSync(backupPath, rawContent, "utf-8");
     console.info(`[Preferences] Backed up legacy desensitize rules to ${backupPath}`);
   }
   writePreferences(preferences) {
-    const dir = path3.dirname(this.filePath);
-    if (!fs3.existsSync(dir)) {
-      fs3.mkdirSync(dir, { recursive: true });
+    const dir = path5.dirname(this.filePath);
+    if (!fs5.existsSync(dir)) {
+      fs5.mkdirSync(dir, { recursive: true });
     }
-    fs3.writeFileSync(this.filePath, JSON.stringify(preferences, null, 2), "utf-8");
+    fs5.writeFileSync(this.filePath, JSON.stringify(preferences, null, 2), "utf-8");
   }
   /**
    * Deep merge: arrays are replaced (not concatenated), objects are merged.
@@ -782,26 +1215,146 @@ var PreferencesManager = class {
 };
 
 // ../../packages/node-runtime/src/merger/index.ts
+function createMessageScope(meta, members, sourceIndex) {
+  const platform = meta?.platform || "unknown";
+  const type = meta?.type || "unknown";
+  if (meta?.groupId) return JSON.stringify([platform, type, "group", meta.groupId]);
+  if (type === "group") return JSON.stringify([platform, type, "source", sourceIndex]);
+  const memberIds = [...new Set(members.map((member) => member.platformId))].sort();
+  if (memberIds.length > 0) {
+    if (type === "private") {
+      if (memberIds.length >= 2) return JSON.stringify([platform, type, "members", memberIds]);
+    } else {
+      return JSON.stringify([platform, type, "name-members", meta?.name || "", memberIds]);
+    }
+  }
+  return JSON.stringify([platform, type, "source", sourceIndex]);
+}
+function createMessageBridgeFamily(meta) {
+  if (!meta || meta.groupId || meta.type !== "group" && meta.type !== "private") return void 0;
+  return JSON.stringify([meta.platform || "unknown", meta.type]);
+}
+function createMessageIdNormalizer(scopes) {
+  const uniqueScopes = [...new Set(scopes)];
+  const scopeIndexes = new Map(uniqueScopes.map((scope, index) => [scope, index]));
+  const namespaceMessageIds = uniqueScopes.length > 1;
+  return (id, scope) => {
+    if (!id || !namespaceMessageIds) return id;
+    return `__chatlab_message_scope__${scopeIndexes.get(scope) ?? 0}__${encodeURIComponent(id)}`;
+  };
+}
+function createMessageScopeResolver(contexts) {
+  const contextBySource = /* @__PURE__ */ new Map();
+  const parents = /* @__PURE__ */ new Map();
+  const firstSourceBySignature = /* @__PURE__ */ new Map();
+  for (const context of contexts) {
+    contextBySource.set(context.sourceIndex, context);
+    parents.set(context.sourceIndex, context.sourceIndex);
+  }
+  const find = (sourceIndex) => {
+    const parent = parents.get(sourceIndex) ?? sourceIndex;
+    if (parent === sourceIndex) return sourceIndex;
+    const root = find(parent);
+    parents.set(sourceIndex, root);
+    return root;
+  };
+  const union = (firstSourceIndex, secondSourceIndex) => {
+    const firstRoot = find(firstSourceIndex);
+    const secondRoot = find(secondSourceIndex);
+    if (firstRoot === secondRoot) return;
+    const root = Math.min(firstRoot, secondRoot);
+    parents.set(firstRoot === root ? secondRoot : firstRoot, root);
+  };
+  return {
+    addMessage(message, context) {
+      if (!context.messageBridgeFamily || !message.platformMessageId) return;
+      const fallbackKey = generateFallbackMessageKey({
+        timestamp: message.timestamp,
+        senderPlatformId: message.senderPlatformId,
+        type: message.type,
+        content: message.content ?? null,
+        replyToMessageId: message.replyToMessageId
+      });
+      const signature = JSON.stringify([context.messageBridgeFamily, message.platformMessageId, fallbackKey]);
+      const firstSourceIndex = firstSourceBySignature.get(signature);
+      if (firstSourceIndex === void 0) {
+        firstSourceBySignature.set(signature, context.sourceIndex);
+        return;
+      }
+      union(firstSourceIndex, context.sourceIndex);
+    },
+    complete() {
+      firstSourceBySignature.clear();
+    },
+    resolve(sourceIndex) {
+      const rootContext = contextBySource.get(find(sourceIndex));
+      return rootContext?.messageScope ?? JSON.stringify(["unknown", "source", sourceIndex]);
+    }
+  };
+}
 function checkConflictsFromSources(dataSources) {
-  const allMessages = [];
-  for (const { source, filename } of dataSources) {
+  const preparedSources = dataSources.map(({ source, filename }, sourceIndex) => {
     const meta = source.getMeta();
-    const platform = meta?.platform || "unknown";
+    const members = source.getMembers();
+    return {
+      source,
+      filename,
+      sourceIndex,
+      platform: meta?.platform || "unknown",
+      messageScope: createMessageScope(meta, members, sourceIndex),
+      messageBridgeFamily: createMessageBridgeFamily(meta)
+    };
+  });
+  const allMessages = [];
+  for (const { source, filename, sourceIndex, platform, messageScope, messageBridgeFamily } of preparedSources) {
     source.streamMessages(1e4, (messages) => {
       for (const msg of messages) {
-        allMessages.push({ msg, source: filename, platform });
+        allMessages.push({ msg, source: filename, sourceIndex, platform, messageScope, messageBridgeFamily });
       }
     });
   }
-  return detectConflictsInMessages(allMessages);
+  const result = detectConflictsInMessages(allMessages);
+  const collidingIds = getCollidingPlatformIdsFromMessages(allMessages);
+  const scopeResolver = createMessageScopeResolver(preparedSources);
+  for (const message of allMessages) scopeResolver.addMessage(message.msg, message);
+  scopeResolver.complete();
+  const normalizeMessageId = createMessageIdNormalizer(
+    preparedSources.map(({ sourceIndex }) => scopeResolver.resolve(sourceIndex))
+  );
+  const dedupState = createMessageDedupState();
+  let totalMessages = 0;
+  for (const { msg, sourceIndex, platform } of allMessages) {
+    const messageScope = scopeResolver.resolve(sourceIndex);
+    const senderPlatformId = normalizePlatformId(msg.senderPlatformId, platform, collidingIds);
+    const duplicate = registerMessageAndCheckDuplicate(
+      {
+        platformMessageId: normalizeMessageId(msg.platformMessageId, messageScope),
+        timestamp: msg.timestamp,
+        senderPlatformId,
+        type: msg.type,
+        content: msg.content ?? null,
+        replyToMessageId: normalizeMessageId(msg.replyToMessageId, messageScope)
+      },
+      dedupState
+    );
+    if (!duplicate) totalMessages++;
+  }
+  return { ...result, totalMessages };
 }
 function buildMergedOutput(dataSources, outputName) {
-  const metas = dataSources.map(({ source, filename }) => ({
-    meta: source.getMeta(),
-    members: source.getMembers(),
-    filename,
-    source
-  }));
+  const metas = dataSources.map(({ source, filename }, sourceIndex) => {
+    const meta = source.getMeta();
+    const members = source.getMembers();
+    return {
+      meta,
+      members,
+      filename,
+      source,
+      sourceIndex,
+      messageScope: createMessageScope(meta, members, sourceIndex),
+      messageBridgeFamily: createMessageBridgeFamily(meta)
+    };
+  });
   const collidingIds = getCollidingPlatformIds(
     metas.map(({ meta, members }) => ({
       platform: meta?.platform || "unknown",
@@ -815,24 +1368,63 @@ function buildMergedOutput(dataSources, outputName) {
     })),
     collidingIds
   );
-  const seenKeys = /* @__PURE__ */ new Set();
+  const uniquePlatforms = [...new Set(metas.map(({ meta }) => meta?.platform || "unknown"))];
+  const scopeResolver = createMessageScopeResolver(metas);
+  for (const context of metas) {
+    if (!context.messageBridgeFamily) continue;
+    context.source.streamMessages(1e4, (messages) => {
+      for (const message of messages) scopeResolver.addMessage(message, context);
+    });
+  }
+  scopeResolver.complete();
+  const normalizeMessageId = createMessageIdNormalizer(
+    metas.map(({ sourceIndex }) => scopeResolver.resolve(sourceIndex))
+  );
+  const dedupState = createMessageDedupState();
+  const fallbackOnlyMessageIndexes = /* @__PURE__ */ new Map();
   const mergedMessages = [];
-  for (const { source, meta } of metas) {
+  for (const { source, meta, sourceIndex } of metas) {
     const platform2 = meta?.platform || "unknown";
+    const messageScope = scopeResolver.resolve(sourceIndex);
     source.streamMessages(1e4, (messages) => {
       for (const msg of messages) {
         const nid = normalizePlatformId(msg.senderPlatformId, platform2, collidingIds);
-        const key = generateMessageKey(msg.timestamp, nid, msg.content ?? null);
-        if (seenKeys.has(key)) continue;
-        seenKeys.add(key);
+        const platformMessageId = normalizeMessageId(msg.platformMessageId, messageScope);
+        const replyToMessageId = normalizeMessageId(msg.replyToMessageId, messageScope);
+        const dedupMessage = {
+          platformMessageId,
+          timestamp: msg.timestamp,
+          senderPlatformId: nid,
+          type: msg.type,
+          content: msg.content ?? null,
+          replyToMessageId
+        };
+        const fallbackKey = generateFallbackMessageKey(dedupMessage);
+        const shouldBackfillId = Boolean(
+          platformMessageId && !dedupState.platformMessageIds.has(platformMessageId) && dedupState.fallbackOnlyKeys.has(fallbackKey)
+        );
+        if (registerMessageAndCheckDuplicate(dedupMessage, dedupState)) {
+          if (shouldBackfillId) {
+            const retainedIndex2 = fallbackOnlyMessageIndexes.get(fallbackKey);
+            if (retainedIndex2 !== void 0) {
+              mergedMessages[retainedIndex2].platformMessageId = platformMessageId;
+              fallbackOnlyMessageIndexes.delete(fallbackKey);
+            }
+          }
+          continue;
+        }
+        const retainedIndex = mergedMessages.length;
         mergedMessages.push({
+          platformMessageId,
           sender: nid,
           accountName: msg.senderAccountName,
           groupNickname: msg.senderGroupNickname,
           timestamp: msg.timestamp,
           type: msg.type,
-          content: msg.content
+          content: msg.content,
+          replyToMessageId
         });
+        if (!platformMessageId) fallbackOnlyMessageIndexes.set(fallbackKey, retainedIndex);
       }
     });
   }
@@ -842,11 +1434,13 @@ function buildMergedOutput(dataSources, outputName) {
     platform: source.getMeta()?.platform || "unknown",
     messageCount: source.getMessageCount()
   }));
-  const uniquePlatforms = [...new Set(metas.map(({ meta }) => meta?.platform || "unknown"))];
   const platform = uniquePlatforms.length === 1 ? uniquePlatforms[0] : "mixed";
   const groupIds = new Set(metas.map(({ meta }) => meta?.groupId).filter(Boolean));
   const groupId = groupIds.size === 1 ? metas.find(({ meta }) => meta?.groupId)?.meta?.groupId : void 0;
   const groupAvatar = groupId ? metas.filter(({ meta }) => meta?.groupId === groupId).pop()?.meta?.groupAvatar : void 0;
+  const ownerIds = new Set(metas.map(({ meta }) => meta?.ownerId).filter((id) => Boolean(id)));
+  const ownerCandidate = uniquePlatforms.length === 1 && ownerIds.size === 1 ? [...ownerIds][0] : void 0;
+  const ownerId = ownerCandidate && memberMap.has(ownerCandidate) ? ownerCandidate : void 0;
   const chatLabData = {
     chatlab: {
       version: CHATLAB_FORMAT_VERSION,
@@ -860,7 +1454,8 @@ function buildMergedOutput(dataSources, outputName) {
       type: metas[0]?.meta?.type || "group",
       sources,
       groupId,
-      groupAvatar
+      groupAvatar,
+      ownerId
     },
     members: Array.from(memberMap.values()),
     messages: mergedMessages
@@ -882,7 +1477,7 @@ function* serializeChatLabToJsonl(data) {
 }
 
 // ../../packages/node-runtime/src/services/analytics.ts
-import * as path4 from "path";
+import * as path6 from "path";
 import { randomUUID as randomUUID2 } from "crypto";
 var DEFAULT_DATA = {
   lastReportDate: null,
@@ -994,7 +1589,7 @@ var AnalyticsService = class {
   // Daily-active deduplication is intentionally process-local unless concurrent runtimes become supported.
   dailyActivePromise = null;
   constructor(systemDir, options) {
-    this.dataPath = path4.join(systemDir, "analytics.json");
+    this.dataPath = path6.join(systemDir, "analytics.json");
     this.options = {
       ...options,
       umami: options.umami ?? getUmamiConfigFromEnv()
@@ -1156,8 +1751,8 @@ var AnalyticsService = class {
 };
 
 // ../../packages/node-runtime/src/services/demo-import.ts
-import * as fs4 from "fs";
-import * as path5 from "path";
+import * as fs6 from "fs";
+import * as path7 from "path";
 var DEMO_BASE_URL = "https://chatlab.fun/assets/demo";
 var DEMO_FILES = ["demo-group.json", "demo-private-B-wukong.json"];
 async function importDemoSessions(options) {
@@ -1189,8 +1784,8 @@ async function importDemoSessions(options) {
     }
     const rebased = rebaseChatLabDemoDocuments(downloaded, options.now?.() ?? /* @__PURE__ */ new Date(), options.targetTimeZone);
     const localPaths = rebased.documents.map((document, index) => {
-      const localPath = path5.join(createdTempDir, DEMO_FILES[index]);
-      fs4.writeFileSync(localPath, document, "utf8");
+      const localPath = path7.join(createdTempDir, DEMO_FILES[index]);
+      fs6.writeFileSync(localPath, document, "utf8");
       return localPath;
     });
     appLogger.info("demo-import", "Demo timeline prepared", {
@@ -1274,7 +1869,6 @@ __export(semantic_index_exports, {
   composeChunkId: () => composeChunkId,
   computeChunkerConfigHash: () => computeChunkerConfigHash,
   computeDbPathHash: () => computeDbPathHash,
-  createChatDbFtsSearcher: () => createChatDbFtsSearcher,
   createChatDbMessageRangeReader: () => createChatDbMessageRangeReader,
   createChatDbMessageSource: () => createChatDbMessageSource,
   createEmbedder: () => createEmbedder,
@@ -1287,24 +1881,22 @@ __export(semantic_index_exports, {
   deriveParentId: () => deriveParentId,
   embedding: () => embedding_exports,
   estimateTokens: () => estimateTokens,
-  extractFtsKeywords: () => extractFtsKeywords,
   formatEvidenceMessages: () => formatEvidenceMessages,
-  hybridSearch: () => hybridSearch,
   isSemanticIndexConfigured: () => isSemanticIndexConfigured,
   isSemanticVoid: () => isSemanticVoid,
   parseParentBounds: () => parseParentBounds,
   persistSemanticIndexConfig: () => persistSemanticIndexConfig,
-  reciprocalRankFusion: () => reciprocalRankFusion,
   resolveModelId: () => resolveModelId,
   resolveSemanticIndexApiKeySet: () => resolveSemanticIndexApiKeySet,
   runWarmup: () => runWarmup,
+  semanticSearch: () => semanticSearch,
   snapshotPathProvider: () => snapshotPathProvider,
   vecTableName: () => vecTableName,
   vecTableSchema: () => vecTableSchema
 });
 
 // ../../packages/node-runtime/src/semantic-index/worker-client.ts
-import path6 from "path";
+import path8 from "path";
 
 // ../../packages/node-runtime/src/semantic-index/worker-thread-transport.ts
 import { Worker } from "worker_threads";
@@ -1686,7 +2278,7 @@ function createSemanticIndexWorkerClient(options) {
 }
 function createSemanticIndexWorkerRuntimeClient(options) {
   const configStore = new SemanticIndexConfigStore(
-    path6.join(options.pathProvider.getAiDataDir(), SEMANTIC_INDEX_CONFIG_FILE)
+    path8.join(options.pathProvider.getAiDataDir(), SEMANTIC_INDEX_CONFIG_FILE)
   );
   return createSemanticIndexWorkerClient({
     configStore,
@@ -1741,7 +2333,23 @@ export {
   NodePathProvider,
   getDefaultNodeUserDataDir,
   getSystemLogsDir,
+  initNlpDir,
+  getNlpDir,
+  getJieba,
+  clearJiebaInstance,
+  segment,
+  batchSegmentWithFrequency,
+  collectPosTagStats,
+  getPosTagDefinitions,
   createJiebaNlpProvider,
+  computeWordFrequency,
+  segmentText,
+  isDictDownloaded,
+  getDictList,
+  loadDictBuffer,
+  downloadDict,
+  deleteDict,
+  ensureDefaultDict,
   PreferencesManager,
   checkConflictsFromSources,
   buildMergedOutput,
@@ -1751,4 +2359,4 @@ export {
   createSemanticIndexWorkerRuntimeClient,
   semantic_index_exports
 };
-//# sourceMappingURL=chunk-2XCNTG4O.mjs.map
+//# sourceMappingURL=chunk-DC7IUDNV.mjs.map