chatlab-cli 0.29.0 → 0.30.0

package/dist/{chunk-QQK7MLCR.mjs → chunk-A2SANKG4.mjs}

@@ -9,7 +9,7 @@ import {
   createEmptyContactsDiagnostics,
   normalizeContactsTimeRangePreset,
   resolveContactsTimeRange
-} from "./chunk-D4WTE7LV.mjs";
+} from "./chunk-CNSYXRJP.mjs";
 import {
   PEOPLE_RELATIONSHIPS_ALGORITHM_VERSION,
   buildPeopleRelationshipsNeighborhoodGraph,
@@ -18,7 +18,7 @@ import {
   computePeopleRelationshipsSnapshot,
   normalizePeopleRelationshipsTimeRangePreset,
   resolvePeopleRelationshipsTimeRange
-} from "./chunk-2MZ6G4AC.mjs";
+} from "./chunk-MUPILWAZ.mjs";
 import {
   BUILTIN_MODELS,
   BUILTIN_PROVIDERS,
@@ -32,6 +32,7 @@ import {
   appLogger,
   batchSegmentChineseWithStats,
   batchSegmentWithFrequency,
+  buildExcludeKeywordsConditions,
   buildSessionInfo,
   clearJiebaInstance,
   clearSessionIndex,
@@ -79,7 +80,7 @@ import {
   updateMemberAliases,
   updateSessionOwnerId,
   writeAuthProfile
-} from "./chunk-WPI6YSAR.mjs";
+} from "./chunk-7BZLS2KW.mjs";
 import {
   __commonJS,
   __export,
@@ -6874,7 +6875,7 @@ async function streamParseFile(filePath, callbacks, formatId) {
 }
 
 // ../../packages/node-runtime/src/nlp/word-frequency.ts
-function buildMessageQuery(timeFilter, memberId) {
+function buildMessageQuery(timeFilter, memberId, excludeKeywords) {
   const conditions = [];
   const params = [];
   if (timeFilter?.startTs !== void 0) {
@@ -6889,6 +6890,11 @@ function buildMessageQuery(timeFilter, memberId) {
     conditions.push("msg.sender_id = ?");
     params.push(memberId);
   }
+  if (excludeKeywords && excludeKeywords.length > 0) {
+    const exclude = buildExcludeKeywordsConditions(excludeKeywords);
+    conditions.push(...exclude.conditions);
+    params.push(...exclude.params);
+  }
   conditions.push("COALESCE(m.account_name, '') != '\u7CFB\u7EDF\u6D88\u606F'");
   conditions.push("msg.type = 0");
   conditions.push("msg.content IS NOT NULL");
@@ -6910,9 +6916,10 @@ function computeWordFrequency(db, params) {
     customPosTags,
     enableStopwords = true,
     dictType = "default",
-    excludeWords
+    excludeWords,
+    excludeKeywords
   } = params;
-  const { clause, params: filterParams } = buildMessageQuery(timeFilter, memberId);
+  const { clause, params: filterParams } = buildMessageQuery(timeFilter, memberId, excludeKeywords);
   const messages = db.prepare(`SELECT msg.content FROM message msg JOIN member m ON msg.sender_id = m.id${clause}`).all(...filterParams);
   if (messages.length === 0) {
     return { words: [], totalWords: 0, totalMessages: 0, uniqueWords: 0 };
@@ -7254,7 +7261,13 @@ async function streamImportSingle(filePath, deps, formatFeature, formatOptions,
     `INSERT INTO meta (name, platform, type, imported_at, group_id, group_avatar, owner_id) VALUES (?, ?, ?, ?, ?, ?, ?)`
   );
   const insertMember = db.prepare(
-    `INSERT OR IGNORE INTO member (platform_id, account_name, group_nickname, avatar, roles) VALUES (?, ?, ?, ?, ?)`
+    `INSERT INTO member (platform_id, account_name, group_nickname, avatar, roles)
+     VALUES (?, ?, ?, ?, ?)
+     ON CONFLICT(platform_id) DO UPDATE SET
+       account_name = COALESCE(NULLIF(excluded.account_name, ''), account_name),
+       group_nickname = COALESCE(NULLIF(excluded.group_nickname, ''), group_nickname),
+       avatar = COALESCE(NULLIF(excluded.avatar, ''), avatar),
+       roles = CASE WHEN excluded.roles != '[]' THEN excluded.roles ELSE roles END`
   );
   const getMemberId = db.prepare(`SELECT id FROM member WHERE platform_id = ?`);
   const insertMessage = db.prepare(
@@ -10550,45 +10563,65 @@ var defaultLogger4 = {
   warn: () => {
   }
 };
+function emptyStats(count) {
+  return {
+    input: count,
+    output: count,
+    cleaned: 0,
+    blacklistRemoved: 0,
+    denoiseRemoved: 0,
+    mergeCombined: 0,
+    desensitizeRulesApplied: 0
+  };
+}
 function preprocessMessages(messages, config, logger = defaultLogger4) {
-  if (!config || !hasAnyEnabled(config)) return messages;
-  if (messages.length === 0) return messages;
-  const inputCount = messages.length;
+  return preprocessMessagesWithStats(messages, config, logger).messages;
+}
+function preprocessMessagesWithStats(messages, config, logger = defaultLogger4) {
+  if (!config || !hasAnyEnabled(config)) return { messages, stats: emptyStats(messages.length) };
+  if (messages.length === 0) return { messages, stats: emptyStats(0) };
+  const stats = emptyStats(messages.length);
   let result = [...messages];
   const applied = [];
   if (config.dataCleaning !== false) {
     const cleaned = applyDataCleaning(result);
     if (cleaned.changed > 0) {
       result = cleaned.messages;
+      stats.cleaned = cleaned.changed;
       applied.push(`dataCleaning: ${cleaned.changed} messages cleaned`);
     }
   }
   if (config.blacklistKeywords.length > 0) {
     const before = result.length;
     result = applyBlacklistFilter(result, config.blacklistKeywords);
+    stats.blacklistRemoved = before - result.length;
     applied.push(`blacklist: ${before} \u2192 ${result.length} (-${before - result.length})`);
   }
   if (config.denoise) {
     const before = result.length;
     result = applyDenoise(result);
+    stats.denoiseRemoved = before - result.length;
     applied.push(`denoise: ${before} \u2192 ${result.length} (-${before - result.length})`);
   }
   if (config.mergeConsecutive) {
     const before = result.length;
     result = applyMergeConsecutive(result, config.mergeWindowSeconds ?? MERGE_WINDOW_DEFAULT);
+    stats.mergeCombined = before - result.length;
     applied.push(`merge: ${before} \u2192 ${result.length} (-${before - result.length})`);
   }
   if (config.desensitize) {
     const enabledRules = (config.desensitizeRules || []).filter((r) => r.enabled);
     if (enabledRules.length > 0) {
       result = applyDesensitize(result, enabledRules, logger);
+      stats.desensitizeRulesApplied = enabledRules.length;
       applied.push(`desensitize: ${enabledRules.length} rules applied`);
     }
   }
-  logger.info("Preprocess", `Pipeline: ${inputCount} \u2192 ${result.length} messages`, {
+  stats.output = result.length;
+  logger.info("Preprocess", `Pipeline: ${stats.input} \u2192 ${result.length} messages`, {
     strategies: applied
   });
-  return result;
+  return { messages: result, stats };
 }
 function hasAnyEnabled(config) {
   return config.dataCleaning !== false || config.mergeConsecutive || config.blacklistKeywords.length > 0 || config.denoise || config.desensitize;
@@ -10683,9 +10716,16 @@ function applyMergeConsecutive(messages, windowSeconds) {
     const sameSender = isSameSender(current, msg);
     const withinWindow = Math.abs(msg.timestamp - current.timestamp) <= windowSeconds;
     if (sameSender && withinWindow) {
+      const mergedIds = [
+        ...current.mergedIds ?? (current.id != null ? [current.id] : []),
+        ...msg.mergedIds ?? (msg.id != null ? [msg.id] : [])
+      ];
       current = {
         ...current,
-        content: [current.content, msg.content].filter(Boolean).join("\n")
+        content: [current.content, msg.content].filter(Boolean).join("\n"),
+        // keep first message's id, track last id for [#start-end] range citations
+        mergedEndId: msg.id ?? current.mergedEndId,
+        ...mergedIds.length > 0 ? { mergedIds } : {}
       };
     } else {
       merged.push(current);
@@ -10706,15 +10746,29 @@ function applyDesensitize(messages, rules, logger) {
   if (compiledRules.length === 0) return messages;
   return messages.map((msg) => {
     if (!msg.content) return msg;
-    let content = msg.content;
-    for (const { regex, replacement } of compiledRules) {
-      regex.lastIndex = 0;
-      content = content.replace(regex, replacement);
-    }
+    const content = applyCompiledRules(msg.content, compiledRules);
     if (content === msg.content) return msg;
     return { ...msg, content };
   });
 }
+function applyCompiledRules(text, compiledRules) {
+  let result = text;
+  for (const { regex, replacement } of compiledRules) {
+    regex.lastIndex = 0;
+    result = result.replace(regex, replacement);
+  }
+  return result;
+}
+function desensitizeText(text, rules, logger = defaultLogger4) {
+  const enabledRules = rules.filter((r) => r.enabled);
+  if (enabledRules.length === 0 || !text) return text;
+  return applyCompiledRules(text, compileRules(enabledRules, logger));
+}
+function matchesBlacklist(text, keywords) {
+  if (keywords.length === 0 || !text) return false;
+  const lower = text.toLowerCase();
+  return keywords.some((kw) => lower.includes(kw.toLowerCase()));
+}
 var regexCache = /* @__PURE__ */ new Map();
 function compileRules(rules, logger) {
   const result = [];
@@ -11040,14 +11094,26 @@ function t(key, locale) {
   return "";
 }
 var MAX_MESSAGE_CONTENT_LENGTH = 200;
-function formatMessageCompact(msg, locale) {
+function formatMessageCompact(msg, locale, options2) {
   const localeStr = isChineseLocale(locale) ? "zh-CN" : "en-US";
   const time = new Date(msg.timestamp * 1e3).toLocaleString(localeStr);
   let content = msg.content || t("noContent", locale);
-  if (content.length > MAX_MESSAGE_CONTENT_LENGTH) {
-    content = content.slice(0, MAX_MESSAGE_CONTENT_LENGTH) + "...";
+  const maxLength = options2?.maxContentLength ?? MAX_MESSAGE_CONTENT_LENGTH;
+  if (maxLength > 0 && content.length > maxLength) {
+    content = content.slice(0, maxLength) + "...";
+  }
+  let idPrefix = "";
+  if (options2?.includeMessageId && msg.id != null) {
+    const range = msg.mergedEndId != null && msg.mergedEndId !== msg.id ? `${msg.id}-${msg.mergedEndId}` : `${msg.id}`;
+    const hitMark = isHitMessage(msg, options2.hitIds) ? "*" : "";
+    idPrefix = `[#${range}${hitMark}] `;
   }
-  return `${time} ${msg.senderName}: ${content}`;
+  return `${idPrefix}${time} ${msg.senderName}: ${content}`;
+}
+function isHitMessage(msg, hitIds) {
+  if (!hitIds) return false;
+  if (msg.id != null && hitIds.has(msg.id)) return true;
+  return msg.mergedIds?.some((id) => hitIds.has(id)) ?? false;
 }
 function formatTimeRange(timeFilter, locale) {
   if (!timeFilter) return t("allTime", locale);
@@ -11081,16 +11147,19 @@ function formatToolResultAsText(details) {
     lines.push("");
     let lastDate = "";
     for (const msg of messages) {
-      const spaceIdx = msg.indexOf(" ");
-      const secondSpaceIdx = msg.indexOf(" ", spaceIdx + 1);
+      const prefixMatch = /^(\[#[^\]]+\] )/.exec(msg);
+      const prefix = prefixMatch ? prefixMatch[1] : "";
+      const body = prefix ? msg.slice(prefix.length) : msg;
+      const spaceIdx = body.indexOf(" ");
+      const secondSpaceIdx = body.indexOf(" ", spaceIdx + 1);
       if (spaceIdx > 0 && secondSpaceIdx > 0) {
-        const date = msg.slice(0, spaceIdx);
-        const rest = msg.slice(spaceIdx + 1);
+        const date = body.slice(0, spaceIdx);
+        const rest = body.slice(spaceIdx + 1);
         if (date !== lastDate) {
           lines.push(`--- ${date} ---`);
           lastDate = date;
         }
-        lines.push(rest);
+        lines.push(prefix + rest);
       } else {
         lines.push(msg);
       }
@@ -11163,14 +11232,26 @@ function applyPreprocessingPipeline(options2) {
     maxToolResultTokens,
     truncationStrategy = "keep_last",
     extraDetails = {},
-    logger
+    logger,
+    includeMessageIds = false,
+    hitIds,
+    maxContentChars
   } = options2;
-  const processed = preprocessMessages(rawMessages, preprocessConfig, logger);
+  const { messages: processed, stats: preprocessStats } = preprocessMessagesWithStats(
+    rawMessages,
+    preprocessConfig,
+    logger
+  );
   let nameMapLine = "";
   if (anonymizeNames) {
     nameMapLine = anonymizeMessageNames(processed, ownerPlatformId);
   }
-  let formatted = processed.map((m) => formatMessageCompact(m, locale));
+  const formatOptions = includeMessageIds || maxContentChars != null ? {
+    includeMessageId: includeMessageIds,
+    hitIds: hitIds ? new Set(hitIds) : void 0,
+    maxContentLength: maxContentChars
+  } : void 0;
+  let formatted = processed.map((m) => formatMessageCompact(m, locale, formatOptions));
   let wasTruncated = false;
   const originalCount = formatted.length;
   if (maxToolResultTokens && maxToolResultTokens > 0) {
@@ -11190,7 +11271,12 @@ function applyPreprocessingPipeline(options2) {
   if (nameMapLine) {
     textContent = nameMapLine + "\n" + textContent;
   }
-  return { text: textContent, details: finalDetails };
+  const stats = {
+    ...preprocessStats,
+    renderedBlocks: formatted.length,
+    truncated: wasTruncated
+  };
+  return { text: textContent, details: finalDetails, stats };
 }
 
 // ../../packages/node-runtime/src/ai/agent/constants.ts
@@ -19316,6 +19402,9 @@ export {
   manualCompress,
   createCompressionLlmAdapter,
   preprocessMessages,
+  preprocessMessagesWithStats,
+  desensitizeText,
+  matchesBlacklist,
   DESENSITIZE_RULES_SCHEMA_VERSION,
   BUILTIN_DESENSITIZE_RULES,
   getDefaultRulesForLocale,
@@ -19440,4 +19529,4 @@ export {
   MergeSessionCache,
   pushImport
 };
-//# sourceMappingURL=chunk-QQK7MLCR.mjs.map
+//# sourceMappingURL=chunk-A2SANKG4.mjs.map