chattercatcher 0.1.13 → 0.1.14

package/dist/cli.js

@@ -8,7 +8,7 @@ import fs13 from "fs/promises";
 // package.json
 var package_default = {
   name: "chattercatcher",
-  version: "0.1.13",
+  version: "0.1.14",
   description: "\u672C\u5730\u4F18\u5148\u7684\u98DE\u4E66/Lark \u5BB6\u5EAD\u7FA4\u77E5\u8BC6\u5E93\u673A\u5668\u4EBA",
   type: "module",
   main: "dist/index.js",
@@ -110,6 +110,10 @@ var appConfigSchema = z.object({
   }),
   schedules: z.object({
     indexing: z.string().default("*/10 * * * *")
+  }),
+  episodes: z.object({
+    windowMinutes: z.number().int().positive().default(10),
+    quietMinutes: z.number().int().positive().default(2)
   })
 });
 var appSecretsSchema = z.object({
@@ -130,7 +134,8 @@ function createDefaultConfig() {
     embedding: {},
     storage: {},
     web: {},
-    schedules: {}
+    schedules: {},
+    episodes: {}
   });
 }
 function createDefaultSecrets() {
@@ -412,6 +417,39 @@ function migrateDatabase(database) {
       tokenize = 'unicode61'
     );
 
+    CREATE TABLE IF NOT EXISTS memory_episodes (
+      id TEXT PRIMARY KEY,
+      chat_id TEXT NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
+      summary TEXT NOT NULL,
+      message_count INTEGER NOT NULL,
+      started_at TEXT NOT NULL,
+      ended_at TEXT NOT NULL,
+      created_at TEXT NOT NULL,
+      UNIQUE(chat_id, started_at, ended_at)
+    );
+
+    CREATE TABLE IF NOT EXISTS memory_episode_messages (
+      episode_id TEXT NOT NULL REFERENCES memory_episodes(id) ON DELETE CASCADE,
+      message_id TEXT NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
+      position INTEGER NOT NULL,
+      PRIMARY KEY (episode_id, message_id)
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS memory_episodes_fts USING fts5(
+      summary,
+      episode_id UNINDEXED,
+      tokenize = 'unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS memory_episodes_delete_fts
+    AFTER DELETE ON memory_episodes
+    BEGIN
+      DELETE FROM memory_episodes_fts WHERE episode_id = old.id;
+    END;
+
+    CREATE INDEX IF NOT EXISTS memory_episode_messages_message_idx
+    ON memory_episode_messages(message_id);
+
     CREATE TABLE IF NOT EXISTS message_chunk_embeddings (
       chunk_id TEXT NOT NULL REFERENCES message_chunks(id) ON DELETE CASCADE,
       model TEXT NOT NULL,
@@ -1268,6 +1306,211 @@ var MessageRepository = class {
   }
 };
 
+// src/episodes/repository.ts
+import crypto3 from "crypto";
+
+// src/episodes/sanitizer.ts
+var SECRET_PATTERNS = [
+  [/-----BEGIN [^-]+ PRIVATE KEY-----[\s\S]*?-----END [^-]+ PRIVATE KEY-----/g, "[REDACTED_SECRET]"],
+  [/(\bAuthorization\s*:\s*Bearer\s+)[A-Za-z0-9._~+/=-]{12,}/gi, "$1[REDACTED_SECRET]"],
+  [/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED_SECRET]@"],
+  [/([?&](?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:id)?|client[_-]?secret)=)[^\s&，。；;]+/gi, "$1[REDACTED_SECRET]"],
+  [/("(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:id)?|client[_-]?secret|private[_-]?key)"\s*:\s*")[^"]+(")/gi, "$1[REDACTED_SECRET]$2"],
+  [/(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|token|secret|password|session(?:id)?|client[_-]?secret)\s*[=:]\s*)[^\s;，。]+/gi, "$1[REDACTED_SECRET]"],
+  [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/g, "[REDACTED_SECRET]"],
+  [/\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g, "[REDACTED_SECRET]"],
+  [/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[REDACTED_SECRET]"]
+];
+function sanitizeEpisodeSummary(summary) {
+  let sanitized = summary;
+  for (const [pattern, replacement] of SECRET_PATTERNS) {
+    sanitized = sanitized.replace(pattern, replacement);
+  }
+  return sanitized;
+}
+
+// src/episodes/repository.ts
+function nowIso3() {
+  return (/* @__PURE__ */ new Date()).toISOString();
+}
+function stableId2(parts) {
+  return crypto3.createHash("sha256").update(parts.join("")).digest("hex").slice(0, 32);
+}
+function escapeFtsQuery2(query) {
+  const terms = query.trim().split(/\s+/).map((term) => term.replace(/[^\p{L}\p{N}_-]+/gu, " ").trim()).flatMap((term) => term.split(/\s+/)).filter(Boolean);
+  if (terms.length === 0) {
+    return '""';
+  }
+  return terms.map((term) => `"${term.replace(/"/g, '""')}"`).join(" OR ");
+}
+function toMillis(value) {
+  const time = Date.parse(value);
+  return Number.isFinite(time) ? time : 0;
+}
+var EpisodeRepository = class {
+  constructor(database) {
+    this.database = database;
+  }
+  database;
+  async summarizeReadyWindows(input2) {
+    const rows = this.database.prepare(
+      `
+          SELECT
+            m.id,
+            m.chat_id AS chatId,
+            c.name AS chatName,
+            m.sender_name AS senderName,
+            m.text,
+            m.sent_at AS sentAt
+          FROM messages m
+          JOIN chats c ON c.id = m.chat_id
+          WHERE NOT EXISTS (
+            SELECT 1 FROM memory_episode_messages mem WHERE mem.message_id = m.id
+          )
+          ORDER BY m.chat_id ASC, m.sent_at ASC
+        `
+    ).all();
+    const byChat = /* @__PURE__ */ new Map();
+    for (const row of rows) {
+      byChat.set(row.chatId, [...byChat.get(row.chatId) ?? [], row]);
+    }
+    const created = [];
+    const nowMs = input2.now.getTime();
+    for (const messages of byChat.values()) {
+      const windows = [];
+      let current = [];
+      for (const message of messages) {
+        const first = current[0];
+        if (first && toMillis(message.sentAt) - toMillis(first.sentAt) > input2.windowMs) {
+          windows.push(current);
+          current = [];
+        }
+        current.push(message);
+      }
+      if (current.length > 0) {
+        windows.push(current);
+      }
+      for (const windowMessages of windows) {
+        const last = windowMessages.at(-1);
+        if (!last || nowMs - toMillis(last.sentAt) < input2.quietMs) {
+          continue;
+        }
+        const first = windowMessages[0];
+        const window = {
+          chatId: first.chatId,
+          chatName: first.chatName,
+          startedAt: first.sentAt,
+          endedAt: last.sentAt,
+          messages: windowMessages
+        };
+        const summary = await input2.summarize(window);
+        created.push(this.insertEpisode(window, summary));
+      }
+    }
+    return created;
+  }
+  insertEpisode(window, summary) {
+    const safeSummary = sanitizeEpisodeSummary(summary);
+    const createdAt = nowIso3();
+    const id = stableId2([window.chatId, window.startedAt, window.endedAt]);
+    const transaction = this.database.transaction(() => {
+      this.database.prepare(
+        `
+            INSERT INTO memory_episodes (id, chat_id, summary, message_count, started_at, ended_at, created_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(chat_id, started_at, ended_at)
+            DO UPDATE SET summary = excluded.summary, message_count = excluded.message_count
+          `
+      ).run(id, window.chatId, safeSummary, window.messages.length, window.startedAt, window.endedAt, createdAt);
+      this.database.prepare("DELETE FROM memory_episode_messages WHERE episode_id = ?").run(id);
+      this.database.prepare("DELETE FROM memory_episodes_fts WHERE episode_id = ?").run(id);
+      const insertMessage = this.database.prepare(
+        "INSERT INTO memory_episode_messages (episode_id, message_id, position) VALUES (?, ?, ?)"
+      );
+      for (const [index2, message] of window.messages.entries()) {
+        insertMessage.run(id, message.id, index2);
+      }
+      this.database.prepare("INSERT INTO memory_episodes_fts (summary, episode_id) VALUES (?, ?)").run(safeSummary, id);
+    });
+    transaction();
+    return {
+      id,
+      chatId: window.chatId,
+      chatName: window.chatName,
+      text: safeSummary,
+      startedAt: window.startedAt,
+      endedAt: window.endedAt,
+      messageIds: window.messages.map((message) => message.id)
+    };
+  }
+  searchEpisodes(query, limit = 8) {
+    const ftsQuery = escapeFtsQuery2(query);
+    return this.database.prepare(
+      `
+          SELECT
+            e.id AS chunkId,
+            e.id AS messageId,
+            'episode' AS platform,
+            e.summary AS text,
+            1.0 AS score,
+            'episode' AS messageType,
+            c.name AS chatName,
+            '\u4F1A\u8BDD\u8BB0\u5FC6' AS senderName,
+            e.ended_at AS sentAt,
+            e.started_at AS startedAt,
+            e.ended_at AS endedAt,
+            (
+              SELECT json_group_array(message_id)
+              FROM (
+                SELECT message_id
+                FROM memory_episode_messages
+                WHERE episode_id = e.id
+                ORDER BY position ASC
+              )
+            ) AS sourceMessageIdsJson
+          FROM memory_episodes_fts fts
+          JOIN memory_episodes e ON e.id = fts.episode_id
+          JOIN chats c ON c.id = e.chat_id
+          WHERE memory_episodes_fts MATCH ?
+          GROUP BY e.id
+          ORDER BY e.ended_at DESC
+          LIMIT ?
+        `
+    ).all(ftsQuery, limit).map((row) => {
+      const item = row;
+      return {
+        ...item,
+        sourceMessageIds: JSON.parse(item.sourceMessageIdsJson)
+      };
+    });
+  }
+};
+
+// src/rag/episode-retriever.ts
+function toEpisodeEvidence(result) {
+  return {
+    id: result.chunkId,
+    text: result.text,
+    score: result.score,
+    source: {
+      type: "episode",
+      label: result.chatName,
+      sender: result.senderName,
+      timestamp: result.endedAt,
+      location: `${result.startedAt} - ${result.endedAt}`
+    }
+  };
+}
+var EpisodeFtsRetriever = class {
+  constructor(episodes) {
+    this.episodes = episodes;
+  }
+  episodes;
+  async retrieve(question) {
+    return this.episodes.searchEpisodes(question, 8).map(toEpisodeEvidence);
+  }
+};
+
 // src/rag/hybrid-retriever.ts
 function normalizeScore(score) {
   if (!Number.isFinite(score)) {
@@ -1275,6 +1518,14 @@ function normalizeScore(score) {
   }
   return Math.max(0, Math.min(1, score));
 }
+function evidenceTimestampMs(evidence) {
+  const timestamp = evidence.source.timestamp;
+  if (!timestamp) {
+    return 0;
+  }
+  const parsed = Date.parse(timestamp);
+  return Number.isFinite(parsed) ? parsed : 0;
+}
 var HybridRetriever = class {
   constructor(retrievers, options = {}) {
     this.retrievers = retrievers;
@@ -1285,19 +1536,19 @@ var HybridRetriever = class {
   async retrieve(question) {
     const results = await Promise.all(this.retrievers.map((retriever) => retriever.retrieve(question)));
     const merged = /* @__PURE__ */ new Map();
-    for (const [retrieverIndex, evidenceList] of results.entries()) {
+    for (const evidenceList of results) {
       for (const evidence of evidenceList) {
         const existing = merged.get(evidence.id);
-        const weightedScore = normalizeScore(evidence.score) + (this.retrievers.length - retrieverIndex) * 0.01;
-        if (!existing || weightedScore > existing.score) {
+        const score = normalizeScore(evidence.score);
+        if (!existing || score > existing.score) {
           merged.set(evidence.id, {
             ...evidence,
-            score: weightedScore
+            score
           });
         }
       }
     }
-    return [...merged.values()].sort((left, right) => right.score - left.score).slice(0, this.options.limit ?? 8);
+    return [...merged.values()].sort((left, right) => right.score - left.score || evidenceTimestampMs(right) - evidenceTimestampMs(left)).slice(0, this.options.limit ?? 8);
   }
 };
 
@@ -1471,7 +1722,10 @@ function hasEmbeddingConfig(config, secrets) {
   return Boolean((config.embedding.baseUrl || config.llm.baseUrl) && config.embedding.model && (secrets.embedding.apiKey || secrets.llm.apiKey));
 }
 async function createHybridRetriever(input2) {
-  const retrievers = [new MessageFtsRetriever(input2.messages, { excludeMessageIds: input2.excludeMessageIds })];
+  const retrievers = [
+    new EpisodeFtsRetriever(new EpisodeRepository(input2.database)),
+    new MessageFtsRetriever(input2.messages, { excludeMessageIds: input2.excludeMessageIds })
+  ];
   const closers = [];
   if (hasEmbeddingConfig(input2.config, input2.secrets)) {
     const vectorStore = new SqliteVectorStore(input2.database, {
@@ -1945,6 +2199,40 @@ async function restoreLocalData(input2) {
   };
 }
 
+// src/episodes/summarizer.ts
+async function summarizeEpisodeWindow(window, model) {
+  const transcript = window.messages.map((message) => `[${message.sentAt}] ${message.senderName}\uFF1A${message.text}`).join("\n");
+  const summary = await model.complete([
+    {
+      role: "system",
+      content: "\u4F60\u662F ChatterCatcher \u7684\u4F1A\u8BDD\u8BB0\u5FC6\u6574\u7406\u6A21\u5757\u3002\u4F60\u7684\u4EFB\u52A1\u662F\u628A\u788E\u7247\u5316\u95F2\u804A\u6574\u7406\u6210\u53EF\u68C0\u7D22\u4E8B\u5B9E\uFF0C\u8865\u5168\u77ED\u6D88\u606F\u3001\u4EE3\u8BCD\u3001\u7F29\u5199\u4E0E\u4E0A\u4E0B\u6587\u4E4B\u95F4\u7684\u5173\u7CFB\u3002\u53EA\u603B\u7ED3\u660E\u786E\u4E8B\u5B9E\uFF0C\u4E0D\u8981\u7F16\u9020\u3002\u4FDD\u7559\u91CD\u8981\u6570\u5B57\u3001\u65E5\u671F\u3001\u94FE\u63A5\u548C\u4EE3\u7801\uFF1B\u5982\u679C\u5185\u5BB9\u50CF\u5BC6\u7801\u3001API key\u3001token \u6216\u5BC6\u94A5\uFF0C\u53EA\u63CF\u8FF0\u5176\u4E0A\u4E0B\u6587\u5173\u7CFB\uFF0C\u4E0D\u8981\u5728\u6458\u8981\u4E2D\u590D\u5199\u539F\u6587\u3002"
+    },
+    {
+      role: "user",
+      content: `\u7FA4\u804A\uFF1A${window.chatName}
+\u65F6\u95F4\uFF1A${window.startedAt} - ${window.endedAt}
+
+\u804A\u5929\u8BB0\u5F55\uFF1A
+${transcript}
+
+\u8BF7\u8F93\u51FA\u4E00\u6BB5\u7B80\u6D01\u7684\u4F1A\u8BDD\u8BB0\u5FC6\u6458\u8981\u3002`
+    }
+  ]);
+  return sanitizeEpisodeSummary(summary);
+}
+
+// src/episodes/manual-process.ts
+async function processEpisodesNow(input2) {
+  const episodes = new EpisodeRepository(input2.database);
+  const created = await episodes.summarizeReadyWindows({
+    now: input2.now ?? /* @__PURE__ */ new Date(),
+    quietMs: input2.config.episodes.quietMinutes * 60 * 1e3,
+    windowMs: input2.config.episodes.windowMinutes * 60 * 1e3,
+    summarize: (window) => summarizeEpisodeWindow(window, input2.model)
+  });
+  return { created: created.length };
+}
+
 // src/feishu/bot-info.ts
 function getOpenApiBaseUrl(domain) {
   return domain === "lark" ? "https://open.larksuite.com/open-apis" : "https://open.feishu.cn/open-apis";
@@ -2416,6 +2704,18 @@ function createFeishuEventDispatcher(options) {
         console.log("\u98DE\u4E66\u6D88\u606F\u91CD\u590D\u6295\u9012\uFF1A\u5DF2\u8DF3\u8FC7\u9644\u4EF6\u5904\u7406\u548C\u56DE\u7B54\u3002");
         return;
       }
+      if (options.episodeProcessor) {
+        const episodeResult = await processEpisodesNow({
+          config: options.config,
+          secrets: options.secrets,
+          database: options.episodeProcessor.database,
+          model: options.episodeProcessor.model,
+          now: options.episodeProcessor.now?.()
+        });
+        if (episodeResult.created > 0) {
+          console.log(`\u98DE\u4E66\u4F1A\u8BDD\u8BB0\u5FC6\u5DF2\u751F\u6210\uFF1A${episodeResult.created}`);
+        }
+      }
       if (result.attachment?.downloaded) {
         console.log(`\u98DE\u4E66\u9644\u4EF6\u5DF2\u4E0B\u8F7D\uFF1A${result.attachment.downloaded.storedPath}`);
         if (result.attachment.indexedMessageId) {
@@ -2463,10 +2763,12 @@ function createFeishuGateway(options) {
   });
   const eventDispatcher = createFeishuEventDispatcher({
     config: options.config,
+    secrets: options.secrets,
     ingestor: options.ingestor,
     questionHandler: options.questionHandler,
     resourceDownloader: options.resourceDownloader,
-    attachmentVectorIndexer: options.attachmentVectorIndexer
+    attachmentVectorIndexer: options.attachmentVectorIndexer,
+    episodeProcessor: options.episodeProcessor
   });
   return {
     async start() {
@@ -2544,7 +2846,7 @@ var FeishuResourceDownloader = class _FeishuResourceDownloader {
 };
 
 // src/files/ingest.ts
-import crypto3 from "crypto";
+import crypto4 from "crypto";
 import fs11 from "fs/promises";
 import path13 from "path";
 
@@ -2608,7 +2910,7 @@ function ensureSupportedTextFile(filePath) {
   }
 }
 function stableStoredName(sourcePath, fileName) {
-  const digest = crypto3.createHash("sha256").update(sourcePath).digest("hex").slice(0, 16);
+  const digest = crypto4.createHash("sha256").update(sourcePath).digest("hex").slice(0, 16);
   return `${digest}-${fileName}`;
 }
 async function ingestLocalFile(input2) {
@@ -3700,6 +4002,8 @@ async function promptForConfiguration(config, secrets) {
     message: "\u7FA4\u804A\u56DE\u7B54\u662F\u5426\u8981\u6C42 @ \u673A\u5668\u4EBA\uFF1F",
     default: config.feishu.requireMention
   });
+  config.episodes.windowMinutes = await number({ message: "\u4F1A\u8BDD\u8BB0\u5FC6\u805A\u5408\u7A97\u53E3\uFF08\u5206\u949F\uFF09", default: config.episodes.windowMinutes, required: true }) ?? config.episodes.windowMinutes;
+  config.episodes.quietMinutes = await number({ message: "\u4F1A\u8BDD\u9759\u9ED8\u591A\u4E45\u540E\u751F\u6210\u8BB0\u5FC6\uFF08\u5206\u949F\uFF09", default: config.episodes.quietMinutes, required: true }) ?? config.episodes.quietMinutes;
 }
 async function tryEnsureFeishuBotOpenId(config, secrets) {
   if (config.feishu.botOpenId || !config.feishu.appId || !secrets.feishu.appSecret) {
@@ -3828,6 +4132,10 @@ async function startGatewayForegroundCommand() {
       store: vectorStore,
       messageIds: [messageId]
     }) : void 0,
+    episodeProcessor: {
+      database,
+      model: createChatModel(config, secrets)
+    },
     questionHandler: new FeishuQuestionHandler({
       config,
       secrets,
@@ -4007,6 +4315,22 @@ processCommand.command("messages").description("\u7ACB\u5373\u5904\u7406\u6D88\u
     database.close();
   }
 });
+processCommand.command("episodes").description("\u7ACB\u5373\u751F\u6210\u4F1A\u8BDD\u8BB0\u5FC6\u5757\uFF0C\u628A\u788E\u7247\u5316\u95F2\u804A\u6574\u7406\u6210\u53EF\u68C0\u7D22\u6458\u8981").action(async () => {
+  const config = await loadConfig();
+  const secrets = await loadSecrets();
+  const database = openDatabase(config);
+  try {
+    const result = await processEpisodesNow({
+      config,
+      secrets,
+      database,
+      model: createChatModel(config, secrets)
+    });
+    console.log(`\u4F1A\u8BDD\u8BB0\u5FC6\u5904\u7406\u5B8C\u6210\uFF1Aepisodes=${result.created}`);
+  } finally {
+    database.close();
+  }
+});
 var files = program.command("files").description("\u7BA1\u7406\u672C\u5730\u6587\u4EF6\u77E5\u8BC6\u6E90");
 files.command("add").description("\u628A\u672C\u5730\u6587\u4EF6\u89E3\u6790\u3001\u4FDD\u5B58\u5230\u6570\u636E\u76EE\u5F55\u5E76\u5199\u5165 RAG \u77E5\u8BC6\u5E93").argument("<paths...>", "\u6587\u4EF6\u8DEF\u5F84\uFF0C\u652F\u6301 txt\u3001md\u3001json\u3001csv\u3001tsv\u3001log\u3001docx\u3001pdf").action(async (paths) => {
   const config = await loadConfig();