chattercatcher 0.1.18 → 0.1.19

package/dist/cli.js

@@ -8,7 +8,7 @@ import fs14 from "fs/promises";
 // package.json
 var package_default = {
   name: "chattercatcher",
-  version: "0.1.18",
+  version: "0.1.19",
   description: "\u672C\u5730\u4F18\u5148\u7684\u98DE\u4E66/Lark \u5BB6\u5EAD\u7FA4\u77E5\u8BC6\u5E93\u673A\u5668\u4EBA",
   type: "module",
   main: "dist/index.js",
@@ -27,7 +27,9 @@ var package_default = {
   files: [
     "assets",
     "dist",
-    "docs",
+    "docs/DEVELOPMENT_PLAN.md",
+    "docs/PRD.md",
+    "docs/TECHNICAL_ARCHITECTURE.md",
     "README.md",
     "AGENTS.md"
   ],
@@ -478,6 +480,22 @@ function migrateDatabase(database) {
     CREATE INDEX IF NOT EXISTS message_chunk_embeddings_model_idx
     ON message_chunk_embeddings(model, dimension);
 
+    CREATE TABLE IF NOT EXISTS qa_logs (
+      id TEXT PRIMARY KEY,
+      chat_id TEXT,
+      question_message_id TEXT,
+      question TEXT NOT NULL,
+      answer TEXT NOT NULL,
+      citations_json TEXT NOT NULL,
+      retrieval_debug_json TEXT NOT NULL,
+      status TEXT NOT NULL CHECK(status IN ('answered','failed')),
+      error TEXT,
+      created_at TEXT NOT NULL
+    );
+
+    CREATE INDEX IF NOT EXISTS qa_logs_created_at_idx ON qa_logs(created_at);
+    CREATE INDEX IF NOT EXISTS qa_logs_chat_idx ON qa_logs(chat_id, created_at);
+
     CREATE TABLE IF NOT EXISTS file_jobs (
       id TEXT PRIMARY KEY,
       source_path TEXT NOT NULL,
@@ -2617,6 +2635,191 @@ async function ensureFeishuBotOpenId(config, secrets, options = {}) {
 // src/feishu/gateway.ts
 import * as lark2 from "@larksuiteoapi/node-sdk";
 
+// src/gateway/indexing-scheduler.ts
+function createIndexingScheduler(options) {
+  const now = options.now ?? (() => /* @__PURE__ */ new Date());
+  const setIntervalFn = options.setIntervalFn ?? setInterval;
+  const clearIntervalFn = options.clearIntervalFn ?? clearInterval;
+  const logger = options.logger ?? console;
+  const parsed = parseCronSchedule(options.schedule);
+  let timer;
+  let running = false;
+  const runDueNow = async () => {
+    if (!parsed || running || !matchesParsedSchedule(parsed, now())) {
+      return;
+    }
+    running = true;
+    try {
+      await options.work();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      logger.error(`\u5B9A\u65F6\u6D88\u606F\u7D22\u5F15\u5931\u8D25\uFF1A${message}`);
+    } finally {
+      running = false;
+    }
+  };
+  return {
+    start() {
+      if (!parsed || timer) {
+        return;
+      }
+      timer = setIntervalFn(() => {
+        void runDueNow();
+      }, 6e4);
+    },
+    stop() {
+      if (!timer) {
+        return;
+      }
+      clearIntervalFn(timer);
+      timer = void 0;
+    },
+    runDueNow
+  };
+}
+function matchesParsedSchedule(schedule, date) {
+  return schedule.minute(date.getMinutes()) && schedule.hour(date.getHours()) && schedule.dayOfMonth(date.getDate()) && schedule.month(date.getMonth() + 1) && schedule.dayOfWeek(date.getDay());
+}
+function parseCronSchedule(schedule) {
+  const fields = schedule.trim().split(/\s+/);
+  if (fields.length !== 5) {
+    return null;
+  }
+  const minute = parseMinuteField(fields[0]);
+  const hour = parseExactOrWildcardField(fields[1], 0, 23);
+  const dayOfMonth = parseExactOrWildcardField(fields[2], 1, 31);
+  const month = parseExactOrWildcardField(fields[3], 1, 12);
+  const dayOfWeek = parseExactOrWildcardField(fields[4], 0, 6);
+  if (!minute || !hour || !dayOfMonth || !month || !dayOfWeek) {
+    return null;
+  }
+  return { minute, hour, dayOfMonth, month, dayOfWeek };
+}
+function parseMinuteField(field) {
+  if (field === "*") {
+    return () => true;
+  }
+  const stepMatch = /^\*\/(\d+)$/.exec(field);
+  if (stepMatch) {
+    const step = Number(stepMatch[1]);
+    if (!Number.isInteger(step) || step <= 0 || step > 59) {
+      return null;
+    }
+    return (value) => value % step === 0;
+  }
+  if (field.includes(",")) {
+    const values = field.split(",").map((part) => parseExactNumber(part, 0, 59));
+    if (values.some((value) => value === null)) {
+      return null;
+    }
+    const allowed = new Set(values);
+    return (value) => allowed.has(value);
+  }
+  const exact = parseExactNumber(field, 0, 59);
+  if (exact === null) {
+    return null;
+  }
+  return (value) => value === exact;
+}
+function parseExactOrWildcardField(field, min, max) {
+  if (field === "*") {
+    return () => true;
+  }
+  const exact = parseExactNumber(field, min, max);
+  if (exact === null) {
+    return null;
+  }
+  return (value) => value === exact;
+}
+function parseExactNumber(field, min, max) {
+  if (!/^\d+$/.test(field)) {
+    return null;
+  }
+  const value = Number(field);
+  if (!Number.isInteger(value) || value < min || value > max) {
+    return null;
+  }
+  return value;
+}
+
+// src/rag/indexer.ts
+async function indexMessageChunks(input2) {
+  const chunks = input2.messageIds ? input2.messages.listMessageChunksByMessageIds(input2.messageIds, input2.limit ?? 1e4) : input2.messages.listAllMessageChunks(input2.limit ?? 1e4);
+  if (chunks.length === 0) {
+    return { chunks: 0, vectors: 0 };
+  }
+  const vectors = await input2.embedding.embedBatch(chunks.map((chunk) => chunk.text));
+  const records = [];
+  for (const [index2, chunk] of chunks.entries()) {
+    const vector = vectors[index2];
+    if (!vector || vector.length === 0) {
+      continue;
+    }
+    records.push({
+      id: chunk.chunkId,
+      vector,
+      evidence: {
+        id: chunk.chunkId,
+        text: chunk.text,
+        score: 1,
+        source: toEvidenceSource3(chunk)
+      }
+    });
+  }
+  await input2.store.upsert(records);
+  return {
+    chunks: chunks.length,
+    vectors: records.length
+  };
+}
+function toEvidenceSource3(chunk) {
+  if (chunk.messageType === "file") {
+    return {
+      type: "file",
+      label: chunk.senderName,
+      timestamp: chunk.sentAt
+    };
+  }
+  return {
+    type: "message",
+    label: chunk.chatName,
+    sender: chunk.senderName,
+    timestamp: chunk.sentAt
+  };
+}
+
+// src/rag/manual-index.ts
+async function processMessagesNow(input2) {
+  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
+  if (!hasEmbeddingConfig(input2.config, input2.secrets)) {
+    return {
+      status: "skipped",
+      reason: "Embedding \u914D\u7F6E\u4E0D\u5B8C\u6574\uFF1BSQLite FTS \u5DF2\u5728\u6D88\u606F\u5165\u5E93\u65F6\u5373\u65F6\u66F4\u65B0\u3002",
+      chunks: 0,
+      vectors: 0,
+      startedAt,
+      finishedAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+  }
+  const vectorStore = new SqliteVectorStore(input2.database, {
+    model: input2.config.embedding.model
+  });
+  const embedding = input2.embedding ?? createEmbeddingModel(input2.config, input2.secrets);
+  const stats = await indexMessageChunks({
+    messages: new MessageRepository(input2.database),
+    embedding,
+    store: vectorStore,
+    limit: input2.limit
+  });
+  return {
+    status: "completed",
+    chunks: stats.chunks,
+    vectors: stats.vectors,
+    startedAt,
+    finishedAt: (/* @__PURE__ */ new Date()).toISOString()
+  };
+}
+
 // src/multimodal/tasks.ts
 import crypto4 from "crypto";
 function nowIso4() {
@@ -3102,6 +3305,108 @@ async function askWithAgenticRag(input2) {
   });
 }
 
+// src/rag/qa-logs.ts
+import crypto5 from "crypto";
+function clampLimit(limit) {
+  return Math.max(1, Math.min(200, Math.trunc(limit)));
+}
+var QaLogRepository = class {
+  constructor(database) {
+    this.database = database;
+  }
+  database;
+  create(input2) {
+    const record = {
+      id: `qa_${crypto5.randomUUID()}`,
+      chatId: input2.chatId ?? null,
+      questionMessageId: input2.questionMessageId ?? null,
+      question: input2.question,
+      answer: input2.answer,
+      citations: input2.citations,
+      retrievalDebug: input2.retrievalDebug,
+      status: input2.status,
+      error: input2.error ?? null,
+      createdAt: input2.createdAt
+    };
+    this.database.prepare(
+      `
+          INSERT INTO qa_logs (
+            id,
+            chat_id,
+            question_message_id,
+            question,
+            answer,
+            citations_json,
+            retrieval_debug_json,
+            status,
+            error,
+            created_at
+          )
+          VALUES (
+            @id,
+            @chatId,
+            @questionMessageId,
+            @question,
+            @answer,
+            @citationsJson,
+            @retrievalDebugJson,
+            @status,
+            @error,
+            @createdAt
+          )
+        `
+    ).run({
+      id: record.id,
+      chatId: record.chatId,
+      questionMessageId: record.questionMessageId,
+      question: record.question,
+      answer: record.answer,
+      citationsJson: JSON.stringify(record.citations),
+      retrievalDebugJson: JSON.stringify(record.retrievalDebug),
+      status: record.status,
+      error: record.error,
+      createdAt: record.createdAt
+    });
+    return record;
+  }
+  listRecent(limit) {
+    const rows = this.database.prepare(
+      `
+          SELECT
+            id,
+            chat_id,
+            question_message_id,
+            question,
+            answer,
+            citations_json,
+            retrieval_debug_json,
+            status,
+            error,
+            created_at
+          FROM qa_logs
+          ORDER BY created_at DESC
+          LIMIT ?
+        `
+    ).all(clampLimit(limit));
+    return rows.map((row) => ({
+      id: row.id,
+      chatId: row.chat_id,
+      questionMessageId: row.question_message_id,
+      question: row.question,
+      answer: row.answer,
+      citations: JSON.parse(row.citations_json),
+      retrievalDebug: JSON.parse(row.retrieval_debug_json),
+      status: row.status,
+      error: row.error,
+      createdAt: row.created_at
+    }));
+  }
+  getCount() {
+    const row = this.database.prepare("SELECT COUNT(*) AS count FROM qa_logs").get();
+    return row.count;
+  }
+};
+
 // src/feishu/question.ts
 function parseTextContent(content) {
   if (!content) {
@@ -3208,6 +3513,7 @@ var FeishuQuestionHandler = class {
       return decision;
     }
     const questionMessageId = payload.event?.message?.message_id;
+    const qaLogs = new QaLogRepository(this.options.database);
     await this.acknowledgeQuestion(decision.chatId, questionMessageId);
     const { tools, close } = await createAgenticRagSearchTools({
       config: this.options.config,
@@ -3223,6 +3529,16 @@ var FeishuQuestionHandler = class {
           tools,
           model: this.options.model
         });
+        qaLogs.create({
+          chatId: decision.chatId,
+          questionMessageId,
+          question: decision.question,
+          answer: result.answer,
+          citations: result.citations,
+          retrievalDebug: { evidenceCount: result.citations.length },
+          status: "answered",
+          createdAt: (/* @__PURE__ */ new Date()).toISOString()
+        });
         const citations = formatCitations(result.citations);
         const text = citations ? `${result.answer}
 
@@ -3231,6 +3547,17 @@ ${citations}` : result.answer;
         await this.sendResponse(decision.chatId, questionMessageId, text);
       } catch (error) {
         const message = error instanceof Error ? error.message : String(error);
+        qaLogs.create({
+          chatId: decision.chatId,
+          questionMessageId,
+          question: decision.question,
+          answer: `\u6682\u65F6\u65E0\u6CD5\u56DE\u7B54\uFF1A${message}`,
+          citations: [],
+          retrievalDebug: {},
+          status: "failed",
+          error: message,
+          createdAt: (/* @__PURE__ */ new Date()).toISOString()
+        });
         await this.sendResponse(decision.chatId, questionMessageId, `\u6682\u65F6\u65E0\u6CD5\u56DE\u7B54\uFF1A${message}`);
       }
       return decision;
@@ -3452,15 +3779,29 @@ function createFeishuGateway(options) {
     episodeProcessor: options.episodeProcessor,
     imageMultimodalProcessor: options.imageMultimodalProcessor
   });
+  const indexingScheduler = options.indexingScheduler ?? (options.indexingProcessor ? createIndexingScheduler({
+    schedule: options.config.schedules.indexing,
+    work: async () => {
+      await processMessagesNow({
+        config: options.config,
+        secrets: options.secrets,
+        database: options.indexingProcessor.database,
+        limit: 1e4
+      });
+    }
+  }) : void 0);
   return {
     async start() {
       try {
         await wsClient.start({ eventDispatcher });
+        indexingScheduler?.start();
       } catch (error) {
+        indexingScheduler?.stop();
         throw formatGatewayStartError(error);
       }
     },
     stop() {
+      indexingScheduler?.stop();
       wsClient.close({ force: true });
     }
   };
@@ -3532,7 +3873,7 @@ var FeishuResourceDownloader = class _FeishuResourceDownloader {
 };
 
 // src/files/ingest.ts
-import crypto5 from "crypto";
+import crypto6 from "crypto";
 import fs11 from "fs/promises";
 import path13 from "path";
 
@@ -3596,7 +3937,7 @@ function ensureSupportedTextFile(filePath) {
   }
 }
 function stableStoredName(sourcePath, fileName) {
-  const digest = crypto5.createHash("sha256").update(sourcePath).digest("hex").slice(0, 16);
+  const digest = crypto6.createHash("sha256").update(sourcePath).digest("hex").slice(0, 16);
   return `${digest}-${fileName}`;
 }
 async function ingestLocalFile(input2) {
@@ -4118,84 +4459,6 @@ function createMultimodalModel(config, secrets) {
   });
 }
 
-// src/rag/indexer.ts
-async function indexMessageChunks(input2) {
-  const chunks = input2.messageIds ? input2.messages.listMessageChunksByMessageIds(input2.messageIds, input2.limit ?? 1e4) : input2.messages.listAllMessageChunks(input2.limit ?? 1e4);
-  if (chunks.length === 0) {
-    return { chunks: 0, vectors: 0 };
-  }
-  const vectors = await input2.embedding.embedBatch(chunks.map((chunk) => chunk.text));
-  const records = [];
-  for (const [index2, chunk] of chunks.entries()) {
-    const vector = vectors[index2];
-    if (!vector || vector.length === 0) {
-      continue;
-    }
-    records.push({
-      id: chunk.chunkId,
-      vector,
-      evidence: {
-        id: chunk.chunkId,
-        text: chunk.text,
-        score: 1,
-        source: toEvidenceSource3(chunk)
-      }
-    });
-  }
-  await input2.store.upsert(records);
-  return {
-    chunks: chunks.length,
-    vectors: records.length
-  };
-}
-function toEvidenceSource3(chunk) {
-  if (chunk.messageType === "file") {
-    return {
-      type: "file",
-      label: chunk.senderName,
-      timestamp: chunk.sentAt
-    };
-  }
-  return {
-    type: "message",
-    label: chunk.chatName,
-    sender: chunk.senderName,
-    timestamp: chunk.sentAt
-  };
-}
-
-// src/rag/manual-index.ts
-async function processMessagesNow(input2) {
-  const startedAt = (/* @__PURE__ */ new Date()).toISOString();
-  if (!hasEmbeddingConfig(input2.config, input2.secrets)) {
-    return {
-      status: "skipped",
-      reason: "Embedding \u914D\u7F6E\u4E0D\u5B8C\u6574\uFF1BSQLite FTS \u5DF2\u5728\u6D88\u606F\u5165\u5E93\u65F6\u5373\u65F6\u66F4\u65B0\u3002",
-      chunks: 0,
-      vectors: 0,
-      startedAt,
-      finishedAt: (/* @__PURE__ */ new Date()).toISOString()
-    };
-  }
-  const vectorStore = new SqliteVectorStore(input2.database, {
-    model: input2.config.embedding.model
-  });
-  const embedding = input2.embedding ?? createEmbeddingModel(input2.config, input2.secrets);
-  const stats = await indexMessageChunks({
-    messages: new MessageRepository(input2.database),
-    embedding,
-    store: vectorStore,
-    limit: input2.limit
-  });
-  return {
-    status: "completed",
-    chunks: stats.chunks,
-    vectors: stats.vectors,
-    startedAt,
-    finishedAt: (/* @__PURE__ */ new Date()).toISOString()
-  };
-}
-
 // src/rag/qa-service.ts
 async function askWithRag(input2) {
   const evidence = await input2.retriever.retrieve(input2.question);
@@ -4444,6 +4707,10 @@ function buildHtml() {
             <h2>\u4F1A\u8BDD\u8BB0\u5FC6</h2>
             <div id="episodes" class="empty">\u6B63\u5728\u8BFB\u53D6...</div>
           </section>
+          <section>
+            <h2>\u95EE\u7B54\u65E5\u5FD7</h2>
+            <div id="qa-logs" class="empty">\u6B63\u5728\u8BFB\u53D6...</div>
+          </section>
         </div>
         <aside>
           <section>
@@ -4474,6 +4741,7 @@ function buildHtml() {
       const chats = document.querySelector("#chats");
       const files = document.querySelector("#files");
       const fileJobs = document.querySelector("#file-jobs");
+      const qaLogs = document.querySelector("#qa-logs");
       const processMessages = document.querySelector("#process-messages");
       const actionStatus = document.querySelector("#action-status");
 
@@ -4665,14 +4933,45 @@ function buildHtml() {
         \`;
       }
 
+      function renderQaLogs(items) {
+        if (items.length === 0) {
+          qaLogs.className = "empty";
+          qaLogs.textContent = "\u8FD8\u6CA1\u6709\u95EE\u7B54\u65E5\u5FD7\u3002";
+          return;
+        }
+        qaLogs.className = "";
+        const rows = items.map((item) => {
+          const citationCount = Array.isArray(item.citations) ? item.citations.length : 0;
+          return [
+            '<article class="message-item">',
+            '  <div class="message-meta">',
+            "    <span>" + escapeHtml(formatDateTime(item.createdAt)) + "</span>",
+            "    <span>" + escapeHtml(item.status) + "</span>",
+            "    <span>" + escapeHtml(citationCount) + " \u6761\u5F15\u7528</span>",
+            "  </div>",
+            "  <div class="message-body"><strong>\u95EE\uFF1A</strong>" + escapeHtml(item.question) + "</div>",
+            "  <div class="message-body"><strong>\u7B54\uFF1A</strong>" + escapeHtml(item.answer) + "</div>",
+            "</article>",
+          ].join("
+");
+        });
+        qaLogs.innerHTML = [
+          '<div class="message-list">',
+          rows.join(""),
+          "</div>",
+        ].join("
+");
+      }
+
       async function load() {
-        const [status, recent, episodeList, chatList, fileList, jobList] = await Promise.all([
+        const [status, recent, episodeList, chatList, fileList, jobList, qaLogList] = await Promise.all([
           fetch("/api/status").then((response) => response.json()),
           fetch("/api/messages/recent?limit=20").then((response) => response.json()),
           fetch("/api/episodes?limit=10").then((response) => response.json()),
           fetch("/api/chats").then((response) => response.json()),
           fetch("/api/files").then((response) => response.json()),
           fetch("/api/file-jobs").then((response) => response.json()),
+          fetch("/api/qa-logs?limit=10").then((response) => response.json()),
         ]);
         renderMetrics(status);
         renderMessages(recent.items);
@@ -4680,6 +4979,7 @@ function buildHtml() {
         renderChats(chatList.items);
         renderFiles(fileList.items);
         renderFileJobs(jobList.items);
+        renderQaLogs(qaLogList.items);
       }
 
       async function processNow() {
@@ -4727,6 +5027,7 @@ function createWebApp(config) {
   const messages = new MessageRepository(database);
   const episodes = new EpisodeRepository(database);
   const fileJobs = new FileJobRepository(database);
+  const qaLogs = new QaLogRepository(database);
   app.addHook("onClose", async () => {
     database.close();
   });
@@ -4737,7 +5038,8 @@ function createWebApp(config) {
       chats: messages.getChatCount(),
       messages: messages.getMessageCount(),
       episodes: episodes.getEpisodeCount(),
-      files: messages.listFiles(1e3).length
+      files: messages.listFiles(1e3).length,
+      qaLogs: qaLogs.getCount()
     },
     rag: {
       mode: "required",
@@ -4778,6 +5080,12 @@ function createWebApp(config) {
       items: episodes.listRecentEpisodes(limit)
     };
   });
+  app.get("/api/qa-logs", async (request) => {
+    const limit = parseLimit(request.query.limit, 20, 100);
+    return {
+      items: qaLogs.listRecent(limit)
+    };
+  });
   app.post("/api/process/messages", async (_request, reply) => {
     try {
       return await processMessagesNow({
@@ -5013,6 +5321,9 @@ async function startGatewayForegroundCommand() {
       database,
       model: createMultimodalModel(config, secrets)
     } : void 0,
+    indexingProcessor: {
+      database
+    },
     questionHandler: new FeishuQuestionHandler({
       config,
       secrets,