@hsingjui/contextweaver 0.0.3 → 0.0.5

package/dist/{chunk-RU4OUBZS.js → chunk-5MZUQNOD.js}

@@ -3,18 +3,20 @@ import {
   batchUpdateVectorIndexHash,
   batchUpsertChunkFts,
   clearVectorIndexHash,
-  isChunksFtsInitialized,
+  isChunksFtsInitialized
+} from "./chunk-VW5RACJC.js";
+import {
   logger
-} from "./chunk-AVBQ77MN.js";
+} from "./chunk-C7XDGBT5.js";
 import {
   getEmbeddingConfig
-} from "./chunk-ESTPBFCW.js";
+} from "./chunk-SKBAE26T.js";
 
 // src/vectorStore/index.ts
-import * as lancedb from "@lancedb/lancedb";
-import path from "path";
-import os from "os";
 import fs from "fs";
+import os from "os";
+import path from "path";
+import * as lancedb from "@lancedb/lancedb";
 var BASE_DIR = path.join(os.homedir(), ".contextweaver");
 var VectorStore = class {
   db = null;
@@ -49,11 +51,14 @@ var VectorStore = class {
     if (this.table) return;
     if (!this.db) throw new Error("VectorStore not initialized");
     if (records.length === 0) return;
-    this.table = await this.db.createTable("chunks", records);
+    this.table = await this.db.createTable(
+      "chunks",
+      records
+    );
   }
   /**
    * 单调版本更新：先插入新版本，再删除旧版本
-   * 
+   *
    * 这保证了：
    * - 最坏情况（崩溃）是新旧版本共存（不缺失）
    * - 正常情况下旧版本被清理
@@ -70,18 +75,20 @@ var VectorStore = class {
       await this.table.add(records);
     }
     if (this.table) {
-      await this.table.delete(`file_path = '${this.escapeString(filePath)}' AND file_hash != '${this.escapeString(newHash)}'`);
+      await this.table.delete(
+        `file_path = '${this.escapeString(filePath)}' AND file_hash != '${this.escapeString(newHash)}'`
+      );
     }
   }
   /**
    * 批量 upsert 多个文件（性能优化版，带分批机制）
-   * 
+   *
    * 流程：
    * 1. 将文件分成小批次（每批最多 BATCH_FILES 个文件或 BATCH_RECORDS 条记录）
    * 2. 每批执行：插入新 records → 删除旧版本
-   * 
+   *
    * 分批是必要的，因为 LanceDB native 模块在处理超大数据时可能崩溃
-   * 
+   *
    * @param files 文件列表，每个包含 path、hash 和 records
    */
   async batchUpsertFiles(files) {
@@ -122,7 +129,9 @@ var VectorStore = class {
         await this.table.add(batchRecords);
       }
       if (this.table && batch.length > 0) {
-        const deleteConditions = batch.map((f) => `(file_path = '${this.escapeString(f.path)}' AND file_hash != '${this.escapeString(f.hash)}')`).join(" OR ");
+        const deleteConditions = batch.map(
+          (f) => `(file_path = '${this.escapeString(f.path)}' AND file_hash != '${this.escapeString(f.hash)}')`
+        ).join(" OR ");
         await this.table.delete(deleteConditions);
       }
     }
@@ -165,7 +174,7 @@ var VectorStore = class {
   }
   /**
    * 批量获取多个文件的 chunks（性能优化：单次查询替代 N 次循环）
-   * 
+   *
    * 适用于 GraphExpander 扩展、词法召回等需要批量获取的场景
    * @returns Map<filePath, ChunkRecord[]>，每个文件的 chunks 已按 chunk_index 排序
    */
@@ -242,9 +251,6 @@ async function closeAllVectorStores() {
   vectorStores.clear();
 }
 
-// src/indexer/index.ts
-import "better-sqlite3";
-
 // src/api/embedding.ts
 var ProgressTracker = class {
   completed = 0;
@@ -253,15 +259,21 @@ var ProgressTracker = class {
   startTime;
   lastLogTime = 0;
   logIntervalMs = 2e3;
-  // 每 3 秒输出一次
-  constructor(total) {
+  // 每 2 秒输出一次
+  onProgress;
+  /** 是否跳过日志（单批次时跳过，避免与索引日志混淆） */
+  skipLogs;
+  constructor(total, onProgress) {
     this.total = total;
     this.startTime = Date.now();
+    this.onProgress = onProgress;
+    this.skipLogs = total <= 1;
   }
   /** 记录一个批次完成 */
   recordBatch(tokens) {
     this.completed++;
     this.totalTokens += tokens;
+    this.onProgress?.(this.completed, this.total);
     const now = Date.now();
     if (now - this.lastLogTime >= this.logIntervalMs) {
       this.logProgress();
@@ -270,6 +282,7 @@ var ProgressTracker = class {
   }
   /** 输出进度 */
   logProgress() {
+    if (this.skipLogs) return;
     const elapsed = (Date.now() - this.startTime) / 1e3;
     const percent = Math.round(this.completed / this.total * 100);
     const rate = this.completed / elapsed;
@@ -287,6 +300,7 @@ var ProgressTracker = class {
   }
   /** 完成时输出最终统计 */
   complete() {
+    if (this.skipLogs) return;
     const elapsed = (Date.now() - this.startTime) / 1e3;
     logger.info(
       {
@@ -391,20 +405,17 @@ var RateLimitController = class {
       },
       "\u901F\u7387\u9650\u5236\uFF1A\u89E6\u53D1 429\uFF0C\u6682\u505C\u6240\u6709\u8BF7\u6C42"
     );
-    let resumeResolve;
+    let resumeResolve = () => {
+    };
     this.pausePromise = new Promise((resolve) => {
       resumeResolve = resolve;
     });
     await sleep(this.backoffMs);
     this.backoffMs = Math.min(this.maxBackoffMs, this.backoffMs * 2);
     this.isPaused = false;
-    const resolvedPromise = this.pausePromise;
     this.pausePromise = null;
     resumeResolve();
-    logger.info(
-      { waitMs: this.backoffMs },
-      "\u901F\u7387\u9650\u5236\uFF1A\u6062\u590D\u8BF7\u6C42"
-    );
+    logger.info({ waitMs: this.backoffMs }, "\u901F\u7387\u9650\u5236\uFF1A\u6062\u590D\u8BF7\u6C42");
   }
   /**
    * 获取当前状态（用于调试）
@@ -444,8 +455,9 @@ var EmbeddingClient = class {
    * 批量获取 Embedding
    * @param texts 待处理的文本数组
    * @param batchSize 每批次发送的文本数量（默认 20）
+   * @param onProgress 可选的进度回调 (completed, total) => void
    */
-  async embedBatch(texts, batchSize = 20) {
+  async embedBatch(texts, batchSize = 20, onProgress) {
     if (texts.length === 0) {
       return [];
     }
@@ -453,7 +465,7 @@ var EmbeddingClient = class {
     for (let i = 0; i < texts.length; i += batchSize) {
       batches.push(texts.slice(i, i + batchSize));
     }
-    const progress = new ProgressTracker(batches.length);
+    const progress = new ProgressTracker(batches.length, onProgress);
     const batchResults = await Promise.all(
       batches.map(
         (batch, batchIndex) => this.processWithRateLimit(batch, batchIndex * batchSize, progress)
@@ -463,10 +475,13 @@ var EmbeddingClient = class {
     return batchResults.flat();
   }
   /**
-   * 带速率限制的批次处理
+   * 带速率限制和网络错误重试的批次处理
    * 使用循环而非递归，避免栈溢出和槽位泄漏
    */
   async processWithRateLimit(texts, startIndex, progress) {
+    const MAX_NETWORK_RETRIES = 3;
+    const INITIAL_RETRY_DELAY_MS = 1e3;
+    let networkRetries = 0;
     while (true) {
       await this.rateLimiter.acquire();
       try {
@@ -474,17 +489,75 @@ var EmbeddingClient = class {
         this.rateLimiter.releaseSuccess();
         return result;
       } catch (err) {
-        const isRateLimited = err.message?.includes("429") || err.message?.includes("rate");
+        const error = err;
+        const errorMessage = error.message || "";
+        const isRateLimited = errorMessage.includes("429") || errorMessage.includes("rate");
+        const isNetworkError = this.isNetworkError(err);
         if (isRateLimited) {
           this.rateLimiter.releaseForRetry();
           await this.rateLimiter.triggerRateLimit();
+          networkRetries = 0;
+        } else if (isNetworkError && networkRetries < MAX_NETWORK_RETRIES) {
+          networkRetries++;
+          const delayMs = INITIAL_RETRY_DELAY_MS * 2 ** (networkRetries - 1);
+          logger.warn(
+            {
+              error: errorMessage,
+              retry: networkRetries,
+              maxRetries: MAX_NETWORK_RETRIES,
+              delayMs
+            },
+            "\u7F51\u7EDC\u9519\u8BEF\uFF0C\u51C6\u5907\u91CD\u8BD5"
+          );
+          this.rateLimiter.releaseForRetry();
+          await sleep(delayMs);
         } else {
           this.rateLimiter.releaseFailure();
+          if (isNetworkError) {
+            logger.error({ error: errorMessage, retries: networkRetries }, "\u7F51\u7EDC\u9519\u8BEF\u91CD\u8BD5\u6B21\u6570\u8017\u5C3D");
+          }
           throw err;
         }
       }
     }
   }
+  /**
+   * 判断是否为网络错误
+   *
+   * 常见网络错误类型：
+   * - terminated: 连接被中断（TLS 断开）
+   * - ECONNRESET: 连接被远端重置
+   * - ETIMEDOUT: 连接超时
+   * - ENOTFOUND: DNS 解析失败
+   * - fetch failed: 通用 fetch 失败
+   * - socket hang up: 套接字意外关闭
+   */
+  isNetworkError(err) {
+    const error = err;
+    const message = (error.message || "").toLowerCase();
+    const code = error.code || "";
+    const networkErrorPatterns = [
+      "terminated",
+      "econnreset",
+      "etimedout",
+      "enotfound",
+      "econnrefused",
+      "fetch failed",
+      "socket hang up",
+      "network",
+      "aborted"
+    ];
+    for (const pattern of networkErrorPatterns) {
+      if (message.includes(pattern)) {
+        return true;
+      }
+    }
+    const networkErrorCodes = ["ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "ECONNREFUSED", "EPIPE"];
+    if (networkErrorCodes.includes(code)) {
+      return true;
+    }
+    return false;
+  }
   /**
    * 处理单个批次（单次请求，不含重试逻辑）
    */
@@ -558,11 +631,12 @@ var Indexer = class {
   }
   /**
    * 处理扫描结果，更新向量索引
-   * 
+   *
    * @param db SQLite 数据库实例
    * @param results 文件处理结果
+   * @param onProgress 可选的进度回调 (indexed, total) => void
    */
-  async indexFiles(db, results) {
+  async indexFiles(db, results, onProgress) {
     if (!this.vectorStore) {
       await this.init();
     }
@@ -606,26 +680,31 @@ var Indexer = class {
       stats.deleted = toDelete.length;
     }
     if (toIndex.length > 0) {
-      const indexResult = await this.batchIndex(db, toIndex);
+      const indexResult = await this.batchIndex(db, toIndex, onProgress);
       stats.indexed = indexResult.success;
       stats.errors = indexResult.errors;
     }
     logger.info(
-      { indexed: stats.indexed, deleted: stats.deleted, errors: stats.errors, skipped: stats.skipped },
+      {
+        indexed: stats.indexed,
+        deleted: stats.deleted,
+        errors: stats.errors,
+        skipped: stats.skipped
+      },
       "\u5411\u91CF\u7D22\u5F15\u5B8C\u6210"
     );
     return stats;
   }
   /**
    * 批量索引文件（性能优化版）
-   * 
+   *
    * 优化策略：
    * 1. Embedding 已批量化（原有）
    * 2. LanceDB 写入批量化：N 次 upsertFile → 1 次 batchUpsertFiles
    * 3. FTS 写入批量化：N 次删除+插入 → 1 次批量删除 + 1 次批量插入
    * 4. 日志汇总化：逐文件日志 → 汇总日志
    */
-  async batchIndex(db, files) {
+  async batchIndex(db, files, onProgress) {
     if (files.length === 0) {
       return { success: 0, errors: 0 };
     }
@@ -646,11 +725,15 @@ var Indexer = class {
     logger.info({ count: allTexts.length, files: files.length }, "\u5F00\u59CB\u6279\u91CF Embedding");
     let embeddings;
     try {
-      const results = await this.embeddingClient.embedBatch(allTexts);
+      const results = await this.embeddingClient.embedBatch(allTexts, 20, onProgress);
       embeddings = results.map((r) => r.embedding);
     } catch (err) {
-      logger.error({ error: err.message, stack: err.stack }, "Embedding \u5931\u8D25");
-      clearVectorIndexHash(db, files.map((f) => f.path));
+      const error = err;
+      logger.error({ error: error.message, stack: error.stack }, "Embedding \u5931\u8D25");
+      clearVectorIndexHash(
+        db,
+        files.map((f) => f.path)
+      );
       return { success: 0, errors: files.length };
     }
     const filesToUpsert = [];
@@ -690,23 +773,35 @@ var Indexer = class {
             filePath: record.file_path,
             chunkIndex: record.chunk_index,
             breadcrumb: record.breadcrumb,
-            content: record.breadcrumb + "\n" + record.display_code
+            content: `${record.breadcrumb}
+${record.display_code}`
           });
         }
         filesToUpsert.push({ path: file.path, hash: file.hash, records });
         successFiles.push({ path: file.path, hash: file.hash });
       } catch (err) {
-        logger.error({ path: file.path, error: err.message, stack: err.stack }, "\u7EC4\u88C5 ChunkRecord \u5931\u8D25");
+        const error = err;
+        logger.error(
+          { path: file.path, error: error.message, stack: error.stack },
+          "\u7EC4\u88C5 ChunkRecord \u5931\u8D25"
+        );
         errorFiles.push(file.path);
       }
     }
     if (filesToUpsert.length > 0) {
       try {
-        await this.vectorStore.batchUpsertFiles(filesToUpsert);
-        logger.info({ files: filesToUpsert.length, chunks: allFtsChunks.length }, "LanceDB \u6279\u91CF\u5199\u5165\u5B8C\u6210");
+        await this.vectorStore?.batchUpsertFiles(filesToUpsert);
+        logger.info(
+          { files: filesToUpsert.length, chunks: allFtsChunks.length },
+          "LanceDB \u6279\u91CF\u5199\u5165\u5B8C\u6210"
+        );
       } catch (err) {
-        logger.error({ error: err.message, stack: err.stack }, "LanceDB \u6279\u91CF\u5199\u5165\u5931\u8D25");
-        clearVectorIndexHash(db, files.map((f) => f.path));
+        const error = err;
+        logger.error({ error: error.message, stack: error.stack }, "LanceDB \u6279\u91CF\u5199\u5165\u5931\u8D25");
+        clearVectorIndexHash(
+          db,
+          files.map((f) => f.path)
+        );
         return { success: 0, errors: files.length };
       }
     }
@@ -715,18 +810,19 @@ var Indexer = class {
         const pathsToDelete = filesToUpsert.map((f) => f.path);
         batchDeleteFileChunksFts(db, pathsToDelete);
         batchUpsertChunkFts(db, allFtsChunks);
-        logger.info({ files: pathsToDelete.length, chunks: allFtsChunks.length }, "FTS \u6279\u91CF\u66F4\u65B0\u5B8C\u6210");
+        logger.info(
+          { files: pathsToDelete.length, chunks: allFtsChunks.length },
+          "FTS \u6279\u91CF\u66F4\u65B0\u5B8C\u6210"
+        );
       } catch (err) {
-        logger.warn({ error: err.message }, "FTS \u6279\u91CF\u66F4\u65B0\u5931\u8D25\uFF08\u5411\u91CF\u7D22\u5F15\u5DF2\u6210\u529F\uFF09");
+        const error = err;
+        logger.warn({ error: error.message }, "FTS \u6279\u91CF\u66F4\u65B0\u5931\u8D25\uFF08\u5411\u91CF\u7D22\u5F15\u5DF2\u6210\u529F\uFF09");
       }
     }
     if (successFiles.length > 0) {
       batchUpdateVectorIndexHash(db, successFiles);
     }
-    logger.info(
-      { success: successFiles.length, errors: errorFiles.length },
-      "\u6279\u91CF\u7D22\u5F15\u5B8C\u6210"
-    );
+    logger.info({ success: successFiles.length, errors: errorFiles.length }, "\u6279\u91CF\u7D22\u5F15\u5B8C\u6210");
     return { success: successFiles.length, errors: errorFiles.length };
   }
   /**
@@ -747,7 +843,7 @@ var Indexer = class {
     if (!this.vectorStore) {
       await this.init();
     }
-    return this.vectorStore.search(queryVector, limit, filter);
+    return this.vectorStore?.search(queryVector, limit, filter);
   }
   /**
    * 文本搜索（先 embedding 再向量搜索）
@@ -763,7 +859,7 @@ var Indexer = class {
     if (!this.vectorStore) {
       await this.init();
     }
-    await this.vectorStore.clear();
+    await this.vectorStore?.clear();
   }
   /**
    * 获取索引统计
@@ -772,7 +868,7 @@ var Indexer = class {
     if (!this.vectorStore) {
       await this.init();
     }
-    const count = await this.vectorStore.count();
+    const count = await this.vectorStore?.count() ?? 0;
     return { totalChunks: count };
   }
 };

package/dist/chunk-C7XDGBT5.js

@@ -0,0 +1,172 @@
+import {
+  isDev,
+  isMcpMode
+} from "./chunk-SKBAE26T.js";
+
+// src/utils/logger.ts
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { Writable } from "stream";
+import pino from "pino";
+var logLevel = isDev ? "debug" : "info";
+var logDir = path.join(os.homedir(), ".contextweaver", "logs");
+var LOG_RETENTION_DAYS = 7;
+function ensureLogDir(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+function getLogFileName() {
+  const now = /* @__PURE__ */ new Date();
+  const dateStr = now.toISOString().split("T")[0];
+  return `app.${dateStr}.log`;
+}
+function formatTime() {
+  const now = /* @__PURE__ */ new Date();
+  const pad = (n) => n.toString().padStart(2, "0");
+  return `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}:${pad(now.getMinutes())}:${pad(now.getSeconds())}`;
+}
+function getLevelLabel(level) {
+  const labels = {
+    10: "TRACE",
+    20: "DEBUG",
+    30: "INFO",
+    40: "WARN",
+    50: "ERROR",
+    60: "FATAL"
+  };
+  return labels[level] || "INFO";
+}
+function cleanupOldLogs(dir) {
+  try {
+    if (!fs.existsSync(dir)) return;
+    const files = fs.readdirSync(dir);
+    const now = Date.now();
+    const maxAge = LOG_RETENTION_DAYS * 24 * 60 * 60 * 1e3;
+    const logPattern = /^app\.(\d{4}-\d{2}-\d{2})\.log$/;
+    for (const file of files) {
+      const match = file.match(logPattern);
+      if (!match) continue;
+      const dateStr = match[1];
+      const fileDate = new Date(dateStr).getTime();
+      if (Number.isNaN(fileDate)) continue;
+      if (now - fileDate > maxAge) {
+        const filePath = path.join(dir, file);
+        try {
+          fs.unlinkSync(filePath);
+          console.error(`[Logger] \u6E05\u7406\u8FC7\u671F\u65E5\u5FD7: ${file}`);
+        } catch {
+        }
+      }
+    }
+  } catch {
+  }
+}
+function createFormattedStream(filePath) {
+  const writeStream = fs.createWriteStream(filePath, { flags: "a" });
+  return new Writable({
+    write(chunk, _encoding, callback) {
+      try {
+        const log = JSON.parse(chunk.toString());
+        const time = formatTime();
+        const level = getLevelLabel(log.level);
+        const msg = log.msg || "";
+        const { level: _l, time: _t, pid: _p, hostname: _h, name: _n, msg: _m, ...extra } = log;
+        let line = `${time} [${level}] ${msg}`;
+        if (Object.keys(extra).length > 0) {
+          const extraLines = JSON.stringify(extra, null, 2).split("\n").map((l, i) => i === 0 ? l : `    ${l}`).join("\n");
+          line += `
+    ${extraLines}`;
+        }
+        writeStream.write(`${line}
+`, callback);
+      } catch {
+        writeStream.write(chunk.toString(), callback);
+      }
+    }
+  });
+}
+function createConsoleStream() {
+  const colors = {
+    10: "\x1B[90m",
+    // TRACE - 灰色
+    20: "\x1B[36m",
+    // DEBUG - 青色
+    30: "\x1B[32m",
+    // INFO - 绿色
+    40: "\x1B[33m",
+    // WARN - 黄色
+    50: "\x1B[31m",
+    // ERROR - 红色
+    60: "\x1B[35m"
+    // FATAL - 品红
+  };
+  const reset = "\x1B[0m";
+  return new Writable({
+    write(chunk, _encoding, callback) {
+      try {
+        const log = JSON.parse(chunk.toString());
+        const time = formatTime();
+        const level = getLevelLabel(log.level);
+        const color = colors[log.level] || "";
+        const msg = log.msg || "";
+        const { level: _l, time: _t, pid: _p, hostname: _h, name: _n, msg: _m, ...extra } = log;
+        let line = `${color}${time} [${level}]${reset} ${msg}`;
+        if (Object.keys(extra).length > 0) {
+          const extraStr = JSON.stringify(extra);
+          line += ` ${color}${extraStr}${reset}`;
+        }
+        process.stdout.write(`${line}
+`, callback);
+      } catch {
+        process.stdout.write(chunk.toString(), callback);
+      }
+    }
+  });
+}
+function createDevLogger() {
+  ensureLogDir(logDir);
+  cleanupOldLogs(logDir);
+  const logPath = path.join(logDir, getLogFileName());
+  const logStream = createFormattedStream(logPath);
+  const consoleStream = createConsoleStream();
+  return pino(
+    {
+      level: logLevel,
+      name: "contextweaver"
+    },
+    // MCP 模式下禁用控制台输出，避免污染 STDIO 协议流
+    isMcpMode ? logStream : pino.multistream([
+      { stream: logStream, level: logLevel },
+      { stream: consoleStream, level: logLevel }
+    ])
+  );
+}
+function createProdLogger() {
+  ensureLogDir(logDir);
+  cleanupOldLogs(logDir);
+  const logPath = path.join(logDir, getLogFileName());
+  const logStream = createFormattedStream(logPath);
+  const consoleStream = createConsoleStream();
+  return pino(
+    {
+      level: logLevel,
+      name: "contextweaver"
+    },
+    // MCP 模式下禁用控制台输出，避免污染 STDIO 协议流
+    isMcpMode ? logStream : pino.multistream([
+      { stream: logStream, level: logLevel },
+      { stream: consoleStream, level: logLevel }
+    ])
+  );
+}
+var logger = isDev ? createDevLogger() : createProdLogger();
+function isDebugEnabled() {
+  return logger.isLevelEnabled("debug");
+}
+
+export {
+  logger,
+  isDebugEnabled
+};