@chiway/contextweaver 1.1.0 → 1.4.0

package/dist/{chunk-7G5V7YT5.js → chunk-EMSMLPMK.js}

@@ -1,9 +1,9 @@
 import {
   generateProjectId
-} from "./chunk-6Z4JEEVJ.js";
+} from "./chunk-RGJSXUFS.js";
 import {
   logger
-} from "./chunk-AMQQK4P7.js";
+} from "./chunk-JVKVSTQ3.js";
 
 // src/mcp/tools/codebaseRetrieval.ts
 import fs from "fs";
@@ -59,8 +59,8 @@ function isProjectIndexed(projectId) {
   return fs.existsSync(dbPath);
 }
 async function ensureIndexed(repoPath, projectId, onProgress) {
-  const { withLock } = await import("./lock-DVY3KJSK.js");
-  const { scan } = await import("./scanner-RFG4YWYI.js");
+  const { withLock } = await import("./lock-FL54LIQL.js");
+  const { scan } = await import("./scanner-2XGJWYHR.js");
   await withLock(
     projectId,
     "index",
@@ -105,7 +105,7 @@ async function handleCodebaseRetrieval(args, onProgress) {
     },
     "MCP codebase-retrieval \u8C03\u7528\u5F00\u59CB"
   );
-  const { checkEmbeddingEnv, checkRerankerEnv } = await import("./config-BWZ6CU3W.js");
+  const { checkEmbeddingEnv, checkRerankerEnv } = await import("./config-LCOJHTCF.js");
   const embeddingCheck = checkEmbeddingEnv();
   const rerankerCheck = checkRerankerEnv();
   const allMissingVars = [...embeddingCheck.missingVars, ...rerankerCheck.missingVars];
@@ -124,7 +124,7 @@ async function handleCodebaseRetrieval(args, onProgress) {
     },
     "MCP \u67E5\u8BE2\u6784\u5EFA"
   );
-  const { SearchService } = await import("./SearchService-MYPOCM3B.js");
+  const { SearchService } = await import("./SearchService-OS7CYHNJ.js");
   const service = new SearchService(projectId, repo_path);
   await service.init();
   logger.debug("SearchService \u521D\u59CB\u5316\u5B8C\u6210");
@@ -283,4 +283,3 @@ export {
   codebaseRetrievalSchema,
   handleCodebaseRetrieval
 };
-//# sourceMappingURL=chunk-7G5V7YT5.js.map

package/dist/{chunk-AMQQK4P7.js → chunk-JVKVSTQ3.js}

@@ -1,7 +1,7 @@
 import {
   isDev,
   isMcpMode
-} from "./chunk-RJURH22T.js";
+} from "./chunk-SKBAE26T.js";
 
 // src/utils/logger.ts
 import fs from "fs";
@@ -168,4 +168,3 @@ export {
   logger,
   isDebugEnabled
 };
-//# sourceMappingURL=chunk-AMQQK4P7.js.map

package/dist/{chunk-6Z4JEEVJ.js → chunk-RGJSXUFS.js}

@@ -1,7 +1,14 @@
 import {
   isDebugEnabled,
   logger
-} from "./chunk-AMQQK4P7.js";
+} from "./chunk-JVKVSTQ3.js";
+
+// src/db/index.ts
+import crypto from "crypto";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import Database from "better-sqlite3";
 
 // src/search/fts.ts
 var tokenizerCache = /* @__PURE__ */ new WeakMap();
@@ -26,7 +33,7 @@ function detectFtsTokenizer(db) {
 function initFilesFts(db) {
   const tokenizer = detectFtsTokenizer(db);
   const tableExists = db.prepare(`
-        SELECT name FROM sqlite_master 
+        SELECT name FROM sqlite_master
         WHERE type='table' AND name='files_fts'
     `).get();
   if (!tableExists) {
@@ -34,25 +41,40 @@ function initFilesFts(db) {
             CREATE VIRTUAL TABLE files_fts USING fts5(
                 path,
                 content,
+                content='files',
+                content_rowid='rowid',
                 tokenize='${tokenizer}'
             );
         `);
-    logger.info(`\u521B\u5EFA files_fts \u8868\uFF0Ctokenizer=${tokenizer}`);
-    syncFilesFts(db);
-  }
-}
-function syncFilesFts(db) {
-  const fileCount = db.prepare("SELECT COUNT(*) as c FROM files WHERE content IS NOT NULL").get().c;
-  const ftsCount = db.prepare("SELECT COUNT(*) as c FROM files_fts").get().c;
-  if (ftsCount < fileCount) {
-    logger.info(`\u540C\u6B65 FTS \u7D22\u5F15: files=${fileCount}, fts=${ftsCount}`);
-    db.exec(`
-            DELETE FROM files_fts;
-            INSERT INTO files_fts(path, content) 
-            SELECT path, content FROM files WHERE content IS NOT NULL;
-        `);
-    logger.info(`FTS \u7D22\u5F15\u540C\u6B65\u5B8C\u6210: ${fileCount} \u6761\u8BB0\u5F55`);
+    logger.info(`\u521B\u5EFA files_fts \u8868\uFF08\u5916\u90E8\u5185\u5BB9\u8868\uFF09\uFF0Ctokenizer=${tokenizer}`);
   }
+  db.exec(`
+        CREATE TRIGGER IF NOT EXISTS files_ai
+        AFTER INSERT ON files
+        WHEN new.content IS NOT NULL
+        BEGIN
+            INSERT INTO files_fts(rowid, path, content)
+            VALUES (new.rowid, new.path, new.content);
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS files_ad
+        AFTER DELETE ON files
+        WHEN old.content IS NOT NULL
+        BEGIN
+            INSERT INTO files_fts(files_fts, rowid, path, content)
+            VALUES('delete', old.rowid, old.path, old.content);
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS files_au
+        AFTER UPDATE ON files
+        WHEN old.content IS NOT NULL OR new.content IS NOT NULL
+        BEGIN
+            INSERT INTO files_fts(files_fts, rowid, path, content)
+            SELECT 'delete', old.rowid, old.path, old.content WHERE old.content IS NOT NULL;
+            INSERT INTO files_fts(rowid, path, content)
+            SELECT new.rowid, new.path, new.content WHERE new.content IS NOT NULL;
+        END;
+    `);
 }
 function initChunksFts(db) {
   const tokenizer = detectFtsTokenizer(db);
@@ -82,13 +104,17 @@ function isChunksFtsInitialized(db) {
   return !!result;
 }
 function batchUpsertChunkFts(db, chunks) {
-  const deleteStmt = db.prepare("DELETE FROM chunks_fts WHERE chunk_id = ?");
+  if (chunks.length === 0) return;
+  const paths = Array.from(new Set(chunks.map((c) => c.filePath)));
+  const deleteByPath = db.prepare("DELETE FROM chunks_fts WHERE file_path = ?");
   const insertStmt = db.prepare(
     "INSERT INTO chunks_fts(chunk_id, file_path, chunk_index, breadcrumb, content) VALUES (?, ?, ?, ?, ?)"
   );
   const transaction = db.transaction((items) => {
+    for (const p of paths) {
+      deleteByPath.run(p);
+    }
     for (const item of items) {
-      deleteStmt.run(item.chunkId);
       insertStmt.run(item.chunkId, item.filePath, item.chunkIndex, item.breadcrumb, item.content);
     }
   });
@@ -172,26 +198,6 @@ function searchChunksFts(db, query, limit) {
   }
   return results.sort((a, b) => b.score - a.score);
 }
-function batchUpsertFileFts(db, files) {
-  const deleteFts = db.prepare("DELETE FROM files_fts WHERE path = ?");
-  const insertFts = db.prepare("INSERT INTO files_fts(path, content) VALUES (?, ?)");
-  const transaction = db.transaction((items) => {
-    for (const item of items) {
-      deleteFts.run(item.path);
-      insertFts.run(item.path, item.content);
-    }
-  });
-  transaction(files);
-}
-function batchDeleteFileFts(db, paths) {
-  const stmt = db.prepare("DELETE FROM files_fts WHERE path = ?");
-  const transaction = db.transaction((items) => {
-    for (const path2 of items) {
-      stmt.run(path2);
-    }
-  });
-  transaction(paths);
-}
 function sanitizeQuery(query) {
   return query.replace(/[():"*^./\\:@#$%&=+[\]{}<>|~`!?,;]/g, " ").replace(/\b(AND|OR|NOT|NEAR)\b/gi, " ").replace(/\s+/g, " ").trim();
 }
@@ -342,11 +348,6 @@ function isFtsInitialized(db) {
 }
 
 // src/db/index.ts
-import crypto from "crypto";
-import fs from "fs";
-import os from "os";
-import path from "path";
-import Database from "better-sqlite3";
 var BASE_DIR = path.join(os.homedir(), ".contextweaver");
 function getDirectoryBirthtime(projectPath) {
   const gitDir = path.join(projectPath, ".git");
@@ -405,6 +406,7 @@ function initDb(projectId) {
       value TEXT NOT NULL
     )
   `);
+  migrateSchema(db);
   initFilesFts(db);
   initChunksFts(db);
   db.pragma("synchronous = NORMAL");
@@ -412,6 +414,148 @@ function initDb(projectId) {
   db.pragma("cache_size = -64000");
   return db;
 }
+var CURRENT_SCHEMA_VERSION = 3;
+var METADATA_KEY_SCHEMA_VERSION = "schema_version";
+function getSchemaVersion(db) {
+  const row = db.prepare("SELECT value FROM metadata WHERE key = ?").get(METADATA_KEY_SCHEMA_VERSION);
+  if (!row) return null;
+  const parsed = parseInt(row.value, 10);
+  return Number.isNaN(parsed) ? null : parsed;
+}
+function setSchemaVersion(db, version) {
+  db.prepare(`
+    INSERT INTO metadata (key, value)
+    VALUES (?, ?)
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+  `).run(METADATA_KEY_SCHEMA_VERSION, String(version));
+}
+function isOldFilesFtsSchema(db) {
+  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
+  if (!row?.sql) return false;
+  return !row.sql.includes("content='files'");
+}
+function migrateSchema(db) {
+  const backupExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts_v1_backup'`).get();
+  const currentFtsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
+  if (backupExists && currentFtsExists && !isOldFilesFtsSchema(db)) {
+    logger.warn("\u68C0\u6D4B\u5230\u6B8B\u7559\u5907\u4EFD\u8868 files_fts_v1_backup\uFF0C\u6E05\u7406\u4E2D");
+    db.exec("DROP TABLE files_fts_v1_backup");
+  }
+  const current = getSchemaVersion(db);
+  if (current === null) {
+    const fileCount = db.prepare("SELECT COUNT(*) as c FROM files").get().c;
+    const ftsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
+    if (fileCount === 0 && !ftsExists) {
+      migrateToV3(db);
+      setSchemaVersion(db, CURRENT_SCHEMA_VERSION);
+      return;
+    }
+  }
+  if ((current ?? 1) < 2) {
+    migrateToV2(db);
+    setSchemaVersion(db, 2);
+  }
+  if ((current ?? 2) < 3) {
+    migrateToV3(db);
+    setSchemaVersion(db, 3);
+  }
+}
+function migrateToV2(db) {
+  const ftsExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='files_fts'`).get();
+  if (!ftsExists) {
+    return;
+  }
+  if (!isOldFilesFtsSchema(db)) {
+    return;
+  }
+  logger.info("\u6267\u884C schema \u8FC1\u79FB v1 \u2192 v2: files_fts \u8F6C\u4E3A\u5916\u90E8\u5185\u5BB9\u8868");
+  db.exec("DROP TABLE files_fts");
+  let tokenizer;
+  try {
+    db.exec(
+      `CREATE VIRTUAL TABLE IF NOT EXISTS _fts_probe USING fts5(content, tokenize='trigram');
+       DROP TABLE IF EXISTS _fts_probe;`
+    );
+    tokenizer = "trigram";
+  } catch {
+    tokenizer = "unicode61";
+  }
+  db.exec(`
+    CREATE VIRTUAL TABLE files_fts USING fts5(
+        path,
+        content,
+        content='files',
+        content_rowid='rowid',
+        tokenize='${tokenizer}'
+    );
+  `);
+  db.exec(`INSERT INTO files_fts(files_fts) VALUES('rebuild')`);
+  logger.info("schema \u8FC1\u79FB v1 \u2192 v2 \u5B8C\u6210");
+}
+function migrateToV3(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS pending_marks (
+      path TEXT PRIMARY KEY,
+      hash TEXT NOT NULL,
+      created_at INTEGER NOT NULL
+    );
+  `);
+  logger.info("schema \u8FC1\u79FB v2 \u2192 v3 \u5B8C\u6210: pending_marks \u8868\u5DF2\u521B\u5EFA");
+}
+function insertPendingMarks(db, items) {
+  if (items.length === 0) return;
+  const now = Date.now();
+  const insert = db.prepare(`
+    INSERT INTO pending_marks (path, hash, created_at)
+    VALUES (?, ?, ?)
+    ON CONFLICT(path) DO UPDATE SET hash = excluded.hash, created_at = excluded.created_at
+  `);
+  const tx = db.transaction((data) => {
+    for (const it of data) {
+      insert.run(it.path, it.hash, now);
+    }
+  });
+  tx(items);
+}
+function deletePendingMarks(db, paths) {
+  if (paths.length === 0) return;
+  const del = db.prepare("DELETE FROM pending_marks WHERE path = ?");
+  const tx = db.transaction((items) => {
+    for (const p of items) del.run(p);
+  });
+  tx(paths);
+}
+function replayPendingMarks(db) {
+  const rows = db.prepare("SELECT path, hash FROM pending_marks").all();
+  if (rows.length === 0) return { applied: 0, discarded: 0 };
+  const update = db.prepare(`
+    UPDATE files SET vector_index_hash = ?
+    WHERE path = ? AND hash = ?
+  `);
+  const del = db.prepare("DELETE FROM pending_marks WHERE path = ?");
+  let applied = 0;
+  let discarded = 0;
+  const tx = db.transaction(() => {
+    for (const r of rows) {
+      const info = update.run(r.hash, r.path, r.hash);
+      if (info.changes > 0) {
+        applied++;
+      } else {
+        discarded++;
+      }
+      del.run(r.path);
+    }
+  });
+  tx();
+  if (applied > 0 || discarded > 0) {
+    logger.info({ applied, discarded }, "pending_marks \u91CD\u653E\u5B8C\u6210");
+  }
+  return { applied, discarded };
+}
+function countPendingMarks(db) {
+  const row = db.prepare("SELECT COUNT(*) as c FROM pending_marks").get();
+  return row.c;
+}
 function closeDb(db) {
   db.close();
 }
@@ -467,15 +611,6 @@ function batchUpsert(db, files) {
     }
   });
   transaction(files);
-  const ftsFiles = [];
-  for (const f of files) {
-    if (f.content !== null) {
-      ftsFiles.push({ path: f.path, content: f.content });
-    }
-  }
-  if (ftsFiles.length > 0) {
-    batchUpsertFileFts(db, ftsFiles);
-  }
 }
 function batchUpdateMtime(db, items) {
   const update = db.prepare("UPDATE files SET mtime = ? WHERE path = ?");
@@ -498,9 +633,6 @@ function batchDelete(db, paths) {
     }
   });
   transaction(paths);
-  if (paths.length > 0) {
-    batchDeleteFileFts(db, paths);
-  }
 }
 function clear(db) {
   db.exec("DELETE FROM files");
@@ -508,6 +640,9 @@ function clear(db) {
   db.exec("DELETE FROM chunks_fts");
 }
 var METADATA_KEY_EMBEDDING_DIMENSIONS = "embedding_dimensions";
+var METADATA_KEY_LANCEDB_MIGRATION_STATE = "lancedb_migration_displaycode_state";
+var METADATA_KEY_LANCEDB_MIGRATION_LOCK = "lancedb_migration_lock";
+var MIGRATION_LOCK_STALE_MS = 10 * 60 * 1e3;
 function getMetadata(db, key) {
   const row = db.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
   return row?.value ?? null;
@@ -528,6 +663,47 @@ function getStoredEmbeddingDimensions(db) {
 function setStoredEmbeddingDimensions(db, dimensions) {
   setMetadata(db, METADATA_KEY_EMBEDDING_DIMENSIONS, String(dimensions));
 }
+function getLanceDbMigrationState(db) {
+  const value = getMetadata(db, METADATA_KEY_LANCEDB_MIGRATION_STATE);
+  if (value === "pending" || value === "done" || value === "aborted") return value;
+  return null;
+}
+function setLanceDbMigrationState(db, state) {
+  setMetadata(db, METADATA_KEY_LANCEDB_MIGRATION_STATE, state);
+}
+function clearAllVectorIndexHash(db) {
+  const info = db.prepare("UPDATE files SET vector_index_hash = NULL").run();
+  return info.changes;
+}
+function tryAcquireLanceDbMigrationLock(db) {
+  const now = Date.now();
+  const pid = process.pid;
+  const lockValue = JSON.stringify({ pid, acquiredAt: now });
+  const existing = getMetadata(db, METADATA_KEY_LANCEDB_MIGRATION_LOCK);
+  if (existing) {
+    try {
+      const parsed = JSON.parse(existing);
+      if (parsed.pid === pid) return true;
+      if (now - parsed.acquiredAt < MIGRATION_LOCK_STALE_MS) return false;
+      logger.warn(
+        { stalePid: parsed.pid, age: now - parsed.acquiredAt },
+        "\u68C0\u6D4B\u5230\u50F5\u5C38\u8FC1\u79FB\u9501\uFF0C\u5F3A\u5236\u593A\u53D6"
+      );
+    } catch {
+    }
+  }
+  db.prepare(`
+    INSERT INTO metadata (key, value)
+    VALUES (?, ?)
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+  `).run(METADATA_KEY_LANCEDB_MIGRATION_LOCK, lockValue);
+  const reread = getMetadata(db, METADATA_KEY_LANCEDB_MIGRATION_LOCK);
+  if (reread !== lockValue) return false;
+  return true;
+}
+function releaseLanceDbMigrationLock(db) {
+  db.prepare("DELETE FROM metadata WHERE key = ?").run(METADATA_KEY_LANCEDB_MIGRATION_LOCK);
+}
 
 export {
   isChunksFtsInitialized,
@@ -539,6 +715,11 @@ export {
   isFtsInitialized,
   generateProjectId,
   initDb,
+  migrateSchema,
+  insertPendingMarks,
+  deletePendingMarks,
+  replayPendingMarks,
+  countPendingMarks,
   closeDb,
   getAllFileMeta,
   getFilesNeedingVectorIndex,
@@ -550,6 +731,10 @@ export {
   batchDelete,
   clear,
   getStoredEmbeddingDimensions,
-  setStoredEmbeddingDimensions
+  setStoredEmbeddingDimensions,
+  getLanceDbMigrationState,
+  setLanceDbMigrationState,
+  clearAllVectorIndexHash,
+  tryAcquireLanceDbMigrationLock,
+  releaseLanceDbMigrationLock
 };
-//# sourceMappingURL=chunk-6Z4JEEVJ.js.map

package/dist/{chunk-RJURH22T.js → chunk-SKBAE26T.js}

@@ -263,4 +263,3 @@ export {
   getRerankerConfig,
   getExcludePatterns
 };
-//# sourceMappingURL=chunk-RJURH22T.js.map

package/dist/{chunk-NQR4CGQ6.js → chunk-X7PAYQMT.js}

@@ -1,9 +1,11 @@
 import {
   closeAllIndexers,
-  closeAllVectorStores,
   getIndexer,
   invalidateAllExpanderCaches
-} from "./chunk-6QMYML5V.js";
+} from "./chunk-AB24E3Z7.js";
+import {
+  closeAllVectorStores
+} from "./chunk-ZOMGPIU6.js";
 import {
   batchDelete,
   batchUpdateMtime,
@@ -17,14 +19,14 @@ import {
   getStoredEmbeddingDimensions,
   initDb,
   setStoredEmbeddingDimensions
-} from "./chunk-6Z4JEEVJ.js";
+} from "./chunk-RGJSXUFS.js";
 import {
   logger
-} from "./chunk-AMQQK4P7.js";
+} from "./chunk-JVKVSTQ3.js";
 import {
   getEmbeddingConfig,
   getExcludePatterns
-} from "./chunk-RJURH22T.js";
+} from "./chunk-SKBAE26T.js";
 
 // src/scanner/index.ts
 import path3 from "path";
@@ -523,6 +525,19 @@ var SourceAdapter = class {
   /**
    * 将字节偏移转换为字符偏移
    */
+  /**
+   * 将 tree-sitter 返回的偏移（可能是 UTF-8 字节或 UTF-16 字符域）
+   * 标准化为 UTF-16 字符域偏移。下游 String.prototype.slice 直接可用。
+   *
+   * 导出供 SemanticSplitter 在生成 ChunkMetadata 时统一域。
+   */
+  toCharOffset(offset) {
+    if (this.domain === "utf16" || this.domain === "unknown") return offset;
+    return this.byteToChar(offset);
+  }
+  /**
+   * 将字节偏移转换为字符偏移（仅 utf8 域有效；utf16/unknown 直接返回原值）
+   */
   byteToChar(byteOffset) {
     if (!this.byteToCharMap) return byteOffset;
     const safeOffset = Math.max(0, Math.min(this.byteToCharMap.length - 1, byteOffset));
@@ -915,11 +930,12 @@ ${displayCode}`,
       const vectorEnd = end;
       const displayCode = this.adapter.slice(start, end);
       const vectorCode = this.adapter.slice(vectorStart, vectorEnd);
+      const toChar = (n) => this.adapter.toCharOffset(n);
       const metadata = {
-        startIndex: start,
-        endIndex: end,
-        rawSpan: { start: prevEnd, end: rawSpanEnd },
-        vectorSpan: { start: vectorStart, end: vectorEnd },
+        startIndex: toChar(start),
+        endIndex: toChar(end),
+        rawSpan: { start: toChar(prevEnd), end: toChar(rawSpanEnd) },
+        vectorSpan: { start: toChar(vectorStart), end: toChar(vectorEnd) },
         filePath,
         language,
         contextPath: w.contextPath
@@ -1351,6 +1367,21 @@ async function scan(rootPath, options = {}) {
       }
     }
     options.onProgress?.(100, 100, "\u7D22\u5F15\u5B8C\u6210");
+    if (options.vectorIndex !== false) {
+      try {
+        const embeddingConfig = getEmbeddingConfig();
+        const indexer = await getIndexer(projectId, embeddingConfig.dimensions);
+        const gcResult = await indexer.gc(db);
+        if (gcResult.orphans > 0) {
+          logger.info({ orphans: gcResult.orphans }, "GC \u5B8C\u6210");
+        } else if (gcResult.truncated) {
+          logger.debug("GC \u8D85\u65F6\u8DF3\u8FC7\uFF0C\u4E0B\u6B21\u626B\u63CF\u91CD\u8BD5");
+        }
+      } catch (err) {
+        const error = err;
+        logger.warn({ error: error.message }, "GC \u8DF3\u8FC7");
+      }
+    }
     invalidateAllExpanderCaches();
     return stats;
   } finally {
@@ -1363,4 +1394,3 @@ async function scan(rootPath, options = {}) {
 export {
   scan
 };
-//# sourceMappingURL=chunk-NQR4CGQ6.js.map