@hsingjui/contextweaver 0.0.1

package/dist/chunk-34YZ2U3O.js

@@ -0,0 +1,1177 @@
+import {
+  closeAllIndexers,
+  closeAllVectorStores,
+  getIndexer
+} from "./chunk-6C2D5Y4R.js";
+import {
+  batchDelete,
+  batchUpdateMtime,
+  batchUpsert,
+  clear,
+  closeDb,
+  generateProjectId,
+  getAllFileMeta,
+  getAllPaths,
+  getFilesNeedingVectorIndex,
+  getStoredEmbeddingDimensions,
+  initDb,
+  logger,
+  setStoredEmbeddingDimensions
+} from "./chunk-5SRSUMKW.js";
+import {
+  getEmbeddingConfig,
+  getExcludePatterns
+} from "./chunk-PN7DP6XL.js";
+
+// src/scanner/crawler.ts
+import { fdir } from "fdir";
+import path2 from "path";
+
+// src/scanner/filter.ts
+import ignore from "ignore";
+import fs from "fs/promises";
+import path from "path";
+
+// src/scanner/language.ts
+var LANGUAGE_MAP = {
+  ".ts": "typescript",
+  ".tsx": "typescript",
+  ".js": "javascript",
+  ".jsx": "javascript",
+  ".mjs": "javascript",
+  ".cjs": "javascript",
+  ".md": "markdown",
+  ".py": "python",
+  ".go": "go",
+  ".rs": "rust",
+  ".java": "java",
+  ".kt": "kotlin",
+  ".swift": "swift",
+  ".cpp": "cpp",
+  ".cc": "cpp",
+  ".cxx": "cpp",
+  ".hpp": "cpp",
+  ".h": "cpp",
+  ".c": "c",
+  ".sh": "shell",
+  ".bash": "shell",
+  ".zsh": "shell",
+  ".fish": "shell",
+  ".ps1": "powershell",
+  ".sql": "sql",
+  ".yaml": "yaml",
+  ".yml": "yaml",
+  ".json": "json",
+  ".toml": "toml",
+  ".xml": "xml",
+  ".html": "html",
+  ".css": "css",
+  ".scss": "scss",
+  ".sass": "sass",
+  ".less": "less",
+  ".vue": "vue",
+  ".svelte": "svelte",
+  ".rb": "ruby",
+  ".php": "php",
+  ".dart": "dart",
+  ".lua": "lua",
+  ".r": "r"
+};
+var ALLOWED_EXTENSIONS = new Set(Object.keys(LANGUAGE_MAP));
+function getLanguage(filePath) {
+  const ext = getFileExtension(filePath);
+  return LANGUAGE_MAP[ext] || "unknown";
+}
+function isAllowedExtension(filePath) {
+  const ext = getFileExtension(filePath);
+  return ALLOWED_EXTENSIONS.has(ext);
+}
+function getFileExtension(filePath) {
+  const ext = filePath.split(".").pop();
+  return ext ? `.${ext.toLowerCase()}` : "";
+}
+
+// src/scanner/filter.ts
+var ignoreInstance = null;
+var lastConfigHash = null;
+var CONFIG_FILES = [
+  ".gitignore",
+  ".contextweaverignore"
+];
+async function generateConfigHash(rootPath) {
+  const crypto2 = await import("crypto");
+  const hashes = [];
+  for (const file of CONFIG_FILES) {
+    const filePath = path.join(rootPath, file);
+    try {
+      const content = await fs.readFile(filePath, "utf-8");
+      hashes.push(crypto2.createHash("sha256").update(content).digest("hex"));
+    } catch {
+    }
+  }
+  const envPatterns = process.env.IGNORE_PATTERNS || "";
+  const envHash = crypto2.createHash("sha256").update(envPatterns).digest("hex");
+  hashes.push(envHash);
+  const combined = hashes.join("|");
+  return crypto2.createHash("sha256").update(combined).digest("hex");
+}
+async function initFilter(rootPath) {
+  const currentHash = await generateConfigHash(rootPath);
+  if (lastConfigHash === currentHash && ignoreInstance) {
+    return;
+  }
+  const ig = ignore();
+  const patterns = getExcludePatterns();
+  ig.add(patterns);
+  const gitignorePath = path.join(rootPath, ".gitignore");
+  try {
+    await fs.access(gitignorePath);
+    ig.add(await fs.readFile(gitignorePath, "utf-8"));
+  } catch {
+  }
+  const cwignorePath = path.join(rootPath, ".contextweaverignore");
+  try {
+    await fs.access(cwignorePath);
+    ig.add(await fs.readFile(cwignorePath, "utf-8"));
+  } catch {
+  }
+  ignoreInstance = ig;
+  lastConfigHash = currentHash;
+}
+function isFiltered(relativePath) {
+  if (!ignoreInstance) {
+    throw new Error("Filter not initialized. Call initFilter() first.");
+  }
+  return ignoreInstance.ignores(relativePath);
+}
+function isAllowedFile(filePath) {
+  return isAllowedExtension(filePath);
+}
+
+// src/scanner/crawler.ts
+async function crawl(rootPath) {
+  const api = new fdir().withFullPaths().withErrors().filter((filePath) => {
+    const relativePath = filePath.replace(rootPath + path2.sep, "").replace(new RegExp(`^${rootPath}/?`), "");
+    return !isFiltered(relativePath) && isAllowedFile(filePath);
+  });
+  const paths = await api.crawl(rootPath).withPromise();
+  return paths;
+}
+
+// src/scanner/processor.ts
+import fs3 from "fs/promises";
+import pLimit from "p-limit";
+import path3 from "path";
+import os from "os";
+
+// src/scanner/hash.ts
+import crypto from "crypto";
+function sha256(data) {
+  return crypto.createHash("sha256").update(data).digest("hex");
+}
+
+// src/utils/encoding.ts
+import fs2 from "fs/promises";
+import chardet from "chardet";
+import iconv from "iconv-lite";
+function normalizeEncoding(encoding) {
+  const map = {
+    "UTF-8": "utf8",
+    "UTF-16 LE": "utf16le",
+    "UTF-16 BE": "utf16be",
+    "UTF-32 LE": "utf32le",
+    "UTF-32 BE": "utf32be",
+    "GB18030": "gb18030",
+    "GBK": "gbk",
+    "GB2312": "gb2312",
+    "Big5": "big5",
+    "Shift_JIS": "shiftjis",
+    "EUC-JP": "eucjp",
+    "EUC-KR": "euckr",
+    "ISO-8859-1": "iso88591",
+    "windows-1252": "win1252",
+    "ASCII": "utf8"
+    // ASCII 是 UTF-8 的子集
+  };
+  return map[encoding] || encoding.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+function detectBOM(buffer) {
+  if (buffer.length >= 3) {
+    if (buffer[0] === 239 && buffer[1] === 187 && buffer[2] === 191) {
+      return "UTF-8";
+    }
+  }
+  if (buffer.length >= 4) {
+    if (buffer[0] === 255 && buffer[1] === 254 && buffer[2] === 0 && buffer[3] === 0) {
+      return "UTF-32 LE";
+    }
+    if (buffer[0] === 0 && buffer[1] === 0 && buffer[2] === 254 && buffer[3] === 255) {
+      return "UTF-32 BE";
+    }
+  }
+  if (buffer.length >= 2) {
+    if (buffer[0] === 255 && buffer[1] === 254) {
+      return "UTF-16 LE";
+    }
+    if (buffer[0] === 254 && buffer[1] === 255) {
+      return "UTF-16 BE";
+    }
+  }
+  return null;
+}
+async function readFileWithEncoding(filePath) {
+  const buffer = await fs2.readFile(filePath);
+  const bom = detectBOM(buffer);
+  let encoding = bom;
+  if (!encoding) {
+    const detected = chardet.detect(buffer);
+    encoding = detected || "UTF-8";
+  }
+  const normalizedEncoding = normalizeEncoding(encoding);
+  let content;
+  try {
+    if (iconv.encodingExists(normalizedEncoding)) {
+      content = iconv.decode(buffer, normalizedEncoding);
+    } else {
+      content = buffer.toString("utf-8");
+    }
+  } catch {
+    content = buffer.toString("utf-8");
+  }
+  return {
+    content,
+    encoding: "utf-8",
+    // 输出始终是 UTF-8
+    originalEncoding: encoding
+  };
+}
+
+// src/chunking/SemanticSplitter.ts
+import "tree-sitter";
+
+// src/chunking/SourceAdapter.ts
+var SourceAdapter = class {
+  code;
+  domain;
+  buffer;
+  // UTF-8 字节偏移 -> 字符偏移的映射表（仅 UTF-8 域使用）
+  byteToCharMap;
+  // UTF-16 前缀和（用于 NWS 计算）
+  nwsPrefixSum;
+  constructor(config) {
+    this.code = config.code;
+    const lenUtf16 = config.code.length;
+    const lenUtf8 = Buffer.byteLength(config.code, "utf8");
+    if (config.endIndex === lenUtf16) {
+      this.domain = "utf16";
+      this.buffer = null;
+      this.byteToCharMap = null;
+    } else if (config.endIndex === lenUtf8) {
+      this.domain = "utf8";
+      this.buffer = Buffer.from(config.code, "utf8");
+      this.byteToCharMap = this.buildByteToCharMap();
+    } else {
+      this.domain = "unknown";
+      this.buffer = null;
+      this.byteToCharMap = null;
+      console.warn(
+        `[SourceAdapter] Index domain unclear: endIndex=${config.endIndex}, utf16Len=${lenUtf16}, utf8Len=${lenUtf8}`
+      );
+    }
+    this.nwsPrefixSum = this.buildNwsPrefixSum();
+  }
+  /**
+   * 获取检测到的索引域
+   */
+  getDomain() {
+    return this.domain;
+  }
+  /**
+   * 安全切片：根据索引域选择正确的切片方式
+   * 
+   * 对于 UTF-8 域，先将字节边界对齐到字符边界，再进行切片
+   * 
+   * @param start Tree-sitter 返回的 startIndex
+   * @param end Tree-sitter 返回的 endIndex
+   * @returns 切片后的字符串
+   */
+  slice(start, end) {
+    if (this.domain === "utf16" || this.domain === "unknown") {
+      return this.code.slice(start, end);
+    }
+    if (!this.byteToCharMap) {
+      return this.code.slice(start, end);
+    }
+    const charStart = this.byteToChar(start);
+    const charEnd = this.byteToChar(end);
+    return this.code.slice(charStart, charEnd);
+  }
+  /**
+   * 计算区间的非空白字符数
+   * 
+   * 注意：NWS 始终在字符域计算，保持语义一致性
+   * 如果索引域是 UTF-8，需要先将字节偏移转换为字符偏移
+   * 
+   * @param start Tree-sitter 返回的 startIndex
+   * @param end Tree-sitter 返回的 endIndex
+   * @returns 非空白字符数
+   */
+  nws(start, end) {
+    let charStart;
+    let charEnd;
+    if (this.domain === "utf8" && this.byteToCharMap) {
+      charStart = this.byteToChar(start);
+      charEnd = this.byteToChar(end);
+    } else {
+      charStart = start;
+      charEnd = end;
+    }
+    const maxIndex = this.nwsPrefixSum.length - 1;
+    const s = Math.max(0, Math.min(maxIndex, charStart));
+    const e = Math.max(0, Math.min(maxIndex, charEnd));
+    return this.nwsPrefixSum[e] - this.nwsPrefixSum[s];
+  }
+  /**
+   * 获取总的非空白字符数
+   */
+  getTotalNws() {
+    return this.nwsPrefixSum[this.nwsPrefixSum.length - 1];
+  }
+  /**
+   * 将字节偏移转换为字符偏移
+   */
+  byteToChar(byteOffset) {
+    if (!this.byteToCharMap) return byteOffset;
+    const safeOffset = Math.max(0, Math.min(this.byteToCharMap.length - 1, byteOffset));
+    return this.byteToCharMap[safeOffset];
+  }
+  /**
+   * 构建字节偏移到字符偏移的映射表
+   * 
+   * 对于 UTF-8 编码，一个字符可能占用 1-4 个字节
+   * 此映射表允许 O(1) 查找任意字节偏移对应的字符偏移
+   */
+  buildByteToCharMap() {
+    const buffer = this.buffer;
+    const map = new Uint32Array(buffer.length + 1);
+    let charIndex = 0;
+    let byteIndex = 0;
+    while (byteIndex < buffer.length) {
+      map[byteIndex] = charIndex;
+      const byte = buffer[byteIndex];
+      let charBytes;
+      if ((byte & 128) === 0) {
+        charBytes = 1;
+      } else if ((byte & 224) === 192) {
+        charBytes = 2;
+      } else if ((byte & 240) === 224) {
+        charBytes = 3;
+      } else if ((byte & 248) === 240) {
+        charBytes = 4;
+      } else {
+        charBytes = 1;
+      }
+      for (let i = 1; i < charBytes && byteIndex + i < buffer.length; i++) {
+        map[byteIndex + i] = charIndex;
+      }
+      byteIndex += charBytes;
+      if (charBytes === 4) {
+        charIndex += 2;
+      } else {
+        charIndex += 1;
+      }
+    }
+    map[buffer.length] = charIndex;
+    return map;
+  }
+  /**
+   * 构建字符域的 NWS 前缀和
+   */
+  buildNwsPrefixSum() {
+    const prefixSum = new Uint32Array(this.code.length + 1);
+    let count = 0;
+    for (let i = 0; i < this.code.length; i++) {
+      const cc = this.code.charCodeAt(i);
+      if (!(cc === 32 || cc === 9 || cc === 10 || cc === 13)) {
+        count++;
+      }
+      prefixSum[i + 1] = count;
+    }
+    return prefixSum;
+  }
+};
+
+// src/chunking/LanguageSpec.ts
+var LANGUAGE_SPECS = {
+  typescript: {
+    hierarchy: /* @__PURE__ */ new Set([
+      // 类和接口
+      "class_declaration",
+      "abstract_class_declaration",
+      "interface_declaration",
+      // 函数
+      "function_declaration",
+      "generator_function_declaration",
+      "method_definition",
+      "arrow_function",
+      // 模块
+      "export_statement",
+      "import_statement"
+    ]),
+    nameFields: ["name", "id"]
+  },
+  javascript: {
+    hierarchy: /* @__PURE__ */ new Set([
+      "class_declaration",
+      "function_declaration",
+      "generator_function_declaration",
+      "method_definition",
+      "arrow_function"
+    ]),
+    nameFields: ["name", "id"]
+  },
+  python: {
+    hierarchy: /* @__PURE__ */ new Set([
+      "class_definition",
+      "function_definition",
+      "decorated_definition"
+    ]),
+    nameFields: ["name"]
+  },
+  go: {
+    hierarchy: /* @__PURE__ */ new Set([
+      // 函数和方法
+      "function_declaration",
+      "method_declaration",
+      // 类型定义
+      "type_spec",
+      "type_declaration",
+      // 结构体和接口
+      "struct_type",
+      "interface_type"
+    ]),
+    nameFields: ["name"]
+  },
+  rust: {
+    hierarchy: /* @__PURE__ */ new Set([
+      // 函数
+      "function_item",
+      // 结构体、枚举、trait
+      "struct_item",
+      "enum_item",
+      "trait_item",
+      // impl 块
+      "impl_item",
+      // 模块
+      "mod_item",
+      // 类型别名
+      "type_item"
+    ]),
+    nameFields: ["name"]
+  },
+  java: {
+    hierarchy: /* @__PURE__ */ new Set([
+      // 类和接口
+      "class_declaration",
+      "interface_declaration",
+      "enum_declaration",
+      "annotation_type_declaration",
+      // 方法和构造函数
+      "method_declaration",
+      "constructor_declaration",
+      // 记录类型 (Java 14+)
+      "record_declaration"
+    ]),
+    nameFields: ["name", "identifier"]
+  }
+};
+function getLanguageSpec(language) {
+  return LANGUAGE_SPECS[language] ?? null;
+}
+
+// src/chunking/SemanticSplitter.ts
+var SemanticSplitter = class {
+  config;
+  adapter;
+  code;
+  language;
+  constructor(config = {}) {
+    const maxChunkSize = config.maxChunkSize ?? 1e3;
+    this.config = {
+      maxChunkSize,
+      minChunkSize: config.minChunkSize ?? 50,
+      chunkOverlap: config.chunkOverlap ?? 100,
+      // 物理字符硬上限：默认为 maxChunkSize * 4（假设 1 token ≈ 4 chars）
+      maxRawChars: config.maxRawChars ?? maxChunkSize * 4
+    };
+  }
+  /**
+   * 对代码进行语义分片
+   * @param tree Tree-sitter 解析树
+   * @param code 源代码字符串
+   * @param filePath 文件路径
+   * @param language 语言标识
+   * @returns 处理后的分片数组
+   */
+  split(tree, code, filePath, language) {
+    this.adapter = new SourceAdapter({
+      code,
+      endIndex: tree.rootNode.endIndex
+    });
+    const domain = this.adapter.getDomain();
+    if (domain === "unknown") {
+      console.warn(
+        `[SemanticSplitter] Unknown index domain for ${filePath}, falling back to simple split`
+      );
+      return this.fallbackSplit(code, filePath, language);
+    }
+    if (domain === "utf8") {
+      console.info(`[SemanticSplitter] Using UTF-8 byte indexing for ${filePath}`);
+    }
+    this.code = code;
+    this.language = language;
+    const initialContext = [filePath];
+    const windows = this.visitNode(tree.rootNode, initialContext);
+    return this.windowsToChunks(windows, filePath, language);
+  }
+  /**
+   * 公开的纯文本分片接口
+   * 
+   * 用于不支持 AST 解析的语言，或作为 AST 解析失败时的降级方案。
+   * 使用 UTF-16 索引（JS 原生字符串），按行切分。
+   * 
+   * @param code 源代码字符串
+   * @param filePath 文件路径
+   * @param language 语言标识
+   * @returns 处理后的分片数组
+   */
+  splitPlainText(code, filePath, language) {
+    return this.fallbackSplit(code, filePath, language);
+  }
+  /**
+   * 降级分片：当索引域不明确时使用
+   * 
+   * 使用 UTF-16 索引（JS 原生字符串），按行切分
+   * 注意：fallback 模式不支持 overlap
+   */
+  fallbackSplit(code, filePath, language) {
+    const adapter = new SourceAdapter({
+      code,
+      endIndex: code.length
+    });
+    const totalSize = adapter.getTotalNws();
+    if (totalSize <= this.config.maxChunkSize) {
+      return [
+        {
+          displayCode: code,
+          vectorText: `// Context: ${filePath}
+${code}`,
+          nwsSize: totalSize,
+          metadata: {
+            startIndex: 0,
+            endIndex: code.length,
+            rawSpan: { start: 0, end: code.length },
+            vectorSpan: { start: 0, end: code.length },
+            filePath,
+            language,
+            contextPath: [filePath]
+          }
+        }
+      ];
+    }
+    const lines = code.split("\n");
+    const chunks = [];
+    let currentLines = [];
+    let currentSize = 0;
+    let lineStartIndex = 0;
+    let chunkStartIndex = 0;
+    let chunkRawStart = 0;
+    for (const line of lines) {
+      const lineEndIndex = lineStartIndex + line.length;
+      const lineNws = adapter.nws(lineStartIndex, lineEndIndex);
+      if (currentSize + lineNws > this.config.maxChunkSize && currentLines.length > 0) {
+        const displayCode = currentLines.join("\n");
+        const chunkEndIndex = chunkStartIndex + displayCode.length;
+        chunks.push({
+          displayCode,
+          vectorText: `// Context: ${filePath}
+${displayCode}`,
+          nwsSize: currentSize,
+          metadata: {
+            startIndex: chunkStartIndex,
+            endIndex: chunkEndIndex,
+            rawSpan: { start: chunkRawStart, end: chunkEndIndex + 1 },
+            // +1 for newline gap
+            vectorSpan: { start: chunkStartIndex, end: chunkEndIndex },
+            filePath,
+            language,
+            contextPath: [filePath]
+          }
+        });
+        chunkRawStart = chunkEndIndex + 1;
+        chunkStartIndex += displayCode.length + 1;
+        currentLines = [line];
+        currentSize = lineNws;
+      } else {
+        currentLines.push(line);
+        currentSize += lineNws;
+      }
+      lineStartIndex = lineEndIndex + 1;
+    }
+    if (currentLines.length > 0) {
+      const displayCode = currentLines.join("\n");
+      const chunkEndIndex = chunkStartIndex + displayCode.length;
+      chunks.push({
+        displayCode,
+        vectorText: `// Context: ${filePath}
+${displayCode}`,
+        nwsSize: currentSize,
+        metadata: {
+          startIndex: chunkStartIndex,
+          endIndex: chunkEndIndex,
+          rawSpan: { start: chunkRawStart, end: code.length },
+          vectorSpan: { start: chunkStartIndex, end: chunkEndIndex },
+          filePath,
+          language,
+          contextPath: [filePath]
+        }
+      });
+    }
+    return chunks;
+  }
+  /**
+   * 递归遍历 AST 节点
+   */
+  visitNode(node, context) {
+    const start = node.startIndex;
+    const end = node.endIndex;
+    const nodeSize = this.adapter.nws(start, end);
+    let nextContext = context;
+    const spec = getLanguageSpec(this.language);
+    if (spec && spec.hierarchy.has(node.type)) {
+      let name = null;
+      for (const child of node.namedChildren) {
+        if (child.type === "identifier" || child.type === "type_identifier" || child.type === "name") {
+          name = child.text;
+          break;
+        }
+      }
+      if (!name && node.firstNamedChild) {
+        const firstChild = node.firstNamedChild;
+        if (firstChild.text.length <= 100 && !firstChild.text.includes("\n")) {
+          name = firstChild.text;
+        }
+      }
+      if (name) {
+        const typePrefix = this.getTypePrefix(node.type);
+        nextContext = [...context, `${typePrefix}${name}`];
+      }
+    }
+    if (nodeSize <= this.config.maxChunkSize) {
+      return [{ nodes: [node], size: nodeSize, contextPath: nextContext }];
+    }
+    const children = node.children;
+    if (children.length === 0) {
+      return [{ nodes: [node], size: nodeSize, contextPath: nextContext }];
+    }
+    let childWindows = [];
+    for (const child of children) {
+      childWindows.push(...this.visitNode(child, nextContext));
+    }
+    return this.mergeAdjacentWindows(childWindows);
+  }
+  /**
+   * 获取节点类型的简短前缀
+   */
+  getTypePrefix(nodeType) {
+    if (nodeType.includes("class")) return "class ";
+    if (nodeType.includes("interface")) return "interface ";
+    if (nodeType.includes("method")) return "method ";
+    if (nodeType.includes("function")) return "function ";
+    return "";
+  }
+  /**
+   * Gap-Aware 相邻窗口合并
+   * 
+   * 使用 NWS + Raw 双预算策略：
+   * - NWS 预算：控制有效代码量
+   * - Raw 预算：控制物理字符数，防止大量注释撑爆 Token
+   */
+  mergeAdjacentWindows(windows) {
+    if (windows.length === 0) return [];
+    const merged = [];
+    let current = windows[0];
+    for (let i = 1; i < windows.length; i++) {
+      const next = windows[i];
+      const currentStart = current.nodes[0].startIndex;
+      const currentEnd = current.nodes[current.nodes.length - 1].endIndex;
+      const nextStart = next.nodes[0].startIndex;
+      const nextEnd = next.nodes[next.nodes.length - 1].endIndex;
+      const gapNws = this.adapter.nws(currentEnd, nextStart);
+      const combinedNws = current.size + gapNws + next.size;
+      const combinedRawLen = nextEnd - currentStart;
+      const isTiny = current.size < this.config.minChunkSize;
+      const fitsNwsBudget = combinedNws <= this.config.maxChunkSize || isTiny && combinedNws < this.config.maxChunkSize * 1.5;
+      const fitsRawBudget = combinedRawLen <= this.config.maxRawChars;
+      if (fitsNwsBudget && fitsRawBudget) {
+        current.nodes.push(...next.nodes);
+        current.size = combinedNws;
+        current.contextPath = this.commonPrefix(current.contextPath, next.contextPath);
+      } else {
+        merged.push(current);
+        current = next;
+      }
+    }
+    merged.push(current);
+    return merged;
+  }
+  /**
+   * 将窗口转换为最终的 ProcessedChunk
+   * 
+   * Gap 归属策略：gap 归属到后一个 chunk（即 chunk 的 rawSpan.start 向前延伸到前一个 chunk 的 endIndex）
+   * Overlap 策略：vectorSpan 向前延伸 chunkOverlap 个 NWS 字符，提升语义检索召回率
+   * 
+   * 保证：所有 rawSpan 拼接后 === 完整文件（不重叠）
+   */
+  windowsToChunks(windows, filePath, language) {
+    if (windows.length === 0) return [];
+    const chunks = [];
+    let prevEnd = 0;
+    const overlap = this.config.chunkOverlap;
+    for (let i = 0; i < windows.length; i++) {
+      const w = windows[i];
+      const start = w.nodes[0].startIndex;
+      const end = w.nodes[w.nodes.length - 1].endIndex;
+      const isLast = i === windows.length - 1;
+      const codeEndIndex = this.adapter.getDomain() === "utf8" ? Buffer.byteLength(this.code, "utf8") : this.code.length;
+      const rawSpanEnd = isLast ? codeEndIndex : end;
+      let vectorStart = start;
+      if (i > 0 && overlap > 0) {
+        const candidateStart = this.findOverlapStart(start, overlap);
+        const overlapRawLen = start - candidateStart;
+        if (overlapRawLen <= this.config.maxRawChars * 0.25) {
+          vectorStart = candidateStart;
+        }
+      }
+      const vectorEnd = end;
+      const displayCode = this.adapter.slice(start, end);
+      const vectorCode = this.adapter.slice(vectorStart, vectorEnd);
+      const metadata = {
+        startIndex: start,
+        endIndex: end,
+        rawSpan: { start: prevEnd, end: rawSpanEnd },
+        vectorSpan: { start: vectorStart, end: vectorEnd },
+        filePath,
+        language,
+        contextPath: w.contextPath
+      };
+      chunks.push({
+        displayCode,
+        vectorText: generateVectorText(vectorCode, w.contextPath),
+        nwsSize: w.size,
+        metadata
+      });
+      prevEnd = end;
+    }
+    return chunks;
+  }
+  /**
+   * 找到 overlap 的起始位置
+   * 
+   * 从 start 位置向前搜索，找到包含 targetNws 个非空白字符的位置
+   * 
+   * @param start 当前 chunk 的起始位置
+   * @param targetNws 目标 overlap 大小（NWS 字符数）
+   * @returns overlap 起始位置
+   */
+  findOverlapStart(start, targetNws) {
+    if (start <= 0 || targetNws <= 0) return start;
+    let low = 0;
+    let high = start;
+    let result = start;
+    while (low <= high) {
+      const mid = Math.floor((low + high) / 2);
+      const nwsInRange = this.adapter.nws(mid, start);
+      if (nwsInRange >= targetNws) {
+        result = mid;
+        low = mid + 1;
+      } else {
+        high = mid - 1;
+      }
+    }
+    return Math.max(0, result);
+  }
+  /**
+   * 计算两个路径数组的最长公共前缀（LCA）
+   * 
+   * 用于合并窗口时更新 contextPath，避免 breadcrumb 误标
+   * 例如：["file", "class A", "method foo"] 和 ["file", "class A", "method bar"]
+   *       => ["file", "class A"]
+   */
+  commonPrefix(a, b) {
+    const result = [];
+    const len = Math.min(a.length, b.length);
+    for (let i = 0; i < len; i++) {
+      if (a[i] === b[i]) {
+        result.push(a[i]);
+      } else {
+        break;
+      }
+    }
+    return result;
+  }
+};
+function generateVectorText(code, contextPath) {
+  const breadcrumb = contextPath.join(" > ");
+  return `// Context: ${breadcrumb}
+${code}`;
+}
+
+// src/chunking/ParserPool.ts
+import Parser2 from "tree-sitter";
+var GRAMMAR_MODULES = {
+  typescript: "tree-sitter-typescript",
+  javascript: "tree-sitter-javascript",
+  python: "tree-sitter-python",
+  go: "tree-sitter-go",
+  rust: "tree-sitter-rust",
+  java: "tree-sitter-java"
+};
+var loadedGrammars = /* @__PURE__ */ new Map();
+var parserCache = /* @__PURE__ */ new Map();
+async function loadGrammar(language) {
+  const cached = loadedGrammars.get(language);
+  if (cached) return cached;
+  const moduleName = GRAMMAR_MODULES[language];
+  if (!moduleName) return null;
+  try {
+    const grammarModule = await import(moduleName);
+    let grammar = null;
+    if (language === "typescript") {
+      grammar = grammarModule.default?.typescript ?? grammarModule.typescript;
+    } else {
+      const exported = grammarModule.default ?? grammarModule;
+      if (exported && typeof exported === "object" && "nodeTypeInfo" in exported) {
+        grammar = exported;
+      } else if (exported?.language) {
+        grammar = exported.language;
+      } else if (exported?.[language]) {
+        grammar = exported[language];
+      }
+    }
+    if (!grammar) {
+      console.error(`[ParserPool] Could not extract grammar for ${language} from module ${moduleName}`);
+      return null;
+    }
+    loadedGrammars.set(language, grammar);
+    return grammar;
+  } catch (err) {
+    console.error(`[ParserPool] Failed to load grammar for ${language}:`, err);
+    return null;
+  }
+}
+async function getParser(language) {
+  const cached = parserCache.get(language);
+  if (cached) return cached;
+  const grammar = await loadGrammar(language);
+  if (!grammar) return null;
+  const parser = new Parser2();
+  parser.setLanguage(grammar);
+  parserCache.set(language, parser);
+  return parser;
+}
+function isLanguageSupported(language) {
+  return language in GRAMMAR_MODULES;
+}
+
+// src/scanner/processor.ts
+var MAX_FILE_SIZE = 500 * 1024;
+var FALLBACK_LANGS = /* @__PURE__ */ new Set([
+  "python",
+  "go",
+  "rust",
+  "java",
+  "markdown",
+  "json"
+]);
+function shouldSkipJson(relPath) {
+  if (relPath.endsWith("-lock.json") || relPath.endsWith("package-lock.json")) {
+    return true;
+  }
+  if (relPath.includes("node_modules/") || relPath.includes("node_modules\\")) {
+    return true;
+  }
+  return false;
+}
+function getAdaptiveConcurrency() {
+  const cpuCount = os.cpus().length;
+  const concurrency = Math.max(4, Math.min(cpuCount - 1, 32));
+  return concurrency;
+}
+var splitter = new SemanticSplitter({
+  maxChunkSize: 500,
+  minChunkSize: 50,
+  chunkOverlap: 40
+  // 混合检索(BM25+向量+rerank)下的保守 overlap
+});
+async function processFile(absPath, relPath, known) {
+  const language = getLanguage(relPath);
+  try {
+    const stat = await fs3.stat(absPath);
+    const mtime = stat.mtimeMs;
+    const size = stat.size;
+    if (size > MAX_FILE_SIZE) {
+      return {
+        absPath,
+        relPath,
+        hash: "",
+        content: null,
+        chunks: [],
+        language,
+        mtime,
+        size,
+        status: "skipped",
+        error: `File too large (${size} bytes > ${MAX_FILE_SIZE} bytes)`
+      };
+    }
+    if (known && known.mtime === mtime && known.size === size) {
+      return {
+        absPath,
+        relPath,
+        hash: known.hash,
+        content: null,
+        chunks: [],
+        language,
+        mtime,
+        size,
+        status: "unchanged"
+      };
+    }
+    const { content, originalEncoding } = await readFileWithEncoding(absPath);
+    if (content.includes("\0")) {
+      return {
+        absPath,
+        relPath,
+        hash: "",
+        content: null,
+        chunks: [],
+        language,
+        mtime,
+        size,
+        status: "skipped",
+        error: `Binary file detected (original encoding: ${originalEncoding})`
+      };
+    }
+    const hash = sha256(content);
+    if (known && known.hash === hash) {
+      return {
+        absPath,
+        relPath,
+        hash,
+        content,
+        chunks: [],
+        language,
+        mtime,
+        size,
+        status: "unchanged"
+      };
+    }
+    if (language === "json" && shouldSkipJson(relPath)) {
+      return {
+        absPath,
+        relPath,
+        hash,
+        content: null,
+        chunks: [],
+        language,
+        mtime,
+        size,
+        status: "skipped",
+        error: "Lock file or node_modules JSON"
+      };
+    }
+    let chunks = [];
+    if (isLanguageSupported(language)) {
+      try {
+        const parser = await getParser(language);
+        if (parser) {
+          const tree = parser.parse(content);
+          chunks = splitter.split(tree, content, relPath, language);
+        }
+      } catch (err) {
+        console.warn(`[Chunking] AST failed for ${relPath}: ${err.message}`);
+      }
+    }
+    if (chunks.length === 0 && FALLBACK_LANGS.has(language)) {
+      chunks = splitter.splitPlainText(content, relPath, language);
+    }
+    return {
+      absPath,
+      relPath,
+      hash,
+      content,
+      chunks,
+      language,
+      mtime,
+      size,
+      status: known ? "modified" : "added"
+    };
+  } catch (err) {
+    return {
+      absPath,
+      relPath,
+      hash: "",
+      content: null,
+      chunks: [],
+      language,
+      mtime: 0,
+      size: 0,
+      status: "error",
+      error: err.message
+    };
+  }
+}
+async function processFiles(rootPath, filePaths, knownFiles) {
+  const concurrency = getAdaptiveConcurrency();
+  const limit = pLimit(concurrency);
+  const tasks = filePaths.map((filePath) => {
+    const relPath = path3.relative(rootPath, filePath);
+    const known = knownFiles.get(relPath);
+    return limit(() => processFile(filePath, relPath, known));
+  });
+  return Promise.all(tasks);
+}
+
+// src/scanner/index.ts
+async function scan(rootPath, options = {}) {
+  const projectId = generateProjectId(rootPath);
+  const db = initDb(projectId);
+  try {
+    await initFilter(rootPath);
+    let forceReindex = options.force ?? false;
+    if (options.vectorIndex !== false) {
+      const currentDimensions = getEmbeddingConfig().dimensions;
+      const storedDimensions = getStoredEmbeddingDimensions(db);
+      if (storedDimensions !== null && storedDimensions !== currentDimensions) {
+        logger.warn(
+          { stored: storedDimensions, current: currentDimensions },
+          "Embedding \u7EF4\u5EA6\u53D8\u5316\uFF0C\u5F3A\u5236\u91CD\u65B0\u7D22\u5F15"
+        );
+        forceReindex = true;
+      }
+      setStoredEmbeddingDimensions(db, currentDimensions);
+    }
+    if (forceReindex) {
+      logger.info("\u5F3A\u5236\u91CD\u65B0\u7D22\u5F15...");
+      clear(db);
+      if (options.vectorIndex !== false) {
+        const embeddingConfig = getEmbeddingConfig();
+        const indexer = await getIndexer(projectId, embeddingConfig.dimensions);
+        await indexer.clear();
+      }
+    }
+    const knownFiles = getAllFileMeta(db);
+    const filePaths = await crawl(rootPath);
+    const scannedPaths = new Set(filePaths.map((p) => p.replace(new RegExp(`^${rootPath}/?`), "")));
+    let processedCount = 0;
+    const results = [];
+    const batchSize = 100;
+    for (let i = 0; i < filePaths.length; i += batchSize) {
+      const batch = filePaths.slice(i, i + batchSize);
+      const batchResults = await processFiles(rootPath, batch, knownFiles);
+      results.push(...batchResults);
+      processedCount += batch.length;
+      options.onProgress?.(processedCount, filePaths.length);
+    }
+    const toAdd = [];
+    const toUpdateMtime = [];
+    const deletedPaths = [];
+    for (const result of results) {
+      switch (result.status) {
+        case "added":
+        case "modified":
+          toAdd.push({
+            path: result.relPath,
+            hash: result.hash,
+            mtime: result.mtime,
+            size: result.size,
+            content: result.content,
+            language: result.language,
+            vectorIndexHash: null
+            // 新文件/修改的文件需要重新索引
+          });
+          break;
+        case "unchanged":
+          toUpdateMtime.push({ path: result.relPath, mtime: result.mtime });
+          break;
+        case "skipped":
+          logger.debug({ path: result.relPath, reason: result.error }, "\u8DF3\u8FC7\u6587\u4EF6");
+          break;
+        case "error":
+          logger.error({ path: result.relPath, error: result.error }, "\u5904\u7406\u6587\u4EF6\u9519\u8BEF");
+          break;
+      }
+    }
+    const allIndexedPaths = getAllPaths(db);
+    for (const indexedPath of allIndexedPaths) {
+      if (!scannedPaths.has(indexedPath)) {
+        deletedPaths.push(indexedPath);
+      }
+    }
+    batchUpsert(db, toAdd);
+    batchUpdateMtime(db, toUpdateMtime);
+    batchDelete(db, deletedPaths);
+    const stats = {
+      totalFiles: filePaths.length,
+      added: results.filter((r) => r.status === "added").length,
+      modified: results.filter((r) => r.status === "modified").length,
+      unchanged: results.filter((r) => r.status === "unchanged").length,
+      deleted: deletedPaths.length,
+      skipped: results.filter((r) => r.status === "skipped").length,
+      errors: results.filter((r) => r.status === "error").length
+    };
+    if (options.vectorIndex !== false) {
+      const embeddingConfig = getEmbeddingConfig();
+      const indexer = await getIndexer(projectId, embeddingConfig.dimensions);
+      const needsVectorIndex = results.filter(
+        (r) => r.status === "added" || r.status === "modified"
+      );
+      const healingPathSet = new Set(getFilesNeedingVectorIndex(db));
+      const healingFiles = results.filter(
+        (r) => r.status === "unchanged" && healingPathSet.has(r.relPath)
+      );
+      if (healingFiles.length > 0) {
+        logger.info({ count: healingFiles.length }, "\u81EA\u6108\uFF1A\u53D1\u73B0\u9700\u8981\u8865\u7D22\u5F15\u7684\u6587\u4EF6");
+      }
+      const deletedResults = deletedPaths.map((path4) => ({
+        absPath: "",
+        relPath: path4,
+        hash: "",
+        content: null,
+        chunks: [],
+        language: "",
+        mtime: 0,
+        size: 0,
+        status: "deleted"
+      }));
+      const allToIndex = [...needsVectorIndex, ...healingFiles, ...deletedResults];
+      if (allToIndex.length > 0) {
+        const indexStats = await indexer.indexFiles(db, allToIndex);
+        stats.vectorIndex = {
+          indexed: indexStats.indexed,
+          deleted: indexStats.deleted,
+          errors: indexStats.errors
+        };
+      }
+    }
+    return stats;
+  } finally {
+    closeDb(db);
+    closeAllIndexers();
+    await closeAllVectorStores();
+  }
+}
+
+export {
+  scan
+};