@iderouter/index-mcp 0.2.0-beta.1

package/scripts/lib/mcp-bench.mjs

@@ -0,0 +1,455 @@
+import crypto from "node:crypto";
+import { execFile, spawn } from "node:child_process";
+import fsSync from "node:fs";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const MCP_PATH = path.resolve(__dirname, "..", "..", "src", "index.js");
+const execFileAsync = promisify(execFile);
+const INDEXABLE_EXTENSIONS = new Set([
+  ".c", ".cc", ".cpp", ".cs", ".css", ".go", ".graphql", ".h", ".hpp", ".html",
+  ".java", ".js", ".json", ".jsx", ".kt", ".kts", ".md", ".mjs", ".php", ".proto",
+  ".py", ".rb", ".rs", ".scss", ".sh", ".sql", ".swift", ".toml", ".ts", ".tsx",
+  ".vue", ".xml", ".yaml", ".yml",
+]);
+
+export function benchRepoId(targetPath) {
+  return crypto.createHash("sha256").update(targetPath).digest("hex").slice(0, 16);
+}
+
+export function benchIndexMetaPath(indexHome, targetPath) {
+  return path.join(indexHome, "indexes", `${benchRepoId(targetPath)}.meta.json`);
+}
+
+export function benchStatusPath(indexHome, targetPath) {
+  return path.join(indexHome, "status", `${benchRepoId(targetPath)}.json`);
+}
+
+export async function readIndexMeta(indexHome, targetPath) {
+  try {
+    const content = await fs.readFile(benchIndexMetaPath(indexHome, targetPath), "utf8");
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+
+export async function readJobStatus(indexHome, targetPath) {
+  try {
+    const content = await fs.readFile(benchStatusPath(indexHome, targetPath), "utf8");
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+
+export function parseDiagnostics(text) {
+  const match = text.match(/Diagnostics:\s(.+?)(?:\.\s(?:Cloud storage:|Cloud sync:|Async indexing guide:)|$)/s);
+  if (!match) return {};
+  const fields = {};
+  for (const part of match[1].split(",")) {
+    const [rawKey, ...rest] = part.trim().split("=");
+    if (!rawKey || rest.length === 0) continue;
+    fields[rawKey.trim()] = rest.join("=").trim();
+  }
+  return fields;
+}
+
+export function normalizeBenchmarkResult(targetPath, finalText, wallMs) {
+  const diagnostics = parseDiagnostics(finalText);
+  return {
+    path: targetPath,
+    wall_ms: wallMs,
+    scan_ms: Number(diagnostics.scan_ms || 0),
+    coarse_ready_ms: Number(diagnostics.coarse_ready_ms || 0),
+    priority_ready_ms: Number(diagnostics.priority_ready_ms || 0),
+    background_ms: Number(diagnostics.background_ms || 0),
+    total_ms: Number(diagnostics.total_ms || 0),
+    strategy_version: diagnostics.strategy_version || "",
+    strategy_source: diagnostics.strategy_source || "",
+    embedding_model_source: diagnostics.embedding_model_source || "",
+    background_fine_target_files: Number(diagnostics.background_fine_target_files || 0),
+    background_fine_target_chunks: Number(diagnostics.background_fine_target_chunks || 0),
+    final_status: finalText.split("\n")[0],
+  };
+}
+
+export function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function escapeRegex(value) {
+  return String(value || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function decodeTomlBasicString(value) {
+  return String(value || "")
+    .replace(/\\\\/g, "\\")
+    .replace(/\\"/g, "\"")
+    .replace(/\\n/g, "\n")
+    .replace(/\\r/g, "\r")
+    .replace(/\\t/g, "\t");
+}
+
+function extractCodexMcpEnvValue(configText, serverName, envKey) {
+  const sectionRegex = new RegExp(
+    `^\\[mcp_servers\\.${escapeRegex(serverName)}\\.env\\]\\s*$([\\s\\S]*?)(?=^\\[|\\Z)`,
+    "m",
+  );
+  const sectionMatch = String(configText || "").match(sectionRegex);
+  if (!sectionMatch) return "";
+  const keyRegex = new RegExp(`^\\s*${escapeRegex(envKey)}\\s*=\\s*"((?:[^"\\\\]|\\\\.)*)"\\s*$`, "m");
+  const keyMatch = sectionMatch[1].match(keyRegex);
+  return keyMatch ? decodeTomlBasicString(keyMatch[1]) : "";
+}
+
+function readCodexMcpEnvFallback(serverName, configPath) {
+  try {
+    const content = fsSync.readFileSync(configPath, "utf8");
+    const result = {};
+    for (const envKey of [
+      "IDEROUTER_API_KEY",
+      "IDEROUTER_BASE_URL",
+      "IDEROUTER_MCP_CONFIG_API_BASE_URL",
+      "IDEROUTER_CLOUD_API_BASE_URL",
+    ]) {
+      const value = extractCodexMcpEnvValue(content, serverName, envKey);
+      if (value) {
+        result[envKey] = value;
+      }
+    }
+    return result;
+  } catch {
+    return {};
+  }
+}
+
+export function resolveBenchmarkEnv(baseEnv = process.env, options = {}) {
+  const nextEnv = { ...baseEnv };
+  if (nextEnv.IDEROUTER_API_KEY) return nextEnv;
+
+  const serverName = String(options.serverName || nextEnv.IDEROUTER_BENCH_MCP_SERVER_NAME || "iderouter-index").trim();
+  const configPath = String(
+    options.configPath ||
+      nextEnv.IDEROUTER_BENCH_CODEX_CONFIG_PATH ||
+      path.join(os.homedir(), ".codex", "config.toml"),
+  );
+  const fallbackEnv = readCodexMcpEnvFallback(serverName, configPath);
+  for (const [key, value] of Object.entries(fallbackEnv)) {
+    if (!nextEnv[key] && value) {
+      nextEnv[key] = value;
+    }
+  }
+  return nextEnv;
+}
+
+function isWorkerProcessAlive(pid) {
+  const numericPid = Number(pid || 0);
+  if (!Number.isInteger(numericPid) || numericPid <= 0) return false;
+  try {
+    process.kill(numericPid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function terminateDetachedIndexWorker(indexHome, targetPath, options = {}) {
+  const graceMs = Math.max(100, Number(options.graceMs) || 1000);
+  const job = await readJobStatus(indexHome, targetPath);
+  const workerPid = Number(job?.workerPid || 0);
+  if (!workerPid || !isWorkerProcessAlive(workerPid)) return false;
+  try {
+    process.kill(workerPid, "SIGTERM");
+  } catch {
+    return false;
+  }
+  const deadline = Date.now() + graceMs;
+  while (Date.now() < deadline) {
+    if (!isWorkerProcessAlive(workerPid)) return true;
+    await sleep(100);
+  }
+  if (!isWorkerProcessAlive(workerPid)) return true;
+  try {
+    process.kill(workerPid, "SIGKILL");
+  } catch {
+    return !isWorkerProcessAlive(workerPid);
+  }
+  return !isWorkerProcessAlive(workerPid);
+}
+
+async function terminateWorkersByTargetPath(targetPath) {
+  if (!targetPath) return;
+  let stdout = "";
+  try {
+    ({ stdout } = await execFileAsync("pgrep", ["-f", targetPath], {
+      maxBuffer: 8 * 1024 * 1024,
+    }));
+  } catch (error) {
+    if (error?.code !== 1) {
+      return;
+    }
+  }
+  const pids = String(stdout || "")
+    .split("\n")
+    .map((line) => Number(line.trim()))
+    .filter((pid) => Number.isInteger(pid) && pid > 0 && pid !== process.pid);
+  for (const pid of pids) {
+    try {
+      process.kill(pid, "SIGTERM");
+    } catch {
+      continue;
+    }
+  }
+  await sleep(250);
+  for (const pid of pids) {
+    if (!isWorkerProcessAlive(pid)) continue;
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      continue;
+    }
+  }
+}
+
+export async function cleanupBenchmarkRun({
+  child,
+  indexHome,
+  targetPath,
+  snapshotDir,
+}) {
+  try {
+    child?.kill?.("SIGTERM");
+  } catch {
+    // ignore
+  }
+  if (indexHome && targetPath) {
+    await terminateDetachedIndexWorker(indexHome, targetPath);
+  }
+  if (targetPath) {
+    await terminateWorkersByTargetPath(targetPath);
+  }
+  if (snapshotDir) {
+    await fs.rm(snapshotDir, { recursive: true, force: true });
+  }
+  if (indexHome) {
+    await fs.rm(indexHome, { recursive: true, force: true });
+  }
+}
+
+export function installBenchmarkSignalCleanup(factory) {
+  let cleaned = false;
+  async function runCleanupAndExit(signal) {
+    if (cleaned) return;
+    cleaned = true;
+    try {
+      const context = typeof factory === "function" ? await factory() : null;
+      if (context) {
+        await cleanupBenchmarkRun(context);
+      }
+    } finally {
+      process.exit(128 + (signal === "SIGINT" ? 2 : 15));
+    }
+  }
+  const onSigint = () => {
+    void runCleanupAndExit("SIGINT");
+  };
+  const onSigterm = () => {
+    void runCleanupAndExit("SIGTERM");
+  };
+  process.once("SIGINT", onSigint);
+  process.once("SIGTERM", onSigterm);
+  return () => {
+    process.off("SIGINT", onSigint);
+    process.off("SIGTERM", onSigterm);
+  };
+}
+
+export class McpClient {
+  constructor(child) {
+    this.child = child;
+    this.nextId = 1;
+    this.pending = new Map();
+    let buffer = "";
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      buffer += chunk;
+      let newline = buffer.indexOf("\n");
+      while (newline >= 0) {
+        const line = buffer.slice(0, newline).trim();
+        buffer = buffer.slice(newline + 1);
+        if (line) {
+          const message = JSON.parse(line);
+          const pending = this.pending.get(message.id);
+          if (pending) {
+            this.pending.delete(message.id);
+            if (message.error) pending.reject(new Error(message.error.message || "MCP error"));
+            else pending.resolve(message.result);
+          }
+        }
+        newline = buffer.indexOf("\n");
+      }
+    });
+  }
+
+  request(method, params) {
+    const id = this.nextId++;
+    const payload = { jsonrpc: "2.0", id, method, params };
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject });
+      this.child.stdin.write(`${JSON.stringify(payload)}\n`);
+    });
+  }
+
+  async initialize() {
+    await this.request("initialize", {
+      protocolVersion: "2024-11-05",
+      capabilities: {},
+      clientInfo: { name: "iderouter-bench", version: "1.0.0" },
+    });
+  }
+
+  async callTool(name, args) {
+    const result = await this.request("tools/call", { name, arguments: args });
+    return result?.content?.[0]?.text || "";
+  }
+}
+
+export function spawnMcpClient(env = process.env) {
+  const resolvedEnv = resolveBenchmarkEnv(env);
+  const child = spawn(process.execPath, [MCP_PATH], {
+    cwd: path.dirname(MCP_PATH),
+    stdio: ["pipe", "pipe", "inherit"],
+    env: resolvedEnv,
+  });
+  return { child, client: new McpClient(child) };
+}
+
+export function isSettledIndexStatus(text) {
+  return (
+    /^Indexed\b/.test(text) ||
+    /^Priority semantic search is ready\b/.test(text) ||
+    /^Background fine rebuild is temporarily paused\b/.test(text) ||
+    /^Last index job .* failed\b/.test(text)
+  );
+}
+
+export function isUsableIndexStatus(text) {
+  return (
+    /^Indexed\b/.test(text) ||
+    /^Priority semantic search is ready\b/.test(text) ||
+    /^Background fine rebuild is temporarily paused\b/.test(text)
+  );
+}
+
+export async function gitWorkspaceStats(targetPath) {
+  try {
+    const { stdout } = await execFileAsync(
+      "git",
+      ["-C", targetPath, "status", "--porcelain=v1", "-z", "--untracked-files=all"],
+      { maxBuffer: 16 * 1024 * 1024 },
+    );
+    const entries = String(stdout || "").split("\0").filter(Boolean);
+    const dirtyPaths = entries.map((entry) => entry.slice(3).trim()).filter(Boolean);
+    const indexableDirtyPaths = dirtyPaths.filter((relativePath) => INDEXABLE_EXTENSIONS.has(path.extname(relativePath)));
+    return {
+      workspace_dirty: dirtyPaths.length > 0,
+      dirty_entries: dirtyPaths.length,
+      indexable_dirty_entries: indexableDirtyPaths.length,
+    };
+  } catch (error) {
+    return {
+      workspace_dirty: null,
+      dirty_entries: 0,
+      indexable_dirty_entries: 0,
+      workspace_dirty_error: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+export async function runIndexBenchmark({
+  targetPath,
+  timeoutMs,
+  pollMs,
+  startArgs,
+  env = process.env,
+  isSettled = isSettledIndexStatus,
+}) {
+  const resolvedEnv = resolveBenchmarkEnv(env);
+  const { child, client } = spawnMcpClient(resolvedEnv);
+  try {
+    await client.initialize();
+    const startedAt = Date.now();
+    const startText = await client.callTool("index_codebase", startArgs);
+    let finalText = startText;
+    const deadline = Date.now() + timeoutMs;
+
+    while (Date.now() < deadline) {
+      await sleep(pollMs);
+      const text = await client.callTool("get_indexing_status", { path: targetPath });
+      finalText = text;
+      if (isSettled(text)) break;
+    }
+
+    const wallMs = Date.now() - startedAt;
+    return {
+      startText,
+      finalText,
+      wallMs,
+      parsed: {
+        ...normalizeBenchmarkResult(targetPath, finalText, wallMs),
+        start_status: startText.split("\n")[0],
+      },
+    };
+    } finally {
+      await cleanupBenchmarkRun({
+        child,
+        indexHome: resolvedEnv.IDEROUTER_INDEX_HOME,
+        targetPath,
+      });
+    }
+}
+
+export async function waitForIndexStatus({
+  targetPath,
+  timeoutMs,
+  pollMs,
+  env = process.env,
+  isSettled = isUsableIndexStatus,
+}) {
+  const resolvedEnv = resolveBenchmarkEnv(env);
+  const { child, client } = spawnMcpClient(resolvedEnv);
+  try {
+    await client.initialize();
+    let finalText = "";
+    const startedAt = Date.now();
+    const deadline = Date.now() + timeoutMs;
+    let settled = false;
+    while (Date.now() < deadline) {
+      const text = await client.callTool("get_indexing_status", { path: targetPath });
+      finalText = text;
+      if (isSettled(text)) {
+        settled = true;
+        break;
+      }
+      await sleep(pollMs);
+    }
+    const wallMs = Date.now() - startedAt;
+    return {
+      finalText,
+      wallMs,
+      settled,
+      parsed: normalizeBenchmarkResult(targetPath, finalText, wallMs),
+    };
+  } finally {
+    await cleanupBenchmarkRun({
+      child,
+      indexHome: resolvedEnv.IDEROUTER_INDEX_HOME,
+      targetPath,
+    });
+  }
+}

package/src/architecture-query-fallback.js

@@ -0,0 +1,50 @@
+function queryMentionsAny(query, words) {
+  const normalized = String(query || "").toLowerCase();
+  return words.some((word) => normalized.includes(word));
+}
+
+function isArchitectureQuery(query) {
+  return queryMentionsAny(query, [
+    "architecture",
+    "architectural",
+    "overview",
+    "design",
+    "explain",
+    "how does",
+    "flow",
+    "end to end",
+    "e2e",
+    "架构",
+    "说明",
+    "总览",
+    "整体",
+    "链路",
+    "流程",
+  ]);
+}
+
+function isRelayArchitectureQuery(query) {
+  return isArchitectureQuery(query) && queryMentionsAny(query, [
+    "relay",
+    "request",
+    "controller",
+    "service",
+    "model",
+    "router",
+    "handler",
+    "后端",
+    "请求",
+  ]);
+}
+
+export function architectureFallbackPathsForQuery(query) {
+  if (!isRelayArchitectureQuery(query)) return [];
+  return [
+    "controller/relay.go",
+    "relay/compatible_handler.go",
+    "relay/common/relay_info.go",
+    "router/relay-router.go",
+    "service/channel_select.go",
+    "model/channel.go",
+  ];
+}

package/src/background-definition-chunks.js

@@ -0,0 +1,199 @@
+import crypto from "node:crypto";
+
+export function backgroundChunkPolicyForPath(relativePathInput, defaults = {}) {
+  const relativePath = String(relativePathInput || "").toLowerCase().replaceAll("\\", "/");
+  const chunkCap = Math.max(1, Number(defaults.chunkCap) || 6);
+  const minimumDefaultChunkCount = Math.max(1, Number(defaults.minimumDefaultChunkCount) || 12);
+  if (relativePath.startsWith("relay/channel/") && relativePath.endsWith("/adaptor.go")) {
+    return {
+      chunkCap: Math.min(chunkCap, 4),
+      minimumDefaultChunkCount: Math.min(minimumDefaultChunkCount, 8),
+    };
+  }
+  if (
+    relativePath === "relay/helper/price.go" ||
+    relativePath === "service/tiered_settle.go" ||
+    relativePath === "service/user_memory.go"
+  ) {
+    return {
+      chunkCap: Math.min(chunkCap, 4),
+      minimumDefaultChunkCount: Math.min(minimumDefaultChunkCount, 8),
+    };
+  }
+  if (relativePath.startsWith("controller/") && relativePath.endsWith(".go")) {
+    return {
+      chunkCap: Math.min(chunkCap, 4),
+      minimumDefaultChunkCount: Math.min(minimumDefaultChunkCount, 8),
+    };
+  }
+  return {
+    chunkCap,
+    minimumDefaultChunkCount,
+  };
+}
+
+function buildLineOffsets(content) {
+  const offsets = [0];
+  for (let index = 0; index < content.length; index += 1) {
+    if (content[index] === "\n") offsets.push(index + 1);
+  }
+  return offsets;
+}
+
+function lineForOffsetFromIndex(lineOffsets, offset) {
+  let low = 0;
+  let high = lineOffsets.length - 1;
+  while (low <= high) {
+    const mid = Math.floor((low + high) / 2);
+    if (lineOffsets[mid] <= offset) low = mid + 1;
+    else high = mid - 1;
+  }
+  return Math.max(1, high + 1);
+}
+
+function splitFile(content, file, options = {}) {
+  const chunkSize = Math.max(1, Number(options.chunkSize) || 600);
+  const overlap = Math.max(0, Math.min(chunkSize - 1, Number(options.overlap) || 80));
+  const chunks = [];
+  const lineOffsets = buildLineOffsets(content);
+  let offset = 0;
+  while (offset < content.length) {
+    const end = Math.min(content.length, offset + chunkSize);
+    const chunkContent = content.slice(offset, end).trim();
+    if (chunkContent) {
+      chunks.push({
+        id: crypto
+          .createHash("sha256")
+          .update(`${file.relativePath}:${offset}:${chunkContent}`)
+          .digest("hex")
+          .slice(0, 24),
+        contentHash: crypto.createHash("sha256").update(chunkContent).digest("hex"),
+        relativePath: file.relativePath,
+        extension: file.extension,
+        startLine: lineForOffsetFromIndex(lineOffsets, offset),
+        endLine: lineForOffsetFromIndex(lineOffsets, end),
+        content: chunkContent,
+      });
+    }
+    if (end === content.length) break;
+    offset = Math.max(end - overlap, offset + 1);
+  }
+  return chunks;
+}
+
+function dedupeDefinitionOffsets(offsets, minGap) {
+  const deduped = [];
+  for (const offset of offsets) {
+    if (!deduped.length || offset - deduped[deduped.length - 1] >= minGap) {
+      deduped.push(offset);
+    }
+  }
+  return deduped;
+}
+
+function defaultChunkCount(fileSize, chunkSize, overlap) {
+  return Math.max(1, Math.ceil(Number(fileSize || 0) / Math.max(1, Number(chunkSize || 0) - Number(overlap || 0))));
+}
+
+export function estimateBackgroundDefinitionChunkCount({
+  fileSize,
+  defaultChunkSize = 600,
+  defaultChunkOverlap = 80,
+  chunkSize = 900,
+  overlap = 120,
+  chunkCap = 6,
+  minimumDefaultChunkCount = 16,
+}) {
+  const defaultEstimate = defaultChunkCount(fileSize, defaultChunkSize, defaultChunkOverlap);
+  if (defaultEstimate <= Math.max(1, Number(minimumDefaultChunkCount) || 1)) {
+    return defaultEstimate;
+  }
+  const priorityEstimate = defaultChunkCount(fileSize, chunkSize, overlap);
+  return Math.max(1, Math.min(defaultEstimate, priorityEstimate, Math.max(1, Number(chunkCap) || 1)));
+}
+
+export function splitBackgroundDefinitionAnchors(content, file, options = {}) {
+  const chunkSize = Math.max(1, Number(options.chunkSize) || 900);
+  const overlap = Math.max(0, Math.min(chunkSize - 1, Number(options.overlap) || 120));
+  const policy = backgroundChunkPolicyForPath(file?.relativePath, options);
+  const chunkCap = policy.chunkCap;
+  const minimumDefaultChunkCount = policy.minimumDefaultChunkCount;
+  const defaultChunkSize = Math.max(1, Number(options.defaultChunkSize) || 600);
+  const defaultChunkOverlap = Math.max(0, Math.min(defaultChunkSize - 1, Number(options.defaultChunkOverlap) || 80));
+  const defaultChunks = splitFile(content, file, { chunkSize, overlap });
+  const estimatedCount = estimateBackgroundDefinitionChunkCount({
+    fileSize: content.length,
+    defaultChunkSize,
+    defaultChunkOverlap,
+    chunkSize,
+    overlap,
+    chunkCap,
+    minimumDefaultChunkCount,
+  });
+  if (defaultChunks.length <= estimatedCount) return defaultChunks;
+
+  const definitionPattern = /^(?:func|type|const|var)\s+[A-Za-z0-9_]+/gm;
+  const offsets = [];
+  let match;
+  while ((match = definitionPattern.exec(content)) !== null) {
+    offsets.push(match.index);
+  }
+  const gap = Math.max(160, Math.floor(chunkSize * 0.6));
+  const anchors = dedupeDefinitionOffsets(offsets, gap);
+  if (anchors.length === 0) return defaultChunks.slice(0, estimatedCount);
+
+  const selectedOffsets = dedupeDefinitionOffsets(
+    [0, ...anchors, Math.max(0, content.length - chunkSize)].sort((left, right) => left - right),
+    gap,
+  );
+  const windowCount = Math.max(1, Math.min(estimatedCount, selectedOffsets.length));
+  if (selectedOffsets.length <= windowCount) {
+    return selectedOffsets.map((offset) => {
+      const end = Math.min(content.length, offset + chunkSize);
+      const chunkContent = content.slice(offset, end).trim();
+      const lineOffsets = buildLineOffsets(content);
+      return {
+        id: crypto
+          .createHash("sha256")
+          .update(`${file.relativePath}:${offset}:${chunkContent}`)
+          .digest("hex")
+          .slice(0, 24),
+        contentHash: crypto.createHash("sha256").update(chunkContent).digest("hex"),
+        relativePath: file.relativePath,
+        extension: file.extension,
+        startLine: lineForOffsetFromIndex(lineOffsets, offset),
+        endLine: lineForOffsetFromIndex(lineOffsets, end),
+        content: chunkContent,
+      };
+    }).filter((chunk) => chunk.content);
+  }
+
+  const picks = new Set();
+  for (let slot = 0; slot < windowCount; slot += 1) {
+    const ratio = windowCount <= 1 ? 0 : slot / (windowCount - 1);
+    const index = Math.min(selectedOffsets.length - 1, Math.round(ratio * (selectedOffsets.length - 1)));
+    picks.add(index);
+  }
+  const lineOffsets = buildLineOffsets(content);
+  return Array.from(picks)
+    .sort((left, right) => left - right)
+    .map((pickIndex) => selectedOffsets[pickIndex])
+    .map((offset) => {
+      const end = Math.min(content.length, offset + chunkSize);
+      const chunkContent = content.slice(offset, end).trim();
+      return {
+        id: crypto
+          .createHash("sha256")
+          .update(`${file.relativePath}:${offset}:${chunkContent}`)
+          .digest("hex")
+          .slice(0, 24),
+        contentHash: crypto.createHash("sha256").update(chunkContent).digest("hex"),
+        relativePath: file.relativePath,
+        extension: file.extension,
+        startLine: lineForOffsetFromIndex(lineOffsets, offset),
+        endLine: lineForOffsetFromIndex(lineOffsets, end),
+        content: chunkContent,
+      };
+    })
+    .filter((chunk) => chunk.content);
+}