@cue-dev/retrieval-core 0.1.3

package/test/chunking.language-aware.test.ts

@@ -0,0 +1,1248 @@
+import Parser from "tree-sitter";
+import { describe, expect, it, vi } from "vitest";
+import { InMemoryQueryCache } from "@cue-dev/data-plane";
+import { getObservability } from "@cue-dev/observability";
+import { InMemoryIndexStore, RetrievalCore } from "../src/index.js";
+import {
+  __isChunkingBoundaryCandidateForTests,
+  __getChunkingParserInitAttemptsForTests,
+  __resetChunkingParserStateForTests,
+  __setChunkingParserLanguageLoaderForTests,
+  buildChunksForFile,
+  getChunkingParserAvailabilitySnapshot
+} from "../src/chunking.js";
+
+async function indexAndListChunks(input: {
+  core: RetrievalCore;
+  store: InMemoryIndexStore;
+  tenant_id: string;
+  workspace_id: string;
+  index_version: string;
+  file: {
+    path: string;
+    language?: string;
+    content: string;
+  };
+}): Promise<
+  Array<{
+    path: string;
+    start_line: number;
+    end_line: number;
+    snippet: string;
+  }>
+> {
+  await input.store.upsertWorkspace({
+    workspace_id: input.workspace_id,
+    tenant_id: input.tenant_id,
+    name: "chunking-test",
+    project_root_path: "/workspace/chunking-test"
+  });
+
+  const report = await input.core.indexArtifact({
+    tenant_id: input.tenant_id,
+    workspace_id: input.workspace_id,
+    index_version: input.index_version,
+    files: [input.file]
+  });
+  expect(report.status).toBe("ready");
+
+  const index = await input.store.getLatestReadyIndex({
+    tenant_id: input.tenant_id,
+    workspace_id: input.workspace_id
+  });
+  expect(index).toBeDefined();
+
+  const chunks = await input.store.listChunksByIndex({
+    tenant_id: input.tenant_id,
+    index_id: index!.index_id
+  });
+  return chunks.map((chunk) => ({
+    path: chunk.path,
+    start_line: chunk.start_line,
+    end_line: chunk.end_line,
+    snippet: chunk.snippet
+  }));
+}
+
+function mockBoundaryTreeFromSource(source: string): Parser.Tree {
+  const lines = source.split("\n");
+  const nodes: Parser.SyntaxNode[] = [];
+
+  const rustFunctionStart = lines.findIndex((line) => line.includes("pub fn"));
+  if (rustFunctionStart >= 0) {
+    let end = rustFunctionStart;
+    while (end < lines.length - 1 && lines[end]?.trim() !== "}") {
+      end += 1;
+    }
+    nodes.push({
+      type: "function_item",
+      startPosition: { row: rustFunctionStart, column: 0 },
+      endPosition: { row: end + 1, column: 0 },
+      parent: null
+    } as unknown as Parser.SyntaxNode);
+  }
+
+  const javaClassStart = lines.findIndex((line) => line.includes("class "));
+  if (javaClassStart >= 0) {
+    let end = lines.length - 1;
+    for (let row = lines.length - 1; row >= javaClassStart; row -= 1) {
+      if ((lines[row] ?? "").trim() === "}") {
+        end = row;
+        break;
+      }
+    }
+    nodes.push({
+      type: "class_declaration",
+      startPosition: { row: javaClassStart, column: 0 },
+      endPosition: { row: end + 1, column: 0 },
+      parent: null
+    } as unknown as Parser.SyntaxNode);
+  }
+
+  return {
+    rootNode: {
+      descendantsOfType(types: string[]): Parser.SyntaxNode[] {
+        return nodes.filter((node) => types.includes(node.type));
+      }
+    }
+  } as unknown as Parser.Tree;
+}
+
+describe("retrieval-core chunking", () => {
+  it("uses language-aware chunking for supported languages when configured", async () => {
+    const store = new InMemoryIndexStore();
+    const observability = getObservability(`retrieval-core-language-aware-${Date.now()}`);
+    const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+      observability,
+      chunkingConfig: {
+        strategy: "language_aware",
+        parse_timeout_ms: 500
+      }
+    });
+
+    const chunks = await indexAndListChunks({
+      core,
+      store,
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-a1",
+      file: {
+        path: "src/feature.ts",
+        language: "typescript",
+        content: [
+          "import { dep } from './dep';",
+          "",
+          "export function alpha(input: number) {",
+          "  const value = dep(input);",
+          "  return value + 1;",
+          "}",
+          "",
+          "export class Greeter {",
+          "  greet(name: string) {",
+          "    return `hello ${name}`;",
+          "  }",
+          "}",
+          "",
+          "export function beta(input: number) {",
+          "  const value = dep(input);",
+          "  return value * 2;",
+          "}"
+        ].join("\n")
+      }
+    });
+
+    expect(chunks.some((chunk) => chunk.snippet.includes("function alpha"))).toBe(true);
+    expect(chunks.some((chunk) => chunk.snippet.includes("class Greeter"))).toBe(true);
+    expect(chunks.some((chunk) => chunk.snippet.includes("function beta"))).toBe(true);
+    expect(chunks.every((chunk) => chunk.start_line >= 1 && chunk.end_line >= chunk.start_line)).toBe(true);
+
+    const strategyCounters = observability.metrics.readCounter("index_chunking_strategy_total");
+    expect(strategyCounters.some((counter) => counter.labels.strategy === "language_aware" && counter.labels.reason === "none")).toBe(
+      true
+    );
+  });
+
+  it("falls back to sliding chunks for unsupported languages", async () => {
+    const store = new InMemoryIndexStore();
+    const observability = getObservability(`retrieval-core-unsupported-${Date.now()}`);
+    const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+      observability,
+      chunkingConfig: {
+        strategy: "language_aware"
+      }
+    });
+
+    const chunks = await indexAndListChunks({
+      core,
+      store,
+      tenant_id: "tenant-b",
+      workspace_id: "ws-b",
+      index_version: "idx-b1",
+      file: {
+        path: "docs/readme.md",
+        language: "markdown",
+        content: Array.from({ length: 220 }, (_, idx) => `line ${idx + 1}: retrieval docs`).join("\n")
+      }
+    });
+
+    expect(chunks.length).toBeGreaterThan(0);
+    const fallbackCounters = observability.metrics.readCounter("index_chunking_fallback_total");
+    expect(
+      fallbackCounters.some(
+        (counter) => counter.labels.reason === "unsupported_language" && counter.labels.language === "markdown"
+      )
+    ).toBe(true);
+  });
+
+  it("falls back with parse_timeout_exceeded when parse latency exceeds the configured timeout", () => {
+    __resetChunkingParserStateForTests();
+    const nowSpy = vi.spyOn(Date, "now");
+    let tick = 0;
+    nowSpy.mockImplementation(() => {
+      tick += 3;
+      return tick;
+    });
+    try {
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        budget_tokenizer: "ranking" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 1,
+        enabled_languages: ["typescript"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const file = {
+        path: "src/timeout.ts",
+        language: "typescript",
+        content: ["export function alpha(input: number) {", "  return input + 1;", "}"].join("\n")
+      };
+
+      const result = buildChunksForFile({ file, config, tokenize });
+      expect(result.strategy).toBe("sliding");
+      expect(result.fallback_reason).toBe("parse_timeout_exceeded");
+      expect(result.language).toBe("typescript");
+    } finally {
+      nowSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("falls back with empty_language_boundaries when parser returns no eligible declaration boundaries", () => {
+    __resetChunkingParserStateForTests();
+    try {
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        budget_tokenizer: "ranking" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["typescript"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const file = {
+        path: "src/no-boundaries.ts",
+        language: "typescript",
+        content: ["const alpha = 1;", "const beta = alpha + 1;", "const gamma = beta + 1;"].join("\n")
+      };
+
+      const result = buildChunksForFile({ file, config, tokenize });
+      expect(result.strategy).toBe("sliding");
+      expect(result.fallback_reason).toBe("empty_language_boundaries");
+      expect(result.language).toBe("typescript");
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("falls back with parse_error when parser throws during language-aware parsing", () => {
+    __resetChunkingParserStateForTests();
+    const parseSpy = vi.spyOn(Parser.prototype as { parse: (input: string) => unknown }, "parse");
+    parseSpy.mockImplementation(() => {
+      throw new Error("forced parse failure");
+    });
+    try {
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        budget_tokenizer: "ranking" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["typescript"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const file = {
+        path: "src/parse-error.ts",
+        language: "typescript",
+        content: ["export function alpha(input: number) {", "  return input + 1;", "}"].join("\n")
+      };
+
+      const result = buildChunksForFile({ file, config, tokenize });
+      expect(result.strategy).toBe("sliding");
+      expect(result.fallback_reason).toBe("parse_error");
+      expect(result.language).toBe("typescript");
+    } finally {
+      parseSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("emits fallback metrics with reason and language labels for parse_timeout_exceeded", async () => {
+    __resetChunkingParserStateForTests();
+    const nowSpy = vi.spyOn(Date, "now");
+    let tick = 0;
+    nowSpy.mockImplementation(() => {
+      tick += 3;
+      return tick;
+    });
+    try {
+      const store = new InMemoryIndexStore();
+      const observability = getObservability(`retrieval-core-timeout-fallback-${Date.now()}`);
+      const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+        observability,
+        chunkingConfig: {
+          strategy: "language_aware",
+          parse_timeout_ms: 1,
+          enabled_languages: ["typescript"]
+        }
+      });
+
+      const chunks = await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-timeout",
+        workspace_id: "ws-timeout",
+        index_version: "idx-timeout-1",
+        file: {
+          path: "src/timeout.ts",
+          language: "typescript",
+          content: ["export function alpha(input: number) {", "  return input + 1;", "}"].join("\n")
+        }
+      });
+      expect(chunks.length).toBeGreaterThan(0);
+
+      const fallbackCounters = observability.metrics.readCounter("index_chunking_fallback_total");
+      expect(
+        fallbackCounters.some(
+          (counter) => counter.labels.reason === "parse_timeout_exceeded" && counter.labels.language === "typescript"
+        )
+      ).toBe(true);
+    } finally {
+      nowSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("emits fallback metrics with reason and language labels for empty_language_boundaries", async () => {
+    __resetChunkingParserStateForTests();
+    try {
+      const store = new InMemoryIndexStore();
+      const observability = getObservability(`retrieval-core-empty-boundaries-${Date.now()}`);
+      const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+        observability,
+        chunkingConfig: {
+          strategy: "language_aware",
+          recursive_semantic_chunking_enabled: false,
+          enabled_languages: ["typescript"]
+        }
+      });
+
+      const chunks = await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-empty",
+        workspace_id: "ws-empty",
+        index_version: "idx-empty-1",
+        file: {
+          path: "src/no-boundaries.ts",
+          language: "typescript",
+          content: ["const alpha = 1;", "const beta = alpha + 1;", "const gamma = beta + 1;"].join("\n")
+        }
+      });
+      expect(chunks.length).toBeGreaterThan(0);
+
+      const fallbackCounters = observability.metrics.readCounter("index_chunking_fallback_total");
+      expect(
+        fallbackCounters.some(
+          (counter) => counter.labels.reason === "empty_language_boundaries" && counter.labels.language === "typescript"
+        )
+      ).toBe(true);
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("emits fallback metrics with reason and language labels for parse_error", async () => {
+    __resetChunkingParserStateForTests();
+    const parseSpy = vi.spyOn(Parser.prototype as { parse: (input: string) => unknown }, "parse");
+    parseSpy.mockImplementation(() => {
+      throw new Error("forced parse failure");
+    });
+    try {
+      const store = new InMemoryIndexStore();
+      const observability = getObservability(`retrieval-core-parse-error-${Date.now()}`);
+      const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+        observability,
+        chunkingConfig: {
+          strategy: "language_aware",
+          enabled_languages: ["typescript"]
+        }
+      });
+
+      const chunks = await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-parse-error",
+        workspace_id: "ws-parse-error",
+        index_version: "idx-parse-error-1",
+        file: {
+          path: "src/parse-error.ts",
+          language: "typescript",
+          content: ["export function alpha(input: number) {", "  return input + 1;", "}"].join("\n")
+        }
+      });
+      expect(chunks.length).toBeGreaterThan(0);
+
+      const fallbackCounters = observability.metrics.readCounter("index_chunking_fallback_total");
+      expect(
+        fallbackCounters.some((counter) => counter.labels.reason === "parse_error" && counter.labels.language === "typescript")
+      ).toBe(true);
+    } finally {
+      parseSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("keeps deterministic line coordinates in sliding mode when lines repeat", async () => {
+    const store = new InMemoryIndexStore();
+    const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+      chunkingConfig: {
+        strategy: "sliding"
+      }
+    });
+
+    const chunks = await indexAndListChunks({
+      core,
+      store,
+      tenant_id: "tenant-c",
+      workspace_id: "ws-c",
+      index_version: "idx-c1",
+      file: {
+        path: "src/repeated.ts",
+        language: "typescript",
+        content: Array.from({ length: 400 }, () => "const token = 1;").join("\n")
+      }
+    });
+
+    expect(chunks.length).toBeGreaterThan(2);
+    const starts = chunks.map((chunk) => chunk.start_line);
+    expect(starts.some((line) => line > 1)).toBe(true);
+    for (let i = 1; i < starts.length; i += 1) {
+      expect(starts[i]).toBeGreaterThan(starts[i - 1] ?? 0);
+    }
+  });
+
+  it("keeps javascript boundary candidate filtering focused on symbol-bearing patterns", () => {
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "javascript",
+        nodeType: "function_expression",
+        parentType: "assignment_expression"
+      })
+    ).toBe(true);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "javascript",
+        nodeType: "function_expression",
+        parentType: "arguments"
+      })
+    ).toBe(false);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "javascript",
+        nodeType: "arrow_function",
+        parentType: "variable_declarator"
+      })
+    ).toBe(true);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "javascript",
+        nodeType: "method_definition",
+        parentType: "object"
+      })
+    ).toBe(true);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "function_expression",
+        parentType: "arguments",
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(false);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "arrow_function",
+        parentType: "arguments",
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(false);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "arrow_function",
+        parentType: "parenthesized_expression",
+        ancestorTypes: ["as_expression", "variable_declarator"],
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(true);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "arrow_function",
+        parentType: "parenthesized_expression",
+        ancestorTypes: ["export_statement"],
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(true);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "arrow_function",
+        parentType: "parenthesized_expression",
+        ancestorTypes: ["arguments"],
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(false);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "class",
+        parentType: "export_statement",
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(true);
+    expect(
+      __isChunkingBoundaryCandidateForTests({
+        parserLanguage: "typescript",
+        nodeType: "class",
+        parentType: "arguments",
+        boundaryStrictness: "semantic_js_ts"
+      })
+    ).toBe(false);
+  });
+
+  it("keeps typescript arrow-function snippets complete with semantic boundary strictness", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 12,
+      chunk_overlap_tokens: 4,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"]
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const file = {
+      path: "src/runtime.ts",
+      language: "typescript",
+      content: [
+        "export const alpha = (input: number) => {",
+        "  const first = input + 1;",
+        "  const second = first * 2;",
+        "  return second;",
+        "};",
+        "",
+        "export const beta = (input: number) => {",
+        "  return alpha(input);",
+        "};"
+      ].join("\n")
+    };
+    const result = buildChunksForFile({ file, config, tokenize });
+    expect(result.strategy).toBe("language_aware");
+    expect(result.fallback_reason).toBeUndefined();
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export const alpha = (input: number) => {"))).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("return second;"))).toBe(true);
+  });
+
+  it("keeps wrapped export-assigned functions as semantic boundaries", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 60,
+      chunk_overlap_tokens: 12,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"]
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const file = {
+      path: "src/export-assigned.ts",
+      language: "typescript",
+      content: [
+        "export default ((input: number) => {",
+        "  return input + 1;",
+        "});"
+      ].join("\n")
+    };
+    const result = buildChunksForFile({ file, config, tokenize });
+    expect(result.strategy).toBe("language_aware");
+    expect(result.fallback_reason).toBeUndefined();
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export default ((input: number) => {"))).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("return input + 1;"))).toBe(true);
+  });
+
+  it("keeps wrapped variable-initialized function expressions as semantic boundaries", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 60,
+      chunk_overlap_tokens: 12,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"]
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const file = {
+      path: "src/variable-wrapped.ts",
+      language: "typescript",
+      content: [
+        "const build = ((input: number) => {",
+        "  return input + 1;",
+        "}) as (input: number) => number;"
+      ].join("\n")
+    };
+    const result = buildChunksForFile({ file, config, tokenize });
+    expect(result.strategy).toBe("language_aware");
+    expect(result.fallback_reason).toBeUndefined();
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("const build = ((input: number) => {"))).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("return input + 1;"))).toBe(true);
+  });
+
+  it("keeps export-assigned class expressions as semantic boundaries", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 60,
+      chunk_overlap_tokens: 12,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"]
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const file = {
+      path: "src/export-class.ts",
+      language: "typescript",
+      content: ["export default class Service {}"].join("\n")
+    };
+    const result = buildChunksForFile({ file, config, tokenize });
+    expect(result.strategy).toBe("language_aware");
+    expect(result.fallback_reason).toBeUndefined();
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export default class Service {}"))).toBe(true);
+  });
+
+  it("keeps object literal method blocks as semantic boundaries", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 60,
+      chunk_overlap_tokens: 12,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"]
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const file = {
+      path: "src/object-methods.ts",
+      language: "typescript",
+      content: [
+        "export const handlers = {",
+        "  onReady() {",
+        "    return 1;",
+        "  }",
+        "};"
+      ].join("\n")
+    };
+    const result = buildChunksForFile({ file, config, tokenize });
+    expect(result.strategy).toBe("language_aware");
+    expect(result.fallback_reason).toBeUndefined();
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export const handlers = {"))).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("onReady()"))).toBe(true);
+  });
+
+  it("treats .mts and .cts as typescript for language-aware chunking", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 80,
+      chunk_overlap_tokens: 16,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript", "javascript"]
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const content = ["export function alpha(input: number) {", "  return input + 1;", "}"].join("\n");
+    const mts = buildChunksForFile({
+      file: {
+        path: "src/runtime.mts",
+        content
+      },
+      config,
+      tokenize
+    });
+    const cts = buildChunksForFile({
+      file: {
+        path: "src/runtime.cts",
+        content
+      },
+      config,
+      tokenize
+    });
+    expect(mts.strategy).toBe("language_aware");
+    expect(cts.strategy).toBe("language_aware");
+    expect(mts.fallback_reason).toBeUndefined();
+    expect(cts.fallback_reason).toBeUndefined();
+  });
+
+  it("uses parser-aware chunking for rust and java declarations without mid-body truncation", () => {
+    __resetChunkingParserStateForTests();
+    const parseSpy = vi.spyOn(Parser.prototype as { parse: (input: string) => unknown }, "parse");
+    parseSpy.mockImplementation((source: string) => mockBoundaryTreeFromSource(source));
+    try {
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 64,
+        chunk_overlap_tokens: 12,
+        budget_tokenizer: "lightweight" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["rust", "java"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+
+      const rustResult = buildChunksForFile({
+        file: {
+          path: "src/counter.rs",
+          content: [
+            "pub struct Counter {",
+            "  value: i32,",
+            "}",
+            "",
+            "impl Counter {",
+            "  pub fn increment(&mut self, amount: i32) {",
+            "    self.value += amount;",
+            "  }",
+            "}",
+            "",
+            "pub fn compute_total(input: i32) -> i32 {",
+            "  let doubled = input * 2;",
+            "  doubled + 1",
+            "}"
+          ].join("\n")
+        },
+        config,
+        tokenize
+      });
+      expect(rustResult.strategy).toBe("language_aware");
+      expect(rustResult.fallback_reason).toBeUndefined();
+      expect(rustResult.language).toBe("rust");
+      const rustFunctionChunk = rustResult.chunks.find((chunk) => chunk.snippet.includes("pub fn compute_total(input: i32) -> i32 {"));
+      expect(rustFunctionChunk).toBeDefined();
+      expect(rustFunctionChunk?.snippet.includes("let doubled = input * 2;")).toBe(true);
+      expect(rustFunctionChunk?.snippet.includes("doubled + 1")).toBe(true);
+
+      const javaResult = buildChunksForFile({
+        file: {
+          path: "src/Calculator.java",
+          content: [
+            "public class Calculator {",
+            "  public int sum(int left, int right) {",
+            "    int total = left + right;",
+            "    return total;",
+            "  }",
+            "",
+            "  public int multiply(int left, int right) {",
+            "    return left * right;",
+            "  }",
+            "}"
+          ].join("\n")
+        },
+        config,
+        tokenize
+      });
+      expect(javaResult.strategy).toBe("language_aware");
+      expect(javaResult.fallback_reason).toBeUndefined();
+      expect(javaResult.language).toBe("java");
+      const javaMethodChunk = javaResult.chunks.find((chunk) => chunk.snippet.includes("public int sum(int left, int right) {"));
+      expect(javaMethodChunk).toBeDefined();
+      expect(javaMethodChunk?.snippet.includes("int total = left + right;")).toBe(true);
+      expect(javaMethodChunk?.snippet.includes("return total;")).toBe(true);
+    } finally {
+      parseSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("resolves rust/java aliases and paths for parser-aware chunking", () => {
+    __resetChunkingParserStateForTests();
+    const parseSpy = vi.spyOn(Parser.prototype as { parse: (input: string) => unknown }, "parse");
+    parseSpy.mockImplementation((source: string) => mockBoundaryTreeFromSource(source));
+    try {
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 80,
+        chunk_overlap_tokens: 16,
+        budget_tokenizer: "lightweight" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["rust", "java"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const rustContent = ["pub fn alpha(input: i32) -> i32 {", "  input + 1", "}"].join("\n");
+      const javaContent = ["public class Demo {", "  int alpha() {", "    return 1;", "  }", "}"].join("\n");
+
+      const rustAlias = buildChunksForFile({
+        file: {
+          path: "src/alpha.txt",
+          language: "rs",
+          content: rustContent
+        },
+        config,
+        tokenize
+      });
+      const rustPath = buildChunksForFile({
+        file: {
+          path: "src/alpha.rs",
+          content: rustContent
+        },
+        config,
+        tokenize
+      });
+      const javaPath = buildChunksForFile({
+        file: {
+          path: "src/Demo.java",
+          content: javaContent
+        },
+        config,
+        tokenize
+      });
+
+      expect(rustAlias.strategy).toBe("language_aware");
+      expect(rustPath.strategy).toBe("language_aware");
+      expect(javaPath.strategy).toBe("language_aware");
+      expect(rustAlias.fallback_reason).toBeUndefined();
+      expect(rustPath.fallback_reason).toBeUndefined();
+      expect(javaPath.fallback_reason).toBeUndefined();
+      expect(rustAlias.language).toBe("rust");
+      expect(rustPath.language).toBe("rust");
+      expect(javaPath.language).toBe("java");
+    } finally {
+      parseSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("falls back with parser_unavailable for rust/java parser load failures and caches deterministically", () => {
+    __resetChunkingParserStateForTests();
+    try {
+      __setChunkingParserLanguageLoaderForTests("rust", () => {
+        throw new Error("forced rust parser load failure");
+      });
+      __setChunkingParserLanguageLoaderForTests("java", () => {
+        throw new Error("forced java parser load failure");
+      });
+
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        budget_tokenizer: "ranking" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["rust", "java"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+
+      const rustFile = {
+        path: "src/runtime.rs",
+        language: "rust",
+        content: ["pub fn alpha() -> i32 {", "  1", "}"].join("\n")
+      };
+      const javaFile = {
+        path: "src/Runtime.java",
+        language: "java",
+        content: ["public class Runtime {", "  int alpha() {", "    return 1;", "  }", "}"].join("\n")
+      };
+
+      const rustFirst = buildChunksForFile({ file: rustFile, config, tokenize });
+      const rustSecond = buildChunksForFile({ file: rustFile, config, tokenize });
+      const javaFirst = buildChunksForFile({ file: javaFile, config, tokenize });
+      const javaSecond = buildChunksForFile({ file: javaFile, config, tokenize });
+
+      expect(rustFirst.fallback_reason).toBe("parser_unavailable");
+      expect(rustSecond.fallback_reason).toBe("parser_unavailable");
+      expect(javaFirst.fallback_reason).toBe("parser_unavailable");
+      expect(javaSecond.fallback_reason).toBe("parser_unavailable");
+      expect(rustFirst.language).toBe("rust");
+      expect(javaFirst.language).toBe("java");
+
+      const attempts = __getChunkingParserInitAttemptsForTests();
+      expect(attempts.rust).toBe(1);
+      expect(attempts.java).toBe(1);
+
+      const snapshot = getChunkingParserAvailabilitySnapshot({
+        enabled_languages: ["rust", "java"]
+      });
+      expect(snapshot.some((entry) => entry.language === "rust" && entry.status === "unavailable")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "java" && entry.status === "unavailable")).toBe(true);
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("falls back with parse_error for rust/java when parser throws during parse", () => {
+    __resetChunkingParserStateForTests();
+    const parseSpy = vi.spyOn(Parser.prototype as { parse: (input: string) => unknown }, "parse");
+    parseSpy.mockImplementation(() => {
+      throw new Error("forced parse failure");
+    });
+    try {
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        budget_tokenizer: "ranking" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["rust", "java"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const rustResult = buildChunksForFile({
+        file: {
+          path: "src/parse-error.rs",
+          language: "rust",
+          content: ["pub fn alpha() -> i32 {", "  1", "}"].join("\n")
+        },
+        config,
+        tokenize
+      });
+      const javaResult = buildChunksForFile({
+        file: {
+          path: "src/ParseError.java",
+          language: "java",
+          content: ["public class ParseError {", "  int alpha() {", "    return 1;", "  }", "}"].join("\n")
+        },
+        config,
+        tokenize
+      });
+
+      expect(rustResult.strategy).toBe("sliding");
+      expect(javaResult.strategy).toBe("sliding");
+      expect(rustResult.fallback_reason).toBe("parse_error");
+      expect(javaResult.fallback_reason).toBe("parse_error");
+      expect(rustResult.language).toBe("rust");
+      expect(javaResult.language).toBe("java");
+    } finally {
+      parseSpy.mockRestore();
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("builds recursive semantic chunks with stable declaration boundaries", () => {
+    const config = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 28,
+      chunk_overlap_tokens: 6,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"],
+      recursive_semantic_chunking_enabled: true,
+      semantic_merge_gap_lines: 1,
+      semantic_merge_max_span_lines: 120
+    };
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const file = {
+      path: "src/recursive-runtime.ts",
+      language: "typescript",
+      content: [
+        "export function alpha(input: number): number {",
+        "  const a = input + 1;",
+        "  return a;",
+        "}",
+        "",
+        "export function beta(input: number): number {",
+        "  const b = alpha(input) * 2;",
+        "  return b;",
+        "}",
+        "",
+        "export function gamma(input: number): number {",
+        "  const c = beta(input) + 3;",
+        "  return c;",
+        "}"
+      ].join("\n")
+    };
+    const result = buildChunksForFile({ file, config, tokenize });
+    expect(result.strategy).toBe("language_aware");
+    expect(result.fallback_reason).toBeUndefined();
+    expect(result.recursive_semantic_chunking_used).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export function alpha"))).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export function beta"))).toBe(true);
+    expect(result.chunks.some((chunk) => chunk.snippet.includes("export function gamma"))).toBe(true);
+    expect(result.chunks.every((chunk) => chunk.start_line >= 1 && chunk.end_line >= chunk.start_line)).toBe(true);
+  });
+
+  it("forward-absorbs comment windows into following code windows when enabled", () => {
+    const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+    const baseConfig = {
+      strategy: "language_aware" as const,
+      fallback_strategy: "sliding" as const,
+      target_chunk_tokens: 30,
+      chunk_overlap_tokens: 4,
+      budget_tokenizer: "lightweight" as const,
+      boundary_strictness: "semantic_js_ts" as const,
+      max_chunks_per_file: 300,
+      parse_timeout_ms: 80,
+      enabled_languages: ["typescript"],
+      recursive_semantic_chunking_enabled: true,
+      semantic_merge_gap_lines: 0,
+      semantic_merge_max_span_lines: 120
+    };
+    const file = {
+      path: "src/comment-absorb.ts",
+      language: "typescript",
+      content: [
+        "// Explanation for alpha",
+        "// Keep this doc attached to alpha",
+        "",
+        "export function alpha(input: number): number {",
+        "  return input + 1;",
+        "}",
+        "",
+        "export function beta(input: number): number {",
+        "  return alpha(input);",
+        "}"
+      ].join("\n")
+    };
+
+    const withAbsorb = buildChunksForFile({
+      file,
+      config: {
+        ...baseConfig,
+        comment_forward_absorb_enabled: true
+      },
+      tokenize
+    });
+    const withoutAbsorb = buildChunksForFile({
+      file,
+      config: {
+        ...baseConfig,
+        comment_forward_absorb_enabled: false
+      },
+      tokenize
+    });
+
+    expect(withAbsorb.chunks.some((chunk) => chunk.snippet.includes("Explanation for alpha") && chunk.snippet.includes("export function alpha"))).toBe(
+      true
+    );
+    expect(
+      withoutAbsorb.chunks.some(
+        (chunk) => chunk.snippet.includes("Explanation for alpha") && !chunk.snippet.includes("export function alpha")
+      )
+    ).toBe(true);
+  });
+
+  it("emits parser availability snapshot and avoids repeated parser init attempts", async () => {
+    __resetChunkingParserStateForTests();
+    try {
+      const store = new InMemoryIndexStore();
+      const observability = getObservability(`retrieval-core-js-py-${Date.now()}`);
+      const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+        observability,
+        chunkingConfig: {
+          strategy: "language_aware",
+          enabled_languages: ["typescript", "javascript", "python", "go", "rust", "java"]
+        }
+      });
+
+      await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-d",
+        workspace_id: "ws-d",
+        index_version: "idx-d1",
+        file: {
+          path: "src/runtime.js",
+          language: "javascript",
+          content: ["export function alpha() {", "  return 1;", "}"].join("\n")
+        }
+      });
+
+      await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-d",
+        workspace_id: "ws-d",
+        index_version: "idx-d2",
+        file: {
+          path: "src/runtime.py",
+          language: "python",
+          content: ["def alpha():", "    return 1"].join("\n")
+        }
+      });
+
+      await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-d",
+        workspace_id: "ws-d",
+        index_version: "idx-d3",
+        file: {
+          path: "src/runtime.rs",
+          language: "rust",
+          content: ["pub fn alpha() -> i32 {", "  1", "}"].join("\n")
+        }
+      });
+
+      await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-d",
+        workspace_id: "ws-d",
+        index_version: "idx-d4",
+        file: {
+          path: "src/Runtime.java",
+          language: "java",
+          content: ["public class Runtime {", "  int alpha() {", "    return 1;", "  }", "}"].join("\n")
+        }
+      });
+
+      const attempts = __getChunkingParserInitAttemptsForTests();
+      expect((attempts.javascript ?? 0) <= 1).toBe(true);
+      expect((attempts.python ?? 0) <= 1).toBe(true);
+      expect((attempts.rust ?? 0) <= 1).toBe(true);
+      expect((attempts.java ?? 0) <= 1).toBe(true);
+
+      const strategyCounters = observability.metrics.readCounter("index_chunking_strategy_total");
+      expect(
+        strategyCounters.some((counter) => counter.labels.language === "javascript")
+      ).toBe(true);
+      expect(strategyCounters.some((counter) => counter.labels.language === "python")).toBe(true);
+      expect(strategyCounters.some((counter) => counter.labels.language === "rust")).toBe(true);
+      expect(strategyCounters.some((counter) => counter.labels.language === "java")).toBe(true);
+
+      const snapshot = getChunkingParserAvailabilitySnapshot({
+        enabled_languages: ["typescript", "javascript", "python", "go", "rust", "java"]
+      });
+      expect(snapshot.some((entry) => entry.language === "javascript" && entry.status === "available")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "python" && entry.status === "available")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "rust" && entry.status === "available")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "java" && entry.status === "available")).toBe(true);
+
+      const availabilityGauges = observability.metrics.readGauge("index_chunking_parser_availability");
+      expect(
+        availabilityGauges.some((point) => point.labels.language === "javascript" && point.labels.status === "available")
+      ).toBe(true);
+      expect(availabilityGauges.some((point) => point.labels.language === "python" && point.labels.status === "available")).toBe(
+        true
+      );
+      expect(availabilityGauges.some((point) => point.labels.language === "rust" && point.labels.status === "available")).toBe(
+        true
+      );
+      expect(availabilityGauges.some((point) => point.labels.language === "java" && point.labels.status === "available")).toBe(
+        true
+      );
+
+      const fallbackCounters = observability.metrics.readCounter("index_chunking_fallback_total");
+      expect(
+        fallbackCounters.some(
+          (counter) =>
+            counter.labels.reason === "parser_unavailable" &&
+            (counter.labels.language === "javascript" ||
+              counter.labels.language === "python" ||
+              counter.labels.language === "rust" ||
+              counter.labels.language === "java")
+        )
+      ).toBe(false);
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("caches parser unavailability and avoids repeated parser init attempts", () => {
+    __resetChunkingParserStateForTests();
+    try {
+      __setChunkingParserLanguageLoaderForTests("python", () => {
+        throw new Error("forced parser load failure");
+      });
+
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        budget_tokenizer: "ranking" as const,
+        boundary_strictness: "legacy" as const,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["python"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const file = {
+        path: "src/runtime.py",
+        language: "python",
+        content: ["def alpha():", "    return 1"].join("\n")
+      };
+
+      const first = buildChunksForFile({ file, config, tokenize });
+      const second = buildChunksForFile({ file, config, tokenize });
+      const third = buildChunksForFile({ file, config, tokenize });
+
+      expect(first.fallback_reason).toBe("parser_unavailable");
+      expect(second.fallback_reason).toBe("parser_unavailable");
+      expect(third.fallback_reason).toBe("parser_unavailable");
+
+      const attempts = __getChunkingParserInitAttemptsForTests();
+      expect(attempts.python).toBe(1);
+
+      const snapshot = getChunkingParserAvailabilitySnapshot({
+        enabled_languages: ["python", "typescript"]
+      });
+      expect(snapshot.some((entry) => entry.language === "python" && entry.status === "unavailable")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "typescript")).toBe(true);
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+});