@rce-mcp/retrieval-core 0.1.0

package/test/chunking.language-aware.test.ts

@@ -0,0 +1,279 @@
+import { describe, expect, it } from "vitest";
+import { InMemoryQueryCache } from "@rce-mcp/data-plane";
+import { getObservability } from "@rce-mcp/observability";
+import { InMemoryIndexStore, RetrievalCore } from "../src/index.js";
+import {
+  __getChunkingParserInitAttemptsForTests,
+  __resetChunkingParserStateForTests,
+  __setChunkingParserLanguageLoaderForTests,
+  buildChunksForFile,
+  getChunkingParserAvailabilitySnapshot
+} from "../src/chunking.js";
+
+async function indexAndListChunks(input: {
+  core: RetrievalCore;
+  store: InMemoryIndexStore;
+  tenant_id: string;
+  workspace_id: string;
+  index_version: string;
+  file: {
+    path: string;
+    language?: string;
+    content: string;
+  };
+}): Promise<
+  Array<{
+    path: string;
+    start_line: number;
+    end_line: number;
+    snippet: string;
+  }>
+> {
+  await input.store.upsertWorkspace({
+    workspace_id: input.workspace_id,
+    tenant_id: input.tenant_id,
+    name: "chunking-test",
+    project_root_path: "/workspace/chunking-test"
+  });
+
+  const report = await input.core.indexArtifact({
+    tenant_id: input.tenant_id,
+    workspace_id: input.workspace_id,
+    index_version: input.index_version,
+    files: [input.file]
+  });
+  expect(report.status).toBe("ready");
+
+  const index = await input.store.getLatestReadyIndex({
+    tenant_id: input.tenant_id,
+    workspace_id: input.workspace_id
+  });
+  expect(index).toBeDefined();
+
+  const chunks = await input.store.listChunksByIndex({
+    tenant_id: input.tenant_id,
+    index_id: index!.index_id
+  });
+  return chunks.map((chunk) => ({
+    path: chunk.path,
+    start_line: chunk.start_line,
+    end_line: chunk.end_line,
+    snippet: chunk.snippet
+  }));
+}
+
+describe("retrieval-core chunking", () => {
+  it("uses language-aware chunking for supported languages when configured", async () => {
+    const store = new InMemoryIndexStore();
+    const observability = getObservability(`retrieval-core-language-aware-${Date.now()}`);
+    const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+      observability,
+      chunkingConfig: {
+        strategy: "language_aware",
+        parse_timeout_ms: 500
+      }
+    });
+
+    const chunks = await indexAndListChunks({
+      core,
+      store,
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-a1",
+      file: {
+        path: "src/feature.ts",
+        language: "typescript",
+        content: [
+          "import { dep } from './dep';",
+          "",
+          "export function alpha(input: number) {",
+          "  const value = dep(input);",
+          "  return value + 1;",
+          "}",
+          "",
+          "export class Greeter {",
+          "  greet(name: string) {",
+          "    return `hello ${name}`;",
+          "  }",
+          "}",
+          "",
+          "export function beta(input: number) {",
+          "  const value = dep(input);",
+          "  return value * 2;",
+          "}"
+        ].join("\n")
+      }
+    });
+
+    expect(chunks.some((chunk) => chunk.snippet.includes("function alpha"))).toBe(true);
+    expect(chunks.some((chunk) => chunk.snippet.includes("class Greeter"))).toBe(true);
+    expect(chunks.some((chunk) => chunk.snippet.includes("function beta"))).toBe(true);
+    expect(chunks.every((chunk) => chunk.start_line >= 1 && chunk.end_line >= chunk.start_line)).toBe(true);
+
+    const strategyCounters = observability.metrics.readCounter("index_chunking_strategy_total");
+    expect(strategyCounters.some((counter) => counter.labels.strategy === "language_aware" && counter.labels.reason === "none")).toBe(
+      true
+    );
+  });
+
+  it("falls back to sliding chunks for unsupported languages", async () => {
+    const store = new InMemoryIndexStore();
+    const observability = getObservability(`retrieval-core-unsupported-${Date.now()}`);
+    const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+      observability,
+      chunkingConfig: {
+        strategy: "language_aware"
+      }
+    });
+
+    const chunks = await indexAndListChunks({
+      core,
+      store,
+      tenant_id: "tenant-b",
+      workspace_id: "ws-b",
+      index_version: "idx-b1",
+      file: {
+        path: "docs/readme.md",
+        language: "markdown",
+        content: Array.from({ length: 220 }, (_, idx) => `line ${idx + 1}: retrieval docs`).join("\n")
+      }
+    });
+
+    expect(chunks.length).toBeGreaterThan(0);
+    const fallbackCounters = observability.metrics.readCounter("index_chunking_fallback_total");
+    expect(
+      fallbackCounters.some(
+        (counter) => counter.labels.reason === "unsupported_language" && counter.labels.language === "markdown"
+      )
+    ).toBe(true);
+  });
+
+  it("keeps deterministic line coordinates in sliding mode when lines repeat", async () => {
+    const store = new InMemoryIndexStore();
+    const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+      chunkingConfig: {
+        strategy: "sliding"
+      }
+    });
+
+    const chunks = await indexAndListChunks({
+      core,
+      store,
+      tenant_id: "tenant-c",
+      workspace_id: "ws-c",
+      index_version: "idx-c1",
+      file: {
+        path: "src/repeated.ts",
+        language: "typescript",
+        content: Array.from({ length: 400 }, () => "const token = 1;").join("\n")
+      }
+    });
+
+    expect(chunks.length).toBeGreaterThan(2);
+    const starts = chunks.map((chunk) => chunk.start_line);
+    expect(starts.some((line) => line > 1)).toBe(true);
+    for (let i = 1; i < starts.length; i += 1) {
+      expect(starts[i]).toBeGreaterThan(starts[i - 1] ?? 0);
+    }
+  });
+
+  it("emits parser availability snapshot and avoids repeated parser init attempts", async () => {
+    __resetChunkingParserStateForTests();
+    try {
+      const store = new InMemoryIndexStore();
+      const observability = getObservability(`retrieval-core-js-py-${Date.now()}`);
+      const core = new RetrievalCore(store, new InMemoryQueryCache(), {
+        observability,
+        chunkingConfig: {
+          strategy: "language_aware"
+        }
+      });
+
+      await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-d",
+        workspace_id: "ws-d",
+        index_version: "idx-d1",
+        file: {
+          path: "src/runtime.js",
+          language: "javascript",
+          content: ["export function alpha() {", "  return 1;", "}"].join("\n")
+        }
+      });
+
+      await indexAndListChunks({
+        core,
+        store,
+        tenant_id: "tenant-d",
+        workspace_id: "ws-d",
+        index_version: "idx-d2",
+        file: {
+          path: "src/runtime.py",
+          language: "python",
+          content: ["def alpha():", "    return 1"].join("\n")
+        }
+      });
+
+      const attempts = __getChunkingParserInitAttemptsForTests();
+      expect((attempts.javascript ?? 0) <= 1).toBe(true);
+      expect((attempts.python ?? 0) <= 1).toBe(true);
+
+      const snapshot = getChunkingParserAvailabilitySnapshot({
+        enabled_languages: ["typescript", "javascript", "python", "go"]
+      });
+      expect(snapshot.some((entry) => entry.language === "javascript")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "python")).toBe(true);
+
+      const availabilityGauges = observability.metrics.readGauge("index_chunking_parser_availability");
+      expect(availabilityGauges.some((point) => point.labels.language === "javascript")).toBe(true);
+      expect(availabilityGauges.some((point) => point.labels.language === "python")).toBe(true);
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+
+  it("caches parser unavailability and avoids repeated parser init attempts", () => {
+    __resetChunkingParserStateForTests();
+    try {
+      __setChunkingParserLanguageLoaderForTests("python", () => {
+        throw new Error("forced parser load failure");
+      });
+
+      const config = {
+        strategy: "language_aware" as const,
+        fallback_strategy: "sliding" as const,
+        target_chunk_tokens: 220,
+        chunk_overlap_tokens: 40,
+        max_chunks_per_file: 300,
+        parse_timeout_ms: 80,
+        enabled_languages: ["python"]
+      };
+      const tokenize = (text: string) => text.split(/\s+/).filter(Boolean);
+      const file = {
+        path: "src/runtime.py",
+        language: "python",
+        content: ["def alpha():", "    return 1"].join("\n")
+      };
+
+      const first = buildChunksForFile({ file, config, tokenize });
+      const second = buildChunksForFile({ file, config, tokenize });
+      const third = buildChunksForFile({ file, config, tokenize });
+
+      expect(first.fallback_reason).toBe("parser_unavailable");
+      expect(second.fallback_reason).toBe("parser_unavailable");
+      expect(third.fallback_reason).toBe("parser_unavailable");
+
+      const attempts = __getChunkingParserInitAttemptsForTests();
+      expect(attempts.python).toBe(1);
+
+      const snapshot = getChunkingParserAvailabilitySnapshot({
+        enabled_languages: ["python", "typescript"]
+      });
+      expect(snapshot.some((entry) => entry.language === "python" && entry.status === "unavailable")).toBe(true);
+      expect(snapshot.some((entry) => entry.language === "typescript")).toBe(true);
+    } finally {
+      __resetChunkingParserStateForTests();
+    }
+  });
+});

package/test/chunking.parser-availability.poc.test.ts

@@ -0,0 +1,60 @@
+import { createRequire } from "node:module";
+import { describe, expect, it } from "vitest";
+import Parser from "tree-sitter";
+import JavaScriptV023 from "tree-sitter-javascript-v023";
+import PythonV023 from "tree-sitter-python-v023";
+import { Language as WebLanguage, Parser as WebParser } from "web-tree-sitter";
+import {
+  __resetChunkingParserStateForTests,
+  getChunkingParserAvailabilitySnapshot
+} from "../src/chunking.js";
+
+const require = createRequire(import.meta.url);
+
+describe("chunking parser availability PoC", () => {
+  it("captures current baseline parser unavailability for javascript and python", () => {
+    __resetChunkingParserStateForTests();
+    const snapshot = getChunkingParserAvailabilitySnapshot({
+      enabled_languages: ["typescript", "javascript", "python", "go"]
+    });
+    const byLanguage = new Map(snapshot.map((entry) => [entry.language, entry]));
+
+    expect(byLanguage.get("typescript")?.status).toBe("available");
+    expect(byLanguage.get("go")?.status).toBe("available");
+    expect(byLanguage.get("javascript")?.status).toBe("unavailable");
+    expect(byLanguage.get("python")?.status).toBe("unavailable");
+  });
+
+  it("proves native availability with tree-sitter-compatible alias grammars", () => {
+    const javascriptParser = new Parser();
+    javascriptParser.setLanguage(JavaScriptV023 as unknown as Parser.Language);
+    const javascriptTree = javascriptParser.parse("function alpha(){ return 1; }\n");
+    expect(javascriptTree?.rootNode?.type).toBe("program");
+
+    const pythonParser = new Parser();
+    pythonParser.setLanguage(PythonV023 as unknown as Parser.Language);
+    const pythonTree = pythonParser.parse("def alpha():\n    return 1\n");
+    expect(pythonTree?.rootNode?.type).toBe("module");
+  });
+
+  it("proves wasm availability with web-tree-sitter for javascript and python", async () => {
+    const runtimeWasm = require.resolve("web-tree-sitter/web-tree-sitter.wasm");
+    await WebParser.init({
+      locateFile() {
+        return runtimeWasm;
+      }
+    });
+
+    const javascriptLanguage = await WebLanguage.load(require.resolve("tree-sitter-javascript/tree-sitter-javascript.wasm"));
+    const javascriptParser = new WebParser();
+    javascriptParser.setLanguage(javascriptLanguage);
+    const javascriptTree = javascriptParser.parse("function alpha(){ return 1; }\n");
+    expect(javascriptTree?.rootNode?.type).toBe("program");
+
+    const pythonLanguage = await WebLanguage.load(require.resolve("tree-sitter-python/tree-sitter-python.wasm"));
+    const pythonParser = new WebParser();
+    pythonParser.setLanguage(pythonLanguage);
+    const pythonTree = pythonParser.parse("def alpha():\n    return 1\n");
+    expect(pythonTree?.rootNode?.type).toBe("module");
+  });
+});

package/test/embedding-provider.test.ts

@@ -0,0 +1,121 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { OpenAICompatibleEmbeddingProvider, RetrievalError } from "../src/index.js";
+
+describe("openai-compatible embedding provider", () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    vi.restoreAllMocks();
+  });
+
+  it("retries on 5xx and succeeds", async () => {
+    let calls = 0;
+    globalThis.fetch = vi.fn(async () => {
+      calls += 1;
+      if (calls === 1) {
+        return new Response(JSON.stringify({ error: "temporary" }), { status: 503 });
+      }
+      return new Response(
+        JSON.stringify({
+          data: [{ index: 0, embedding: [0.1, 0.2, 0.3] }]
+        }),
+        { status: 200 }
+      );
+    }) as typeof fetch;
+
+    const provider = new OpenAICompatibleEmbeddingProvider({
+      base_url: "https://router.tumuer.me/v1",
+      api_key: "test-key",
+      model: "Qwen/Qwen3-Embedding-4B",
+      dimensions: 3,
+      timeout_ms: 200,
+      batch_size: 16,
+      max_retries: 2
+    });
+
+    const vectors = await provider.embed({
+      texts: ["hello"],
+      purpose: "query"
+    });
+
+    expect(vectors).toEqual([[0.1, 0.2, 0.3]]);
+    expect(calls).toBe(2);
+  });
+
+  it("does not retry auth/config 4xx failures", async () => {
+    let calls = 0;
+    globalThis.fetch = vi.fn(async () => {
+      calls += 1;
+      return new Response(JSON.stringify({ error: "bad key" }), { status: 401 });
+    }) as typeof fetch;
+
+    const provider = new OpenAICompatibleEmbeddingProvider({
+      base_url: "https://router.tumuer.me/v1",
+      api_key: "test-key",
+      dimensions: 3
+    });
+
+    await expect(
+      provider.embed({
+        texts: ["hello"],
+        purpose: "query"
+      })
+    ).rejects.toMatchObject({
+      code: "UPSTREAM_FAILURE"
+    } satisfies Partial<RetrievalError>);
+    expect(calls).toBe(1);
+  });
+
+  it("rejects non-numeric vectors", async () => {
+    globalThis.fetch = vi.fn(async () => {
+      return new Response(
+        JSON.stringify({
+          data: [{ index: 0, embedding: [0.1, "bad", 0.3] }]
+        }),
+        { status: 200 }
+      );
+    }) as typeof fetch;
+
+    const provider = new OpenAICompatibleEmbeddingProvider({
+      base_url: "https://router.tumuer.me/v1",
+      api_key: "test-key",
+      dimensions: 3
+    });
+
+    await expect(
+      provider.embed({
+        texts: ["hello"],
+        purpose: "query"
+      })
+    ).rejects.toMatchObject({
+      code: "UPSTREAM_FAILURE"
+    } satisfies Partial<RetrievalError>);
+  });
+
+  it("rejects vectors with unexpected dimensions", async () => {
+    globalThis.fetch = vi.fn(async () => {
+      return new Response(
+        JSON.stringify({
+          data: [{ index: 0, embedding: [0.1, 0.2] }]
+        }),
+        { status: 200 }
+      );
+    }) as typeof fetch;
+
+    const provider = new OpenAICompatibleEmbeddingProvider({
+      base_url: "https://router.tumuer.me/v1",
+      api_key: "test-key",
+      dimensions: 3
+    });
+
+    await expect(
+      provider.embed({
+        texts: ["hello"],
+        purpose: "query"
+      })
+    ).rejects.toMatchObject({
+      code: "UPSTREAM_FAILURE"
+    } satisfies Partial<RetrievalError>);
+  });
+});