@rce-mcp/retrieval-core 0.1.0

package/test/enhance-confidence.test.ts

@@ -0,0 +1,357 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { InMemoryQueryCache } from "@rce-mcp/data-plane";
+import { InMemoryIndexStore, RetrievalCore } from "../src/index.js";
+
+describe("enhance prompt confidence controls", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("adds clarification questions and filters risky refs when retrieval confidence is low", async () => {
+    const core = new RetrievalCore(new InMemoryIndexStore(), new InMemoryQueryCache());
+    vi.spyOn(core, "searchContext").mockResolvedValue({
+      trace_id: "trc-low-confidence",
+      results: [
+        {
+          path: "docs/reference/hooks.md",
+          start_line: 1,
+          end_line: 20,
+          snippet: "addHook lifecycle ordering and examples",
+          score: 0.71,
+          reason: "semantic match"
+        },
+        {
+          path: "test/hooks.test.js",
+          start_line: 1,
+          end_line: 25,
+          snippet: "addHook lifecycle should preserve plugin encapsulation",
+          score: 0.707,
+          reason: "path and token overlap"
+        },
+        {
+          path: "lib/hooks.js",
+          start_line: 1,
+          end_line: 40,
+          snippet: "function addHook(name, fn) { return hooks.add(name, fn) }",
+          score: 0.704,
+          reason: "exact symbol match"
+        },
+        {
+          path: "examples/hooks.js",
+          start_line: 1,
+          end_line: 22,
+          snippet: "example for addHook lifecycle behavior",
+          score: 0.703,
+          reason: "semantic match"
+        },
+        {
+          path: "lib/error-handler.js",
+          start_line: 1,
+          end_line: 30,
+          snippet: "function setErrorHandler(fn) { return fn }",
+          score: 0.702,
+          reason: "path and token overlap"
+        }
+      ],
+      search_metadata: {
+        latency_ms: 15,
+        retrieval_mode: "hybrid",
+        index_version: "idx-1"
+      }
+    });
+
+    const output = await core.enhancePrompt({
+      trace_id: "trc-low-confidence",
+      tenant_id: "tenant-dev",
+      workspace_id: "ws-dev",
+      request: {
+        prompt: "Refine addHook lifecycle runtime behavior and preserve plugin compatibility.",
+        project_root_path: "/workspace/dev",
+        conversation_history: [{ role: "user", content: "Avoid docs and tests as primary context." }]
+      }
+    });
+
+    expect(output.warnings.some((warning) => /Low retrieval confidence/i.test(warning))).toBe(true);
+    expect(output.questions.length).toBeGreaterThan(0);
+    expect(output.context_refs.length).toBeGreaterThan(0);
+    expect(output.context_refs.some((ref) => ref.path === "lib/hooks.js")).toBe(true);
+    expect(output.context_refs.every((ref) => !/^docs\/|^test\/|^examples\//.test(ref.path))).toBe(true);
+  });
+
+  it("builds enhancer retrieval query from history symbols and implementation hints", async () => {
+    const core = new RetrievalCore(new InMemoryIndexStore(), new InMemoryQueryCache());
+    const searchSpy = vi.spyOn(core, "searchContext").mockResolvedValue({
+      trace_id: "trc-query-build",
+      results: [
+        {
+          path: "lib/error-handler.js",
+          start_line: 1,
+          end_line: 20,
+          snippet: "function setErrorHandler(fn) { return fn }",
+          score: 0.95,
+          reason: "exact symbol match"
+        },
+        {
+          path: "lib/hooks.js",
+          start_line: 1,
+          end_line: 20,
+          snippet: "function addHook(name, fn) { return hooks.add(name, fn) }",
+          score: 0.3,
+          reason: "path and token overlap"
+        },
+        {
+          path: "fastify.js",
+          start_line: 1,
+          end_line: 20,
+          snippet: "function setErrorHandler(fn) { return buildErrorHandler(fn) }",
+          score: 0.28,
+          reason: "path and token overlap"
+        }
+      ],
+      search_metadata: {
+        latency_ms: 10,
+        retrieval_mode: "hybrid",
+        index_version: "idx-2"
+      }
+    });
+
+    await core.enhancePrompt({
+      trace_id: "trc-query-build",
+      tenant_id: "tenant-dev",
+      workspace_id: "ws-dev",
+      request: {
+        prompt: "Improve request pipeline error handling and keep behavior stable.",
+        project_root_path: "/workspace/dev",
+        conversation_history: [
+          { role: "user", content: "Focus on `setErrorHandler` and onRequest ordering." },
+          { role: "assistant", content: "Likely files include lib/error-handler.js and fastify.js." }
+        ]
+      }
+    });
+
+    expect(searchSpy).toHaveBeenCalledTimes(1);
+    const retrievalQuery = searchSpy.mock.calls[0]?.[0].request.query ?? "";
+    expect(retrievalQuery).toContain("setErrorHandler");
+    expect(retrievalQuery).toContain("lib/error-handler.js");
+    expect(retrievalQuery).toMatch(/query_intent:\s+symbol-heavy/);
+    expect(retrievalQuery).toMatch(/retrieval_hints:\s+.*setErrorHandler.*lib\/error-handler\.js/);
+  });
+
+  it("routes symbol-heavy prompts to tighter pre-rerank caps", async () => {
+    const core = new RetrievalCore(new InMemoryIndexStore(), new InMemoryQueryCache(), {
+      enhancerConfig: {
+        max_candidates_pre_rerank: 4
+      }
+    });
+    const searchSpy = vi.spyOn(core, "searchContext").mockResolvedValue({
+      trace_id: "trc-symbol-intent-cap",
+      results: [
+        {
+          path: "lib/error-handler.js",
+          start_line: 1,
+          end_line: 20,
+          snippet: "function setErrorHandler(fn) { return buildErrorHandler(fn) }",
+          score: 0.92,
+          reason: "exact symbol match"
+        },
+        {
+          path: "src/server.ts",
+          start_line: 1,
+          end_line: 20,
+          snippet: "addHook onRequest preHandler onError lifecycle",
+          score: 0.9,
+          reason: "path and token overlap"
+        },
+        {
+          path: "internal/router.ts",
+          start_line: 1,
+          end_line: 20,
+          snippet: "registerRoute and dispatchRequest internals",
+          score: 0.89,
+          reason: "path and token overlap"
+        },
+        {
+          path: "app/runtime.ts",
+          start_line: 1,
+          end_line: 20,
+          snippet: "runtime hook orchestration",
+          score: 0.88,
+          reason: "semantic match"
+        }
+      ],
+      search_metadata: {
+        latency_ms: 10,
+        retrieval_mode: "hybrid",
+        index_version: "idx-symbol-intent"
+      }
+    });
+
+    const output = await core.enhancePrompt({
+      trace_id: "trc-symbol-intent-cap",
+      tenant_id: "tenant-dev",
+      workspace_id: "ws-dev",
+      request: {
+        prompt: "Refine `setErrorHandler` + `addHook` + `onRequest` + `preHandler` flow.",
+        project_root_path: "/workspace/dev",
+        conversation_history: [{ role: "user", content: "Target exact lifecycle symbols only." }]
+      }
+    });
+
+    const retrievalQuery = searchSpy.mock.calls[0]?.[0].request.query ?? "";
+    expect(retrievalQuery).toMatch(/query_intent:\s+symbol-heavy/);
+    expect(output.context_refs.length).toBeLessThanOrEqual(3);
+  });
+
+  it("routes conceptual prompts without forcing implementation-only filtering", async () => {
+    const core = new RetrievalCore(new InMemoryIndexStore(), new InMemoryQueryCache());
+    const searchSpy = vi.spyOn(core, "searchContext").mockResolvedValue({
+      trace_id: "trc-conceptual-intent",
+      results: [
+        {
+          path: "docs/architecture/retrieval.md",
+          start_line: 1,
+          end_line: 30,
+          snippet: "retrieval architecture and ranking tradeoffs",
+          score: 0.91,
+          reason: "path and token overlap"
+        },
+        {
+          path: "src/retrieval/rerank.ts",
+          start_line: 1,
+          end_line: 20,
+          snippet: "rerank implementation details",
+          score: 0.89,
+          reason: "semantic match"
+        }
+      ],
+      search_metadata: {
+        latency_ms: 9,
+        retrieval_mode: "hybrid",
+        index_version: "idx-conceptual-intent"
+      }
+    });
+
+    const output = await core.enhancePrompt({
+      trace_id: "trc-conceptual-intent",
+      tenant_id: "tenant-dev",
+      workspace_id: "ws-dev",
+      request: {
+        prompt: "Explain architecture tradeoffs for retrieval ranking and docs guidance.",
+        project_root_path: "/workspace/dev",
+        conversation_history: [{ role: "user", content: "Need high-level conceptual guidance." }]
+      }
+    });
+
+    const retrievalQuery = searchSpy.mock.calls[0]?.[0].request.query ?? "";
+    expect(retrievalQuery).toMatch(/query_intent:\s+conceptual/);
+    expect(output.context_refs.some((ref) => ref.path.startsWith("docs/"))).toBe(true);
+  });
+
+  it("applies negation-aware filtering for docs/tests/examples/archive paths", async () => {
+    const core = new RetrievalCore(new InMemoryIndexStore(), new InMemoryQueryCache());
+    vi.spyOn(core, "searchContext").mockResolvedValue({
+      trace_id: "trc-negation-filter",
+      results: [
+        {
+          path: "docs/reference/blueprints.md",
+          start_line: 1,
+          end_line: 20,
+          snippet: "blueprints runtime design notes",
+          score: 0.93,
+          reason: "path and token overlap"
+        },
+        {
+          path: "tests/blueprints.test.py",
+          start_line: 1,
+          end_line: 20,
+          snippet: "test blueprint registration order",
+          score: 0.92,
+          reason: "path and token overlap"
+        },
+        {
+          path: "examples/blueprint_demo.py",
+          start_line: 1,
+          end_line: 20,
+          snippet: "example blueprint usage",
+          score: 0.91,
+          reason: "semantic match"
+        },
+        {
+          path: "src/flask/_archive/blueprints_legacy.py",
+          start_line: 1,
+          end_line: 30,
+          snippet: "legacy blueprint implementation",
+          score: 0.9,
+          reason: "semantic match"
+        },
+        {
+          path: "src/flask/blueprints.py",
+          start_line: 1,
+          end_line: 40,
+          snippet: "def register_blueprint(app, blueprint): return app",
+          score: 0.89,
+          reason: "exact symbol match"
+        }
+      ],
+      search_metadata: {
+        latency_ms: 12,
+        retrieval_mode: "hybrid",
+        index_version: "idx-negation"
+      }
+    });
+
+    const output = await core.enhancePrompt({
+      trace_id: "trc-negation-filter",
+      tenant_id: "tenant-dev",
+      workspace_id: "ws-dev",
+      request: {
+        prompt: "Improve blueprints runtime behavior and avoid docs/tests/examples/archive context.",
+        project_root_path: "/workspace/dev",
+        conversation_history: [{ role: "user", content: "Keep implementation focused." }]
+      }
+    });
+
+    expect(output.context_refs.length).toBeGreaterThan(0);
+    expect(output.context_refs[0]?.path).toBe("src/flask/blueprints.py");
+    expect(output.context_refs.every((ref) => !/docs\/|tests\/|examples\/|_archive\//.test(ref.path))).toBe(true);
+  });
+
+  it("caps expansion hints and emits path-form hint expansions", async () => {
+    const core = new RetrievalCore(new InMemoryIndexStore(), new InMemoryQueryCache(), {
+      enhancerConfig: {
+        max_expansion_hints: 6
+      }
+    });
+    const searchSpy = vi.spyOn(core, "searchContext").mockResolvedValue({
+      trace_id: "trc-hint-cap",
+      results: [],
+      search_metadata: {
+        latency_ms: 8,
+        retrieval_mode: "hybrid",
+        index_version: "idx-cap"
+      }
+    });
+
+    await core.enhancePrompt({
+      trace_id: "trc-hint-cap",
+      tenant_id: "tenant-dev",
+      workspace_id: "ws-dev",
+      request: {
+        prompt: "Harden JSON provider and blueprints runtime behavior without docs.",
+        project_root_path: "/workspace/dev",
+        conversation_history: [{ role: "user", content: "Focus on `render_template_string` and json provider internals." }]
+      }
+    });
+
+    const retrievalQuery = searchSpy.mock.calls[0]?.[0].request.query ?? "";
+    expect(retrievalQuery).toContain("json/provider");
+    const hintLine = retrievalQuery
+      .split("\n")
+      .find((line) => line.startsWith("retrieval_hints:"));
+    const hintCount = (hintLine ?? "")
+      .replace(/^retrieval_hints:\s*/u, "")
+      .split(/\s+/)
+      .filter(Boolean).length;
+    expect(hintCount).toBeLessThanOrEqual(6);
+  });
+});

package/test/integration.test.ts

@@ -0,0 +1,324 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { newDb } from "pg-mem";
+import { InMemoryQueryCache, PostgresIndexRepository } from "@rce-mcp/data-plane";
+import { DeterministicEmbeddingProvider, RetrievalCore, RetrievalError } from "../src/index.js";
+
+describe("retrieval-core postgres integration", () => {
+  let repository: PostgresIndexRepository;
+  let close: () => Promise<void>;
+
+  beforeEach(async () => {
+    const db = newDb({ autoCreateForeignKeyIndices: true });
+    const { Pool } = db.adapters.createPg();
+    const pool = new Pool();
+
+    repository = new PostgresIndexRepository(pool, {
+      preferPgVector: false,
+      chunkEmbeddingDimensions: 24
+    });
+    await repository.migrate();
+
+    close = async () => {
+      await pool.end();
+    };
+  });
+
+  afterEach(async () => {
+    await close();
+  });
+
+  it("handles indexing lifecycle with added/modified/deleted and secret exclusion", async () => {
+    await repository.upsertWorkspace({
+      workspace_id: "ws-a",
+      tenant_id: "tenant-a",
+      name: "workspace-a",
+      project_root_path: "/workspace/a"
+    });
+
+    const core = new RetrievalCore(repository, new InMemoryQueryCache());
+
+    const reportV1 = await core.indexArtifact({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-v1",
+      files: [
+        {
+          path: "src/app.ts",
+          language: "typescript",
+          content: "export function add(a: number, b: number) { return a + b; }"
+        },
+        {
+          path: "src/utils.ts",
+          language: "typescript",
+          content: "export function clamp(v: number) { return Math.max(0, v); }"
+        },
+        {
+          path: "src/secrets.env",
+          content: "AWS_ACCESS_KEY_ID=AKIAABCDEFGHIJKLMNOP"
+        }
+      ]
+    });
+
+    expect(reportV1.counts.added).toBe(2);
+    expect(reportV1.counts.skipped).toBe(1);
+    expect(reportV1.warnings[0]?.category).toBe("secret_exclusion");
+
+    const readyAfterV1 = await repository.getLatestReadyIndex({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a"
+    });
+    expect(readyAfterV1).toBeDefined();
+    const metadataV1 = await repository.getIndexMetadata?.({
+      tenant_id: "tenant-a",
+      index_id: readyAfterV1!.index_id
+    });
+    expect(metadataV1?.chunking_strategy).toBe("sliding");
+    expect(metadataV1?.embedding_provider).toBe("deterministic");
+    expect(metadataV1?.embedding_model).toBe("pseudo-sha256");
+    expect(metadataV1?.embedding_dimensions).toBe(24);
+
+    const reportV2 = await core.indexArtifact({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-v2",
+      files: [
+        {
+          path: "src/app.ts",
+          language: "typescript",
+          content: "export function add(a: number, b: number) { return a + b + 1; }"
+        },
+        {
+          path: "src/new.ts",
+          language: "typescript",
+          content: "export function multiply(a: number, b: number) { return a * b; }"
+        },
+        {
+          path: "src/secrets.env",
+          content: "Bearer abcdefghijklmnopqrstuvwxyz12345"
+        }
+      ]
+    });
+
+    expect(reportV2.counts.modified).toBe(1);
+    expect(reportV2.counts.added).toBe(1);
+    expect(reportV2.counts.deleted).toBe(1);
+    expect(reportV2.counts.skipped).toBe(1);
+
+    const search = await core.searchContext({
+      trace_id: "trc-test",
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      request: {
+        project_root_path: "/workspace/a",
+        query: "multiply function",
+        top_k: 8
+      }
+    });
+
+    expect(search.search_metadata.index_version).toBe("idx-v2");
+    expect(search.results.some((row) => row.path === "src/new.ts")).toBe(true);
+  });
+
+  it("serves retrieval from persisted index across retrieval-core instances", async () => {
+    await repository.upsertWorkspace({
+      workspace_id: "ws-a",
+      tenant_id: "tenant-a",
+      name: "workspace-a",
+      project_root_path: "/workspace/a"
+    });
+
+    const writer = new RetrievalCore(repository, new InMemoryQueryCache());
+    await writer.indexArtifact({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-v1",
+      files: [
+        {
+          path: "src/db.ts",
+          language: "typescript",
+          content: "export const DB_NAME = 'primary';"
+        }
+      ]
+    });
+
+    const reader = new RetrievalCore(repository, new InMemoryQueryCache());
+    const result = await reader.searchContext({
+      trace_id: "trc-read",
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      request: {
+        project_root_path: "/workspace/a",
+        query: "DB_NAME",
+        top_k: 8
+      }
+    });
+
+    expect(result.results.length).toBeGreaterThan(0);
+    expect(result.results[0]?.path).toBe("src/db.ts");
+  });
+
+  it("enforces tenant isolation on retrieval", async () => {
+    await repository.upsertWorkspace({
+      workspace_id: "ws-a",
+      tenant_id: "tenant-a",
+      name: "workspace-a",
+      project_root_path: "/workspace/a"
+    });
+    await repository.upsertWorkspace({
+      workspace_id: "ws-b",
+      tenant_id: "tenant-b",
+      name: "workspace-b",
+      project_root_path: "/workspace/b"
+    });
+
+    const core = new RetrievalCore(repository, new InMemoryQueryCache());
+
+    await core.indexArtifact({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-a1",
+      files: [{ path: "src/a.ts", content: "export const TENANT_A = true;" }]
+    });
+
+    await core.indexArtifact({
+      tenant_id: "tenant-b",
+      workspace_id: "ws-b",
+      index_version: "idx-b1",
+      files: [{ path: "src/b.ts", content: "export const TENANT_B = true;" }]
+    });
+
+    const tenantBSearch = await core.searchContext({
+      trace_id: "trc-b",
+      tenant_id: "tenant-b",
+      workspace_id: "ws-b",
+      request: {
+        project_root_path: "/workspace/b",
+        query: "TENANT_B",
+        top_k: 8
+      }
+    });
+    expect(tenantBSearch.results[0]?.path).toBe("src/b.ts");
+
+    await expect(
+      core.searchContext({
+        trace_id: "trc-cross",
+        tenant_id: "tenant-b",
+        workspace_id: "ws-a",
+        request: {
+          project_root_path: "/workspace/a",
+          query: "TENANT_A",
+          top_k: 8
+        }
+      })
+    ).rejects.toMatchObject({
+      code: "NOT_FOUND"
+    } satisfies Partial<RetrievalError>);
+  });
+
+  it("persists configured embedding metadata on index finalize", async () => {
+    await repository.upsertWorkspace({
+      workspace_id: "ws-a",
+      tenant_id: "tenant-a",
+      name: "workspace-a",
+      project_root_path: "/workspace/a"
+    });
+
+    const core = new RetrievalCore(repository, new InMemoryQueryCache(), {
+      embeddingProvider: new DeterministicEmbeddingProvider({
+        model: "pseudo-sha256-custom",
+        dimensions: 32,
+        version: "v2"
+      })
+    });
+
+    await core.indexArtifact({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-meta",
+      files: [{ path: "src/a.ts", content: "export const A = 1;" }]
+    });
+
+    const latest = await repository.getLatestReadyIndex({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a"
+    });
+    expect(latest).toBeDefined();
+    const metadata = await repository.getIndexMetadata?.({
+      tenant_id: "tenant-a",
+      index_id: latest!.index_id
+    });
+    expect(metadata?.embedding_provider).toBe("deterministic");
+    expect(metadata?.embedding_model).toBe("pseudo-sha256-custom");
+    expect(metadata?.embedding_dimensions).toBe(32);
+    expect(metadata?.embedding_version).toBe("v2");
+  });
+
+  it("returns INVALID_ARGUMENT when query embedding config is incompatible with latest ready index", async () => {
+    await repository.upsertWorkspace({
+      workspace_id: "ws-a",
+      tenant_id: "tenant-a",
+      name: "workspace-a",
+      project_root_path: "/workspace/a"
+    });
+
+    const writer = new RetrievalCore(repository, new InMemoryQueryCache());
+    await writer.indexArtifact({
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      index_version: "idx-v1",
+      files: [{ path: "src/a.ts", content: "export const A = 1;" }]
+    });
+
+    const request = {
+      trace_id: "trc-mismatch",
+      tenant_id: "tenant-a",
+      workspace_id: "ws-a",
+      request: {
+        project_root_path: "/workspace/a",
+        query: "A constant",
+        top_k: 5
+      }
+    } as const;
+
+    const cases = [
+      {
+        name: "provider mismatch",
+        options: {
+          embeddingDescriptor: {
+            provider: "openai_compatible",
+            model: "Qwen/Qwen3-Embedding-4B",
+            dimensions: 24
+          }
+        }
+      },
+      {
+        name: "model mismatch",
+        options: {
+          embeddingDescriptor: {
+            provider: "deterministic",
+            model: "pseudo-sha256-v2",
+            dimensions: 24
+          }
+        }
+      },
+      {
+        name: "dimension mismatch",
+        options: {
+          embeddingDescriptor: {
+            provider: "deterministic",
+            model: "pseudo-sha256",
+            dimensions: 64
+          }
+        }
+      }
+    ] as const;
+
+    for (const testCase of cases) {
+      const reader = new RetrievalCore(repository, new InMemoryQueryCache(), testCase.options);
+      await expect(reader.searchContext(request)).rejects.toMatchObject({
+        code: "INVALID_ARGUMENT"
+      } satisfies Partial<RetrievalError>);
+      await expect(reader.searchContext(request)).rejects.toThrow(/reindex required/i);
+    }
+  });
+});