@cue-dev/retrieval-core 0.1.3

package/test/remote-sync.integration.test.ts

@@ -0,0 +1,350 @@
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+  RemoteSyncHttpResponseError,
+  buildRemoteSyncDeltaFromState,
+  collectProjectFileStats,
+  runRemoteAdaptiveSync,
+  runRemoteDeltaSync,
+  type RemoteSyncStateFile
+} from "../src/index.js";
+
+const tempDirs: string[] = [];
+
+async function createProject(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), "cue-remote-sync-test-"));
+  tempDirs.push(dir);
+  await mkdir(join(dir, "src"), { recursive: true });
+  return dir;
+}
+
+afterEach(async () => {
+  while (tempDirs.length > 0) {
+    const dir = tempDirs.pop();
+    if (!dir) {
+      continue;
+    }
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+describe("runRemoteDeltaSync", () => {
+  it("excludes .tmp directories from remote sync deltas", async () => {
+    const root = await createProject();
+    await mkdir(join(root, ".tmp", "cache"), { recursive: true });
+    await writeFile(join(root, "src", "keep.ts"), "export const KEEP = true;\n");
+    await writeFile(join(root, ".tmp", "cache", "noise.ts"), "export const NOISE = true;\n");
+
+    const result = await buildRemoteSyncDeltaFromState({
+      project_root_path: root,
+      force_full_upsert: true
+    });
+
+    expect(result.delta.upsert_files.map((file) => file.path)).toEqual(["src/keep.ts"]);
+    expect(Object.keys(result.next_files)).toEqual(["src/keep.ts"]);
+  });
+
+  it("excludes common dependency/cache/vcs directories and binary/media file suffixes", async () => {
+    const root = await createProject();
+    await mkdir(join(root, "vendor", "pkg"), { recursive: true });
+    await mkdir(join(root, ".venv", "bin"), { recursive: true });
+    await mkdir(join(root, "venv", "bin"), { recursive: true });
+    await mkdir(join(root, "__pycache__"), { recursive: true });
+    await mkdir(join(root, ".pytest_cache"), { recursive: true });
+    await mkdir(join(root, ".svn"), { recursive: true });
+    await mkdir(join(root, ".hg"), { recursive: true });
+    await writeFile(join(root, "src", "keep.ts"), "export const KEEP = true;\n");
+    await writeFile(join(root, "vendor", "pkg", "noise.ts"), "export const NOISE = true;\n");
+    await writeFile(join(root, ".venv", "bin", "activate"), "source venv/bin/activate\n");
+    await writeFile(join(root, "venv", "bin", "activate"), "source venv/bin/activate\n");
+    await writeFile(join(root, "__pycache__", "module.pyc"), "noise\n");
+    await writeFile(join(root, ".pytest_cache", "state"), "noise\n");
+    await writeFile(join(root, ".svn", "entries"), "noise\n");
+    await writeFile(join(root, ".hg", "store"), "noise\n");
+    await writeFile(join(root, "native.dll"), "noise\n");
+    await writeFile(join(root, "native.so"), "noise\n");
+    await writeFile(join(root, "binary.exe"), "noise\n");
+    await writeFile(join(root, "cache.pyc"), "noise\n");
+    await writeFile(join(root, "image.png"), "noise\n");
+    await writeFile(join(root, "image.jpg"), "noise\n");
+    await writeFile(join(root, "image.jpeg"), "noise\n");
+    await writeFile(join(root, "image.gif"), "noise\n");
+    await writeFile(join(root, "image.webp"), "noise\n");
+    await writeFile(join(root, "image.svg"), "noise\n");
+    await writeFile(join(root, "video.mp4"), "noise\n");
+    await writeFile(join(root, "video.mov"), "noise\n");
+    await writeFile(join(root, "video.avi"), "noise\n");
+    await writeFile(join(root, "video.mkv"), "noise\n");
+    await writeFile(join(root, "blob.bin"), "noise\n");
+    await writeFile(join(root, "native.dylib"), "noise\n");
+    await writeFile(join(root, "spec.pdf"), "noise\n");
+
+    const result = await buildRemoteSyncDeltaFromState({
+      project_root_path: root,
+      force_full_upsert: true
+    });
+
+    expect(result.delta.upsert_files.map((file) => file.path)).toEqual(["src/keep.ts"]);
+    expect(Object.keys(result.next_files)).toEqual(["src/keep.ts"]);
+  });
+
+  it("applies .contextignore and .cueignore patterns to remote sync deltas", async () => {
+    const root = await createProject();
+    await mkdir(join(root, "src", "generated"), { recursive: true });
+    await mkdir(join(root, "generated"), { recursive: true });
+    await mkdir(join(root, "build"), { recursive: true });
+
+    await writeFile(
+      join(root, ".contextignore"),
+      "# comments and blank lines are ignored\n\n**/generated/**\n"
+    );
+    await writeFile(join(root, ".cueignore"), "build/\n");
+    await writeFile(join(root, "src", "keep.ts"), "export const KEEP = true;\n");
+    await writeFile(join(root, "src", "generated", "types.ts"), "export const GENERATED = true;\n");
+    await writeFile(join(root, "generated", "root.ts"), "export const ROOT_GENERATED = true;\n");
+    await writeFile(join(root, "build", "bundle.ts"), "export const BUNDLE = true;\n");
+
+    const result = await buildRemoteSyncDeltaFromState({
+      project_root_path: root,
+      force_full_upsert: true
+    });
+
+    expect(result.delta.upsert_files.map((file) => file.path)).toEqual(["src/keep.ts"]);
+    expect(Object.keys(result.next_files)).toEqual(["src/keep.ts"]);
+  });
+
+  it("maps .mjs/.cjs to javascript and .mts/.cts to typescript language metadata during scans", async () => {
+    const root = await createProject();
+    await writeFile(join(root, "src", "runtime.mjs"), "export const runtime = true;\n");
+    await writeFile(join(root, "src", "loader.cjs"), "module.exports = { loader: true };\n");
+    await writeFile(join(root, "src", "types.mts"), "export const typed = true as const;\n");
+    await writeFile(join(root, "src", "legacy.cts"), "export = { typed: true };\n");
+
+    const stats = await collectProjectFileStats(root);
+    expect(stats.get("src/runtime.mjs")?.language).toBe("javascript");
+    expect(stats.get("src/loader.cjs")?.language).toBe("javascript");
+    expect(stats.get("src/types.mts")?.language).toBe("typescript");
+    expect(stats.get("src/legacy.cts")?.language).toBe("typescript");
+  });
+
+  it("applies add/modify/delete changes incrementally", async () => {
+    const root = await createProject();
+    await writeFile(join(root, "src", "a.ts"), "export const A = 1;\n");
+    await writeFile(join(root, "src", "b.ts"), "export const B = 1;\n");
+
+    const calls: Array<{ upserts: string[]; deleted: string[]; base?: string }> = [];
+    const first = await runRemoteDeltaSync({
+      project_root_path: root,
+      workspace_id: "ws-1",
+      max_body_bytes: 1024 * 1024,
+      push_delta: async (request) => {
+        calls.push({
+          upserts: request.upsert_files.map((file) => file.path).sort(),
+          deleted: [...request.deleted_paths].sort(),
+          base: request.base_index_version
+        });
+        return {
+          workspace_id: "ws-1",
+          index_version: "idx-v1"
+        };
+      }
+    });
+
+    expect(first.changed).toBe(true);
+    expect(first.applied_delta).toEqual({ upsert_files: 2, deleted_paths: 0 });
+    expect(calls).toHaveLength(1);
+    expect(calls[0]?.upserts).toEqual(["src/a.ts", "src/b.ts"]);
+    expect(calls[0]?.deleted).toEqual([]);
+
+    await new Promise((resolveSleep) => setTimeout(resolveSleep, 5));
+    await rm(join(root, "src", "b.ts"));
+    await writeFile(join(root, "src", "a.ts"), "export const A = 2;\n");
+    await writeFile(join(root, "src", "c.ts"), "export const C = 1;\n");
+
+    const secondCalls: Array<{ upserts: string[]; deleted: string[]; base?: string }> = [];
+    const second = await runRemoteDeltaSync({
+      project_root_path: root,
+      workspace_id: "ws-1",
+      previous_state: first.state,
+      max_body_bytes: 1024 * 1024,
+      push_delta: async (request) => {
+        secondCalls.push({
+          upserts: request.upsert_files.map((file) => file.path).sort(),
+          deleted: [...request.deleted_paths].sort(),
+          base: request.base_index_version
+        });
+        return {
+          workspace_id: "ws-1",
+          index_version: "idx-v2"
+        };
+      }
+    });
+
+    expect(second.changed).toBe(true);
+    expect(second.applied_delta).toEqual({ upsert_files: 2, deleted_paths: 1 });
+    expect(secondCalls).toHaveLength(1);
+    expect(secondCalls[0]?.upserts).toEqual(["src/a.ts", "src/c.ts"]);
+    expect(secondCalls[0]?.deleted).toEqual(["src/b.ts"]);
+    expect(secondCalls[0]?.base).toBe("idx-v1");
+    expect(Object.keys(second.state.files).sort()).toEqual(["src/a.ts", "src/c.ts"]);
+  });
+
+  it("returns unchanged state without remote call when no files changed", async () => {
+    const root = await createProject();
+    await writeFile(join(root, "src", "stable.ts"), "export const STABLE = true;\n");
+
+    const initial = await buildRemoteSyncDeltaFromState({
+      project_root_path: root,
+      force_full_upsert: true
+    });
+
+    const previousState: RemoteSyncStateFile = {
+      mode: "remote_delta_v1",
+      workspace_id: "ws-1",
+      last_index_version: "idx-v1",
+      files: initial.next_files,
+      updated_at: new Date().toISOString()
+    };
+
+    let called = false;
+    const result = await runRemoteDeltaSync({
+      project_root_path: root,
+      previous_state: previousState,
+      workspace_id: "ws-1",
+      max_body_bytes: 1024 * 1024,
+      push_delta: async () => {
+        called = true;
+        return {
+          workspace_id: "ws-1",
+          index_version: "idx-v2"
+        };
+      }
+    });
+
+    expect(called).toBe(false);
+    expect(result.changed).toBe(false);
+    expect(result.index_version).toBe("idx-v1");
+  });
+
+  it("retries once with full-upsert when base index is stale", async () => {
+    const root = await createProject();
+    await writeFile(join(root, "src", "retry.ts"), "export const RETRY = 1;\n");
+
+    const previousBuild = await buildRemoteSyncDeltaFromState({
+      project_root_path: root,
+      force_full_upsert: true
+    });
+
+    const previousState: RemoteSyncStateFile = {
+      mode: "remote_delta_v1",
+      workspace_id: "ws-1",
+      last_index_version: "idx-stale",
+      files: previousBuild.next_files,
+      updated_at: new Date().toISOString()
+    };
+
+    await new Promise((resolveSleep) => setTimeout(resolveSleep, 5));
+    await writeFile(join(root, "src", "retry.ts"), "export const RETRY = 2;\n");
+
+    const bases: Array<string | undefined> = [];
+    let callCount = 0;
+    const result = await runRemoteDeltaSync({
+      project_root_path: root,
+      previous_state: previousState,
+      workspace_id: "ws-1",
+      max_body_bytes: 1024 * 1024,
+      retries: 1,
+      push_delta: async (request) => {
+        callCount += 1;
+        bases.push(request.base_index_version);
+        if (callCount === 1) {
+          throw new RemoteSyncHttpResponseError("INVALID_ARGUMENT: base index version is stale", 400);
+        }
+        return {
+          workspace_id: "ws-1",
+          index_version: "idx-v2"
+        };
+      }
+    });
+
+    expect(callCount).toBe(2);
+    expect(bases[0]).toBe("idx-stale");
+    expect(bases[1]).toBeUndefined();
+    expect(result.index_version).toBe("idx-v2");
+  });
+
+  it("uses blob_commit_v2 protocol when advertised and falls back on unsupported endpoint", async () => {
+    const root = await createProject();
+    await writeFile(join(root, "src", "adaptive.ts"), "export const ADAPTIVE = 1;\n");
+
+    const deltaCalls: Array<{ upserts: number; deletes: number }> = [];
+    const v2Result = await runRemoteAdaptiveSync({
+      project_root_path: root,
+      workspace_id: "ws-1",
+      capabilities: {
+        max_body_bytes: 1024 * 1024,
+        sync_protocols: ["delta_v1", "blob_commit_v2"],
+        max_blob_bytes: 128 * 1024,
+        max_blob_batch_bytes: 512 * 1024,
+        max_commit_body_bytes: 512 * 1024,
+        upload_concurrency_hint: 2
+      },
+      push_delta: async (request) => {
+        deltaCalls.push({
+          upserts: request.upsert_files.length,
+          deletes: request.deleted_paths.length
+        });
+        return {
+          workspace_id: "ws-1",
+          index_version: "idx-delta"
+        };
+      },
+      upload_blobs: async (request) => ({
+        accepted_hashes: request.blobs.map((blob) => blob.hash),
+        already_present_hashes: [],
+        rejected: []
+      }),
+      commit_v2: async () => ({
+        workspace_id: "ws-1",
+        index_version: "idx-v2"
+      })
+    });
+    expect(v2Result.protocol).toBe("blob_commit_v2");
+    expect(v2Result.index_version).toBe("idx-v2");
+    expect(deltaCalls).toHaveLength(0);
+
+    await new Promise((resolveSleep) => setTimeout(resolveSleep, 5));
+    await writeFile(join(root, "src", "adaptive.ts"), "export const ADAPTIVE = 2;\n");
+
+    const fallbackResult = await runRemoteAdaptiveSync({
+      project_root_path: root,
+      workspace_id: "ws-1",
+      previous_state: v2Result.state,
+      capabilities: {
+        max_body_bytes: 1024 * 1024,
+        sync_protocols: ["delta_v1", "blob_commit_v2"]
+      },
+      push_delta: async (request) => {
+        deltaCalls.push({
+          upserts: request.upsert_files.length,
+          deletes: request.deleted_paths.length
+        });
+        return {
+          workspace_id: "ws-1",
+          index_version: "idx-delta-fallback"
+        };
+      },
+      upload_blobs: async () => {
+        throw new RemoteSyncHttpResponseError("NOT_FOUND: /v1/sync/blobs/upload", 404);
+      },
+      commit_v2: async () => ({
+        workspace_id: "ws-1",
+        index_version: "idx-ignored"
+      })
+    });
+    expect(fallbackResult.protocol).toBe("delta_v1");
+    expect(deltaCalls.length).toBeGreaterThan(0);
+  });
+});

package/test/smart-cutoff.config.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, it } from "vitest";
+import {
+  __applySmartCutoffCandidatesForTests,
+  mergeRetrievalScoringConfig,
+  resolveRetrievalScoringProfile
+} from "../src/index.js";
+
+function sampleCandidates(scores: number[]) {
+  return scores.map((score, index) => ({
+    path: `src/candidate-${index + 1}.ts`,
+    start_line: index * 10 + 1,
+    end_line: index * 10 + 5,
+    snippet: `export const candidate${index + 1} = ${score};`,
+    score,
+    reason: "semantic match" as const
+  }));
+}
+
+describe("retrieval smart cutoff", () => {
+  it("applies delta and ratio guards after the safe-harbor minimum", () => {
+    const baseline = resolveRetrievalScoringProfile("baseline");
+    const config = mergeRetrievalScoringConfig(baseline.config, {
+      rerank: {
+        smart_cutoff_enabled: true,
+        smart_cutoff_min_k: 2,
+        smart_cutoff_max_k: 8,
+        smart_cutoff_min_score: 0.25,
+        smart_cutoff_top_ratio: 0.5,
+        smart_cutoff_delta_abs: 0.2
+      }
+    }).rerank;
+
+    const output = __applySmartCutoffCandidatesForTests({
+      config,
+      candidates: sampleCandidates([0.92, 0.88, 0.56, 0.55, 0.41])
+    });
+
+    expect(output).toHaveLength(2);
+    expect(output[0]?.score).toBe(0.92);
+    expect(output[1]?.score).toBe(0.88);
+  });
+
+  it("keeps at least min_k results even when score floors are not met", () => {
+    const baseline = resolveRetrievalScoringProfile("baseline");
+    const config = mergeRetrievalScoringConfig(baseline.config, {
+      rerank: {
+        smart_cutoff_enabled: true,
+        smart_cutoff_min_k: 2,
+        smart_cutoff_max_k: 8,
+        smart_cutoff_min_score: 0.8,
+        smart_cutoff_top_ratio: 0.95,
+        smart_cutoff_delta_abs: 0.01
+      }
+    }).rerank;
+
+    const output = __applySmartCutoffCandidatesForTests({
+      config,
+      candidates: sampleCandidates([0.91, 0.31, 0.3, 0.29])
+    });
+
+    expect(output).toHaveLength(2);
+    expect(output[0]?.score).toBe(0.91);
+    expect(output[1]?.score).toBe(0.31);
+  });
+
+  it("rejects invalid smart cutoff ranges in scoring config", () => {
+    const baseline = resolveRetrievalScoringProfile("baseline");
+
+    expect(() =>
+      mergeRetrievalScoringConfig(baseline.config, {
+        rerank: {
+          smart_cutoff_min_k: 3,
+          smart_cutoff_max_k: 2
+        }
+      })
+    ).toThrow(/smart_cutoff_max_k must be >= smart_cutoff_min_k/);
+
+    expect(() =>
+      mergeRetrievalScoringConfig(baseline.config, {
+        rerank: {
+          smart_cutoff_top_ratio: 0
+        }
+      })
+    ).toThrow(/smart_cutoff_top_ratio must be in \(0, 1\]/);
+  });
+});

package/test/snippet-integrity.config.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+import {
+  DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG,
+  mergeRetrievalSnippetIntegrityConfig
+} from "../src/index.js";
+
+describe("retrieval-core snippet integrity config validation", () => {
+  it("uses dark-launch defaults", () => {
+    expect(DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG).toMatchObject({
+      enabled: false,
+      target_languages: ["typescript", "tsx", "javascript", "jsx"],
+      max_contiguous_gap_lines: 6,
+      marker_template_version: "v1",
+      repair_enabled: false,
+      repair_max_envelope_lines: 260,
+      repair_max_snippet_chars: 3600
+    });
+  });
+
+  it("accepts valid overrides and normalizes aliases", () => {
+    const merged = mergeRetrievalSnippetIntegrityConfig(DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG, {
+      enabled: true,
+      target_languages: ["ts", "js", "tsx"],
+      max_contiguous_gap_lines: 10,
+      repair_enabled: true,
+      repair_max_envelope_lines: 400,
+      repair_max_snippet_chars: 4200
+    });
+    expect(merged.enabled).toBe(true);
+    expect(merged.target_languages).toEqual(["typescript", "javascript", "tsx"]);
+    expect(merged.max_contiguous_gap_lines).toBe(10);
+    expect(merged.marker_template_version).toBe("v1");
+    expect(merged.repair_enabled).toBe(true);
+    expect(merged.repair_max_envelope_lines).toBe(400);
+    expect(merged.repair_max_snippet_chars).toBe(4200);
+  });
+
+  it("rejects unsupported target languages", () => {
+    expect(() =>
+      mergeRetrievalSnippetIntegrityConfig(DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG, {
+        target_languages: ["python"]
+      })
+    ).toThrow(/unsupported target language/);
+  });
+
+  it("rejects invalid repair caps", () => {
+    expect(() =>
+      mergeRetrievalSnippetIntegrityConfig(DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG, {
+        repair_max_envelope_lines: 0
+      })
+    ).toThrow(/repair_max_envelope_lines must be a positive integer/);
+
+    expect(() =>
+      mergeRetrievalSnippetIntegrityConfig(DEFAULT_RETRIEVAL_SNIPPET_INTEGRITY_CONFIG, {
+        repair_max_snippet_chars: -1
+      })
+    ).toThrow(/repair_max_snippet_chars must be a positive integer/);
+  });
+});

package/tsconfig.build.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "composite": true,
+    "declaration": true,
+    "declarationMap": false,
+    "rootDir": "src",
+    "outDir": "dist",
+    "tsBuildInfoFile": "dist/.tsbuildinfo"
+  },
+  "include": ["src/**/*.ts"],
+  "references": [
+    { "path": "../contracts/tsconfig.build.json" },
+    { "path": "../data-plane/tsconfig.build.json" },
+    { "path": "../observability/tsconfig.build.json" }
+  ]
+}

package/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "include": ["src/**/*.ts"]
+}