@cue-dev/retrieval-core 0.1.3

package/src/remote-sync.ts

@@ -0,0 +1,1119 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+import { loadIndexingIgnoreMatcher, normalizeRepoRelativePath } from "./indexing-ignore.js";
+
+export const REMOTE_SYNC_STATE_MODE = "remote_delta_v1" as const;
+export const DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES = 1_000_000;
+
+const DEFAULT_EXCLUDED_DIRS = new Set([
+  ".tmp",
+  ".cache",
+  ".git",
+  ".hg",
+  ".next",
+  ".nuxt",
+  ".pytest_cache",
+  ".cue",
+  ".cue-tool",
+  ".svn",
+  ".svelte-kit",
+  ".turbo",
+  ".venv",
+  "__pycache__",
+  "build",
+  "coverage",
+  "dist",
+  "node_modules",
+  "out",
+  "target",
+  "vendor",
+  "venv"
+]);
+
+const DEFAULT_EXCLUDED_FILES = new Set([
+  "bun.lock",
+  "bun.lockb",
+  "Cargo.lock",
+  "composer.lock",
+  "Gemfile.lock",
+  "package-lock.json",
+  "Pipfile.lock",
+  "pnpm-lock.yaml",
+  "yarn.lock"
+]);
+
+const DEFAULT_EXCLUDED_FILE_SUFFIXES = new Set([
+  ".min.js",
+  ".min.css",
+  ".map",
+  ".exe",
+  ".dll",
+  ".so",
+  ".bin",
+  ".dylib",
+  ".pyc",
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".webp",
+  ".svg",
+  ".mp4",
+  ".mov",
+  ".avi",
+  ".mkv",
+  ".pdf"
+]);
+
+export interface RemoteSyncUploadCandidate {
+  path: string;
+  content: string;
+  language?: string;
+}
+
+export interface RemoteSyncProjectFileStat {
+  path: string;
+  full_path: string;
+  size: number;
+  mtime_ms: number;
+  language?: string;
+}
+
+export interface RemoteSyncStateEntry {
+  content_hash: string;
+  size: number;
+  mtime_ms: number;
+  language?: string;
+}
+
+export interface RemoteSyncStateFile {
+  mode: typeof REMOTE_SYNC_STATE_MODE;
+  workspace_id?: string;
+  last_index_version?: string;
+  files: Record<string, RemoteSyncStateEntry>;
+  updated_at: string;
+}
+
+export interface RemoteSyncDeltaPayload {
+  upsert_files: RemoteSyncUploadCandidate[];
+  deleted_paths: string[];
+}
+
+export interface RemoteSyncDeltaBatch {
+  upsert_files: RemoteSyncUploadCandidate[];
+  deleted_paths: string[];
+  approx_bytes: number;
+}
+
+export interface BuildRemoteSyncDeltaResult {
+  delta: RemoteSyncDeltaPayload;
+  upsert_state_entries: Record<string, RemoteSyncStateEntry>;
+  next_files: Record<string, RemoteSyncStateEntry>;
+}
+
+export interface RemoteSyncScanOptions {
+  max_file_size_bytes?: number;
+  excluded_dirs?: Set<string>;
+  excluded_files?: Set<string>;
+  excluded_file_suffixes?: Set<string>;
+}
+
+export class RemoteSyncHttpResponseError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+    readonly payload?: unknown
+  ) {
+    super(message);
+  }
+}
+
+export interface RunRemoteDeltaSyncInput {
+  project_root_path: string;
+  scan_root_path?: string;
+  workspace_id?: string;
+  previous_state?: RemoteSyncStateFile;
+  force_full_upsert?: boolean;
+  max_body_bytes: number;
+  retries?: number;
+  initial_delay_ms?: number;
+  stale_base_error?: (error: unknown) => boolean;
+  persist_state?: (state: RemoteSyncStateFile) => Promise<void>;
+  on_batch_processed?: (event: {
+    batch_index: number;
+    batch_count: number;
+    approx_bytes: number;
+    upsert_files: number;
+    deleted_paths: number;
+    latency_ms: number;
+  }) => void | Promise<void>;
+  push_delta: (request: {
+    workspace_id?: string;
+    project_root_path: string;
+    base_index_version?: string;
+    upsert_files: RemoteSyncUploadCandidate[];
+    deleted_paths: string[];
+  }) => Promise<{
+    workspace_id?: string;
+    index_version?: string;
+  }>;
+}
+
+export interface RunRemoteDeltaSyncResult {
+  state: RemoteSyncStateFile;
+  changed: boolean;
+  workspace_id?: string;
+  index_version?: string;
+  applied_delta: {
+    upsert_files: number;
+    deleted_paths: number;
+  };
+  stats: {
+    batches_total: number;
+    bytes_total: number;
+    latency_ms: number;
+  };
+}
+
+export interface RemoteSyncCapabilities {
+  max_body_bytes: number;
+  sync_protocols?: string[];
+  max_blob_bytes?: number;
+  max_blob_batch_bytes?: number;
+  max_commit_body_bytes?: number;
+  upload_concurrency_hint?: number;
+}
+
+export interface RunRemoteAdaptiveSyncInput {
+  project_root_path: string;
+  scan_root_path?: string;
+  workspace_id?: string;
+  previous_state?: RemoteSyncStateFile;
+  force_full_upsert?: boolean;
+  capabilities: RemoteSyncCapabilities;
+  retries?: number;
+  initial_delay_ms?: number;
+  stale_base_error?: (error: unknown) => boolean;
+  persist_state?: (state: RemoteSyncStateFile) => Promise<void>;
+  push_delta: RunRemoteDeltaSyncInput["push_delta"];
+  upload_blobs: (request: {
+    workspace_id?: string;
+    project_root_path: string;
+    blobs: Array<{
+      hash: string;
+      content: string;
+      size_bytes: number;
+    }>;
+  }) => Promise<{
+    accepted_hashes: string[];
+    already_present_hashes: string[];
+    rejected: Array<{ hash: string; reason: string }>;
+  }>;
+  commit_v2: (request: {
+    workspace_id?: string;
+    project_root_path: string;
+    base_index_version?: string;
+    upsert_files: Array<{
+      path: string;
+      blob_hash: string;
+      language?: string;
+      generated?: boolean;
+      binary?: boolean;
+      updated_at?: string;
+    }>;
+    deleted_paths: string[];
+  }) => Promise<{
+    workspace_id?: string;
+    index_version?: string;
+  }>;
+  on_upload_strategy_change?: (event: {
+    previous_concurrency: number;
+    next_concurrency: number;
+    reason: "success" | "error";
+  }) => void | Promise<void>;
+}
+
+export interface RunRemoteAdaptiveSyncResult extends RunRemoteDeltaSyncResult {
+  protocol: "delta_v1" | "blob_commit_v2";
+}
+
+function nowIso(): string {
+  return new Date().toISOString();
+}
+
+function extensionToLanguage(path: string): string | undefined {
+  if (path.endsWith(".ts") || path.endsWith(".tsx") || path.endsWith(".mts") || path.endsWith(".cts")) {
+    return "typescript";
+  }
+  if (path.endsWith(".js") || path.endsWith(".jsx") || path.endsWith(".mjs") || path.endsWith(".cjs")) {
+    return "javascript";
+  }
+  if (path.endsWith(".py")) return "python";
+  if (path.endsWith(".go")) return "go";
+  if (path.endsWith(".rs")) return "rust";
+  if (path.endsWith(".java")) return "java";
+  if (path.endsWith(".json")) return "json";
+  if (path.endsWith(".md")) return "markdown";
+  if (path.endsWith(".yml") || path.endsWith(".yaml")) return "yaml";
+  return undefined;
+}
+
+function looksBinary(content: string): boolean {
+  return content.includes("\0");
+}
+
+function sha256Text(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function shouldExcludeFile(path: string, excludedFiles: Set<string>, excludedSuffixes: Set<string>): boolean {
+  const lower = path.toLowerCase();
+  if (excludedFiles.has(path) || excludedFiles.has(lower)) {
+    return true;
+  }
+  for (const suffix of excludedSuffixes) {
+    if (lower.endsWith(suffix)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function resolveScanOptions(options?: RemoteSyncScanOptions): Required<RemoteSyncScanOptions> {
+  return {
+    max_file_size_bytes: options?.max_file_size_bytes ?? DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES,
+    excluded_dirs: options?.excluded_dirs ?? DEFAULT_EXCLUDED_DIRS,
+    excluded_files: options?.excluded_files ?? DEFAULT_EXCLUDED_FILES,
+    excluded_file_suffixes: options?.excluded_file_suffixes ?? DEFAULT_EXCLUDED_FILE_SUFFIXES
+  };
+}
+
+export async function collectProjectFileStats(
+  project_root_path: string,
+  options?: RemoteSyncScanOptions
+): Promise<Map<string, RemoteSyncProjectFileStat>> {
+  const root = resolve(project_root_path);
+  const resolvedOptions = resolveScanOptions(options);
+  const ignoreMatcher = await loadIndexingIgnoreMatcher(root);
+  const output = new Map<string, RemoteSyncProjectFileStat>();
+
+  async function walk(dir: string): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      const repoPath = normalizeRepoRelativePath(relative(root, fullPath));
+
+      if (entry.isDirectory()) {
+        if (resolvedOptions.excluded_dirs.has(entry.name) || ignoreMatcher.shouldIgnorePath(repoPath, "dir")) {
+          continue;
+        }
+        await walk(fullPath);
+        continue;
+      }
+
+      if (!entry.isFile()) {
+        continue;
+      }
+
+      if (shouldExcludeFile(entry.name, resolvedOptions.excluded_files, resolvedOptions.excluded_file_suffixes)) {
+        continue;
+      }
+      if (ignoreMatcher.shouldIgnorePath(repoPath, "file")) {
+        continue;
+      }
+      const fileStat = await stat(fullPath);
+      if (fileStat.size > resolvedOptions.max_file_size_bytes) {
+        continue;
+      }
+
+      output.set(repoPath, {
+        path: repoPath,
+        full_path: fullPath,
+        size: fileStat.size,
+        mtime_ms: Math.trunc(fileStat.mtimeMs),
+        language: extensionToLanguage(repoPath)
+      });
+    }
+  }
+
+  await walk(root);
+  return output;
+}
+
+export async function collectUploadCandidates(
+  project_root_path: string,
+  options?: RemoteSyncScanOptions
+): Promise<RemoteSyncUploadCandidate[]> {
+  const stats = await collectProjectFileStats(project_root_path, options);
+  const output: RemoteSyncUploadCandidate[] = [];
+
+  for (const repoPath of [...stats.keys()].sort((a, b) => a.localeCompare(b))) {
+    const fileStat = stats.get(repoPath)!;
+    let content: string;
+    try {
+      content = await readFile(fileStat.full_path, "utf8");
+    } catch {
+      continue;
+    }
+    if (looksBinary(content)) {
+      continue;
+    }
+
+    output.push({
+      path: repoPath,
+      content,
+      ...(fileStat.language ? { language: fileStat.language } : {})
+    });
+  }
+
+  return output;
+}
+
+export async function buildRemoteSyncDeltaFromState(input: {
+  project_root_path: string;
+  previous_state?: RemoteSyncStateFile;
+  force_full_upsert: boolean;
+  options?: RemoteSyncScanOptions;
+}): Promise<BuildRemoteSyncDeltaResult> {
+  const projectFiles = await collectProjectFileStats(input.project_root_path, input.options);
+  const previousFiles = input.previous_state?.files ?? {};
+  const upsertFiles: RemoteSyncUploadCandidate[] = [];
+  const deletedPaths: string[] = [];
+  const upsertStateEntries: Record<string, RemoteSyncStateEntry> = {};
+  const nextFiles: Record<string, RemoteSyncStateEntry> = {};
+  const sortedPaths = [...projectFiles.keys()].sort((a, b) => a.localeCompare(b));
+
+  for (const repoPath of sortedPaths) {
+    const projectFile = projectFiles.get(repoPath)!;
+    const previous = previousFiles[repoPath];
+    const statChanged =
+      !previous ||
+      previous.size !== projectFile.size ||
+      previous.mtime_ms !== projectFile.mtime_ms ||
+      input.force_full_upsert;
+
+    if (!statChanged) {
+      nextFiles[repoPath] = previous;
+      continue;
+    }
+
+    let content: string;
+    try {
+      content = await readFile(projectFile.full_path, "utf8");
+    } catch {
+      if (previous) {
+        nextFiles[repoPath] = previous;
+      }
+      continue;
+    }
+
+    if (looksBinary(content)) {
+      if (previous) {
+        nextFiles[repoPath] = previous;
+      }
+      continue;
+    }
+
+    const contentHash = sha256Text(content);
+    const nextEntry: RemoteSyncStateEntry = {
+      content_hash: contentHash,
+      size: projectFile.size,
+      mtime_ms: projectFile.mtime_ms,
+      ...(projectFile.language ? { language: projectFile.language } : {})
+    };
+
+    nextFiles[repoPath] = nextEntry;
+    if (!input.force_full_upsert && previous?.content_hash === contentHash) {
+      continue;
+    }
+
+    upsertFiles.push({
+      path: repoPath,
+      content,
+      ...(projectFile.language ? { language: projectFile.language } : {})
+    });
+    upsertStateEntries[repoPath] = nextEntry;
+  }
+
+  for (const previousPath of Object.keys(previousFiles)) {
+    if (!projectFiles.has(previousPath)) {
+      deletedPaths.push(previousPath);
+    }
+  }
+
+  return {
+    delta: {
+      upsert_files: upsertFiles,
+      deleted_paths: deletedPaths
+    },
+    upsert_state_entries: upsertStateEntries,
+    next_files: nextFiles
+  };
+}
+
+export function estimateRemoteSyncDeltaRequestSize(input: {
+  project_root_path: string;
+  workspace_id?: string;
+  base_index_version?: string;
+  upsert_files: RemoteSyncUploadCandidate[];
+  deleted_paths: string[];
+}): number {
+  const payload = {
+    project_root_path: input.project_root_path,
+    ...(input.workspace_id ? { workspace_id: input.workspace_id } : {}),
+    ...(input.base_index_version ? { base_index_version: input.base_index_version } : {}),
+    upsert_files: input.upsert_files.map((file) => ({
+      path: file.path,
+      content: file.content,
+      ...(file.language ? { language: file.language } : {})
+    })),
+    deleted_paths: input.deleted_paths
+  };
+  return Buffer.byteLength(JSON.stringify(payload), "utf8");
+}
+
+export function splitRemoteSyncDeltaIntoBatches(input: {
+  project_root_path: string;
+  workspace_id?: string;
+  base_index_version?: string;
+  delta: RemoteSyncDeltaPayload;
+  max_body_bytes: number;
+}): RemoteSyncDeltaBatch[] {
+  const budget = Math.max(256 * 1024, Math.floor(input.max_body_bytes * 0.8));
+  const upserts = [...input.delta.upsert_files].sort((a, b) => a.path.localeCompare(b.path));
+  const deleted = [...new Set(input.delta.deleted_paths)].sort((a, b) => a.localeCompare(b));
+  const batches: RemoteSyncDeltaBatch[] = [];
+  let currentUpserts: RemoteSyncUploadCandidate[] = [];
+  let currentDeleted: string[] = [];
+
+  const flush = () => {
+    if (currentUpserts.length === 0 && currentDeleted.length === 0) {
+      return;
+    }
+    batches.push({
+      upsert_files: currentUpserts,
+      deleted_paths: currentDeleted,
+      approx_bytes: estimateRemoteSyncDeltaRequestSize({
+        project_root_path: input.project_root_path,
+        workspace_id: input.workspace_id,
+        base_index_version: input.base_index_version,
+        upsert_files: currentUpserts,
+        deleted_paths: currentDeleted
+      })
+    });
+    currentUpserts = [];
+    currentDeleted = [];
+  };
+
+  const canFit = (nextUpserts: RemoteSyncUploadCandidate[], nextDeleted: string[]) =>
+    estimateRemoteSyncDeltaRequestSize({
+      project_root_path: input.project_root_path,
+      workspace_id: input.workspace_id,
+      base_index_version: input.base_index_version,
+      upsert_files: nextUpserts,
+      deleted_paths: nextDeleted
+    }) <= budget;
+
+  for (const file of upserts) {
+    const candidateUpserts = [...currentUpserts, file];
+    if (canFit(candidateUpserts, currentDeleted)) {
+      currentUpserts = candidateUpserts;
+      continue;
+    }
+    flush();
+    if (!canFit([file], [])) {
+      throw new Error(`delta upsert payload too large for path ${file.path}`);
+    }
+    currentUpserts = [file];
+  }
+
+  for (const path of deleted) {
+    const candidateDeleted = [...currentDeleted, path];
+    if (canFit(currentUpserts, candidateDeleted)) {
+      currentDeleted = candidateDeleted;
+      continue;
+    }
+    flush();
+    if (!canFit([], [path])) {
+      throw new Error(`delta delete payload too large for path ${path}`);
+    }
+    currentDeleted = [path];
+  }
+
+  flush();
+  return batches;
+}
+
+export async function readRemoteSyncState(path: string): Promise<RemoteSyncStateFile | undefined> {
+  if (!existsSync(path)) {
+    return undefined;
+  }
+
+  try {
+    const parsed = JSON.parse(await readFile(path, "utf8")) as Partial<RemoteSyncStateFile>;
+    if (
+      !parsed ||
+      parsed.mode !== REMOTE_SYNC_STATE_MODE ||
+      typeof parsed.files !== "object" ||
+      parsed.files === null ||
+      Array.isArray(parsed.files)
+    ) {
+      return undefined;
+    }
+
+    return {
+      mode: REMOTE_SYNC_STATE_MODE,
+      workspace_id: parsed.workspace_id,
+      last_index_version: parsed.last_index_version,
+      files: parsed.files as Record<string, RemoteSyncStateEntry>,
+      updated_at: typeof parsed.updated_at === "string" ? parsed.updated_at : nowIso()
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+export async function writeRemoteSyncState(path: string, state: RemoteSyncStateFile): Promise<void> {
+  await writeFile(
+    path,
+    JSON.stringify(
+      {
+        ...state,
+        mode: REMOTE_SYNC_STATE_MODE,
+        updated_at: nowIso()
+      },
+      null,
+      2
+    )
+  );
+}
+
+export function isStaleBaseIndexError(error: unknown): boolean {
+  if (!(error instanceof RemoteSyncHttpResponseError)) {
+    return false;
+  }
+  if (error.status !== 400) {
+    return false;
+  }
+  const message = error.message.toLowerCase();
+  return message.includes("invalid_argument") && message.includes("base index version");
+}
+
+export function isDeltaUnsupportedError(error: unknown): boolean {
+  if (error instanceof RemoteSyncHttpResponseError) {
+    if ([404, 405, 501].includes(error.status)) {
+      return true;
+    }
+    const message = error.message.toLowerCase();
+    return message.includes("not_found") && message.includes("push-delta");
+  }
+
+  if (error instanceof Error) {
+    const message = error.message.toLowerCase();
+    return message.includes("push-delta") && (message.includes("not found") || message.includes("404"));
+  }
+
+  return false;
+}
+
+export function isBlobCommitV2UnsupportedError(error: unknown): boolean {
+  if (error instanceof RemoteSyncHttpResponseError) {
+    if ([404, 405, 501].includes(error.status)) {
+      return true;
+    }
+    const message = error.message.toLowerCase();
+    return (
+      (message.includes("blob_commit_v2") && message.includes("disabled")) ||
+      message.includes("commit-v2") ||
+      message.includes("blobs/upload")
+    ) && message.includes("not found");
+  }
+
+  if (error instanceof Error) {
+    const message = error.message.toLowerCase();
+    return (
+      (message.includes("commit-v2") || message.includes("blobs/upload")) &&
+      (message.includes("not found") || message.includes("404"))
+    );
+  }
+
+  return false;
+}
+
+function isRetryableAdaptiveError(error: unknown): boolean {
+  if (error instanceof RemoteSyncHttpResponseError) {
+    return error.status === 429 || error.status >= 500;
+  }
+  return false;
+}
+
+function computeBlobBatches(input: {
+  upsert_files: RemoteSyncUploadCandidate[];
+  max_blob_bytes: number;
+  max_blob_batch_bytes: number;
+}): Array<
+  Array<{
+    hash: string;
+    content: string;
+    size_bytes: number;
+  }>
+> {
+  const deduped = new Map<string, { hash: string; content: string; size_bytes: number }>();
+  for (const file of input.upsert_files) {
+    const sizeBytes = Buffer.byteLength(file.content, "utf8");
+    if (sizeBytes > input.max_blob_bytes) {
+      throw new Error(`blob payload too large for path ${file.path}`);
+    }
+    const hash = sha256Text(file.content);
+    if (!deduped.has(hash)) {
+      deduped.set(hash, {
+        hash,
+        content: file.content,
+        size_bytes: sizeBytes
+      });
+    }
+  }
+
+  const blobs = [...deduped.values()].sort((a, b) => a.hash.localeCompare(b.hash));
+  const batches: Array<Array<{ hash: string; content: string; size_bytes: number }>> = [];
+  let current: Array<{ hash: string; content: string; size_bytes: number }> = [];
+  let currentBytes = 0;
+  for (const blob of blobs) {
+    if (blob.size_bytes > input.max_blob_batch_bytes) {
+      throw new Error(`blob ${blob.hash} exceeds max blob batch bytes`);
+    }
+    if (current.length > 0 && currentBytes + blob.size_bytes > input.max_blob_batch_bytes) {
+      batches.push(current);
+      current = [];
+      currentBytes = 0;
+    }
+    current.push(blob);
+    currentBytes += blob.size_bytes;
+  }
+  if (current.length > 0) {
+    batches.push(current);
+  }
+  return batches;
+}
+
+function supportsBlobCommitV2(capabilities: RemoteSyncCapabilities): boolean {
+  const protocols = capabilities.sync_protocols ?? [];
+  return protocols.includes("blob_commit_v2");
+}
+
+export async function runRemoteAdaptiveSync(input: RunRemoteAdaptiveSyncInput): Promise<RunRemoteAdaptiveSyncResult> {
+  const runStartedAt = Date.now();
+  const retries = input.retries ?? 3;
+  const initialDelayMs = input.initial_delay_ms ?? 500;
+  const staleBaseError = input.stale_base_error ?? isStaleBaseIndexError;
+
+  if (!supportsBlobCommitV2(input.capabilities)) {
+    const delta = await runRemoteDeltaSync({
+      project_root_path: input.project_root_path,
+      scan_root_path: input.scan_root_path,
+      workspace_id: input.workspace_id,
+      previous_state: input.previous_state,
+      force_full_upsert: input.force_full_upsert,
+      max_body_bytes: input.capabilities.max_body_bytes,
+      retries,
+      initial_delay_ms: initialDelayMs,
+      stale_base_error: staleBaseError,
+      persist_state: input.persist_state,
+      push_delta: input.push_delta
+    });
+    return {
+      ...delta,
+      protocol: "delta_v1"
+    };
+  }
+
+  const execute = async (previousState: RemoteSyncStateFile | undefined, forceFullUpsert: boolean) => {
+    const scanRootPath = input.scan_root_path ?? input.project_root_path;
+    const deltaBuild = await buildRemoteSyncDeltaFromState({
+      project_root_path: scanRootPath,
+      previous_state: previousState,
+      force_full_upsert: forceFullUpsert
+    });
+
+    const nextWorkspaceId = input.workspace_id ?? previousState?.workspace_id;
+    if (deltaBuild.delta.upsert_files.length === 0 && deltaBuild.delta.deleted_paths.length === 0) {
+      const unchangedState: RemoteSyncStateFile = {
+        mode: REMOTE_SYNC_STATE_MODE,
+        workspace_id: nextWorkspaceId,
+        last_index_version: previousState?.last_index_version,
+        files: deltaBuild.next_files,
+        updated_at: nowIso()
+      };
+      await input.persist_state?.(unchangedState);
+      return {
+        state: unchangedState,
+        changed: false,
+        workspace_id: unchangedState.workspace_id,
+        index_version: unchangedState.last_index_version,
+        applied_delta: {
+          upsert_files: 0,
+          deleted_paths: 0
+        },
+        stats: {
+          batches_total: 0,
+          bytes_total: 0,
+          latency_ms: Date.now() - runStartedAt
+        },
+        protocol: "blob_commit_v2" as const
+      };
+    }
+
+    const maxBlobBytes = input.capabilities.max_blob_bytes ?? 128 * 1024;
+    const maxBlobBatchBytes = input.capabilities.max_blob_batch_bytes ?? Math.max(maxBlobBytes, 1024 * 1024);
+    const maxCommitBodyBytes = input.capabilities.max_commit_body_bytes ?? input.capabilities.max_body_bytes;
+    let currentConcurrency = Math.max(
+      1,
+      Math.min(input.capabilities.upload_concurrency_hint ?? 4, 16)
+    );
+
+    const blobBatches = computeBlobBatches({
+      upsert_files: deltaBuild.delta.upsert_files,
+      max_blob_bytes: maxBlobBytes,
+      max_blob_batch_bytes: maxBlobBatchBytes
+    });
+    const pending = [...blobBatches];
+    const acknowledged = new Set<string>();
+    let bytesTotal = 0;
+
+    while (pending.length > 0) {
+      const wave = pending.splice(0, currentConcurrency);
+      const settled = await Promise.allSettled(
+        wave.map(async (batch) =>
+          retryWithBackoff({
+            retries,
+            initial_delay_ms: initialDelayMs,
+            fn: async () =>
+              input.upload_blobs({
+                workspace_id: nextWorkspaceId,
+                project_root_path: input.project_root_path,
+                blobs: batch
+              })
+          })
+        )
+      );
+
+      let waveHadRetryableError = false;
+      for (let idx = 0; idx < settled.length; idx += 1) {
+        const result = settled[idx]!;
+        const batch = wave[idx]!;
+        if (result.status === "rejected") {
+          if (isRetryableAdaptiveError(result.reason)) {
+            pending.push(batch);
+            waveHadRetryableError = true;
+            continue;
+          }
+          throw result.reason;
+        }
+
+        const payload = result.value;
+        for (const hash of payload.accepted_hashes) {
+          acknowledged.add(hash.toLowerCase());
+        }
+        for (const hash of payload.already_present_hashes) {
+          acknowledged.add(hash.toLowerCase());
+        }
+        if (payload.rejected.length > 0) {
+          throw new Error(`blob upload rejected: ${payload.rejected[0]!.reason}`);
+        }
+        bytesTotal += batch.reduce((sum, blob) => sum + blob.size_bytes, 0);
+      }
+
+      if (waveHadRetryableError) {
+        const previous = currentConcurrency;
+        currentConcurrency = Math.max(1, Math.floor(currentConcurrency / 2));
+        await input.on_upload_strategy_change?.({
+          previous_concurrency: previous,
+          next_concurrency: currentConcurrency,
+          reason: "error"
+        });
+      } else if (currentConcurrency < Math.max(1, input.capabilities.upload_concurrency_hint ?? 4)) {
+        const previous = currentConcurrency;
+        currentConcurrency += 1;
+        await input.on_upload_strategy_change?.({
+          previous_concurrency: previous,
+          next_concurrency: currentConcurrency,
+          reason: "success"
+        });
+      }
+    }
+
+    const dedupedUpserts = new Map<string, RemoteSyncUploadCandidate>();
+    for (const file of deltaBuild.delta.upsert_files) {
+      dedupedUpserts.set(file.path, file);
+    }
+    const commitUpserts = [...dedupedUpserts.values()]
+      .sort((a, b) => a.path.localeCompare(b.path))
+      .map((file) => ({
+        path: file.path,
+        blob_hash: sha256Text(file.content),
+        ...(file.language ? { language: file.language } : {})
+      }));
+    const commitDeletes = [...new Set(deltaBuild.delta.deleted_paths)].sort((a, b) => a.localeCompare(b));
+
+    const missingHash = commitUpserts.find((file) => !acknowledged.has(file.blob_hash.toLowerCase()));
+    if (missingHash) {
+      throw new Error(`missing uploaded blob hash for commit: ${missingHash.blob_hash}`);
+    }
+
+    if (
+      Buffer.byteLength(
+        JSON.stringify({
+          project_root_path: input.project_root_path,
+          workspace_id: nextWorkspaceId,
+          base_index_version: previousState?.last_index_version,
+          upsert_files: commitUpserts,
+          deleted_paths: commitDeletes
+        }),
+        "utf8"
+      ) > maxCommitBodyBytes
+    ) {
+      throw new Error("commit-v2 payload exceeds max_commit_body_bytes");
+    }
+
+    const commit = await retryWithBackoff({
+      retries,
+      initial_delay_ms: initialDelayMs,
+      fn: async () =>
+        input.commit_v2({
+          workspace_id: nextWorkspaceId,
+          project_root_path: input.project_root_path,
+          ...(previousState?.last_index_version ? { base_index_version: previousState.last_index_version } : {}),
+          upsert_files: commitUpserts,
+          deleted_paths: commitDeletes
+        })
+    });
+
+    const finalWorkspace = commit.workspace_id ?? nextWorkspaceId;
+    const finalIndexVersion = commit.index_version ?? previousState?.last_index_version;
+    const finalState: RemoteSyncStateFile = {
+      mode: REMOTE_SYNC_STATE_MODE,
+      workspace_id: finalWorkspace,
+      last_index_version: finalIndexVersion,
+      files: deltaBuild.next_files,
+      updated_at: nowIso()
+    };
+    await input.persist_state?.(finalState);
+
+    return {
+      state: finalState,
+      changed: true,
+      workspace_id: finalWorkspace,
+      index_version: finalIndexVersion,
+      applied_delta: {
+        upsert_files: commitUpserts.length,
+        deleted_paths: commitDeletes.length
+      },
+      stats: {
+        batches_total: blobBatches.length,
+        bytes_total: bytesTotal,
+        latency_ms: Date.now() - runStartedAt
+      },
+      protocol: "blob_commit_v2" as const
+    };
+  };
+
+  const forceFullUpsert = input.force_full_upsert ?? false;
+  try {
+    return await execute(input.previous_state, forceFullUpsert);
+  } catch (error) {
+    if (isBlobCommitV2UnsupportedError(error)) {
+      const delta = await runRemoteDeltaSync({
+        project_root_path: input.project_root_path,
+        scan_root_path: input.scan_root_path,
+        workspace_id: input.workspace_id,
+        previous_state: input.previous_state,
+        force_full_upsert: input.force_full_upsert,
+        max_body_bytes: input.capabilities.max_body_bytes,
+        retries,
+        initial_delay_ms: initialDelayMs,
+        stale_base_error: staleBaseError,
+        persist_state: input.persist_state,
+        push_delta: input.push_delta
+      });
+      return {
+        ...delta,
+        protocol: "delta_v1"
+      };
+    }
+    if (!forceFullUpsert && input.previous_state && staleBaseError(error)) {
+      return execute(undefined, true);
+    }
+    throw error;
+  }
+}
+
+export async function retryWithBackoff<T>(input: {
+  fn: () => Promise<T>;
+  retries: number;
+  initial_delay_ms: number;
+}): Promise<T> {
+  let attempt = 0;
+  let lastError: unknown;
+  while (attempt < input.retries) {
+    try {
+      return await input.fn();
+    } catch (error) {
+      lastError = error;
+      attempt += 1;
+      if (attempt >= input.retries) {
+        break;
+      }
+      await new Promise((resolveSleep) => setTimeout(resolveSleep, input.initial_delay_ms * 2 ** (attempt - 1)));
+    }
+  }
+  throw lastError;
+}
+
+export async function runRemoteDeltaSync(input: RunRemoteDeltaSyncInput): Promise<RunRemoteDeltaSyncResult> {
+  const runStartedAt = Date.now();
+  const retries = input.retries ?? 3;
+  const initialDelayMs = input.initial_delay_ms ?? 500;
+  const isStaleError = input.stale_base_error ?? isStaleBaseIndexError;
+
+  const execute = async (previousState: RemoteSyncStateFile | undefined, forceFullUpsert: boolean): Promise<RunRemoteDeltaSyncResult> => {
+    const scanRootPath = input.scan_root_path ?? input.project_root_path;
+    const deltaBuild = await buildRemoteSyncDeltaFromState({
+      project_root_path: scanRootPath,
+      previous_state: previousState,
+      force_full_upsert: forceFullUpsert
+    });
+
+    const nextWorkspaceId = input.workspace_id ?? previousState?.workspace_id;
+
+    if (deltaBuild.delta.upsert_files.length === 0 && deltaBuild.delta.deleted_paths.length === 0) {
+      const unchangedState: RemoteSyncStateFile = {
+        mode: REMOTE_SYNC_STATE_MODE,
+        workspace_id: nextWorkspaceId,
+        last_index_version: previousState?.last_index_version,
+        files: deltaBuild.next_files,
+        updated_at: nowIso()
+      };
+      await input.persist_state?.(unchangedState);
+      return {
+        state: unchangedState,
+        changed: false,
+        workspace_id: unchangedState.workspace_id,
+        index_version: unchangedState.last_index_version,
+        applied_delta: {
+          upsert_files: 0,
+          deleted_paths: 0
+        },
+        stats: {
+          batches_total: 0,
+          bytes_total: 0,
+          latency_ms: Date.now() - runStartedAt
+        }
+      };
+    }
+
+    let currentBaseIndexVersion = previousState?.last_index_version;
+    let currentWorkspaceId = nextWorkspaceId;
+    const progressFiles: Record<string, RemoteSyncStateEntry> = { ...(previousState?.files ?? {}) };
+    let uploadedCount = 0;
+    let deletedCount = 0;
+    let bytesTotal = 0;
+
+    const batches = splitRemoteSyncDeltaIntoBatches({
+      project_root_path: input.project_root_path,
+      workspace_id: currentWorkspaceId,
+      base_index_version: currentBaseIndexVersion,
+      delta: deltaBuild.delta,
+      max_body_bytes: input.max_body_bytes
+    });
+
+    let latestState: RemoteSyncStateFile | undefined;
+
+    for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
+      const batch = batches[batchIndex]!;
+      const batchStartedAt = Date.now();
+      const result = await retryWithBackoff({
+        retries,
+        initial_delay_ms: initialDelayMs,
+        fn: async () =>
+          input.push_delta({
+            workspace_id: currentWorkspaceId,
+            project_root_path: input.project_root_path,
+            ...(currentBaseIndexVersion ? { base_index_version: currentBaseIndexVersion } : {}),
+            upsert_files: batch.upsert_files,
+            deleted_paths: batch.deleted_paths
+          })
+      });
+
+      currentWorkspaceId = result.workspace_id ?? currentWorkspaceId;
+      currentBaseIndexVersion = result.index_version ?? currentBaseIndexVersion;
+
+      for (const file of batch.upsert_files) {
+        const entry = deltaBuild.upsert_state_entries[file.path];
+        if (entry) {
+          progressFiles[file.path] = entry;
+        }
+      }
+      for (const path of batch.deleted_paths) {
+        delete progressFiles[path];
+      }
+
+      uploadedCount += batch.upsert_files.length;
+      deletedCount += batch.deleted_paths.length;
+      bytesTotal += batch.approx_bytes;
+
+      await input.on_batch_processed?.({
+        batch_index: batchIndex,
+        batch_count: batches.length,
+        approx_bytes: batch.approx_bytes,
+        upsert_files: batch.upsert_files.length,
+        deleted_paths: batch.deleted_paths.length,
+        latency_ms: Date.now() - batchStartedAt
+      });
+
+      latestState = {
+        mode: REMOTE_SYNC_STATE_MODE,
+        workspace_id: currentWorkspaceId,
+        last_index_version: currentBaseIndexVersion,
+        files: progressFiles,
+        updated_at: nowIso()
+      };
+      await input.persist_state?.(latestState);
+    }
+
+    const finalState = latestState ?? {
+      mode: REMOTE_SYNC_STATE_MODE,
+      workspace_id: currentWorkspaceId,
+      last_index_version: currentBaseIndexVersion,
+      files: progressFiles,
+      updated_at: nowIso()
+    };
+
+    return {
+      state: finalState,
+      changed: true,
+      workspace_id: currentWorkspaceId,
+      index_version: currentBaseIndexVersion,
+      applied_delta: {
+        upsert_files: uploadedCount,
+        deleted_paths: deletedCount
+      },
+      stats: {
+        batches_total: batches.length,
+        bytes_total: bytesTotal,
+        latency_ms: Date.now() - runStartedAt
+      }
+    };
+  };
+
+  const forceFullUpsert = input.force_full_upsert ?? false;
+  try {
+    return await execute(input.previous_state, forceFullUpsert);
+  } catch (error) {
+    if (!forceFullUpsert && input.previous_state && isStaleError(error)) {
+      return execute(undefined, true);
+    }
+    throw error;
+  }
+}