@rce-mcp/retrieval-core 0.1.0

package/src/remote-sync.ts

@@ -0,0 +1,651 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+
+export const REMOTE_SYNC_STATE_MODE = "remote_delta_v1" as const;
+export const DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES = 1_000_000;
+
+const DEFAULT_EXCLUDED_DIRS = new Set([
+  ".cache",
+  ".git",
+  ".next",
+  ".nuxt",
+  ".rce",
+  ".rce-tool",
+  ".svelte-kit",
+  ".turbo",
+  "build",
+  "coverage",
+  "dist",
+  "node_modules",
+  "out",
+  "target"
+]);
+
+const DEFAULT_EXCLUDED_FILES = new Set([
+  "bun.lock",
+  "bun.lockb",
+  "Cargo.lock",
+  "composer.lock",
+  "Gemfile.lock",
+  "package-lock.json",
+  "Pipfile.lock",
+  "pnpm-lock.yaml",
+  "yarn.lock"
+]);
+
+const DEFAULT_EXCLUDED_FILE_SUFFIXES = new Set([".min.js", ".min.css", ".map"]);
+
+export interface RemoteSyncUploadCandidate {
+  path: string;
+  content: string;
+  language?: string;
+}
+
+export interface RemoteSyncProjectFileStat {
+  path: string;
+  full_path: string;
+  size: number;
+  mtime_ms: number;
+  language?: string;
+}
+
+export interface RemoteSyncStateEntry {
+  content_hash: string;
+  size: number;
+  mtime_ms: number;
+  language?: string;
+}
+
+export interface RemoteSyncStateFile {
+  mode: typeof REMOTE_SYNC_STATE_MODE;
+  workspace_id?: string;
+  last_index_version?: string;
+  files: Record<string, RemoteSyncStateEntry>;
+  updated_at: string;
+}
+
+export interface RemoteSyncDeltaPayload {
+  upsert_files: RemoteSyncUploadCandidate[];
+  deleted_paths: string[];
+}
+
+export interface RemoteSyncDeltaBatch {
+  upsert_files: RemoteSyncUploadCandidate[];
+  deleted_paths: string[];
+  approx_bytes: number;
+}
+
+export interface BuildRemoteSyncDeltaResult {
+  delta: RemoteSyncDeltaPayload;
+  upsert_state_entries: Record<string, RemoteSyncStateEntry>;
+  next_files: Record<string, RemoteSyncStateEntry>;
+}
+
+export interface RemoteSyncScanOptions {
+  max_file_size_bytes?: number;
+  excluded_dirs?: Set<string>;
+  excluded_files?: Set<string>;
+  excluded_file_suffixes?: Set<string>;
+}
+
+export class RemoteSyncHttpResponseError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+    readonly payload?: unknown
+  ) {
+    super(message);
+  }
+}
+
+export interface RunRemoteDeltaSyncInput {
+  project_root_path: string;
+  scan_root_path?: string;
+  workspace_id?: string;
+  previous_state?: RemoteSyncStateFile;
+  force_full_upsert?: boolean;
+  max_body_bytes: number;
+  retries?: number;
+  initial_delay_ms?: number;
+  stale_base_error?: (error: unknown) => boolean;
+  persist_state?: (state: RemoteSyncStateFile) => Promise<void>;
+  push_delta: (request: {
+    workspace_id?: string;
+    project_root_path: string;
+    base_index_version?: string;
+    upsert_files: RemoteSyncUploadCandidate[];
+    deleted_paths: string[];
+  }) => Promise<{
+    workspace_id?: string;
+    index_version?: string;
+  }>;
+}
+
+export interface RunRemoteDeltaSyncResult {
+  state: RemoteSyncStateFile;
+  changed: boolean;
+  workspace_id?: string;
+  index_version?: string;
+  applied_delta: {
+    upsert_files: number;
+    deleted_paths: number;
+  };
+}
+
+function nowIso(): string {
+  return new Date().toISOString();
+}
+
+function extensionToLanguage(path: string): string | undefined {
+  if (path.endsWith(".ts") || path.endsWith(".tsx")) return "typescript";
+  if (path.endsWith(".js") || path.endsWith(".jsx")) return "javascript";
+  if (path.endsWith(".py")) return "python";
+  if (path.endsWith(".go")) return "go";
+  if (path.endsWith(".rs")) return "rust";
+  if (path.endsWith(".java")) return "java";
+  if (path.endsWith(".json")) return "json";
+  if (path.endsWith(".md")) return "markdown";
+  if (path.endsWith(".yml") || path.endsWith(".yaml")) return "yaml";
+  return undefined;
+}
+
+function looksBinary(content: string): boolean {
+  return content.includes("\0");
+}
+
+function sha256Text(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function shouldExcludeFile(path: string, excludedFiles: Set<string>, excludedSuffixes: Set<string>): boolean {
+  const lower = path.toLowerCase();
+  if (excludedFiles.has(path) || excludedFiles.has(lower)) {
+    return true;
+  }
+  for (const suffix of excludedSuffixes) {
+    if (lower.endsWith(suffix)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function resolveScanOptions(options?: RemoteSyncScanOptions): Required<RemoteSyncScanOptions> {
+  return {
+    max_file_size_bytes: options?.max_file_size_bytes ?? DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES,
+    excluded_dirs: options?.excluded_dirs ?? DEFAULT_EXCLUDED_DIRS,
+    excluded_files: options?.excluded_files ?? DEFAULT_EXCLUDED_FILES,
+    excluded_file_suffixes: options?.excluded_file_suffixes ?? DEFAULT_EXCLUDED_FILE_SUFFIXES
+  };
+}
+
+export async function collectProjectFileStats(
+  project_root_path: string,
+  options?: RemoteSyncScanOptions
+): Promise<Map<string, RemoteSyncProjectFileStat>> {
+  const root = resolve(project_root_path);
+  const resolvedOptions = resolveScanOptions(options);
+  const output = new Map<string, RemoteSyncProjectFileStat>();
+
+  async function walk(dir: string): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        if (resolvedOptions.excluded_dirs.has(entry.name)) {
+          continue;
+        }
+        await walk(join(dir, entry.name));
+        continue;
+      }
+
+      if (!entry.isFile()) {
+        continue;
+      }
+
+      if (shouldExcludeFile(entry.name, resolvedOptions.excluded_files, resolvedOptions.excluded_file_suffixes)) {
+        continue;
+      }
+
+      const fullPath = join(dir, entry.name);
+      const fileStat = await stat(fullPath);
+      if (fileStat.size > resolvedOptions.max_file_size_bytes) {
+        continue;
+      }
+
+      const repoPath = relative(root, fullPath).replace(/\\/g, "/");
+      output.set(repoPath, {
+        path: repoPath,
+        full_path: fullPath,
+        size: fileStat.size,
+        mtime_ms: Math.trunc(fileStat.mtimeMs),
+        language: extensionToLanguage(repoPath)
+      });
+    }
+  }
+
+  await walk(root);
+  return output;
+}
+
+export async function collectUploadCandidates(
+  project_root_path: string,
+  options?: RemoteSyncScanOptions
+): Promise<RemoteSyncUploadCandidate[]> {
+  const stats = await collectProjectFileStats(project_root_path, options);
+  const output: RemoteSyncUploadCandidate[] = [];
+
+  for (const repoPath of [...stats.keys()].sort((a, b) => a.localeCompare(b))) {
+    const fileStat = stats.get(repoPath)!;
+    let content: string;
+    try {
+      content = await readFile(fileStat.full_path, "utf8");
+    } catch {
+      continue;
+    }
+    if (looksBinary(content)) {
+      continue;
+    }
+
+    output.push({
+      path: repoPath,
+      content,
+      ...(fileStat.language ? { language: fileStat.language } : {})
+    });
+  }
+
+  return output;
+}
+
+export async function buildRemoteSyncDeltaFromState(input: {
+  project_root_path: string;
+  previous_state?: RemoteSyncStateFile;
+  force_full_upsert: boolean;
+  options?: RemoteSyncScanOptions;
+}): Promise<BuildRemoteSyncDeltaResult> {
+  const projectFiles = await collectProjectFileStats(input.project_root_path, input.options);
+  const previousFiles = input.previous_state?.files ?? {};
+  const upsertFiles: RemoteSyncUploadCandidate[] = [];
+  const deletedPaths: string[] = [];
+  const upsertStateEntries: Record<string, RemoteSyncStateEntry> = {};
+  const nextFiles: Record<string, RemoteSyncStateEntry> = {};
+  const sortedPaths = [...projectFiles.keys()].sort((a, b) => a.localeCompare(b));
+
+  for (const repoPath of sortedPaths) {
+    const projectFile = projectFiles.get(repoPath)!;
+    const previous = previousFiles[repoPath];
+    const statChanged =
+      !previous ||
+      previous.size !== projectFile.size ||
+      previous.mtime_ms !== projectFile.mtime_ms ||
+      input.force_full_upsert;
+
+    if (!statChanged) {
+      nextFiles[repoPath] = previous;
+      continue;
+    }
+
+    let content: string;
+    try {
+      content = await readFile(projectFile.full_path, "utf8");
+    } catch {
+      if (previous) {
+        nextFiles[repoPath] = previous;
+      }
+      continue;
+    }
+
+    if (looksBinary(content)) {
+      if (previous) {
+        nextFiles[repoPath] = previous;
+      }
+      continue;
+    }
+
+    const contentHash = sha256Text(content);
+    const nextEntry: RemoteSyncStateEntry = {
+      content_hash: contentHash,
+      size: projectFile.size,
+      mtime_ms: projectFile.mtime_ms,
+      ...(projectFile.language ? { language: projectFile.language } : {})
+    };
+
+    nextFiles[repoPath] = nextEntry;
+    if (!input.force_full_upsert && previous?.content_hash === contentHash) {
+      continue;
+    }
+
+    upsertFiles.push({
+      path: repoPath,
+      content,
+      ...(projectFile.language ? { language: projectFile.language } : {})
+    });
+    upsertStateEntries[repoPath] = nextEntry;
+  }
+
+  for (const previousPath of Object.keys(previousFiles)) {
+    if (!projectFiles.has(previousPath)) {
+      deletedPaths.push(previousPath);
+    }
+  }
+
+  return {
+    delta: {
+      upsert_files: upsertFiles,
+      deleted_paths: deletedPaths
+    },
+    upsert_state_entries: upsertStateEntries,
+    next_files: nextFiles
+  };
+}
+
+export function estimateRemoteSyncDeltaRequestSize(input: {
+  project_root_path: string;
+  workspace_id?: string;
+  base_index_version?: string;
+  upsert_files: RemoteSyncUploadCandidate[];
+  deleted_paths: string[];
+}): number {
+  const payload = {
+    project_root_path: input.project_root_path,
+    ...(input.workspace_id ? { workspace_id: input.workspace_id } : {}),
+    ...(input.base_index_version ? { base_index_version: input.base_index_version } : {}),
+    upsert_files: input.upsert_files.map((file) => ({
+      path: file.path,
+      content: file.content,
+      ...(file.language ? { language: file.language } : {})
+    })),
+    deleted_paths: input.deleted_paths
+  };
+  return Buffer.byteLength(JSON.stringify(payload), "utf8");
+}
+
+export function splitRemoteSyncDeltaIntoBatches(input: {
+  project_root_path: string;
+  workspace_id?: string;
+  base_index_version?: string;
+  delta: RemoteSyncDeltaPayload;
+  max_body_bytes: number;
+}): RemoteSyncDeltaBatch[] {
+  const budget = Math.max(256 * 1024, Math.floor(input.max_body_bytes * 0.8));
+  const upserts = [...input.delta.upsert_files].sort((a, b) => a.path.localeCompare(b.path));
+  const deleted = [...new Set(input.delta.deleted_paths)].sort((a, b) => a.localeCompare(b));
+  const batches: RemoteSyncDeltaBatch[] = [];
+  let currentUpserts: RemoteSyncUploadCandidate[] = [];
+  let currentDeleted: string[] = [];
+
+  const flush = () => {
+    if (currentUpserts.length === 0 && currentDeleted.length === 0) {
+      return;
+    }
+    batches.push({
+      upsert_files: currentUpserts,
+      deleted_paths: currentDeleted,
+      approx_bytes: estimateRemoteSyncDeltaRequestSize({
+        project_root_path: input.project_root_path,
+        workspace_id: input.workspace_id,
+        base_index_version: input.base_index_version,
+        upsert_files: currentUpserts,
+        deleted_paths: currentDeleted
+      })
+    });
+    currentUpserts = [];
+    currentDeleted = [];
+  };
+
+  const canFit = (nextUpserts: RemoteSyncUploadCandidate[], nextDeleted: string[]) =>
+    estimateRemoteSyncDeltaRequestSize({
+      project_root_path: input.project_root_path,
+      workspace_id: input.workspace_id,
+      base_index_version: input.base_index_version,
+      upsert_files: nextUpserts,
+      deleted_paths: nextDeleted
+    }) <= budget;
+
+  for (const file of upserts) {
+    const candidateUpserts = [...currentUpserts, file];
+    if (canFit(candidateUpserts, currentDeleted)) {
+      currentUpserts = candidateUpserts;
+      continue;
+    }
+    flush();
+    if (!canFit([file], [])) {
+      throw new Error(`delta upsert payload too large for path ${file.path}`);
+    }
+    currentUpserts = [file];
+  }
+
+  for (const path of deleted) {
+    const candidateDeleted = [...currentDeleted, path];
+    if (canFit(currentUpserts, candidateDeleted)) {
+      currentDeleted = candidateDeleted;
+      continue;
+    }
+    flush();
+    if (!canFit([], [path])) {
+      throw new Error(`delta delete payload too large for path ${path}`);
+    }
+    currentDeleted = [path];
+  }
+
+  flush();
+  return batches;
+}
+
+export async function readRemoteSyncState(path: string): Promise<RemoteSyncStateFile | undefined> {
+  if (!existsSync(path)) {
+    return undefined;
+  }
+
+  try {
+    const parsed = JSON.parse(await readFile(path, "utf8")) as Partial<RemoteSyncStateFile>;
+    if (
+      !parsed ||
+      parsed.mode !== REMOTE_SYNC_STATE_MODE ||
+      typeof parsed.files !== "object" ||
+      parsed.files === null ||
+      Array.isArray(parsed.files)
+    ) {
+      return undefined;
+    }
+
+    return {
+      mode: REMOTE_SYNC_STATE_MODE,
+      workspace_id: parsed.workspace_id,
+      last_index_version: parsed.last_index_version,
+      files: parsed.files as Record<string, RemoteSyncStateEntry>,
+      updated_at: typeof parsed.updated_at === "string" ? parsed.updated_at : nowIso()
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+export async function writeRemoteSyncState(path: string, state: RemoteSyncStateFile): Promise<void> {
+  await writeFile(
+    path,
+    JSON.stringify(
+      {
+        ...state,
+        mode: REMOTE_SYNC_STATE_MODE,
+        updated_at: nowIso()
+      },
+      null,
+      2
+    )
+  );
+}
+
+export function isStaleBaseIndexError(error: unknown): boolean {
+  if (!(error instanceof RemoteSyncHttpResponseError)) {
+    return false;
+  }
+  if (error.status !== 400) {
+    return false;
+  }
+  const message = error.message.toLowerCase();
+  return message.includes("invalid_argument") && message.includes("base index version");
+}
+
+export function isDeltaUnsupportedError(error: unknown): boolean {
+  if (error instanceof RemoteSyncHttpResponseError) {
+    if ([404, 405, 501].includes(error.status)) {
+      return true;
+    }
+    const message = error.message.toLowerCase();
+    return message.includes("not_found") && message.includes("push-delta");
+  }
+
+  if (error instanceof Error) {
+    const message = error.message.toLowerCase();
+    return message.includes("push-delta") && (message.includes("not found") || message.includes("404"));
+  }
+
+  return false;
+}
+
+export async function retryWithBackoff<T>(input: {
+  fn: () => Promise<T>;
+  retries: number;
+  initial_delay_ms: number;
+}): Promise<T> {
+  let attempt = 0;
+  let lastError: unknown;
+  while (attempt < input.retries) {
+    try {
+      return await input.fn();
+    } catch (error) {
+      lastError = error;
+      attempt += 1;
+      if (attempt >= input.retries) {
+        break;
+      }
+      await new Promise((resolveSleep) => setTimeout(resolveSleep, input.initial_delay_ms * 2 ** (attempt - 1)));
+    }
+  }
+  throw lastError;
+}
+
+export async function runRemoteDeltaSync(input: RunRemoteDeltaSyncInput): Promise<RunRemoteDeltaSyncResult> {
+  const retries = input.retries ?? 3;
+  const initialDelayMs = input.initial_delay_ms ?? 500;
+  const isStaleError = input.stale_base_error ?? isStaleBaseIndexError;
+
+  const execute = async (previousState: RemoteSyncStateFile | undefined, forceFullUpsert: boolean): Promise<RunRemoteDeltaSyncResult> => {
+    const scanRootPath = input.scan_root_path ?? input.project_root_path;
+    const deltaBuild = await buildRemoteSyncDeltaFromState({
+      project_root_path: scanRootPath,
+      previous_state: previousState,
+      force_full_upsert: forceFullUpsert
+    });
+
+    const nextWorkspaceId = input.workspace_id ?? previousState?.workspace_id;
+
+    if (deltaBuild.delta.upsert_files.length === 0 && deltaBuild.delta.deleted_paths.length === 0) {
+      const unchangedState: RemoteSyncStateFile = {
+        mode: REMOTE_SYNC_STATE_MODE,
+        workspace_id: nextWorkspaceId,
+        last_index_version: previousState?.last_index_version,
+        files: deltaBuild.next_files,
+        updated_at: nowIso()
+      };
+      await input.persist_state?.(unchangedState);
+      return {
+        state: unchangedState,
+        changed: false,
+        workspace_id: unchangedState.workspace_id,
+        index_version: unchangedState.last_index_version,
+        applied_delta: {
+          upsert_files: 0,
+          deleted_paths: 0
+        }
+      };
+    }
+
+    let currentBaseIndexVersion = previousState?.last_index_version;
+    let currentWorkspaceId = nextWorkspaceId;
+    const progressFiles: Record<string, RemoteSyncStateEntry> = { ...(previousState?.files ?? {}) };
+    let uploadedCount = 0;
+    let deletedCount = 0;
+
+    const batches = splitRemoteSyncDeltaIntoBatches({
+      project_root_path: input.project_root_path,
+      workspace_id: currentWorkspaceId,
+      base_index_version: currentBaseIndexVersion,
+      delta: deltaBuild.delta,
+      max_body_bytes: input.max_body_bytes
+    });
+
+    let latestState: RemoteSyncStateFile | undefined;
+
+    for (const batch of batches) {
+      const result = await retryWithBackoff({
+        retries,
+        initial_delay_ms: initialDelayMs,
+        fn: async () =>
+          input.push_delta({
+            workspace_id: currentWorkspaceId,
+            project_root_path: input.project_root_path,
+            ...(currentBaseIndexVersion ? { base_index_version: currentBaseIndexVersion } : {}),
+            upsert_files: batch.upsert_files,
+            deleted_paths: batch.deleted_paths
+          })
+      });
+
+      currentWorkspaceId = result.workspace_id ?? currentWorkspaceId;
+      currentBaseIndexVersion = result.index_version ?? currentBaseIndexVersion;
+
+      for (const file of batch.upsert_files) {
+        const entry = deltaBuild.upsert_state_entries[file.path];
+        if (entry) {
+          progressFiles[file.path] = entry;
+        }
+      }
+      for (const path of batch.deleted_paths) {
+        delete progressFiles[path];
+      }
+
+      uploadedCount += batch.upsert_files.length;
+      deletedCount += batch.deleted_paths.length;
+
+      latestState = {
+        mode: REMOTE_SYNC_STATE_MODE,
+        workspace_id: currentWorkspaceId,
+        last_index_version: currentBaseIndexVersion,
+        files: progressFiles,
+        updated_at: nowIso()
+      };
+      await input.persist_state?.(latestState);
+    }
+
+    const finalState = latestState ?? {
+      mode: REMOTE_SYNC_STATE_MODE,
+      workspace_id: currentWorkspaceId,
+      last_index_version: currentBaseIndexVersion,
+      files: progressFiles,
+      updated_at: nowIso()
+    };
+
+    return {
+      state: finalState,
+      changed: true,
+      workspace_id: currentWorkspaceId,
+      index_version: currentBaseIndexVersion,
+      applied_delta: {
+        upsert_files: uploadedCount,
+        deleted_paths: deletedCount
+      }
+    };
+  };
+
+  const forceFullUpsert = input.force_full_upsert ?? false;
+  try {
+    return await execute(input.previous_state, forceFullUpsert);
+  } catch (error) {
+    if (!forceFullUpsert && input.previous_state && isStaleError(error)) {
+      return execute(undefined, true);
+    }
+    throw error;
+  }
+}