@rce-mcp/retrieval-core 0.1.0

package/dist/remote-sync.d.ts

@@ -0,0 +1,116 @@
+export declare const REMOTE_SYNC_STATE_MODE: "remote_delta_v1";
+export declare const DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES = 1000000;
+export interface RemoteSyncUploadCandidate {
+    path: string;
+    content: string;
+    language?: string;
+}
+export interface RemoteSyncProjectFileStat {
+    path: string;
+    full_path: string;
+    size: number;
+    mtime_ms: number;
+    language?: string;
+}
+export interface RemoteSyncStateEntry {
+    content_hash: string;
+    size: number;
+    mtime_ms: number;
+    language?: string;
+}
+export interface RemoteSyncStateFile {
+    mode: typeof REMOTE_SYNC_STATE_MODE;
+    workspace_id?: string;
+    last_index_version?: string;
+    files: Record<string, RemoteSyncStateEntry>;
+    updated_at: string;
+}
+export interface RemoteSyncDeltaPayload {
+    upsert_files: RemoteSyncUploadCandidate[];
+    deleted_paths: string[];
+}
+export interface RemoteSyncDeltaBatch {
+    upsert_files: RemoteSyncUploadCandidate[];
+    deleted_paths: string[];
+    approx_bytes: number;
+}
+export interface BuildRemoteSyncDeltaResult {
+    delta: RemoteSyncDeltaPayload;
+    upsert_state_entries: Record<string, RemoteSyncStateEntry>;
+    next_files: Record<string, RemoteSyncStateEntry>;
+}
+export interface RemoteSyncScanOptions {
+    max_file_size_bytes?: number;
+    excluded_dirs?: Set<string>;
+    excluded_files?: Set<string>;
+    excluded_file_suffixes?: Set<string>;
+}
+export declare class RemoteSyncHttpResponseError extends Error {
+    readonly status: number;
+    readonly payload?: unknown | undefined;
+    constructor(message: string, status: number, payload?: unknown | undefined);
+}
+export interface RunRemoteDeltaSyncInput {
+    project_root_path: string;
+    scan_root_path?: string;
+    workspace_id?: string;
+    previous_state?: RemoteSyncStateFile;
+    force_full_upsert?: boolean;
+    max_body_bytes: number;
+    retries?: number;
+    initial_delay_ms?: number;
+    stale_base_error?: (error: unknown) => boolean;
+    persist_state?: (state: RemoteSyncStateFile) => Promise<void>;
+    push_delta: (request: {
+        workspace_id?: string;
+        project_root_path: string;
+        base_index_version?: string;
+        upsert_files: RemoteSyncUploadCandidate[];
+        deleted_paths: string[];
+    }) => Promise<{
+        workspace_id?: string;
+        index_version?: string;
+    }>;
+}
+export interface RunRemoteDeltaSyncResult {
+    state: RemoteSyncStateFile;
+    changed: boolean;
+    workspace_id?: string;
+    index_version?: string;
+    applied_delta: {
+        upsert_files: number;
+        deleted_paths: number;
+    };
+}
+export declare function collectProjectFileStats(project_root_path: string, options?: RemoteSyncScanOptions): Promise<Map<string, RemoteSyncProjectFileStat>>;
+export declare function collectUploadCandidates(project_root_path: string, options?: RemoteSyncScanOptions): Promise<RemoteSyncUploadCandidate[]>;
+export declare function buildRemoteSyncDeltaFromState(input: {
+    project_root_path: string;
+    previous_state?: RemoteSyncStateFile;
+    force_full_upsert: boolean;
+    options?: RemoteSyncScanOptions;
+}): Promise<BuildRemoteSyncDeltaResult>;
+export declare function estimateRemoteSyncDeltaRequestSize(input: {
+    project_root_path: string;
+    workspace_id?: string;
+    base_index_version?: string;
+    upsert_files: RemoteSyncUploadCandidate[];
+    deleted_paths: string[];
+}): number;
+export declare function splitRemoteSyncDeltaIntoBatches(input: {
+    project_root_path: string;
+    workspace_id?: string;
+    base_index_version?: string;
+    delta: RemoteSyncDeltaPayload;
+    max_body_bytes: number;
+}): RemoteSyncDeltaBatch[];
+export declare function readRemoteSyncState(path: string): Promise<RemoteSyncStateFile | undefined>;
+export declare function writeRemoteSyncState(path: string, state: RemoteSyncStateFile): Promise<void>;
+export declare function isStaleBaseIndexError(error: unknown): boolean;
+export declare function isDeltaUnsupportedError(error: unknown): boolean;
+export declare function retryWithBackoff<T>(input: {
+    fn: () => Promise<T>;
+    retries: number;
+    initial_delay_ms: number;
+}): Promise<T>;
+export declare function runRemoteDeltaSync(input: RunRemoteDeltaSyncInput): Promise<RunRemoteDeltaSyncResult>;

package/dist/remote-sync.js

@@ -0,0 +1,476 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+export const REMOTE_SYNC_STATE_MODE = "remote_delta_v1";
+export const DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES = 1_000_000;
+const DEFAULT_EXCLUDED_DIRS = new Set([
+    ".cache",
+    ".git",
+    ".next",
+    ".nuxt",
+    ".rce",
+    ".rce-tool",
+    ".svelte-kit",
+    ".turbo",
+    "build",
+    "coverage",
+    "dist",
+    "node_modules",
+    "out",
+    "target"
+]);
+const DEFAULT_EXCLUDED_FILES = new Set([
+    "bun.lock",
+    "bun.lockb",
+    "Cargo.lock",
+    "composer.lock",
+    "Gemfile.lock",
+    "package-lock.json",
+    "Pipfile.lock",
+    "pnpm-lock.yaml",
+    "yarn.lock"
+]);
+const DEFAULT_EXCLUDED_FILE_SUFFIXES = new Set([".min.js", ".min.css", ".map"]);
+export class RemoteSyncHttpResponseError extends Error {
+    status;
+    payload;
+    constructor(message, status, payload) {
+        super(message);
+        this.status = status;
+        this.payload = payload;
+    }
+}
+function nowIso() {
+    return new Date().toISOString();
+}
+function extensionToLanguage(path) {
+    if (path.endsWith(".ts") || path.endsWith(".tsx"))
+        return "typescript";
+    if (path.endsWith(".js") || path.endsWith(".jsx"))
+        return "javascript";
+    if (path.endsWith(".py"))
+        return "python";
+    if (path.endsWith(".go"))
+        return "go";
+    if (path.endsWith(".rs"))
+        return "rust";
+    if (path.endsWith(".java"))
+        return "java";
+    if (path.endsWith(".json"))
+        return "json";
+    if (path.endsWith(".md"))
+        return "markdown";
+    if (path.endsWith(".yml") || path.endsWith(".yaml"))
+        return "yaml";
+    return undefined;
+}
+function looksBinary(content) {
+    return content.includes("\0");
+}
+function sha256Text(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+function shouldExcludeFile(path, excludedFiles, excludedSuffixes) {
+    const lower = path.toLowerCase();
+    if (excludedFiles.has(path) || excludedFiles.has(lower)) {
+        return true;
+    }
+    for (const suffix of excludedSuffixes) {
+        if (lower.endsWith(suffix)) {
+            return true;
+        }
+    }
+    return false;
+}
+function resolveScanOptions(options) {
+    return {
+        max_file_size_bytes: options?.max_file_size_bytes ?? DEFAULT_REMOTE_SYNC_MAX_FILE_SIZE_BYTES,
+        excluded_dirs: options?.excluded_dirs ?? DEFAULT_EXCLUDED_DIRS,
+        excluded_files: options?.excluded_files ?? DEFAULT_EXCLUDED_FILES,
+        excluded_file_suffixes: options?.excluded_file_suffixes ?? DEFAULT_EXCLUDED_FILE_SUFFIXES
+    };
+}
+export async function collectProjectFileStats(project_root_path, options) {
+    const root = resolve(project_root_path);
+    const resolvedOptions = resolveScanOptions(options);
+    const output = new Map();
+    async function walk(dir) {
+        const entries = await readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isDirectory()) {
+                if (resolvedOptions.excluded_dirs.has(entry.name)) {
+                    continue;
+                }
+                await walk(join(dir, entry.name));
+                continue;
+            }
+            if (!entry.isFile()) {
+                continue;
+            }
+            if (shouldExcludeFile(entry.name, resolvedOptions.excluded_files, resolvedOptions.excluded_file_suffixes)) {
+                continue;
+            }
+            const fullPath = join(dir, entry.name);
+            const fileStat = await stat(fullPath);
+            if (fileStat.size > resolvedOptions.max_file_size_bytes) {
+                continue;
+            }
+            const repoPath = relative(root, fullPath).replace(/\\/g, "/");
+            output.set(repoPath, {
+                path: repoPath,
+                full_path: fullPath,
+                size: fileStat.size,
+                mtime_ms: Math.trunc(fileStat.mtimeMs),
+                language: extensionToLanguage(repoPath)
+            });
+        }
+    }
+    await walk(root);
+    return output;
+}
+export async function collectUploadCandidates(project_root_path, options) {
+    const stats = await collectProjectFileStats(project_root_path, options);
+    const output = [];
+    for (const repoPath of [...stats.keys()].sort((a, b) => a.localeCompare(b))) {
+        const fileStat = stats.get(repoPath);
+        let content;
+        try {
+            content = await readFile(fileStat.full_path, "utf8");
+        }
+        catch {
+            continue;
+        }
+        if (looksBinary(content)) {
+            continue;
+        }
+        output.push({
+            path: repoPath,
+            content,
+            ...(fileStat.language ? { language: fileStat.language } : {})
+        });
+    }
+    return output;
+}
+export async function buildRemoteSyncDeltaFromState(input) {
+    const projectFiles = await collectProjectFileStats(input.project_root_path, input.options);
+    const previousFiles = input.previous_state?.files ?? {};
+    const upsertFiles = [];
+    const deletedPaths = [];
+    const upsertStateEntries = {};
+    const nextFiles = {};
+    const sortedPaths = [...projectFiles.keys()].sort((a, b) => a.localeCompare(b));
+    for (const repoPath of sortedPaths) {
+        const projectFile = projectFiles.get(repoPath);
+        const previous = previousFiles[repoPath];
+        const statChanged = !previous ||
+            previous.size !== projectFile.size ||
+            previous.mtime_ms !== projectFile.mtime_ms ||
+            input.force_full_upsert;
+        if (!statChanged) {
+            nextFiles[repoPath] = previous;
+            continue;
+        }
+        let content;
+        try {
+            content = await readFile(projectFile.full_path, "utf8");
+        }
+        catch {
+            if (previous) {
+                nextFiles[repoPath] = previous;
+            }
+            continue;
+        }
+        if (looksBinary(content)) {
+            if (previous) {
+                nextFiles[repoPath] = previous;
+            }
+            continue;
+        }
+        const contentHash = sha256Text(content);
+        const nextEntry = {
+            content_hash: contentHash,
+            size: projectFile.size,
+            mtime_ms: projectFile.mtime_ms,
+            ...(projectFile.language ? { language: projectFile.language } : {})
+        };
+        nextFiles[repoPath] = nextEntry;
+        if (!input.force_full_upsert && previous?.content_hash === contentHash) {
+            continue;
+        }
+        upsertFiles.push({
+            path: repoPath,
+            content,
+            ...(projectFile.language ? { language: projectFile.language } : {})
+        });
+        upsertStateEntries[repoPath] = nextEntry;
+    }
+    for (const previousPath of Object.keys(previousFiles)) {
+        if (!projectFiles.has(previousPath)) {
+            deletedPaths.push(previousPath);
+        }
+    }
+    return {
+        delta: {
+            upsert_files: upsertFiles,
+            deleted_paths: deletedPaths
+        },
+        upsert_state_entries: upsertStateEntries,
+        next_files: nextFiles
+    };
+}
+export function estimateRemoteSyncDeltaRequestSize(input) {
+    const payload = {
+        project_root_path: input.project_root_path,
+        ...(input.workspace_id ? { workspace_id: input.workspace_id } : {}),
+        ...(input.base_index_version ? { base_index_version: input.base_index_version } : {}),
+        upsert_files: input.upsert_files.map((file) => ({
+            path: file.path,
+            content: file.content,
+            ...(file.language ? { language: file.language } : {})
+        })),
+        deleted_paths: input.deleted_paths
+    };
+    return Buffer.byteLength(JSON.stringify(payload), "utf8");
+}
+export function splitRemoteSyncDeltaIntoBatches(input) {
+    const budget = Math.max(256 * 1024, Math.floor(input.max_body_bytes * 0.8));
+    const upserts = [...input.delta.upsert_files].sort((a, b) => a.path.localeCompare(b.path));
+    const deleted = [...new Set(input.delta.deleted_paths)].sort((a, b) => a.localeCompare(b));
+    const batches = [];
+    let currentUpserts = [];
+    let currentDeleted = [];
+    const flush = () => {
+        if (currentUpserts.length === 0 && currentDeleted.length === 0) {
+            return;
+        }
+        batches.push({
+            upsert_files: currentUpserts,
+            deleted_paths: currentDeleted,
+            approx_bytes: estimateRemoteSyncDeltaRequestSize({
+                project_root_path: input.project_root_path,
+                workspace_id: input.workspace_id,
+                base_index_version: input.base_index_version,
+                upsert_files: currentUpserts,
+                deleted_paths: currentDeleted
+            })
+        });
+        currentUpserts = [];
+        currentDeleted = [];
+    };
+    const canFit = (nextUpserts, nextDeleted) => estimateRemoteSyncDeltaRequestSize({
+        project_root_path: input.project_root_path,
+        workspace_id: input.workspace_id,
+        base_index_version: input.base_index_version,
+        upsert_files: nextUpserts,
+        deleted_paths: nextDeleted
+    }) <= budget;
+    for (const file of upserts) {
+        const candidateUpserts = [...currentUpserts, file];
+        if (canFit(candidateUpserts, currentDeleted)) {
+            currentUpserts = candidateUpserts;
+            continue;
+        }
+        flush();
+        if (!canFit([file], [])) {
+            throw new Error(`delta upsert payload too large for path ${file.path}`);
+        }
+        currentUpserts = [file];
+    }
+    for (const path of deleted) {
+        const candidateDeleted = [...currentDeleted, path];
+        if (canFit(currentUpserts, candidateDeleted)) {
+            currentDeleted = candidateDeleted;
+            continue;
+        }
+        flush();
+        if (!canFit([], [path])) {
+            throw new Error(`delta delete payload too large for path ${path}`);
+        }
+        currentDeleted = [path];
+    }
+    flush();
+    return batches;
+}
+export async function readRemoteSyncState(path) {
+    if (!existsSync(path)) {
+        return undefined;
+    }
+    try {
+        const parsed = JSON.parse(await readFile(path, "utf8"));
+        if (!parsed ||
+            parsed.mode !== REMOTE_SYNC_STATE_MODE ||
+            typeof parsed.files !== "object" ||
+            parsed.files === null ||
+            Array.isArray(parsed.files)) {
+            return undefined;
+        }
+        return {
+            mode: REMOTE_SYNC_STATE_MODE,
+            workspace_id: parsed.workspace_id,
+            last_index_version: parsed.last_index_version,
+            files: parsed.files,
+            updated_at: typeof parsed.updated_at === "string" ? parsed.updated_at : nowIso()
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+export async function writeRemoteSyncState(path, state) {
+    await writeFile(path, JSON.stringify({
+        ...state,
+        mode: REMOTE_SYNC_STATE_MODE,
+        updated_at: nowIso()
+    }, null, 2));
+}
+export function isStaleBaseIndexError(error) {
+    if (!(error instanceof RemoteSyncHttpResponseError)) {
+        return false;
+    }
+    if (error.status !== 400) {
+        return false;
+    }
+    const message = error.message.toLowerCase();
+    return message.includes("invalid_argument") && message.includes("base index version");
+}
+export function isDeltaUnsupportedError(error) {
+    if (error instanceof RemoteSyncHttpResponseError) {
+        if ([404, 405, 501].includes(error.status)) {
+            return true;
+        }
+        const message = error.message.toLowerCase();
+        return message.includes("not_found") && message.includes("push-delta");
+    }
+    if (error instanceof Error) {
+        const message = error.message.toLowerCase();
+        return message.includes("push-delta") && (message.includes("not found") || message.includes("404"));
+    }
+    return false;
+}
+export async function retryWithBackoff(input) {
+    let attempt = 0;
+    let lastError;
+    while (attempt < input.retries) {
+        try {
+            return await input.fn();
+        }
+        catch (error) {
+            lastError = error;
+            attempt += 1;
+            if (attempt >= input.retries) {
+                break;
+            }
+            await new Promise((resolveSleep) => setTimeout(resolveSleep, input.initial_delay_ms * 2 ** (attempt - 1)));
+        }
+    }
+    throw lastError;
+}
+export async function runRemoteDeltaSync(input) {
+    const retries = input.retries ?? 3;
+    const initialDelayMs = input.initial_delay_ms ?? 500;
+    const isStaleError = input.stale_base_error ?? isStaleBaseIndexError;
+    const execute = async (previousState, forceFullUpsert) => {
+        const scanRootPath = input.scan_root_path ?? input.project_root_path;
+        const deltaBuild = await buildRemoteSyncDeltaFromState({
+            project_root_path: scanRootPath,
+            previous_state: previousState,
+            force_full_upsert: forceFullUpsert
+        });
+        const nextWorkspaceId = input.workspace_id ?? previousState?.workspace_id;
+        if (deltaBuild.delta.upsert_files.length === 0 && deltaBuild.delta.deleted_paths.length === 0) {
+            const unchangedState = {
+                mode: REMOTE_SYNC_STATE_MODE,
+                workspace_id: nextWorkspaceId,
+                last_index_version: previousState?.last_index_version,
+                files: deltaBuild.next_files,
+                updated_at: nowIso()
+            };
+            await input.persist_state?.(unchangedState);
+            return {
+                state: unchangedState,
+                changed: false,
+                workspace_id: unchangedState.workspace_id,
+                index_version: unchangedState.last_index_version,
+                applied_delta: {
+                    upsert_files: 0,
+                    deleted_paths: 0
+                }
+            };
+        }
+        let currentBaseIndexVersion = previousState?.last_index_version;
+        let currentWorkspaceId = nextWorkspaceId;
+        const progressFiles = { ...(previousState?.files ?? {}) };
+        let uploadedCount = 0;
+        let deletedCount = 0;
+        const batches = splitRemoteSyncDeltaIntoBatches({
+            project_root_path: input.project_root_path,
+            workspace_id: currentWorkspaceId,
+            base_index_version: currentBaseIndexVersion,
+            delta: deltaBuild.delta,
+            max_body_bytes: input.max_body_bytes
+        });
+        let latestState;
+        for (const batch of batches) {
+            const result = await retryWithBackoff({
+                retries,
+                initial_delay_ms: initialDelayMs,
+                fn: async () => input.push_delta({
+                    workspace_id: currentWorkspaceId,
+                    project_root_path: input.project_root_path,
+                    ...(currentBaseIndexVersion ? { base_index_version: currentBaseIndexVersion } : {}),
+                    upsert_files: batch.upsert_files,
+                    deleted_paths: batch.deleted_paths
+                })
+            });
+            currentWorkspaceId = result.workspace_id ?? currentWorkspaceId;
+            currentBaseIndexVersion = result.index_version ?? currentBaseIndexVersion;
+            for (const file of batch.upsert_files) {
+                const entry = deltaBuild.upsert_state_entries[file.path];
+                if (entry) {
+                    progressFiles[file.path] = entry;
+                }
+            }
+            for (const path of batch.deleted_paths) {
+                delete progressFiles[path];
+            }
+            uploadedCount += batch.upsert_files.length;
+            deletedCount += batch.deleted_paths.length;
+            latestState = {
+                mode: REMOTE_SYNC_STATE_MODE,
+                workspace_id: currentWorkspaceId,
+                last_index_version: currentBaseIndexVersion,
+                files: progressFiles,
+                updated_at: nowIso()
+            };
+            await input.persist_state?.(latestState);
+        }
+        const finalState = latestState ?? {
+            mode: REMOTE_SYNC_STATE_MODE,
+            workspace_id: currentWorkspaceId,
+            last_index_version: currentBaseIndexVersion,
+            files: progressFiles,
+            updated_at: nowIso()
+        };
+        return {
+            state: finalState,
+            changed: true,
+            workspace_id: currentWorkspaceId,
+            index_version: currentBaseIndexVersion,
+            applied_delta: {
+                upsert_files: uploadedCount,
+                deleted_paths: deletedCount
+            }
+        };
+    };
+    const forceFullUpsert = input.force_full_upsert ?? false;
+    try {
+        return await execute(input.previous_state, forceFullUpsert);
+    }
+    catch (error) {
+        if (!forceFullUpsert && input.previous_state && isStaleError(error)) {
+            return execute(undefined, true);
+        }
+        throw error;
+    }
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@rce-mcp/retrieval-core",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@rce-mcp/contracts": "0.1.0",
+    "@rce-mcp/data-plane": "0.1.0",
+    "@rce-mcp/observability": "0.1.0",
+    "tree-sitter": "^0.22.4",
+    "tree-sitter-go": "^0.23.4",
+    "tree-sitter-javascript": "^0.25.0",
+    "tree-sitter-python": "^0.25.0",
+    "tree-sitter-typescript": "^0.23.2"
+  },
+  "devDependencies": {
+    "pg-mem": "^3.0.5",
+    "tree-sitter-go-v025": "npm:tree-sitter-go@0.25.0",
+    "tree-sitter-javascript-v023": "npm:tree-sitter-javascript@0.23.1",
+    "tree-sitter-javascript-v025": "npm:tree-sitter-javascript@0.25.0",
+    "tree-sitter-python-v023": "npm:tree-sitter-python@0.23.6",
+    "tree-sitter-python-v025": "npm:tree-sitter-python@0.25.0",
+    "tree-sitter-v025": "npm:tree-sitter@0.25.0",
+    "web-tree-sitter": "^0.26.5"
+  }
+}