crewx-bridge 0.2.7 → 0.2.8

package/dist/daemon.js

@@ -1,34 +1,29 @@
-import { execFile } from "node:child_process";
+import { randomBytes } from "node:crypto";
 import { hostname } from "node:os";
-import { promisify } from "node:util";
+import { probeAdapter } from "crewx-agent-cli/adapters";
 import { BridgeCommandPayloadSchema, } from "crewx-agent-protocol";
 import { BridgeApi } from "./api.js";
 import { BRIDGE_VERSION, ADAPTERS, DEFAULT_POLL_INTERVAL_MS } from "./constants.js";
+import { BridgeDaemonLock, } from "./daemon-lock.js";
 import { BridgeError, errorMessage, redactSecrets } from "./errors.js";
 import { folderDescriptors, verifyLaunchFolder } from "./folders.js";
 import { BridgeJournal } from "./journal.js";
 import { validateStartPolicy } from "./policy.js";
 import { BridgeRunLogs, publicRunDiagnostic } from "./run-logs.js";
 import { ProcessSupervisor } from "./supervisor.js";
-const execFileAsync = promisify(execFile);
-async function probeRuntime(adapter) {
-    try {
-        const result = await execFileAsync(adapter, ["--version"], {
-            encoding: "utf8",
-            timeout: 5_000,
-            maxBuffer: 64 * 1024,
-        });
-        const output = `${result.stdout}\n${result.stderr}`;
-        const version = output.match(/\b\d+(?:\.\d+){1,3}(?:[-+][0-9A-Za-z._-]+)?\b/)?.[0];
-        return { available: true, ...(version ? { version } : {}) };
-    }
-    catch {
-        return { available: false };
-    }
+const CAPABILITY_REFRESH_INTERVAL_MS = 5 * 60 * 1_000;
+const LEGACY_OUTCOME_RETRY_INTERVAL_MS = 5 * 60 * 1_000;
+const MAX_IDLE_CLAIM_DELAY_MS = 8_000;
+export function adaptiveIdleClaimDelay(serverMinimumMs, consecutiveEmptyClaims, random = Math.random) {
+    const serverMinimum = Math.max(250, Math.ceil(serverMinimumMs));
+    const exponential = Math.min(MAX_IDLE_CLAIM_DELAY_MS, DEFAULT_POLL_INTERVAL_MS *
+        2 ** Math.max(0, Math.min(8, consecutiveEmptyClaims - 1)));
+    const sampled = Math.floor(Math.max(0, Math.min(1, random())) * exponential);
+    return Math.max(serverMinimum, Math.min(MAX_IDLE_CLAIM_DELAY_MS, sampled));
 }
-export async function probeCapabilities() {
+export async function probeCapabilities(runtimeProbe = probeAdapter) {
     const runtimes = await Promise.all(ADAPTERS.map(async (adapter) => {
-        const result = await probeRuntime(adapter);
+        const result = await runtimeProbe(adapter);
         return {
             adapter,
             available: result.available,
@@ -45,12 +40,65 @@ export function policySummary(config) {
         max_concurrent_runs: config.policy.maxConcurrentRuns,
     };
 }
+export function bridgeHealth(journal) {
+    const quarantined = journal.quarantinedOutcomes(16);
+    const backlog = journal
+        .unreportedOutcomes(129)
+        .filter((outcome) => outcome.reconciliation?.compatibility !== "legacy_lease");
+    const issues = quarantined.map((outcome) => {
+        const status = outcome.reconciliation?.status;
+        const recovery = `Stop the service, then run \`crewx-bridge outcomes retry ${outcome.id} --yes\` or ` +
+            `\`crewx-bridge outcomes resolve ${outcome.id} --accept-server-state --yes\`, then start the service.`;
+        return {
+            code: status === 404
+                ? "outcome_command_missing"
+                : status === 409
+                    ? "outcome_reconciliation_conflict"
+                    : status === 401 || status === 403
+                        ? "outcome_auth_rejected"
+                        : "outcome_reconciliation_rejected",
+            message: status === 401 || status === 403
+                ? `CrewX rejected this machine credential. Re-enroll the machine from CrewX before retrying outcome ${outcome.id}. ${recovery}`
+                : `CrewX could not reconcile local outcome ${outcome.id}${status ? ` (HTTP ${status})` : ""}. New work is paused. ${recovery}`,
+            command_id: outcome.id,
+            occurred_at: outcome.reconciliation?.quarantinedAt ??
+                outcome.reconciliation?.lastAttemptAt ??
+                outcome.updatedAt,
+        };
+    });
+    if (backlog.length > 128 && issues.length < 16) {
+        issues.push({
+            code: "outcome_reconciliation_backlog",
+            message: "CrewX Bridge paused new work because too many completed commands are awaiting server confirmation.",
+            occurred_at: backlog[0]?.reconciliation?.lastAttemptAt ??
+                backlog[0]?.updatedAt ??
+                new Date().toISOString(),
+        });
+    }
+    else {
+        for (const outcome of backlog.slice(0, 16 - issues.length)) {
+            issues.push({
+                code: "outcome_reconciliation_pending",
+                message: "A completed local command is waiting for CrewX to confirm its outcome.",
+                command_id: outcome.id,
+                occurred_at: outcome.reconciliation?.lastAttemptAt ?? outcome.updatedAt,
+            });
+        }
+    }
+    return {
+        status: issues.length > 0 ? "degraded" : "healthy",
+        issues,
+    };
+}
 function commandResult(runStatus, pid) {
     return {
         run_status: runStatus,
         ...(pid !== undefined ? { pid } : {}),
     };
 }
+function isTerminalRunStatus(status) {
+    return status === "stopped" || status === "failed" || status === "lost";
+}
 export class BridgeDaemon {
     config;
     api;
@@ -59,8 +107,16 @@ export class BridgeDaemon {
     capabilities;
     log;
     runLogs;
+    capabilityProbe;
+    now;
+    random;
     lastHeartbeat = 0;
-    constructor(config, api, journal, supervisor, capabilities, log = () => undefined, runLogs = new BridgeRunLogs()) {
+    lastCapabilityProbe;
+    outcomeReconcileFailures = 0;
+    nextOutcomeReconcileAt = 0;
+    consecutiveEmptyClaims = 0;
+    nextClaimAt = 0;
+    constructor(config, api, journal, supervisor, capabilities, log = () => undefined, runLogs = new BridgeRunLogs(), capabilityProbe = probeCapabilities, now = Date.now, random = Math.random) {
         this.config = config;
         this.api = api;
         this.journal = journal;
@@ -68,17 +124,36 @@ export class BridgeDaemon {
         this.capabilities = capabilities;
         this.log = log;
         this.runLogs = runLogs;
+        this.capabilityProbe = capabilityProbe;
+        this.now = now;
+        this.random = random;
+        this.lastCapabilityProbe = this.now();
     }
     async initialize() {
         await this.journal.load();
         await this.api.syncFolders({ folders: folderDescriptors(this.config) });
+        await this.reconcileOutcomes(true);
         await this.heartbeat();
     }
     async heartbeat() {
+        if (this.now() - this.lastCapabilityProbe >=
+            CAPABILITY_REFRESH_INTERVAL_MS) {
+            this.lastCapabilityProbe = this.now();
+            try {
+                this.capabilities = await this.capabilityProbe();
+            }
+            catch (error) {
+                this.log(`Runtime capability refresh failed; retaining the last known inventory: ${redactSecrets(errorMessage(error))}.`);
+            }
+        }
         const runs = [];
         const reportedRunIds = new Set();
         for (const run of this.journal.activeRuns()) {
-            if (this.supervisor.isRunning(run.id, run.pid)) {
+            const supervised = this.supervisor.isRunning(run.id, run.pid) ||
+                Boolean(run.pid &&
+                    run.processNonce &&
+                    this.supervisor.adopt?.(run.id, run.pid, run.processNonce));
+            if (supervised) {
                 runs.push({
                     id: run.id,
                     status: run.status === "starting" ? "starting" : "running",
@@ -143,71 +218,279 @@ export class BridgeDaemon {
             capabilities: this.capabilities,
             policy: policySummary(this.config),
             runs,
+            health: bridgeHealth(this.journal),
         });
-        this.lastHeartbeat = Date.now();
+        this.lastHeartbeat = this.now();
     }
     async tick() {
-        if (Date.now() - this.lastHeartbeat >=
+        await this.reconcileOutcomes();
+        if (this.now() - this.lastHeartbeat >=
             this.config.heartbeatInterval * 1_000) {
             await this.heartbeat();
         }
+        if (this.journal.requiresClaimBackpressure()) {
+            return this.delayUntilHeartbeat(5_000);
+        }
+        const untilClaim = this.nextClaimAt - this.now();
+        if (untilClaim > 0) {
+            return this.delayUntilHeartbeat(untilClaim);
+        }
         const response = await this.api.claim();
-        if (!response.command)
-            return response.retry_after_ms;
+        if (!response.command) {
+            this.consecutiveEmptyClaims += 1;
+            const delay = adaptiveIdleClaimDelay(response.retry_after_ms, this.consecutiveEmptyClaims, this.random);
+            this.nextClaimAt = this.now() + delay;
+            return this.delayUntilHeartbeat(delay);
+        }
+        this.consecutiveEmptyClaims = 0;
+        this.nextClaimAt = 0;
         await this.execute(response.command);
         return 250;
     }
-    async execute(command) {
-        const existing = this.journal.getCommand(command.id);
-        if (existing?.state === "succeeded" && existing.result) {
-            await this.api.complete(command.id, {
-                lease_token: command.lease_token,
-                result: existing.result,
-            });
+    delayUntilHeartbeat(delay) {
+        const untilHeartbeat = this.lastHeartbeat +
+            this.config.heartbeatInterval * 1_000 -
+            this.now();
+        return Math.max(250, Math.min(delay, Math.max(250, untilHeartbeat)));
+    }
+    async reconcileOutcomes(force = false) {
+        if (!force && this.now() < this.nextOutcomeReconcileAt)
+            return;
+        const pending = this.journal.unreportedOutcomes(32);
+        if (pending.length === 0) {
+            this.outcomeReconcileFailures = 0;
+            this.nextOutcomeReconcileAt = 0;
             return;
         }
-        if (existing?.state === "failed") {
-            await this.api.fail(command.id, {
-                lease_token: command.lease_token,
-                code: existing.errorCode ?? "local_execution_failed",
-                message: "This command previously failed locally.",
-            });
+        let retainedFailure = false;
+        for (const outcome of pending) {
+            let request;
+            if (outcome.state === "succeeded") {
+                if (!outcome.result?.run_status) {
+                    retainedFailure = true;
+                    await this.journal.recordReconciliationFailure(outcome.id, {
+                        quarantined: true,
+                        errorCode: "invalid_local_outcome",
+                        errorMessage: "The durable local success is missing its run status and cannot be reconciled automatically.",
+                    });
+                    continue;
+                }
+                request = {
+                    outcome: "succeeded",
+                    result: {
+                        ...outcome.result,
+                        run_status: outcome.result.run_status,
+                    },
+                };
+            }
+            else {
+                request = {
+                    outcome: "failed",
+                    error: {
+                        code: outcome.errorCode &&
+                            /^[a-z0-9_]{1,80}$/.test(outcome.errorCode)
+                            ? outcome.errorCode
+                            : "local_execution_failed",
+                        message: publicRunDiagnostic(outcome.errorMessage ??
+                            "This command previously failed locally.") ||
+                            "The Bridge command failed locally. Check the protected runner log on the connected machine.",
+                    },
+                };
+            }
+            try {
+                await this.api.reconcileOutcome(outcome.id, request);
+                await this.journal.markCommandReported(outcome.id);
+            }
+            catch (error) {
+                const status = error instanceof BridgeError ? error.status : undefined;
+                const endpointUnsupported = status === 405 ||
+                    (error instanceof BridgeError &&
+                        error.serverCode === "endpoint_unsupported");
+                if (endpointUnsupported) {
+                    await this.journal.recordReconciliationFailure(outcome.id, {
+                        quarantined: false,
+                        status,
+                        errorCode: "outcome_endpoint_unsupported",
+                        errorMessage: "This CrewX server uses legacy lease-bound outcome reporting. The durable outcome will be sent when the command is redelivered.",
+                        compatibility: "legacy_lease",
+                    });
+                    this.nextOutcomeReconcileAt =
+                        this.now() + LEGACY_OUTCOME_RETRY_INTERVAL_MS;
+                    this.log(`CrewX does not expose durable outcome reconciliation yet; retaining ${outcome.id} for legacy lease redelivery.`);
+                    return;
+                }
+                retainedFailure = true;
+                const previousAttempts = outcome.reconciliation?.attempts ?? 0;
+                const quarantined = (error instanceof BridgeError && !error.retryable) ||
+                    (!(error instanceof BridgeError) && previousAttempts >= 7);
+                await this.journal.recordReconciliationFailure(outcome.id, {
+                    quarantined,
+                    ...(status !== undefined ? { status } : {}),
+                    errorCode: error instanceof BridgeError
+                        ? error.serverCode ?? error.code
+                        : "outcome_reconciliation_failed",
+                    errorMessage: publicRunDiagnostic(errorMessage(error)) ||
+                        "CrewX could not reconcile this command outcome.",
+                });
+                this.outcomeReconcileFailures += 1;
+                const retryCap = Math.min(30_000, 1_000 * 2 ** Math.min(5, this.outcomeReconcileFailures - 1));
+                const jitter = Math.floor(Math.random() * (retryCap + 1));
+                const retryAfter = error instanceof BridgeError ? error.retryAfterMs : undefined;
+                this.nextOutcomeReconcileAt =
+                    this.now() + Math.max(1_000, jitter, retryAfter ?? 0);
+                this.log(`CrewX has not confirmed durable command outcome ${outcome.id}; it remains queued for reconciliation: ${redactSecrets(errorMessage(error))}`);
+                if (!quarantined)
+                    return;
+            }
+        }
+        if (!retainedFailure) {
+            this.outcomeReconcileFailures = 0;
+            this.nextOutcomeReconcileAt = 0;
+        }
+    }
+    async execute(command) {
+        const existing = this.journal.getCommand(command.id);
+        if (existing?.state === "succeeded" || existing?.state === "failed") {
+            await this.reportOutcome(command);
             return;
         }
+        let parsedPayload;
         try {
-            const payload = BridgeCommandPayloadSchema.parse(command.payload);
-            if (payload.type === "start_agent") {
-                await this.start(command, payload);
+            parsedPayload = BridgeCommandPayloadSchema.parse(command.payload);
+            if (parsedPayload.type === "start_agent") {
+                await this.start(command, parsedPayload);
             }
             else {
-                await this.stop(command, payload.run_id);
+                await this.stop(command, parsedPayload.run_id);
             }
         }
         catch (error) {
+            const recorded = this.journal.getCommand(command.id);
+            if (recorded?.state === "succeeded" || recorded?.state === "failed") {
+                await this.reportOutcome(command);
+                return;
+            }
+            if (recorded?.state === "prepared") {
+                this.log(`Command ${command.id} is prepared locally; acknowledgement delivery is pending and will be retried after redelivery.`);
+                return;
+            }
+            if (parsedPayload &&
+                (await this.recoverAppliedOutcome(command, parsedPayload))) {
+                return;
+            }
             const code = error instanceof BridgeError ? error.code : "invalid_or_failed_command";
-            const message = redactSecrets(errorMessage(error), [
+            const message = publicRunDiagnostic(redactSecrets(errorMessage(error), [
                 command.lease_token,
                 command.secret?.agent_token ?? "",
-            ]).slice(0, 1000);
+            ])).slice(0, 1000) ||
+                "The Bridge command failed locally. Check the protected runner log on the connected machine.";
             await this.journal.recordCommand(command.id, {
                 state: "failed",
                 runId: command.payload.run_id,
                 type: command.payload.type,
                 errorCode: code,
+                errorMessage: message,
             });
-            await this.api.fail(command.id, {
-                lease_token: command.lease_token,
-                code,
-                message,
-            });
+            await this.reportOutcome(command);
             this.log(`Command ${command.id} failed: ${code}.`);
         }
     }
+    async reportOutcome(command) {
+        const outcome = this.journal.getCommand(command.id);
+        if (!outcome)
+            return;
+        try {
+            if (outcome.state === "succeeded" && outcome.result) {
+                await this.api.complete(command.id, {
+                    lease_token: command.lease_token,
+                    result: outcome.result,
+                });
+                await this.journal.markCommandReported(command.id);
+                return;
+            }
+            if (outcome.state === "failed") {
+                const message = publicRunDiagnostic(outcome.errorMessage ??
+                    "This command previously failed locally.") ||
+                    "The Bridge command failed locally. Check the protected runner log on the connected machine.";
+                await this.api.fail(command.id, {
+                    lease_token: command.lease_token,
+                    code: outcome.errorCode ?? "local_execution_failed",
+                    message,
+                });
+                await this.journal.markCommandReported(command.id);
+            }
+        }
+        catch (error) {
+            this.log(`Command ${command.id} has a durable local outcome, but CrewX has not confirmed it yet: ${redactSecrets(errorMessage(error), [command.lease_token])}`);
+        }
+    }
+    async recoverAppliedOutcome(command, payload) {
+        const recorded = this.journal.getCommand(command.id);
+        if (recorded?.state !== "accepted")
+            return false;
+        const run = this.journal.getRun(payload.run_id);
+        if (payload.type === "start_agent") {
+            const active = this.supervisor
+                .activeRuns()
+                .find((candidate) => candidate.id === payload.run_id);
+            if (!active || !this.supervisor.isRunning(payload.run_id, active.pid)) {
+                return false;
+            }
+            const result = commandResult("running", active.pid);
+            try {
+                await this.journal.recordRun(payload.run_id, {
+                    status: "running",
+                    pid: active.pid,
+                    ...(run?.processNonce
+                        ? { processNonce: run.processNonce }
+                        : {}),
+                    adapter: payload.adapter,
+                    folderId: payload.folder_id,
+                    startedAt: run?.startedAt ?? new Date().toISOString(),
+                });
+                await this.journal.recordCommand(command.id, {
+                    state: "succeeded",
+                    runId: payload.run_id,
+                    type: "start_agent",
+                    result,
+                });
+                await this.reportOutcome(command);
+            }
+            catch (journalError) {
+                this.log(`Started run ${payload.run_id}, but its durable completion could not be recorded: ${redactSecrets(errorMessage(journalError))}`);
+            }
+            return true;
+        }
+        if (this.supervisor.isRunning(payload.run_id, run?.pid))
+            return false;
+        const result = commandResult("stopped", run?.pid);
+        try {
+            if (run) {
+                await this.journal.recordRun(payload.run_id, {
+                    ...run,
+                    status: "stopped",
+                    endedAt: run.endedAt ?? new Date().toISOString(),
+                });
+            }
+            await this.journal.recordCommand(command.id, {
+                state: "succeeded",
+                runId: payload.run_id,
+                type: "stop_agent",
+                result,
+            });
+            await this.reportOutcome(command);
+        }
+        catch (journalError) {
+            this.log(`Stopped run ${payload.run_id}, but its durable completion could not be recorded: ${redactSecrets(errorMessage(journalError))}`);
+        }
+        return true;
+    }
     async start(command, rawPayload) {
         const payload = validateStartPolicy(this.config, rawPayload, this.supervisor.activeCount());
         if (!command.secret) {
             throw new BridgeError("missing_command_secret", "The start command does not include a run credential.");
         }
+        const folder = await verifyLaunchFolder(this.config, payload.folder_id);
         const existingRun = this.journal.getRun(payload.run_id);
         const existingCommand = this.journal.getCommand(command.id);
         if (existingCommand?.state === "accepted") {
@@ -220,39 +503,62 @@ export class BridgeDaemon {
                     type: "start_agent",
                     result,
                 });
-                await this.api.complete(command.id, {
-                    lease_token: command.lease_token,
+                await this.reportOutcome(command);
+                return;
+            }
+            if (existingRun && isTerminalRunStatus(existingRun.status)) {
+                const result = commandResult(existingRun.status, existingRun.pid);
+                await this.journal.recordCommand(command.id, {
+                    state: "succeeded",
+                    runId: payload.run_id,
+                    type: "start_agent",
                     result,
                 });
+                await this.reportOutcome(command);
                 return;
             }
-            throw new BridgeError("ambiguous_previous_start", "A prior start was accepted but its process cannot be verified; refusing to spawn a duplicate.");
+            if (!existingRun) {
+                // The remote acknowledgement was durable, but the local effect had not
+                // begun. It is safe to continue without spawning a duplicate.
+            }
+            else {
+                throw new BridgeError("ambiguous_previous_start", "A prior start was accepted but its process cannot be verified; refusing to spawn a duplicate.");
+            }
+        }
+        else {
+            await this.journal.recordCommand(command.id, {
+                state: "prepared",
+                runId: payload.run_id,
+                type: "start_agent",
+            });
+            await this.api.acknowledge(command.id, {
+                lease_token: command.lease_token,
+            });
+            await this.journal.recordCommand(command.id, {
+                state: "accepted",
+                runId: payload.run_id,
+                type: "start_agent",
+            });
         }
-        const folder = await verifyLaunchFolder(this.config, payload.folder_id);
-        await this.journal.recordCommand(command.id, {
-            state: "accepted",
-            runId: payload.run_id,
-            type: "start_agent",
-        });
-        await this.api.acknowledge(command.id, {
-            lease_token: command.lease_token,
-        });
         const startedAt = new Date().toISOString();
+        const processNonce = randomBytes(32).toString("base64url");
         await this.journal.recordRun(payload.run_id, {
             status: "starting",
             pid: null,
+            processNonce,
             adapter: payload.adapter,
             folderId: payload.folder_id,
             startedAt,
         });
         let pid;
         try {
-            pid = this.supervisor.start(payload, command.secret, folder);
+            pid = this.supervisor.start(payload, command.secret, folder, processNonce);
         }
         catch (error) {
             await this.journal.recordRun(payload.run_id, {
                 status: "failed",
                 pid: null,
+                processNonce,
                 adapter: payload.adapter,
                 folderId: payload.folder_id,
                 startedAt,
@@ -265,6 +571,7 @@ export class BridgeDaemon {
         await this.journal.recordRun(payload.run_id, {
             status: "running",
             pid,
+            processNonce,
             adapter: payload.adapter,
             folderId: payload.folder_id,
             startedAt,
@@ -276,10 +583,7 @@ export class BridgeDaemon {
             type: "start_agent",
             result,
         });
-        await this.api.complete(command.id, {
-            lease_token: command.lease_token,
-            result,
-        });
+        await this.reportOutcome(command);
         this.log(`Started ${payload.adapter} run ${payload.run_id}.`);
     }
     async stop(command, runId) {
@@ -293,23 +597,31 @@ export class BridgeDaemon {
                 type: "stop_agent",
                 result,
             });
-            await this.api.complete(command.id, {
+            await this.reportOutcome(command);
+            return;
+        }
+        if (existing?.state !== "accepted") {
+            await this.journal.recordCommand(command.id, {
+                state: "prepared",
+                runId,
+                type: "stop_agent",
+            });
+            await this.api.acknowledge(command.id, {
                 lease_token: command.lease_token,
-                result,
             });
-            return;
+            await this.journal.recordCommand(command.id, {
+                state: "accepted",
+                runId,
+                type: "stop_agent",
+            });
         }
-        await this.journal.recordCommand(command.id, {
-            state: "accepted",
-            runId,
-            type: "stop_agent",
-        });
-        await this.api.acknowledge(command.id, {
-            lease_token: command.lease_token,
-        });
         await this.supervisor.stop(runId, run?.pid);
         if (run) {
-            await this.journal.recordRun(runId, { ...run, status: "stopped" });
+            await this.journal.recordRun(runId, {
+                ...run,
+                status: "stopped",
+                endedAt: run.endedAt ?? new Date().toISOString(),
+            });
         }
         const result = commandResult("stopped", run?.pid);
         await this.journal.recordCommand(command.id, {
@@ -318,10 +630,7 @@ export class BridgeDaemon {
             type: "stop_agent",
             result,
         });
-        await this.api.complete(command.id, {
-            lease_token: command.lease_token,
-            result,
-        });
+        await this.reportOutcome(command);
         this.log(`Stopped run ${runId}.`);
     }
 }
@@ -332,70 +641,87 @@ export async function runBridge(options) {
     if (typeof process.getuid === "function" && process.getuid() === 0) {
         throw new BridgeError("root_refused", "CrewX Bridge refuses to run as root.");
     }
-    const api = options.api ?? new BridgeApi(options.config);
-    const journal = options.journal ?? new BridgeJournal();
-    const log = options.log ?? (() => undefined);
-    const runLogs = options.runLogs ?? new BridgeRunLogs();
-    const supervisor = options.supervisor ??
-        new ProcessSupervisor(options.config, undefined, undefined, ({ runId, pid, code, signal, expectedStop, error, errorCode, errorMessage: runErrorMessage, logPath, }) => {
-            const recorded = journal.getRun(runId);
-            if (recorded) {
-                void journal
-                    .recordRun(runId, {
-                    ...recorded,
-                    status: expectedStop || (!error && code === 0) ? "stopped" : "failed",
-                    endedAt: new Date().toISOString(),
-                    exitCode: code,
-                    signal,
-                    ...(errorCode ? { errorCode } : {}),
-                    ...(runErrorMessage ? { errorMessage: runErrorMessage } : {}),
-                })
-                    .catch((journalError) => {
-                    log(`Could not record the exit for agent run ${runId}: ${redactSecrets(errorMessage(journalError))}.`);
-                });
-            }
-            if (error) {
-                log(`Agent run ${runId} (PID ${pid}) failed: ${redactSecrets(runErrorMessage ?? errorMessage(error))}.${logPath ? ` Local log: ${logPath}` : ""}`);
-                return;
-            }
-            const result = signal
-                ? `signal ${signal}`
-                : `exit code ${code ?? "unknown"}`;
-            log(`Agent run ${runId} (PID ${pid}) exited with ${result}.${runErrorMessage ? ` ${runErrorMessage}` : ""}${logPath ? ` Local log: ${logPath}` : ""}`);
-        }, runLogs);
-    const capabilities = options.capabilities ?? (await probeCapabilities());
-    const daemon = new BridgeDaemon(options.config, api, journal, supervisor, capabilities, log, runLogs);
-    await daemon.initialize();
-    if (options.once) {
-        await daemon.tick();
-        return;
-    }
+    const daemonLock = options.daemonLock ?? new BridgeDaemonLock();
+    await daemonLock.acquire();
     try {
-        while (!options.signal?.aborted) {
-            let delay = DEFAULT_POLL_INTERVAL_MS;
-            try {
-                delay = await daemon.tick();
+        const api = options.api ?? new BridgeApi(options.config);
+        const journal = options.journal ?? new BridgeJournal();
+        const log = options.log ?? (() => undefined);
+        const runLogs = options.runLogs ?? new BridgeRunLogs();
+        const supervisor = options.supervisor ??
+            new ProcessSupervisor(options.config, undefined, undefined, ({ runId, pid, code, signal, expectedStop, error, errorCode, errorMessage: runErrorMessage, logPath, }) => {
+                const recorded = journal.getRun(runId);
+                if (recorded) {
+                    void journal
+                        .recordRun(runId, {
+                        ...recorded,
+                        status: expectedStop || (!error && code === 0)
+                            ? "stopped"
+                            : "failed",
+                        endedAt: new Date().toISOString(),
+                        exitCode: code,
+                        signal,
+                        ...(errorCode ? { errorCode } : {}),
+                        ...(runErrorMessage ? { errorMessage: runErrorMessage } : {}),
+                    })
+                        .catch((journalError) => {
+                        log(`Could not record the exit for agent run ${runId}: ${redactSecrets(errorMessage(journalError))}.`);
+                    });
+                }
+                if (error) {
+                    log(`Agent run ${runId} (PID ${pid}) failed: ${redactSecrets(runErrorMessage ?? errorMessage(error))}.${logPath ? ` Local log: ${logPath}` : ""}`);
+                    return;
+                }
+                const result = signal
+                    ? `signal ${signal}`
+                    : `exit code ${code ?? "unknown"}`;
+                log(`Agent run ${runId} (PID ${pid}) exited with ${result}.${runErrorMessage ? ` ${runErrorMessage}` : ""}${logPath ? ` Local log: ${logPath}` : ""}`);
+            }, runLogs);
+        const capabilities = options.capabilities ?? (await probeCapabilities());
+        const daemon = new BridgeDaemon(options.config, api, journal, supervisor, capabilities, log, runLogs);
+        await daemon.initialize();
+        if (options.once) {
+            await daemon.tick();
+            return;
+        }
+        let consecutiveFailures = 0;
+        try {
+            while (!options.signal?.aborted) {
+                let delay = DEFAULT_POLL_INTERVAL_MS;
+                try {
+                    delay = await daemon.tick();
+                    consecutiveFailures = 0;
+                }
+                catch (error) {
+                    options.log?.(`Control loop error: ${redactSecrets(errorMessage(error))}`);
+                    consecutiveFailures += 1;
+                    const retryCap = Math.min(30_000, DEFAULT_POLL_INTERVAL_MS * 2 ** Math.min(8, consecutiveFailures));
+                    const jitter = Math.floor(Math.random() * (retryCap + 1));
+                    const serverDelay = error instanceof BridgeError ? error.retryAfterMs : undefined;
+                    delay =
+                        error instanceof BridgeError && !error.retryable
+                            ? Math.max(30_000, serverDelay ?? 0)
+                            : Math.max(250, jitter, serverDelay ?? 0);
+                }
+                await new Promise((resolve) => {
+                    const timer = setTimeout(resolve, delay);
+                    options.signal?.addEventListener("abort", () => {
+                        clearTimeout(timer);
+                        resolve();
+                    }, { once: true });
+                });
             }
-            catch (error) {
-                options.log?.(`Control loop error: ${redactSecrets(errorMessage(error))}`);
-                delay = Math.min(30_000, DEFAULT_POLL_INTERVAL_MS * 2);
+        }
+        finally {
+            const activeRuns = supervisor.activeCount();
+            if (activeRuns > 0) {
+                log(`Bridge is stopping ${activeRuns} supervised agent ${activeRuns === 1 ? "run" : "runs"}. Relaunch them from CrewX after the service returns.`);
             }
-            const jitter = Math.floor(Math.random() * Math.min(500, delay / 4));
-            await new Promise((resolve) => {
-                const timer = setTimeout(resolve, delay + jitter);
-                options.signal?.addEventListener("abort", () => {
-                    clearTimeout(timer);
-                    resolve();
-                }, { once: true });
-            });
+            await supervisor.stopAll();
         }
     }
     finally {
-        const activeRuns = supervisor.activeCount();
-        if (activeRuns > 0) {
-            log(`Bridge is stopping ${activeRuns} supervised agent ${activeRuns === 1 ? "run" : "runs"}. Relaunch them from CrewX after the service returns.`);
-        }
-        await supervisor.stopAll();
+        await daemonLock.release();
     }
 }
 export function supportedAdapters() {