@fitlab-ai/agent-infra 0.9.6 → 0.9.7

package/dist/lib/sandbox/control/server.js

@@ -1,12 +1,15 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import { randomUUID } from 'node:crypto';
 import { getProcessStartTime } from '../../server/process-state.js';
 import { assertGitWorktreeBinding } from '../../git/worktree-identity.js';
-import { controlError, SANDBOX_CONTROL_STATUS_INTERVAL_MS } from './protocol.js';
+import { controlError, DEFAULT_SANDBOX_CONTROL_TIMING } from './protocol.js';
 import { prepareSandboxControlExecution } from './executor.js';
 import { appendSandboxControlAudit, atomicWriteJson, cleanupStaleSandboxControlLease, executionPath, readActiveLease, readExecution, terminateSandboxControlExecution, writeSandboxControlStatus } from './state.js';
 import { validateSandboxControlRequest } from './protocol.js';
-import { acquireSandboxControlBrokerStartup, isSandboxControlRootQuiescing, readSandboxControlManifest } from './lifecycle.js';
+import { inspectSandboxControlContainer } from './container-identity.js';
+import { acquireSandboxControlBrokerStartup, garbageCollectSandboxControlRoot, isSandboxControlRootQuiescing, readSandboxControlManifest } from './lifecycle.js';
+import { nextSandboxControlBackoff } from './timing.js';
 function assertRealDirectory(directory, parent) {
     const stat = fs.lstatSync(directory);
     if (!stat.isDirectory() || stat.isSymbolicLink())
@@ -73,10 +76,12 @@ function bindingReason(manifest) {
         return 'SANDBOX_WORKTREE_BINDING_LOST';
     }
 }
-function recoverProcessing(manifest) {
+function recoverProcessing(manifest, brokerOwns) {
     for (const entry of fs.readdirSync(manifest.processingDir, { withFileTypes: true })) {
         if (!entry.isDirectory() || !/^[a-f0-9-]{16,64}$/.test(entry.name))
             continue;
+        if (!brokerOwns())
+            return false;
         const descriptor = executionPath(manifest, entry.name);
         const existingResponse = responsePath(manifest, entry.name);
         if (fs.existsSync(descriptor)) {
@@ -84,6 +89,8 @@ function recoverProcessing(manifest) {
             if (!terminateSandboxControlExecution(execution)) {
                 throw new Error(`SANDBOX_CONTROL_EXECUTION_STILL_RUNNING: ${entry.name}`);
             }
+            if (!brokerOwns())
+                return false;
             appendSandboxControlAudit(manifest, 'orphan-tree-terminated', { requestId: entry.name });
             let terminal = false;
             if (fs.existsSync(existingResponse)) {
@@ -96,14 +103,22 @@ function recoverProcessing(manifest) {
                     terminal = false;
                 }
             }
-            if (!terminal)
+            if (!terminal) {
+                if (!brokerOwns())
+                    return false;
                 writeResponse(manifest, unknown(entry.name));
+            }
         }
         else {
+            if (!brokerOwns())
+                return false;
             writeResponse(manifest, notExecuted(entry.name));
         }
+        if (!brokerOwns())
+            return false;
         fs.rmSync(path.join(manifest.processingDir, entry.name), { recursive: true, force: true });
     }
+    return true;
 }
 function delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
@@ -111,19 +126,38 @@ function delay(ms) {
 export function sandboxControlSafeEnv(env = process.env) {
     return Object.fromEntries(Object.entries(env).filter(([key]) => !key.toUpperCase().startsWith('AGENT_INFRA_CONTROL_')));
 }
-export async function serveSandboxControl(manifestPath, signal = new AbortController().signal) {
+export async function serveSandboxControl(manifestPath, signal = new AbortController().signal, options = {}) {
+    const timing = options.timing ?? DEFAULT_SANDBOX_CONTROL_TIMING;
+    const inspectContainer = options.inspectContainer ?? ((value) => inspectSandboxControlContainer(value));
+    const bindingCheck = options.bindingCheck ?? bindingReason;
     const manifest = readSandboxControlManifest(manifestPath);
     const root = path.dirname(manifestPath);
     const startTime = getProcessStartTime(process.pid);
     if (!startTime)
         throw new Error('SANDBOX_CONTROL_BROKER_IDENTITY_UNAVAILABLE');
-    const releaseStartup = await acquireSandboxControlBrokerStartup(root, { pid: process.pid, startTime });
+    const brokerId = randomUUID();
+    const releaseStartup = await acquireSandboxControlBrokerStartup(root, { pid: process.pid, startTime, brokerId });
     const requestsDir = path.join(manifest.channelDir, 'requests');
     const responsesDir = path.join(manifest.channelDir, 'responses');
     const consumedDir = path.join(root, 'consumed');
-    const broker = { pid: process.pid, startTime };
+    const broker = {
+        version: 3,
+        pid: process.pid,
+        startTime,
+        brokerId,
+        token: manifest.token,
+        generation: manifest.generation
+    };
     const brokerPath = path.join(root, 'broker.json');
-    const brokerRecord = `${JSON.stringify({ version: 2, ...broker, token: manifest.token, generation: manifest.generation })}\n`;
+    const brokerRecord = `${JSON.stringify(broker)}\n`;
+    const brokerOwns = () => {
+        try {
+            return fs.readFileSync(brokerPath, 'utf8') === brokerRecord;
+        }
+        catch {
+            return false;
+        }
+    };
     try {
         for (const directory of [manifest.channelDir, requestsDir, responsesDir, consumedDir, manifest.publicStatusDir, manifest.processingDir]) {
             fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
@@ -147,11 +181,24 @@ export async function serveSandboxControl(manifestPath, signal = new AbortContro
     let active = null;
     let lastState = '';
     let lastStatusAt = 0;
+    let nextBindingCheckAt = 0;
+    let nextContainerHeartbeatAt = Date.now() + timing.containerHeartbeatMs;
+    let containerReasonCode = null;
+    let containerBackoffMs = timing.parkedBindingInitialMs;
+    let bindingReasonCode = null;
+    let bindingBackoffMs = timing.parkedBindingInitialMs;
     try {
+        if (!brokerOwns())
+            return;
         writeSandboxControlStatus(manifest, broker, 'starting', null, null);
+        if (!brokerOwns())
+            return;
         appendSandboxControlAudit(manifest, 'broker-start', { pid: broker.pid });
-        recoverProcessing(manifest);
+        if (!recoverProcessing(manifest, brokerOwns))
+            return;
         while (!signal.aborted) {
+            if (!brokerOwns())
+                break;
             let current;
             try {
                 current = readSandboxControlManifest(manifestPath);
@@ -161,22 +208,78 @@ export async function serveSandboxControl(manifestPath, signal = new AbortContro
             }
             if (current.token !== manifest.token || current.generation !== manifest.generation)
                 break;
+            const heartbeatNow = Date.now();
+            if (heartbeatNow >= nextContainerHeartbeatAt) {
+                let observation;
+                try {
+                    observation = await inspectContainer(manifest);
+                }
+                catch (error) {
+                    observation = { state: 'unknown', reason: controlError(error).code };
+                }
+                if (!brokerOwns())
+                    break;
+                if (observation.state === 'found') {
+                    containerReasonCode = null;
+                    containerBackoffMs = timing.parkedBindingInitialMs;
+                    nextContainerHeartbeatAt = Date.now() + timing.containerHeartbeatMs;
+                }
+                else if (observation.state === 'unknown') {
+                    containerReasonCode = 'SANDBOX_CONTROL_CONTAINER_UNKNOWN';
+                    appendSandboxControlAudit(manifest, 'container-heartbeat-unknown', { reason: observation.reason });
+                    const backoff = nextSandboxControlBackoff(containerBackoffMs, timing.slowCheckMs);
+                    nextContainerHeartbeatAt = Date.now() + backoff.delayMs;
+                    containerBackoffMs = backoff.nextDelayMs;
+                }
+                else {
+                    try {
+                        await garbageCollectSandboxControlRoot(root, {
+                            timeoutMs: timing.quiesceDeadlineMs,
+                            inspectContainer: (timeoutMs) => inspectContainer(manifest),
+                            selfOwner: broker
+                        });
+                        active = null;
+                        return;
+                    }
+                    catch (error) {
+                        if (!brokerOwns())
+                            break;
+                        containerReasonCode = 'SANDBOX_CONTROL_CONTAINER_ABSENT_GC_FAILED';
+                        appendSandboxControlAudit(manifest, 'container-gc-failed', { reason: controlError(error).code });
+                        const backoff = nextSandboxControlBackoff(containerBackoffMs, timing.slowCheckMs);
+                        nextContainerHeartbeatAt = Date.now() + backoff.delayMs;
+                        containerBackoffMs = backoff.nextDelayMs;
+                    }
+                }
+            }
             if (active?.settled) {
+                if (!brokerOwns())
+                    break;
                 if (active.result) {
+                    if (!brokerOwns())
+                        break;
                     writeResponse(manifest, {
                         version: 2, id: active.request.id, phase: 'completed', exitCode: active.result.exitCode,
                         stdout: active.result.stdout, stderr: active.result.stderr, error: null
                     });
                 }
                 else {
+                    if (!brokerOwns())
+                        break;
                     writeResponse(manifest, unknown(active.request.id));
                 }
+                if (!brokerOwns())
+                    break;
                 fs.rmSync(path.join(manifest.processingDir, active.request.id), { recursive: true, force: true });
                 active = null;
             }
             let reasonCode = null;
             try {
+                if (!brokerOwns())
+                    break;
                 if (cleanupStaleSandboxControlLease(manifest)) {
+                    if (!brokerOwns())
+                        break;
                     appendSandboxControlAudit(manifest, 'lease-stale-cleanup');
                 }
                 if (readActiveLease(manifest))
@@ -185,18 +288,32 @@ export async function serveSandboxControl(manifestPath, signal = new AbortContro
             catch {
                 reasonCode = 'SANDBOX_CONTROL_HANDOFF_ACTIVE';
             }
-            reasonCode ??= bindingReason(manifest);
+            const now = Date.now();
+            if (now >= nextBindingCheckAt) {
+                bindingReasonCode = bindingCheck(manifest);
+                if (bindingReasonCode) {
+                    const backoff = nextSandboxControlBackoff(bindingBackoffMs, timing.slowCheckMs);
+                    nextBindingCheckAt = now + backoff.delayMs;
+                    bindingBackoffMs = backoff.nextDelayMs;
+                }
+                else {
+                    nextBindingCheckAt = now + timing.slowCheckMs;
+                    bindingBackoffMs = timing.parkedBindingInitialMs;
+                }
+            }
+            reasonCode ??= containerReasonCode;
+            reasonCode ??= bindingReasonCode;
             const state = reasonCode ? 'parked' : active ? 'busy' : 'healthy';
             const stateKey = `${state}:${reasonCode ?? ''}:${active?.request.id ?? ''}`;
-            const now = Date.now();
-            if (stateKey !== lastState || now - lastStatusAt >= SANDBOX_CONTROL_STATUS_INTERVAL_MS) {
+            if (brokerOwns() && (stateKey !== lastState || now - lastStatusAt >= timing.controlTickMs)) {
                 writeSandboxControlStatus(manifest, broker, state, reasonCode, active?.request.id ?? null, now);
                 lastStatusAt = now;
             }
-            if (stateKey !== lastState) {
+            if (brokerOwns() && stateKey !== lastState) {
                 appendSandboxControlAudit(manifest, 'broker-state', { state, reasonCode, requestId: active?.request.id ?? null });
                 lastState = stateKey;
             }
+            let retiring = false;
             for (const name of fs.readdirSync(requestsDir).sort()) {
                 if (!/^[a-f0-9-]{16,64}\.json$/.test(name))
                     continue;
@@ -204,14 +321,26 @@ export async function serveSandboxControl(manifestPath, signal = new AbortContro
                 const source = path.join(requestsDir, name);
                 let claimed = null;
                 try {
+                    if (!brokerOwns()) {
+                        retiring = true;
+                        break;
+                    }
                     claimed = claimRequest(manifest, source, id);
+                    if (!brokerOwns()) {
+                        retiring = true;
+                        break;
+                    }
                     consumeRequest(consumedDir, id);
+                    if (!brokerOwns()) {
+                        retiring = true;
+                        break;
+                    }
                     if (reasonCode)
                         throw new Error(reasonCode);
                     if (active)
                         throw new Error('SANDBOX_CONTROL_BUSY');
                     const request = validateSandboxControlRequest(JSON.parse(fs.readFileSync(claimed, 'utf8')), manifest);
-                    if (bindingReason(manifest))
+                    if (bindingCheck(manifest))
                         throw new Error('SANDBOX_WORKTREE_BINDING_LOST');
                     if (readActiveLease(manifest))
                         throw new Error('SANDBOX_CONTROL_HANDOFF_ACTIVE');
@@ -221,43 +350,90 @@ export async function serveSandboxControl(manifestPath, signal = new AbortContro
                     const execution = { request, prepared, result: null, failure: null, settled: false };
                     active = execution;
                     prepared.completion.then((result) => { execution.result = result; execution.settled = true; }, (error) => { execution.failure = error; execution.settled = true; });
+                    if (!brokerOwns()) {
+                        prepared.terminate(false);
+                        active = null;
+                        retiring = true;
+                        continue;
+                    }
+                    if (!brokerOwns()) {
+                        prepared.terminate(false);
+                        active = null;
+                        retiring = true;
+                        continue;
+                    }
                     writeResponse(manifest, {
                         version: 2, id, phase: 'accepted', exitCode: null, stdout: '', stderr: '', error: null
                     });
+                    if (!brokerOwns()) {
+                        prepared.terminate(false);
+                        active = null;
+                        retiring = true;
+                        break;
+                    }
                     try {
-                        prepared.start();
+                        prepared.start(brokerOwns);
                     }
                     catch (error) {
-                        prepared.terminate();
+                        const owned = brokerOwns();
+                        prepared.terminate(owned);
+                        if (!owned || !brokerOwns()) {
+                            active = null;
+                            retiring = true;
+                            break;
+                        }
                         writeResponse(manifest, unknown(id));
+                        if (!brokerOwns()) {
+                            active = null;
+                            retiring = true;
+                            break;
+                        }
                         fs.rmSync(path.join(manifest.processingDir, id), { recursive: true, force: true });
                         active = null;
                         claimed = null;
+                        if (!brokerOwns()) {
+                            retiring = true;
+                            break;
+                        }
                         appendSandboxControlAudit(manifest, 'executor-gate-failed', { requestId: id });
                         continue;
                     }
                 }
                 catch (error) {
+                    if (!brokerOwns()) {
+                        retiring = true;
+                        break;
+                    }
                     writeResponse(manifest, rejected(id, error));
+                    if (!brokerOwns()) {
+                        retiring = true;
+                        break;
+                    }
                     if (claimed)
                         fs.rmSync(path.dirname(claimed), { recursive: true, force: true });
                 }
             }
-            await delay(25);
+            if (retiring || !brokerOwns())
+                break;
+            await delay(timing.controlTickMs);
         }
     }
     finally {
         if (active) {
-            active.prepared.terminate();
-            writeResponse(manifest, unknown(active.request.id));
+            const owned = brokerOwns();
+            active.prepared.terminate(owned);
+            if (owned && brokerOwns())
+                writeResponse(manifest, unknown(active.request.id));
         }
-        appendSandboxControlAudit(manifest, 'broker-stop', { pid: broker.pid });
-        try {
-            if (fs.readFileSync(brokerPath, 'utf8') === brokerRecord)
-                fs.unlinkSync(brokerPath);
-        }
-        catch {
-            // A newer owner or recovery path owns the record.
+        if (brokerOwns()) {
+            appendSandboxControlAudit(manifest, 'broker-stop', { pid: broker.pid, brokerId: broker.brokerId });
+            try {
+                if (fs.readFileSync(brokerPath, 'utf8') === brokerRecord)
+                    fs.unlinkSync(brokerPath);
+            }
+            catch {
+                // A newer owner or recovery path owns the record.
+            }
         }
     }
 }

package/dist/lib/sandbox/control/state.js

@@ -32,15 +32,17 @@ export function statusPath(manifest) {
 }
 export function writeSandboxControlStatus(manifest, broker, state, reasonCode, activeRequestId, now = Date.now()) {
     const status = {
-        version: 1, generation: manifest.generation, broker, state, reasonCode, activeRequestId, updatedAt: now
+        version: 2, generation: manifest.generation, broker, state, reasonCode, activeRequestId, updatedAt: now
     };
     atomicWriteJson(statusPath(manifest), status, 0o600, false);
     return status;
 }
 export function parseSandboxControlStatus(value) {
     const status = value;
-    if (!status || status.version !== 1 || typeof status.generation !== 'string'
-        || !status.broker || !Number.isSafeInteger(status.broker.pid) || typeof status.broker.startTime !== 'string'
+    if (!status || status.version !== 2 || typeof status.generation !== 'string'
+        || !status.broker || !Number.isSafeInteger(status.broker.pid) || typeof status.broker.startTime !== 'number'
+        || !Number.isSafeInteger(status.broker.startTime) || typeof status.broker.brokerId !== 'string'
+        || status.broker.brokerId.length === 0
         || !['starting', 'healthy', 'busy', 'parked'].includes(status.state ?? '')
         || (status.reasonCode !== null && typeof status.reasonCode !== 'string')
         || (status.activeRequestId !== null && typeof status.activeRequestId !== 'string')
@@ -56,9 +58,10 @@ export function readActiveLease(manifest, now = Date.now()) {
     if (!fs.existsSync(filePath))
         return null;
     const lease = readJsonFile(filePath);
-    if (!lease || lease.version !== 1
+    if (!lease || lease.version !== 2
         || typeof lease.nonce !== 'string' || !lease.owner || !Number.isSafeInteger(lease.owner.pid)
-        || typeof lease.owner.startTime !== 'string' || !Number.isSafeInteger(lease.issuedAt)
+        || typeof lease.owner.startTime !== 'number' || !Number.isSafeInteger(lease.owner.startTime)
+        || !Number.isSafeInteger(lease.issuedAt)
         || !Number.isSafeInteger(lease.expiresAt) || typeof lease.generation !== 'string'
         || (lease.taskId !== null && typeof lease.taskId !== 'string')
         || typeof lease.branch !== 'string' || typeof lease.reason !== 'string') {
@@ -75,9 +78,10 @@ export function cleanupStaleSandboxControlLease(manifest, now = Date.now()) {
         return false;
     const raw = fs.readFileSync(filePath, 'utf8');
     const lease = JSON.parse(raw);
-    if (!lease || lease.version !== 1
+    if (!lease || lease.version !== 2
         || typeof lease.nonce !== 'string' || !lease.owner || !Number.isSafeInteger(lease.owner.pid)
-        || typeof lease.owner.startTime !== 'string' || !Number.isSafeInteger(lease.issuedAt)
+        || typeof lease.owner.startTime !== 'number' || !Number.isSafeInteger(lease.owner.startTime)
+        || !Number.isSafeInteger(lease.issuedAt)
         || !Number.isSafeInteger(lease.expiresAt) || typeof lease.generation !== 'string'
         || (lease.taskId !== null && typeof lease.taskId !== 'string')
         || typeof lease.branch !== 'string' || typeof lease.reason !== 'string') {
@@ -85,7 +89,7 @@ export function cleanupStaleSandboxControlLease(manifest, now = Date.now()) {
     }
     const stale = lease.generation !== manifest.generation || lease.taskId !== manifest.taskId
         || lease.branch !== manifest.branch || lease.expiresAt <= now || !processIdentityMatches({
-        version: 1, pid: lease.owner.pid, startTime: lease.owner.startTime
+        pid: lease.owner.pid, startTime: lease.owner.startTime
     });
     if (!stale)
         return false;
@@ -109,9 +113,10 @@ export function executionPath(manifest, requestId) {
 }
 export function readExecution(filePath) {
     const execution = readJsonFile(filePath);
-    if (!execution || execution.version !== 1 || typeof execution.generation !== 'string'
+    if (!execution || execution.version !== 2 || typeof execution.generation !== 'string'
         || typeof execution.requestId !== 'string' || typeof execution.nonce !== 'string'
-        || !execution.child || !Number.isSafeInteger(execution.child.pid) || typeof execution.child.startTime !== 'string'
+        || !execution.child || !Number.isSafeInteger(execution.child.pid) || typeof execution.child.startTime !== 'number'
+        || !Number.isSafeInteger(execution.child.startTime)
         || !['prepared', 'running', 'terminating'].includes(execution.phase ?? '')
         || !Number.isSafeInteger(execution.updatedAt))
         throw new Error('SANDBOX_CONTROL_EXECUTION_INVALID');
@@ -140,7 +145,7 @@ export function terminateSandboxControlExecution(execution, options = {}) {
     };
     const treeAlive = () => {
         if (platform === 'win32' || !processGroupId) {
-            return processIdentityMatches({ version: 1, pid: execution.child.pid, startTime: execution.child.startTime });
+            return processIdentityMatches({ pid: execution.child.pid, startTime: execution.child.startTime });
         }
         return processGroupAlive(processGroupId);
     };
@@ -156,20 +161,25 @@ export function terminateSandboxControlExecution(execution, options = {}) {
     catch {
         // The identity check below distinguishes an already-exited tree from a failed termination.
     }
-    const deadline = Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS);
-    while (Date.now() < deadline) {
+    const deadline = options.deadlineAt ?? (Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS));
+    const forceAt = options.forceAt ?? (options.deadlineAt === undefined
+        ? deadline
+        : Math.min(deadline, Date.now() + Math.floor(Math.max(0, deadline - Date.now()) / 2)));
+    while (Date.now() < forceAt) {
         if (!treeAlive())
             return true;
         Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
     }
-    if (platform !== 'win32') {
+    if (platform !== 'win32' && options.allowForce !== false
+        && (options.deadlineAt === undefined || Date.now() < deadline)) {
         try {
             process.kill(processGroupId ? -processGroupId : execution.child.pid, 'SIGKILL');
         }
         catch {
             // The final identity check handles an already-exited tree.
         }
-        const killDeadline = Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS);
+        const killDeadline = options.deadlineAt
+            ?? (Date.now() + (options.timeoutMs ?? SANDBOX_CONTROL_EXECUTION_STOP_MS));
         while (Date.now() < killDeadline) {
             if (!treeAlive())
                 return true;

package/dist/lib/sandbox/control/timing.js

@@ -0,0 +1,8 @@
+export function nextSandboxControlBackoff(currentDelayMs, capMs) {
+    const delayMs = Math.min(currentDelayMs, capMs);
+    return {
+        delayMs,
+        nextDelayMs: Math.min(capMs, delayMs * 2)
+    };
+}
+//# sourceMappingURL=timing.js.map

package/dist/lib/sandbox/mounts.js

@@ -1,19 +1,7 @@
 import path from 'node:path';
 import { shareBranchDir, shareCommonDir, shellConfigDirCandidates, worktreeDirCandidates } from './constants.js';
-import { sandboxWorkspaceViewStatePaths } from './workspace-view.js';
 export function sandboxCoreBindMounts(config, branch, overrides) {
     const taskBound = Boolean(overrides.taskSource && overrides.taskId);
-    const workspaceMounts = sandboxWorkspaceViewStatePaths(overrides.workspaceViewRoot).map(({ state, hostPath }) => taskBound && state === 'active'
-        ? {
-            hostPaths: [path.join(hostPath, '.short-ids.json')],
-            containerPath: '/workspace/.agents/workspace/active/.short-ids.json',
-            readOnly: true
-        }
-        : {
-            hostPaths: [hostPath],
-            containerPath: path.posix.join('/workspace/.agents/workspace', state),
-            readOnly: true
-        });
     const mounts = [
         {
             hostPaths: overrides.worktree
@@ -22,7 +10,11 @@ export function sandboxCoreBindMounts(config, branch, overrides) {
             containerPath: '/workspace',
             readOnly: false
         },
-        ...workspaceMounts,
+        {
+            hostPaths: [overrides.workspaceViewRoot],
+            containerPath: '/workspace/.agents/workspace',
+            readOnly: true
+        },
     ];
     if (taskBound) {
         mounts.push({