@fitlab-ai/agent-infra 0.9.5 → 0.9.7

package/dist/lib/sandbox/commands/start.js

@@ -8,9 +8,9 @@ const USAGE = `Usage: ai sandbox start [--recreate] <branch | TASK-id | N>
 
 Start an existing sandbox container that has stopped (for example after the
 Docker daemon was restarted or replaced). The container must already exist:
-if none is found, run 'ai sandbox create <branch>' first. A container that is
-already running is checked without modifying healthy runtime state. --recreate
-allows container-only replacement after in-place recovery fails.`;
+if none is found, run 'ai sandbox create <branch>' first. Without --recreate, a
+container that is already running is checked without modifying healthy runtime
+state. --recreate replaces the existing container even when it is healthy.`;
 export function parseStartArgs(args) {
     let recreate = false;
     let help = false;
@@ -62,6 +62,7 @@ export async function start(args) {
         workspace: target.workspace,
         row: found,
         allowRecreate: parsed.recreate,
+        forceRecreate: parsed.recreate,
         recreate: async () => {
             const { create } = await import('./create.js');
             await create([target.requestedRef, '--no-refresh']);

package/dist/lib/sandbox/control/client.js

@@ -1,15 +1,75 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { randomUUID } from 'node:crypto';
-import { SANDBOX_CONTROL_MAX_BYTES, isSandboxControlFamily } from './protocol.js';
+import { SANDBOX_CONTROL_ADMISSION_WINDOW_MS, SANDBOX_CONTROL_MAX_BYTES, SANDBOX_CONTROL_STATUS_STALE_MS, isSandboxControlFamily } from './protocol.js';
+import { readSandboxControlStatus } from './state.js';
+const SANDBOX_CONTROL_RESPONSE_SETTLE_MS = 250;
+export class SandboxControlClientError extends Error {
+    detail;
+    accepted;
+    constructor(detail, accepted = false) {
+        super(detail.message);
+        this.name = 'SandboxControlClientError';
+        this.detail = detail;
+        this.accepted = accepted;
+    }
+}
 function sleep(milliseconds) {
     Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
 }
+function clientError(code, message, retryable, accepted = false) {
+    throw new SandboxControlClientError({ code, message: `${code}: ${message}`, retryable }, accepted);
+}
+function preflight(statusDir, generation, now = Date.now()) {
+    let status;
+    try {
+        status = readSandboxControlStatus(statusDir);
+    }
+    catch {
+        clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker status is missing or invalid', true);
+    }
+    if (status.generation !== generation) {
+        clientError('SANDBOX_CONTROL_GENERATION_INVALID', 'broker status generation does not match this sandbox', false);
+    }
+    if (now - status.updatedAt > SANDBOX_CONTROL_STATUS_STALE_MS || status.updatedAt > now + 1_000) {
+        clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker status heartbeat is stale', true);
+    }
+    if (status.state === 'busy')
+        clientError('SANDBOX_CONTROL_BUSY', 'broker is executing another request', true);
+    if (status.state === 'parked') {
+        const code = status.reasonCode ?? 'SANDBOX_CONTROL_BROKER_UNAVAILABLE';
+        clientError(code, 'broker is parked until the host restores a valid binding', true);
+    }
+    if (status.state !== 'healthy')
+        clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker is not ready', true);
+}
+function parseResponse(raw, id) {
+    const response = JSON.parse(raw);
+    if (response.version !== 2 || response.id !== id
+        || !['accepted', 'completed', 'rejected'].includes(response.phase)
+        || (response.phase === 'completed' && !Number.isInteger(response.exitCode))) {
+        clientError('SANDBOX_CONTROL_RESPONSE_INVALID', 'broker response is invalid', false);
+    }
+    return response;
+}
+function cancelPendingRequest(requestPath) {
+    try {
+        fs.unlinkSync(requestPath);
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return false;
+        throw error;
+    }
+}
 function exchangeSandboxControl(request, params) {
     const channelDir = params.channelDir ?? process.env.AGENT_INFRA_CONTROL_DIR ?? '/run/agent-infra/control';
+    const statusDir = params.statusDir ?? process.env.AGENT_INFRA_CONTROL_STATUS_DIR ?? '/run/agent-infra/control-status';
+    preflight(statusDir, request.generation);
     const encoded = `${JSON.stringify(request)}\n`;
     if (Buffer.byteLength(encoded, 'utf8') > SANDBOX_CONTROL_MAX_BYTES) {
-        throw new Error('SANDBOX_CONTROL_REQUEST_TOO_LARGE: request exceeds the control limit');
+        clientError('SANDBOX_CONTROL_REQUEST_TOO_LARGE', 'request exceeds the control limit', false);
     }
     const requestsDir = path.join(channelDir, 'requests');
     const responsesDir = path.join(channelDir, 'responses');
@@ -19,39 +79,95 @@ function exchangeSandboxControl(request, params) {
     fs.writeFileSync(temporary, encoded, { mode: 0o600, flag: 'wx' });
     fs.renameSync(temporary, requestPath);
     const deadline = Date.now() + (params.timeoutMs ?? 30_000);
+    let accepted = false;
+    let malformedResponseRaw = null;
+    let malformedResponseObservedAt = 0;
     while (Date.now() < deadline) {
         if (fs.existsSync(responsePath)) {
-            const raw = fs.readFileSync(responsePath, 'utf8');
-            fs.unlinkSync(responsePath);
-            const response = JSON.parse(raw);
-            if (response.version !== 1 || response.id !== request.id || !Number.isInteger(response.exitCode)) {
-                throw new Error('SANDBOX_CONTROL_RESPONSE_INVALID: broker response is invalid');
+            let raw;
+            try {
+                raw = fs.readFileSync(responsePath, 'utf8');
+            }
+            catch (error) {
+                if (error.code === 'ENOENT')
+                    continue;
+                throw error;
+            }
+            let response;
+            try {
+                response = parseResponse(raw, request.id);
+                malformedResponseRaw = null;
+            }
+            catch (error) {
+                if (!(error instanceof SyntaxError))
+                    throw error;
+                const now = Date.now();
+                if (malformedResponseRaw !== raw) {
+                    malformedResponseRaw = raw;
+                    malformedResponseObservedAt = now;
+                }
+                else if (now - malformedResponseObservedAt >= SANDBOX_CONTROL_RESPONSE_SETTLE_MS) {
+                    clientError('SANDBOX_CONTROL_RESPONSE_INVALID', 'broker response remained malformed', false, accepted);
+                }
+                sleep(25);
+                continue;
+            }
+            if (response.phase === 'accepted') {
+                accepted = true;
+            }
+            else {
+                fs.rmSync(responsePath, { force: true });
+                return response;
+            }
+        }
+        if (!accepted) {
+            try {
+                preflight(statusDir, request.generation);
+            }
+            catch (error) {
+                if (cancelPendingRequest(requestPath))
+                    throw error;
+                clientError('SANDBOX_CONTROL_RESULT_UNKNOWN', 'request was claimed before broker availability was lost; inspect domain state before retrying', false, true);
             }
-            return response;
         }
         sleep(25);
     }
-    throw new Error(`SANDBOX_CONTROL_TIMEOUT: broker did not respond within ${params.timeoutMs ?? 30_000}ms`);
+    if (accepted) {
+        clientError('SANDBOX_CONTROL_RESULT_UNKNOWN', 'accepted request did not produce a final result; inspect domain state before retrying', false, true);
+    }
+    if (!cancelPendingRequest(requestPath)) {
+        clientError('SANDBOX_CONTROL_RESULT_UNKNOWN', 'request was claimed but no acceptance result was observed; inspect domain state before retrying', false, true);
+    }
+    clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', `broker did not accept the request within ${params.timeoutMs ?? 30_000}ms`, true);
+}
+function authority(params) {
+    const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
+    const generation = params.generation ?? process.env.AGENT_INFRA_CONTROL_GENERATION;
+    if (!token || !generation)
+        clientError('SANDBOX_CONTROL_UNAVAILABLE', 'control authority is missing', false);
+    return { token, generation };
 }
 export function requestSandboxControl(params) {
-    if (!isSandboxControlFamily(params.family)) {
-        throw new Error(`SANDBOX_CONTROL_COMMAND_DENIED: '${params.family}' is not allowed`);
-    }
+    if (!isSandboxControlFamily(params.family))
+        clientError('SANDBOX_CONTROL_COMMAND_DENIED', `'${params.family}' is not allowed`, false);
     if (params.family === 'task-create')
-        throw new Error("SANDBOX_CONTROL_COMMAND_DENIED: 'task-create' requires a typed candidate");
-    const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
-    if (!token)
-        throw new Error('SANDBOX_CONTROL_UNAVAILABLE: control token is missing');
-    const id = randomUUID();
-    const request = { version: 1, id, token, family: params.family, args: params.args };
+        clientError('SANDBOX_CONTROL_COMMAND_DENIED', "'task-create' requires a typed candidate", false);
+    const auth = authority(params);
+    const issuedAt = Date.now();
+    const request = {
+        version: 2, id: randomUUID(), ...auth, issuedAt,
+        expiresAt: issuedAt + SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
+        family: params.family, args: params.args
+    };
     return exchangeSandboxControl(request, params);
 }
 export function requestSandboxTaskCreate(params) {
-    const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
-    if (!token)
-        throw new Error('SANDBOX_CONTROL_UNAVAILABLE: control token is missing');
+    const auth = authority(params);
+    const issuedAt = Date.now();
     const request = {
-        version: 1, id: randomUUID(), token, family: 'task-create', candidate: params.candidate
+        version: 2, id: randomUUID(), ...auth, issuedAt,
+        expiresAt: issuedAt + SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
+        family: 'task-create', candidate: params.candidate
     };
     return exchangeSandboxControl(request, { ...params, timeoutMs: params.timeoutMs ?? 120_000 });
 }

package/dist/lib/sandbox/control/container-identity.js

@@ -0,0 +1,64 @@
+import { commandForEngine, runEngine, runProbe } from '../shell.js';
+function isAuthoritativeNotFound(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return /(?:no such container|container .* not found|cannot find container)/i.test(message);
+}
+export function classifySandboxContainerInspection(expected, inspection) {
+    if ('notFound' in inspection)
+        return { state: 'absent', id: expected.id };
+    if ('error' in inspection) {
+        return isAuthoritativeNotFound(inspection.error)
+            ? { state: 'absent', id: expected.id }
+            : { state: 'unknown', reason: `container inspection failed: ${inspection.error instanceof Error ? inspection.error.message : String(inspection.error)}` };
+    }
+    const response = inspection;
+    if (typeof response.id !== 'string' || response.id !== expected.id
+        || typeof response.running !== 'boolean' || !response.labels
+        || typeof response.labels !== 'object' || Array.isArray(response.labels)) {
+        return { state: 'unknown', reason: 'container inspection response is incomplete' };
+    }
+    const labels = response.labels;
+    for (const [key, value] of Object.entries(expected.labels)) {
+        if (labels[key] !== value)
+            return { state: 'unknown', reason: 'container labels do not match the manifest' };
+    }
+    return {
+        state: 'found',
+        id: response.id,
+        running: response.running,
+        labels: Object.fromEntries(Object.entries(labels).flatMap(([key, value]) => typeof value === 'string' ? [[key, value]] : []))
+    };
+}
+export async function inspectSandboxControlContainer(manifest, options = {}) {
+    try {
+        const args = ['inspect', '--format', '{{json .}}', manifest.containerIdentity.id];
+        let raw;
+        if (options.run) {
+            raw = options.run(manifest.engine, 'docker', args, options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs });
+        }
+        else {
+            const command = commandForEngine(manifest.engine, 'docker', args);
+            const result = (options.probe ?? runProbe)(command.cmd, command.args, options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs });
+            if (result.status !== 0) {
+                const stderr = typeof result.stderr === 'string' ? result.stderr : result.stderr?.toString('utf8') ?? '';
+                return classifySandboxContainerInspection(manifest.containerIdentity, {
+                    error: new Error(stderr || `container inspection exited with ${result.status ?? 'unknown'}`)
+                });
+            }
+            raw = typeof result.stdout === 'string' ? result.stdout : result.stdout?.toString('utf8') ?? '';
+        }
+        const parsed = JSON.parse(raw);
+        const value = Array.isArray(parsed) && parsed.length === 1 ? parsed[0] : parsed;
+        if (!value || Array.isArray(value))
+            return { state: 'unknown', reason: 'container inspection response is incomplete' };
+        return classifySandboxContainerInspection(manifest.containerIdentity, {
+            id: value.Id,
+            running: value.State?.Running,
+            labels: value.Config?.Labels
+        });
+    }
+    catch (error) {
+        return classifySandboxContainerInspection(manifest.containerIdentity, { error });
+    }
+}
+//# sourceMappingURL=container-identity.js.map

package/dist/lib/sandbox/control/executor.js

@@ -0,0 +1,142 @@
+import { spawn, spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { getProcessStartTime } from '../../server/process-state.js';
+import { createTask } from '../../task/create-service.js';
+import { bindSandboxControlTask } from './protocol.js';
+import { atomicWriteJson, executionPath, terminateSandboxControlExecution } from './state.js';
+function safeEnv(env) {
+    return Object.fromEntries(Object.entries(env).filter(([key]) => !key.toUpperCase().startsWith('AGENT_INFRA_CONTROL_')));
+}
+export function nodeEntryArgs(entry, args) {
+    return path.extname(entry) === '.ts'
+        ? ['--experimental-strip-types', '--no-warnings', entry, ...args]
+        : [entry, ...args];
+}
+async function waitForStartTime(pid, timeoutMs = 2_000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const startTime = getProcessStartTime(pid);
+        if (startTime !== null)
+            return startTime;
+        await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+    throw new Error('SANDBOX_CONTROL_EXECUTOR_IDENTITY_UNAVAILABLE');
+}
+export async function prepareSandboxControlExecution(params) {
+    const nonce = randomUUID();
+    const child = spawn(process.execPath, nodeEntryArgs(params.internalCliPath, [
+        'sandbox-control', 'execute', '--request', params.requestPath, '--nonce', nonce
+    ]), {
+        cwd: params.manifest.repoRoot,
+        detached: process.platform !== 'win32',
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: { ...safeEnv(process.env), AGENT_INFRA_EXECUTOR_MANIFEST: params.manifestPath }
+    });
+    if (typeof child.pid !== 'number')
+        throw new Error('SANDBOX_CONTROL_EXECUTOR_SPAWN_FAILED');
+    let stdout = '';
+    let stderr = '';
+    child.stdout?.setEncoding('utf8');
+    child.stderr?.setEncoding('utf8');
+    child.stdout?.on('data', (chunk) => { stdout += chunk; });
+    child.stderr?.on('data', (chunk) => { stderr += chunk; });
+    const completion = new Promise((resolve, reject) => {
+        child.once('error', reject);
+        child.once('close', (code) => resolve({ exitCode: code ?? 1, stdout, stderr }));
+    });
+    const startTime = await waitForStartTime(child.pid);
+    const execution = {
+        version: 2,
+        generation: params.manifest.generation,
+        requestId: params.request.id,
+        nonce,
+        child: {
+            pid: child.pid,
+            startTime,
+            processGroupId: process.platform === 'win32' ? null : child.pid
+        },
+        phase: 'prepared',
+        updatedAt: Date.now()
+    };
+    atomicWriteJson(executionPath(params.manifest, params.request.id), execution);
+    return {
+        execution,
+        start(canWrite = () => true) {
+            if (!child.connected)
+                throw new Error('SANDBOX_CONTROL_EXECUTOR_GATE_CLOSED');
+            if (!canWrite())
+                throw new Error('SANDBOX_CONTROL_OWNER_LOST');
+            atomicWriteJson(executionPath(params.manifest, params.request.id), { ...execution, phase: 'running', updatedAt: Date.now() });
+            if (!canWrite())
+                throw new Error('SANDBOX_CONTROL_OWNER_LOST');
+            child.send({ version: 1, nonce });
+        },
+        completion,
+        terminate(updateState = true) {
+            if (updateState) {
+                atomicWriteJson(executionPath(params.manifest, params.request.id), {
+                    ...execution, phase: 'terminating', updatedAt: Date.now()
+                });
+            }
+            return terminateSandboxControlExecution(execution);
+        }
+    };
+}
+function waitForGate(nonce, timeoutMs = 2_000) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_TIMEOUT')), timeoutMs);
+        const onDisconnect = () => {
+            clearTimeout(timer);
+            reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_CLOSED'));
+        };
+        process.once('disconnect', onDisconnect);
+        process.once('message', (message) => {
+            clearTimeout(timer);
+            process.off('disconnect', onDisconnect);
+            const value = message;
+            if (!value || value.version !== 1 || value.nonce !== nonce) {
+                reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_INVALID'));
+                return;
+            }
+            resolve();
+        });
+    });
+}
+function executeRequest(manifest, request) {
+    if (request.family === 'task-create') {
+        const result = createTask(request.candidate, { repoRoot: manifest.repoRoot });
+        return {
+            exitCode: result.status === 'blocked' ? 2 : result.status === 'failed' ? 1 : 0,
+            stdout: `${JSON.stringify(result)}\n`,
+            stderr: ''
+        };
+    }
+    const boundArgs = bindSandboxControlTask(request, manifest.taskId);
+    if (request.family === 'task-orchestration')
+        boundArgs.push('--git-worktree-root', manifest.worktreeRoot);
+    const result = spawnSync(process.execPath, nodeEntryArgs(process.argv[1], [request.family, ...boundArgs]), { cwd: manifest.repoRoot, encoding: 'utf8', env: safeEnv(process.env) });
+    return { exitCode: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
+}
+export async function runSandboxControlExecutor(requestPath, nonce) {
+    await waitForGate(nonce);
+    process.disconnect?.();
+    const root = fs.realpathSync.native(process.cwd());
+    const request = JSON.parse(fs.readFileSync(requestPath, 'utf8'));
+    const manifestPath = process.env.AGENT_INFRA_EXECUTOR_MANIFEST;
+    if (!manifestPath)
+        throw new Error('SANDBOX_CONTROL_EXECUTOR_MANIFEST_MISSING');
+    const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+    if (fs.realpathSync.native(manifest.repoRoot) !== root)
+        throw new Error('SANDBOX_CONTROL_EXECUTOR_ROOT_INVALID');
+    const result = executeRequest(manifest, request);
+    process.stdout.write(result.stdout);
+    process.stderr.write(result.stderr);
+    process.exitCode = result.exitCode;
+}
+export function disconnectExecutor(child) {
+    if (child.connected)
+        child.disconnect();
+}
+//# sourceMappingURL=executor.js.map