@fitlab-ai/agent-infra 0.9.5 → 0.9.7

package/lib/sandbox/commands/start.ts

@@ -17,9 +17,9 @@ const USAGE = `Usage: ai sandbox start [--recreate] <branch | TASK-id | N>
 
 Start an existing sandbox container that has stopped (for example after the
 Docker daemon was restarted or replaced). The container must already exist:
-if none is found, run 'ai sandbox create <branch>' first. A container that is
-already running is checked without modifying healthy runtime state. --recreate
-allows container-only replacement after in-place recovery fails.`;
+if none is found, run 'ai sandbox create <branch>' first. Without --recreate, a
+container that is already running is checked without modifying healthy runtime
+state. --recreate replaces the existing container even when it is healthy.`;
 
 export function parseStartArgs(args: string[]): { target: string; recreate: boolean; help: boolean } {
   let recreate = false;
@@ -83,6 +83,7 @@ export async function start(args: string[]): Promise<void> {
     workspace: target.workspace,
     row: found,
     allowRecreate: parsed.recreate,
+    forceRecreate: parsed.recreate,
     recreate: async () => {
       const { create } = await import('./create.ts');
       await create([target.requestedRef, '--no-refresh']);

package/lib/sandbox/control/client.ts

@@ -2,26 +2,91 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { randomUUID } from 'node:crypto';
 import {
+  SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
   SANDBOX_CONTROL_MAX_BYTES,
+  SANDBOX_CONTROL_STATUS_STALE_MS,
   isSandboxControlFamily,
+  type SandboxControlError,
   type SandboxControlRequest,
   type SandboxControlResponse,
   type SandboxTaskCreateRequest
 } from './protocol.ts';
+import { readSandboxControlStatus } from './state.ts';
 import type { TaskCreateCandidateV1 } from '../../task/create.ts';
 
+const SANDBOX_CONTROL_RESPONSE_SETTLE_MS = 250;
+
+export class SandboxControlClientError extends Error {
+  readonly detail: SandboxControlError;
+  readonly accepted: boolean;
+  constructor(detail: SandboxControlError, accepted = false) {
+    super(detail.message);
+    this.name = 'SandboxControlClientError';
+    this.detail = detail;
+    this.accepted = accepted;
+  }
+}
+
 function sleep(milliseconds: number): void {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
 }
 
+function clientError(code: string, message: string, retryable: boolean, accepted = false): never {
+  throw new SandboxControlClientError({ code, message: `${code}: ${message}`, retryable }, accepted);
+}
+
+function preflight(statusDir: string, generation: string, now = Date.now()): void {
+  let status;
+  try {
+    status = readSandboxControlStatus(statusDir);
+  } catch {
+    clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker status is missing or invalid', true);
+  }
+  if (status.generation !== generation) {
+    clientError('SANDBOX_CONTROL_GENERATION_INVALID', 'broker status generation does not match this sandbox', false);
+  }
+  if (now - status.updatedAt > SANDBOX_CONTROL_STATUS_STALE_MS || status.updatedAt > now + 1_000) {
+    clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker status heartbeat is stale', true);
+  }
+  if (status.state === 'busy') clientError('SANDBOX_CONTROL_BUSY', 'broker is executing another request', true);
+  if (status.state === 'parked') {
+    const code = status.reasonCode ?? 'SANDBOX_CONTROL_BROKER_UNAVAILABLE';
+    clientError(code, 'broker is parked until the host restores a valid binding', true);
+  }
+  if (status.state !== 'healthy') clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker is not ready', true);
+}
+
+function parseResponse(raw: string, id: string): SandboxControlResponse {
+  const response = JSON.parse(raw) as SandboxControlResponse;
+  if (response.version !== 2 || response.id !== id
+    || !['accepted', 'completed', 'rejected'].includes(response.phase)
+    || (response.phase === 'completed' && !Number.isInteger(response.exitCode))) {
+    clientError('SANDBOX_CONTROL_RESPONSE_INVALID', 'broker response is invalid', false);
+  }
+  return response;
+}
+
+function cancelPendingRequest(requestPath: string): boolean {
+  try {
+    fs.unlinkSync(requestPath);
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
 function exchangeSandboxControl(request: SandboxControlRequest, params: Readonly<{
   channelDir?: string;
+  statusDir?: string;
   timeoutMs?: number;
 }>): SandboxControlResponse {
   const channelDir = params.channelDir ?? process.env.AGENT_INFRA_CONTROL_DIR ?? '/run/agent-infra/control';
+  const statusDir = params.statusDir ?? process.env.AGENT_INFRA_CONTROL_STATUS_DIR ?? '/run/agent-infra/control-status';
+  preflight(statusDir, request.generation);
   const encoded = `${JSON.stringify(request)}\n`;
   if (Buffer.byteLength(encoded, 'utf8') > SANDBOX_CONTROL_MAX_BYTES) {
-    throw new Error('SANDBOX_CONTROL_REQUEST_TOO_LARGE: request exceeds the control limit');
+    clientError('SANDBOX_CONTROL_REQUEST_TOO_LARGE', 'request exceeds the control limit', false);
   }
   const requestsDir = path.join(channelDir, 'requests');
   const responsesDir = path.join(channelDir, 'responses');
@@ -31,49 +96,108 @@ function exchangeSandboxControl(request: SandboxControlRequest, params: Readonly
   fs.writeFileSync(temporary, encoded, { mode: 0o600, flag: 'wx' });
   fs.renameSync(temporary, requestPath);
   const deadline = Date.now() + (params.timeoutMs ?? 30_000);
+  let accepted = false;
+  let malformedResponseRaw: string | null = null;
+  let malformedResponseObservedAt = 0;
   while (Date.now() < deadline) {
     if (fs.existsSync(responsePath)) {
-      const raw = fs.readFileSync(responsePath, 'utf8');
-      fs.unlinkSync(responsePath);
-      const response = JSON.parse(raw) as SandboxControlResponse;
-      if (response.version !== 1 || response.id !== request.id || !Number.isInteger(response.exitCode)) {
-        throw new Error('SANDBOX_CONTROL_RESPONSE_INVALID: broker response is invalid');
+      let raw: string;
+      try {
+        raw = fs.readFileSync(responsePath, 'utf8');
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue;
+        throw error;
+      }
+      let response: SandboxControlResponse;
+      try {
+        response = parseResponse(raw, request.id);
+        malformedResponseRaw = null;
+      } catch (error) {
+        if (!(error instanceof SyntaxError)) throw error;
+        const now = Date.now();
+        if (malformedResponseRaw !== raw) {
+          malformedResponseRaw = raw;
+          malformedResponseObservedAt = now;
+        } else if (now - malformedResponseObservedAt >= SANDBOX_CONTROL_RESPONSE_SETTLE_MS) {
+          clientError('SANDBOX_CONTROL_RESPONSE_INVALID', 'broker response remained malformed', false, accepted);
+        }
+        sleep(25);
+        continue;
+      }
+      if (response.phase === 'accepted') {
+        accepted = true;
+      } else {
+        fs.rmSync(responsePath, { force: true });
+        return response;
+      }
+    }
+    if (!accepted) {
+      try {
+        preflight(statusDir, request.generation);
+      } catch (error) {
+        if (cancelPendingRequest(requestPath)) throw error;
+        clientError(
+          'SANDBOX_CONTROL_RESULT_UNKNOWN',
+          'request was claimed before broker availability was lost; inspect domain state before retrying',
+          false,
+          true
+        );
       }
-      return response;
     }
     sleep(25);
   }
-  throw new Error(`SANDBOX_CONTROL_TIMEOUT: broker did not respond within ${params.timeoutMs ?? 30_000}ms`);
+  if (accepted) {
+    clientError(
+      'SANDBOX_CONTROL_RESULT_UNKNOWN',
+      'accepted request did not produce a final result; inspect domain state before retrying',
+      false,
+      true
+    );
+  }
+  if (!cancelPendingRequest(requestPath)) {
+    clientError(
+      'SANDBOX_CONTROL_RESULT_UNKNOWN',
+      'request was claimed but no acceptance result was observed; inspect domain state before retrying',
+      false,
+      true
+    );
+  }
+  clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', `broker did not accept the request within ${params.timeoutMs ?? 30_000}ms`, true);
+}
+
+function authority(params: { token?: string; generation?: string }): { token: string; generation: string } {
+  const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
+  const generation = params.generation ?? process.env.AGENT_INFRA_CONTROL_GENERATION;
+  if (!token || !generation) clientError('SANDBOX_CONTROL_UNAVAILABLE', 'control authority is missing', false);
+  return { token, generation };
 }
 
 export function requestSandboxControl(params: Readonly<{
-  family: string;
-  args: string[];
-  channelDir?: string;
-  token?: string;
-  timeoutMs?: number;
+  family: string; args: string[]; channelDir?: string; statusDir?: string;
+  token?: string; generation?: string; timeoutMs?: number;
 }>): SandboxControlResponse {
-  if (!isSandboxControlFamily(params.family)) {
-    throw new Error(`SANDBOX_CONTROL_COMMAND_DENIED: '${params.family}' is not allowed`);
-  }
-  if (params.family === 'task-create') throw new Error("SANDBOX_CONTROL_COMMAND_DENIED: 'task-create' requires a typed candidate");
-  const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
-  if (!token) throw new Error('SANDBOX_CONTROL_UNAVAILABLE: control token is missing');
-  const id = randomUUID();
-  const request: SandboxControlRequest = { version: 1, id, token, family: params.family, args: params.args };
+  if (!isSandboxControlFamily(params.family)) clientError('SANDBOX_CONTROL_COMMAND_DENIED', `'${params.family}' is not allowed`, false);
+  if (params.family === 'task-create') clientError('SANDBOX_CONTROL_COMMAND_DENIED', "'task-create' requires a typed candidate", false);
+  const auth = authority(params);
+  const issuedAt = Date.now();
+  const request: SandboxControlRequest = {
+    version: 2, id: randomUUID(), ...auth, issuedAt,
+    expiresAt: issuedAt + SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
+    family: params.family, args: params.args
+  };
   return exchangeSandboxControl(request, params);
 }
 
 export function requestSandboxTaskCreate(params: Readonly<{
-  candidate: TaskCreateCandidateV1;
-  channelDir?: string;
-  token?: string;
-  timeoutMs?: number;
+  candidate: TaskCreateCandidateV1; channelDir?: string; statusDir?: string;
+  token?: string; generation?: string; timeoutMs?: number;
 }>): SandboxControlResponse {
-  const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
-  if (!token) throw new Error('SANDBOX_CONTROL_UNAVAILABLE: control token is missing');
+  const auth = authority(params);
+  const issuedAt = Date.now();
   const request: SandboxTaskCreateRequest = {
-    version: 1, id: randomUUID(), token, family: 'task-create', candidate: params.candidate
+    version: 2, id: randomUUID(), ...auth, issuedAt,
+    expiresAt: issuedAt + SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
+    family: 'task-create', candidate: params.candidate
   };
   return exchangeSandboxControl(request, { ...params, timeoutMs: params.timeoutMs ?? 120_000 });
 }

package/lib/sandbox/control/container-identity.ts

@@ -0,0 +1,93 @@
+import type { SandboxControlManifest } from './protocol.ts';
+import { commandForEngine, runEngine, runProbe } from '../shell.ts';
+
+export type ContainerObservation =
+  | Readonly<{ state: 'found'; id: string; running: boolean; labels: Readonly<Record<string, string>> }>
+  | Readonly<{ state: 'absent'; id: string }>
+  | Readonly<{ state: 'unknown'; reason: string }>;
+
+export type ContainerInspection = Readonly<{
+  id?: unknown;
+  running?: unknown;
+  labels?: unknown;
+}>;
+
+function isAuthoritativeNotFound(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /(?:no such container|container .* not found|cannot find container)/i.test(message);
+}
+
+export function classifySandboxContainerInspection(
+  expected: SandboxControlManifest['containerIdentity'],
+  inspection: ContainerInspection | { notFound: true } | { error: unknown }
+): ContainerObservation {
+  if ('notFound' in inspection) return { state: 'absent', id: expected.id };
+  if ('error' in inspection) {
+    return isAuthoritativeNotFound(inspection.error)
+      ? { state: 'absent', id: expected.id }
+      : { state: 'unknown', reason: `container inspection failed: ${inspection.error instanceof Error ? inspection.error.message : String(inspection.error)}` };
+  }
+  const response = inspection as ContainerInspection;
+  if (typeof response.id !== 'string' || response.id !== expected.id
+    || typeof response.running !== 'boolean' || !response.labels
+    || typeof response.labels !== 'object' || Array.isArray(response.labels)) {
+    return { state: 'unknown', reason: 'container inspection response is incomplete' };
+  }
+  const labels = response.labels as Record<string, unknown>;
+  for (const [key, value] of Object.entries(expected.labels)) {
+    if (labels[key] !== value) return { state: 'unknown', reason: 'container labels do not match the manifest' };
+  }
+  return {
+    state: 'found',
+    id: response.id,
+    running: response.running,
+    labels: Object.fromEntries(
+      Object.entries(labels).flatMap(([key, value]) => typeof value === 'string' ? [[key, value]] : [])
+    )
+  };
+}
+
+export async function inspectSandboxControlContainer(
+  manifest: SandboxControlManifest,
+  options: { run?: typeof runEngine; probe?: typeof runProbe; timeoutMs?: number } = {}
+): Promise<ContainerObservation> {
+  try {
+    const args = ['inspect', '--format', '{{json .}}', manifest.containerIdentity.id];
+    let raw: string;
+    if (options.run) {
+      raw = options.run(manifest.engine, 'docker', args, options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs });
+    } else {
+      const command = commandForEngine(manifest.engine, 'docker', args);
+      const result = (options.probe ?? runProbe)(
+        command.cmd,
+        command.args,
+        options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs }
+      );
+      if (result.status !== 0) {
+        const stderr = typeof result.stderr === 'string' ? result.stderr : result.stderr?.toString('utf8') ?? '';
+        return classifySandboxContainerInspection(manifest.containerIdentity, {
+          error: new Error(stderr || `container inspection exited with ${result.status ?? 'unknown'}`)
+        });
+      }
+      raw = typeof result.stdout === 'string' ? result.stdout : result.stdout?.toString('utf8') ?? '';
+    }
+    const parsed = JSON.parse(raw) as {
+      Id?: unknown;
+      State?: { Running?: unknown };
+      Config?: { Labels?: unknown };
+    } | Array<{
+      Id?: unknown;
+      State?: { Running?: unknown };
+      Config?: { Labels?: unknown };
+    }>;
+    const value = Array.isArray(parsed) && parsed.length === 1 ? parsed[0] : parsed;
+    if (!value || Array.isArray(value)) return { state: 'unknown', reason: 'container inspection response is incomplete' };
+    return classifySandboxContainerInspection(manifest.containerIdentity, {
+      id: value.Id,
+      running: value.State?.Running,
+      labels: value.Config?.Labels
+    });
+  } catch (error) {
+    return classifySandboxContainerInspection(manifest.containerIdentity, { error });
+  }
+}

package/lib/sandbox/control/executor.ts

@@ -0,0 +1,167 @@
+import { spawn, spawnSync, type ChildProcess } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { getProcessStartTime } from '../../server/process-state.ts';
+import { createTask } from '../../task/create-service.ts';
+import { bindSandboxControlTask, type SandboxControlExecution, type SandboxControlManifest, type SandboxControlRequest } from './protocol.ts';
+import { atomicWriteJson, executionPath, terminateSandboxControlExecution } from './state.ts';
+
+export type SandboxControlExecutionResult = {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+};
+
+export type PreparedSandboxControlExecution = {
+  execution: SandboxControlExecution;
+  start(canWrite?: () => boolean): void;
+  completion: Promise<SandboxControlExecutionResult>;
+  terminate(updateState?: boolean): boolean;
+};
+
+function safeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  return Object.fromEntries(Object.entries(env).filter(([key]) => !key.toUpperCase().startsWith('AGENT_INFRA_CONTROL_')));
+}
+
+export function nodeEntryArgs(entry: string, args: string[]): string[] {
+  return path.extname(entry) === '.ts'
+    ? ['--experimental-strip-types', '--no-warnings', entry, ...args]
+    : [entry, ...args];
+}
+
+async function waitForStartTime(pid: number, timeoutMs = 2_000): Promise<number> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const startTime = getProcessStartTime(pid);
+    if (startTime !== null) return startTime;
+    await new Promise<void>((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error('SANDBOX_CONTROL_EXECUTOR_IDENTITY_UNAVAILABLE');
+}
+
+export async function prepareSandboxControlExecution(params: {
+  manifest: SandboxControlManifest;
+  manifestPath: string;
+  request: SandboxControlRequest;
+  requestPath: string;
+  internalCliPath: string;
+}): Promise<PreparedSandboxControlExecution> {
+  const nonce = randomUUID();
+  const child = spawn(
+    process.execPath,
+    nodeEntryArgs(params.internalCliPath, [
+      'sandbox-control', 'execute', '--request', params.requestPath, '--nonce', nonce
+    ]),
+    {
+      cwd: params.manifest.repoRoot,
+      detached: process.platform !== 'win32',
+      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      env: { ...safeEnv(process.env), AGENT_INFRA_EXECUTOR_MANIFEST: params.manifestPath }
+    }
+  );
+  if (typeof child.pid !== 'number') throw new Error('SANDBOX_CONTROL_EXECUTOR_SPAWN_FAILED');
+  let stdout = '';
+  let stderr = '';
+  child.stdout?.setEncoding('utf8');
+  child.stderr?.setEncoding('utf8');
+  child.stdout?.on('data', (chunk: string) => { stdout += chunk; });
+  child.stderr?.on('data', (chunk: string) => { stderr += chunk; });
+  const completion = new Promise<SandboxControlExecutionResult>((resolve, reject) => {
+    child.once('error', reject);
+    child.once('close', (code) => resolve({ exitCode: code ?? 1, stdout, stderr }));
+  });
+  const startTime = await waitForStartTime(child.pid);
+  const execution: SandboxControlExecution = {
+    version: 2,
+    generation: params.manifest.generation,
+    requestId: params.request.id,
+    nonce,
+    child: {
+      pid: child.pid,
+      startTime,
+      processGroupId: process.platform === 'win32' ? null : child.pid
+    },
+    phase: 'prepared',
+    updatedAt: Date.now()
+  };
+  atomicWriteJson(executionPath(params.manifest, params.request.id), execution);
+  return {
+    execution,
+    start(canWrite = () => true) {
+      if (!child.connected) throw new Error('SANDBOX_CONTROL_EXECUTOR_GATE_CLOSED');
+      if (!canWrite()) throw new Error('SANDBOX_CONTROL_OWNER_LOST');
+      atomicWriteJson(executionPath(params.manifest, params.request.id), { ...execution, phase: 'running', updatedAt: Date.now() });
+      if (!canWrite()) throw new Error('SANDBOX_CONTROL_OWNER_LOST');
+      child.send({ version: 1, nonce });
+    },
+    completion,
+    terminate(updateState = true) {
+      if (updateState) {
+        atomicWriteJson(executionPath(params.manifest, params.request.id), {
+          ...execution, phase: 'terminating', updatedAt: Date.now()
+        });
+      }
+      return terminateSandboxControlExecution(execution);
+    }
+  };
+}
+
+function waitForGate(nonce: string, timeoutMs = 2_000): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_TIMEOUT')), timeoutMs);
+    const onDisconnect = () => {
+      clearTimeout(timer);
+      reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_CLOSED'));
+    };
+    process.once('disconnect', onDisconnect);
+    process.once('message', (message: unknown) => {
+      clearTimeout(timer);
+      process.off('disconnect', onDisconnect);
+      const value = message as { version?: unknown; nonce?: unknown } | null;
+      if (!value || value.version !== 1 || value.nonce !== nonce) {
+        reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_INVALID'));
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function executeRequest(manifest: SandboxControlManifest, request: SandboxControlRequest): SandboxControlExecutionResult {
+  if (request.family === 'task-create') {
+    const result = createTask(request.candidate, { repoRoot: manifest.repoRoot });
+    return {
+      exitCode: result.status === 'blocked' ? 2 : result.status === 'failed' ? 1 : 0,
+      stdout: `${JSON.stringify(result)}\n`,
+      stderr: ''
+    };
+  }
+  const boundArgs = bindSandboxControlTask(request, manifest.taskId!);
+  if (request.family === 'task-orchestration') boundArgs.push('--git-worktree-root', manifest.worktreeRoot);
+  const result = spawnSync(
+    process.execPath,
+    nodeEntryArgs(process.argv[1]!, [request.family, ...boundArgs]),
+    { cwd: manifest.repoRoot, encoding: 'utf8', env: safeEnv(process.env) }
+  );
+  return { exitCode: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
+}
+
+export async function runSandboxControlExecutor(requestPath: string, nonce: string): Promise<void> {
+  await waitForGate(nonce);
+  process.disconnect?.();
+  const root = fs.realpathSync.native(process.cwd());
+  const request = JSON.parse(fs.readFileSync(requestPath, 'utf8')) as SandboxControlRequest;
+  const manifestPath = process.env.AGENT_INFRA_EXECUTOR_MANIFEST;
+  if (!manifestPath) throw new Error('SANDBOX_CONTROL_EXECUTOR_MANIFEST_MISSING');
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as SandboxControlManifest;
+  if (fs.realpathSync.native(manifest.repoRoot) !== root) throw new Error('SANDBOX_CONTROL_EXECUTOR_ROOT_INVALID');
+  const result = executeRequest(manifest, request);
+  process.stdout.write(result.stdout);
+  process.stderr.write(result.stderr);
+  process.exitCode = result.exitCode;
+}
+
+export function disconnectExecutor(child: ChildProcess): void {
+  if (child.connected) child.disconnect();
+}