@fitlab-ai/agent-infra 0.9.5 → 0.9.6

package/lib/sandbox/control/client.ts

@@ -2,26 +2,91 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { randomUUID } from 'node:crypto';
 import {
+  SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
   SANDBOX_CONTROL_MAX_BYTES,
+  SANDBOX_CONTROL_STATUS_STALE_MS,
   isSandboxControlFamily,
+  type SandboxControlError,
   type SandboxControlRequest,
   type SandboxControlResponse,
   type SandboxTaskCreateRequest
 } from './protocol.ts';
+import { readSandboxControlStatus } from './state.ts';
 import type { TaskCreateCandidateV1 } from '../../task/create.ts';
 
+const SANDBOX_CONTROL_RESPONSE_SETTLE_MS = 250;
+
+export class SandboxControlClientError extends Error {
+  readonly detail: SandboxControlError;
+  readonly accepted: boolean;
+  constructor(detail: SandboxControlError, accepted = false) {
+    super(detail.message);
+    this.name = 'SandboxControlClientError';
+    this.detail = detail;
+    this.accepted = accepted;
+  }
+}
+
 function sleep(milliseconds: number): void {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
 }
 
+function clientError(code: string, message: string, retryable: boolean, accepted = false): never {
+  throw new SandboxControlClientError({ code, message: `${code}: ${message}`, retryable }, accepted);
+}
+
+function preflight(statusDir: string, generation: string, now = Date.now()): void {
+  let status;
+  try {
+    status = readSandboxControlStatus(statusDir);
+  } catch {
+    clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker status is missing or invalid', true);
+  }
+  if (status.generation !== generation) {
+    clientError('SANDBOX_CONTROL_GENERATION_INVALID', 'broker status generation does not match this sandbox', false);
+  }
+  if (now - status.updatedAt > SANDBOX_CONTROL_STATUS_STALE_MS || status.updatedAt > now + 1_000) {
+    clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker status heartbeat is stale', true);
+  }
+  if (status.state === 'busy') clientError('SANDBOX_CONTROL_BUSY', 'broker is executing another request', true);
+  if (status.state === 'parked') {
+    const code = status.reasonCode ?? 'SANDBOX_CONTROL_BROKER_UNAVAILABLE';
+    clientError(code, 'broker is parked until the host restores a valid binding', true);
+  }
+  if (status.state !== 'healthy') clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', 'broker is not ready', true);
+}
+
+function parseResponse(raw: string, id: string): SandboxControlResponse {
+  const response = JSON.parse(raw) as SandboxControlResponse;
+  if (response.version !== 2 || response.id !== id
+    || !['accepted', 'completed', 'rejected'].includes(response.phase)
+    || (response.phase === 'completed' && !Number.isInteger(response.exitCode))) {
+    clientError('SANDBOX_CONTROL_RESPONSE_INVALID', 'broker response is invalid', false);
+  }
+  return response;
+}
+
+function cancelPendingRequest(requestPath: string): boolean {
+  try {
+    fs.unlinkSync(requestPath);
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
 function exchangeSandboxControl(request: SandboxControlRequest, params: Readonly<{
   channelDir?: string;
+  statusDir?: string;
   timeoutMs?: number;
 }>): SandboxControlResponse {
   const channelDir = params.channelDir ?? process.env.AGENT_INFRA_CONTROL_DIR ?? '/run/agent-infra/control';
+  const statusDir = params.statusDir ?? process.env.AGENT_INFRA_CONTROL_STATUS_DIR ?? '/run/agent-infra/control-status';
+  preflight(statusDir, request.generation);
   const encoded = `${JSON.stringify(request)}\n`;
   if (Buffer.byteLength(encoded, 'utf8') > SANDBOX_CONTROL_MAX_BYTES) {
-    throw new Error('SANDBOX_CONTROL_REQUEST_TOO_LARGE: request exceeds the control limit');
+    clientError('SANDBOX_CONTROL_REQUEST_TOO_LARGE', 'request exceeds the control limit', false);
   }
   const requestsDir = path.join(channelDir, 'requests');
   const responsesDir = path.join(channelDir, 'responses');
@@ -31,49 +96,108 @@ function exchangeSandboxControl(request: SandboxControlRequest, params: Readonly
   fs.writeFileSync(temporary, encoded, { mode: 0o600, flag: 'wx' });
   fs.renameSync(temporary, requestPath);
   const deadline = Date.now() + (params.timeoutMs ?? 30_000);
+  let accepted = false;
+  let malformedResponseRaw: string | null = null;
+  let malformedResponseObservedAt = 0;
   while (Date.now() < deadline) {
     if (fs.existsSync(responsePath)) {
-      const raw = fs.readFileSync(responsePath, 'utf8');
-      fs.unlinkSync(responsePath);
-      const response = JSON.parse(raw) as SandboxControlResponse;
-      if (response.version !== 1 || response.id !== request.id || !Number.isInteger(response.exitCode)) {
-        throw new Error('SANDBOX_CONTROL_RESPONSE_INVALID: broker response is invalid');
+      let raw: string;
+      try {
+        raw = fs.readFileSync(responsePath, 'utf8');
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue;
+        throw error;
+      }
+      let response: SandboxControlResponse;
+      try {
+        response = parseResponse(raw, request.id);
+        malformedResponseRaw = null;
+      } catch (error) {
+        if (!(error instanceof SyntaxError)) throw error;
+        const now = Date.now();
+        if (malformedResponseRaw !== raw) {
+          malformedResponseRaw = raw;
+          malformedResponseObservedAt = now;
+        } else if (now - malformedResponseObservedAt >= SANDBOX_CONTROL_RESPONSE_SETTLE_MS) {
+          clientError('SANDBOX_CONTROL_RESPONSE_INVALID', 'broker response remained malformed', false, accepted);
+        }
+        sleep(25);
+        continue;
+      }
+      if (response.phase === 'accepted') {
+        accepted = true;
+      } else {
+        fs.rmSync(responsePath, { force: true });
+        return response;
+      }
+    }
+    if (!accepted) {
+      try {
+        preflight(statusDir, request.generation);
+      } catch (error) {
+        if (cancelPendingRequest(requestPath)) throw error;
+        clientError(
+          'SANDBOX_CONTROL_RESULT_UNKNOWN',
+          'request was claimed before broker availability was lost; inspect domain state before retrying',
+          false,
+          true
+        );
       }
-      return response;
     }
     sleep(25);
   }
-  throw new Error(`SANDBOX_CONTROL_TIMEOUT: broker did not respond within ${params.timeoutMs ?? 30_000}ms`);
+  if (accepted) {
+    clientError(
+      'SANDBOX_CONTROL_RESULT_UNKNOWN',
+      'accepted request did not produce a final result; inspect domain state before retrying',
+      false,
+      true
+    );
+  }
+  if (!cancelPendingRequest(requestPath)) {
+    clientError(
+      'SANDBOX_CONTROL_RESULT_UNKNOWN',
+      'request was claimed but no acceptance result was observed; inspect domain state before retrying',
+      false,
+      true
+    );
+  }
+  clientError('SANDBOX_CONTROL_BROKER_UNAVAILABLE', `broker did not accept the request within ${params.timeoutMs ?? 30_000}ms`, true);
+}
+
+function authority(params: { token?: string; generation?: string }): { token: string; generation: string } {
+  const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
+  const generation = params.generation ?? process.env.AGENT_INFRA_CONTROL_GENERATION;
+  if (!token || !generation) clientError('SANDBOX_CONTROL_UNAVAILABLE', 'control authority is missing', false);
+  return { token, generation };
 }
 
 export function requestSandboxControl(params: Readonly<{
-  family: string;
-  args: string[];
-  channelDir?: string;
-  token?: string;
-  timeoutMs?: number;
+  family: string; args: string[]; channelDir?: string; statusDir?: string;
+  token?: string; generation?: string; timeoutMs?: number;
 }>): SandboxControlResponse {
-  if (!isSandboxControlFamily(params.family)) {
-    throw new Error(`SANDBOX_CONTROL_COMMAND_DENIED: '${params.family}' is not allowed`);
-  }
-  if (params.family === 'task-create') throw new Error("SANDBOX_CONTROL_COMMAND_DENIED: 'task-create' requires a typed candidate");
-  const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
-  if (!token) throw new Error('SANDBOX_CONTROL_UNAVAILABLE: control token is missing');
-  const id = randomUUID();
-  const request: SandboxControlRequest = { version: 1, id, token, family: params.family, args: params.args };
+  if (!isSandboxControlFamily(params.family)) clientError('SANDBOX_CONTROL_COMMAND_DENIED', `'${params.family}' is not allowed`, false);
+  if (params.family === 'task-create') clientError('SANDBOX_CONTROL_COMMAND_DENIED', "'task-create' requires a typed candidate", false);
+  const auth = authority(params);
+  const issuedAt = Date.now();
+  const request: SandboxControlRequest = {
+    version: 2, id: randomUUID(), ...auth, issuedAt,
+    expiresAt: issuedAt + SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
+    family: params.family, args: params.args
+  };
   return exchangeSandboxControl(request, params);
 }
 
 export function requestSandboxTaskCreate(params: Readonly<{
-  candidate: TaskCreateCandidateV1;
-  channelDir?: string;
-  token?: string;
-  timeoutMs?: number;
+  candidate: TaskCreateCandidateV1; channelDir?: string; statusDir?: string;
+  token?: string; generation?: string; timeoutMs?: number;
 }>): SandboxControlResponse {
-  const token = params.token ?? process.env.AGENT_INFRA_CONTROL_TOKEN;
-  if (!token) throw new Error('SANDBOX_CONTROL_UNAVAILABLE: control token is missing');
+  const auth = authority(params);
+  const issuedAt = Date.now();
   const request: SandboxTaskCreateRequest = {
-    version: 1, id: randomUUID(), token, family: 'task-create', candidate: params.candidate
+    version: 2, id: randomUUID(), ...auth, issuedAt,
+    expiresAt: issuedAt + SANDBOX_CONTROL_ADMISSION_WINDOW_MS,
+    family: 'task-create', candidate: params.candidate
   };
   return exchangeSandboxControl(request, { ...params, timeoutMs: params.timeoutMs ?? 120_000 });
 }

package/lib/sandbox/control/executor.ts

@@ -0,0 +1,163 @@
+import { spawn, spawnSync, type ChildProcess } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { getProcessStartTime } from '../../server/process-state.ts';
+import { createTask } from '../../task/create-service.ts';
+import { bindSandboxControlTask, type SandboxControlExecution, type SandboxControlManifest, type SandboxControlRequest } from './protocol.ts';
+import { atomicWriteJson, executionPath, terminateSandboxControlExecution } from './state.ts';
+
+export type SandboxControlExecutionResult = {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+};
+
+export type PreparedSandboxControlExecution = {
+  execution: SandboxControlExecution;
+  start(): void;
+  completion: Promise<SandboxControlExecutionResult>;
+  terminate(): boolean;
+};
+
+function safeEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  return Object.fromEntries(Object.entries(env).filter(([key]) => !key.toUpperCase().startsWith('AGENT_INFRA_CONTROL_')));
+}
+
+export function nodeEntryArgs(entry: string, args: string[]): string[] {
+  return path.extname(entry) === '.ts'
+    ? ['--experimental-strip-types', '--no-warnings', entry, ...args]
+    : [entry, ...args];
+}
+
+async function waitForStartTime(pid: number, timeoutMs = 2_000): Promise<string> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const startTime = getProcessStartTime(pid);
+    if (startTime) return startTime;
+    await new Promise<void>((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error('SANDBOX_CONTROL_EXECUTOR_IDENTITY_UNAVAILABLE');
+}
+
+export async function prepareSandboxControlExecution(params: {
+  manifest: SandboxControlManifest;
+  manifestPath: string;
+  request: SandboxControlRequest;
+  requestPath: string;
+  internalCliPath: string;
+}): Promise<PreparedSandboxControlExecution> {
+  const nonce = randomUUID();
+  const child = spawn(
+    process.execPath,
+    nodeEntryArgs(params.internalCliPath, [
+      'sandbox-control', 'execute', '--request', params.requestPath, '--nonce', nonce
+    ]),
+    {
+      cwd: params.manifest.repoRoot,
+      detached: process.platform !== 'win32',
+      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+      env: { ...safeEnv(process.env), AGENT_INFRA_EXECUTOR_MANIFEST: params.manifestPath }
+    }
+  );
+  if (typeof child.pid !== 'number') throw new Error('SANDBOX_CONTROL_EXECUTOR_SPAWN_FAILED');
+  let stdout = '';
+  let stderr = '';
+  child.stdout?.setEncoding('utf8');
+  child.stderr?.setEncoding('utf8');
+  child.stdout?.on('data', (chunk: string) => { stdout += chunk; });
+  child.stderr?.on('data', (chunk: string) => { stderr += chunk; });
+  const completion = new Promise<SandboxControlExecutionResult>((resolve, reject) => {
+    child.once('error', reject);
+    child.once('close', (code) => resolve({ exitCode: code ?? 1, stdout, stderr }));
+  });
+  const startTime = await waitForStartTime(child.pid);
+  const execution: SandboxControlExecution = {
+    version: 1,
+    generation: params.manifest.generation,
+    requestId: params.request.id,
+    nonce,
+    child: {
+      pid: child.pid,
+      startTime,
+      processGroupId: process.platform === 'win32' ? null : child.pid
+    },
+    phase: 'prepared',
+    updatedAt: Date.now()
+  };
+  atomicWriteJson(executionPath(params.manifest, params.request.id), execution);
+  return {
+    execution,
+    start() {
+      if (!child.connected) throw new Error('SANDBOX_CONTROL_EXECUTOR_GATE_CLOSED');
+      atomicWriteJson(executionPath(params.manifest, params.request.id), { ...execution, phase: 'running', updatedAt: Date.now() });
+      child.send({ version: 1, nonce });
+    },
+    completion,
+    terminate() {
+      atomicWriteJson(executionPath(params.manifest, params.request.id), {
+        ...execution, phase: 'terminating', updatedAt: Date.now()
+      });
+      return terminateSandboxControlExecution(execution);
+    }
+  };
+}
+
+function waitForGate(nonce: string, timeoutMs = 2_000): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_TIMEOUT')), timeoutMs);
+    const onDisconnect = () => {
+      clearTimeout(timer);
+      reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_CLOSED'));
+    };
+    process.once('disconnect', onDisconnect);
+    process.once('message', (message: unknown) => {
+      clearTimeout(timer);
+      process.off('disconnect', onDisconnect);
+      const value = message as { version?: unknown; nonce?: unknown } | null;
+      if (!value || value.version !== 1 || value.nonce !== nonce) {
+        reject(new Error('SANDBOX_CONTROL_EXECUTOR_GATE_INVALID'));
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function executeRequest(manifest: SandboxControlManifest, request: SandboxControlRequest): SandboxControlExecutionResult {
+  if (request.family === 'task-create') {
+    const result = createTask(request.candidate, { repoRoot: manifest.repoRoot });
+    return {
+      exitCode: result.status === 'blocked' ? 2 : result.status === 'failed' ? 1 : 0,
+      stdout: `${JSON.stringify(result)}\n`,
+      stderr: ''
+    };
+  }
+  const boundArgs = bindSandboxControlTask(request, manifest.taskId!);
+  if (request.family === 'task-orchestration') boundArgs.push('--git-worktree-root', manifest.worktreeRoot);
+  const result = spawnSync(
+    process.execPath,
+    nodeEntryArgs(process.argv[1]!, [request.family, ...boundArgs]),
+    { cwd: manifest.repoRoot, encoding: 'utf8', env: safeEnv(process.env) }
+  );
+  return { exitCode: result.status ?? 1, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
+}
+
+export async function runSandboxControlExecutor(requestPath: string, nonce: string): Promise<void> {
+  await waitForGate(nonce);
+  process.disconnect?.();
+  const root = fs.realpathSync.native(process.cwd());
+  const request = JSON.parse(fs.readFileSync(requestPath, 'utf8')) as SandboxControlRequest;
+  const manifestPath = process.env.AGENT_INFRA_EXECUTOR_MANIFEST;
+  if (!manifestPath) throw new Error('SANDBOX_CONTROL_EXECUTOR_MANIFEST_MISSING');
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as SandboxControlManifest;
+  if (fs.realpathSync.native(manifest.repoRoot) !== root) throw new Error('SANDBOX_CONTROL_EXECUTOR_ROOT_INVALID');
+  const result = executeRequest(manifest, request);
+  process.stdout.write(result.stdout);
+  process.stderr.write(result.stderr);
+  process.exitCode = result.exitCode;
+}
+
+export function disconnectExecutor(child: ChildProcess): void {
+  if (child.connected) child.disconnect();
+}