agl 1.0.17 → 21.0.2

package/ts/classes.terminalmanager.ts

@@ -0,0 +1,1207 @@
+import * as plugins from './plugins.js';
+import type { IControllerTerminal, TControllerTerminalId } from '../ts_interfaces/index.js';
+import type { IControllerResourceDocument } from './interfaces.projects.js';
+import { createSanitizedRuntimeEnvironment } from './functions.runtimeenvironment.js';
+
+export interface ITerminalOutputPayload {
+  terminalId: TControllerTerminalId;
+  dataBase64: string;
+  reset?: boolean;
+  ended?: boolean;
+}
+
+export interface IControllerPtySpawner {
+  ensurePtySupport(): Promise<void>;
+  execSpawnStreamingInteractiveControlPty(
+    commandArg: string,
+    argsArg?: string[],
+    optionsArg?: plugins.smartshell.IPtyDirectSpawnOptions,
+  ): Promise<plugins.smartshell.IExecResultPtyStreaming>;
+}
+
+export interface IControllerTerminalShell {
+  executable: string;
+  args: string[];
+  environment: NodeJS.ProcessEnv;
+}
+
+export interface IControllerTerminalManagerOptions {
+  resolveProjectDirectory: (projectIdArg: string) => Promise<string>;
+  sendOutput: (peerIdArg: string, payloadArg: ITerminalOutputPayload) => Promise<void>;
+  onTerminalsChanged: (projectIdArg: string) => void;
+  onTerminalStopped: (
+    projectIdArg: string,
+    terminalIdArg: TControllerTerminalId,
+    exitCodeArg?: number,
+  ) => Promise<void>;
+  ptySpawner?: IControllerPtySpawner;
+  resolveShell?: () => Promise<IControllerTerminalShell>;
+  cleanupTimeoutMs?: number;
+}
+
+interface IProjectAdmissionState {
+  activeOperations: number;
+  retiring: boolean;
+  retired: boolean;
+  drainPromise?: Promise<void>;
+  resolveDrain?: () => void;
+  retirementPromise?: Promise<boolean>;
+}
+
+interface IOutputSlab {
+  buffer: Buffer;
+  start: number;
+  end: number;
+}
+
+interface ITerminalOutputStore {
+  slabs: IOutputSlab[];
+  start: number;
+  end: number;
+  bytes: number;
+}
+
+interface ITerminalPeerState {
+  peerId: string;
+  generation: number;
+  cursor: number;
+  detaching: boolean;
+  finalizing: boolean;
+  deliveryPromise?: Promise<void>;
+}
+
+type TTerminalFinalState = 'running' | 'failed-unconfirmed' | 'exited';
+
+interface IControllerOwnedTerminal extends Omit<IControllerTerminal, 'id'> {
+  id: TControllerTerminalId;
+}
+
+interface ITerminalEntry {
+  projectId: string;
+  terminal: IControllerOwnedTerminal;
+  execution: plugins.smartshell.IExecResultPtyStreaming;
+  peers: Map<string, ITerminalPeerState>;
+  sizesByPeer: Map<string, { rows: number; cols: number }>;
+  output: ITerminalOutputStore;
+  controlQueue: Promise<void>;
+  pendingControlOperations: number;
+  sizeRevision: number;
+  sizeReconciliationActive: boolean;
+  sizeReconciliationPromise?: Promise<void>;
+  closing: boolean;
+  finalState: TTerminalFinalState;
+  exitCode?: number;
+  finalError?: Error;
+  signalError?: Error;
+  rootExitPromise: Promise<void>;
+  resolveRootExit: () => void;
+  exitSubscription?: plugins.smartshell.IPtySubscription;
+  finalizePromise?: Promise<void>;
+  closeLifecyclePromise?: Promise<void>;
+  finalizationDeadlineAt?: number;
+}
+
+type TTerminalManagerState = 'new' | 'open' | 'closing' | 'closed';
+
+const maxTerminals = 32;
+const maxTerminalsPerProject = 8;
+const maxScrollbackBytes = 2 * 1024 * 1024;
+const maxPtyCaptureBytes = 4 * 1024 * 1024;
+const maxInputBytes = 8 * 1024;
+const maxOutputFrameBytes = 64 * 1024;
+const maxPeersPerTerminal = 32;
+const maxPendingControlOperations = 256;
+const defaultCleanupTimeoutMs = 10_000;
+
+const normalizeError = (errorArg: unknown): Error =>
+  errorArg instanceof Error ? errorArg : new Error(String(errorArg));
+
+const toTerminalDto = (terminalArg: IControllerOwnedTerminal): IControllerTerminal => ({
+  ...terminalArg,
+  id: { ...terminalArg.id },
+});
+
+const appendBoundedChunk = (
+  slabsArg: IOutputSlab[],
+  currentBytesArg: number,
+  chunkArg: Buffer,
+): number => {
+  if (chunkArg.byteLength === 0) return currentBytesArg;
+  let sourceOffset = 0;
+  let currentBytes = currentBytesArg;
+  if (chunkArg.byteLength >= maxScrollbackBytes) {
+    slabsArg.splice(0, slabsArg.length);
+    sourceOffset = chunkArg.byteLength - maxScrollbackBytes;
+    currentBytes = 0;
+  }
+
+  while (sourceOffset < chunkArg.byteLength) {
+    let slab = slabsArg.at(-1);
+    if (!slab || slab.end >= slab.buffer.byteLength) {
+      slab = {
+        buffer: Buffer.allocUnsafe(maxOutputFrameBytes),
+        start: 0,
+        end: 0,
+      };
+      slabsArg.push(slab);
+    }
+    const copyBytes = Math.min(
+      chunkArg.byteLength - sourceOffset,
+      slab.buffer.byteLength - slab.end,
+    );
+    chunkArg.copy(slab.buffer, slab.end, sourceOffset, sourceOffset + copyBytes);
+    slab.end += copyBytes;
+    sourceOffset += copyBytes;
+    currentBytes += copyBytes;
+  }
+
+  let excess = currentBytes - maxScrollbackBytes;
+  while (excess > 0) {
+    const first = slabsArg[0];
+    const firstBytes = first.end - first.start;
+    if (firstBytes <= excess) {
+      slabsArg.shift();
+      currentBytes -= firstBytes;
+      excess -= firstBytes;
+    } else {
+      first.start += excess;
+      currentBytes -= excess;
+      excess = 0;
+    }
+  }
+  return currentBytes;
+};
+
+const appendOutput = (outputArg: ITerminalOutputStore, chunkArg: Buffer): void => {
+  if (chunkArg.byteLength === 0) return;
+  outputArg.end += chunkArg.byteLength;
+  outputArg.bytes = appendBoundedChunk(outputArg.slabs, outputArg.bytes, chunkArg);
+  outputArg.start = outputArg.end - outputArg.bytes;
+};
+
+const validateExecutable = async (
+  candidateArg: string,
+  platformArg: NodeJS.Platform,
+): Promise<string> => {
+  const pathApi = platformArg === 'win32' ? plugins.path.win32 : plugins.path.posix;
+  if (!pathApi.isAbsolute(candidateArg)) {
+    throw new Error('The terminal shell executable must be an absolute path.');
+  }
+  const executable = await plugins.fs.promises.realpath(candidateArg);
+  const stats = await plugins.fs.promises.stat(executable);
+  if (!stats.isFile()) throw new Error('The terminal shell executable is not a file.');
+  if (platformArg !== 'win32') {
+    await plugins.fs.promises.access(executable, plugins.fs.constants.X_OK);
+  }
+  return executable;
+};
+
+export const resolveControllerTerminalShell = async (
+  environmentArg: NodeJS.ProcessEnv = process.env,
+  platformArg: NodeJS.Platform = process.platform,
+  validateExecutableArg: (
+    candidateArg: string,
+    platformArg: NodeJS.Platform,
+  ) => Promise<string> = validateExecutable,
+): Promise<IControllerTerminalShell> => {
+  const environment = createSanitizedRuntimeEnvironment(environmentArg);
+  if (platformArg === 'win32') {
+    const candidates: string[] = [];
+    if (environmentArg.COMSPEC && plugins.path.win32.isAbsolute(environmentArg.COMSPEC)) {
+      candidates.push(environmentArg.COMSPEC);
+    }
+    const systemRoot = environmentArg.SystemRoot ?? environmentArg.SYSTEMROOT;
+    if (systemRoot && plugins.path.win32.isAbsolute(systemRoot)) {
+      candidates.push(plugins.path.win32.join(systemRoot, 'System32', 'cmd.exe'));
+    }
+    for (const candidate of [...new Set(candidates)]) {
+      try {
+        const executable = await validateExecutableArg(candidate, platformArg);
+        delete environment.SHELL;
+        environment.COMSPEC = executable;
+        return { executable, args: [], environment };
+      } catch {
+        // Try the trusted SystemRoot fallback before failing closed.
+      }
+    }
+    throw new Error('No trusted Windows terminal shell executable is available.');
+  }
+
+  const candidates = [
+    ...(environmentArg.SHELL && plugins.path.posix.isAbsolute(environmentArg.SHELL)
+      ? [environmentArg.SHELL]
+      : []),
+    '/bin/sh',
+  ];
+  for (const candidate of [...new Set(candidates)]) {
+    try {
+      const executable = await validateExecutableArg(candidate, platformArg);
+      delete environment.COMSPEC;
+      environment.SHELL = executable;
+      return { executable, args: [], environment };
+    } catch {
+      // Fall through to /bin/sh, then fail closed.
+    }
+  }
+  throw new Error('No trusted POSIX terminal shell executable is available.');
+};
+
+const waitUntil = async (promiseArg: Promise<void>, deadlineAtArg: number): Promise<void> => {
+  const remainingMs = deadlineAtArg - Date.now();
+  if (remainingMs <= 0) throw new Error('Terminal cleanup exceeded its deadline.');
+  let timeout: NodeJS.Timeout | undefined;
+  try {
+    await Promise.race([
+      promiseArg,
+      new Promise<never>((_resolve, reject) => {
+        timeout = setTimeout(() => reject(new Error('Terminal cleanup exceeded its deadline.')), remainingMs);
+      }),
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+};
+
+export class ControllerTerminalManager {
+  private readonly ptySpawner: IControllerPtySpawner;
+  private readonly resolveShell: () => Promise<IControllerTerminalShell>;
+  private readonly cleanupTimeoutMs: number;
+  private readonly entries = new Map<string, ITerminalEntry>();
+  private readonly projectStates = new Map<string, IProjectAdmissionState>();
+  private readonly activeRetirements = new Set<Promise<boolean>>();
+  private readonly pendingRemovalCleanups = new Map<Promise<void>, string>();
+  private state: TTerminalManagerState = 'new';
+  private shell?: IControllerTerminalShell;
+  private initPromise?: Promise<void>;
+  private closePromise?: Promise<void>;
+  private pendingCreates = 0;
+  private readonly pendingCreatesByProject = new Map<string, number>();
+  private peerGeneration = 0;
+
+  constructor(private readonly options: IControllerTerminalManagerOptions) {
+    this.ptySpawner = options.ptySpawner ?? new plugins.smartshell.Smartshell({
+      executor: 'sh',
+      sourceFilePaths: [],
+    });
+    this.resolveShell = options.resolveShell ?? (() => resolveControllerTerminalShell());
+    this.cleanupTimeoutMs = options.cleanupTimeoutMs ?? defaultCleanupTimeoutMs;
+    if (
+      !Number.isSafeInteger(this.cleanupTimeoutMs)
+      || this.cleanupTimeoutMs < 1
+      || this.cleanupTimeoutMs > 60_000
+    ) {
+      throw new Error('cleanupTimeoutMs must be an integer between 1 and 60000.');
+    }
+  }
+
+  public async init(): Promise<void> {
+    if (this.state === 'open') return;
+    if (this.initPromise) return this.initPromise;
+    if (this.state !== 'new') throw new Error('The terminal manager cannot be initialized now.');
+    const initPromise = (async () => {
+      const [shell] = await Promise.all([
+        this.resolveShell(),
+        this.ptySpawner.ensurePtySupport(),
+      ]);
+      if (this.state !== 'new') throw new Error('Terminal manager initialization was cancelled.');
+      this.shell = shell;
+      this.state = 'open';
+    })();
+    this.initPromise = initPromise;
+    try {
+      await initPromise;
+    } finally {
+      if (this.initPromise === initPromise) this.initPromise = undefined;
+    }
+  }
+
+  public async listTerminals(projectIdArg: string): Promise<IControllerTerminal[]> {
+    return this.runProjectOperation(projectIdArg, async () => {
+      await this.options.resolveProjectDirectory(projectIdArg);
+      return [...this.entries.values()]
+        .filter((entry) => entry.projectId === projectIdArg && entry.finalState !== 'exited')
+        .map((entry) => toTerminalDto(entry.terminal));
+    });
+  }
+
+  public async createTerminal(projectIdArg: string, titleArg?: string): Promise<IControllerTerminal> {
+    return this.runProjectOperation(projectIdArg, async () => {
+      this.reserveCreate(projectIdArg);
+      try {
+        const directory = await this.options.resolveProjectDirectory(projectIdArg);
+        const shell = this.requireShell();
+        const terminalNativeId = this.createTerminalId();
+        return this.spawnTerminal({
+          projectId: projectIdArg,
+          terminalNativeId,
+          title: titleArg ?? plugins.path.basename(shell.executable),
+          command: shell.executable,
+          args: [...shell.args],
+          cwd: directory,
+          environment: { ...shell.environment },
+        });
+      } finally {
+        this.releaseCreate(projectIdArg);
+      }
+    });
+  }
+
+  public async resolveTerminalResourceMetadata(
+    projectIdArg: string,
+    titleArg?: string,
+  ): Promise<{ title: string; terminal: NonNullable<IControllerResourceDocument['terminal']> }> {
+    const directory = await this.options.resolveProjectDirectory(projectIdArg);
+    const shell = this.requireShell();
+    return {
+      title: titleArg ?? plugins.path.basename(shell.executable),
+      terminal: {
+        command: shell.executable,
+        args: [...shell.args],
+        cwd: directory,
+        stoppedAt: new Date(),
+      },
+    };
+  }
+
+  public async startTerminalResource(resourceArg: IControllerResourceDocument): Promise<IControllerTerminal> {
+    if (resourceArg.kind !== 'terminal' || !resourceArg.terminal) {
+      throw new Error('The resource is not a terminal.');
+    }
+    const terminal = resourceArg.terminal;
+    return this.runProjectOperation(resourceArg.projectId, async () => {
+      this.reserveCreate(resourceArg.projectId);
+      try {
+        if (this.entries.has(resourceArg.id)) {
+          throw new Error('The terminal resource is already running.');
+        }
+        const directory = await this.options.resolveProjectDirectory(resourceArg.projectId);
+        if (directory !== terminal.cwd) {
+          throw new Error('The terminal resource project directory binding changed.');
+        }
+        const shell = this.requireShell();
+        return this.spawnTerminal({
+          projectId: resourceArg.projectId,
+          terminalNativeId: resourceArg.id,
+          title: resourceArg.title,
+          command: terminal.command,
+          args: [...terminal.args],
+          cwd: directory,
+          environment: { ...shell.environment },
+        });
+      } finally {
+        this.releaseCreate(resourceArg.projectId);
+      }
+    });
+  }
+
+  public async stopTerminalResource(projectIdArg: string, resourceIdArg: string): Promise<void> {
+    const entry = this.entries.get(resourceIdArg);
+    if (!entry || entry.projectId !== projectIdArg) return;
+    await this.runProjectOperation(projectIdArg, async () => {
+      await this.closeEntry(entry, Date.now() + this.cleanupTimeoutMs);
+    });
+  }
+
+  public isTerminalRunning(projectIdArg: string, resourceIdArg: string): boolean {
+    const entry = this.entries.get(resourceIdArg);
+    return entry?.projectId === projectIdArg && entry.finalState === 'running';
+  }
+
+  private async spawnTerminal(inputArg: {
+    projectId: string;
+    terminalNativeId: string;
+    title: string;
+    command: string;
+    args: string[];
+    cwd: string;
+    environment: NodeJS.ProcessEnv;
+  }): Promise<IControllerTerminal> {
+    let entry: ITerminalEntry | undefined;
+    const output: ITerminalOutputStore = {
+      slabs: [],
+      start: 0,
+      end: 0,
+      bytes: 0,
+    };
+    const execution = await this.ptySpawner.execSpawnStreamingInteractiveControlPty(
+      inputArg.command,
+      [...inputArg.args],
+      {
+        cwd: inputArg.cwd,
+        env: { ...inputArg.environment },
+        maxBuffer: maxPtyCaptureBytes,
+        ptyCols: 120,
+        ptyRows: 30,
+        ptyTerm: 'xterm-256color',
+        onData: (chunkArg) => {
+          const chunk = Buffer.from(chunkArg, 'utf8');
+          if (entry) this.appendOutput(entry, chunk);
+          else appendOutput(output, chunk);
+        },
+      },
+    );
+    let resolveRootExit!: () => void;
+    const rootExitPromise = new Promise<void>((resolve) => {
+      resolveRootExit = resolve;
+    });
+    const createdEntry: ITerminalEntry = {
+      projectId: inputArg.projectId,
+      terminal: {
+        id: { harnessId: 'controller', nativeId: inputArg.terminalNativeId },
+        title: inputArg.title,
+        command: inputArg.command,
+        cwd: inputArg.cwd,
+      },
+      execution,
+      peers: new Map(),
+      sizesByPeer: new Map(),
+      output,
+      controlQueue: Promise.resolve(),
+      pendingControlOperations: 0,
+      sizeRevision: 0,
+      sizeReconciliationActive: false,
+      closing: false,
+      finalState: 'running',
+      rootExitPromise,
+      resolveRootExit,
+    };
+    entry = createdEntry;
+    this.entries.set(inputArg.terminalNativeId, createdEntry);
+    createdEntry.exitSubscription = execution.ptyProcess.onExit(
+      (eventArg) => this.confirmRootExit(createdEntry, eventArg.exitCode),
+    );
+    if (createdEntry.finalState === 'exited') createdEntry.exitSubscription.dispose();
+    void execution.finalPromise.then(
+      (resultArg) => this.confirmRootExit(createdEntry, resultArg.exitCode),
+      (errorArg) => {
+        if (createdEntry.finalState === 'running') createdEntry.finalState = 'failed-unconfirmed';
+        createdEntry.finalError = normalizeError(errorArg);
+      },
+    );
+    if (createdEntry.finalState === 'exited' || this.state !== 'open') {
+      await this.closeEntry(createdEntry, Date.now() + this.cleanupTimeoutMs);
+      throw new Error('The terminal exited while it was being created.');
+    }
+    const projectState = this.projectState(inputArg.projectId);
+    if (projectState.retiring || projectState.retired) {
+      await this.closeEntry(createdEntry, Date.now() + this.cleanupTimeoutMs);
+      throw new Error('The project stopped accepting terminal operations.');
+    }
+    this.options.onTerminalsChanged(inputArg.projectId);
+    return toTerminalDto(createdEntry.terminal);
+  }
+
+  public async renameTerminal(
+    projectIdArg: string,
+    terminalIdArg: string,
+    titleArg: string,
+  ): Promise<IControllerTerminal> {
+    const entry = this.requireEntry(projectIdArg, terminalIdArg);
+    return this.runProjectOperation(projectIdArg, () => (
+      this.runEntryControl(entry, async () => {
+        this.assertEntryRunning(entry);
+        entry.terminal.title = titleArg;
+        this.options.onTerminalsChanged(projectIdArg);
+        return toTerminalDto(entry.terminal);
+      })
+    ));
+  }
+
+  public async renameTerminalResource(
+    projectIdArg: string,
+    terminalIdArg: string,
+    titleArg: string,
+  ): Promise<void> {
+    const entry = this.entries.get(terminalIdArg);
+    if (!entry || entry.projectId !== projectIdArg || entry.finalState === 'exited') return;
+    await this.runProjectOperation(projectIdArg, () => (
+      this.runEntryControl(entry, async () => {
+        this.assertEntryRunning(entry);
+        entry.terminal.title = titleArg;
+        this.options.onTerminalsChanged(projectIdArg);
+      })
+    ));
+  }
+
+  public async removeTerminal(projectIdArg: string, terminalIdArg: string): Promise<boolean> {
+    const entry = this.entries.get(terminalIdArg);
+    return this.runProjectOperation(projectIdArg, async () => {
+      if (!entry || entry.projectId !== projectIdArg) return false;
+      await this.closeEntry(entry, Date.now() + this.cleanupTimeoutMs);
+      return true;
+    });
+  }
+
+  public async attach(
+    peerIdArg: string,
+    projectIdArg: string,
+    terminalIdArg: string,
+    isPeerConnectedArg: () => boolean = () => true,
+  ): Promise<void> {
+    const entry = this.requireEntry(projectIdArg, terminalIdArg);
+    return this.runProjectOperation(projectIdArg, () => (
+      this.runEntryControl(entry, async () => {
+        this.assertEntryRunning(entry);
+        if (!isPeerConnectedArg()) throw new Error('The terminal peer disconnected before attach.');
+        const previous = entry.peers.get(peerIdArg);
+        if (previous) {
+          this.detachPeerState(entry, previous);
+          if (previous.deliveryPromise) {
+            await waitUntil(
+              previous.deliveryPromise.catch(() => undefined),
+              Date.now() + this.cleanupTimeoutMs,
+            ).catch(() => undefined);
+          }
+          await this.applyMinimumSizeUntilStable(entry);
+        }
+        if (entry.peers.size >= maxPeersPerTerminal) {
+          throw new Error(`At most ${maxPeersPerTerminal} peers may attach to one terminal.`);
+        }
+        const peer: ITerminalPeerState = {
+          peerId: peerIdArg,
+          generation: ++this.peerGeneration,
+          cursor: entry.output.start,
+          detaching: false,
+          finalizing: false,
+        };
+        entry.peers.set(peerIdArg, peer);
+        try {
+          await this.schedulePeerDelivery(entry, peer, true);
+          if (!isPeerConnectedArg() || entry.peers.get(peerIdArg) !== peer) {
+            throw new Error('The terminal peer disconnected during attach.');
+          }
+        } catch (errorArg) {
+          this.detachPeerState(entry, peer);
+          throw errorArg;
+        }
+      })
+    ));
+  }
+
+  public async detach(peerIdArg: string, projectIdArg: string, terminalIdArg: string): Promise<void> {
+    const entry = this.entries.get(terminalIdArg);
+    return this.runProjectOperation(projectIdArg, () => {
+      if (!entry || entry.projectId !== projectIdArg) return Promise.resolve();
+      return this.runEntryControl(entry, async () => {
+        const peer = entry.peers.get(peerIdArg);
+        if (!peer) return;
+        this.detachPeerState(entry, peer);
+        if (peer.deliveryPromise) {
+          await waitUntil(
+            peer.deliveryPromise.catch(() => undefined),
+            Date.now() + this.cleanupTimeoutMs,
+          ).catch(() => undefined);
+        }
+        await this.applyMinimumSizeUntilStable(entry);
+      });
+    });
+  }
+
+  public async detachPeer(peerIdArg: string): Promise<void> {
+    const reconciliations: Promise<void>[] = [];
+    for (const entry of this.entries.values()) {
+      const peer = entry.peers.get(peerIdArg);
+      if (!peer) continue;
+      this.detachPeerState(entry, peer);
+      reconciliations.push(this.requestSizeReconciliation(entry));
+    }
+    await Promise.allSettled(reconciliations);
+  }
+
+  public async input(
+    peerIdArg: string,
+    projectIdArg: string,
+    terminalIdArg: string,
+    dataArg: Buffer,
+  ): Promise<void> {
+    if (dataArg.byteLength > maxInputBytes) {
+      throw new Error('Terminal input exceeds the per-message limit.');
+    }
+    const entry = this.requireEntry(projectIdArg, terminalIdArg);
+    return this.runProjectOperation(projectIdArg, () => (
+      this.runEntryControl(entry, async () => {
+        this.assertAttached(entry, peerIdArg);
+        await entry.execution.sendInput(dataArg.toString('utf8'));
+      })
+    ));
+  }
+
+  public async resize(
+    peerIdArg: string,
+    projectIdArg: string,
+    terminalIdArg: string,
+    rowsArg: number,
+    colsArg: number,
+  ): Promise<void> {
+    const entry = this.requireEntry(projectIdArg, terminalIdArg);
+    return this.runProjectOperation(projectIdArg, () => (
+      this.runEntryControl(entry, async () => {
+        this.assertAttached(entry, peerIdArg);
+        entry.sizesByPeer.set(peerIdArg, { rows: rowsArg, cols: colsArg });
+        entry.sizeRevision += 1;
+        await this.applyMinimumSizeUntilStable(entry);
+      })
+    ));
+  }
+
+  public async retireProject(
+    projectIdArg: string,
+    retireArg: () => Promise<boolean>,
+  ): Promise<boolean> {
+    if (this.state !== 'open') throw new Error('The terminal manager is not accepting operations.');
+    const projectState = this.projectState(projectIdArg);
+    if (projectState.retiring || projectState.retired) {
+      throw new Error('The project is already leaving terminal admission.');
+    }
+    projectState.retiring = true;
+    const deadlineAt = Date.now() + this.cleanupTimeoutMs;
+    const retirementPromise = (async () => {
+      await this.closeProjectEntries(projectIdArg, deadlineAt);
+      await waitUntil(this.waitForProjectDrain(projectState), deadlineAt);
+      await this.closeProjectEntries(projectIdArg, deadlineAt);
+      if ([...this.pendingRemovalCleanups.values()].includes(projectIdArg)) {
+        await waitUntil(this.waitForRemovalCleanups(projectIdArg), deadlineAt);
+      }
+      const retired = await retireArg();
+      projectState.retired = retired;
+      return retired;
+    })();
+    projectState.retirementPromise = retirementPromise;
+    this.activeRetirements.add(retirementPromise);
+    try {
+      return await retirementPromise;
+    } finally {
+      this.activeRetirements.delete(retirementPromise);
+      if (projectState.retirementPromise === retirementPromise) {
+        projectState.retirementPromise = undefined;
+      }
+      projectState.retiring = false;
+      this.pruneProjectState(projectIdArg, projectState);
+    }
+  }
+
+  public async closeAll(): Promise<void> {
+    if (this.state === 'closed') return;
+    if (this.closePromise) return this.closePromise;
+    this.state = 'closing';
+    const deadlineAt = Date.now() + this.cleanupTimeoutMs;
+    const closePromise = (async () => {
+      const initialClosures = [...this.entries.values()]
+        .map((entry) => this.closeEntry(entry, deadlineAt));
+      const coordination = Promise.all([
+        Promise.allSettled(initialClosures).then(() => undefined),
+        this.waitForAllProjectDrains(),
+        Promise.allSettled([...this.activeRetirements]).then(() => undefined),
+      ]).then(() => this.waitForRemovalCleanups());
+      let coordinationError: unknown;
+      try {
+        await waitUntil(coordination, deadlineAt);
+      } catch (errorArg) {
+        coordinationError = errorArg;
+      }
+      const finalOutcomes = await Promise.allSettled(
+        [...this.entries.values()].map((entry) => this.closeEntry(entry, deadlineAt)),
+      );
+      let cleanupError: unknown;
+      if (this.pendingRemovalCleanups.size > 0) {
+        try {
+          await waitUntil(this.waitForRemovalCleanups(), deadlineAt);
+        } catch (errorArg) {
+          cleanupError = errorArg;
+          if (this.entries.size === 0) this.pendingRemovalCleanups.clear();
+        }
+      }
+      const activeOperations = [...this.projectStates.values()]
+        .reduce((sum, projectState) => sum + projectState.activeOperations, 0);
+      const errors: unknown[] = [];
+      if (this.entries.size > 0) {
+        errors.push(new Error('One or more terminal roots remain unconfirmed.'));
+      }
+      if (activeOperations > 0) {
+        errors.push(new Error('One or more terminal operations remain active.'));
+      }
+      if (this.activeRetirements.size > 0) {
+        errors.push(new Error('One or more project retirements remain active.'));
+      }
+      if (this.pendingRemovalCleanups.size > 0) {
+        errors.push(new Error('One or more terminal layout cleanups remain active.'));
+      }
+      if (errors.length > 0 && coordinationError) errors.push(coordinationError);
+      if (errors.length > 0 && cleanupError) errors.push(cleanupError);
+      if (this.entries.size > 0) {
+        errors.push(...finalOutcomes.flatMap((outcome) => (
+          outcome.status === 'rejected' ? [outcome.reason] : []
+        )));
+      }
+      if (errors.length > 0) {
+        throw new AggregateError(errors, 'The terminal manager did not close every PTY.');
+      }
+      this.state = 'closed';
+    })();
+    this.closePromise = closePromise;
+    try {
+      await closePromise;
+    } finally {
+      if (this.closePromise === closePromise) this.closePromise = undefined;
+    }
+  }
+
+  private projectState(projectIdArg: string): IProjectAdmissionState {
+    let state = this.projectStates.get(projectIdArg);
+    if (!state) {
+      const newState: IProjectAdmissionState = {
+        activeOperations: 0,
+        retiring: false,
+        retired: false,
+      };
+      this.projectStates.set(projectIdArg, newState);
+      state = newState;
+    }
+    return state;
+  }
+
+  private pruneProjectState(projectIdArg: string, projectStateArg: IProjectAdmissionState): void {
+    if (
+      this.projectStates.get(projectIdArg) === projectStateArg
+      && projectStateArg.activeOperations === 0
+      && !projectStateArg.retiring
+      && !projectStateArg.retirementPromise
+    ) {
+      this.projectStates.delete(projectIdArg);
+    }
+  }
+
+  private async runProjectOperation<T>(
+    projectIdArg: string,
+    operationArg: () => Promise<T>,
+  ): Promise<T> {
+    if (this.state !== 'open') throw new Error('The terminal manager is not accepting operations.');
+    const projectState = this.projectState(projectIdArg);
+    if (projectState.retiring || projectState.retired) {
+      throw new Error('The project is not accepting terminal operations.');
+    }
+    if (projectState.activeOperations === 0) {
+      let resolveDrain!: () => void;
+      projectState.drainPromise = new Promise<void>((resolve) => {
+        resolveDrain = resolve;
+      });
+      projectState.resolveDrain = resolveDrain;
+    }
+    projectState.activeOperations += 1;
+    try {
+      return await operationArg();
+    } finally {
+      projectState.activeOperations -= 1;
+      if (projectState.activeOperations === 0) {
+        projectState.resolveDrain?.();
+        projectState.resolveDrain = undefined;
+        projectState.drainPromise = undefined;
+        this.pruneProjectState(projectIdArg, projectState);
+      }
+    }
+  }
+
+  private waitForProjectDrain(projectStateArg: IProjectAdmissionState): Promise<void> {
+    return projectStateArg.drainPromise ?? Promise.resolve();
+  }
+
+  private async waitForAllProjectDrains(): Promise<void> {
+    await Promise.all([...this.projectStates.values()].map((state) => this.waitForProjectDrain(state)));
+  }
+
+  private async waitForRemovalCleanups(projectIdArg?: string): Promise<void> {
+    while (true) {
+      const pending = [...this.pendingRemovalCleanups.entries()]
+        .filter(([, projectId]) => projectIdArg === undefined || projectId === projectIdArg)
+        .map(([cleanup]) => cleanup);
+      if (pending.length === 0) return;
+      await Promise.all(pending);
+    }
+  }
+
+  private reserveCreate(projectIdArg: string): void {
+    const activeForProject = [...this.entries.values()]
+      .filter((entry) => entry.projectId === projectIdArg && entry.finalState !== 'exited').length;
+    const pendingForProject = this.pendingCreatesByProject.get(projectIdArg) ?? 0;
+    if (this.entries.size + this.pendingCreates >= maxTerminals) {
+      throw new Error(`At most ${maxTerminals} controller terminals may run concurrently.`);
+    }
+    if (activeForProject + pendingForProject >= maxTerminalsPerProject) {
+      throw new Error(`At most ${maxTerminalsPerProject} terminals may run in one project.`);
+    }
+    this.pendingCreates += 1;
+    this.pendingCreatesByProject.set(projectIdArg, pendingForProject + 1);
+  }
+
+  private releaseCreate(projectIdArg: string): void {
+    this.pendingCreates -= 1;
+    const remaining = (this.pendingCreatesByProject.get(projectIdArg) ?? 1) - 1;
+    if (remaining <= 0) this.pendingCreatesByProject.delete(projectIdArg);
+    else this.pendingCreatesByProject.set(projectIdArg, remaining);
+  }
+
+  private createTerminalId(): string {
+    for (let attempt = 0; attempt < 8; attempt += 1) {
+      const id = `terminal_${plugins.crypto.randomBytes(16).toString('base64url')}`;
+      if (!this.entries.has(id)) return id;
+    }
+    throw new Error('Unable to allocate a unique terminal identifier.');
+  }
+
+  private requireShell(): IControllerTerminalShell {
+    if (!this.shell || this.state !== 'open') throw new Error('The terminal manager is not initialized.');
+    return this.shell;
+  }
+
+  private requireEntry(projectIdArg: string, terminalIdArg: string): ITerminalEntry {
+    const entry = this.entries.get(terminalIdArg);
+    if (!entry || entry.projectId !== projectIdArg || entry.finalState === 'exited') {
+      throw new Error('The requested terminal does not exist in this project.');
+    }
+    return entry;
+  }
+
+  private assertEntryRunning(entryArg: ITerminalEntry): void {
+    if (entryArg.closing || entryArg.finalState !== 'running') {
+      throw new Error('The terminal is not accepting operations.');
+    }
+  }
+
+  private assertAttached(entryArg: ITerminalEntry, peerIdArg: string): ITerminalPeerState {
+    this.assertEntryRunning(entryArg);
+    const peer = entryArg.peers.get(peerIdArg);
+    if (!peer || peer.detaching) throw new Error('The terminal is not attached by this peer.');
+    return peer;
+  }
+
+  private runEntryControl<T>(
+    entryArg: ITerminalEntry,
+    operationArg: () => Promise<T>,
+    internalArg = false,
+  ): Promise<T> {
+    if (!internalArg && entryArg.pendingControlOperations >= maxPendingControlOperations) {
+      return Promise.reject(new Error('The terminal control queue is full.'));
+    }
+    if (!internalArg) entryArg.pendingControlOperations += 1;
+    const operation = entryArg.controlQueue.then(async () => {
+      if (entryArg.closing || entryArg.finalState !== 'running') {
+        throw new Error('The terminal is not accepting operations.');
+      }
+      return operationArg();
+    });
+    entryArg.controlQueue = operation.then(() => undefined, () => undefined);
+    if (internalArg) return operation;
+    return operation.finally(() => {
+      entryArg.pendingControlOperations -= 1;
+    });
+  }
+
+  private appendOutput(entryArg: ITerminalEntry, chunkArg: Buffer): void {
+    if (chunkArg.byteLength === 0 || entryArg.finalState === 'exited') return;
+    appendOutput(entryArg.output, chunkArg);
+    if (entryArg.closing) return;
+    for (const peer of entryArg.peers.values()) {
+      if (!peer.deliveryPromise) void this.schedulePeerDelivery(entryArg, peer);
+    }
+  }
+
+  private schedulePeerDelivery(
+    entryArg: ITerminalEntry,
+    peerArg: ITerminalPeerState,
+    forceResetArg = false,
+  ): Promise<void> {
+    if (peerArg.detaching || peerArg.finalizing || entryArg.peers.get(peerArg.peerId) !== peerArg) {
+      return Promise.resolve();
+    }
+    if (peerArg.deliveryPromise) return peerArg.deliveryPromise;
+    const targetEnd = entryArg.output.end;
+    const needsReset = forceResetArg || peerArg.cursor < entryArg.output.start;
+    const start = needsReset ? entryArg.output.start : Math.min(peerArg.cursor, targetEnd);
+    const delivery = this.deliverRange(entryArg, peerArg, start, targetEnd, needsReset);
+    peerArg.deliveryPromise = delivery;
+    void delivery.catch(() => {
+      if (entryArg.peers.get(peerArg.peerId) === peerArg) {
+        this.detachPeerState(entryArg, peerArg);
+        void this.requestSizeReconciliation(entryArg);
+      }
+    }).finally(() => {
+      if (peerArg.deliveryPromise === delivery) peerArg.deliveryPromise = undefined;
+      if (
+        !peerArg.detaching
+        && !peerArg.finalizing
+        && entryArg.peers.get(peerArg.peerId) === peerArg
+        && peerArg.cursor < entryArg.output.end
+      ) {
+        void this.schedulePeerDelivery(entryArg, peerArg);
+      }
+    });
+    return delivery;
+  }
+
+  private async deliverRange(
+    entryArg: ITerminalEntry,
+    peerArg: ITerminalPeerState,
+    startArg: number,
+    endArg: number,
+    resetArg: boolean,
+    deadlineAtArg?: number,
+  ): Promise<void> {
+    let cursor = startArg;
+    let resetPending = resetArg;
+    let sentFrame = false;
+    while (cursor < endArg) {
+      if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return;
+      if (cursor < entryArg.output.start) {
+        cursor = entryArg.output.start;
+        peerArg.cursor = cursor;
+        resetPending = true;
+      }
+      if (cursor >= endArg) break;
+      const frame = this.readOutputFrame(entryArg, cursor, endArg);
+      if (!frame || frame.byteLength === 0) break;
+      const send = this.sendPeerOutput(entryArg, peerArg, frame, resetPending);
+      if (deadlineAtArg === undefined) await send;
+      else await waitUntil(send, deadlineAtArg);
+      if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return;
+      cursor += frame.byteLength;
+      peerArg.cursor = cursor;
+      resetPending = false;
+      sentFrame = true;
+    }
+    if (resetPending && !sentFrame) {
+      const send = this.sendPeerOutput(entryArg, peerArg, Buffer.alloc(0), true);
+      if (deadlineAtArg === undefined) await send;
+      else await waitUntil(send, deadlineAtArg);
+      peerArg.cursor = Math.max(peerArg.cursor, entryArg.output.start);
+    }
+  }
+
+  private readOutputFrame(
+    entryArg: ITerminalEntry,
+    startArg: number,
+    endArg: number,
+  ): Buffer | undefined {
+    if (startArg >= endArg) return undefined;
+    let slabStart = entryArg.output.start;
+    for (const slab of entryArg.output.slabs) {
+      const slabBytes = slab.end - slab.start;
+      const slabEnd = slabStart + slabBytes;
+      if (slabEnd <= startArg) {
+        slabStart = slabEnd;
+        continue;
+      }
+      if (slabStart >= endArg) break;
+      const offset = Math.max(0, startArg - slabStart);
+      const frameBytes = Math.min(
+        maxOutputFrameBytes,
+        slabBytes - offset,
+        endArg - startArg,
+      );
+      return slab.buffer.subarray(slab.start + offset, slab.start + offset + frameBytes);
+    }
+    return undefined;
+  }
+
+  private async sendPeerOutput(
+    entryArg: ITerminalEntry,
+    peerArg: ITerminalPeerState,
+    dataArg: Buffer,
+    resetArg: boolean,
+  ): Promise<void> {
+    if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return;
+    await this.options.sendOutput(peerArg.peerId, {
+      terminalId: { ...entryArg.terminal.id },
+      dataBase64: dataArg.toString('base64'),
+      ...(resetArg ? { reset: true } : {}),
+    });
+  }
+
+  private detachPeerState(entryArg: ITerminalEntry, peerArg: ITerminalPeerState): void {
+    peerArg.detaching = true;
+    if (entryArg.peers.get(peerArg.peerId) === peerArg) {
+      entryArg.peers.delete(peerArg.peerId);
+      if (entryArg.sizesByPeer.delete(peerArg.peerId)) entryArg.sizeRevision += 1;
+    }
+  }
+
+  private async applyMinimumSize(entryArg: ITerminalEntry): Promise<void> {
+    if (entryArg.sizesByPeer.size === 0 || entryArg.closing) return;
+    let rows = Number.POSITIVE_INFINITY;
+    let cols = Number.POSITIVE_INFINITY;
+    for (const size of entryArg.sizesByPeer.values()) {
+      rows = Math.min(rows, size.rows);
+      cols = Math.min(cols, size.cols);
+    }
+    if (Number.isFinite(rows) && Number.isFinite(cols)) {
+      await entryArg.execution.resize(cols, rows);
+    }
+  }
+
+  private async applyMinimumSizeUntilStable(entryArg: ITerminalEntry): Promise<void> {
+    if (entryArg.sizeReconciliationActive || entryArg.closing) return;
+    entryArg.sizeReconciliationActive = true;
+    try {
+      let appliedRevision: number;
+      do {
+        appliedRevision = entryArg.sizeRevision;
+        await this.applyMinimumSize(entryArg);
+      } while (!entryArg.closing && appliedRevision !== entryArg.sizeRevision);
+    } finally {
+      entryArg.sizeReconciliationActive = false;
+    }
+  }
+
+  private requestSizeReconciliation(entryArg: ITerminalEntry): Promise<void> {
+    if (entryArg.closing || entryArg.sizeReconciliationActive) return Promise.resolve();
+    if (entryArg.sizeReconciliationPromise) return entryArg.sizeReconciliationPromise;
+    const reconciliation = this.runEntryControl(
+      entryArg,
+      () => this.applyMinimumSizeUntilStable(entryArg),
+      true,
+    );
+    entryArg.sizeReconciliationPromise = reconciliation;
+    void reconciliation.catch(() => undefined).finally(() => {
+      if (entryArg.sizeReconciliationPromise === reconciliation) {
+        entryArg.sizeReconciliationPromise = undefined;
+      }
+    });
+    return reconciliation;
+  }
+
+  private confirmRootExit(entryArg: ITerminalEntry, exitCodeArg?: number): void {
+    if (entryArg.finalState === 'exited') return;
+    entryArg.finalState = 'exited';
+    if (exitCodeArg !== undefined) entryArg.exitCode = exitCodeArg;
+    entryArg.exitSubscription?.dispose();
+    entryArg.resolveRootExit();
+    entryArg.finalizePromise = this.finalizeExitedEntry(
+      entryArg,
+      entryArg.finalizationDeadlineAt ?? Date.now() + this.cleanupTimeoutMs,
+    );
+  }
+
+  private async finalizeExitedPeer(
+    entryArg: ITerminalEntry,
+    peerArg: ITerminalPeerState,
+    finalOutputEndArg: number,
+    deadlineAtArg: number,
+  ): Promise<void> {
+    if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return;
+    peerArg.finalizing = true;
+    try {
+      if (peerArg.deliveryPromise) {
+        await waitUntil(peerArg.deliveryPromise, deadlineAtArg);
+      }
+      if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return;
+      const needsReset = peerArg.cursor < entryArg.output.start;
+      await this.deliverRange(
+        entryArg,
+        peerArg,
+        needsReset ? entryArg.output.start : peerArg.cursor,
+        finalOutputEndArg,
+        needsReset,
+        deadlineAtArg,
+      );
+      if (
+        peerArg.detaching
+        || entryArg.peers.get(peerArg.peerId) !== peerArg
+        || peerArg.cursor < finalOutputEndArg
+      ) return;
+      await waitUntil(this.options.sendOutput(peerArg.peerId, {
+        terminalId: { ...entryArg.terminal.id },
+        dataBase64: '',
+        ended: true,
+      }), deadlineAtArg);
+    } catch {
+      // A stalled peer is detached without an ended frame; output can never race ended.
+    } finally {
+      this.detachPeerState(entryArg, peerArg);
+    }
+  }
+
+  private async finalizeExitedEntry(
+    entryArg: ITerminalEntry,
+    deadlineAtArg: number,
+  ): Promise<void> {
+    if (this.entries.get(entryArg.terminal.id.nativeId) !== entryArg) return;
+    const finalOutputEnd = entryArg.output.end;
+    await Promise.allSettled([...entryArg.peers.values()].map((peer) => (
+      this.finalizeExitedPeer(entryArg, peer, finalOutputEnd, deadlineAtArg)
+    )));
+    this.entries.delete(entryArg.terminal.id.nativeId);
+    entryArg.peers.clear();
+    entryArg.sizesByPeer.clear();
+    const terminalId = { ...entryArg.terminal.id };
+    const removalCleanup = Promise.resolve()
+      .then(() => this.options.onTerminalStopped(entryArg.projectId, terminalId, entryArg.exitCode))
+      .catch(() => undefined);
+    this.pendingRemovalCleanups.set(removalCleanup, entryArg.projectId);
+    try {
+      await waitUntil(removalCleanup, deadlineAtArg);
+    } catch {
+      // Resource ownership is already released; startup pruning recovers a timed-out cleanup.
+    } finally {
+      this.pendingRemovalCleanups.delete(removalCleanup);
+    }
+    try {
+      this.options.onTerminalsChanged(entryArg.projectId);
+    } catch {
+      // Root ownership is already released; a notification callback cannot restore it.
+    }
+  }
+
+  private async closeEntry(entryArg: ITerminalEntry, deadlineAtArg: number): Promise<void> {
+    if (this.entries.get(entryArg.terminal.id.nativeId) !== entryArg) return;
+    if (entryArg.finalState === 'exited') {
+      await waitUntil(entryArg.finalizePromise ?? Promise.resolve(), deadlineAtArg);
+      return;
+    }
+    if (!entryArg.closeLifecyclePromise) {
+      entryArg.closing = true;
+      entryArg.finalizationDeadlineAt = deadlineAtArg;
+      const lifecycle = (async () => {
+        await entryArg.rootExitPromise;
+        await entryArg.finalizePromise;
+      })();
+      entryArg.closeLifecyclePromise = lifecycle;
+      try {
+        const signal = entryArg.execution.kill();
+        void signal.catch((errorArg) => {
+          entryArg.signalError = normalizeError(errorArg);
+        });
+      } catch (errorArg) {
+        entryArg.signalError = normalizeError(errorArg);
+      }
+    }
+    try {
+      await waitUntil(entryArg.closeLifecyclePromise, deadlineAtArg);
+    } catch (errorArg) {
+      if (this.entries.get(entryArg.terminal.id.nativeId) !== entryArg) return;
+      throw entryArg.signalError
+        ? new AggregateError(
+            [entryArg.signalError, errorArg],
+            'The terminal root could not be confirmed stopped.',
+          )
+        : errorArg;
+    }
+  }
+
+  private async closeProjectEntries(projectIdArg: string, deadlineAtArg: number): Promise<void> {
+    const outcomes = await Promise.allSettled(
+      [...this.entries.values()]
+        .filter((entry) => entry.projectId === projectIdArg)
+        .map((entry) => this.closeEntry(entry, deadlineAtArg)),
+    );
+    const errors = outcomes.flatMap((outcome) => (
+      outcome.status === 'rejected' ? [outcome.reason] : []
+    ));
+    const remaining = [...this.entries.values()]
+      .filter((entry) => entry.projectId === projectIdArg);
+    if (remaining.length > 0) {
+      errors.push(new Error('One or more project terminal roots remain unconfirmed.'));
+    }
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'The project terminal roots could not all be confirmed stopped.');
+    }
+  }
+}