agl 1.0.17 → 21.0.2

package/ts/classes.browserresourcehost.ts

@@ -0,0 +1,1205 @@
+import * as plugins from './plugins.js';
+import type { IControllerResourceRuntimeHost } from './classes.resourcecoordinator.js';
+import type { IControllerResourceDocument } from './interfaces.projects.js';
+import {
+  flexIpcMaximumBrowserChannels,
+  type IFlexBrowserChannelBinding,
+} from './interfaces.flexipc.js';
+
+const maximumPendingFlexCapabilityIssuances = 128;
+const maximumActiveHumanBrowserViews = 64;
+const maximumTrackedCapabilityRevocations = 512;
+const maximumCleanupOperationWaiters = 512;
+const browserCapabilityAuthorizationTimeoutMs = 10_000;
+const browserResourceCleanupTimeoutMs = 40_000;
+
+const waitForSignal = async <TValue>(
+  promiseArg: Promise<TValue>,
+  signalArg: AbortSignal,
+): Promise<TValue> => new Promise<TValue>((resolve, reject) => {
+  const abort = () => {
+    signalArg.removeEventListener('abort', abort);
+    reject(signalArg.reason ?? new DOMException('The operation was aborted.', 'AbortError'));
+  };
+  signalArg.addEventListener('abort', abort, { once: true });
+  if (signalArg.aborted) {
+    abort();
+    return;
+  }
+  void promiseArg.then(
+    (value) => {
+      signalArg.removeEventListener('abort', abort);
+      resolve(value);
+    },
+    (errorArg) => {
+      signalArg.removeEventListener('abort', abort);
+      reject(errorArg);
+    },
+  );
+});
+
+const waitForCleanupDeadline = async <TValue>(
+  promiseArg: Promise<TValue>,
+  deadlineAtArg: number,
+  messageArg: string,
+  signalArg?: AbortSignal,
+): Promise<TValue> => new Promise<TValue>((resolve, reject) => {
+  let timeout: NodeJS.Timeout | undefined;
+  let settled = false;
+  const finish = (operationArg: () => void) => {
+    if (settled) return;
+    settled = true;
+    if (timeout) clearTimeout(timeout);
+    signalArg?.removeEventListener('abort', abort);
+    operationArg();
+  };
+  const abort = () => finish(() => reject(
+    signalArg?.reason ?? new DOMException('The operation was aborted.', 'AbortError'),
+  ));
+  void promiseArg.then(
+    (value) => finish(() => resolve(value)),
+    (errorArg) => finish(() => reject(errorArg)),
+  );
+  if (signalArg?.aborted) {
+    abort();
+    return;
+  }
+  signalArg?.addEventListener('abort', abort, { once: true });
+  const remainingMs = deadlineAtArg - Date.now();
+  if (remainingMs <= 0) {
+    finish(() => reject(new Error(messageArg)));
+    return;
+  }
+  timeout = setTimeout(() => finish(() => reject(new Error(messageArg))), remainingMs);
+});
+
+export interface IControllerBrowserResourceHostOptions {
+  runtimeDirectory: string;
+  authorizeCapability: plugins.browserRuntime.IBrowserRuntimeOptions['authorizeCapability'];
+  isFlexRunActive?(bindingArg: IFlexBrowserChannelBinding, runIdArg: string): boolean;
+  beforeOperation?: plugins.browserRuntime.IBrowserRuntimeOptions['beforeOperation'];
+  audit?: plugins.browserRuntime.IBrowserRuntimeOptions['audit'];
+}
+
+export interface IControllerBrowserHumanView {
+  lease: plugins.browserRuntime.BrowserRuntimeLease;
+  activate(listenerArg: (eventArg: plugins.browserRuntime.TBrowserRuntimeEvent) => void): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface IControllerFlexBrowserChannel {
+  binding: IFlexBrowserChannelBinding;
+  capabilityToken: string;
+}
+
+interface IActiveFlexBrowserChannel {
+  binding: IFlexBrowserChannelBinding;
+  runId: string;
+  capabilityId: string;
+  inbound: plugins.stream.PassThrough;
+  peer: plugins.browserRuntime.BrowserRuntimeFramedServerPeer;
+  notifyClosed: boolean;
+  closing: boolean;
+  peerCloseOperation?: IObservedCleanupOperation;
+  peerClosed: boolean;
+  capabilityRevoked: boolean;
+}
+
+interface IFlexBrowserAuthorization {
+  binding: IFlexBrowserChannelBinding;
+  runId: string;
+}
+
+interface IFlexBrowserChannelAdmission {
+  projectId: string;
+  sessionId: string;
+  runId: string;
+  peerId: string;
+  abortController: AbortController;
+  settled: Promise<void>;
+  resolveSettled(): void;
+}
+
+interface IHumanBrowserViewAdmission {
+  abortController: AbortController;
+  settled: Promise<void>;
+  resolveSettled(): void;
+}
+
+type TBrowserCapabilityOwner = 'flex' | 'human';
+
+type TCleanupOperationOutcome =
+  | { status: 'fulfilled' }
+  | { status: 'rejected'; reason: unknown };
+
+interface IObservedCleanupOperation {
+  outcome?: TCleanupOperationOutcome;
+  waiters: Set<(outcomeArg: TCleanupOperationOutcome) => void>;
+}
+
+interface ICapabilityRevocationOperation extends IObservedCleanupOperation {
+  runtime: plugins.browserRuntime.BrowserRuntime;
+  owner: TBrowserCapabilityOwner;
+  onFulfilled: Set<() => void>;
+}
+
+interface IRuntimeStopOperation extends IObservedCleanupOperation {
+  runtime: plugins.browserRuntime.BrowserRuntime;
+}
+
+const observeCleanupOperation = (
+  operationArg: () => Promise<void>,
+  onSettledArg?: (outcomeArg: TCleanupOperationOutcome) => void,
+): IObservedCleanupOperation => {
+  const operation: IObservedCleanupOperation = { waiters: new Set() };
+  const settle = (outcomeArg: TCleanupOperationOutcome) => {
+    if (operation.outcome) return;
+    operation.outcome = outcomeArg;
+    onSettledArg?.(outcomeArg);
+    const waiters = [...operation.waiters];
+    operation.waiters.clear();
+    for (const waiter of waiters) waiter(outcomeArg);
+  };
+  void Promise.resolve().then(operationArg).then(
+    () => settle({ status: 'fulfilled' }),
+    (reason) => settle({ status: 'rejected', reason }),
+  );
+  return operation;
+};
+
+const waitForObservedCleanup = async (
+  operationArg: IObservedCleanupOperation,
+  deadlineAtArg: number,
+  messageArg: string,
+  signalArg?: AbortSignal,
+): Promise<TCleanupOperationOutcome> => {
+  if (operationArg.outcome) return operationArg.outcome;
+  signalArg?.throwIfAborted();
+  const remainingMs = deadlineAtArg - Date.now();
+  if (remainingMs <= 0) throw new Error(messageArg);
+  if (operationArg.waiters.size >= maximumCleanupOperationWaiters) {
+    throw new Error('Browser cleanup waiter capacity was reached.');
+  }
+  return new Promise<TCleanupOperationOutcome>((resolve, reject) => {
+    let timeout: NodeJS.Timeout | undefined;
+    let settled = false;
+    const finish = (finishArg: () => void) => {
+      if (settled) return;
+      settled = true;
+      if (timeout) clearTimeout(timeout);
+      operationArg.waiters.delete(complete);
+      signalArg?.removeEventListener('abort', abort);
+      finishArg();
+    };
+    const complete = (outcomeArg: TCleanupOperationOutcome) => {
+      finish(() => resolve(outcomeArg));
+    };
+    const abort = () => finish(() => reject(
+      signalArg?.reason ?? new DOMException('The operation was aborted.', 'AbortError'),
+    ));
+    operationArg.waiters.add(complete);
+    signalArg?.addEventListener('abort', abort, { once: true });
+    if (operationArg.outcome) {
+      complete(operationArg.outcome);
+      return;
+    }
+    if (signalArg?.aborted) {
+      abort();
+      return;
+    }
+    timeout = setTimeout(() => finish(() => reject(new Error(messageArg))), remainingMs);
+  });
+};
+
+export class ControllerBrowserResourceHost implements IControllerResourceRuntimeHost {
+  private runtime?: plugins.browserRuntime.BrowserRuntime;
+  private started = false;
+  private unavailable = false;
+  private stopping = false;
+  private cleanupTimeoutMs = browserResourceCleanupTimeoutMs;
+  private startPromise?: Promise<void>;
+  private stopPromise?: Promise<void>;
+  private readonly flexAuthorizationBindings = new Map<string, IFlexBrowserAuthorization>();
+  private readonly flexChannels = new Map<string, IActiveFlexBrowserChannel>();
+  private readonly flexChannelAdmissions = new Map<string, IFlexBrowserChannelAdmission>();
+  private readonly capabilityRevocationOperations = new Map<string, ICapabilityRevocationOperation>();
+  private readonly lateCapabilityIssuanceTasks = new Map<Promise<void>, TBrowserCapabilityOwner>();
+  private readonly humanViews = new Set<IControllerBrowserHumanView>();
+  private readonly humanViewAdmissions = new Set<IHumanBrowserViewAdmission>();
+  private admissionAbortController = new AbortController();
+  private readonly runtimeStopOperations = new WeakMap<
+    plugins.browserRuntime.BrowserRuntime,
+    IRuntimeStopOperation
+  >();
+
+  constructor(private readonly options: IControllerBrowserResourceHostOptions) {}
+
+  public async start(signalArg?: AbortSignal): Promise<void> {
+    if (this.started || this.unavailable) return;
+    if (this.startPromise) return await waitForSignal(
+      this.startPromise,
+      signalArg ?? new AbortController().signal,
+    );
+    if (this.stopping) throw new Error('Browser resource host cleanup is incomplete.');
+    if (this.admissionAbortController.signal.aborted) {
+      this.admissionAbortController = new AbortController();
+    }
+    const startPromise = this.performStart(signalArg);
+    this.startPromise = startPromise;
+    try {
+      await startPromise;
+    } finally {
+      if (this.startPromise === startPromise) this.startPromise = undefined;
+    }
+  }
+
+  private async performStart(signalArg?: AbortSignal): Promise<void> {
+    signalArg?.throwIfAborted();
+    const runtime = new plugins.browserRuntime.BrowserRuntime({
+      runtimeDirectory: this.options.runtimeDirectory,
+      authorizationTimeoutMs: browserCapabilityAuthorizationTimeoutMs,
+      authorizeCapability: this.options.authorizeCapability,
+      ...(this.options.beforeOperation === undefined ? {} : {
+        beforeOperation: this.options.beforeOperation,
+      }),
+      ...(this.options.audit === undefined ? {} : { audit: this.options.audit }),
+    });
+    this.runtime = runtime;
+    try {
+      await (signalArg ? waitForSignal(runtime.start(), signalArg) : runtime.start());
+      signalArg?.throwIfAborted();
+      if (this.stopping) throw new Error('Browser resource host cleanup began during startup.');
+      this.started = true;
+    } catch (errorArg) {
+      if (
+        errorArg instanceof plugins.browserRuntime.BrowserRuntimeError
+        && (errorArg.code === 'CONFINEMENT_FAILED' || errorArg.code === 'INVALID_INPUT')
+      ) {
+        await runtime.stop().catch((cleanupErrorArg) => {
+          throw new AggregateError(
+            [errorArg, cleanupErrorArg],
+            'BrowserRuntime startup and cleanup failed.',
+          );
+        });
+        if (this.runtime === runtime) this.runtime = undefined;
+        this.unavailable = true;
+        return;
+      }
+      await runtime.stop().catch((cleanupErrorArg) => {
+        throw new AggregateError(
+          [errorArg, cleanupErrorArg],
+          'BrowserRuntime startup and cleanup failed.',
+        );
+      });
+      if (this.runtime === runtime) this.runtime = undefined;
+      throw errorArg;
+    }
+  }
+
+  public async stop(): Promise<void> {
+    if (this.stopPromise) return this.stopPromise;
+    const stopPromise = this.performStop();
+    this.stopPromise = stopPromise;
+    try {
+      await stopPromise;
+    } finally {
+      if (this.stopPromise === stopPromise) this.stopPromise = undefined;
+    }
+  }
+
+  private async performStop(): Promise<void> {
+    const runtime = this.runtime;
+    if (!runtime) {
+      if (
+        this.humanViews.size > 0
+        || this.flexChannels.size > 0
+        || this.flexChannelAdmissions.size > 0
+        || this.humanViewAdmissions.size > 0
+        || this.capabilityRevocationOperations.size > 0
+        || this.lateCapabilityIssuanceTasks.size > 0
+      ) throw new Error('Browser resource host cleanup has no runtime owner.');
+      this.stopping = false;
+      return;
+    }
+    this.stopping = true;
+    if (!this.admissionAbortController.signal.aborted) {
+      this.admissionAbortController.abort(new Error('Browser resource host is stopping.'));
+    }
+    const deadlineAt = Date.now() + this.cleanupTimeoutMs;
+    const cleanupErrors: unknown[] = [];
+    for (const cleanup of [
+      () => this.cancelAndDrainFlexChannelAdmissions(deadlineAt),
+      () => this.drainHumanViewAdmissions(deadlineAt),
+      () => this.closeAllHumanViews(deadlineAt),
+      () => this.closeAllFlexChannels(undefined, deadlineAt),
+      () => this.drainLateCapabilityIssuanceTasks(deadlineAt),
+      () => this.retryPendingCapabilityRevocations(deadlineAt),
+    ]) {
+      await cleanup().catch((errorArg) => cleanupErrors.push(errorArg));
+    }
+    if (cleanupErrors.length > 0) {
+      throw new AggregateError(cleanupErrors, 'Browser resource host cleanup is incomplete.');
+    }
+    await this.stopRuntime(runtime, deadlineAt);
+    if (this.runtime === runtime) this.runtime = undefined;
+    this.started = false;
+    this.stopping = false;
+  }
+
+  public authorizeFlexCapability(
+    requestArg: plugins.browserRuntime.TBrowserCapabilityAuthorizationRequest,
+  ): boolean {
+    if (requestArg.role !== 'agent' || requestArg.source !== 'flex') return false;
+    const expected = this.flexAuthorizationBindings.get(requestArg.channelId);
+    return expected !== undefined
+      && this.isFlexRunActive(expected.binding, expected.runId)
+      && expected.binding.projectId === requestArg.projectId
+      && expected.binding.browserResourceId === requestArg.browserResourceId
+      && expected.binding.attachmentAuthorityId === requestArg.attachmentAuthorityId
+      && expected.binding.attachmentRevision === requestArg.attachmentRevision
+      && expected.binding.actorId === requestArg.actorId
+      && expected.binding.peerId === requestArg.peerId
+      && expected.binding.scopeId === requestArg.scopeId
+      && expected.binding.channelId === requestArg.channelId
+      && expected.binding.runId === requestArg.runId
+      && expected.binding.sessionId.harnessId === requestArg.sessionId.harnessId
+      && expected.binding.sessionId.nativeId === requestArg.sessionId.nativeId;
+  }
+
+  public async openFlexChannel(inputArg: {
+    resource: IControllerResourceDocument;
+    scopeId: string;
+    sessionId: string;
+    runId: string;
+    channelId: string;
+    peerId: string;
+    signal?: AbortSignal;
+    sendFrame(channelIdArg: string, bytesArg: Buffer): Promise<void>;
+    notifyClosed(channelIdArg: string): Promise<void>;
+  }): Promise<IControllerFlexBrowserChannel> {
+    const runtime = this.runtime;
+    const resource = inputArg.resource;
+    if (
+      !runtime
+      || !this.started
+      || resource.kind !== 'browser'
+      || resource.lifecycle !== 'active'
+      || resource.pendingAttachment !== undefined
+      || resource.sessionId?.harnessId !== 'flex'
+      || resource.sessionId.nativeId !== inputArg.sessionId
+      || resource.projectId !== inputArg.scopeId
+      || this.flexChannels.has(inputArg.channelId)
+      || this.flexChannelAdmissions.has(inputArg.channelId)
+      || this.stopping
+    ) throw new Error('The Flex browser channel is unavailable.');
+    if (
+      this.flexChannels.size + this.flexChannelAdmissions.size
+      >= flexIpcMaximumBrowserChannels
+    ) throw new Error('The Flex browser channel capacity was reached.');
+    inputArg.signal?.throwIfAborted();
+    this.admissionAbortController.signal.throwIfAborted();
+    let resolveAdmissionSettled!: () => void;
+    const admission: IFlexBrowserChannelAdmission = {
+      projectId: resource.projectId,
+      sessionId: inputArg.sessionId,
+      runId: inputArg.runId,
+      peerId: inputArg.peerId,
+      abortController: new AbortController(),
+      settled: new Promise<void>((resolve) => { resolveAdmissionSettled = resolve; }),
+      resolveSettled: () => resolveAdmissionSettled(),
+    };
+    const abortAdmission = (signalArg: AbortSignal) => {
+      if (!admission.abortController.signal.aborted) {
+        admission.abortController.abort(
+          signalArg.reason ?? new DOMException('The operation was aborted.', 'AbortError'),
+        );
+      }
+    };
+    const abortFromInput = () => abortAdmission(inputArg.signal!);
+    const abortFromHost = () => abortAdmission(this.admissionAbortController.signal);
+    inputArg.signal?.addEventListener('abort', abortFromInput, { once: true });
+    this.admissionAbortController.signal.addEventListener('abort', abortFromHost, { once: true });
+    this.flexChannelAdmissions.set(inputArg.channelId, admission);
+    if (inputArg.signal?.aborted) abortFromInput();
+    if (this.admissionAbortController.signal.aborted) abortFromHost();
+    const binding: IFlexBrowserChannelBinding = {
+      projectId: resource.projectId,
+      browserResourceId: resource.id,
+      attachmentAuthorityId: resource.attachmentAuthorityId,
+      attachmentRevision: resource.attachmentRevision,
+      actorId: `flex-run:${plugins.crypto.createHash('sha256')
+        .update(JSON.stringify([inputArg.scopeId, inputArg.sessionId, inputArg.runId]))
+        .digest('hex')}`,
+      peerId: inputArg.peerId,
+      role: 'agent',
+      source: 'flex',
+      scopeId: inputArg.scopeId,
+      channelId: inputArg.channelId,
+      runId: inputArg.runId,
+      sessionId: { harnessId: 'flex', nativeId: inputArg.sessionId },
+    };
+    const inbound = new plugins.stream.PassThrough();
+    const outbound = new plugins.stream.Writable({
+      write: (chunkArg, _encodingArg, callbackArg) => {
+        const bytes = Buffer.from(chunkArg as Uint8Array);
+        void inputArg.sendFrame(inputArg.channelId, bytes).then(
+          () => callbackArg(),
+          (errorArg) => callbackArg(errorArg instanceof Error ? errorArg : new Error('Frame send failed.')),
+        );
+      },
+      destroy: (errorArg, callbackArg) => {
+        const channel = this.flexChannels.get(inputArg.channelId);
+        if (channel?.notifyClosed) {
+          channel.notifyClosed = false;
+          void inputArg.notifyClosed(inputArg.channelId).catch(() => undefined);
+          void this.closeFlexChannel(inputArg.channelId, true).catch(() => undefined);
+        }
+        callbackArg(errorArg);
+      },
+    });
+    this.flexAuthorizationBindings.set(inputArg.channelId, {
+      binding,
+      runId: inputArg.runId,
+    });
+    let capabilityId = '';
+    let issuancePromise: ReturnType<plugins.browserRuntime.BrowserRuntime['issueCapability']> | undefined;
+    let channelRegistered = false;
+    try {
+      admission.abortController.signal.throwIfAborted();
+      if (!this.isFlexRunActive(binding, inputArg.runId)) {
+        throw new Error('The Flex browser run is no longer active.');
+      }
+      await this.retryPendingCapabilityRevocations(
+        Date.now() + browserCapabilityAuthorizationTimeoutMs,
+        admission.abortController.signal,
+      );
+      if (this.lateCapabilityIssuanceTasks.size >= maximumPendingFlexCapabilityIssuances) {
+        throw new Error('Browser capability issuance capacity was reached.');
+      }
+      issuancePromise = runtime.issueCapability({
+        ...binding,
+        expiresInMs: 24 * 60 * 60 * 1000,
+      });
+      const issued = await waitForSignal(issuancePromise, admission.abortController.signal);
+      issuancePromise = undefined;
+      capabilityId = issued.capabilityId;
+      const peer = runtime.attachTrustedFramedPeer({
+        ...binding,
+        readable: inbound,
+        writable: outbound,
+      });
+      this.flexChannels.set(inputArg.channelId, {
+        binding,
+        runId: inputArg.runId,
+        capabilityId,
+        inbound,
+        peer,
+        notifyClosed: true,
+        closing: false,
+        peerClosed: false,
+        capabilityRevoked: false,
+      });
+      channelRegistered = true;
+      admission.abortController.signal.throwIfAborted();
+      if (!this.isFlexRunActive(binding, inputArg.runId)) {
+        throw new Error('The Flex browser run is no longer active.');
+      }
+      return { binding, capabilityToken: issued.capabilityToken };
+    } catch (errorArg) {
+      const cleanupErrors: unknown[] = [];
+      if (issuancePromise) {
+        this.trackLateCapabilityIssuance(runtime, issuancePromise, 'flex');
+        issuancePromise = undefined;
+      }
+      if (channelRegistered) {
+        const channel = this.flexChannels.get(inputArg.channelId);
+        if (channel) channel.notifyClosed = false;
+        await this.closeFlexChannel(
+          inputArg.channelId,
+          false,
+          undefined,
+          Date.now() + this.cleanupTimeoutMs,
+        ).catch((cleanupErrorArg) => cleanupErrors.push(cleanupErrorArg));
+      } else {
+        this.flexAuthorizationBindings.delete(inputArg.channelId);
+        inbound.destroy();
+        outbound.destroy();
+        if (capabilityId) {
+          this.trackCapabilityRevocation(runtime, capabilityId, 'flex');
+          await this.awaitCapabilityRevocation(
+            capabilityId,
+            Date.now() + this.cleanupTimeoutMs,
+          ).catch((cleanupErrorArg) => cleanupErrors.push(cleanupErrorArg));
+        }
+      }
+      if (cleanupErrors.length > 0) {
+        throw new AggregateError(
+          [errorArg, ...cleanupErrors],
+          'Flex browser channel admission and cleanup failed.',
+        );
+      }
+      throw errorArg;
+    } finally {
+      inputArg.signal?.removeEventListener('abort', abortFromInput);
+      this.admissionAbortController.signal.removeEventListener('abort', abortFromHost);
+      this.flexChannelAdmissions.delete(inputArg.channelId);
+      admission.resolveSettled();
+    }
+  }
+
+  public async receiveFlexFrame(channelIdArg: string, bytesArg: Buffer, peerIdArg: string): Promise<void> {
+    const channel = this.flexChannels.get(channelIdArg);
+    if (
+      !channel
+      || channel.binding.peerId !== peerIdArg
+      || channel.closing
+      || !this.isFlexRunActive(channel.binding, channel.runId)
+    ) {
+      if (channel && !channel.closing && !this.isFlexRunActive(channel.binding, channel.runId)) {
+        void this.closeFlexChannel(channelIdArg, true).catch(() => undefined);
+      }
+      throw new Error('The Flex browser channel is unavailable.');
+    }
+    await new Promise<void>((resolve, reject) => {
+      channel.inbound.write(bytesArg, (errorArg) => errorArg ? reject(errorArg) : resolve());
+    });
+  }
+
+  public async closeFlexChannel(
+    channelIdArg: string,
+    notifyChildArg = false,
+    peerIdArg?: string,
+    deadlineAtArg = Date.now() + this.cleanupTimeoutMs,
+  ): Promise<void> {
+    const channel = this.flexChannels.get(channelIdArg);
+    if (!channel) return;
+    if (peerIdArg !== undefined && channel.binding.peerId !== peerIdArg) {
+      throw new Error('The Flex browser channel is unavailable.');
+    }
+    if (notifyChildArg) channel.notifyClosed = true;
+    else channel.notifyClosed = false;
+    channel.closing = true;
+    this.flexAuthorizationBindings.delete(channelIdArg);
+    const peerCleanup = (async () => {
+      if (channel.peerClosed) return;
+      channel.peerCloseOperation ??= observeCleanupOperation(
+        () => channel.peer.close(),
+        (outcomeArg) => {
+          if (outcomeArg.status === 'fulfilled') channel.peerClosed = true;
+        },
+      );
+      const outcome = await waitForObservedCleanup(
+        channel.peerCloseOperation,
+        deadlineAtArg,
+        'Flex browser peer cleanup timed out.',
+      );
+      if (outcome.status === 'rejected') {
+        channel.peerCloseOperation = undefined;
+        throw outcome.reason;
+      }
+    })();
+    const capabilityCleanup = (async () => {
+      if (channel.capabilityRevoked) return;
+      const runtime = this.runtime;
+      if (!runtime) throw new Error('BrowserRuntime is unavailable during channel cleanup.');
+      this.trackCapabilityRevocation(
+        runtime,
+        channel.capabilityId,
+        'flex',
+        () => { channel.capabilityRevoked = true; },
+      );
+      await this.awaitCapabilityRevocation(channel.capabilityId, deadlineAtArg);
+      channel.capabilityRevoked = true;
+    })();
+    const results = await Promise.allSettled([peerCleanup, capabilityCleanup]);
+    if (channel.peerClosed && channel.capabilityRevoked) {
+      channel.inbound.destroy();
+      this.flexChannels.delete(channelIdArg);
+    }
+    const errors = results
+      .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'Flex browser channel cleanup is incomplete.');
+    }
+  }
+
+  public async closeAllFlexChannels(
+    peerIdArg?: string,
+    deadlineAtArg = Date.now() + this.cleanupTimeoutMs,
+  ): Promise<void> {
+    const admissions = [...this.flexChannelAdmissions.entries()].filter(([, admission]) => (
+      peerIdArg === undefined || admission.peerId === peerIdArg
+    ));
+    for (const [channelId, admission] of admissions) {
+      this.flexAuthorizationBindings.delete(channelId);
+      if (!admission.abortController.signal.aborted) {
+        admission.abortController.abort(new Error('Flex browser channel admission was cancelled.'));
+      }
+    }
+    const admissionResults = await Promise.allSettled(admissions.map(([, admission]) => (
+      waitForCleanupDeadline(
+        admission.settled,
+        deadlineAtArg,
+        'Flex browser channel admission cleanup timed out.',
+      )
+    )));
+    const channels = [...this.flexChannels.values()].filter((channel) => (
+      peerIdArg === undefined || channel.binding.peerId === peerIdArg
+    ));
+    const results = await Promise.allSettled(
+      channels.map((channel) => this.closeFlexChannel(
+        channel.binding.channelId,
+        true,
+        undefined,
+        deadlineAtArg,
+      )),
+    );
+    const errors = [...admissionResults, ...results]
+      .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (peerIdArg === undefined) {
+      await this.drainLateCapabilityIssuanceTasks(deadlineAtArg, 'flex')
+        .catch((errorArg) => errors.push(errorArg));
+      await this.retryPendingCapabilityRevocations(deadlineAtArg, undefined, 'flex')
+        .catch((errorArg) => errors.push(errorArg));
+    }
+    if (errors.length > 0) throw new AggregateError(errors, 'Flex browser channel cleanup is incomplete.');
+  }
+
+  public async closeFlexChannelsForRun(
+    projectIdArg: string,
+    sessionIdArg: string,
+    runIdArg: string,
+  ): Promise<void> {
+    const deadlineAt = Date.now() + this.cleanupTimeoutMs;
+    const admissions = [...this.flexChannelAdmissions.entries()].filter(([, admission]) => (
+      admission.projectId === projectIdArg
+      && admission.sessionId === sessionIdArg
+      && admission.runId === runIdArg
+    ));
+    for (const [channelId, admission] of admissions) {
+      this.flexAuthorizationBindings.delete(channelId);
+      if (!admission.abortController.signal.aborted) {
+        admission.abortController.abort(new Error('Flex browser run ended.'));
+      }
+    }
+    const admissionResults = await Promise.allSettled(admissions.map(([, admission]) => (
+      waitForCleanupDeadline(
+        admission.settled,
+        deadlineAt,
+        'Flex browser channel admission cleanup timed out.',
+      )
+    )));
+    const channels = [...this.flexChannels.values()].filter((channel) => (
+      channel.binding.projectId === projectIdArg
+      && channel.binding.sessionId.nativeId === sessionIdArg
+      && channel.runId === runIdArg
+    ));
+    const channelResults = await Promise.allSettled(channels.map((channel) => (
+      this.closeFlexChannel(channel.binding.channelId, true, undefined, deadlineAt)
+    )));
+    const errors = [...admissionResults, ...channelResults]
+      .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'Flex browser run channel cleanup is incomplete.');
+    }
+  }
+
+  public async disconnectPeer(peerIdArg: string): Promise<void> {
+    await this.runtime?.disconnectPeer(peerIdArg);
+  }
+
+  public async openHumanView(inputArg: {
+    resource: IControllerResourceDocument;
+    actorId: string;
+    peerId: string;
+    signal: AbortSignal;
+    onDisconnect(signalArg: AbortSignal): Promise<void> | void;
+  }): Promise<IControllerBrowserHumanView> {
+    const runtime = this.runtime;
+    if (!runtime || !this.started || inputArg.resource.kind !== 'browser' || this.stopping) {
+      throw new Error('BrowserRuntime is unavailable.');
+    }
+    if (this.humanViews.size + this.humanViewAdmissions.size >= maximumActiveHumanBrowserViews) {
+      throw new Error('Human browser view capacity was reached.');
+    }
+    inputArg.signal.throwIfAborted();
+    this.admissionAbortController.signal.throwIfAborted();
+    let resolveAdmissionSettled!: () => void;
+    const admission: IHumanBrowserViewAdmission = {
+      abortController: new AbortController(),
+      settled: new Promise<void>((resolve) => { resolveAdmissionSettled = resolve; }),
+      resolveSettled: () => resolveAdmissionSettled(),
+    };
+    const abortAdmission = (signalArg: AbortSignal) => {
+      if (!admission.abortController.signal.aborted) {
+        admission.abortController.abort(
+          signalArg.reason ?? new DOMException('The operation was aborted.', 'AbortError'),
+        );
+      }
+    };
+    const abortFromInput = () => abortAdmission(inputArg.signal);
+    const abortFromHost = () => abortAdmission(this.admissionAbortController.signal);
+    inputArg.signal.addEventListener('abort', abortFromInput, { once: true });
+    this.admissionAbortController.signal.addEventListener('abort', abortFromHost, { once: true });
+    this.humanViewAdmissions.add(admission);
+    if (inputArg.signal.aborted) abortFromInput();
+    if (this.admissionAbortController.signal.aborted) abortFromHost();
+    const binding = {
+      projectId: inputArg.resource.projectId,
+      browserResourceId: inputArg.resource.id,
+      attachmentAuthorityId: inputArg.resource.attachmentAuthorityId,
+      attachmentRevision: inputArg.resource.attachmentRevision,
+      actorId: inputArg.actorId,
+      peerId: inputArg.peerId,
+      role: 'human' as const,
+      source: 'human' as const,
+    };
+    let humanView: IControllerBrowserHumanView | undefined;
+    let subscription: plugins.browserRuntime.IBrowserRuntimeFrameSubscription | undefined;
+    let activationPromise: Promise<void> | undefined;
+    let subscriptionCloseOperation: IObservedCleanupOperation | undefined;
+    let subscriptionClosed = false;
+    let capabilityRevoked = false;
+    let closing = false;
+    let runtimeDisconnectActive = false;
+    let runtimeDisconnectComplete = false;
+    let capabilityId = '';
+    let issuancePromise: ReturnType<plugins.browserRuntime.BrowserRuntime['issueCapability']> | undefined;
+    const closeSubscription = async (deadlineAtArg: number): Promise<void> => {
+      if (subscriptionClosed || !subscription) return;
+      subscriptionCloseOperation ??= observeCleanupOperation(
+        () => subscription!.close(),
+        (outcomeArg) => {
+          if (outcomeArg.status === 'fulfilled') subscriptionClosed = true;
+        },
+      );
+      const outcome = await waitForObservedCleanup(
+        subscriptionCloseOperation,
+        deadlineAtArg,
+        'Browser view subscription cleanup timed out.',
+      );
+      if (outcome.status === 'rejected') {
+        subscriptionCloseOperation = undefined;
+        throw outcome.reason;
+      }
+    };
+    try {
+      await this.retryPendingCapabilityRevocations(
+        Date.now() + browserCapabilityAuthorizationTimeoutMs,
+        admission.abortController.signal,
+      );
+      if (this.lateCapabilityIssuanceTasks.size >= maximumPendingFlexCapabilityIssuances) {
+        throw new Error('Browser capability issuance capacity was reached.');
+      }
+      issuancePromise = runtime.issueCapability({
+        ...binding,
+        disconnect: async (signalArg) => {
+          closing = true;
+          runtimeDisconnectActive = true;
+          try {
+            await inputArg.onDisconnect(signalArg);
+          } finally {
+            runtimeDisconnectActive = false;
+            runtimeDisconnectComplete = true;
+            subscriptionClosed = true;
+            capabilityRevoked = true;
+            if (humanView) this.humanViews.delete(humanView);
+          }
+        },
+      });
+      const issued = await waitForSignal(issuancePromise, admission.abortController.signal);
+      issuancePromise = undefined;
+      capabilityId = issued.capabilityId;
+      const lease = await runtime.acquireLease({
+        ...binding,
+        capabilityToken: issued.capabilityToken,
+        signal: admission.abortController.signal,
+      });
+      admission.abortController.signal.throwIfAborted();
+      humanView = {
+        lease,
+        activate: (listenerArg) => {
+          if (closing) return Promise.reject(new Error('Browser view cleanup has begun.'));
+          if (subscription && !subscriptionClosed) return Promise.resolve();
+          if (subscriptionClosed) {
+            subscription = undefined;
+            subscriptionCloseOperation = undefined;
+            subscriptionClosed = false;
+          }
+          activationPromise ??= (async () => {
+            let initialStateReceived = false;
+            const candidate = await lease.subscribeEvents((eventArg) => {
+              if (!initialStateReceived) {
+                if (eventArg.type !== 'state') {
+                  throw new Error('BrowserRuntime did not publish an initial state event.');
+                }
+                initialStateReceived = true;
+              }
+              listenerArg(eventArg);
+            });
+            subscription = candidate;
+            if (!initialStateReceived) {
+              await closeSubscription(Date.now() + this.cleanupTimeoutMs);
+              throw new Error('BrowserRuntime did not publish an initial state event.');
+            }
+            if (closing) {
+              await closeSubscription(Date.now() + this.cleanupTimeoutMs);
+              throw new Error('Browser view cleanup began during activation.');
+            }
+          })().finally(() => {
+            if (!subscription || subscriptionClosed) activationPromise = undefined;
+          });
+          return activationPromise;
+        },
+        close: async () => {
+          closing = true;
+          const deadlineAt = Date.now() + this.cleanupTimeoutMs;
+          if (runtimeDisconnectActive) {
+            await closeSubscription(deadlineAt);
+            return;
+          }
+          const errors: unknown[] = [];
+          if (activationPromise) {
+            await waitForCleanupDeadline(
+              activationPromise.catch(() => undefined),
+              deadlineAt,
+              'Browser view activation cleanup timed out.',
+            ).catch((errorArg) => errors.push(errorArg));
+          }
+          if (!subscriptionClosed) {
+            if (subscription) {
+              await closeSubscription(deadlineAt).catch((errorArg) => errors.push(errorArg));
+            } else {
+              subscriptionClosed = true;
+            }
+          }
+          if (!capabilityRevoked) {
+            this.trackCapabilityRevocation(
+              runtime,
+              capabilityId,
+              'human',
+              () => { capabilityRevoked = true; },
+            );
+            await this.awaitCapabilityRevocation(capabilityId, deadlineAt).then(
+              () => { capabilityRevoked = true; },
+              (errorArg) => errors.push(errorArg),
+            );
+          }
+          if (runtimeDisconnectComplete || capabilityRevoked) subscriptionClosed = true;
+          if (errors.length > 0) throw new AggregateError(errors, 'Browser view cleanup is incomplete.');
+          this.humanViews.delete(humanView!);
+        },
+      };
+      this.humanViews.add(humanView);
+      return humanView;
+    } catch (errorArg) {
+      const cleanupErrors: unknown[] = [];
+      if (issuancePromise) {
+        this.trackLateCapabilityIssuance(runtime, issuancePromise, 'human');
+      } else if (capabilityId && !capabilityRevoked) {
+        this.trackCapabilityRevocation(runtime, capabilityId, 'human');
+        await this.awaitCapabilityRevocation(
+          capabilityId,
+          Date.now() + this.cleanupTimeoutMs,
+        ).catch((cleanupErrorArg) => cleanupErrors.push(cleanupErrorArg));
+      }
+      if (humanView) this.humanViews.delete(humanView);
+      if (cleanupErrors.length > 0) {
+        throw new AggregateError(
+          [errorArg, ...cleanupErrors],
+          'Browser view admission and cleanup failed.',
+        );
+      }
+      throw errorArg;
+    } finally {
+      inputArg.signal.removeEventListener('abort', abortFromInput);
+      this.admissionAbortController.signal.removeEventListener('abort', abortFromHost);
+      this.humanViewAdmissions.delete(admission);
+      admission.resolveSettled();
+    }
+  }
+
+  public async startResource(
+    resourceArg: IControllerResourceDocument,
+    signalArg: AbortSignal,
+  ): Promise<void> {
+    signalArg.throwIfAborted();
+    await this.reconcileAttachment(resourceArg, signalArg);
+  }
+
+  public async stopResource(
+    resourceArg: IControllerResourceDocument,
+    signalArg: AbortSignal,
+  ): Promise<void> {
+    signalArg.throwIfAborted();
+    if (!this.runtime || this.stopping) return;
+    await this.runtime.terminateResource(resourceArg.projectId, resourceArg.id);
+  }
+
+  public isRunning(resourceArg: IControllerResourceDocument): boolean {
+    if (!this.runtime || this.stopping) return false;
+    const registration = this.runtime.listResources().find((candidate) => (
+      candidate.projectId === resourceArg.projectId
+      && candidate.browserResourceId === resourceArg.id
+    ));
+    return registration?.incarnationStatus === 'running';
+  }
+
+  public isAvailable(): boolean {
+    return this.started && !this.stopping;
+  }
+
+  public async reconcileAttachment(
+    resourceArg: IControllerResourceDocument,
+    signalArg: AbortSignal,
+  ): Promise<void> {
+    signalArg.throwIfAborted();
+    if (resourceArg.kind !== 'browser') return;
+    if (this.unavailable) return;
+    if (!this.runtime || this.stopping) throw new Error('BrowserRuntime is not started.');
+    const attachmentBinding = {
+      attachmentAuthorityId: resourceArg.attachmentAuthorityId,
+      attachmentRevision: resourceArg.attachmentRevision,
+      sessionId: resourceArg.sessionId,
+    };
+    const existing = this.runtime.listResources().find((candidate) => (
+      candidate.projectId === resourceArg.projectId
+      && candidate.browserResourceId === resourceArg.id
+    ));
+    if (!existing) {
+      this.runtime.registerResource({
+        projectId: resourceArg.projectId,
+        browserResourceId: resourceArg.id,
+        attachmentBinding,
+      });
+      return;
+    }
+    await this.runtime.applyAttachmentBinding({
+      projectId: resourceArg.projectId,
+      browserResourceId: resourceArg.id,
+      attachmentBinding,
+    });
+  }
+
+  public async retireResource(
+    resourceArg: IControllerResourceDocument,
+    signalArg: AbortSignal,
+  ): Promise<void> {
+    signalArg.throwIfAborted();
+    if (!this.runtime || this.stopping || resourceArg.kind !== 'browser') return;
+    const registered = this.runtime.listResources().some((candidate) => (
+      candidate.projectId === resourceArg.projectId
+      && candidate.browserResourceId === resourceArg.id
+    ));
+    if (!registered) {
+      this.runtime.registerResource({
+        projectId: resourceArg.projectId,
+        browserResourceId: resourceArg.id,
+        attachmentBinding: {
+          attachmentAuthorityId: resourceArg.attachmentAuthorityId,
+          attachmentRevision: resourceArg.attachmentRevision,
+          sessionId: resourceArg.sessionId,
+        },
+      });
+    }
+    await this.runtime.retireResource(resourceArg.projectId, resourceArg.id);
+  }
+
+  public get availability(): 'available' | 'unavailable' {
+    return this.started && !this.stopping ? 'available' : 'unavailable';
+  }
+
+  private isFlexRunActive(bindingArg: IFlexBrowserChannelBinding, runIdArg: string): boolean {
+    if (bindingArg.runId !== runIdArg) return false;
+    return this.options.isFlexRunActive?.(bindingArg, runIdArg) ?? true;
+  }
+
+  private trackCapabilityRevocation(
+    runtimeArg: plugins.browserRuntime.BrowserRuntime,
+    capabilityIdArg: string,
+    ownerArg: TBrowserCapabilityOwner,
+    onFulfilledArg?: () => void,
+  ): void {
+    const existing = this.capabilityRevocationOperations.get(capabilityIdArg);
+    if (existing && existing.owner !== ownerArg) {
+      throw new Error('Browser capability cleanup ownership changed unexpectedly.');
+    }
+    if (existing && existing.outcome?.status !== 'rejected') {
+      if (onFulfilledArg) existing.onFulfilled.add(onFulfilledArg);
+      return;
+    }
+    const onFulfilled = new Set(existing?.onFulfilled ?? []);
+    if (onFulfilledArg) onFulfilled.add(onFulfilledArg);
+    if (existing) this.capabilityRevocationOperations.delete(capabilityIdArg);
+    if (this.capabilityRevocationOperations.size >= maximumTrackedCapabilityRevocations) {
+      throw new Error('Browser capability cleanup capacity was reached.');
+    }
+    const operation = observeCleanupOperation(
+      () => runtimeArg.revokeCapability(capabilityIdArg),
+      (outcomeArg) => {
+        if (outcomeArg.status === 'fulfilled') {
+          for (const callback of operation.onFulfilled) callback();
+          operation.onFulfilled.clear();
+          if (this.capabilityRevocationOperations.get(capabilityIdArg) === operation) {
+            this.capabilityRevocationOperations.delete(capabilityIdArg);
+          }
+        }
+      },
+    ) as ICapabilityRevocationOperation;
+    operation.runtime = runtimeArg;
+    operation.owner = ownerArg;
+    operation.onFulfilled = onFulfilled;
+    this.capabilityRevocationOperations.set(capabilityIdArg, operation);
+  }
+
+  private async awaitCapabilityRevocation(
+    capabilityIdArg: string,
+    deadlineAtArg: number,
+    signalArg?: AbortSignal,
+  ): Promise<void> {
+    const operation = this.capabilityRevocationOperations.get(capabilityIdArg);
+    if (!operation) return;
+    const outcome = await waitForObservedCleanup(
+      operation,
+      deadlineAtArg,
+      'Browser capability cleanup timed out.',
+      signalArg,
+    );
+    if (outcome.status === 'rejected') throw outcome.reason;
+  }
+
+  private trackLateCapabilityIssuance(
+    runtimeArg: plugins.browserRuntime.BrowserRuntime,
+    issuancePromiseArg: ReturnType<plugins.browserRuntime.BrowserRuntime['issueCapability']>,
+    ownerArg: TBrowserCapabilityOwner,
+  ): void {
+    const task = (async () => {
+      const issued = await issuancePromiseArg;
+      this.trackCapabilityRevocation(runtimeArg, issued.capabilityId, ownerArg);
+      await this.awaitCapabilityRevocation(
+        issued.capabilityId,
+        Date.now() + this.cleanupTimeoutMs,
+      );
+    })();
+    this.lateCapabilityIssuanceTasks.set(task, ownerArg);
+    void task.catch(() => undefined).finally(() => this.lateCapabilityIssuanceTasks.delete(task));
+  }
+
+  private async closeAllHumanViews(deadlineAtArg: number): Promise<void> {
+    const results = await Promise.allSettled(
+      [...this.humanViews].map((view) => waitForCleanupDeadline(
+        view.close(),
+        deadlineAtArg,
+        'Human browser view cleanup timed out.',
+      )),
+    );
+    const errors = results
+      .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'Human browser view cleanup is incomplete.');
+    }
+  }
+
+  private async retryPendingCapabilityRevocations(
+    deadlineAtArg: number,
+    signalArg?: AbortSignal,
+    ownerArg?: TBrowserCapabilityOwner,
+  ): Promise<void> {
+    const entries = [...this.capabilityRevocationOperations.entries()].filter(([, operation]) => (
+      ownerArg === undefined || operation.owner === ownerArg
+    ));
+    for (const [capabilityId, operation] of entries) {
+      if (operation.outcome?.status === 'rejected') {
+        this.trackCapabilityRevocation(operation.runtime, capabilityId, operation.owner);
+      }
+    }
+    const results = await Promise.allSettled(
+      [...this.capabilityRevocationOperations.entries()]
+        .filter(([, operation]) => ownerArg === undefined || operation.owner === ownerArg)
+        .map(([capabilityId]) => (
+          this.awaitCapabilityRevocation(capabilityId, deadlineAtArg, signalArg)
+        )),
+    );
+    const errors = results
+      .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'Pending browser capability cleanup is incomplete.');
+    }
+  }
+
+  private async cancelAndDrainFlexChannelAdmissions(deadlineAtArg: number): Promise<void> {
+    const admissions = [...this.flexChannelAdmissions.entries()];
+    for (const [channelId, admission] of admissions) {
+      this.flexAuthorizationBindings.delete(channelId);
+      if (!admission.abortController.signal.aborted) {
+        admission.abortController.abort(new Error('Browser resource host is stopping.'));
+      }
+    }
+    await Promise.all(admissions.map(([, admission]) => waitForCleanupDeadline(
+      admission.settled,
+      deadlineAtArg,
+      'Flex browser channel admission cleanup timed out.',
+    )));
+  }
+
+  private async drainHumanViewAdmissions(deadlineAtArg: number): Promise<void> {
+    const admissions = [...this.humanViewAdmissions];
+    for (const admission of admissions) {
+      if (!admission.abortController.signal.aborted) {
+        admission.abortController.abort(new Error('Browser resource host is stopping.'));
+      }
+    }
+    await Promise.all(admissions.map((admission) => waitForCleanupDeadline(
+      admission.settled,
+      deadlineAtArg,
+      'Human browser view admission cleanup timed out.',
+    )));
+  }
+
+  private async drainLateCapabilityIssuanceTasks(
+    deadlineAtArg: number,
+    ownerArg?: TBrowserCapabilityOwner,
+  ): Promise<void> {
+    const tasks = [...this.lateCapabilityIssuanceTasks.entries()]
+      .filter(([, owner]) => ownerArg === undefined || owner === ownerArg)
+      .map(([task]) => task);
+    const results = await Promise.allSettled(tasks.map((task) => waitForCleanupDeadline(
+      task,
+      deadlineAtArg,
+      'Late browser capability issuance cleanup timed out.',
+    )));
+    const errors = results
+      .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+      .map((result) => result.reason);
+    if (errors.length > 0) {
+      throw new AggregateError(errors, 'Late browser capability issuance cleanup is incomplete.');
+    }
+  }
+
+  private async stopRuntime(
+    runtimeArg: plugins.browserRuntime.BrowserRuntime,
+    deadlineAtArg: number,
+  ): Promise<void> {
+    let operation = this.runtimeStopOperations.get(runtimeArg);
+    if (operation?.outcome?.status === 'rejected') {
+      this.runtimeStopOperations.delete(runtimeArg);
+      operation = undefined;
+    }
+    if (!operation) {
+      operation = observeCleanupOperation(() => runtimeArg.stop()) as IRuntimeStopOperation;
+      operation.runtime = runtimeArg;
+      this.runtimeStopOperations.set(runtimeArg, operation);
+    }
+    const outcome = await waitForObservedCleanup(
+      operation,
+      deadlineAtArg,
+      'BrowserRuntime shutdown timed out.',
+    );
+    if (outcome.status === 'rejected') throw outcome.reason;
+  }
+}