@modelprofile.com/browser-runtime 3.1.1 → 4.0.0

package/ts/classes.runtime.ts

@@ -2,6 +2,7 @@ import * as plugins from './plugins.js';
 import { validateAgentAction } from './actions.js';
 import { BrowserArtifactStore } from './classes.artifactstore.js';
 import { BrowserEgressProxy } from './classes.egressproxy.js';
+import { BrowserRuntimeOwnership } from './classes.runtimeownership.js';
 import { runProductionConfinementProbe } from './confinement.js';
 import { BrowserRuntimeError } from './errors.js';
 import {
@@ -89,6 +90,7 @@ interface INormalizedRuntimeOptions {
   maxOutstandingFrames: number;
   frameAcknowledgementTimeoutMs: number;
   egress: NonNullable<IBrowserRuntimeOptions['egress']>;
+  ownership?: IBrowserRuntimeTestingOptions['ownership'];
   beforeLeasePublication?: IBrowserRuntimeTestingOptions['beforeLeasePublication'];
   beforeIdleTermination?: IBrowserRuntimeTestingOptions['beforeIdleTermination'];
   beforeFrameFailureTermination?: IBrowserRuntimeTestingOptions['beforeFrameFailureTermination'];
@@ -233,9 +235,8 @@ const identifierMaximum = 256;
 
 export class BrowserRuntime {
   private readonly options: INormalizedRuntimeOptions;
-  private readonly profileRoot: string;
-  private readonly artifactRoot: string;
-  private readonly lockPath: string;
+  private readonly ownership: BrowserRuntimeOwnership;
+  private profileRoot?: string;
   private readonly artifactStore: BrowserArtifactStore;
   private readonly runtimeAuthorityId = randomId(18);
   private readonly slots = new Map<string, IResourceSlot>();
@@ -247,11 +248,10 @@ export class BrowserRuntime {
   private readonly capabilityMutex = new TransitionMutex();
   private readonly cleanupOperations = new Set<Promise<void>>();
   private readonly framedPeers = new Set<BrowserRuntimeFramedServerPeer>();
-  private lockHandle?: plugins.fsPromises.FileHandle;
-  private lockHandleClosed = false;
   private startPromise?: Promise<void>;
   private stopPromise?: Promise<void>;
   private stopCleanupPromise?: Promise<void>;
+  private localCleanupPending = false;
   private lifecycleState: 'stopped' | 'starting' | 'running' | 'stopping' = 'stopped';
   private lifecycleEpoch = 0;
   private lifecycleController = new AbortController();
@@ -473,17 +473,21 @@ export class BrowserRuntime {
         10_000,
       ),
       egress: options.egress ?? {},
+      ownership: testingOptions?.ownership,
       beforeLeasePublication: testingOptions?.beforeLeasePublication,
       beforeIdleTermination: testingOptions?.beforeIdleTermination,
       beforeFrameFailureTermination: testingOptions?.beforeFrameFailureTermination,
       afterFrameFailureFence: testingOptions?.afterFrameFailureFence,
       beforeFramedPeerLeasePublication: testingOptions?.beforeFramedPeerLeasePublication,
     };
-    this.profileRoot = plugins.path.join(options.runtimeDirectory, 'profiles');
-    this.artifactRoot = plugins.path.join(options.runtimeDirectory, 'artifacts');
-    this.lockPath = plugins.path.join(options.runtimeDirectory, 'runtime.lock');
+    this.ownership = new BrowserRuntimeOwnership({
+      runtimeDirectory: options.runtimeDirectory,
+      uid: environment.uid ?? -1,
+      testing: testingOptions?.ownership,
+      onOwnershipLost: () => this.handleOwnershipLost(),
+    });
     this.artifactStore = new BrowserArtifactStore({
-      rootDirectory: this.artifactRoot,
+      rootDirectory: plugins.path.join(options.runtimeDirectory, 'artifacts'),
       ...(options.artifacts ?? {}),
     });
   }
@@ -492,6 +496,13 @@ export class BrowserRuntime {
     if (this.lifecycleState === 'running') return Promise.resolve();
     if (this.startPromise) return this.startPromise;
     if (this.stopCleanupPromise) return this.stopCleanupPromise.then(() => this.start());
+    if (
+      this.lifecycleState === 'stopping'
+      || this.localCleanupPending
+      || this.ownership.cleanupPending
+    ) {
+      return this.stop().then(() => this.start());
+    }
     this.lifecycleState = 'starting';
     const epoch = ++this.lifecycleEpoch;
     this.lifecycleController = new AbortController();
@@ -515,7 +526,8 @@ export class BrowserRuntime {
     if (this.stopPromise) return this.stopPromise;
     if (
       this.lifecycleState === 'stopped'
-      && !this.lockHandle
+      && !this.localCleanupPending
+      && !this.ownership.cleanupPending
       && !this.startPromise
       && !this.stopCleanupPromise
     ) {
@@ -528,8 +540,13 @@ export class BrowserRuntime {
       const startup = this.startPromise;
       const cleanup = (async () => {
         await startup?.catch(() => undefined);
-        await this.stopInternal();
-        this.lifecycleState = 'stopped';
+        try {
+          await this.stopInternal();
+        } finally {
+          if (!this.localCleanupPending && !this.ownership.cleanupPending) {
+            this.lifecycleState = 'stopped';
+          }
+        }
       })();
       this.stopCleanupPromise = cleanup;
       void cleanup.then(
@@ -1321,7 +1338,7 @@ export class BrowserRuntime {
   ): Promise<Uint8Array> {
     this.requireHuman(lease);
     this.requireValidLease(lease);
-    return this.artifactStore.read(
+    return this.requireArtifactStore().read(
       lease.capability.projectId,
       lease.capability.browserResourceId,
       artifactId,
@@ -1332,7 +1349,7 @@ export class BrowserRuntime {
   public async deleteLeaseArtifact(lease: ILeaseRecord, artifactId: string): Promise<void> {
     this.requireHuman(lease);
     this.requireValidLease(lease);
-    await this.artifactStore.delete(
+    await this.requireArtifactStore().delete(
       lease.capability.projectId,
       lease.capability.browserResourceId,
       artifactId,
@@ -1360,30 +1377,31 @@ export class BrowserRuntime {
     if (this.options.environment.uid === undefined || this.options.environment.uid === 0) {
       throw new BrowserRuntimeError('FENCED', 'browser runtime requires a non-root uid');
     }
-    await plugins.fsPromises.mkdir(this.options.runtimeDirectory, { recursive: true, mode: 0o700 });
-    await plugins.fsPromises.chmod(this.options.runtimeDirectory, 0o700);
-    if (!this.lockHandle) {
+    const generation = await this.ownership.acquire();
+    this.profileRoot = generation.profileRoot;
+    try {
+      this.artifactStore.setRootDirectoryForRuntime(generation.artifactRoot);
+      await plugins.fsPromises.mkdir(this.profileRoot, { recursive: false, mode: 0o700 });
+      await this.secureNewPrivateDirectory(this.profileRoot);
+      await this.artifactStore.start();
+    } catch (error) {
       try {
-        this.lockHandle = await plugins.fsPromises.open(this.lockPath, 'wx', 0o600);
-        this.lockHandleClosed = false;
-        await this.lockHandle.chmod(0o600);
-      } catch (error) {
-        if ((error as NodeJS.ErrnoException).code === 'EEXIST') {
-          throw new BrowserRuntimeError('LOCKED');
-        }
-        throw error;
+        await this.ownership.release('startup-failure');
+        this.profileRoot = undefined;
+      } catch {
+        throw new BrowserRuntimeError('FENCED', 'runtime startup cleanup is incomplete');
       }
+      throw error;
     }
-    await plugins.fsPromises.mkdir(this.profileRoot, { recursive: true, mode: 0o700 });
-    await plugins.fsPromises.chmod(this.profileRoot, 0o700);
-    const profileEntries = await plugins.fsPromises.readdir(this.profileRoot);
-    if (profileEntries.length > 0) {
-      throw new BrowserRuntimeError('FENCED', 'profile root is not empty');
-    }
-    await this.artifactStore.start();
   }
 
   private async stopInternal(): Promise<void> {
+    this.localCleanupPending = true;
+    const errors: unknown[] = [];
+    const localErrors: unknown[] = [];
+    if (this.ownership.owned) {
+      await this.ownership.assertCurrentOwnership().catch((error) => errors.push(error));
+    }
     const peerResults = await Promise.allSettled(
       [...this.framedPeers].map((peer) => peer.close()),
     );
@@ -1402,34 +1420,98 @@ export class BrowserRuntime {
     while (this.cleanupOperations.size > 0) {
       await Promise.allSettled([...this.cleanupOperations]);
     }
-    const errors = [...peerResults, ...revokeResults, ...retirementResults]
+    localErrors.push(...[...peerResults, ...revokeResults, ...retirementResults]
       .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
-      .map((result) => result.reason);
+      .map((result) => result.reason));
+    await this.artifactStore.stop().catch((error) => localErrors.push(error));
+    errors.push(...localErrors);
+    await this.ownership.release('stop').catch((error) => errors.push(error));
+    if (localErrors.length === 0) {
+      this.localCleanupPending = false;
+      this.clearStoppedRuntimeState();
+    }
     if (errors.length > 0) {
       throw new AggregateError(errors, 'Browser runtime cleanup is incomplete');
     }
-    await this.artifactStore.stop();
-    if (this.lockHandle) {
-      if (!this.lockHandleClosed) {
-        await this.lockHandle.close();
-        this.lockHandleClosed = true;
-      }
-      await plugins.fsPromises.rm(this.lockPath, { force: true });
-      this.lockHandle = undefined;
-      this.lockHandleClosed = false;
-    }
+  }
+
+  private clearStoppedRuntimeState(): void {
+    this.profileRoot = undefined;
     this.slots.clear();
+    this.capabilitiesByDigest.clear();
+    this.capabilitiesById.clear();
+    this.failedRevocations.clear();
+    this.framedPeers.clear();
     this.retiredResourceIds.fill(0);
     this.retiredResourceKeys.fill(0);
     this.startPromise = undefined;
   }
 
+  private async secureNewPrivateDirectory(pathArg: string): Promise<void> {
+    const uid = this.options.environment.uid;
+    if (uid === undefined) throw new BrowserRuntimeError('FENCED');
+    await plugins.fsPromises.chmod(pathArg, 0o700);
+    const handle = await plugins.fsPromises.open(
+      pathArg,
+      plugins.fs.constants.O_RDONLY
+        | plugins.fs.constants.O_DIRECTORY
+        | plugins.fs.constants.O_NOFOLLOW,
+    );
+    try {
+      await handle.chmod(0o700);
+      await plugins.fsPromises.chmod(pathArg, 0o700);
+      const [pathStat, handleStat] = await Promise.all([
+        plugins.fsPromises.lstat(pathArg, { bigint: true }),
+        handle.stat({ bigint: true }),
+      ]);
+      if (
+        !pathStat.isDirectory()
+        || pathStat.isSymbolicLink()
+        || pathStat.uid !== BigInt(uid)
+        || (pathStat.mode & 0o777n) !== 0o700n
+        || pathStat.dev !== handleStat.dev
+        || pathStat.ino !== handleStat.ino
+      ) throw new BrowserRuntimeError('FENCED', 'runtime directory is not private');
+    } finally {
+      await handle.close();
+    }
+  }
+
   private requireRunning(): void {
-    if (this.lifecycleState !== 'running' || this.lifecycleController.signal.aborted) {
+    if (
+      this.lifecycleState !== 'running'
+      || this.lifecycleController.signal.aborted
+      || !this.ownership.owned
+    ) {
       throw new BrowserRuntimeError('NOT_RUNNING');
     }
   }
 
+  private requireArtifactStore(): BrowserArtifactStore {
+    this.requireRunning();
+    return this.artifactStore;
+  }
+
+  private handleOwnershipLost(): void {
+    if (this.lifecycleState === 'stopped' || this.lifecycleState === 'stopping') return;
+    this.lifecycleEpoch += 1;
+    this.lifecycleController.abort(new BrowserRuntimeError('FENCED'));
+    this.lifecycleState = 'stopping';
+    void this.stopAfterOwnershipLoss();
+  }
+
+  private async stopAfterOwnershipLoss(): Promise<void> {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      try {
+        await this.stop();
+        return;
+      } catch {
+        if (!this.localCleanupPending && !this.ownership.cleanupPending) return;
+        await Promise.resolve();
+      }
+    }
+  }
+
   private async acquireAgentLease(
     slot: IResourceSlot,
     capability: TCapabilityRecord,
@@ -1570,7 +1652,9 @@ export class BrowserRuntime {
     if (slot.session) return;
     this.assertSlotAvailable(slot);
     this.reserveLaunch(slot);
-    const profileDirectory = plugins.path.join(this.profileRoot, randomId(24));
+    const profileRoot = this.profileRoot;
+    if (!profileRoot) throw new BrowserRuntimeError('FENCED');
+    const profileDirectory = plugins.path.join(profileRoot, randomId(24));
     let profileCreated = false;
     let proxy: BrowserEgressProxy | undefined;
     let session: ILiveBrowserSessionLike | undefined;
@@ -1578,7 +1662,7 @@ export class BrowserRuntime {
     try {
       await plugins.fsPromises.mkdir(profileDirectory, { mode: 0o700 });
       profileCreated = true;
-      await plugins.fsPromises.chmod(profileDirectory, 0o700);
+      await this.secureNewPrivateDirectory(profileDirectory);
       proxy = new BrowserEgressProxy({
         projectId: slot.projectId,
         browserResourceId: slot.browserResourceId,
@@ -2199,7 +2283,7 @@ export class BrowserRuntime {
         signal.throwIfAborted();
         this.requireValidLease(lease);
         if (lease.slot.session !== session) throw new BrowserRuntimeError('ABORTED');
-        const artifact = await this.artifactStore.store(
+        const artifact = await this.requireArtifactStore().store(
           lease.capability.projectId,
           lease.capability.browserResourceId,
           snapshot.mimeType,
@@ -2277,6 +2361,7 @@ export class BrowserRuntime {
             lastError: {
               code: truncateString(state.lastError.code, 128),
               fatal: state.lastError.fatal,
+              message: truncateString(state.lastError.message, 2048),
               ...(state.lastError.tabId
                 ? { tabId: truncateString(state.lastError.tabId, 128) }
                 : {}),
@@ -2712,26 +2797,52 @@ export class BrowserRuntime {
       }
       return;
     }
-    if (event.type === 'error' && event.error.fatal && slot.lease) {
-      if (refreshForSession) {
-        this.failFrameRefresh(refreshForSession, 'FRAME_STREAM_FAILED');
-      } else {
-        this.trackCleanup(this.revokeCapabilityRecord(slot.lease.capability));
-      }
-    }
-    if (!subscription || subscription.closed) return;
-    if (event.type === 'state') {
-      this.pushToSubscription(subscription, { type: 'state', state: this.resourceState(event.state) });
-    } else {
-      this.pushToSubscription(subscription, {
+    if (event.type === 'error') {
+      const runtimeEvent = {
         type: 'error',
         error: {
           code: truncateString(event.error.code, 128),
           fatal: event.error.fatal,
+          message: truncateString(event.error.message, 2048),
           ...(event.error.tabId ? { tabId: truncateString(event.error.tabId, 128) } : {}),
         },
-      });
+      } satisfies TBrowserRuntimeEvent;
+      let fatalLease = event.error.fatal && slot.lease?.capability.state === 'active'
+        ? slot.lease
+        : undefined;
+      if (fatalLease) {
+        try {
+          this.requireValidLease(fatalLease);
+        } catch {
+          fatalLease = undefined;
+        }
+      }
+      if (event.error.fatal && !fatalLease) return;
+      if (fatalLease) {
+        const fatalSubscription = subscription
+          && !subscription.closed
+          && subscription.lease === fatalLease
+          ? subscription
+          : undefined;
+        if (refreshForSession?.lease === fatalLease) {
+          this.failFrameRefresh(refreshForSession, 'FRAME_STREAM_FAILED');
+        }
+        this.invalidateCapabilityRecord(fatalLease.capability);
+        try {
+          fatalSubscription?.listener(runtimeEvent);
+        } catch {
+          // Fatal revocation already owns cleanup for a failing listener.
+        } finally {
+          this.trackCleanup(this.revokeCapabilityRecord(fatalLease.capability));
+        }
+        return;
+      }
+      if (!subscription || subscription.closed) return;
+      this.pushToSubscription(subscription, runtimeEvent);
+      return;
     }
+    if (!subscription || subscription.closed) return;
+    this.pushToSubscription(subscription, { type: 'state', state: this.resourceState(event.state) });
   }
 
   private pushToSubscription(