@vgai/engine 0.5.14 → 0.5.16

package/src/adapter/ingest/scene-capture.ts

@@ -1,11 +1,11 @@
 /**
  * Scene capture — the core primitive of unmodified-game ingestion.
  *
- * An external three.js game owns its own `WebGLRenderer`, `Scene`, camera, and
- * render loop. To let the editor inspect/edit that live scene WITHOUT touching
- * the game's code, we need a handle to the game's `Scene`+camera the moment it
- * first renders. The robust way to obtain it is an **accessor trap** on
- * `WebGLRenderer.prototype.render`:
+ * An external three.js game owns its own renderer, `Scene`, camera, and render
+ * loop. To let the editor inspect/edit that live scene WITHOUT touching the
+ * game's code, we need a handle to the game's `Scene`+camera the moment it
+ * first renders. For WebGL, the robust way to obtain it is an **accessor trap**
+ * on `WebGLRenderer.prototype.render`:
  *
  *   - `WebGLRenderer` assigns `this.render` as an OWN instance property inside
  *     its constructor (not on the prototype), and `THREE.WebGLRenderer` is a
@@ -20,12 +20,16 @@
  * (`docs/ingest-study-spike/vgai-ingest-adapter.js`), proven against an
  * unmodified `three.js/examples/games_fps` game.
  *
- * CRITICAL: the trap must be installed on the SAME `three` module instance the
- * game uses. In a bundler/dev-server that dedupes `three` (one `node_modules/
- * three`), an external ESM game's `import 'three'` resolves to that one instance,
- * so passing the host's `THREE` here traps the game's renderer too. A game that
- * bundles its own copy of `three` cannot be captured this way (the module-identity
- * gatekeeper).
+ * DOM-backed addon renderers use ordinary prototype methods instead. The host
+ * supplies those exact shared classes through `additionalRendererCtors`, and
+ * the same observer captures their `(scene, camera)` pair without inventing a
+ * second world model. CSS3DRenderer is the first implementer.
+ *
+ * CRITICAL: every trap must be installed on the SAME `three` module/addon
+ * instance the game uses. In a bundler/dev-server that dedupes `three` (one
+ * `node_modules/three`), an external ESM game's imports resolve to those shared
+ * classes. A game that bundles its own copy cannot be captured this way (the
+ * module-identity gatekeeper).
  *
  * Wave 17 adds an OPTIONAL, ADDITIVE composer capture: a game rendering through
  * its own three.js addon `EffectComposer`
@@ -51,20 +55,43 @@
 
 import type * as THREE from 'three';
 import {
+  documentVisibilityClock,
   startVisibleCaptureWindow,
   type VisibilityClock,
   type VisibleCaptureWindow,
 } from './visible-capture-window';
 
+/** The renderer operations the ingest host may lawfully use after capture.
+ * WebGLRenderer supplies every member; DOM-backed Three addon renderers such as
+ * CSS3DRenderer deliberately omit the WebGL-only lifecycle operations. */
+export interface CapturedThreeRenderer {
+  readonly domElement: HTMLElement;
+  readonly info?: THREE.WebGLInfo | undefined;
+  render(scene: THREE.Scene, camera: THREE.Camera): unknown;
+  setSize(width: number, height: number, updateStyle?: boolean): void;
+  setAnimationLoop?(callback: ((time: number) => void) | null): void;
+  dispose?(): void;
+  getPixelRatio?(): number;
+  getContext?(): unknown;
+  getRenderTarget?(): unknown;
+}
+
 /** A live runtime captured from an external game on its first rendered frame. */
 export interface CapturedRuntime {
   scene: THREE.Scene;
   camera: THREE.Camera;
-  renderer: THREE.WebGLRenderer;
+  renderer: CapturedThreeRenderer;
 }
 
 /** Options for {@link installSceneCapture}. */
 export interface SceneCaptureOptions {
+  /**
+   * Additional shared Three addon renderer classes whose prototype `render`
+   * method carries the same `(scene, camera)` pair as WebGLRenderer. CSS3DRenderer
+   * is the first implementer. The caller must pass the exact class its game
+   * imports; a bundled private copy remains deliberately uncapturable.
+   */
+  additionalRendererCtors?: readonly unknown[];
   /**
    * True when `renderer` is one the HOST constructed for its own drawing.
    *
@@ -230,7 +257,7 @@ export interface SceneCaptureHandle {
   getDrawCount(): number;
   /** The game's last (non-null) `setAnimationLoop` callback for a renderer, so the
    *  host can pause (set null) and resume (re-set it) the game's own loop. */
-  getAnimationLoop(renderer: THREE.WebGLRenderer): ((time: number) => void) | null;
+  getAnimationLoop(renderer: CapturedThreeRenderer): ((time: number) => void) | null;
   /**
    * Wave 17 (D-C3): resize every captured `EffectComposer` that renders
    * through the captured renderer to `w`×`h`, matching its pixel ratio to
@@ -265,6 +292,10 @@ interface EffectComposerCtorLike {
   prototype: { render?: (...args: unknown[]) => unknown };
 }
 
+interface PrototypeRendererCtorLike {
+  prototype: { render?: (...args: unknown[]) => unknown };
+}
+
 /**
  * Install the render accessor trap on `threeNamespace.WebGLRenderer.prototype`.
  * Pass the host's `three` module so the game (which shares it) is trapped.
@@ -318,7 +349,29 @@ export function installSceneCapture(
   // the GETTER returns a wrapper that records each non-null callback before forwarding.
   const REAL_SAL = Symbol('vgai.realSetAnimationLoop');
   const loopCallbacks = new WeakMap<object, (time: number) => void>();
+  const loopedRenderers = new Set<object>();
+  /** Visibility clock of the in-flight `waitForCapture`, if any. */
+  let waitingVisibility: VisibilityClock | null = null;
   const priorSAL = Object.getOwnPropertyDescriptor(proto, 'setAnimationLoop');
+
+  function pumpHiddenLoops(): void {
+    if (
+      captured ||
+      !waitingVisibility ||
+      !(waitingVisibility.suspended?.() ?? waitingVisibility.hidden())
+    ) {
+      return;
+    }
+    for (const renderer of loopedRenderers) {
+      try {
+        loopCallbacks.get(renderer)?.(0);
+      } catch {
+        /* a throwing game frame must not kill the waiter */
+      }
+      if (captured) return;
+    }
+  }
+
   Object.defineProperty(proto, 'setAnimationLoop', {
     configurable: true,
     set(this: Record<symbol, unknown>, fn: unknown) {
@@ -327,7 +380,14 @@ export function installSceneCapture(
     get(this: Record<symbol, unknown>) {
       const self = this;
       return function setAnimationLoop(this: unknown, cb: unknown) {
-        if (cb) loopCallbacks.set(self as object, cb as (time: number) => void);
+        if (cb) {
+          loopCallbacks.set(self as object, cb as (time: number) => void);
+          loopedRenderers.add(self as object);
+          // Hidden tabs park rAF. Play/eval still need a first frame, so when
+          // a waiter is parked we drive the game's own loop once — the same
+          // class of tick `waitSimTime` already uses for a hidden document.
+          queueMicrotask(pumpHiddenLoops);
+        }
         const real = self[REAL_SAL];
         return typeof real === 'function'
           ? (real as (...a: unknown[]) => unknown).call(self, cb)
@@ -371,7 +431,7 @@ export function installSceneCapture(
     captured = {
       scene: scene as THREE.Scene,
       camera: camera as THREE.Camera,
-      renderer: self as THREE.WebGLRenderer,
+      renderer: self as CapturedThreeRenderer,
     };
     opts?.onWorldAdoption?.({
       phase: 'adopted',
@@ -405,6 +465,17 @@ export function installSceneCapture(
     });
   }
 
+  /** Observe the common `(scene, camera)` render contract once, regardless of
+   * whether it came from WebGLRenderer's constructor-assigned method or an
+   * addon's ordinary prototype method. */
+  function observeRendererRender(self: unknown, scene: unknown, camera: unknown): void {
+    if (!captured || self === captured.renderer) drawCount++;
+    if (opts?.isHostRenderer?.(self) === true) return;
+    if (!(scene as { isScene?: boolean })?.isScene) return;
+    if (captured) recordAlternateWorld(self, scene, camera);
+    else adoptWorld(self, scene, camera);
+  }
+
   Object.defineProperty(proto, 'render', {
     configurable: true,
     set(this: Record<symbol, unknown>, fn: unknown) {
@@ -423,7 +494,6 @@ export function installSceneCapture(
         // `setAnimationLoop`, so they ARE gateable) and the instrument was
         // reporting someone else's frames. Before capture every render still
         // counts: that is how the first game frame is detected at all.
-        if (!captured || (self as unknown) === captured.renderer) drawCount++;
         // Never capture a scene the HOST owns. The trap sits on the shared
         // `WebGLRenderer.prototype`, so the editor's own viewport renders
         // arrive here too — and when a game bundles its own MISMATCHED three,
@@ -434,15 +504,39 @@ export function installSceneCapture(
         // Whether that happened at all came down to whether the editor
         // rendered a frame inside the game's capture window, so the same
         // session could pass or fail on timing alone.
-        if (opts?.isHostRenderer?.(self) !== true && (scene as { isScene?: boolean })?.isScene) {
-          if (captured) recordAlternateWorld(self, scene, camera);
-          else adoptWorld(self, scene, camera);
-        }
+        observeRendererRender(self, scene, camera);
         return forwardRender(self, args);
       };
     },
   });
 
+  // DOM-backed Three renderers (CSS3DRenderer is the first) use a normal
+  // prototype method rather than WebGLRenderer's constructor assignment. Trap
+  // only classes explicitly supplied by the host, and restore each byte-for-byte.
+  const additionalRendererRestores: Array<() => void> = [];
+  for (const candidate of opts?.additionalRendererCtors ?? []) {
+    const ctor = candidate as PrototypeRendererCtorLike;
+    const rendererProto = ctor?.prototype;
+    const priorRender = rendererProto?.render;
+    if (!rendererProto || typeof priorRender !== 'function' || rendererProto === proto) continue;
+    rendererProto.render = function (this: unknown, ...args: unknown[]) {
+      observeRendererRender(this, args[0], args[1]);
+      const hooks = renderPassHooks;
+      if (hooks === null || !captured || this !== captured.renderer) {
+        return priorRender.apply(this, args);
+      }
+      hooks.before();
+      try {
+        return priorRender.apply(this, args);
+      } finally {
+        hooks.after();
+      }
+    };
+    additionalRendererRestores.push(() => {
+      rendererProto.render = priorRender;
+    });
+  }
+
   // ---- Wave 17 (D-C2): additive composer capture, only when a composer ctor
   // was passed. `EffectComposer` methods (including `render`) live on the
   // PROTOTYPE (a plain ES class — the constructor never does `this.render =
@@ -494,7 +588,8 @@ export function installSceneCapture(
       // touches `renderer.getPixelRatio()` — a non-composer game's captured
       // renderer need not even expose that method for this to stay a no-op.
       if (!captured || composers.size === 0) return;
-      const pixelRatio = captured.renderer.getPixelRatio();
+      const pixelRatio = captured.renderer.getPixelRatio?.();
+      if (pixelRatio === undefined) return;
       for (const composer of composers) {
         composer.setSize(w, h);
         composer.setPixelRatio(pixelRatio);
@@ -517,12 +612,13 @@ export function installSceneCapture(
           onExpire: () => {
             const i = waiters.indexOf(wrapped);
             if (i >= 0) waiters.splice(i, 1);
+            waitingVisibility = null;
             opts.onWait?.(null);
             reject(
               new Error(
                 `Scene capture timed out after ${timeoutMs}ms of VISIBLE time ` +
-                  `(${Math.round(captureWindow.elapsedHiddenMs())}ms hidden, which is not counted — ` +
-                  'a hidden tab cannot render) — the game never rendered, ' +
+                  `(${Math.round(captureWindow.elapsedHiddenMs())}ms browser-suspended, which is ` +
+                  'not counted because no frame can be presented) — the game never rendered, ' +
                   'or it bundles its own (un-shared) copy of three.',
               ),
             );
@@ -530,11 +626,17 @@ export function installSceneCapture(
         });
         const wrapped = (rt: CapturedRuntime) => {
           captureWindow.cancel();
+          waitingVisibility = null;
           opts.onWait?.(null);
           resolve(rt);
         };
         waiters.push(wrapped);
+        waitingVisibility = opts.visibility ?? documentVisibilityClock();
         opts.onWait?.(captureWindow);
+        // A waiter that starts already-hidden (the normal `vgai play` path
+        // against a backgrounded tab) must not wait for a human to foreground
+        // it. Pump any loop the game has already registered.
+        queueMicrotask(pumpHiddenLoops);
       });
     },
     uninstall() {
@@ -577,6 +679,7 @@ export function installSceneCapture(
       // prototype method — no per-instance own-property to restore, unlike
       // the renderer/loop traps above).
       restoreComposerTrap();
+      for (const restore of additionalRendererRestores) restore();
     },
   };
 }

package/src/adapter/ingest/visible-capture-window.ts

@@ -46,6 +46,16 @@ export interface VisibilityClock {
   now(): number;
   /** True while the document is hidden (no rAF, so no frame can be captured). */
   hidden(): boolean;
+  /**
+   * True while the browser is not presenting frames. This is wider than
+   * `hidden()`: WebKit also stops rAF for an unfocused window (and can suspend
+   * the page entirely) while `document.hidden` still reads false.
+   *
+   * Optional for compatibility with injected clocks; absent means `hidden()`.
+   */
+  suspended?(): boolean;
+  /** Why {@link suspended} is true, when the clock can say. */
+  suspensionReason?(): 'hidden' | 'unfocused' | 'page-suspended' | null;
   /** Subscribe to visibility transitions; returns the unsubscribe. */
   subscribe(onChange: () => void): () => void;
 }
@@ -55,13 +65,46 @@ export interface VisibilityClock {
  *  honest answer for a headless caller — it has no tab to background. */
 export function documentVisibilityClock(): VisibilityClock {
   const doc = typeof document === 'undefined' ? null : document;
+  const win = doc?.defaultView ?? null;
+  let pageSuspended = false;
+  const reason = (): 'hidden' | 'unfocused' | 'page-suspended' | null => {
+    if (pageSuspended) return 'page-suspended';
+    if (doc?.hidden === true) return 'hidden';
+    // WebKit stops rAF when the browser window becomes inactive, including
+    // when another app is in front. Page Visibility can still say `visible`
+    // in that state. Focus is a platform fact, not a Safari/UA sniff, and in
+    // browsers that keep rendering while unfocused this merely parks the
+    // expiry clock until either a frame arrives or focus returns.
+    if (doc && typeof doc.hasFocus === 'function' && !doc.hasFocus()) return 'unfocused';
+    return null;
+  };
   return {
     now: () => (typeof performance === 'undefined' ? Date.now() : performance.now()),
     hidden: () => doc?.hidden === true,
+    suspended: () => reason() !== null,
+    suspensionReason: reason,
     subscribe(onChange) {
       if (!doc) return () => {};
+      const onPageHide = () => {
+        pageSuspended = true;
+        onChange();
+      };
+      const onPageShow = () => {
+        pageSuspended = false;
+        onChange();
+      };
       doc.addEventListener('visibilitychange', onChange);
-      return () => doc.removeEventListener('visibilitychange', onChange);
+      win?.addEventListener('blur', onChange);
+      win?.addEventListener('focus', onChange);
+      win?.addEventListener('pagehide', onPageHide);
+      win?.addEventListener('pageshow', onPageShow);
+      return () => {
+        doc.removeEventListener('visibilitychange', onChange);
+        win?.removeEventListener('blur', onChange);
+        win?.removeEventListener('focus', onChange);
+        win?.removeEventListener('pagehide', onPageHide);
+        win?.removeEventListener('pageshow', onPageShow);
+      };
     },
   };
 }
@@ -137,19 +180,26 @@ export interface VisibleCaptureWindow {
   readonly budgetMs: number;
   /** Visible milliseconds spent so far. */
   elapsedVisibleMs(): number;
-  /** Milliseconds this window has spent parked while hidden (not counted). */
+  /** Milliseconds this window has spent browser-suspended (not counted). */
   elapsedHiddenMs(): number;
-  /** Whether the wait is parked right now because the document is hidden. */
+  /**
+   * Whether the document itself is hidden. Kept distinct from browser
+   * suspension so status never calls an unfocused, still-visible window hidden.
+   */
   isHidden(): boolean;
+  /** Whether the expiry budget is parked because the browser is not presenting frames. */
+  isSuspended(): boolean;
+  /** The browser condition parking the budget, when observable. */
+  suspensionReason(): 'hidden' | 'unfocused' | 'page-suspended' | null;
   /** Stop the timer and drop the visibility listener. Idempotent. */
   cancel(): void;
 }
 
 /**
  * Start a window that calls `onExpire` after `budgetMs` of VISIBLE time.
- * While the document is hidden the timer is disarmed entirely (so a throttled
- * background timer cannot fire it late either) and re-armed with the remaining
- * budget when the document comes back.
+ * While browser frame presentation is suspended the timer is disarmed entirely
+ * (so a throttled background timer cannot fire it late either) and re-armed
+ * with the remaining budget when frame presentation resumes.
  */
 export function startVisibleCaptureWindow(opts: {
   budgetMs: number;
@@ -164,7 +214,8 @@ export function startVisibleCaptureWindow(opts: {
   const setTimer = opts.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
   const clearTimer = opts.clearTimer ?? ((h) => clearTimeout(h as ReturnType<typeof setTimeout>));
 
-  let state = beginVisibleBudget(opts.budgetMs, clock.now(), clock.hidden());
+  const suspended = () => clock.suspended?.() ?? clock.hidden();
+  let state = beginVisibleBudget(opts.budgetMs, clock.now(), suspended());
   let timer: unknown = null;
   let done = false;
 
@@ -192,7 +243,7 @@ export function startVisibleCaptureWindow(opts: {
 
   const unsubscribe = clock.subscribe(() => {
     if (done) return;
-    state = applyVisibility(state, clock.now(), clock.hidden());
+    state = applyVisibility(state, clock.now(), suspended());
     arm();
   });
 
@@ -202,7 +253,10 @@ export function startVisibleCaptureWindow(opts: {
     budgetMs: opts.budgetMs,
     elapsedVisibleMs: () => visibleElapsedMs(state, clock.now()),
     elapsedHiddenMs: () => hiddenElapsedMs(state, clock.now()),
-    isHidden: () => state.hidden,
+    isHidden: () => clock.hidden(),
+    isSuspended: () => state.hidden,
+    suspensionReason: () =>
+      state.hidden ? (clock.suspensionReason?.() ?? (clock.hidden() ? 'hidden' : null)) : null,
     cancel() {
       if (done) return;
       done = true;