@lightningjs/renderer 2.12.1 → 2.13.1

package/src/core/CoreTextureManager.ts

@@ -28,6 +28,10 @@ import { TextureType, type Texture } from './textures/Texture.js';
 import { EventEmitter } from '../common/EventEmitter.js';
 import { getTimeStamp } from './platform.js';
 import type { Stage } from './Stage.js';
+import {
+  validateCreateImageBitmap,
+  type CreateImageBitmapSupport,
+} from './lib/validateImageBitmap.js';
 
 /**
  * Augmentable map of texture class types
@@ -45,12 +49,6 @@ export interface TextureMap {
   RenderTexture: typeof RenderTexture;
 }
 
-export interface CreateImageBitmapSupport {
-  basic: boolean; // Supports createImageBitmap(image)
-  options: boolean; // Supports createImageBitmap(image, options)
-  full: boolean; // Supports createImageBitmap(image, sx, sy, sw, sh, options)
-}
-
 export type ExtractProps<Type> = Type extends { z$__type__Props: infer Props }
   ? Props
   : never;
@@ -63,6 +61,11 @@ export interface TextureManagerDebugInfo {
   keyCacheSize: number;
 }
 
+export interface TextureManagerSettings {
+  numImageWorkers: number;
+  createImageBitmapSupport: 'auto' | 'basic' | 'options' | 'full';
+}
+
 export type ResizeModeOptions =
   | {
       /**
@@ -180,6 +183,7 @@ export class CoreTextureManager extends EventEmitter {
   private uploadTextureQueue: Array<Texture> = [];
   private initialized = false;
   private stage: Stage;
+  private numImageWorkers: number;
 
   imageWorkerManager: ImageWorkerManager | null = null;
   hasCreateImageBitmap = !!self.createImageBitmap;
@@ -210,48 +214,34 @@ export class CoreTextureManager extends EventEmitter {
    */
   frameTime = 0;
 
-  constructor(stage: Stage, numImageWorkers: number) {
+  constructor(stage: Stage, settings: TextureManagerSettings) {
     super();
+
+    const { numImageWorkers, createImageBitmapSupport } = settings;
     this.stage = stage;
-    this.validateCreateImageBitmap()
-      .then((result) => {
-        this.hasCreateImageBitmap =
-          result.basic || result.options || result.full;
-        this.imageBitmapSupported = result;
+    this.numImageWorkers = numImageWorkers;
 
-        if (!this.hasCreateImageBitmap) {
+    if (createImageBitmapSupport === 'auto') {
+      validateCreateImageBitmap()
+        .then((result) => {
+          this.initialize(result);
+        })
+        .catch((e) => {
           console.warn(
             '[Lightning] createImageBitmap is not supported on this browser. ImageTexture will be slower.',
           );
-        }
 
-        if (
-          this.hasCreateImageBitmap &&
-          this.hasWorker &&
-          numImageWorkers > 0
-        ) {
-          this.imageWorkerManager = new ImageWorkerManager(
-            numImageWorkers,
-            result,
-          );
-        } else {
-          console.warn(
-            '[Lightning] Imageworker is 0 or not supported on this browser. Image loading will be slower.',
-          );
-        }
-
-        this.initialized = true;
-        this.emit('initialized');
-      })
-      .catch((e) => {
-        console.warn(
-          '[Lightning] createImageBitmap is not supported on this browser. ImageTexture will be slower.',
-        );
-
-        // initialized without image worker manager and createImageBitmap
-        this.initialized = true;
-        this.emit('initialized');
+          // initialized without image worker manager and createImageBitmap
+          this.initialized = true;
+          this.emit('initialized');
+        });
+    } else {
+      this.initialize({
+        basic: createImageBitmapSupport === 'basic',
+        options: createImageBitmapSupport === 'options',
+        full: createImageBitmapSupport === 'full',
       });
+    }
 
     this.registerTextureType('ImageTexture', ImageTexture);
     this.registerTextureType('ColorTexture', ColorTexture);
@@ -260,77 +250,6 @@ export class CoreTextureManager extends EventEmitter {
     this.registerTextureType('RenderTexture', RenderTexture);
   }
 
-  private async validateCreateImageBitmap(): Promise<CreateImageBitmapSupport> {
-    // Test if createImageBitmap is supported using a simple 1x1 PNG image
-    // prettier-ignore
-    const pngBinaryData = new Uint8Array([
-      0x89, 0x50, 0x4e, 0x47,
-      0x0d, 0x0a, 0x1a, 0x0a, // PNG signature
-      0x00, 0x00, 0x00, 0x0d, // IHDR chunk length
-      0x49, 0x48, 0x44, 0x52, // "IHDR" chunk type
-      0x00, 0x00, 0x00, 0x01, // Width: 1
-      0x00, 0x00, 0x00, 0x01, // Height: 1
-      0x01,                   // Bit depth: 1
-      0x03,                   // Color type: Indexed
-      0x00,                   // Compression method: Deflate
-      0x00,                   // Filter method: None
-      0x00,                   // Interlace method: None
-      0x25, 0xdb, 0x56, 0xca, // CRC for IHDR
-      0x00, 0x00, 0x00, 0x03, // PLTE chunk length
-      0x50, 0x4c, 0x54, 0x45, // "PLTE" chunk type
-      0x00, 0x00, 0x00,       // Palette entry: Black
-      0xa7, 0x7a, 0x3d, 0xda, // CRC for PLTE
-      0x00, 0x00, 0x00, 0x01, // tRNS chunk length
-      0x74, 0x52, 0x4e, 0x53, // "tRNS" chunk type
-      0x00,                   // Transparency for black: Fully transparent
-      0x40, 0xe6, 0xd8, 0x66, // CRC for tRNS
-      0x00, 0x00, 0x00, 0x0a, // IDAT chunk length
-      0x49, 0x44, 0x41, 0x54, // "IDAT" chunk type
-      0x08, 0xd7,             // Deflate header
-      0x63, 0x60, 0x00, 0x00,
-      0x00, 0x02, 0x00, 0x01, // Zlib-compressed data
-      0xe2, 0x21, 0xbc, 0x33, // CRC for IDAT
-      0x00, 0x00, 0x00, 0x00, // IEND chunk length
-      0x49, 0x45, 0x4e, 0x44, // "IEND" chunk type
-      0xae, 0x42, 0x60, 0x82, // CRC for IEND
-    ]);
-
-    const support: CreateImageBitmapSupport = {
-      basic: false,
-      options: false,
-      full: false,
-    };
-
-    // Test basic createImageBitmap support
-    const blob = new Blob([pngBinaryData], { type: 'image/png' });
-    const bitmap = await createImageBitmap(blob);
-    bitmap.close?.();
-    support.basic = true;
-
-    // Test createImageBitmap with options support
-    try {
-      const options = { premultiplyAlpha: 'none' as const };
-      const bitmapWithOptions = await createImageBitmap(blob, options);
-      bitmapWithOptions.close?.();
-      support.options = true;
-    } catch (e) {
-      /* ignore */
-    }
-
-    // Test createImageBitmap with full options support
-    try {
-      const bitmapWithFullOptions = await createImageBitmap(blob, 0, 0, 1, 1, {
-        premultiplyAlpha: 'none',
-      });
-      bitmapWithFullOptions.close?.();
-      support.full = true;
-    } catch (e) {
-      /* ignore */
-    }
-
-    return support;
-  }
-
   registerTextureType<Type extends keyof TextureMap>(
     textureType: Type,
     textureClass: TextureMap[Type],
@@ -338,6 +257,36 @@ export class CoreTextureManager extends EventEmitter {
     this.txConstructors[textureType] = textureClass;
   }
 
+  private initialize(support: CreateImageBitmapSupport) {
+    this.hasCreateImageBitmap =
+      support.basic || support.options || support.full;
+    this.imageBitmapSupported = support;
+
+    if (!this.hasCreateImageBitmap) {
+      console.warn(
+        '[Lightning] createImageBitmap is not supported on this browser. ImageTexture will be slower.',
+      );
+    }
+
+    if (
+      this.hasCreateImageBitmap &&
+      this.hasWorker &&
+      this.numImageWorkers > 0
+    ) {
+      this.imageWorkerManager = new ImageWorkerManager(
+        this.numImageWorkers,
+        support,
+      );
+    } else {
+      console.warn(
+        '[Lightning] Imageworker is 0 or not supported on this browser. Image loading will be slower.',
+      );
+    }
+
+    this.initialized = true;
+    this.emit('initialized');
+  }
+
   /**
    * Enqueue a texture for downloading its source image.
    */
@@ -392,15 +341,7 @@ export class CoreTextureManager extends EventEmitter {
 
   orphanTexture(texture: Texture): void {
     // if it is part of the download or upload queue, remove it
-    const downloadIndex = this.downloadTextureSourceQueue.indexOf(texture);
-    if (downloadIndex !== -1) {
-      this.downloadTextureSourceQueue.splice(downloadIndex, 1);
-    }
-
-    const uploadIndex = this.uploadTextureQueue.indexOf(texture);
-    if (uploadIndex !== -1) {
-      this.uploadTextureQueue.splice(uploadIndex, 1);
-    }
+    this.removeTextureFromQueue(texture);
 
     if (texture.type === TextureType.subTexture) {
       // ignore subtextures
@@ -419,11 +360,42 @@ export class CoreTextureManager extends EventEmitter {
   loadTexture(texture: Texture, priority?: boolean): void {
     this.stage.txMemManager.removeFromOrphanedTextures(texture);
 
-    if (texture.state !== 'initial' && texture.state !== 'freed') {
+    if (texture.type === TextureType.subTexture) {
+      // ignore subtextures - they get loaded through their parent
       return;
     }
 
-    texture.setState('loading');
+    // if the texture is already loaded, don't load it again
+    if (
+      texture.ctxTexture !== undefined &&
+      texture.ctxTexture.state === 'loaded'
+    ) {
+      texture.setState('loaded');
+      return;
+    }
+
+    // if the texture is already being processed, don't load it again
+    if (
+      this.downloadTextureSourceQueue.includes(texture) === true ||
+      this.uploadTextureQueue.includes(texture) === true
+    ) {
+      return;
+    }
+
+    // if the texture is already loading, free it, this can happen if the texture is
+    // orphaned and then reloaded
+    if (
+      texture.ctxTexture !== undefined &&
+      texture.ctxTexture.state === 'loading'
+    ) {
+      // if the texture has texture data, queue it for upload
+      if (texture.textureData !== null) {
+        this.enqueueUploadTexture(texture);
+      }
+
+      // else we will have to re-download the texture
+      texture.free();
+    }
 
     // if we're not initialized, just queue the texture into the priority queue
     if (this.initialized === false) {
@@ -435,15 +407,17 @@ export class CoreTextureManager extends EventEmitter {
     // Technically the noise texture shouldn't either, but it's a special case
     // and not really used in production so who cares ¯\_(ツ)_/¯
     if (
-      texture.type === TextureType.subTexture ||
-      texture.type === TextureType.color ||
-      texture.type === TextureType.renderToTexture
+      (texture.type === TextureType.color ||
+        texture.type === TextureType.renderToTexture) &&
+      texture.state !== 'initial'
     ) {
       texture.setState('fetched');
       this.enqueueUploadTexture(texture);
       return;
     }
 
+    texture.setState('loading');
+
     // prioritize the texture for immediate loading
     if (priority === true) {
       texture
@@ -466,11 +440,21 @@ export class CoreTextureManager extends EventEmitter {
    * @param texture Texture to upload
    */
   uploadTexture(texture: Texture): void {
-    if (texture.state !== 'fetched') {
+    if (
+      this.stage.txMemManager.doNotExceedCriticalThreshold === true &&
+      this.stage.txMemManager.criticalCleanupRequested === true
+    ) {
+      // we're at a critical memory threshold, don't upload textures
+      this.enqueueUploadTexture(texture);
       return;
     }
 
     const coreContext = texture.loadCtxTexture();
+    if (coreContext !== null && coreContext.state === 'loaded') {
+      texture.setState('loaded');
+      return;
+    }
+
     coreContext.load();
   }
 
@@ -576,6 +560,23 @@ export class CoreTextureManager extends EventEmitter {
     }
   }
 
+  /**
+   * Remove texture from the queue's
+   *
+   * @param texture - The texture to remove
+   */
+  removeTextureFromQueue(texture: Texture): void {
+    const downloadIndex = this.downloadTextureSourceQueue.indexOf(texture);
+    if (downloadIndex !== -1) {
+      this.downloadTextureSourceQueue.splice(downloadIndex, 1);
+    }
+
+    const uploadIndex = this.uploadTextureQueue.indexOf(texture);
+    if (uploadIndex !== -1) {
+      this.uploadTextureQueue.splice(uploadIndex, 1);
+    }
+  }
+
   /**
    * Resolve a parent texture from the cache or fallback to the provided texture.
    *

package/src/core/Stage.ts

@@ -20,6 +20,7 @@ import { startLoop, getTimeStamp } from './platform.js';
 import { assertTruthy, setPremultiplyMode } from '../utils.js';
 import { AnimationManager } from './animations/AnimationManager.js';
 import {
+  UpdateType,
   CoreNode,
   CoreNodeRenderState,
   type CoreNodeProps,
@@ -77,6 +78,7 @@ export interface StageOptions {
   inspector: boolean;
   strictBounds: boolean;
   textureProcessingTimeLimit: number;
+  createImageBitmapSupport: 'auto' | 'basic' | 'options' | 'full';
 }
 
 export type StageFpsUpdateHandler = (
@@ -102,7 +104,7 @@ export class Stage {
   public readonly shManager: CoreShaderManager;
   public readonly renderer: CoreRenderer;
   public readonly root: CoreNode;
-  public readonly boundsMargin: [number, number, number, number];
+  public boundsMargin: [number, number, number, number];
   public readonly defShaderCtr: BaseShaderController;
   public readonly strictBound: Bound;
   public readonly preloadBound: Bound;
@@ -150,10 +152,14 @@ export class Stage {
       textureMemory,
       renderEngine,
       fontEngines,
+      createImageBitmapSupport,
     } = options;
 
     this.eventBus = options.eventBus;
-    this.txManager = new CoreTextureManager(this, numImageWorkers);
+    this.txManager = new CoreTextureManager(this, {
+      numImageWorkers,
+      createImageBitmapSupport,
+    });
 
     // Wait for the Texture Manager to initialize
     // once it does, request a render
@@ -240,6 +246,7 @@ export class Stage {
       height: appHeight,
       alpha: 1,
       autosize: false,
+      boundsMargin: null,
       clipping: false,
       color: 0x00000000,
       colorTop: 0x00000000,
@@ -307,6 +314,7 @@ export class Stage {
    * Create default PixelTexture
    */
   createDefaultTexture() {
+    console.log('Creating default texture');
     (this.defaultTexture as ColorTexture) = this.txManager.createTexture(
       'ColorTexture',
       {
@@ -315,7 +323,6 @@ export class Stage {
     );
 
     assertTruthy(this.defaultTexture instanceof ColorTexture);
-
     this.txManager.loadTexture(this.defaultTexture, true);
 
     // Mark the default texture as ALWAYS renderable
@@ -372,8 +379,13 @@ export class Stage {
     renderer.reset();
 
     // Check if we need to cleanup textures
-    if (this.txMemManager.criticalCleanupRequested) {
-      this.txMemManager.cleanup();
+    if (this.txMemManager.criticalCleanupRequested === true) {
+      this.txMemManager.cleanup(false);
+
+      if (this.txMemManager.criticalCleanupRequested === true) {
+        // If we still need to cleanup, request another but aggressive cleanup
+        this.txMemManager.cleanup(true);
+      }
     }
 
     // If we have RTT nodes draw them first
@@ -636,6 +648,14 @@ export class Stage {
     return new CoreTextNode(this, resolvedProps, resolvedTextRenderer);
   }
 
+  setBoundsMargin(value: number | [number, number, number, number]) {
+    this.boundsMargin = Array.isArray(value)
+      ? value
+      : [value, value, value, value];
+
+    this.root.setUpdateType(UpdateType.RenderBounds);
+  }
+
   /**
    * Resolves the default property values for a Node
    *
@@ -669,6 +689,7 @@ export class Stage {
       height: props.height ?? 0,
       alpha: props.alpha ?? 1,
       autosize: props.autosize ?? false,
+      boundsMargin: props.boundsMargin ?? null,
       clipping: props.clipping ?? false,
       color,
       colorTop: props.colorTop ?? color,
@@ -716,7 +737,7 @@ export class Stage {
    * @remarks
    * This method is used to cleanup orphaned textures that are no longer in use.
    */
-  cleanup() {
-    this.txMemManager.cleanup();
+  cleanup(aggressive: boolean) {
+    this.txMemManager.cleanup(aggressive);
   }
 }

package/src/core/TextureMemoryManager.ts

@@ -19,7 +19,7 @@
 import { isProductionEnvironment } from '../utils.js';
 import { getTimeStamp } from './platform.js';
 import type { Stage } from './Stage.js';
-import type { Texture } from './textures/Texture.js';
+import { TextureType, type Texture } from './textures/Texture.js';
 import { bytesToMb } from './utils.js';
 
 export interface TextureMemoryManagerSettings {
@@ -53,13 +53,13 @@ export interface TextureMemoryManagerSettings {
   targetThresholdLevel: number;
 
   /**
-   * Interval between Texture Cleanups (in milliseconds)
+   * Interval between non-aggressive Texture Cleanups (in milliseconds)
    *
    * @remarks
-   * Texture Memory Manager will perform a Texture Cleanup no more
-   * frequently than this interval generally when the scene becomes idle.
+   * Texture Memory Manager will perform a non aggressive Texture Cleanup no more
+   * frequently than this interval when the scene becomes idle.
    *
-   * @defaultValue `30,000` (30 seconds)
+   * @defaultValue `5,000` (5 seconds)
    */
   cleanupInterval: number;
 
@@ -80,6 +80,13 @@ export interface TextureMemoryManagerSettings {
    * @defaultValue `25e6` (25 MB)
    */
   baselineMemoryAllocation: number;
+
+  /**
+   * Do not exceed critical threshold
+   *
+   * @defaultValue `false`
+   */
+  doNotExceedCriticalThreshold: boolean;
 }
 
 export interface MemoryInfo {
@@ -117,7 +124,10 @@ export class TextureMemoryManager {
   private debugLogging: boolean;
   private lastCleanupTime = 0;
   private baselineMemoryAllocation: number;
+
   public criticalCleanupRequested = false;
+  public doNotExceedCriticalThreshold: boolean;
+
   /**
    * The current frame time in milliseconds
    *
@@ -129,18 +139,22 @@ export class TextureMemoryManager {
   public frameTime = 0;
 
   constructor(private stage: Stage, settings: TextureMemoryManagerSettings) {
-    const { criticalThreshold } = settings;
+    const { criticalThreshold, doNotExceedCriticalThreshold } = settings;
+    this.doNotExceedCriticalThreshold = doNotExceedCriticalThreshold || false;
     this.criticalThreshold = Math.round(criticalThreshold);
     const targetFraction = Math.max(
       0,
       Math.min(1, settings.targetThresholdLevel),
     );
-    this.targetThreshold = Math.round(criticalThreshold * targetFraction);
     this.cleanupInterval = settings.cleanupInterval;
     this.debugLogging = settings.debugLogging;
     this.baselineMemoryAllocation = Math.round(
       settings.baselineMemoryAllocation,
     );
+    this.targetThreshold = Math.max(
+      Math.round(criticalThreshold * targetFraction),
+      this.baselineMemoryAllocation,
+    );
     this.memUsed = Math.round(settings.baselineMemoryAllocation);
 
     if (settings.debugLogging) {
@@ -229,24 +243,11 @@ export class TextureMemoryManager {
     );
   }
 
-  cleanup() {
-    const critical = this.criticalCleanupRequested;
-    this.lastCleanupTime = this.frameTime;
-    this.criticalCleanupRequested = false;
-
-    if (critical === true) {
-      this.stage.queueFrameEvent('criticalCleanup', {
-        memUsed: this.memUsed,
-        criticalThreshold: this.criticalThreshold,
-      });
-    }
-
-    if (this.debugLogging) {
-      console.log(
-        `[TextureMemoryManager] Cleaning up textures. Critical: ${critical}`,
-      );
-    }
+  checkCriticalCleanup() {
+    return this.memUsed > this.criticalThreshold;
+  }
 
+  cleanupQuick(critical: boolean) {
     // Free non-renderable textures until we reach the target threshold
     const memTarget = this.targetThreshold;
     const txManager = this.stage.txManager;
@@ -255,10 +256,13 @@ export class TextureMemoryManager {
     while (
       this.memUsed >= memTarget &&
       this.orphanedTextures.length > 0 &&
-      // if it a non-critical cleanup, we will only cleanup for 10ms
       (critical || getTimeStamp() - timestamp < 10)
     ) {
-      const texture = this.orphanedTextures.shift()!;
+      const texture = this.orphanedTextures.shift();
+
+      if (texture === undefined) {
+        continue;
+      }
 
       if (texture.renderable === true) {
         // If the texture is renderable, we can't free it up
@@ -268,6 +272,80 @@ export class TextureMemoryManager {
       texture.free();
       txManager.removeTextureFromCache(texture);
     }
+  }
+
+  cleanupDeep(critical: boolean) {
+    // Free non-renderable textures until we reach the target threshold
+    const memTarget = critical ? this.criticalThreshold : this.targetThreshold;
+    const txManager = this.stage.txManager;
+
+    // sort by renderability
+    const filteredAndSortedTextures: Texture[] = [];
+    const textures = [...this.loadedTextures.keys()];
+    for (let i = 0; i < textures.length; i++) {
+      const texture = textures[i];
+      if (texture === undefined) {
+        continue;
+      }
+
+      if (
+        texture.type === TextureType.image ||
+        texture.type === TextureType.noise ||
+        texture.type === TextureType.renderToTexture
+      ) {
+        if (texture.renderable === true) {
+          filteredAndSortedTextures.push(texture);
+        } else {
+          filteredAndSortedTextures.unshift(texture);
+        }
+      }
+    }
+
+    while (this.memUsed >= memTarget && filteredAndSortedTextures.length > 0) {
+      const texture = filteredAndSortedTextures.shift();
+      if (texture === undefined) {
+        continue;
+      }
+
+      if (texture.preventCleanup === true) {
+        continue;
+      }
+
+      if (texture.renderable === true) {
+        break;
+      }
+
+      texture.free();
+      this.removeFromOrphanedTextures(texture);
+      txManager.removeTextureFromCache(texture);
+      txManager.removeTextureFromQueue(texture);
+    }
+  }
+
+  cleanup(aggressive: boolean = false) {
+    const critical = this.criticalCleanupRequested;
+    this.lastCleanupTime = this.frameTime;
+
+    if (critical === true) {
+      this.stage.queueFrameEvent('criticalCleanup', {
+        memUsed: this.memUsed,
+        criticalThreshold: this.criticalThreshold,
+      });
+    }
+
+    if (this.debugLogging === true) {
+      console.log(
+        `[TextureMemoryManager] Cleaning up textures. Critical: ${critical}. Aggressive: ${aggressive}`,
+      );
+    }
+
+    // try a quick cleanup first
+    this.cleanupQuick(critical);
+
+    // if we're still above the target threshold, do a deep cleanup
+    if (aggressive === true && this.memUsed >= this.criticalThreshold) {
+      this.cleanupDeep(critical);
+    }
 
     if (this.memUsed >= this.criticalThreshold) {
       this.stage.queueFrameEvent('criticalCleanupFailed', {
@@ -280,6 +358,8 @@ export class TextureMemoryManager {
           `[TextureMemoryManager] Memory usage above critical threshold after cleanup: ${this.memUsed}`,
         );
       }
+    } else {
+      this.criticalCleanupRequested = false;
     }
   }
 
@@ -299,8 +379,9 @@ export class TextureMemoryManager {
           acc + (texture.renderable ? this.loadedTextures.get(texture)! : 0)
         );
       },
-      0,
+      this.baselineMemoryAllocation,
     );
+
     return {
       criticalThreshold: this.criticalThreshold,
       targetThreshold: this.targetThreshold,