@luma.gl/core 9.2.5 → 9.3.0-alpha.10

package/src/portable/shader-block-writer.ts

@@ -0,0 +1,254 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {CompositeUniformValue, UniformValue} from '../adapter/types/uniforms';
+import {getScratchArrayBuffer} from '../utils/array-utils-flat';
+import {isNumberArray} from '../utils/is-array';
+import {log} from '../utils/log';
+import type {
+  CompositeShaderType,
+  VariableShaderType
+} from '../shadertypes/shader-types/shader-types';
+import {
+  getLeafLayoutInfo,
+  isCompositeShaderTypeStruct,
+  type ShaderBlockLayout
+} from '../shadertypes/shader-types/shader-block-layout';
+
+/**
+ * Serializes nested JavaScript uniform values according to a {@link ShaderBlockLayout}.
+ */
+export class ShaderBlockWriter {
+  /** Layout metadata used to flatten and serialize values. */
+  readonly layout: ShaderBlockLayout;
+
+  /**
+   * Creates a writer for a precomputed shader-block layout.
+   */
+  constructor(layout: ShaderBlockLayout) {
+    this.layout = layout;
+  }
+
+  /**
+   * Returns `true` if the flattened layout contains the given field.
+   */
+  has(name: string): boolean {
+    return Boolean(this.layout.fields[name]);
+  }
+
+  /**
+   * Returns offset and size metadata for a flattened field.
+   */
+  get(name: string): {offset: number; size: number} | undefined {
+    const entry = this.layout.fields[name];
+    return entry ? {offset: entry.offset, size: entry.size} : undefined;
+  }
+
+  /**
+   * Flattens nested composite values into leaf-path values understood by {@link UniformBlock}.
+   *
+   * Top-level values may be supplied either in nested object form matching the
+   * declared composite shader types or as already-flattened leaf-path values.
+   */
+  getFlatUniformValues(
+    uniformValues: Readonly<Record<string, CompositeUniformValue>>
+  ): Record<string, UniformValue> {
+    const flattenedUniformValues: Record<string, UniformValue> = {};
+
+    for (const [name, value] of Object.entries(uniformValues)) {
+      const uniformType = this.layout.uniformTypes[name];
+      if (uniformType) {
+        this._flattenCompositeValue(flattenedUniformValues, name, uniformType, value);
+      } else if (this.layout.fields[name]) {
+        flattenedUniformValues[name] = value as UniformValue;
+      }
+    }
+
+    return flattenedUniformValues;
+  }
+
+  /**
+   * Serializes the supplied values into buffer-backed binary data.
+   *
+   * The returned view length matches {@link ShaderBlockLayout.byteLength}, which
+   * is the exact packed size of the block.
+   */
+  getData(uniformValues: Readonly<Record<string, CompositeUniformValue>>): Uint8Array {
+    const buffer = getScratchArrayBuffer(this.layout.byteLength);
+    new Uint8Array(buffer, 0, this.layout.byteLength).fill(0);
+    const typedArrays = {
+      i32: new Int32Array(buffer),
+      u32: new Uint32Array(buffer),
+      f32: new Float32Array(buffer),
+      f16: new Uint16Array(buffer)
+    };
+
+    const flattenedUniformValues = this.getFlatUniformValues(uniformValues);
+    for (const [name, value] of Object.entries(flattenedUniformValues)) {
+      this._writeLeafValue(typedArrays, name, value);
+    }
+
+    return new Uint8Array(buffer, 0, this.layout.byteLength);
+  }
+
+  /**
+   * Recursively flattens nested values using the declared composite shader type.
+   */
+  private _flattenCompositeValue(
+    flattenedUniformValues: Record<string, UniformValue>,
+    baseName: string,
+    uniformType: CompositeShaderType,
+    value: CompositeUniformValue | undefined
+  ): void {
+    if (value === undefined) {
+      return;
+    }
+
+    if (typeof uniformType === 'string' || this.layout.fields[baseName]) {
+      flattenedUniformValues[baseName] = value as UniformValue;
+      return;
+    }
+
+    if (Array.isArray(uniformType)) {
+      const elementType = uniformType[0] as CompositeShaderType;
+      const length = uniformType[1] as number;
+
+      if (Array.isArray(elementType)) {
+        throw new Error(`Nested arrays are not supported for ${baseName}`);
+      }
+
+      if (typeof elementType === 'string' && isNumberArray(value)) {
+        this._flattenPackedArray(flattenedUniformValues, baseName, elementType, length, value);
+        return;
+      }
+
+      if (!Array.isArray(value)) {
+        log.warn(`Unsupported uniform array value for ${baseName}:`, value)();
+        return;
+      }
+
+      for (let index = 0; index < Math.min(value.length, length); index++) {
+        const elementValue = value[index];
+        if (elementValue === undefined) {
+          continue;
+        }
+
+        this._flattenCompositeValue(
+          flattenedUniformValues,
+          `${baseName}[${index}]`,
+          elementType,
+          elementValue
+        );
+      }
+      return;
+    }
+
+    if (isCompositeShaderTypeStruct(uniformType) && isCompositeUniformObject(value)) {
+      for (const [key, subValue] of Object.entries(value)) {
+        if (subValue === undefined) {
+          continue;
+        }
+
+        const nestedName = `${baseName}.${key}`;
+        this._flattenCompositeValue(flattenedUniformValues, nestedName, uniformType[key], subValue);
+      }
+      return;
+    }
+
+    log.warn(`Unsupported uniform value for ${baseName}:`, value)();
+  }
+
+  /**
+   * Expands tightly packed numeric arrays into per-element leaf fields.
+   */
+  private _flattenPackedArray(
+    flattenedUniformValues: Record<string, UniformValue>,
+    baseName: string,
+    elementType: VariableShaderType,
+    length: number,
+    value: UniformValue
+  ): void {
+    const numericValue = value as Readonly<ArrayLike<number>>;
+    const elementLayout = getLeafLayoutInfo(elementType, this.layout.layout);
+    const packedElementLength = elementLayout.components;
+
+    for (let index = 0; index < length; index++) {
+      const start = index * packedElementLength;
+      if (start >= numericValue.length) {
+        break;
+      }
+
+      if (packedElementLength === 1) {
+        flattenedUniformValues[`${baseName}[${index}]`] = Number(numericValue[start]);
+      } else {
+        flattenedUniformValues[`${baseName}[${index}]`] = sliceNumericArray(
+          value,
+          start,
+          start + packedElementLength
+        ) as UniformValue;
+      }
+    }
+  }
+
+  /**
+   * Writes one flattened leaf value into its typed-array view.
+   */
+  private _writeLeafValue(
+    typedArrays: Record<string, any>,
+    name: string,
+    value: UniformValue
+  ): void {
+    const entry = this.layout.fields[name];
+    if (!entry) {
+      log.warn(`Uniform ${name} not found in layout`)();
+      return;
+    }
+
+    const {type, components, columns, rows, offset, columnStride} = entry;
+    const array = typedArrays[type];
+
+    if (components === 1) {
+      array[offset] = Number(value);
+      return;
+    }
+
+    const sourceValue = value as Readonly<ArrayLike<number>>;
+
+    if (columns === 1) {
+      for (let componentIndex = 0; componentIndex < components; componentIndex++) {
+        array[offset + componentIndex] = Number(sourceValue[componentIndex] ?? 0);
+      }
+      return;
+    }
+
+    let sourceIndex = 0;
+    for (let columnIndex = 0; columnIndex < columns; columnIndex++) {
+      const columnOffset = offset + columnIndex * columnStride;
+      for (let rowIndex = 0; rowIndex < rows; rowIndex++) {
+        array[columnOffset + rowIndex] = Number(sourceValue[sourceIndex++] ?? 0);
+      }
+    }
+  }
+}
+
+/**
+ * Type guard for nested uniform objects.
+ */
+function isCompositeUniformObject(
+  value: CompositeUniformValue
+): value is Record<string, CompositeUniformValue | undefined> {
+  return (
+    Boolean(value) &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    !ArrayBuffer.isView(value)
+  );
+}
+
+/**
+ * Slices a numeric array-like value without changing its numeric representation.
+ */
+function sliceNumericArray(value: UniformValue, start: number, end: number): number[] {
+  return Array.prototype.slice.call(value, start, end) as number[];
+}

package/src/portable/uniform-block.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {VariableShaderType} from '../shadertypes/data-types/shader-types';
+import type {VariableShaderType} from '../shadertypes/shader-types/shader-types';
 import type {UniformValue} from '../adapter/types/uniforms';
 import {
   ShaderLayout,

package/src/portable/uniform-store.ts

@@ -2,13 +2,44 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {VariableShaderType} from '../shadertypes/data-types/shader-types';
-import type {UniformValue} from '../adapter/types/uniforms';
+import type {CompositeShaderType} from '../shadertypes/shader-types/shader-types';
+import type {CompositeUniformValue} from '../adapter/types/uniforms';
 import type {Device} from '../adapter/device';
 import {Buffer} from '../adapter/resources/buffer';
 import {log} from '../utils/log';
+import {
+  makeShaderBlockLayout,
+  type ShaderBlockLayout
+} from '../shadertypes/shader-types/shader-block-layout';
 import {UniformBlock} from './uniform-block';
-import {UniformBufferLayout} from './uniform-buffer-layout';
+import {ShaderBlockWriter} from './shader-block-writer';
+
+/** Definition of a single managed uniform block. */
+export type UniformStoreBlockDefinition = {
+  /** Declared shader types for the block's uniforms. */
+  uniformTypes?: Record<string, CompositeShaderType>;
+  /** Reserved for future prop-level defaults. */
+  defaultProps?: Record<string, unknown>;
+  /** Initial uniform values written into the backing block. */
+  defaultUniforms?: Record<string, CompositeUniformValue>;
+  /** Explicit shader-block layout override. */
+  layout?: 'std140' | 'wgsl-uniform' | 'wgsl-storage';
+};
+
+/** Uniform block definitions keyed by block name. */
+export type UniformStoreBlocks<TPropGroups extends Record<string, Record<string, unknown>>> =
+  Record<keyof TPropGroups, UniformStoreBlockDefinition>;
+
+/**
+ * Smallest buffer size that can be used for uniform buffers.
+ *
+ * This is an allocation policy rather than part of {@link ShaderBlockLayout}.
+ * Layouts report the exact packed size, while the store applies any minimum
+ * buffer-size rule when allocating GPU buffers.
+ *
+ * TODO - does this depend on device?
+ */
+const minUniformBufferSize = 1024;
 
 /**
  * A uniform store holds a uniform values for one or more uniform blocks,
@@ -23,41 +54,37 @@ export class UniformStore<
     Record<string, unknown>
   >
 > {
+  /** Device used to infer layout and allocate buffers. */
+  readonly device: Device;
   /** Stores the uniform values for each uniform block */
   uniformBlocks = new Map<keyof TPropGroups, UniformBlock>();
-  /** Can generate data for a uniform buffer for each block from data */
-  uniformBufferLayouts = new Map<keyof TPropGroups, UniformBufferLayout>();
+  /** Flattened layout metadata for each block. */
+  shaderBlockLayouts = new Map<keyof TPropGroups, ShaderBlockLayout>();
+  /** Serializers for block-backed uniform data. */
+  shaderBlockWriters = new Map<keyof TPropGroups, ShaderBlockWriter>();
   /** Actual buffer for the blocks */
   uniformBuffers = new Map<keyof TPropGroups, Buffer>();
 
   /**
-   * Create a new UniformStore instance
-   * @param blocks
+   * Creates a new {@link UniformStore} for the supplied device and block definitions.
    */
-  constructor(
-    blocks: Record<
-      keyof TPropGroups,
-      {
-        uniformTypes?: Record<string, VariableShaderType>;
-        uniformSizes?: Record<string, number>;
-        defaultProps?: Record<string, unknown>;
-        defaultUniforms?: Record<string, UniformValue>;
-      }
-    >
-  ) {
+  constructor(device: Device, blocks: UniformStoreBlocks<TPropGroups>) {
+    this.device = device;
+
     for (const [bufferName, block] of Object.entries(blocks)) {
       const uniformBufferName = bufferName as keyof TPropGroups;
 
       // Create a layout object to help us generate correctly formatted binary uniform buffers
-      const uniformBufferLayout = new UniformBufferLayout(
-        block.uniformTypes ?? {},
-        block.uniformSizes ?? {}
-      );
-      this.uniformBufferLayouts.set(uniformBufferName, uniformBufferLayout);
+      const shaderBlockLayout = makeShaderBlockLayout(block.uniformTypes ?? {}, {
+        layout: block.layout ?? getDefaultUniformBufferLayout(device)
+      });
+      const shaderBlockWriter = new ShaderBlockWriter(shaderBlockLayout);
+      this.shaderBlockLayouts.set(uniformBufferName, shaderBlockLayout);
+      this.shaderBlockWriters.set(uniformBufferName, shaderBlockWriter);
 
       // Create a Uniform block to store the uniforms for each buffer.
       const uniformBlock = new UniformBlock({name: bufferName});
-      uniformBlock.setUniforms(block.defaultUniforms || {});
+      uniformBlock.setUniforms(shaderBlockWriter.getFlatUniformValues(block.defaultUniforms || {}));
       this.uniformBlocks.set(uniformBufferName, uniformBlock);
     }
   }
@@ -71,13 +98,20 @@ export class UniformStore<
 
   /**
    * Set uniforms
-   * Makes all properties partial
+   *
+   * Makes all group properties partial and eagerly propagates changes to any
+   * managed GPU buffers.
    */
   setUniforms(
     uniforms: Partial<{[group in keyof TPropGroups]: Partial<TPropGroups[group]>}>
   ): void {
     for (const [blockName, uniformValues] of Object.entries(uniforms)) {
-      this.uniformBlocks.get(blockName)?.setUniforms(uniformValues);
+      const uniformBufferName = blockName as keyof TPropGroups;
+      const shaderBlockWriter = this.shaderBlockWriters.get(uniformBufferName);
+      const flattenedUniforms = shaderBlockWriter?.getFlatUniformValues(
+        (uniformValues || {}) as Record<string, CompositeUniformValue>
+      );
+      this.uniformBlocks.get(uniformBufferName)?.setUniforms(flattenedUniforms || {});
       // We leverage logging in updateUniformBuffers(), even though slightly less efficient
       // this.updateUniformBuffer(blockName);
     }
@@ -85,24 +119,33 @@ export class UniformStore<
     this.updateUniformBuffers();
   }
 
-  /** Get the required minimum length of the uniform buffer */
+  /**
+   * Returns the allocation size for the named uniform buffer.
+   *
+   * This may exceed the packed layout size because minimum buffer-size policy is
+   * applied at the store layer.
+   */
   getUniformBufferByteLength(uniformBufferName: keyof TPropGroups): number {
-    return this.uniformBufferLayouts.get(uniformBufferName)?.byteLength || 0;
+    const packedByteLength = this.shaderBlockLayouts.get(uniformBufferName)?.byteLength || 0;
+    return Math.max(packedByteLength, minUniformBufferSize);
   }
 
-  /** Get formatted binary memory that can be uploaded to a buffer */
+  /**
+   * Returns packed binary data that can be uploaded to the named uniform buffer.
+   *
+   * The returned view length matches the packed block size and is not padded to
+   * the store's minimum allocation size.
+   */
   getUniformBufferData(uniformBufferName: keyof TPropGroups): Uint8Array {
     const uniformValues = this.uniformBlocks.get(uniformBufferName)?.getAllUniforms() || {};
-    // @ts-ignore
-    return this.uniformBufferLayouts.get(uniformBufferName)?.getData(uniformValues);
+    const shaderBlockWriter = this.shaderBlockWriters.get(uniformBufferName);
+    return shaderBlockWriter?.getData(uniformValues) || new Uint8Array(0);
   }
 
   /**
-   * Creates an unmanaged uniform buffer (umnanaged means that application is responsible for destroying it)
-   * The new buffer is initialized with current / supplied values
+   * Creates an unmanaged uniform buffer initialized with the current or supplied values.
    */
   createUniformBuffer(
-    device: Device,
     uniformBufferName: keyof TPropGroups,
     uniforms?: Partial<{[group in keyof TPropGroups]: Partial<TPropGroups[group]>}>
   ): Buffer {
@@ -110,7 +153,7 @@ export class UniformStore<
       this.setUniforms(uniforms);
     }
     const byteLength = this.getUniformBufferByteLength(uniformBufferName);
-    const uniformBuffer = device.createBuffer({
+    const uniformBuffer = this.device.createBuffer({
       usage: Buffer.UNIFORM | Buffer.COPY_DST,
       byteLength
     });
@@ -120,11 +163,11 @@ export class UniformStore<
     return uniformBuffer;
   }
 
-  /** Get the managed uniform buffer. "managed" resources are destroyed when the uniformStore is destroyed. */
-  getManagedUniformBuffer(device: Device, uniformBufferName: keyof TPropGroups): Buffer {
+  /** Returns the managed uniform buffer for the named block. */
+  getManagedUniformBuffer(uniformBufferName: keyof TPropGroups): Buffer {
     if (!this.uniformBuffers.get(uniformBufferName)) {
       const byteLength = this.getUniformBufferByteLength(uniformBufferName);
-      const uniformBuffer = device.createBuffer({
+      const uniformBuffer = this.device.createBuffer({
         usage: Buffer.UNIFORM | Buffer.COPY_DST,
         byteLength
       });
@@ -135,7 +178,11 @@ export class UniformStore<
     return this.uniformBuffers.get(uniformBufferName);
   }
 
-  /** Updates all uniform buffers where values have changed */
+  /**
+   * Updates every managed uniform buffer whose source uniforms have changed.
+   *
+   * @returns The first redraw reason encountered, or `false` if nothing changed.
+   */
   updateUniformBuffers(): false | string {
     let reason: false | string = false;
     for (const uniformBufferName of this.uniformBlocks.keys()) {
@@ -148,7 +195,11 @@ export class UniformStore<
     return reason;
   }
 
-  /** Update one uniform buffer. Only updates if values have changed */
+  /**
+   * Updates one managed uniform buffer if its corresponding block is dirty.
+   *
+   * @returns The redraw reason for the update, or `false` if no write occurred.
+   */
   updateUniformBuffer(uniformBufferName: keyof TPropGroups): false | string {
     const uniformBlock = this.uniformBlocks.get(uniformBufferName);
     let uniformBuffer = this.uniformBuffers.get(uniformBufferName);
@@ -174,3 +225,10 @@ export class UniformStore<
     return reason;
   }
 }
+
+/**
+ * Returns the default uniform-buffer layout for the supplied device.
+ */
+function getDefaultUniformBufferLayout(device: Device): 'std140' | 'wgsl-uniform' | 'wgsl-storage' {
+  return device.type === 'webgpu' ? 'wgsl-uniform' : 'std140';
+}

package/src/shadertypes/data-types/data-type-decoder.ts

@@ -0,0 +1,105 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray, TypedArrayConstructor} from '../../types';
+import {
+  PrimitiveDataType,
+  SignedDataType,
+  NormalizedDataType,
+  DataTypeInfo,
+  TypedArrayConstructorT
+} from './data-types';
+
+export class DataTypeDecoder {
+  /**
+   * Gets info about a data type constant (signed or normalized)
+   * @returns underlying primitive / signed types, byte length, normalization, integer, signed flags
+   */
+  getDataTypeInfo<T extends NormalizedDataType = NormalizedDataType>(type: T): DataTypeInfo<T> {
+    const [signedType, primitiveType, byteLength] = NORMALIZED_TYPE_MAP[type];
+    const normalized: boolean = type.includes('norm');
+    const integer: boolean = !normalized && !type.startsWith('float');
+    const signed: boolean = type.startsWith('s');
+    return {
+      signedType: signedType as DataTypeInfo<T>['signedType'],
+      primitiveType: primitiveType as DataTypeInfo<T>['primitiveType'],
+      byteLength: byteLength as DataTypeInfo<T>['byteLength'],
+      normalized: normalized as DataTypeInfo<T>['normalized'],
+      integer: integer as DataTypeInfo<T>['integer'],
+      signed: signed as DataTypeInfo<T>['signed']
+      // TODO - add webglOnly flag
+    };
+  }
+
+  /** Build a vertex format from a signed data type and a component */
+  getNormalizedDataType(signedDataType: SignedDataType): NormalizedDataType {
+    const dataType: NormalizedDataType = signedDataType;
+    // prettier-ignore
+    switch (dataType) {
+      case 'uint8': return 'unorm8';
+      case 'sint8': return 'snorm8';
+      case 'uint16': return 'unorm16';
+      case 'sint16': return 'snorm16';
+      default: return dataType;
+    }
+  }
+
+  /** Align offset to 1, 2 or 4 elements (4, 8 or 16 bytes) */
+  alignTo(size: number, count: number): number {
+    // prettier-ignore
+    switch (count) {
+      case 1: return size; // Pad upwards to even multiple of 2
+      case 2: return size + (size % 2); // Pad upwards to even multiple of 2
+      default: return size + ((4 - (size % 4)) % 4); // Pad upwards to even multiple of 4
+    }
+  }
+
+  /** Returns the VariableShaderType that corresponds to a typed array */
+  getDataType(arrayOrType: TypedArray | TypedArrayConstructor): SignedDataType {
+    const Constructor = ArrayBuffer.isView(arrayOrType) ? arrayOrType.constructor : arrayOrType;
+    if (Constructor === Uint8ClampedArray) {
+      return 'uint8';
+    }
+    const info = Object.values(NORMALIZED_TYPE_MAP).find(entry => Constructor === entry[4]);
+    if (!info) {
+      throw new Error(Constructor.name);
+    }
+    return info[0];
+  }
+
+  /** Returns the TypedArray that corresponds to a shader data type */
+  getTypedArrayConstructor<T extends NormalizedDataType>(
+    type: NormalizedDataType
+  ): TypedArrayConstructorT<T> {
+    const [, , , , Constructor] = NORMALIZED_TYPE_MAP[type];
+    return Constructor as unknown as TypedArrayConstructorT<T>;
+  }
+}
+
+/** Entry point for decoding luma.gl data types */
+export const dataTypeDecoder = new DataTypeDecoder();
+
+const NORMALIZED_TYPE_MAP = {
+  uint8: ['uint8', 'u32', 1, false, Uint8Array],
+  sint8: ['sint8', 'i32', 1, false, Int8Array],
+  unorm8: ['uint8', 'f32', 1, true, Uint8Array],
+  snorm8: ['sint8', 'f32', 1, true, Int8Array],
+  uint16: ['uint16', 'u32', 2, false, Uint16Array],
+  sint16: ['sint16', 'i32', 2, false, Int16Array],
+  unorm16: ['uint16', 'u32', 2, true, Uint16Array],
+  snorm16: ['sint16', 'i32', 2, true, Int16Array],
+  float16: ['float16', 'f16', 2, false, Uint16Array],
+  float32: ['float32', 'f32', 4, false, Float32Array],
+  uint32: ['uint32', 'u32', 4, false, Uint32Array],
+  sint32: ['sint32', 'i32', 4, false, Int32Array]
+} satisfies Record<
+  NormalizedDataType,
+  [
+    SignedDataType,
+    PrimitiveDataType,
+    bytes: 1 | 2 | 4,
+    normalized: boolean,
+    arrayConstructor: TypedArrayConstructor
+  ]
+>;