@loaders.gl/scene 5.0.0-alpha.9 → 5.0.0-beta.2

package/src/gltf/lib/extensions/EXT_texture_avif.ts

@@ -0,0 +1,56 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// GLTF EXTENSION: EXT_TEXTURE_AVIF
+// https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Vendor/EXT_texture_avif
+/* eslint-disable camelcase */
+
+import type {GLTF_EXT_texture_avif} from '../types/gltf-json-schema';
+import type {GLTFWithBuffers} from '../types/gltf-types';
+import type {GLTFLoaderOptions} from '../../gltf-loader-types';
+
+import {getSupportedImageFormats} from '@loaders.gl/images';
+import {GLTFIterator} from '../api/gltf-iterator';
+
+const EXT_TEXTURE_AVIF = 'EXT_texture_avif';
+
+/** Extension name. */
+export const name = EXT_TEXTURE_AVIF;
+
+/**
+ * Selects AVIF image sources when the active image decoder supports AVIF.
+ *
+ * Optional AVIF sources retain their ordinary texture source when unsupported. Required AVIF
+ * sources fail before image loading so callers never receive a texture with an invalid source.
+ *
+ * @param gltfData Parsed glTF JSON whose texture sources may be updated in place.
+ * @param options glTF loader options. The extension currently has no custom options.
+ */
+export async function preprocess(
+  gltfData: GLTFWithBuffers,
+  options: GLTFLoaderOptions
+): Promise<void> {
+  void options;
+  const iterator = new GLTFIterator(gltfData);
+  // Do not probe window-only image APIs for documents that do not use AVIF.
+  if (!iterator.hasExtension(EXT_TEXTURE_AVIF)) return;
+
+  const supportedImageFormats = await getSupportedImageFormats();
+  if (!supportedImageFormats.has('image/avif')) {
+    if (iterator.isExtensionRequired(EXT_TEXTURE_AVIF)) {
+      throw new Error(`gltf: Required extension ${EXT_TEXTURE_AVIF} not supported by browser`);
+    }
+    return;
+  }
+
+  for (const texture of iterator.textures) {
+    const extension = iterator.getExtension<GLTF_EXT_texture_avif>(texture, EXT_TEXTURE_AVIF);
+    if (extension) {
+      texture.source = extension.source;
+    }
+    iterator.removeExtension(texture, EXT_TEXTURE_AVIF);
+  }
+
+  iterator.removeExtension(EXT_TEXTURE_AVIF);
+}

package/src/gltf/lib/extensions/EXT_texture_webp.ts

@@ -0,0 +1,46 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// GLTF EXTENSION: EXT_TEXTURE_WEBP
+// https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/EXT_TEXTURE_WEBP
+/* eslint-disable camelcase */
+
+import type {GLTF_EXT_texture_webp} from '../types/gltf-json-schema';
+import type {GLTFWithBuffers} from '../types/gltf-types';
+import type {GLTFLoaderOptions} from '../../gltf-loader-types';
+
+import {isImageFormatSupported} from '@loaders.gl/images';
+import {GLTFIterator} from '../api/gltf-iterator';
+
+const EXT_TEXTURE_WEBP = 'EXT_texture_webp';
+
+/** Extension name */
+export const name = EXT_TEXTURE_WEBP;
+
+/**
+ * Replaces a texture source reference with the extension texture
+ * Done in preprocess() to prevent load of default image
+ */
+export function preprocess(gltfData: GLTFWithBuffers, options: GLTFLoaderOptions): void {
+  const iterator = new GLTFIterator(gltfData);
+
+  if (!isImageFormatSupported('image/webp')) {
+    if (iterator.isExtensionRequired(EXT_TEXTURE_WEBP)) {
+      throw new Error(`gltf: Required extension ${EXT_TEXTURE_WEBP} not supported by browser`);
+    }
+    return;
+  }
+
+  for (const texture of iterator.textures) {
+    const extension = iterator.getExtension<GLTF_EXT_texture_webp>(texture, EXT_TEXTURE_WEBP);
+    if (extension) {
+      // TODO - if multiple texture extensions are present which one wins?
+      texture.source = extension.source;
+    }
+    iterator.removeExtension(texture, EXT_TEXTURE_WEBP);
+  }
+
+  // Remove the top-level extension
+  iterator.removeExtension(EXT_TEXTURE_WEBP);
+}

package/src/gltf/lib/extensions/KHR_audio_emitter.ts

@@ -0,0 +1,136 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import type {LoaderContext} from '@loaders.gl/core/loader-utils';
+import type {GLTFLoaderOptions} from '../../gltf-loader-types';
+import type {GLTFWithBuffers} from '../types/gltf-types';
+import type {
+  GLTF_KHR_audio_emitter,
+  GLTFKHRAudioAttachment,
+  GLTFAudioClip,
+  GLTFAudioEmitter
+} from '../types/gltf-khr-audio-emitter-schema';
+import {resolveUrl} from '../gltf-utils/resolve-url';
+
+/** Name of the draft audio emitter extension. */
+export const name = 'KHR_audio_emitter';
+
+/** Resolves audio references and optionally loads encoded clips, without creating playback objects. */
+export async function decode(
+  gltfData: GLTFWithBuffers,
+  options: GLTFLoaderOptions,
+  context?: LoaderContext
+): Promise<void> {
+  const extension = gltfData.json.extensions?.[name] as GLTF_KHR_audio_emitter | undefined;
+  const clips: GLTFAudioClip[] = (extension?.audio || []).map(audio => ({audio}));
+  const sources = extension?.sources || [];
+  const emitters = extension?.emitters || [];
+  for (const source of sources) {
+    if (source.audio !== undefined) getReference(clips, source.audio, 'audio');
+  }
+  for (const emitter of emitters) {
+    if (emitter.type !== 'global' && emitter.type !== 'positional') {
+      throw new Error(`${name}: invalid emitter type ${emitter.type}.`);
+    }
+    if (emitter.type === 'global' && emitter.positional !== undefined) {
+      throw new Error(`${name}: global emitters cannot have positional settings.`);
+    }
+    for (const sourceIndex of emitter.sources || []) getReference(sources, sourceIndex, 'source');
+  }
+  const descriptors: GLTFAudioEmitter[] = [];
+  for (const attachmentType of ['nodes', 'scenes'] as const) {
+    for (const [attachmentIndex, object] of (gltfData.json[attachmentType] || []).entries()) {
+      const attachment = object.extensions?.[name] as GLTFKHRAudioAttachment | undefined;
+      if (!attachment) continue;
+      if (!Array.isArray(attachment.emitters) || !attachment.emitters.length) {
+        throw new Error(`${name}: ${attachmentType} ${attachmentIndex} requires emitters.`);
+      }
+      for (const emitterIndex of attachment.emitters) {
+        const emitter = getReference(emitters, emitterIndex, 'emitter');
+        if (attachmentType === 'scenes' && emitter.type !== 'global') {
+          throw new Error(`${name}: scenes may only reference global emitters.`);
+        }
+        descriptors.push({
+          emitterIndex,
+          nodeIndex: attachmentType === 'nodes' ? attachmentIndex : null,
+          sceneIndex: attachmentType === 'scenes' ? attachmentIndex : null,
+          emitter,
+          sources: (emitter.sources || []).map(sourceIndex => {
+            const source = sources[sourceIndex];
+            return {
+              sourceIndex,
+              source,
+              clip: source.audio === undefined ? undefined : clips[source.audio]
+            };
+          })
+        });
+      }
+    }
+  }
+  if (!extension) return;
+  // Validate resource declarations before performing external requests.
+  for (const clip of clips) validateClip(gltfData, clip);
+  if (options.gltf?.loadAudio !== false) {
+    await Promise.all(clips.map(clip => loadClip(gltfData, clip, options, context)));
+  }
+  gltfData.audioClips = clips;
+  gltfData.audioEmitters = descriptors;
+}
+
+/** Looks up a nonnegative integral extension index with an actionable error. */
+function getReference<Value>(values: Value[], index: number, kind: string): Value {
+  if (!Number.isInteger(index) || index < 0 || !values[index]) {
+    throw new Error(`${name}: invalid ${kind} index ${index}.`);
+  }
+  return values[index];
+}
+
+/** Checks resource declarations even when resource loading is disabled. */
+function validateClip(gltfData: GLTFWithBuffers, clip: GLTFAudioClip): void {
+  const {audio} = clip;
+  if (audio.bufferView !== undefined) {
+    getReference(gltfData.json.bufferViews || [], audio.bufferView, 'bufferView');
+    if (!audio.mimeType) throw new Error(`${name}: buffer-view audio requires mimeType.`);
+  } else if (!audio.uri) {
+    throw new Error(`${name}: audio requires a uri or bufferView.`);
+  }
+}
+
+/** Loads encoded bytes through the loader context or a bounded embedded buffer view. */
+async function loadClip(
+  gltfData: GLTFWithBuffers,
+  clip: GLTFAudioClip,
+  options: GLTFLoaderOptions,
+  context?: LoaderContext
+): Promise<void> {
+  const {audio} = clip;
+  if (audio.bufferView !== undefined) {
+    const bufferView = gltfData.json.bufferViews![audio.bufferView];
+    const buffer = gltfData.buffers[bufferView.buffer];
+    if (!buffer) {
+      if (options.gltf?.loadBuffers === false) return;
+      throw new Error(`${name}: audio buffer ${bufferView.buffer} is not loaded.`);
+    }
+    const byteOffset = bufferView.byteOffset ?? 0;
+    const byteLength = bufferView.byteLength;
+    if (
+      !Number.isSafeInteger(byteOffset) ||
+      byteOffset < 0 ||
+      !Number.isSafeInteger(byteLength) ||
+      byteLength < 0 ||
+      byteOffset + byteLength > buffer.byteLength ||
+      buffer.byteOffset + byteOffset + byteLength > buffer.arrayBuffer.byteLength
+    ) {
+      throw new Error(`${name}: audio bufferView exceeds its buffer.`);
+    }
+    clip.data = new Uint8Array(buffer.arrayBuffer, buffer.byteOffset + byteOffset, byteLength);
+    return;
+  }
+  if (!context?.fetch) throw new Error(`${name}: a loader context is required to load audio.`);
+  const url = resolveUrl(audio.uri!, options, context);
+  const response = await context.fetch(url);
+  if (!response.ok) throw new Error(`${name}: failed to load audio ${url} (${response.status}).`);
+  clip.url = url;
+  clip.data = new Uint8Array(await response.arrayBuffer());
+}

package/src/gltf/lib/extensions/KHR_binary_gltf.ts

@@ -0,0 +1,56 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// GLTF 1.0 EXTENSION: KHR_binary_glTF
+// https://github.com/KhronosGroup/glTF/tree/master/extensions/1.0/Khronos/KHR_binary_glTF
+/* eslint-disable camelcase */
+
+import type {GLTF_KHR_binary_glTF} from '../types/gltf-json-schema';
+import type {GLTFWithBuffers} from '../types/gltf-types';
+
+import {GLTFIterator} from '../api/gltf-iterator';
+
+const KHR_BINARY_GLTF = 'KHR_binary_glTF';
+
+/** Extension name */
+export const name = KHR_BINARY_GLTF;
+
+/** Convert embedded glTF 1 images and the named binary body to core glTF 2 fields. */
+export function preprocess(gltfData: GLTFWithBuffers): void {
+  const iterator = new GLTFIterator(gltfData);
+
+  // Note: json.buffers.binary_glTF also needs to be replaced
+  // This is currently done during gltf normalization
+
+  // Image and shader nodes can have the extension
+  // https://github.com/KhronosGroup/glTF/blob/master/extensions/1.0/Khronos/KHR_binary_glTF/schema/image.KHR_binary_glTF.schema.json
+  for (const image of iterator.images) {
+    const extension = iterator.getExtension<GLTF_KHR_binary_glTF>(image, KHR_BINARY_GLTF);
+    // Copy core image fields, omitting extension-only dimensions.
+    if (extension) {
+      image.bufferView = extension.bufferView;
+      image.mimeType = extension.mimeType;
+      delete image.uri;
+    }
+    iterator.removeExtension(image, KHR_BINARY_GLTF);
+  }
+
+  // TODO shaders - At least traverse and throw error if used?
+  // https://github.com/KhronosGroup/glTF/blob/master/extensions/1.0/Khronos/KHR_binary_glTF/schema/shader.KHR_binary_glTF.schema.json
+
+  // Only the reserved GLB body has an ignored URI; external buffers retain theirs.
+  for (const buffer of iterator.data.buffers || []) {
+    if ((buffer as typeof buffer & {id?: string}).id === 'binary_glTF') {
+      delete buffer.uri;
+    }
+  }
+
+  // Remove the top-level extension as it has now been processed
+  iterator.removeExtension(KHR_BINARY_GLTF);
+}
+
+// KHR_binary_gltf is a 1.0 extension that is supported natively by 2.0
+// export function encode() {
+//   throw new Error(KHR_BINARY_GLTF);
+// }

package/src/gltf/lib/extensions/KHR_draco_mesh_compression.ts

@@ -0,0 +1,170 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_draco_mesh_compression
+// Only TRIANGLES: 0x0004 and TRIANGLE_STRIP: 0x0005 are supported
+/* eslint-disable camelcase */
+
+import type {LoaderContext} from '@loaders.gl/core/loader-utils';
+import {sliceArrayBuffer, parseFromContext} from '@loaders.gl/core/loader-utils';
+
+import {DracoLoader} from '@loaders.gl/scene/draco-loader-types';
+import type {DracoLoaderOptions, DracoMesh} from '@loaders.gl/scene';
+
+import type {
+  GLTFWithBuffers,
+  GLTFAccessor,
+  GLTFMeshPrimitive,
+  GLTF_KHR_draco_mesh_compression
+} from '../types/gltf-types';
+import type {GLTFLoaderOptions} from '../../gltf-loader-types';
+
+import {GLTFIterator} from '../api/gltf-iterator';
+import {getGLTFAccessors, getGLTFAccessor} from '../gltf-utils/gltf-attribute-utils';
+import {getTypedArrayForBufferView} from '../gltf-utils/get-typed-array';
+
+const KHR_DRACO_MESH_COMPRESSION = 'KHR_draco_mesh_compression';
+
+/** Extension name */
+export const name = KHR_DRACO_MESH_COMPRESSION;
+
+export function preprocess(
+  gltfData: GLTFWithBuffers,
+  options: GLTFLoaderOptions,
+  context: LoaderContext
+): void {
+  const iterator = new GLTFIterator(gltfData);
+  for (const mesh of iterator.meshes) {
+    for (const primitive of iterator.getReferences(mesh).primitives) {
+      if (iterator.getExtension(primitive, KHR_DRACO_MESH_COMPRESSION)) {
+        // TODO - Remove fallback accessors to make sure we don't load unnecessary buffers
+      }
+    }
+  }
+}
+
+export async function decode(
+  gltfData: GLTFWithBuffers,
+  options: GLTFLoaderOptions,
+  context: LoaderContext
+): Promise<void> {
+  if (!options?.gltf?.decompressMeshes) {
+    return;
+  }
+
+  const iterator = new GLTFIterator(gltfData);
+  const promises: Promise<void>[] = [];
+  for (const mesh of iterator.meshes) {
+    for (const primitive of iterator.getReferences(mesh).primitives) {
+      if (iterator.getExtension(primitive, KHR_DRACO_MESH_COMPRESSION)) {
+        promises.push(decompressPrimitive(iterator, primitive, options, context));
+      }
+    }
+  }
+
+  // Decompress meshes in parallel
+  await Promise.all(promises);
+
+  // We have now decompressed all primitives, so remove the top-level extension
+  iterator.removeExtension(KHR_DRACO_MESH_COMPRESSION);
+}
+
+// DECODE
+
+// Unpacks one mesh primitive and removes the extension from the primitive
+// DracoDecoder needs to be imported and registered by app
+// Returns: Promise that resolves when all pending draco decoder jobs for this mesh complete
+
+// TODO - Implement fallback behavior per KHR_DRACO_MESH_COMPRESSION spec
+
+async function decompressPrimitive(
+  iterator: GLTFIterator,
+  primitive: GLTFMeshPrimitive,
+  options: GLTFLoaderOptions,
+  context: LoaderContext
+): Promise<void> {
+  const dracoExtension = iterator.getExtension<GLTF_KHR_draco_mesh_compression>(
+    primitive,
+    KHR_DRACO_MESH_COMPRESSION
+  );
+  if (!dracoExtension) {
+    return;
+  }
+
+  const buffer = getTypedArrayForBufferView(
+    iterator.data,
+    iterator.gltf.buffers,
+    dracoExtension.bufferView
+  );
+  // A decoder worker can transfer its input, so it must not own the shared glTF buffer.
+  const compressedData = sliceArrayBuffer(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+
+  const dracoOptions: DracoLoaderOptions = {
+    ...options,
+    draco: {
+      ...options.draco,
+      // Internal glTF decoding consumes the legacy mesh contract.
+      shape: 'mesh',
+      // Normalize compressed strips to the triangle indices returned below.
+      topology: 'triangle-list',
+      decoderProfile: options.draco?.decoderProfile || 'gltf',
+      extraAttributes: dracoExtension.attributes
+    }
+  };
+
+  // TODO - remove hack: The entire tileset might be included, too expensive to serialize
+  delete dracoOptions['3d-tiles'];
+  const decodedData = (await parseFromContext(
+    compressedData,
+    DracoLoader,
+    dracoOptions,
+    context
+  )) as DracoMesh;
+
+  const decodedAttributes: {[key: string]: GLTFAccessor} = getGLTFAccessors(decodedData.attributes);
+
+  // A declared compressed attribute cannot fall back to its raw accessor: it may have no buffer view.
+  for (const attributeName of Object.keys(dracoExtension.attributes)) {
+    if (!Object.hasOwn(decodedAttributes, attributeName)) {
+      throw new Error(`glTF: Draco decompression did not produce attribute ${attributeName}`);
+    }
+  }
+
+  // Restore accessor bounds and integer normalization from the glTF declaration.
+  for (const [attributeName, decodedAttribute] of Object.entries(decodedAttributes)) {
+    if (attributeName in primitive.attributes) {
+      const accessorIndex: number = primitive.attributes[attributeName];
+      const accessor = iterator.data.accessors?.[accessorIndex];
+      if (accessor?.normalized !== undefined) {
+        decodedAttribute.normalized = accessor.normalized;
+      }
+      if (accessor?.min && accessor?.max) {
+        decodedAttribute.min = accessor.min;
+        decodedAttribute.max = accessor.max;
+      }
+    }
+  }
+
+  // Attributes outside the Draco bitstream still reference ordinary glTF accessors.
+  // @ts-ignore
+  primitive.attributes = {...primitive.attributes, ...decodedAttributes};
+  if (decodedData.indices) {
+    // @ts-ignore
+    primitive.indices = getGLTFAccessor(decodedData.indices);
+    primitive.mode = 4;
+  }
+
+  // Extension has been processed, delete it
+  iterator.removeExtension(primitive, KHR_DRACO_MESH_COMPRESSION);
+
+  checkPrimitive(primitive);
+}
+
+// UTILS
+
+function checkPrimitive(primitive: GLTFMeshPrimitive): void {
+  if (!primitive.attributes || Object.keys(primitive.attributes).length === 0) {
+    throw new Error('glTF: Empty primitive detected: Draco decompression failure?');
+  }
+}