@loaders.gl/tile-converter 5.0.0-alpha.7 → 5.0.0-alpha.8

package/src/{i3s-converter → v4/i3s-converter}/helpers/geometry-converter.ts

@@ -21,7 +21,6 @@ import {Ellipsoid} from '@math.gl/geospatial';
 /* @ts-expect-error TS2732: Cannot find module 'node:module' or its corresponding type declarations. */
 import {createRequire} from 'node:module';
 /* @ts-expect-error TS2732: Cannot find module 'node:module' or its corresponding type declarations. */
-import {dirname, join} from 'node:path';
 
 import {DracoWriterWorker} from '@loaders.gl/draco';
 import {assert, encode} from '@loaders.gl/core';
@@ -34,6 +33,7 @@ import {
 import md5 from 'md5';
 import {v4 as uuidv4} from 'uuid';
 import {generateAttributes} from './geometry-attributes';
+import {encodeStringAttribute} from './encode-string-attribute';
 import {createBoundingVolumesFromGeometry} from './coordinate-converter';
 import {
   ConvertedAttributes,
@@ -1028,15 +1028,18 @@ async function mergeMaterials(
     material1.mergedMaterials &&
     material2.mergedMaterials
   ) {
-    const buffer1 = Buffer.from(material1.texture.bufferView.data);
-    const buffer2 = Buffer.from(material2.texture.bufferView.data);
+    const firstImageBytes = new Uint8Array(material1.texture.bufferView.data);
+    const secondImageBytes = new Uint8Array(material2.texture.bufferView.data);
     try {
       // @ts-ignore
       const {joinImages} = await import('join-images');
-      const sharpData = await joinImages([buffer1, buffer2], {direction: 'horizontal'});
-      material1.texture.bufferView.data = await sharpData
+      const sharpData = await joinImages([firstImageBytes, secondImageBytes], {
+        direction: 'horizontal'
+      });
+      const encodedImage = await sharpData
         .toFormat(material1.texture.mimeType === 'image/png' ? 'png' : 'jpeg')
         .toBuffer();
+      material1.texture.bufferView.data = new Uint8Array(encodedImage);
     } catch (error) {
       // eslint-disable-next-line no-console
       console.log(
@@ -1458,10 +1461,10 @@ function generateAttributeBuffer(type: string, value: any): ArrayBuffer {
       attributeBuffer = generateDoubleAttributeBuffer(value);
       break;
     case STRING_TYPE:
-      attributeBuffer = generateStringAttributeBuffer(value);
+      attributeBuffer = encodeStringAttribute(value);
       break;
     default:
-      attributeBuffer = generateStringAttributeBuffer(value);
+      attributeBuffer = encodeStringAttribute(value);
   }
 
   return attributeBuffer;
@@ -1497,7 +1500,7 @@ function getAttributeType(key: string, attributeStorageInfo: any[]): string {
 /**
  * Convert short integer to attribute arrayBuffer.
  * @param featureIds
- * @returns - Buffer with objectId data.
+ * @returns ArrayBuffer with objectId data.
  */
 function generateShortIntegerAttributeBuffer(featureIds: any[]): ArrayBuffer {
   const count = new Uint32Array([featureIds.length]);
@@ -1508,7 +1511,7 @@ function generateShortIntegerAttributeBuffer(featureIds: any[]): ArrayBuffer {
 /**
  * Convert double to attribute arrayBuffer.
  * @param featureIds
- * @returns - Buffer with objectId data.
+ * @returns ArrayBuffer with objectId data.
  */
 function generateDoubleAttributeBuffer(featureIds: any[]): ArrayBuffer {
   const count = new Uint32Array([featureIds.length]);
@@ -1518,36 +1521,6 @@ function generateDoubleAttributeBuffer(featureIds: any[]): ArrayBuffer {
   return concatenateArrayBuffers(count.buffer, padding.buffer, valuesArray.buffer);
 }
 
-/**
- * Convert batch table attributes to array buffer with batch table data.
- * @param batchAttributes
- * @returns - Buffer with batch table data.
- */
-function generateStringAttributeBuffer(batchAttributes: any[]): ArrayBuffer {
-  const stringCountArray = new Uint32Array([batchAttributes.length]);
-  let totalNumberOfBytes = 0;
-  const stringSizesArray = new Uint32Array(batchAttributes.length);
-  const stringBufferArray: ArrayBuffer[] = [];
-
-  for (let index = 0; index < batchAttributes.length; index++) {
-    const currentString = `${String(batchAttributes[index])}\0`;
-    const currentStringBuffer = Buffer.from(currentString);
-    const currentStringSize = currentStringBuffer.length;
-    totalNumberOfBytes += currentStringSize;
-    stringSizesArray[index] = currentStringSize;
-    stringBufferArray.push(currentStringBuffer);
-  }
-
-  const totalBytes = new Uint32Array([totalNumberOfBytes]);
-
-  return concatenateArrayBuffers(
-    stringCountArray.buffer,
-    totalBytes.buffer,
-    stringSizesArray.buffer,
-    ...stringBufferArray
-  );
-}
-
 /**
  * Convert featureIds to BigUint64Array.
  * @param featureIds
@@ -1642,14 +1615,14 @@ async function generateCompressedGeometry(
   );
 }
 
+/** Resolve the published Node Draco writer, falling back to the dependency installer destination. */
 function getLocalDracoWriterWorkerUrl(): string {
   try {
     const require = createRequire(import.meta.url);
-    const packageJsonPath = require.resolve('@loaders.gl/draco/package.json');
-    return join(dirname(packageJsonPath), 'src/workers/draco-writer-worker-node.cjs');
+    return require.resolve('@loaders.gl/draco/draco-writer-worker-node.cjs');
   } catch {
     // Fall back to monorepo-local path
-    return './modules/draco/src/workers/draco-writer-worker-node.cjs';
+    return './modules/draco/dist/draco-writer-worker-node.cjs';
   }
 }
 

package/src/{i3s-converter → v4/i3s-converter}/helpers/gltf-attributes.ts

@@ -2,6 +2,7 @@ import type {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import type {GLTFAccessorPostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
 import {Matrix4, TypedArray, Vector3} from '@math.gl/core';
 import {BoundingSphere, OrientedBoundingBox} from '@math.gl/culling';
+import {getImageData} from '@loaders.gl/images';
 import {Ellipsoid} from '@math.gl/geospatial';
 import {GLTFAttributesData} from '../types';
 
@@ -30,15 +31,15 @@ export function prepareDataForAttributesConversion(
       if (imageObject?.image?.compressed) {
         return null;
       }
-      const data = imageObject?.image?.data;
+      const {data, width, height} = getImageData(imageObject.image);
       const dataCopy = new Uint8Array(data.length);
       dataCopy.set(data);
       return {
         data: dataCopy,
         compressed: false,
-        height: imageObject.image.height,
-        width: imageObject.image.width,
-        components: imageObject.image.components,
+        height,
+        width,
+        components: data.length / (width * height),
         mimeType: imageObject.mimeType
       };
     }) || [];

package/src/{i3s-converter → v4/i3s-converter}/helpers/load-3d-tiles.ts

@@ -119,15 +119,11 @@ export async function loadFromArchive(
     const hashTable = await loadHashTable(fileProvider);
     const archive = new Tiles3DArchive(fileProvider, hashTable, tz3Path);
     const fileSystem = new ZipFileSystem(archive);
-    // const loadOptionsWithoutDeprecatedFetch = {...loadOptions} as Tiles3DLoaderOptions & {
-    //   fetch?: unknown;
-    // };
-    // delete loadOptionsWithoutDeprecatedFetch.fetch;
-    // const mergedOptions = {
-    //   ...loadOptionsWithoutDeprecatedFetch,
-    //   core: {...loadOptions?.core, fetch: fileSystem.fetch.bind(fileSystem)}
-    // }
-    const content = await load(filename, loader, loadOptions);
+    const archiveLoadOptions = {
+      ...loadOptions,
+      core: {...loadOptions?.core, fetch: fileSystem.fetch.bind(fileSystem)}
+    };
+    const content = await load(filename, loader, archiveLoadOptions);
     await fileSystem.destroy();
     return content;
   }

package/src/{i3s-converter → v4/i3s-converter}/i3s-converter.ts

@@ -49,7 +49,7 @@ import {validateNodeBoundingVolumes} from './helpers/node-debug';
 import {KTX2BasisWriterWorker} from '@loaders.gl/textures';
 import {LoaderWithParser} from '@loaders.gl/loader-utils';
 import {I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s';
-import {ImageWriter} from '@loaders.gl/images';
+import {getImageData, ImageWriter} from '@loaders.gl/images';
 import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
 import {
   GLTFPrimitiveModeString,
@@ -143,7 +143,7 @@ export default class I3SConverter {
     basis: {
       format: 'rgba32',
       // We need to load local fs workers because nodejs can't load workers from the Internet
-      workerUrl: './modules/textures/dist/basis-worker-node.js'
+      workerUrl: './modules/textures/dist/basis-worker-node.cjs'
     },
     // We need to load local fs workers because nodejs can't load workers from the Internet
     draco: {workerUrl: './modules/draco/dist/draco-worker-node.js'},
@@ -1266,18 +1266,17 @@ export default class I3SConverter {
 
           if (this.generateTextures) {
             formats.push({name: '1', format: 'ktx2'});
-            // For Node.js texture.image.data is type of Buffer
-            const copyArrayBuffer = texture.image.data.subarray();
-            const arrayToEncode = new Uint8Array(copyArrayBuffer);
+            const imageData = getImageData(texture.image);
+            const arrayToEncode = new Uint8Array(imageData.data);
             const ktx2TextureData = encode(
-              {...texture.image, data: arrayToEncode},
+              {...imageData, data: arrayToEncode},
               // @ts-expect-error - Worker encoder typing is still WIP
               KTX2BasisWriterWorker,
               {
                 ...KTX2BasisWriterWorker.options,
                 ['ktx2-basis-writer']: {
                   // We need to load local fs workers because nodejs can't load workers from the Internet
-                  workerUrl: './modules/textures/dist/ktx2-basis-writer-worker-node.js'
+                  workerUrl: './modules/textures/dist/ktx2-basis-writer-worker-node.cjs'
                 },
                 reuseWorkers: true,
                 _nodeWorkers: true,

package/src/{i3s-server → v4/i3s-server}/README.md

@@ -6,8 +6,8 @@ The server provides I3S Rest endpoints per specification https://github.com/Esri
 ## Development
 
 - install tsx globally `yarn global add tsx`
-- uncomment "local debug" lines in `modules/tile-converter/src/i3s-server/app.ts` and `modules/tile-converter/src/i3s-server/bin/www.ts` (see comments);
-- run `PORT=8080 HTTPS_PORT=4443 I3sLayerPath="./path/to/dataset" DEBUG=i3s-server:* npx tsx ./src/i3s-server/bin/www.ts`
+- uncomment "local debug" lines in `src/v4/i3s-server/app.ts` and `src/v4/i3s-server/bin/www.ts` (see comments);
+- run `PORT=8080 HTTPS_PORT=4443 I3sLayerPath="./path/to/dataset" DEBUG=i3s-server:* npx tsx ./src/v4/i3s-server/bin/www.ts`
 
 ## Usage
 

package/src/{i3s-server → v4/i3s-server}/routes/slpk-router.ts

@@ -29,6 +29,7 @@ router.get('*', (req, res, next) => {
         const json = JSON.parse(textDecoder.decode(file));
         res.send(json);
       } catch (_e) {
+        // Express 4 sends typed arrays as JSON; adapt only at this Node.js HTTP boundary.
         res.send(Buffer.from(file));
       }
     } else {

package/src/v4/index.ts

@@ -0,0 +1,2 @@
+export {default as I3SConverter} from './i3s-converter/i3s-converter';
+export {default as Tiles3DConverter} from './3d-tiles-converter/3d-tiles-converter';

package/src/v5/browser-sink.ts

@@ -0,0 +1,129 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TileConversionError} from './conversion-api.js';
+import type {TileConversionReport, TileConversionSink} from './conversion-api.js';
+
+const BLOB_SIZE_GETTER = Object.getOwnPropertyDescriptor(Blob.prototype, 'size')?.get;
+
+/** One named browser output resource. */
+export interface BrowserTileConversionResource {
+  /** Stable relative output path supplied by the target profile. */
+  readonly resourceId: string;
+  /** Parts combined into a Blob after the total output budget is checked. */
+  readonly parts: readonly BlobPart[];
+  /** MIME type recorded on the resulting Blob. */
+  readonly contentType?: string;
+}
+
+/** A named output resource retained by the bounded browser sink. */
+export interface BrowserTileConversionFile {
+  /** Stable relative output path supplied by the target profile. */
+  readonly resourceId: string;
+  /** Browser-native data for download or application-provided packaging. */
+  readonly blob: Blob;
+}
+
+/** Memory limit for a browser conversion sink. */
+export interface BoundedMemoryTileConversionSinkOptions {
+  /** Maximum combined Blob size retained by this sink, in bytes. */
+  readonly maxTotalBytes: number;
+}
+
+/** Browser sink with read access to its bounded output resources. */
+export interface BoundedMemoryTileConversionSink
+  extends TileConversionSink<BrowserTileConversionResource> {
+  /** Returns the retained files in stable resource-ID order. */
+  getFiles(): readonly BrowserTileConversionFile[];
+}
+
+/**
+ * Creates a browser-only sink that retains output as Blobs under a total byte limit.
+ *
+ * The sink does not download or archive files. Applications can use `getFiles()` after
+ * `convertTileset` completes and package or download the returned Blobs as needed.
+ *
+ * @param options - Maximum in-memory output size.
+ * @returns A sink that stores browser-native Blobs and clears them on abort.
+ */
+export function createBoundedMemoryTileConversionSink(
+  options: BoundedMemoryTileConversionSinkOptions
+): BoundedMemoryTileConversionSink {
+  if (!Number.isSafeInteger(options.maxTotalBytes) || options.maxTotalBytes < 0) {
+    throw new TileConversionError(
+      'INVALID_OUTPUT_MEMORY_LIMIT',
+      'The browser output memory limit must be a non-negative safe integer'
+    );
+  }
+
+  const files = new Map<string, BrowserTileConversionFile>();
+  let totalBytes = 0;
+  let closed = false;
+
+  return {
+    getFiles() {
+      return [...files.values()].sort((left, right) =>
+        left.resourceId < right.resourceId ? -1 : left.resourceId > right.resourceId ? 1 : 0
+      );
+    },
+    async write(resource, signal) {
+      if (signal?.aborted) {
+        throw signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
+      }
+      if (closed) {
+        throw new TileConversionError(
+          'RESOURCE_SINK_CLOSED',
+          'Cannot write after sink finalization'
+        );
+      }
+      if (!resource.resourceId || files.has(resource.resourceId)) {
+        throw new TileConversionError(
+          'INVALID_BROWSER_RESOURCE_ID',
+          'Browser output resource IDs must be non-empty and unique'
+        );
+      }
+
+      const byteLength = resource.parts.reduce((total, part) => total + measureBlobPart(part), 0);
+      if (!Number.isSafeInteger(byteLength) || totalBytes + byteLength > options.maxTotalBytes) {
+        throw new TileConversionError(
+          'OUTPUT_MEMORY_LIMIT_EXCEEDED',
+          `Browser output exceeds the configured ${options.maxTotalBytes} byte memory limit`
+        );
+      }
+
+      const blob = new Blob([...resource.parts], {type: resource.contentType});
+      files.set(resource.resourceId, {resourceId: resource.resourceId, blob});
+      totalBytes += blob.size;
+    },
+    async finalize(_report: TileConversionReport) {
+      if (closed) {
+        throw new TileConversionError('RESOURCE_SINK_CLOSED', 'Sink has already been finalized');
+      }
+      closed = true;
+    },
+    async abort() {
+      files.clear();
+      totalBytes = 0;
+      closed = true;
+    }
+  };
+}
+
+/** Measures supported Blob parts without copying them into a Blob first. */
+function measureBlobPart(part: BlobPart): number {
+  if (typeof part === 'string') {
+    return new TextEncoder().encode(part).byteLength;
+  }
+  if (BLOB_SIZE_GETTER) {
+    try {
+      return BLOB_SIZE_GETTER.call(part) as number;
+    } catch {
+      // Keep checking the remaining supported Blob part types.
+    }
+  }
+  if (part instanceof ArrayBuffer || ArrayBuffer.isView(part)) {
+    return part.byteLength;
+  }
+  throw new TileConversionError('UNSUPPORTED_BROWSER_RESOURCE', 'Unsupported browser Blob part');
+}

package/src/v5/browser-source.ts

@@ -0,0 +1,155 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TileConversionError} from './conversion-api.js';
+import type {TileConversionSource} from './conversion-api.js';
+
+/** Metadata available before a browser resource is read. */
+export interface BrowserTileConversionInspection {
+  /** Stable resource path assigned to the input tile. */
+  readonly resourceId: string;
+  /** Optional MIME type supplied by the caller or source Blob. */
+  readonly contentType?: string;
+}
+
+/** Raw bytes read from one browser tile resource. */
+export interface BrowserTileConversionInputResource {
+  /** Stable resource path assigned to the input tile. */
+  readonly resourceId: string;
+  /** Complete resource bytes for a profile-specific codec. */
+  readonly data: Uint8Array;
+  /** Optional MIME type supplied by the caller, Blob, or URL response. */
+  readonly contentType?: string;
+}
+
+/** Browser input and size limit for a single-resource conversion. */
+export interface BrowserTileConversionSourceOptions {
+  /** Input URL or already available browser Blob. */
+  readonly input: string | Blob;
+  /** Stable path to associate with the input resource. */
+  readonly resourceId: string;
+  /** Maximum number of input bytes held in memory. */
+  readonly maxInputBytes: number;
+  /** Optional MIME type override. */
+  readonly contentType?: string;
+}
+
+/**
+ * Creates a browser source for one mesh resource provided as a URL or Blob.
+ *
+ * URL responses are read incrementally and canceled if they exceed `maxInputBytes`.
+ * Tileset JSON traversal and format decoding remain profile-specific source and codec work.
+ *
+ * @param options - Input, stable resource ID, and maximum input size.
+ * @returns A source that yields one complete resource to a v5 codec.
+ */
+export function createBrowserTileConversionSource(
+  options: BrowserTileConversionSourceOptions
+): TileConversionSource<BrowserTileConversionInspection, BrowserTileConversionInputResource> {
+  if (
+    !options.resourceId ||
+    !Number.isSafeInteger(options.maxInputBytes) ||
+    options.maxInputBytes < 0
+  ) {
+    throw new TileConversionError(
+      'INVALID_BROWSER_SOURCE_OPTIONS',
+      'Browser sources require a resource ID and a non-negative safe input byte limit'
+    );
+  }
+
+  return {
+    async inspect(signal) {
+      throwIfAborted(signal);
+      const contentType = options.contentType ?? getBlobContentType(options.input);
+      return {
+        resourceId: options.resourceId,
+        ...(contentType ? {contentType} : {})
+      };
+    },
+    async *read(inspection, signal) {
+      throwIfAborted(signal);
+      const input = options.input;
+      let data: Uint8Array;
+      let contentType = inspection.contentType;
+
+      if (typeof input === 'string') {
+        const response = await fetch(input, {signal});
+        if (!response.ok) {
+          throw new TileConversionError(
+            'BROWSER_SOURCE_FETCH_FAILED',
+            `Could not fetch browser input (${response.status} ${response.statusText})`
+          );
+        }
+        contentType = contentType ?? response.headers.get('content-type') ?? undefined;
+        data = await readResponseBytes(response, options.maxInputBytes, signal);
+      } else {
+        if (input.size > options.maxInputBytes) {
+          throw createInputLimitError(options.maxInputBytes);
+        }
+        data = new Uint8Array(await input.arrayBuffer());
+      }
+
+      throwIfAborted(signal);
+      yield {
+        resourceId: inspection.resourceId,
+        data,
+        ...(contentType ? {contentType} : {})
+      };
+    }
+  };
+}
+
+/** Reads a response stream while enforcing the input byte limit. */
+async function readResponseBytes(
+  response: Response,
+  maxInputBytes: number,
+  signal?: AbortSignal
+): Promise<Uint8Array> {
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new TileConversionError(
+      'BROWSER_SOURCE_STREAM_UNAVAILABLE',
+      'Browser URL inputs must provide a readable response stream'
+    );
+  }
+
+  const data = new Uint8Array(maxInputBytes);
+  let totalBytes = 0;
+  try {
+    while (true) {
+      throwIfAborted(signal);
+      const {done, value} = await reader.read();
+      if (done) break;
+      if (value.byteLength > maxInputBytes - totalBytes) {
+        await reader.cancel();
+        throw createInputLimitError(maxInputBytes);
+      }
+      data.set(value, totalBytes);
+      totalBytes += value.byteLength;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return data.subarray(0, totalBytes);
+}
+
+/** Returns a Blob's MIME type without requiring a same-realm Blob constructor. */
+function getBlobContentType(input: string | Blob): string | undefined {
+  return typeof input === 'string' ? undefined : input.type || undefined;
+}
+
+/** Creates the typed error used when browser input exceeds its configured limit. */
+function createInputLimitError(maxInputBytes: number): TileConversionError {
+  return new TileConversionError(
+    'INPUT_RESOURCE_TOO_LARGE',
+    `Browser input exceeds the configured ${maxInputBytes} byte memory limit`
+  );
+}
+
+/** Throws the signal reason when a browser source operation has been canceled. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
+  }
+}