@loaders.gl/core 4.4.0-alpha.1 → 4.4.0-alpha.9

package/src/iterators/make-iterator/make-string-iterator.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import {ensureArrayBuffer} from '@loaders.gl/loader-utils';
 import type {IteratorOptions} from './make-iterator';
 
 const DEFAULT_CHUNK_SIZE = 256 * 1024;
@@ -27,6 +28,6 @@ export function* makeStringIterator(
     offset += chunkLength;
 
     // yield an ArrayBuffer chunk
-    yield textEncoder.encode(chunk);
+    yield ensureArrayBuffer(textEncoder.encode(chunk));
   }
 }

package/src/lib/api/encode-table.ts

@@ -6,6 +6,7 @@
 /* global TextEncoder, TextDecoder */
 import {
   concatenateArrayBuffers,
+  ensureArrayBuffer,
   WriterOptionsType,
   WriterWithEncoder
 } from '@loaders.gl/loader-utils';
@@ -22,7 +23,7 @@ export async function encodeTable<WriterT extends WriterWithEncoder = WriterWith
 
   if (writer.encodeText) {
     const text = await writer.encodeText(data, options);
-    return new TextEncoder().encode(text);
+    return ensureArrayBuffer(new TextEncoder().encode(text));
   }
 
   if (writer.encodeInBatches) {

package/src/lib/api/encode.ts

@@ -9,7 +9,13 @@ import type {
   WriterDataType,
   WriterBatchType
 } from '@loaders.gl/loader-utils';
-import {canEncodeWithWorker, NodeFile, resolvePath, isBrowser} from '@loaders.gl/loader-utils';
+import {
+  canEncodeWithWorker,
+  NodeFile,
+  resolvePath,
+  isBrowser,
+  ensureArrayBuffer
+} from '@loaders.gl/loader-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
 import {fetchFile} from '../fetch/fetch-file';
 import {getLoaderOptions} from './loader-options';
@@ -52,7 +58,7 @@ export function encodeSync<WriterT extends WriterWithEncoder>(
     return writer.encodeSync(data, options);
   }
   if (writer.encodeTextSync) {
-    return new TextEncoder().encode(writer.encodeTextSync(data, options));
+    return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));
   }
   throw new Error(`Writer ${writer.name} could not synchronously encode data`);
 }

package/src/lib/api/load.ts

@@ -12,7 +12,7 @@ import type {
   LoaderArrayOptionsType,
   LoaderArrayReturnType
 } from '@loaders.gl/loader-utils';
-import {isBlob} from '../../javascript-utils/is-type';
+import {isBlob} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {getFetchFunction} from '../loader-utils/get-fetch-function';
 

package/src/lib/api/parse-in-batches.ts

@@ -7,13 +7,15 @@ import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
 import type {
   Loader,
   LoaderWithParser,
+  StrictLoaderOptions,
   LoaderOptions,
   LoaderContext,
   BatchableDataType,
   LoaderOptionsType,
   LoaderBatchType,
   LoaderArrayOptionsType,
-  LoaderArrayBatchType
+  LoaderArrayBatchType,
+  TransformBatches
 } from '@loaders.gl/loader-utils';
 import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
@@ -80,7 +82,7 @@ export async function parseInBatches(
   // Signature: parseInBatches(data, options, url) - Uses registered loaders
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
     context = undefined; // context not supported in short signature
-    options = loaders as LoaderOptions;
+    options = loaders as unknown as LoaderOptions;
     loaders = undefined;
   }
 
@@ -92,21 +94,21 @@ export async function parseInBatches(
 
   // Chooses a loader and normalizes it
   // Note - only uses URL and contentType for streams and iterator inputs
-  const loader = await selectLoader(data as ArrayBuffer, loaders, options);
+  const loader = await selectLoader(data, loaders, options);
   // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
   if (!loader) {
     return [];
   }
 
   // Normalize options
-  options = normalizeOptions(options, loader, loaderArray, url);
+  const strictOptions = normalizeOptions(options, loader, loaderArray, url);
   context = getLoaderContext(
     {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
-    options,
+    strictOptions,
     context || null
   );
 
-  return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);
+  return await parseWithLoaderInBatches(loader as LoaderWithParser, data, strictOptions, context);
 }
 
 /**
@@ -115,13 +117,13 @@ export async function parseInBatches(
 async function parseWithLoaderInBatches(
   loader: LoaderWithParser,
   data: BatchableDataType,
-  options: LoaderOptions,
+  options: StrictLoaderOptions,
   context: LoaderContext
 ): Promise<AsyncIterable<unknown>> {
   const outputIterator = await parseToOutputIterator(loader, data, options, context);
 
   // Generate metadata batch if requested
-  if (!options.metadata) {
+  if (!options?.core?.metadata) {
     return outputIterator;
   }
 
@@ -155,14 +157,17 @@ async function parseWithLoaderInBatches(
 async function parseToOutputIterator(
   loader: LoaderWithParser,
   data: BatchableDataType,
-  options: LoaderOptions,
+  options: StrictLoaderOptions,
   context: LoaderContext
 ): Promise<AsyncIterable<unknown>> {
   // Get an iterator from the input
   const inputIterator = await getAsyncIterableFromData(data, options);
 
   // Apply any iterator transforms (options.transforms)
-  const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);
+  const transformedIterator = await applyInputTransforms(
+    inputIterator,
+    options?.core?.transforms || []
+  );
 
   // If loader supports parseInBatches, we are done
   if (loader.parseInBatches) {
@@ -174,9 +179,11 @@ async function parseToOutputIterator(
 
 // Fallback: load atomically using `parse` concatenating input iterator into single chunk
 async function* parseChunkInBatches(
-  transformedIterator: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
+  transformedIterator:
+    | Iterable<ArrayBufferLike | ArrayBufferView>
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>,
   loader: Loader,
-  options: LoaderOptions,
+  options: StrictLoaderOptions,
   context: LoaderContext
 ): AsyncIterable<Batch> {
   const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
@@ -185,7 +192,7 @@ async function* parseChunkInBatches(
     arrayBuffer,
     loader,
     // TODO - Hack: supply loaders MIME type to ensure we match it
-    {...options, mimeType: loader.mimeTypes[0]},
+    {...options, core: {...options?.core, mimeType: loader.mimeTypes[0]}},
     context
   );
 
@@ -215,19 +222,19 @@ function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
   return batch;
 }
 
-type TransformBatches = (
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
-) => AsyncIterable<ArrayBuffer>;
-
 /**
  * Create an iterator chain with any transform iterators (crypto, decompression)
  * @param inputIterator
  * @param options
  */
 async function applyInputTransforms(
-  inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  inputIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
   transforms: TransformBatches[] = []
-): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {
+): Promise<
+  AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>
+> {
   let iteratorChain = inputIterator;
   for await (const transformBatches of transforms) {
     iteratorChain = transformBatches(iteratorChain);

package/src/lib/api/parse-sync.ts

@@ -11,7 +11,8 @@ import type {
   LoaderOptionsType,
   LoaderReturnType,
   LoaderArrayOptionsType,
-  LoaderArrayReturnType
+  LoaderArrayReturnType,
+  StrictLoaderOptions
 } from '@loaders.gl/loader-utils';
 import {selectLoaderSync} from './select-loader';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
@@ -84,7 +85,7 @@ export function parseSync(
   }
 
   // Normalize options
-  options = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
+  const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
 
   // Extract a url for auto detection
   const url = getResourceUrl(data);
@@ -94,18 +95,18 @@ export function parseSync(
   };
   context = getLoaderContext(
     {url, _parseSync: parse, _parse: parse, loaders: loaders as Loader[]},
-    options,
+    strictOptions,
     context || null
   );
 
-  return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);
+  return parseWithLoaderSync(loader as LoaderWithParser, data, strictOptions, context);
 }
 
 // TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
 function parseWithLoaderSync(
   loader: LoaderWithParser,
   data: SyncDataType,
-  options: LoaderOptions,
+  options: StrictLoaderOptions,
   context: LoaderContext
 ) {
   data = getArrayBufferOrStringFromDataSync(data, loader, options);

package/src/lib/api/parse.ts

@@ -11,12 +11,17 @@ import type {
   LoaderOptionsType,
   LoaderReturnType,
   LoaderArrayOptionsType,
-  LoaderArrayReturnType
+  LoaderArrayReturnType,
+  StrictLoaderOptions
+} from '@loaders.gl/loader-utils';
+import {
+  parseWithWorker,
+  canParseWithWorker,
+  mergeOptions,
+  isResponse
 } from '@loaders.gl/loader-utils';
-import {parseWithWorker, canParseWithWorker, mergeOptions} from '@loaders.gl/loader-utils';
 import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
-import {isResponse} from '../../javascript-utils/is-type';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
@@ -100,18 +105,18 @@ export async function parse(
   }
 
   // Normalize options
-  // @ts-expect-error
-  options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
+  // @ts-expect-error candidateLoaders
+  const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
 
   // Get a context (if already present, will be unchanged)
   context = getLoaderContext(
     // @ts-expect-error
     {url, _parse: parse, loaders: candidateLoaders},
-    options,
+    strictOptions,
     context || null
   );
 
-  return await parseWithLoader(loader, data, options, context);
+  return await parseWithLoader(loader, data, strictOptions, context);
 }
 
 // TODO: support progress and abort
@@ -119,7 +124,7 @@ export async function parse(
 async function parseWithLoader(
   loader: Loader,
   data,
-  options: LoaderOptions,
+  options: StrictLoaderOptions,
   context: LoaderContext
 ): Promise<unknown> {
   validateWorkerVersion(loader);
@@ -128,9 +133,8 @@ async function parseWithLoader(
 
   if (isResponse(data)) {
     // Serialize to support passing the response to web worker
-    const response = data as Response;
-    const {ok, redirected, status, statusText, type, url} = response;
-    const headers = Object.fromEntries(response.headers.entries());
+    const {ok, redirected, status, statusText, type, url} = data;
+    const headers = Object.fromEntries(data.headers.entries());
     // @ts-expect-error TODO - fix this
     context.response = {headers, ok, redirected, status, statusText, type, url};
   }

package/src/lib/api/select-loader.ts

@@ -2,14 +2,21 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-utils';
-import {compareArrayBuffers, path, log} from '@loaders.gl/loader-utils';
+import type {LoaderContext, LoaderOptions, Loader, DataType} from '@loaders.gl/loader-utils';
+import {
+  compareArrayBuffers,
+  path,
+  log,
+  isBlob,
+  ensureArrayBuffer,
+  isArrayBufferLike
+} from '@loaders.gl/loader-utils';
 import {TypedArray} from '@loaders.gl/schema';
 import {normalizeLoader} from '../loader-utils/normalize-loader';
+import {normalizeLoaderOptions} from '../loader-utils/option-utils';
 import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
 import {compareMIMETypes} from '../utils/mime-type-utils';
 import {getRegisteredLoaders} from './register-loaders';
-import {isBlob} from '../../javascript-utils/is-type';
 import {stripQueryString} from '../utils/url-utils';
 
 const EXT_PATTERN = /\.([^.]+)$/;
@@ -28,7 +35,7 @@ const EXT_PATTERN = /\.([^.]+)$/;
  * @param context used internally, applications should not provide this parameter
  */
 export async function selectLoader(
-  data: Response | Blob | ArrayBuffer | string,
+  data: DataType,
   loaders: Loader[] | Loader = [],
   options?: LoaderOptions,
   context?: LoaderContext
@@ -37,8 +44,16 @@ export async function selectLoader(
     return null;
   }
 
+  const normalizedOptions = normalizeLoaderOptions(options || {});
+  normalizedOptions.core ||= {};
+
   // First make a sync attempt, disabling exceptions
-  let loader = selectLoaderSync(data, loaders, {...options, nothrow: true}, context);
+  let loader = selectLoaderSync(
+    data,
+    loaders,
+    {...normalizedOptions, core: {...normalizedOptions.core, nothrow: true}},
+    context
+  );
   if (loader) {
     return loader;
   }
@@ -46,12 +61,12 @@ export async function selectLoader(
   // For Blobs and Files, try to asynchronously read a small initial slice and test again with that
   // to see if we can detect by initial content
   if (isBlob(data)) {
-    data = await (data as Blob).slice(0, 10).arrayBuffer();
-    loader = selectLoaderSync(data, loaders, options, context);
+    data = await data.slice(0, 10).arrayBuffer();
+    loader = selectLoaderSync(data, loaders, normalizedOptions, context);
   }
 
   // no loader available
-  if (!loader && !options?.nothrow) {
+  if (!loader && !normalizedOptions.core.nothrow) {
     throw new Error(getNoValidLoaderMessage(data));
   }
 
@@ -68,7 +83,7 @@ export async function selectLoader(
  * @param context used internally, applications should not provide this parameter
  */
 export function selectLoaderSync(
-  data: Response | Blob | ArrayBuffer | string,
+  data: DataType,
   loaders: Loader[] | Loader = [],
   options?: LoaderOptions,
   context?: LoaderContext
@@ -77,6 +92,9 @@ export function selectLoaderSync(
     return null;
   }
 
+  const normalizedOptions = normalizeLoaderOptions(options || {});
+  normalizedOptions.core ||= {};
+
   // eslint-disable-next-line complexity
   // if only a single loader was provided (not as array), force its use
   // TODO - Should this behavior be kept and documented?
@@ -92,17 +110,17 @@ export function selectLoaderSync(
     candidateLoaders = candidateLoaders.concat(loaders);
   }
   // Then fall back to registered loaders
-  if (!options?.ignoreRegisteredLoaders) {
+  if (!normalizedOptions.core.ignoreRegisteredLoaders) {
     candidateLoaders.push(...getRegisteredLoaders());
   }
 
   // TODO - remove support for legacy loaders
   normalizeLoaders(candidateLoaders);
 
-  const loader = selectLoaderInternal(data, candidateLoaders, options, context);
+  const loader = selectLoaderInternal(data, candidateLoaders, normalizedOptions, context);
 
   // no loader available
-  if (!loader && !options?.nothrow) {
+  if (!loader && !normalizedOptions.core.nothrow) {
     throw new Error(getNoValidLoaderMessage(data));
   }
 
@@ -112,7 +130,7 @@ export function selectLoaderSync(
 /** Implements loaders selection logic */
 // eslint-disable-next-line complexity
 function selectLoaderInternal(
-  data: Response | Blob | ArrayBuffer | string,
+  data: DataType,
   loaders: Loader[],
   options?: LoaderOptions,
   context?: LoaderContext
@@ -126,9 +144,9 @@ function selectLoaderInternal(
   let reason: string = '';
 
   // if options.mimeType is supplied, it takes precedence
-  if (options?.mimeType) {
-    loader = findLoaderByMIMEType(loaders, options?.mimeType);
-    reason = `match forced by supplied MIME type ${options?.mimeType}`;
+  if (options?.core?.mimeType) {
+    loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
+    reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
   }
 
   // Look up loader by url
@@ -146,8 +164,8 @@ function selectLoaderInternal(
   reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
 
   // Look up loader by fallback mime type
-  if (options?.fallbackMimeType) {
-    loader = loader || findLoaderByMIMEType(loaders, options?.fallbackMimeType);
+  if (options?.core?.fallbackMimeType) {
+    loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
     reason = reason || (loader ? `matched fallback MIME type ${type}` : '');
   }
 
@@ -171,7 +189,7 @@ function validHTTPResponse(data: unknown): boolean {
 }
 
 /** Generate a helpful message to help explain why loader selection failed. */
-function getNoValidLoaderMessage(data: string | ArrayBuffer | Response | Blob): string {
+function getNoValidLoaderMessage(data: DataType): string {
   const url = getResourceUrl(data);
   const type = getResourceMIMEType(data);
 
@@ -264,23 +282,23 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
   return tests.some((test) => data.startsWith(test as string));
 }
 
-function testDataAgainstBinary(data: ArrayBuffer, byteOffset: number, loader: Loader): boolean {
+function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
   const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
   return tests.some((test) => testBinary(data, byteOffset, loader, test));
 }
 
 function testBinary(
-  data: ArrayBuffer,
+  data: ArrayBufferLike,
   byteOffset: number,
   loader: Loader,
   test?: ArrayBuffer | string | ((b: ArrayBuffer) => boolean)
 ): boolean {
-  if (test instanceof ArrayBuffer) {
+  if (isArrayBufferLike(test)) {
     return compareArrayBuffers(test, data, test.byteLength);
   }
   switch (typeof test) {
     case 'function':
-      return test(data);
+      return test(ensureArrayBuffer(data));
 
     case 'string':
       // Magic bytes check: If `test` is a string, check if binary data starts with that strings
@@ -305,7 +323,7 @@ function getFirstCharacters(data: string | ArrayBuffer | TypedArray, length: num
   return '';
 }
 
-function getMagicString(arrayBuffer: ArrayBuffer, byteOffset: number, length: number): string {
+function getMagicString(arrayBuffer: ArrayBufferLike, byteOffset: number, length: number): string {
   if (arrayBuffer.byteLength < byteOffset + length) {
     return '';
   }