@loaders.gl/core 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/lib/api/encode-table.ts

@@ -0,0 +1,71 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc
+
+/* global TextEncoder, TextDecoder */
+import {concatenateArrayBuffers, Writer, WriterOptionsType} from '@loaders.gl/loader-utils';
+import {Table} from '@loaders.gl/schema';
+
+export async function encodeTable<WriterT extends Writer = Writer>(
+  data: Table,
+  writer: WriterT,
+  options?: WriterOptionsType<WriterT>
+): Promise<ArrayBuffer> {
+  if (writer.encode) {
+    return await writer.encode(data, options);
+  }
+
+  if (writer.encodeText) {
+    const text = await writer.encodeText(data, options);
+    return new TextEncoder().encode(text);
+  }
+
+  if (writer.encodeInBatches) {
+    // Create an iterator representing the data
+    // TODO - Assumes this is a table
+    const batches = encodeTableInBatches(data, writer, options);
+
+    // Concatenate the output
+    const chunks: ArrayBuffer[] = [];
+    for await (const batch of batches) {
+      chunks.push(batch);
+    }
+    return concatenateArrayBuffers(...chunks);
+  }
+
+  throw new Error('Writer could not encode data');
+}
+
+export async function encodeTableAsText<WriterT extends Writer = Writer>(
+  data: Table,
+  writer: WriterT,
+  options?: WriterOptionsType<WriterT>
+): Promise<string> {
+  if (writer.text && writer.encodeText) {
+    return await writer.encodeText(data, options);
+  }
+
+  if (writer.text && (writer.encode || writer.encodeInBatches)) {
+    const arrayBuffer = await encodeTable(data, writer, options);
+    return new TextDecoder().decode(arrayBuffer);
+  }
+  throw new Error('Writer could not encode data as text');
+}
+
+export function encodeTableInBatches<WriterT extends Writer = Writer>(
+  data: Table,
+  writer: WriterT,
+  options?: WriterOptionsType<WriterT>
+): AsyncIterable<ArrayBuffer> {
+  if (writer.encodeInBatches) {
+    const dataIterator = getIterator(data);
+    // @ts-expect-error
+    return writer.encodeInBatches(dataIterator, options);
+  }
+  // TODO -fall back to atomic encode?
+  throw new Error('Writer could not encode data in batches');
+}
+
+function getIterator(data) {
+  const dataIterator = [{table: data, start: 0, end: data.length}];
+  return dataIterator;
+}

package/src/lib/api/encode.ts

@@ -1,8 +1,10 @@
-import type {Writer, LoaderOptions} from '@loaders.gl/loader-utils';
+import {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils';
+import {processOnWorker} from '@loaders.gl/worker-utils';
 import {concatenateArrayBuffers, resolvePath} from '@loaders.gl/loader-utils';
 import {isBrowser} from '@loaders.gl/loader-utils';
 import {writeFile} from '../fetch/write-file';
 import {fetchFile} from '../fetch/fetch-file';
+import {getLoaderOptions} from './loader-options';
 
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
@@ -10,8 +12,15 @@ import {fetchFile} from '../fetch/fetch-file';
 export async function encode(
   data: any,
   writer: Writer,
-  options?: LoaderOptions
+  options?: WriterOptions
 ): Promise<ArrayBuffer> {
+  const globalOptions = getLoaderOptions() as WriterOptions;
+  // const globalOptions: WriterOptions = {}; // getWriterOptions();
+  options = {...globalOptions, ...options};
+  if (canEncodeWithWorker(writer, options)) {
+    return await processOnWorker(writer, data, options);
+  }
+
   // TODO Merge default writer options with options argument like it is done in load module.
   if (writer.encode) {
     return await writer.encode(data, options);
@@ -63,7 +72,7 @@ export async function encode(
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
  */
-export function encodeSync(data: any, writer: Writer, options?: LoaderOptions): ArrayBuffer {
+export function encodeSync(data: any, writer: Writer, options?: WriterOptions): ArrayBuffer {
   if (writer.encodeSync) {
     return writer.encodeSync(data, options);
   }
@@ -79,7 +88,7 @@ export function encodeSync(data: any, writer: Writer, options?: LoaderOptions):
 export async function encodeText(
   data: any,
   writer: Writer,
-  options?: LoaderOptions
+  options?: WriterOptions
 ): Promise<string> {
   if (writer.text && writer.encodeText) {
     return await writer.encodeText(data, options);
@@ -99,10 +108,11 @@ export async function encodeText(
 export function encodeInBatches(
   data: any,
   writer: Writer,
-  options?: LoaderOptions
+  options?: WriterOptions
 ): AsyncIterable<ArrayBuffer> {
   if (writer.encodeInBatches) {
     const dataIterator = getIterator(data);
+    // @ts-expect-error
     return writer.encodeInBatches(dataIterator, options);
   }
   // TODO -fall back to atomic encode?

package/src/lib/api/load-in-batches.ts

@@ -1,6 +1,6 @@
 import type {LoaderWithParser, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
-import {getFetchFunction} from '../loader-utils/option-utils';
+import {getFetchFunction} from '../loader-utils/get-fetch-function';
 
 import {parseInBatches} from './parse-in-batches';
 

package/src/lib/api/load.ts

@@ -1,7 +1,10 @@
+// loaders.gl, MIT license
+
 import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
 import {isBlob} from '../../javascript-utils/is-type';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
-import {getFetchFunction} from '../loader-utils/option-utils';
+import {getFetchFunction} from '../loader-utils/get-fetch-function';
 
 import {parse} from './parse';
 
@@ -14,22 +17,46 @@ import {parse} from './parse';
  * @param options
  * @param context
  */
+
+export async function load<LoaderT extends Loader>(
+  url: string | DataType,
+  loader: LoaderT,
+  options?: LoaderOptionsType<LoaderT>,
+  context?: LoaderContext
+): Promise<LoaderReturnType<LoaderT>>;
+
+export async function load<
+  LoaderT extends Loader, // eslint-disable-line  @typescript-eslint/no-unused-vars
+  LoaderOptionsT extends LoaderOptions = LoaderOptions
+>(
+  url: string | DataType,
+  loaders: Loader[] | LoaderOptions,
+  options?: LoaderOptionsT,
+  context?: LoaderContext
+): Promise<any>;
+
 // implementation signature
-export async function load(
+export async function load<LoaderOptionsT extends LoaderOptions>(
   url: string | DataType,
-  loaders?: Loader | Loader[] | LoaderOptions,
-  options?: LoaderOptions,
+  loaders?: Loader[] | LoaderOptions,
+  options?: LoaderOptionsT,
   context?: LoaderContext
 ): Promise<any> {
+  let resolvedLoaders: Loader | Loader[];
+  let resolvedOptions: LoaderOptionsT | undefined;
+
   // Signature: load(url, options)
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
+    resolvedLoaders = [];
+    resolvedOptions = loaders as LoaderOptionsT;
     context = undefined; // context not supported in short signature
-    options = loaders as LoaderOptions;
-    loaders = undefined;
+  } else {
+    resolvedLoaders = loaders as Loader | Loader[];
+    resolvedOptions = options;
   }
 
   // Select fetch function
-  const fetch = getFetchFunction(options);
+  const fetch = getFetchFunction(resolvedOptions);
 
   // at this point, `url` could be already loaded binary data
   let data = url;
@@ -41,9 +68,13 @@ export async function load(
 
   if (isBlob(url)) {
     // The fetch response object will contain blob.name
+    // @ts-expect-error TODO - This may not work for overridden fetch functions
     data = await fetch(url);
   }
 
   // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
-  return await parse(data, loaders, options);
+  // return await parse(data, loaders as Loader[], options);
+  return Array.isArray(resolvedLoaders)
+    ? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
+    : await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
 }

package/src/lib/api/loader-options.ts

@@ -0,0 +1,2 @@
+export {setGlobalOptions as setLoaderOptions} from '../loader-utils/option-utils';
+export {getGlobalLoaderOptions as getLoaderOptions} from '../loader-utils/option-utils';

package/src/lib/api/parse-in-batches.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getLoaderContext} from '../loader-utils/loader-context';
 import {getAsyncIterableFromData} from '../loader-utils/get-data';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
 // Ensure `parse` is available in context if loader falls back to `parse`
@@ -32,6 +32,8 @@ export async function parseInBatches(
 ): Promise<AsyncIterable<any>> {
   assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
 
+  const loaderArray = Array.isArray(loaders) ? loaders : undefined;
+
   // Signature: parseInBatches(data, options, url) - Uses registered loaders
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
     context = undefined; // context not supported in short signature
@@ -43,11 +45,11 @@ export async function parseInBatches(
   options = options || {};
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader and normalizes it
   // Note - only uses URL and contentType for streams and iterator inputs
-  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);
+  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader | Loader[], options);
   // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
   if (!loader) {
     // @ts-ignore
@@ -55,14 +57,11 @@ export async function parseInBatches(
   }
 
   // Normalize options
-  // @ts-ignore
-  options = normalizeOptions(options, loader, loaders, url);
-  // @ts-ignore
+  options = normalizeOptions(options, loader, loaderArray, url);
   context = getLoaderContext(
-    // @ts-ignore
-    {url, parseInBatches, parse, loaders: loaders as Loader[]},
+    {url, parseInBatches, parse, loaders: loaderArray},
     options,
-    context
+    context || null
   );
 
   return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);

package/src/lib/api/parse-sync.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 
 /**
  * Parses `data` synchronously using a specified loader
@@ -52,12 +52,16 @@ export function parseSync(
   options = normalizeOptions(options, loader, candidateLoaders);
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   const parse = () => {
-    throw new Error('parseSync called parse');
+    throw new Error('parseSync called parse (which is async');
   };
-  context = getLoaderContext({url, parseSync, parse, loaders: loaders as Loader[]}, options);
+  context = getLoaderContext(
+    {url, parseSync, parse, loaders: loaders as Loader[]},
+    options,
+    context || null
+  );
 
   return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);
 }

package/src/lib/api/parse.ts

@@ -1,13 +1,41 @@
+// loaders.gl, MIT license
+
 import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
 import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {parseWithWorker, canParseWithWorker} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
+import {isResponse} from '../../javascript-utils/is-type';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
+// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
+
+export async function parse<
+  LoaderT extends Loader,
+  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+>(
+  data: DataType | Promise<DataType>,
+  loader: LoaderT,
+  options?: OptionsT,
+  context?: LoaderContext
+): Promise<LoaderReturnType<LoaderT>>;
+
+export async function parse(
+  data: DataType | Promise<DataType>,
+  loaders: Loader[],
+  options?: LoaderOptions,
+  context?: LoaderContext
+): Promise<any>;
+
+export async function parse(
+  data: DataType | Promise<DataType>,
+  options?: LoaderOptions
+): Promise<any>;
+
 /**
  * Parses `data` using a specified loader
  * @param data
@@ -15,6 +43,7 @@ import {selectLoader} from './select-loader';
  * @param options
  * @param context
  */
+// implementation signature
 export async function parse(
   data: DataType | Promise<DataType>,
   loaders?: Loader | Loader[] | LoaderOptions,
@@ -32,10 +61,10 @@ export async function parse(
   }
 
   data = await data; // Resolve any promise
-  options = options || {};
+  options = options || ({} as LoaderOptions); // Could be invalid...
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader (and normalizes it)
   // Also use any loaders in the context, new loaders take priority
@@ -49,10 +78,10 @@ export async function parse(
   }
 
   // Normalize options
-  options = normalizeOptions(options, loader, candidateLoaders, url);
+  options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
 
   // Get a context (if already present, will be unchanged)
-  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context);
+  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context || null);
 
   return await parseWithLoader(loader, data, options, context);
 }
@@ -62,6 +91,14 @@ export async function parse(
 async function parseWithLoader(loader, data, options, context) {
   validateWorkerVersion(loader);
 
+  if (isResponse(data)) {
+    // Serialize to support passing the response to web worker
+    const response = data as Response;
+    const {ok, redirected, status, statusText, type, url} = response;
+    const headers = Object.fromEntries(response.headers.entries());
+    context.response = {headers, ok, redirected, status, statusText, type, url};
+  }
+
   data = await getArrayBufferOrStringFromData(data, loader, options);
 
   // First check for synchronous text parser, wrap results in promises

package/src/lib/api/register-loaders.ts

@@ -11,7 +11,7 @@ const getGlobalLoaderRegistry = () => {
 };
 
 /** Register a list of global loaders */
-export function registerLoaders(loaders: Loader[]) {
+export function registerLoaders(loaders: Loader[] | Loader) {
   const loaderRegistry = getGlobalLoaderRegistry();
 
   loaders = Array.isArray(loaders) ? loaders : [loaders];

package/src/lib/api/select-loader.ts

@@ -1,9 +1,11 @@
 import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-utils';
 import {compareArrayBuffers, path} from '@loaders.gl/loader-utils';
 import {normalizeLoader} from '../loader-utils/normalize-loader';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {log} from '../utils/log';
+import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
 import {getRegisteredLoaders} from './register-loaders';
 import {isBlob} from '../../javascript-utils/is-type';
+import {stripQueryString} from '../utils/url-utils';
 
 const EXT_PATTERN = /\.([^.]+)$/;
 
@@ -103,30 +105,46 @@ export function selectLoaderSync(
 }
 
 /** Implements loaders selection logic */
+// eslint-disable-next-line complexity
 function selectLoaderInternal(
   data: Response | Blob | ArrayBuffer | string,
   loaders: Loader[],
   options?: LoaderOptions,
   context?: LoaderContext
 ) {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
-  const testUrl = url || context?.url;
+  const testUrl = stripQueryString(url) || context?.url;
 
   let loader: Loader | null = null;
+  let reason: string = '';
 
   // if options.mimeType is supplied, it takes precedence
   if (options?.mimeType) {
     loader = findLoaderByMIMEType(loaders, options?.mimeType);
+    reason = `match forced by supplied MIME type ${options?.mimeType}`;
   }
+
   // Look up loader by url
   loader = loader || findLoaderByUrl(loaders, testUrl);
+  reason = reason || (loader ? `matched url ${testUrl}` : '');
+
   // Look up loader by mime type
   loader = loader || findLoaderByMIMEType(loaders, type);
+  reason = reason || (loader ? `matched MIME type ${type}` : '');
+
   // Look for loader via initial bytes (Note: not always accessible (e.g. Response, stream, async iterator)
   loader = loader || findLoaderByInitialBytes(loaders, data);
+  reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
+
   // Look up loader by fallback mime type
   loader = loader || findLoaderByMIMEType(loaders, options?.fallbackMimeType);
+  reason = reason || (loader ? `matched fallback MIME type ${type}` : '');
+
+  if (reason) {
+    log.log(1, `selectLoader selected ${loader?.name}: ${reason}.`);
+  }
 
   return loader;
 }
@@ -145,7 +163,8 @@ function validHTTPResponse(data: any): boolean {
 
 /** Generate a helpful message to help explain why loader selection failed. */
 function getNoValidLoaderMessage(data): string {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
   let message = 'No valid loader found (';
   message += url ? `${path.filename(url)}, ` : 'no url provided, ';

package/src/lib/init.ts

@@ -0,0 +1,17 @@
+// loaders.gl, MIT license
+import {log} from './utils/log';
+
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const version = typeof __VERSION__ !== 'undefined' ? __VERSION__ : '';
+
+// @ts-ignore
+if (!globalThis.loaders) {
+  log.log(1, `loaders.gl ${version}`)();
+
+  globalThis.loaders = Object.assign(globalThis.loaders || {}, {
+    VERSION: version,
+    log
+  });
+}
+// @ts-ignore
+export default globalThis.loaders;

package/src/lib/loader-utils/get-fetch-function.ts

@@ -0,0 +1,38 @@
+// loaders.gl, MIT license
+
+import type {LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import {isObject} from '../../javascript-utils/is-type';
+import {fetchFile} from '../fetch/fetch-file';
+import {getGlobalLoaderOptions} from './option-utils';
+
+/**
+ * Gets the current fetch function from options and context
+ * @param options
+ * @param context
+ */
+export function getFetchFunction(
+  options?: LoaderOptions,
+  context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
+) {
+  const globalOptions = getGlobalLoaderOptions();
+
+  const fetchOptions = options || globalOptions;
+
+  // options.fetch can be a function
+  if (typeof fetchOptions.fetch === 'function') {
+    return fetchOptions.fetch;
+  }
+
+  // options.fetch can be an options object
+  if (isObject(fetchOptions.fetch)) {
+    return (url) => fetchFile(url, fetchOptions as RequestInit);
+  }
+
+  // else refer to context (from parent loader) if available
+  if (context?.fetch) {
+    return context?.fetch;
+  }
+
+  // else return the default fetch function
+  return fetchFile;
+}

package/src/lib/loader-utils/loader-context.ts

@@ -1,5 +1,7 @@
 import type {Loader, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
-import {getFetchFunction} from './option-utils';
+import {getFetchFunction} from './get-fetch-function';
+import {extractQueryString, stripQueryString} from '../utils/url-utils';
+import {path} from '@loaders.gl/loader-utils';
 
 /**
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -12,26 +14,35 @@ import {getFetchFunction} from './option-utils';
  */
 export function getLoaderContext(
   context: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>,
-  options?: LoaderOptions,
-  previousContext: LoaderContext | null = null
+  options: LoaderOptions,
+  parentContext: LoaderContext | null
 ): LoaderContext {
   // For recursive calls, we already have a context
   // TODO - add any additional loaders to context?
-  if (previousContext) {
-    return previousContext;
+  if (parentContext) {
+    return parentContext;
   }
 
-  const resolvedContext: LoaderContext = {
+  const newContext: LoaderContext = {
     fetch: getFetchFunction(options, context),
     ...context
   };
 
+  // Parse URLs so that subloaders can easily generate correct strings
+  if (newContext.url) {
+    const baseUrl = stripQueryString(newContext.url);
+    newContext.baseUrl = baseUrl;
+    newContext.queryString = extractQueryString(newContext.url);
+    newContext.filename = path.filename(baseUrl);
+    newContext.baseUrl = path.dirname(baseUrl);
+  }
+
   // Recursive loading does not use single loader
-  if (!Array.isArray(resolvedContext.loaders)) {
-    resolvedContext.loaders = null;
+  if (!Array.isArray(newContext.loaders)) {
+    newContext.loaders = null;
   }
 
-  return resolvedContext;
+  return newContext;
 }
 
 // eslint-disable-next-line complexity

package/src/lib/loader-utils/loggers.ts

@@ -1,5 +1,5 @@
 // probe.gl Log compatible loggers
-import {Log} from 'probe.gl';
+import {Log} from '@probe.gl/log';
 
 export const probeLog = new Log({id: 'loaders.gl'});
 

package/src/lib/loader-utils/option-defaults.ts

@@ -1,4 +1,5 @@
 import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import {isBrowser} from '@loaders.gl/loader-utils';
 import {ConsoleLog} from './loggers';
 
 export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
@@ -12,7 +13,8 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
   worker: true, // By default, use worker if provided by loader.
   maxConcurrency: 3, // How many worker instances should be created for each loader.
   maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
-  reuseWorkers: true, // By default reuse workers,
+  reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
+  _nodeWorkers: false, // By default do not support node workers
   _workerType: '', // 'test' to use locally generated workers
 
   limit: 0,