@loaders.gl/core 4.0.0-alpha.6 → 4.0.0-alpha.7

package/dist/null-worker-node.js

@@ -0,0 +1,234 @@
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+
+// ../worker-utils/src/lib/worker-utils/get-transfer-list.ts
+function getTransferList(object, recursive = true, transfers) {
+  const transfersSet = transfers || new Set();
+  if (!object) {
+  } else if (isTransferable(object)) {
+    transfersSet.add(object);
+  } else if (isTransferable(object.buffer)) {
+    transfersSet.add(object.buffer);
+  } else if (ArrayBuffer.isView(object)) {
+  } else if (recursive && typeof object === "object") {
+    for (const key in object) {
+      getTransferList(object[key], recursive, transfersSet);
+    }
+  }
+  return transfers === void 0 ? Array.from(transfersSet) : [];
+}
+function isTransferable(object) {
+  if (!object) {
+    return false;
+  }
+  if (object instanceof ArrayBuffer) {
+    return true;
+  }
+  if (typeof MessagePort !== "undefined" && object instanceof MessagePort) {
+    return true;
+  }
+  if (typeof ImageBitmap !== "undefined" && object instanceof ImageBitmap) {
+    return true;
+  }
+  if (typeof OffscreenCanvas !== "undefined" && object instanceof OffscreenCanvas) {
+    return true;
+  }
+  return false;
+}
+
+// ../worker-utils/src/lib/worker-farm/worker-body.ts
+function getParentPort() {
+  let parentPort;
+  try {
+    eval("globalThis.parentPort = require('worker_threads').parentPort");
+    parentPort = globalThis.parentPort;
+  } catch {
+  }
+  return parentPort;
+}
+var onMessageWrapperMap = new Map();
+var WorkerBody = class {
+  static inWorkerThread() {
+    return typeof self !== "undefined" || Boolean(getParentPort());
+  }
+  static set onmessage(onMessage) {
+    function handleMessage(message) {
+      const parentPort3 = getParentPort();
+      const { type, payload } = parentPort3 ? message : message.data;
+      onMessage(type, payload);
+    }
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      parentPort2.on("message", handleMessage);
+      parentPort2.on("exit", () => console.debug("Node worker closing"));
+    } else {
+      globalThis.onmessage = handleMessage;
+    }
+  }
+  static addEventListener(onMessage) {
+    let onMessageWrapper = onMessageWrapperMap.get(onMessage);
+    if (!onMessageWrapper) {
+      onMessageWrapper = (message) => {
+        if (!isKnownMessage(message)) {
+          return;
+        }
+        const parentPort3 = getParentPort();
+        const { type, payload } = parentPort3 ? message : message.data;
+        onMessage(type, payload);
+      };
+    }
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      console.error("not implemented");
+    } else {
+      globalThis.addEventListener("message", onMessageWrapper);
+    }
+  }
+  static removeEventListener(onMessage) {
+    const onMessageWrapper = onMessageWrapperMap.get(onMessage);
+    onMessageWrapperMap.delete(onMessage);
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      console.error("not implemented");
+    } else {
+      globalThis.removeEventListener("message", onMessageWrapper);
+    }
+  }
+  static postMessage(type, payload) {
+    const data = { source: "loaders.gl", type, payload };
+    const transferList = getTransferList(payload);
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      parentPort2.postMessage(data, transferList);
+    } else {
+      globalThis.postMessage(data, transferList);
+    }
+  }
+};
+function isKnownMessage(message) {
+  const { type, data } = message;
+  return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
+}
+
+// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+var requestId = 0;
+function createLoaderWorker(loader) {
+  if (!WorkerBody.inWorkerThread()) {
+    return;
+  }
+  WorkerBody.onmessage = async (type, payload) => {
+    switch (type) {
+      case "process":
+        try {
+          const { input, options = {}, context = {} } = payload;
+          const result = await parseData({
+            loader,
+            arrayBuffer: input,
+            options,
+            context: __spreadProps(__spreadValues({}, context), {
+              parse: parseOnMainThread
+            })
+          });
+          WorkerBody.postMessage("done", { result });
+        } catch (error) {
+          const message = error instanceof Error ? error.message : "";
+          WorkerBody.postMessage("error", { error: message });
+        }
+        break;
+      default:
+    }
+  };
+}
+function parseOnMainThread(arrayBuffer, options) {
+  return new Promise((resolve, reject) => {
+    const id = requestId++;
+    const onMessage = (type, payload2) => {
+      if (payload2.id !== id) {
+        return;
+      }
+      switch (type) {
+        case "done":
+          WorkerBody.removeEventListener(onMessage);
+          resolve(payload2.result);
+          break;
+        case "error":
+          WorkerBody.removeEventListener(onMessage);
+          reject(payload2.error);
+          break;
+        default:
+      }
+    };
+    WorkerBody.addEventListener(onMessage);
+    const payload = { id, input: arrayBuffer, options };
+    WorkerBody.postMessage("process", payload);
+  });
+}
+async function parseData({ loader, arrayBuffer, options, context }) {
+  let data;
+  let parser;
+  if (loader.parseSync || loader.parse) {
+    data = arrayBuffer;
+    parser = loader.parseSync || loader.parse;
+  } else if (loader.parseTextSync) {
+    const textDecoder = new TextDecoder();
+    data = textDecoder.decode(arrayBuffer);
+    parser = loader.parseTextSync;
+  } else {
+    throw new Error(`Could not load data with ${loader.name} loader`);
+  }
+  options = __spreadProps(__spreadValues({}, options), {
+    modules: loader && loader.options && loader.options.modules || {},
+    worker: false
+  });
+  return await parser(data, __spreadValues({}, options), context, loader);
+}
+
+// src/null-loader.ts
+var VERSION = true ? "4.0.0-alpha.7" : "latest";
+function parseSync(arrayBuffer, options, context) {
+  if (!options.null.echoParameters)
+    return null;
+  context = context && JSON.parse(JSON.stringify(context));
+  return { arrayBuffer, options, context };
+}
+var NullLoader = {
+  name: "Null loader",
+  id: "null",
+  module: "core",
+  version: VERSION,
+  mimeTypes: ["application/x.empty"],
+  extensions: ["null"],
+  parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
+  parseSync,
+  parseInBatches: async function* generator(asyncIterator, options, context) {
+    for await (const batch of asyncIterator) {
+      yield parseSync(batch, options, context);
+    }
+  },
+  tests: [() => false],
+  options: {
+    null: {
+      echoParameters: false
+    }
+  }
+};
+
+// src/workers/null-worker.ts
+createLoaderWorker(NullLoader);

package/dist/null-worker.js

@@ -184,7 +184,7 @@
   }
 
   // src/null-loader.ts
-  var VERSION = true ? "4.0.0-alpha.6" : "latest";
+  var VERSION = true ? "4.0.0-alpha.7" : "latest";
   function parseSync(arrayBuffer, options, context) {
     if (!options.null.echoParameters)
       return null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/core",
-  "version": "4.0.0-alpha.6",
+  "version": "4.0.0-alpha.7",
   "description": "The core API for working with loaders.gl loaders and writers",
   "license": "MIT",
   "publishConfig": {
@@ -36,15 +36,16 @@
     "README.md"
   ],
   "scripts": {
-    "pre-build": "npm run build-bundle && npm run build-worker",
+    "pre-build": "npm run build-bundle && npm run build-worker && npm run build-worker-node",
     "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js",
-    "build-worker": "esbuild src/workers/null-worker.ts --bundle --outfile=dist/null-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
+    "build-worker": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker.js --bundle --target=esnext --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.6",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.6",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.7",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.7",
     "@probe.gl/log": "^4.0.2"
   },
-  "gitHead": "acc1985050dfaa0f1f0c066f8da5bce7454a046c"
+  "gitHead": "afb59c4d8e5d8ebb9c28f111cb0c96c5527d0ffd"
 }

package/src/lib/api/load.ts

@@ -26,7 +26,7 @@ export async function load<LoaderT extends Loader>(
 ): Promise<LoaderReturnType<LoaderT>>;
 
 export async function load<
-  LoaderT extends Loader,
+  LoaderT extends Loader, // eslint-disable-line  @typescript-eslint/no-unused-vars
   LoaderOptionsT extends LoaderOptions = LoaderOptions
 >(
   url: string | DataType,

package/src/lib/api/parse-in-batches.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getLoaderContext} from '../loader-utils/loader-context';
 import {getAsyncIterableFromData} from '../loader-utils/get-data';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
 // Ensure `parse` is available in context if loader falls back to `parse`
@@ -32,6 +32,8 @@ export async function parseInBatches(
 ): Promise<AsyncIterable<any>> {
   assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
 
+  const loaderArray = Array.isArray(loaders) ? loaders : undefined;
+
   // Signature: parseInBatches(data, options, url) - Uses registered loaders
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
     context = undefined; // context not supported in short signature
@@ -43,11 +45,11 @@ export async function parseInBatches(
   options = options || {};
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader and normalizes it
   // Note - only uses URL and contentType for streams and iterator inputs
-  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);
+  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader | Loader[], options);
   // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
   if (!loader) {
     // @ts-ignore
@@ -55,14 +57,11 @@ export async function parseInBatches(
   }
 
   // Normalize options
-  // @ts-ignore
-  options = normalizeOptions(options, loader, loaders, url);
-  // @ts-ignore
+  options = normalizeOptions(options, loader, loaderArray, url);
   context = getLoaderContext(
-    // @ts-ignore
-    {url, parseInBatches, parse, loaders: loaders as Loader[]},
+    {url, parseInBatches, parse, loaders: loaderArray},
     options,
-    context
+    context || null
   );
 
   return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);

package/src/lib/api/parse-sync.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 
 /**
  * Parses `data` synchronously using a specified loader
@@ -52,12 +52,16 @@ export function parseSync(
   options = normalizeOptions(options, loader, candidateLoaders);
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   const parse = () => {
-    throw new Error('parseSync called parse');
+    throw new Error('parseSync called parse (which is async');
   };
-  context = getLoaderContext({url, parseSync, parse, loaders: loaders as Loader[]}, options);
+  context = getLoaderContext(
+    {url, parseSync, parse, loaders: loaders as Loader[]},
+    options,
+    context || null
+  );
 
   return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);
 }

package/src/lib/api/parse.ts

@@ -9,7 +9,7 @@ import {isResponse} from '../../javascript-utils/is-type';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
 // type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
@@ -64,7 +64,7 @@ export async function parse(
   options = options || ({} as LoaderOptions); // Could be invalid...
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader (and normalizes it)
   // Also use any loaders in the context, new loaders take priority
@@ -81,7 +81,7 @@ export async function parse(
   options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
 
   // Get a context (if already present, will be unchanged)
-  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context);
+  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context || null);
 
   return await parseWithLoader(loader, data, options, context);
 }

package/src/lib/api/select-loader.ts

@@ -2,9 +2,10 @@ import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-util
 import {compareArrayBuffers, path} from '@loaders.gl/loader-utils';
 import {normalizeLoader} from '../loader-utils/normalize-loader';
 import {log} from '../utils/log';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
 import {getRegisteredLoaders} from './register-loaders';
 import {isBlob} from '../../javascript-utils/is-type';
+import {stripQueryString} from '../utils/url-utils';
 
 const EXT_PATTERN = /\.([^.]+)$/;
 
@@ -111,9 +112,10 @@ function selectLoaderInternal(
   options?: LoaderOptions,
   context?: LoaderContext
 ) {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
-  const testUrl = url || context?.url;
+  const testUrl = stripQueryString(url) || context?.url;
 
   let loader: Loader | null = null;
   let reason: string = '';
@@ -161,7 +163,8 @@ function validHTTPResponse(data: any): boolean {
 
 /** Generate a helpful message to help explain why loader selection failed. */
 function getNoValidLoaderMessage(data): string {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
   let message = 'No valid loader found (';
   message += url ? `${path.filename(url)}, ` : 'no url provided, ';

package/src/lib/loader-utils/loader-context.ts

@@ -1,5 +1,7 @@
 import type {Loader, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
 import {getFetchFunction} from './get-fetch-function';
+import {extractQueryString, stripQueryString} from '../utils/url-utils';
+import {path} from '@loaders.gl/loader-utils';
 
 /**
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -12,26 +14,35 @@ import {getFetchFunction} from './get-fetch-function';
  */
 export function getLoaderContext(
   context: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>,
-  options?: LoaderOptions,
-  previousContext: LoaderContext | null = null
+  options: LoaderOptions,
+  parentContext: LoaderContext | null
 ): LoaderContext {
   // For recursive calls, we already have a context
   // TODO - add any additional loaders to context?
-  if (previousContext) {
-    return previousContext;
+  if (parentContext) {
+    return parentContext;
   }
 
-  const resolvedContext: LoaderContext = {
+  const newContext: LoaderContext = {
     fetch: getFetchFunction(options, context),
     ...context
   };
 
+  // Parse URLs so that subloaders can easily generate correct strings
+  if (newContext.url) {
+    const baseUrl = stripQueryString(newContext.url);
+    newContext.baseUrl = baseUrl;
+    newContext.queryString = extractQueryString(newContext.url);
+    newContext.filename = path.filename(baseUrl);
+    newContext.baseUrl = path.dirname(baseUrl);
+  }
+
   // Recursive loading does not use single loader
-  if (!Array.isArray(resolvedContext.loaders)) {
-    resolvedContext.loaders = null;
+  if (!Array.isArray(newContext.loaders)) {
+    newContext.loaders = null;
   }
 
-  return resolvedContext;
+  return newContext;
 }
 
 // eslint-disable-next-line complexity

package/src/lib/utils/resource-utils.ts

@@ -1,51 +1,73 @@
+// loaders.gl, MIT license
+
 import {isResponse, isBlob} from '../../javascript-utils/is-type';
 import {parseMIMEType, parseMIMETypeFromURL} from './mime-type-utils';
+import {stripQueryString} from './url-utils';
 
-const QUERY_STRING_PATTERN = /\?.*/;
+/**
+ * A loadable resource. Includes:
+ * `Response`, `Blob` (`File` is a subclass), string URLs and data URLs
+ */
+export type Resource = Response | Blob | string;
 
 /**
- * Returns an object with `url` and (MIME) `type` fields
- * If it cannot determine url or type, the corresponding value will be an empty string
+ * Returns the URL associated with this resource.
+ * The returned value may include a query string and need further processing.
+ * If it cannot determine url, the corresponding value will be an empty string
  *
- * @param resource Any type, but only Responses, string URLs and data URLs are processed
+ * @todo string parameters are assumed to be URLs
+ */
+export function getResourceUrl(resource: unknown): string {
+  // If resource is a `Response`, it contains the information directly as a field
+  if (isResponse(resource)) {
+    const response = resource as Response;
+    return response.url;
+  }
+
+  // If the resource is a Blob or a File (subclass of Blob)
+  if (isBlob(resource)) {
+    const blob = resource as Blob;
+    // File objects have a "name" property. Blob objects don't have any
+    // url (name) information
+    return blob.name || '';
+  }
+
+  if (typeof resource === 'string') {
+    return resource;
+  }
+
+  // Unknown
+  return '';
+}
+
+/**
+ * Returns the URL associated with this resource.
+ * The returned value may include a query string and need further processing.
+ * If it cannot determine url, the corresponding value will be an empty string
  *
  * @todo string parameters are assumed to be URLs
  */
-export function getResourceUrlAndType(resource: any): {url: string; type: string} {
+export function getResourceMIMEType(resource: unknown): string {
   // If resource is a response, it contains the information directly
   if (isResponse(resource)) {
-    const url = stripQueryString(resource.url || '');
-    const contentTypeHeader = resource.headers.get('content-type') || '';
-    return {
-      url,
-      type: parseMIMEType(contentTypeHeader) || parseMIMETypeFromURL(url)
-    };
+    const response = resource as Response;
+    const contentTypeHeader = response.headers.get('content-type') || '';
+    const noQueryUrl = stripQueryString(response.url);
+    return parseMIMEType(contentTypeHeader) || parseMIMETypeFromURL(noQueryUrl);
   }
 
   // If the resource is a Blob or a File (subclass of Blob)
   if (isBlob(resource)) {
-    return {
-      // File objects have a "name" property. Blob objects don't have any
-      // url (name) information
-      url: stripQueryString(resource.name || ''),
-      type: resource.type || ''
-    };
+    const blob = resource as Blob;
+    return blob.type || '';
   }
 
   if (typeof resource === 'string') {
-    return {
-      // TODO this could mess up data URL but it doesn't matter as it is just used for inference
-      url: stripQueryString(resource),
-      // If a data url
-      type: parseMIMETypeFromURL(resource)
-    };
+    return parseMIMETypeFromURL(resource);
   }
 
   // Unknown
-  return {
-    url: '',
-    type: ''
-  };
+  return '';
 }
 
 /**
@@ -55,12 +77,14 @@ export function getResourceUrlAndType(resource: any): {url: string; type: string
 
   * @note string parameters are NOT assumed to be URLs
   */
-export function getResourceContentLength(resource: any): number {
+export function getResourceContentLength(resource: unknown): number {
   if (isResponse(resource)) {
-    return resource.headers['content-length'] || -1;
+    const response = resource as Response;
+    return response.headers['content-length'] || -1;
   }
   if (isBlob(resource)) {
-    return resource.size;
+    const blob = resource as Blob;
+    return blob.size;
   }
   if (typeof resource === 'string') {
     // TODO - handle data URL?
@@ -74,7 +98,3 @@ export function getResourceContentLength(resource: any): number {
   }
   return -1;
 }
-
-function stripQueryString(url) {
-  return url.replace(QUERY_STRING_PATTERN, '');
-}

package/src/lib/utils/response-utils.ts

@@ -1,5 +1,5 @@
 import {isResponse} from '../../javascript-utils/is-type';
-import {getResourceContentLength, getResourceUrlAndType} from './resource-utils';
+import {getResourceContentLength, getResourceUrl, getResourceMIMEType} from './resource-utils';
 
 /**
  * Returns a Response object
@@ -22,7 +22,8 @@ export async function makeResponse(resource: any): Promise<Response> {
 
   // `new Response(File)` does not preserve content-type and URL
   // so we add them here
-  const {url, type} = getResourceUrlAndType(resource);
+  const url = getResourceUrl(resource);
+  const type = getResourceMIMEType(resource);
   if (type) {
     headers['content-type'] = type;
   }

package/src/lib/utils/url-utils.ts

@@ -0,0 +1,12 @@
+// loaders.gl, MIT license
+
+const QUERY_STRING_PATTERN = /\?.*/;
+
+export function extractQueryString(url): string {
+  const matches = url.match(QUERY_STRING_PATTERN);
+  return matches && matches[0];
+}
+
+export function stripQueryString(url): string {
+  return url.replace(QUERY_STRING_PATTERN, '');
+}