@loaders.gl/core 4.0.0-alpha.5 → 4.0.0-alpha.7

package/dist/null-worker-node.js

@@ -0,0 +1,234 @@
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+
+// ../worker-utils/src/lib/worker-utils/get-transfer-list.ts
+function getTransferList(object, recursive = true, transfers) {
+  const transfersSet = transfers || new Set();
+  if (!object) {
+  } else if (isTransferable(object)) {
+    transfersSet.add(object);
+  } else if (isTransferable(object.buffer)) {
+    transfersSet.add(object.buffer);
+  } else if (ArrayBuffer.isView(object)) {
+  } else if (recursive && typeof object === "object") {
+    for (const key in object) {
+      getTransferList(object[key], recursive, transfersSet);
+    }
+  }
+  return transfers === void 0 ? Array.from(transfersSet) : [];
+}
+function isTransferable(object) {
+  if (!object) {
+    return false;
+  }
+  if (object instanceof ArrayBuffer) {
+    return true;
+  }
+  if (typeof MessagePort !== "undefined" && object instanceof MessagePort) {
+    return true;
+  }
+  if (typeof ImageBitmap !== "undefined" && object instanceof ImageBitmap) {
+    return true;
+  }
+  if (typeof OffscreenCanvas !== "undefined" && object instanceof OffscreenCanvas) {
+    return true;
+  }
+  return false;
+}
+
+// ../worker-utils/src/lib/worker-farm/worker-body.ts
+function getParentPort() {
+  let parentPort;
+  try {
+    eval("globalThis.parentPort = require('worker_threads').parentPort");
+    parentPort = globalThis.parentPort;
+  } catch {
+  }
+  return parentPort;
+}
+var onMessageWrapperMap = new Map();
+var WorkerBody = class {
+  static inWorkerThread() {
+    return typeof self !== "undefined" || Boolean(getParentPort());
+  }
+  static set onmessage(onMessage) {
+    function handleMessage(message) {
+      const parentPort3 = getParentPort();
+      const { type, payload } = parentPort3 ? message : message.data;
+      onMessage(type, payload);
+    }
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      parentPort2.on("message", handleMessage);
+      parentPort2.on("exit", () => console.debug("Node worker closing"));
+    } else {
+      globalThis.onmessage = handleMessage;
+    }
+  }
+  static addEventListener(onMessage) {
+    let onMessageWrapper = onMessageWrapperMap.get(onMessage);
+    if (!onMessageWrapper) {
+      onMessageWrapper = (message) => {
+        if (!isKnownMessage(message)) {
+          return;
+        }
+        const parentPort3 = getParentPort();
+        const { type, payload } = parentPort3 ? message : message.data;
+        onMessage(type, payload);
+      };
+    }
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      console.error("not implemented");
+    } else {
+      globalThis.addEventListener("message", onMessageWrapper);
+    }
+  }
+  static removeEventListener(onMessage) {
+    const onMessageWrapper = onMessageWrapperMap.get(onMessage);
+    onMessageWrapperMap.delete(onMessage);
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      console.error("not implemented");
+    } else {
+      globalThis.removeEventListener("message", onMessageWrapper);
+    }
+  }
+  static postMessage(type, payload) {
+    const data = { source: "loaders.gl", type, payload };
+    const transferList = getTransferList(payload);
+    const parentPort2 = getParentPort();
+    if (parentPort2) {
+      parentPort2.postMessage(data, transferList);
+    } else {
+      globalThis.postMessage(data, transferList);
+    }
+  }
+};
+function isKnownMessage(message) {
+  const { type, data } = message;
+  return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
+}
+
+// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+var requestId = 0;
+function createLoaderWorker(loader) {
+  if (!WorkerBody.inWorkerThread()) {
+    return;
+  }
+  WorkerBody.onmessage = async (type, payload) => {
+    switch (type) {
+      case "process":
+        try {
+          const { input, options = {}, context = {} } = payload;
+          const result = await parseData({
+            loader,
+            arrayBuffer: input,
+            options,
+            context: __spreadProps(__spreadValues({}, context), {
+              parse: parseOnMainThread
+            })
+          });
+          WorkerBody.postMessage("done", { result });
+        } catch (error) {
+          const message = error instanceof Error ? error.message : "";
+          WorkerBody.postMessage("error", { error: message });
+        }
+        break;
+      default:
+    }
+  };
+}
+function parseOnMainThread(arrayBuffer, options) {
+  return new Promise((resolve, reject) => {
+    const id = requestId++;
+    const onMessage = (type, payload2) => {
+      if (payload2.id !== id) {
+        return;
+      }
+      switch (type) {
+        case "done":
+          WorkerBody.removeEventListener(onMessage);
+          resolve(payload2.result);
+          break;
+        case "error":
+          WorkerBody.removeEventListener(onMessage);
+          reject(payload2.error);
+          break;
+        default:
+      }
+    };
+    WorkerBody.addEventListener(onMessage);
+    const payload = { id, input: arrayBuffer, options };
+    WorkerBody.postMessage("process", payload);
+  });
+}
+async function parseData({ loader, arrayBuffer, options, context }) {
+  let data;
+  let parser;
+  if (loader.parseSync || loader.parse) {
+    data = arrayBuffer;
+    parser = loader.parseSync || loader.parse;
+  } else if (loader.parseTextSync) {
+    const textDecoder = new TextDecoder();
+    data = textDecoder.decode(arrayBuffer);
+    parser = loader.parseTextSync;
+  } else {
+    throw new Error(`Could not load data with ${loader.name} loader`);
+  }
+  options = __spreadProps(__spreadValues({}, options), {
+    modules: loader && loader.options && loader.options.modules || {},
+    worker: false
+  });
+  return await parser(data, __spreadValues({}, options), context, loader);
+}
+
+// src/null-loader.ts
+var VERSION = true ? "4.0.0-alpha.7" : "latest";
+function parseSync(arrayBuffer, options, context) {
+  if (!options.null.echoParameters)
+    return null;
+  context = context && JSON.parse(JSON.stringify(context));
+  return { arrayBuffer, options, context };
+}
+var NullLoader = {
+  name: "Null loader",
+  id: "null",
+  module: "core",
+  version: VERSION,
+  mimeTypes: ["application/x.empty"],
+  extensions: ["null"],
+  parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
+  parseSync,
+  parseInBatches: async function* generator(asyncIterator, options, context) {
+    for await (const batch of asyncIterator) {
+      yield parseSync(batch, options, context);
+    }
+  },
+  tests: [() => false],
+  options: {
+    null: {
+      echoParameters: false
+    }
+  }
+};
+
+// src/workers/null-worker.ts
+createLoaderWorker(NullLoader);

package/dist/null-worker.js

@@ -35,16 +35,33 @@
   }
 
   // ../worker-utils/src/lib/worker-farm/worker-body.ts
+  function getParentPort() {
+    let parentPort;
+    try {
+      eval("globalThis.parentPort = require('worker_threads').parentPort");
+      parentPort = globalThis.parentPort;
+    } catch {
+    }
+    return parentPort;
+  }
   var onMessageWrapperMap = new Map();
   var WorkerBody = class {
+    static inWorkerThread() {
+      return typeof self !== "undefined" || Boolean(getParentPort());
+    }
     static set onmessage(onMessage) {
-      self.onmessage = (message) => {
-        if (!isKnownMessage(message)) {
-          return;
-        }
-        const { type, payload } = message.data;
+      function handleMessage(message) {
+        const parentPort3 = getParentPort();
+        const { type, payload } = parentPort3 ? message : message.data;
         onMessage(type, payload);
-      };
+      }
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        parentPort2.on("message", handleMessage);
+        parentPort2.on("exit", () => console.debug("Node worker closing"));
+      } else {
+        globalThis.onmessage = handleMessage;
+      }
     }
     static addEventListener(onMessage) {
       let onMessageWrapper = onMessageWrapperMap.get(onMessage);
@@ -53,22 +70,36 @@
           if (!isKnownMessage(message)) {
             return;
           }
-          const { type, payload } = message.data;
+          const parentPort3 = getParentPort();
+          const { type, payload } = parentPort3 ? message : message.data;
           onMessage(type, payload);
         };
       }
-      self.addEventListener("message", onMessageWrapper);
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        console.error("not implemented");
+      } else {
+        globalThis.addEventListener("message", onMessageWrapper);
+      }
     }
     static removeEventListener(onMessage) {
       const onMessageWrapper = onMessageWrapperMap.get(onMessage);
       onMessageWrapperMap.delete(onMessage);
-      self.removeEventListener("message", onMessageWrapper);
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        console.error("not implemented");
+      } else {
+        globalThis.removeEventListener("message", onMessageWrapper);
+      }
     }
     static postMessage(type, payload) {
-      if (self) {
-        const data = { source: "loaders.gl", type, payload };
-        const transferList = getTransferList(payload);
-        self.postMessage(data, transferList);
+      const data = { source: "loaders.gl", type, payload };
+      const transferList = getTransferList(payload);
+      const parentPort2 = getParentPort();
+      if (parentPort2) {
+        parentPort2.postMessage(data, transferList);
+      } else {
+        globalThis.postMessage(data, transferList);
       }
     }
   };
@@ -80,19 +111,20 @@
   // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
   var requestId = 0;
   function createLoaderWorker(loader) {
-    if (typeof self === "undefined") {
+    if (!WorkerBody.inWorkerThread()) {
       return;
     }
     WorkerBody.onmessage = async (type, payload) => {
       switch (type) {
         case "process":
           try {
-            const { input, options = {} } = payload;
+            const { input, options = {}, context = {} } = payload;
             const result = await parseData({
               loader,
               arrayBuffer: input,
               options,
               context: {
+                ...context,
                 parse: parseOnMainThread
               }
             });
@@ -152,7 +184,13 @@
   }
 
   // src/null-loader.ts
-  var VERSION = true ? "4.0.0-alpha.5" : "latest";
+  var VERSION = true ? "4.0.0-alpha.7" : "latest";
+  function parseSync(arrayBuffer, options, context) {
+    if (!options.null.echoParameters)
+      return null;
+    context = context && JSON.parse(JSON.stringify(context));
+    return { arrayBuffer, options, context };
+  }
   var NullLoader = {
     name: "Null loader",
     id: "null",
@@ -160,16 +198,18 @@
     version: VERSION,
     mimeTypes: ["application/x.empty"],
     extensions: ["null"],
-    parse: async (arrayBuffer) => arrayBuffer,
-    parseSync: (arrayBuffer) => arrayBuffer,
-    parseInBatches: async function* generator(asyncIterator) {
+    parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
+    parseSync,
+    parseInBatches: async function* generator(asyncIterator, options, context) {
       for await (const batch of asyncIterator) {
-        yield batch;
+        yield parseSync(batch, options, context);
       }
     },
     tests: [() => false],
     options: {
-      null: {}
+      null: {
+        echoParameters: false
+      }
     }
   };
 

package/dist/workers/null-worker.js

@@ -1,4 +1,5 @@
-import { createLoaderWorker } from '@loaders.gl/loader-utils';
-import { NullLoader } from '../null-loader';
-createLoaderWorker(NullLoader);
-//# sourceMappingURL=null-worker.js.map
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const null_loader_1 = require("../null-loader");
+(0, loader_utils_1.createLoaderWorker)(null_loader_1.NullLoader);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@loaders.gl/core",
-  "version": "4.0.0-alpha.5",
-  "description": "Framework-independent loaders for 3D graphics formats",
+  "version": "4.0.0-alpha.7",
+  "description": "The core API for working with loaders.gl loaders and writers",
   "license": "MIT",
   "publishConfig": {
     "access": "public"
@@ -18,8 +18,8 @@
     "point cloud"
   ],
   "types": "dist/index.d.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "browser": {
     "fs": false,
@@ -36,15 +36,16 @@
     "README.md"
   ],
   "scripts": {
-    "pre-build": "npm run build-bundle && npm run build-worker",
+    "pre-build": "npm run build-bundle && npm run build-worker && npm run build-worker-node",
     "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js",
-    "build-worker": "esbuild src/workers/null-worker.ts --bundle --outfile=dist/null-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
+    "build-worker": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker.js --bundle --target=esnext --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.5",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.5",
-    "probe.gl": "^3.4.0"
+    "@loaders.gl/loader-utils": "4.0.0-alpha.7",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.7",
+    "@probe.gl/log": "^4.0.2"
   },
-  "gitHead": "7a71a54bdf1ddf985cc3af3db90b82e7fa97d025"
+  "gitHead": "afb59c4d8e5d8ebb9c28f111cb0c96c5527d0ffd"
 }

package/src/index.ts

@@ -1,3 +1,20 @@
+// loaders.gl, MIT license
+
+// TYPES
+export type {
+  Loader,
+  LoaderWithParser,
+  LoaderContext,
+  LoaderOptions,
+  Writer,
+  WriterOptions,
+  DataType,
+  SyncDataType,
+  BatchableDataType,
+  IFileSystem,
+  IRandomAccessReadFileSystem
+} from '@loaders.gl/loader-utils';
+
 // FILE READING AND WRITING
 export {fetchFile} from './lib/fetch/fetch-file';
 
@@ -6,7 +23,7 @@ export {readFileSync} from './lib/fetch/read-file';
 export {writeFile, writeFileSync} from './lib/fetch/write-file';
 
 // CONFIGURATION
-export {setLoaderOptions} from './lib/api/set-loader-options';
+export {setLoaderOptions, getLoaderOptions} from './lib/api/loader-options';
 export {registerLoaders} from './lib/api/register-loaders';
 export {selectLoader, selectLoaderSync} from './lib/api/select-loader';
 
@@ -19,6 +36,7 @@ export {load} from './lib/api/load';
 export {loadInBatches} from './lib/api/load-in-batches';
 
 // ENCODING (ENCODING AND WRITING)
+export {encodeTable, encodeTableAsText, encodeTableInBatches} from './lib/api/encode-table';
 export {encode, encodeSync, encodeInBatches, encodeText, encodeURLtoURL} from './lib/api/encode';
 export {save, saveSync} from './lib/api/save';
 

package/src/iterators/make-stream/make-dom-stream.ts

@@ -24,7 +24,6 @@ export function makeStream<ArrayBuffer>(
       // Create a byte stream (enables `Response(stream).arrayBuffer()`)
       // Only supported on Chrome
       // See: https://developer.mozilla.org/en-US/docs/Web/API/ReadableByteStreamController
-      // @ts-expect-error Only Chrome supports byte streams
       type: 'bytes',
 
       async pull(controller) {
@@ -34,6 +33,7 @@ export function makeStream<ArrayBuffer>(
             controller.close();
           } else {
             // TODO - ignores controller.desiredSize
+            // @ts-expect-error Unclear why value is not correctly typed
             controller.enqueue(new Uint8Array(value));
           }
         } catch (error) {

package/src/iterators/make-stream/make-node-stream.ts

@@ -1,4 +1,10 @@
-import {Readable, ReadableOptions} from 'stream';
+import type {ReadableOptions} from 'stream';
+import * as Stream from 'stream';
+
+class _Readable {}
+
+type ReadableType = Stream.Readable | _Readable;
+const Readable = Stream.Readable || _Readable;
 
 export type MakeStreamOptions = ReadableOptions;
 
@@ -6,7 +12,7 @@ export type MakeStreamOptions = ReadableOptions;
 export function makeStream<ArrayBuffer>(
   source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
   options?: ReadableOptions
-): Readable {
+): ReadableType {
   const iterator = source[Symbol.asyncIterator]
     ? source[Symbol.asyncIterator]()
     : source[Symbol.iterator]();

package/src/lib/api/encode-table.ts

@@ -0,0 +1,71 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc
+
+/* global TextEncoder, TextDecoder */
+import {concatenateArrayBuffers, Writer, WriterOptionsType} from '@loaders.gl/loader-utils';
+import {Table} from '@loaders.gl/schema';
+
+export async function encodeTable<WriterT extends Writer = Writer>(
+  data: Table,
+  writer: WriterT,
+  options?: WriterOptionsType<WriterT>
+): Promise<ArrayBuffer> {
+  if (writer.encode) {
+    return await writer.encode(data, options);
+  }
+
+  if (writer.encodeText) {
+    const text = await writer.encodeText(data, options);
+    return new TextEncoder().encode(text);
+  }
+
+  if (writer.encodeInBatches) {
+    // Create an iterator representing the data
+    // TODO - Assumes this is a table
+    const batches = encodeTableInBatches(data, writer, options);
+
+    // Concatenate the output
+    const chunks: ArrayBuffer[] = [];
+    for await (const batch of batches) {
+      chunks.push(batch);
+    }
+    return concatenateArrayBuffers(...chunks);
+  }
+
+  throw new Error('Writer could not encode data');
+}
+
+export async function encodeTableAsText<WriterT extends Writer = Writer>(
+  data: Table,
+  writer: WriterT,
+  options?: WriterOptionsType<WriterT>
+): Promise<string> {
+  if (writer.text && writer.encodeText) {
+    return await writer.encodeText(data, options);
+  }
+
+  if (writer.text && (writer.encode || writer.encodeInBatches)) {
+    const arrayBuffer = await encodeTable(data, writer, options);
+    return new TextDecoder().decode(arrayBuffer);
+  }
+  throw new Error('Writer could not encode data as text');
+}
+
+export function encodeTableInBatches<WriterT extends Writer = Writer>(
+  data: Table,
+  writer: WriterT,
+  options?: WriterOptionsType<WriterT>
+): AsyncIterable<ArrayBuffer> {
+  if (writer.encodeInBatches) {
+    const dataIterator = getIterator(data);
+    // @ts-expect-error
+    return writer.encodeInBatches(dataIterator, options);
+  }
+  // TODO -fall back to atomic encode?
+  throw new Error('Writer could not encode data in batches');
+}
+
+function getIterator(data) {
+  const dataIterator = [{table: data, start: 0, end: data.length}];
+  return dataIterator;
+}

package/src/lib/api/encode.ts

@@ -1,8 +1,10 @@
-import type {Writer, LoaderOptions} from '@loaders.gl/loader-utils';
+import {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils';
+import {processOnWorker} from '@loaders.gl/worker-utils';
 import {concatenateArrayBuffers, resolvePath} from '@loaders.gl/loader-utils';
 import {isBrowser} from '@loaders.gl/loader-utils';
 import {writeFile} from '../fetch/write-file';
 import {fetchFile} from '../fetch/fetch-file';
+import {getLoaderOptions} from './loader-options';
 
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
@@ -10,8 +12,15 @@ import {fetchFile} from '../fetch/fetch-file';
 export async function encode(
   data: any,
   writer: Writer,
-  options?: LoaderOptions
+  options?: WriterOptions
 ): Promise<ArrayBuffer> {
+  const globalOptions = getLoaderOptions() as WriterOptions;
+  // const globalOptions: WriterOptions = {}; // getWriterOptions();
+  options = {...globalOptions, ...options};
+  if (canEncodeWithWorker(writer, options)) {
+    return await processOnWorker(writer, data, options);
+  }
+
   // TODO Merge default writer options with options argument like it is done in load module.
   if (writer.encode) {
     return await writer.encode(data, options);
@@ -63,7 +72,7 @@ export async function encode(
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
  */
-export function encodeSync(data: any, writer: Writer, options?: LoaderOptions): ArrayBuffer {
+export function encodeSync(data: any, writer: Writer, options?: WriterOptions): ArrayBuffer {
   if (writer.encodeSync) {
     return writer.encodeSync(data, options);
   }
@@ -79,7 +88,7 @@ export function encodeSync(data: any, writer: Writer, options?: LoaderOptions):
 export async function encodeText(
   data: any,
   writer: Writer,
-  options?: LoaderOptions
+  options?: WriterOptions
 ): Promise<string> {
   if (writer.text && writer.encodeText) {
     return await writer.encodeText(data, options);
@@ -99,10 +108,11 @@ export async function encodeText(
 export function encodeInBatches(
   data: any,
   writer: Writer,
-  options?: LoaderOptions
+  options?: WriterOptions
 ): AsyncIterable<ArrayBuffer> {
   if (writer.encodeInBatches) {
     const dataIterator = getIterator(data);
+    // @ts-expect-error
     return writer.encodeInBatches(dataIterator, options);
   }
   // TODO -fall back to atomic encode?

package/src/lib/api/load-in-batches.ts

@@ -1,6 +1,6 @@
 import type {LoaderWithParser, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
-import {getFetchFunction} from '../loader-utils/option-utils';
+import {getFetchFunction} from '../loader-utils/get-fetch-function';
 
 import {parseInBatches} from './parse-in-batches';