@loaders.gl/loader-utils 4.0.0-alpha.4 → 4.0.0-alpha.6

package/dist/lib/worker-loader-utils/parse-with-worker.js

@@ -1,64 +1,81 @@
-import { WorkerFarm, getWorkerURL } from '@loaders.gl/worker-utils';
-export function canParseWithWorker(loader, options) {
-  if (!WorkerFarm.isSupported()) {
-    return false;
-  }
-
-  return loader.worker && (options === null || options === void 0 ? void 0 : options.worker);
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseWithWorker = exports.canParseWithWorker = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const worker_utils_2 = require("@loaders.gl/worker-utils");
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+function canParseWithWorker(loader, options) {
+    if (!worker_utils_2.WorkerFarm.isSupported()) {
+        return false;
+    }
+    // Node workers are still experimental
+    if (!worker_utils_1.isBrowser && !options?._nodeWorkers) {
+        return false;
+    }
+    return loader.worker && options?.worker;
 }
-export async function parseWithWorker(loader, data, options, context, parseOnMainThread) {
-  const name = loader.id;
-  const url = getWorkerURL(loader, options);
-  const workerFarm = WorkerFarm.getWorkerFarm(options);
-  const workerPool = workerFarm.getWorkerPool({
-    name,
-    url
-  });
-  options = JSON.parse(JSON.stringify(options));
-  const job = await workerPool.startJob('process-on-worker', onMessage.bind(null, parseOnMainThread));
-  job.postMessage('process', {
-    input: data,
-    options
-  });
-  const result = await job.result;
-  return await result.result;
+exports.canParseWithWorker = canParseWithWorker;
+/**
+ * this function expects that the worker function sends certain messages,
+ * this can be automated if the worker is wrapper by a call to createLoaderWorker in @loaders.gl/loader-utils.
+ */
+async function parseWithWorker(loader, data, options, context, parseOnMainThread) {
+    const name = loader.id; // TODO
+    const url = (0, worker_utils_2.getWorkerURL)(loader, options);
+    const workerFarm = worker_utils_2.WorkerFarm.getWorkerFarm(options);
+    const workerPool = workerFarm.getWorkerPool({ name, url });
+    // options.log object contains functions which cannot be transferred
+    // context.fetch & context.parse functions cannot be transferred
+    // TODO - decide how to handle logging on workers
+    options = JSON.parse(JSON.stringify(options));
+    context = JSON.parse(JSON.stringify(context || {}));
+    const job = await workerPool.startJob('process-on-worker', 
+    // @ts-expect-error
+    onMessage.bind(null, parseOnMainThread) // eslint-disable-line @typescript-eslint/no-misused-promises
+    );
+    job.postMessage('process', {
+        // @ts-ignore
+        input: data,
+        options,
+        context
+    });
+    const result = await job.result;
+    // TODO - what is going on here?
+    return await result.result;
 }
-
+exports.parseWithWorker = parseWithWorker;
+/**
+ * Handle worker's responses to the main thread
+ * @param job
+ * @param type
+ * @param payload
+ */
 async function onMessage(parseOnMainThread, job, type, payload) {
-  switch (type) {
-    case 'done':
-      job.done(payload);
-      break;
-
-    case 'error':
-      job.error(new Error(payload.error));
-      break;
-
-    case 'process':
-      const {
-        id,
-        input,
-        options
-      } = payload;
-
-      try {
-        const result = await parseOnMainThread(input, options);
-        job.postMessage('done', {
-          id,
-          result
-        });
-      } catch (error) {
-        const message = error instanceof Error ? error.message : 'unknown error';
-        job.postMessage('error', {
-          id,
-          error: message
-        });
-      }
-
-      break;
-
-    default:
-      console.warn("parse-with-worker unknown message ".concat(type));
-  }
+    switch (type) {
+        case 'done':
+            job.done(payload);
+            break;
+        case 'error':
+            job.error(new Error(payload.error));
+            break;
+        case 'process':
+            // Worker is asking for main thread to parseO
+            const { id, input, options } = payload;
+            try {
+                const result = await parseOnMainThread(input, options);
+                job.postMessage('done', { id, result });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : 'unknown error';
+                job.postMessage('error', { id, error: message });
+            }
+            break;
+        default:
+            // eslint-disable-next-line
+            console.warn(`parse-with-worker unknown message ${type}`);
+    }
 }
-//# sourceMappingURL=parse-with-worker.js.map

package/dist/types.d.ts

@@ -0,0 +1,220 @@
+/// <reference types="node" />
+export type TypedIntArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Int32Array | Uint32Array;
+export type TypedFloatArray = Uint16Array | Float32Array | Float64Array;
+export type TypedArray = TypedIntArray | TypedFloatArray;
+export type NumericArray = Array<number> | TypedIntArray | TypedFloatArray;
+type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
+/**
+ * Core Loader Options
+ */
+export type LoaderOptions = {
+    /** fetch options or a custom fetch function */
+    fetch?: typeof fetch | FetchLike | RequestInit | null;
+    /** Do not throw on errors */
+    nothrow?: boolean;
+    /** loader selection, search first for supplied mimeType */
+    mimeType?: string;
+    /** loader selection, provide fallback mimeType is server does not provide */
+    fallbackMimeType?: string;
+    /** loader selection, avoid searching registered loaders */
+    ignoreRegisteredLoaders?: boolean;
+    /** Experimental: Supply a logger to the parser */
+    log?: any;
+    /** Size of each batch. `auto` matches batches to size of incoming chunks */
+    batchSize?: number | 'auto';
+    /** Minimal amount of time between batches */
+    batchDebounceMs?: number;
+    /** Stop loading after a given number of rows (compare SQL limit clause) */
+    limit?: 0;
+    /** Experimental: Stop loading after reaching */
+    _limitMB?: 0;
+    /** Generate metadata batches */
+    metadata?: boolean;
+    /** Transforms to run on incoming batches */
+    transforms?: TransformBatches[];
+    /** CDN load workers from */
+    CDN?: string | null;
+    /** Set to `false` to disable workers */
+    worker?: boolean;
+    /** Number of concurrent workers (per loader) on desktop browser */
+    maxConcurrency?: number;
+    /** Number of concurrent workers (per loader) on mobile browsers */
+    maxMobileConcurrency?: number;
+    /** Set to `false` to prevent reuse workers */
+    reuseWorkers?: boolean;
+    /** Whether to use workers under Node.js (experimental) */
+    _nodeWorkers?: boolean;
+    /** set to 'test' to run local worker */
+    _workerType?: string;
+    /** @deprecated `options.batchType` removed, Use `options.<loader>.type` instead */
+    batchType?: 'row' | 'columnar' | 'arrow';
+    /** @deprecated `options.throw removed`, Use `options.nothrow` instead */
+    throws?: boolean;
+    /** @deprecated `options.dataType` no longer used */
+    dataType?: never;
+    /** @deprecated `options.uri` no longer used */
+    uri?: never;
+    /** @deprecated `options.method` removed. Use `options.fetch.method` */
+    method?: never;
+    /** @deprecated `options.headers` removed. Use `options.fetch.headers` */
+    headers?: never;
+    /** @deprecated `options.body` removed. Use `options.fetch.body` */
+    body?: never;
+    /** @deprecated `options.mode` removed. Use `options.fetch.mode` */
+    mode?: never;
+    /** @deprecated `options.credentials` removed. Use `options.fetch.credentials` */
+    credentials?: never;
+    /** @deprecated `options.cache` removed. Use `options.fetch.cache` */
+    cache?: never;
+    /** @deprecated `options.redirect` removed. Use `options.fetch.redirect` */
+    redirect?: never;
+    /** @deprecated `options.referrer` removed. Use `options.fetch.referrer` */
+    referrer?: never;
+    /** @deprecated `options.referrerPolicy` removed. Use `options.fetch.referrerPolicy` */
+    referrerPolicy?: never;
+    /** @deprecated `options.integrity` removed. Use `options.fetch.integrity` */
+    integrity?: never;
+    /** @deprecated `options.keepalive` removed. Use `options.fetch.keepalive` */
+    keepalive?: never;
+    /** @deprecated `options.signal` removed. Use `options.fetch.signal` */
+    signal?: never;
+    [loaderId: string]: unknown;
+};
+type PreloadOptions = {
+    [key: string]: unknown;
+};
+/**
+ * A worker loader definition that can be used with `@loaders.gl/core` functions
+ */
+export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = {
+    dataType?: DataT;
+    batchType?: BatchT;
+    options: LoaderOptionsT;
+    deprecatedOptions?: Record<string, string | Record<string, string>>;
+    name: string;
+    id: string;
+    module: string;
+    version: string;
+    worker?: string | boolean;
+    category?: string;
+    extensions: string[];
+    mimeTypes: string[];
+    binary?: boolean;
+    text?: boolean;
+    tests?: (((ArrayBuffer: ArrayBuffer) => boolean) | ArrayBuffer | string)[];
+    supported?: boolean;
+    testText?: (string: string) => boolean;
+};
+/**
+ * A "bundled" loader definition that can be used with `@loaders.gl/core` functions
+ * If a worker loader is supported it will also be supported.
+ */
+export type LoaderWithParser<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = Loader<DataT, BatchT, LoaderOptionsT> & {
+    testText?: (string: string) => boolean;
+    parse: (arrayBuffer: ArrayBuffer, options?: LoaderOptionsT, context?: LoaderContext) => Promise<DataT>;
+    preload?: Preload;
+    parseSync?: (arrayBuffer: ArrayBuffer, options?: LoaderOptionsT, context?: LoaderContext) => DataT;
+    parseText?: (text: string, options?: LoaderOptionsT) => Promise<DataT>;
+    parseTextSync?: (text: string, options?: LoaderOptionsT) => DataT;
+    parseInBatches?: (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptionsT, context?: LoaderContext) => AsyncIterable<BatchT>;
+    parseFileInBatches?: (file: Blob, options?: LoaderOptionsT, context?: LoaderContext) => AsyncIterable<BatchT>;
+};
+export type LoaderContext = {
+    loaders?: Loader[] | null;
+    url?: string;
+    fetch: typeof fetch | FetchLike;
+    response?: Response;
+    parse: (arrayBuffer: ArrayBuffer, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext) => Promise<any>;
+    parseSync?: (arrayBuffer: ArrayBuffer, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext) => any;
+    parseInBatches?: (iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext) => AsyncIterable<any> | Promise<AsyncIterable<any>>;
+};
+type Preload = (url: string, options?: PreloadOptions) => any;
+/** Typescript helper to extract options type from a generic loader type */
+export type LoaderOptionsType<T = Loader> = T extends Loader<any, any, infer Options> ? Options : never;
+/** Typescript helper to extract data type from a generic loader type */
+export type LoaderReturnType<T = Loader> = T extends Loader<infer Return, any, any> ? Return : never;
+/** Typescript helper to extract batch type from a generic loader type */
+export type LoaderBatchType<T = Loader> = T extends Loader<any, infer Batch, any> ? Batch : never;
+/** Options for writers */
+export type WriterOptions = {
+    /** worker source. If is set will be used instead of loading worker from the Internet */
+    souce?: string | null;
+    /** writer-specific options */
+    [writerId: string]: any;
+};
+/**
+ * A writer definition that can be used with `@loaders.gl/core` functions
+ */
+export type Writer<DataT = unknown, BatchT = unknown, WriterOptionsT = WriterOptions> = {
+    name: string;
+    id: string;
+    module: string;
+    version: string;
+    worker?: string | boolean;
+    extensions?: string[];
+    mimeTypes?: string[];
+    binary?: boolean;
+    text?: boolean;
+    options: WriterOptionsT;
+    deprecatedOptions?: Record<string, string>;
+    encodeSync?: EncodeSync;
+    encodeURLtoURL?: EncodeURLtoURL;
+    encode?(data: DataT, options?: WriterOptionsT): Promise<ArrayBuffer>;
+    encodeText?(table: DataT, options?: WriterOptionsT): Promise<string> | string;
+    encodeInBatches?(data: AsyncIterable<any>, options?: WriterOptionsT): AsyncIterable<ArrayBuffer>;
+};
+type EncodeSync = (data: any, options?: WriterOptions) => ArrayBuffer;
+type EncodeURLtoURL = (inputUrl: string, outputUrl: string, options?: WriterOptions) => Promise<string>;
+/** Typescript helper to extract the writer options type from a generic writer type */
+export type WriterOptionsType<T = Writer> = T extends Writer<unknown, unknown, infer Options> ? Options : never;
+export type TransformBatches = (asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>) => AsyncIterable<ArrayBuffer>;
+/** Types that can be synchronously parsed */
+export type SyncDataType = string | ArrayBuffer;
+/** Types that can be parsed async */
+export type DataType = string | ArrayBuffer | File | Blob | Response | ReadableStream | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>;
+/** Types that can be parsed in batches */
+export type BatchableDataType = DataType | Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer> | Promise<AsyncIterable<ArrayBuffer>>;
+/**
+ * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
+ */
+export interface IFileSystem {
+    /**
+     * Return a list of file names
+     * @param dirname directory name. file system root directory if omitted
+     */
+    readdir(dirname?: string, options?: {
+        recursive?: boolean;
+    }): Promise<string[]>;
+    /**
+     * Gets information from a local file from the filesystem
+     * @param filename file name to stat
+     * @param options currently unused
+     * @throws if filename is not in local filesystem
+     */
+    stat(filename: string, options?: object): Promise<{
+        size: number;
+    }>;
+    /**
+     * Fetches a local file from the filesystem (or a URL)
+     * @param filename
+     * @param options
+     */
+    fetch(filename: string, options?: object): Promise<Response>;
+}
+type ReadOptions = {
+    buffer?: ArrayBuffer;
+    offset?: number;
+    length?: number;
+    position?: number;
+};
+export interface IRandomAccessReadFileSystem extends IFileSystem {
+    open(path: string, flags: string | number, mode?: any): Promise<any>;
+    close(fd: any): Promise<void>;
+    fstat(fd: any): Promise<object>;
+    read(fd: any, options?: ReadOptions): Promise<{
+        bytesRead: number;
+        buffer: Buffer;
+    }>;
+}
+export {};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAEA,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,UAAU,GACV,iBAAiB,GACjB,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,CAAC;AAEhB,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,YAAY,GAAG,YAAY,CAAC;AAExE,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;AAEzD,MAAM,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,aAAa,GAAG,eAAe,CAAC;AAE3E,KAAK,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;AAI3E;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,+CAA+C;IAC/C,KAAK,CAAC,EAAE,OAAO,KAAK,GAAG,SAAS,GAAG,WAAW,GAAG,IAAI,CAAC;IACtD,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAGlC,kDAAkD;IAClD,GAAG,CAAC,EAAE,GAAG,CAAC;IAIV,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,6CAA6C;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,CAAC,CAAC;IACV,gDAAgD;IAChD,QAAQ,CAAC,EAAE,CAAC,CAAC;IACb,gCAAgC;IAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,4CAA4C;IAC5C,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAIhC,4BAA4B;IAC5B,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,wCAAwC;IACxC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,mEAAmE;IACnE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,mEAAmE;IACnE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,8CAA8C;IAC9C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,mFAAmF;IACnF,SAAS,CAAC,EAAE,KAAK,GAAG,UAAU,GAAG,OAAO,CAAC;IACzC,yEAAyE;IACzE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,oDAAoD;IACpD,QAAQ,CAAC,EAAE,KAAK,CAAC;IACjB,+CAA+C;IAC/C,GAAG,CAAC,EAAE,KAAK,CAAC;IACZ,uEAAuE;IACvE,MAAM,CAAC,EAAE,KAAK,CAAC;IACf,yEAAyE;IACzE,OAAO,CAAC,EAAE,KAAK,CAAC;IAChB,mEAAmE;IACnE,IAAI,CAAC,EAAE,KAAK,CAAC;IACb,mEAAmE;IACnE,IAAI,CAAC,EAAE,KAAK,CAAC;IACb,iFAAiF;IACjF,WAAW,CAAC,EAAE,KAAK,CAAC;IACpB,qEAAqE;IACrE,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,KAAK,CAAC;IACjB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,KAAK,CAAC;IACjB,uFAAuF;IACvF,cAAc,CAAC,EAAE,KAAK,CAAC;IACvB,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,uEAAuE;IACvE,MAAM,CAAC,EAAE,KAAK,CAAC;IAGf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B,CAAC;AAEF,KAAK,cAAc,GAAG;IACpB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,MAAM,CAAC,KAAK,GAAG,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,cAAc,GAAG,aAAa,IAAI;IAE9E,QAAQ,CAAC,EAAE,KAAK,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,OAAO,EAAE,cAAc,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAGpE,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAG1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,WAAW,KAAK,OAAO,CAAC,GAAG,WAAW,GAAG,MAAM,CAAC,EAAE,CAAC;IAG3E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;CACxC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,CAAC,KAAK,GAAG,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,cAAc,GAAG,aAAa,IAAI,MAAM,CAC9F,KAAK,EACL,MAAM,EACN,cAAc,CACf,GAAG;IAEF,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAEvC,KAAK,EAAE,CACL,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,cAAc,EACxB,OAAO,CAAC,EAAE,aAAa,KACpB,OAAO,CAAC,KAAK,CAAC,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,CACV,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,cAAc,EACxB,OAAO,CAAC,EAAE,aAAa,KACpB,KAAK,CAAC;IACX,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IACvE,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,KAAK,KAAK,CAAC;IAClE,cAAc,CAAC,EAAE,CACf,QAAQ,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EAC5D,OAAO,CAAC,EAAE,cAAc,EACxB,OAAO,CAAC,EAAE,aAAa,KACpB,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3B,kBAAkB,CAAC,EAAE,CACnB,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE,cAAc,EACxB,OAAO,CAAC,EAAE,aAAa,KACpB,aAAa,CAAC,MAAM,CAAC,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,KAAK,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;IAChC,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,KAAK,EAAE,CACL,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,KACpB,OAAO,CAAC,GAAG,CAAC,CAAC;IAClB,SAAS,CAAC,EAAE,CACV,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,KACpB,GAAG,CAAC;IACT,cAAc,CAAC,EAAE,CACf,QAAQ,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EAC5D,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,KACpB,aAAa,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;CACvD,CAAC;AAyBF,KAAK,OAAO,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,KAAK,GAAG,CAAC;AAE9D,2EAA2E;AAC3E,MAAM,MAAM,iBAAiB,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC,GACjF,OAAO,GACP,KAAK,CAAC;AACV,wEAAwE;AACxE,MAAM,MAAM,gBAAgB,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,MAAM,CAAC,MAAM,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAC/E,MAAM,GACN,KAAK,CAAC;AACV,yEAAyE;AACzE,MAAM,MAAM,eAAe,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,MAAM,CAAC,GAAG,EAAE,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC;AAIlG,0BAA0B;AAC1B,MAAM,MAAM,aAAa,GAAG;IAC1B,wFAAwF;IACxF,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,8BAA8B;IAC9B,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,MAAM,CAAC,KAAK,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,EAAE,cAAc,GAAG,aAAa,IAAI;IACtF,IAAI,EAAE,MAAM,CAAC;IAEb,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAG1B,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf,OAAO,EAAE,cAAc,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAI3C,UAAU,CAAC,EAAE,UAAU,CAAC;IAExB,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,MAAM,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACrE,UAAU,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IAC9E,eAAe,CAAC,CAAC,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;CAClG,CAAC;AAGF,KAAK,UAAU,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,aAAa,KAAK,WAAW,CAAC;AAItE,KAAK,cAAc,GAAG,CACpB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,aAAa,KACpB,OAAO,CAAC,MAAM,CAAC,CAAC;AAErB,sFAAsF;AACtF,MAAM,MAAM,iBAAiB,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC,GACzF,OAAO,GACP,KAAK,CAAC;AAIV,MAAM,MAAM,gBAAgB,GAAG,CAC7B,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,KAC9D,aAAa,CAAC,WAAW,CAAC,CAAC;AAEhC,6CAA6C;AAC7C,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,WAAW,CAAC;AAEhD,qCAAqC;AACrC,MAAM,MAAM,QAAQ,GAChB,MAAM,GACN,WAAW,GACX,IAAI,GACJ,IAAI,GACJ,QAAQ,GACR,cAAc,GACd,QAAQ,CAAC,WAAW,CAAC,GACrB,aAAa,CAAC,WAAW,CAAC,CAAC;AAE/B,0CAA0C;AAC1C,MAAM,MAAM,iBAAiB,GACzB,QAAQ,GACR,QAAQ,CAAC,WAAW,CAAC,GACrB,aAAa,CAAC,WAAW,CAAC,GAC1B,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC;AAExC;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9E;;;;;OAKG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;IAElE;;;;OAIG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CAC9D;AAED,KAAK,WAAW,GAAG;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAC,CAAC;AAC/F,MAAM,WAAW,2BAA4B,SAAQ,WAAW;IAC9D,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrE,KAAK,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,KAAK,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;CACpF"}

package/dist/types.js

@@ -1,2 +1,3 @@
-export {};
-//# sourceMappingURL=types.js.map
+"use strict";
+// Typed arrays
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/workers/json-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=json-worker.d.ts.map

package/dist/workers/json-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-worker.d.ts","sourceRoot":"","sources":["../../src/workers/json-worker.ts"],"names":[],"mappings":""}

package/dist/workers/json-worker.js

@@ -1,4 +1,5 @@
-import { createLoaderWorker } from '../lib/worker-loader-utils/create-loader-worker';
-import { JSONLoader } from '../json-loader';
-createLoaderWorker(JSONLoader);
-//# sourceMappingURL=json-worker.js.map
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const create_loader_worker_1 = require("../lib/worker-loader-utils/create-loader-worker");
+const json_loader_1 = require("../json-loader");
+(0, create_loader_worker_1.createLoaderWorker)(json_loader_1.JSONLoader);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/loader-utils",
-  "version": "4.0.0-alpha.4",
+  "version": "4.0.0-alpha.6",
   "description": "Framework-independent loaders for 3D graphics formats",
   "license": "MIT",
   "publishConfig": {
@@ -17,9 +17,9 @@
     "mesh",
     "point cloud"
   ],
-  "types": "src/index.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "files": [
     "src",
@@ -27,29 +27,19 @@
     "README.md"
   ],
   "browser": {
-    "./src/lib/node/buffer.ts": false,
-    "./src/lib/node/buffer.js": false,
-    "./dist/es5/lib/node/buffer.js": false,
-    "./dist/esm/lib/node/buffer.js": false,
-    "./src/lib/node/fs.ts": false,
-    "./src/lib/node/fs.js": false,
-    "./dist/es5/lib/node/fs.js": false,
-    "./dist/esm/lib/node/fs.js": false,
-    "./src/lib/node/util.ts": false,
-    "./src/lib/node/util.js": false,
-    "./dist/es5/lib/node/util.js": false,
-    "./dist/esm/lib/node/util.js": false,
+    "./src/lib/node/buffer.ts": "./src/lib/node/buffer.browser.ts",
+    "./src/lib/node/buffer.js": "./src/lib/node/buffer.browser.js",
+    "./dist/lib/node/buffer.js": "./dist/lib/node/buffer.browser.js",
+    "./dist/es5/lib/node/buffer.js": "./dist/es5/lib/node/buffer.browser.js",
+    "./dist/esm/lib/node/buffer.js": "./dist/esm/lib/node/buffer.browser.js",
     "fs": false,
-    "util": false
-  },
-  "scripts": {
-    "pre-build-disabled": "npm run build-bundle",
-    "build-bundle": "webpack --config ../../scripts/webpack/bundle.js"
+    "stream": false
   },
+  "scripts": {},
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.4",
-    "@probe.gl/stats": "^3.4.0"
+    "@loaders.gl/worker-utils": "4.0.0-alpha.6",
+    "@probe.gl/stats": "^4.0.2"
   },
-  "gitHead": "53026061b3c8871f7e96d3a5826125cc6613bddc"
+  "gitHead": "acc1985050dfaa0f1f0c066f8da5bce7454a046c"
 }

package/src/index.ts

@@ -1,11 +1,18 @@
 // TYPES
 export type {
+  // loaders
   Loader,
   LoaderWithParser,
   LoaderContext,
   LoaderOptions,
+  LoaderOptionsType,
+  LoaderReturnType,
+  LoaderBatchType,
+  // writers
   Writer,
   WriterOptions,
+  WriterOptionsType,
+  // misc
   DataType,
   SyncDataType,
   BatchableDataType,
@@ -25,31 +32,31 @@ export {
   document
 } from './lib/env-utils/globals';
 
+export {mergeLoaderOptions} from './lib/option-utils/merge-loader-options';
+
 // LOADERS.GL-SPECIFIC WORKER UTILS
 export {createLoaderWorker} from './lib/worker-loader-utils/create-loader-worker';
 export {parseWithWorker, canParseWithWorker} from './lib/worker-loader-utils/parse-with-worker';
+export {canEncodeWithWorker} from './lib/worker-loader-utils/encode-with-worker';
 
 // PARSER UTILS
 export {parseJSON} from './lib/parser-utils/parse-json';
 
 // MEMORY COPY UTILS
 export {
-  toArrayBuffer,
   sliceArrayBuffer,
   concatenateArrayBuffers,
   concatenateTypedArrays,
   compareArrayBuffers
 } from './lib/binary-utils/array-buffer-utils';
 export {padToNBytes, copyToArray, copyArrayBuffer} from './lib/binary-utils/memory-copy-utils';
-export {
-  copyPaddedArrayBufferToDataView,
-  copyPaddedStringToDataView
-} from './lib/binary-utils/binary-copy-utils';
 export {
   padStringToByteAlignment,
   copyStringToDataView,
-  copyBinaryToDataView
-} from './lib/binary-utils/encode-utils';
+  copyBinaryToDataView,
+  copyPaddedArrayBufferToDataView,
+  copyPaddedStringToDataView
+} from './lib/binary-utils/dataview-copy-utils';
 export {getFirstCharacters, getMagicString} from './lib/binary-utils/get-first-characters';
 
 // ITERATOR UTILS
@@ -75,24 +82,31 @@ export {JSONLoader} from './json-loader';
 
 // Node.js emulation (can be used in browser)
 
-// `path` replacement (avoids bundling big path polyfill)
-import * as path from './lib/path-utils/path';
-export {path};
-
 // Avoid direct use of `Buffer` which pulls in 50KB polyfill
-export {isBuffer, toBuffer, bufferToArrayBuffer} from './lib/binary-utils/buffer-utils';
+export {isBuffer, toBuffer, toArrayBuffer} from './lib/binary-utils/memory-conversion-utils';
 
 // Note.js wrappers (can be safely imported, but not used in browser)
 
-// Use instead of importing 'util'
-import * as util from './lib/node/util';
-export {util};
-// TODO - remove
-export {promisify} from './lib/node/util';
+// Use instead of importing 'util' to avoid node dependencies
+export {promisify1, promisify2} from './lib/node/promisify';
+
+// `path` replacement (avoids bundling big path polyfill)
+import * as path from './lib/path-utils/path';
+export {path};
 
-// Use instead of importing 'fs';`
+// Use instead of importing 'fs' to avoid node dependencies`
 import * as fs from './lib/node/fs';
 export {fs};
 
+// Use instead of importing 'stream' to avoid node dependencies`
+import * as stream from './lib/node/stream';
+export {stream};
+
 // EXPERIMENTAL
+export type {ReadableFile} from './lib/filesystems/readable-file';
+export {makeReadableFile} from './lib/filesystems/readable-file';
+
+export type {WritableFile} from './lib/filesystems/writable-file';
+export {makeWritableFile} from './lib/filesystems/writable-file';
+
 export {default as _NodeFileSystem} from './lib/filesystems/node-filesystem';

package/src/json-loader.ts

@@ -1,14 +1,17 @@
-import type {LoaderWithParser} from './types';
+import type {LoaderWithParser, LoaderOptions} from './types';
+import type {Table, TableBatch} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+export type JSONLoaderOptions = LoaderOptions;
+
 /**
  * A JSON Micro loader (minimal bundle size)
  * Alternative to `@loaders.gl/json`
  */
-export const JSONLoader = {
+export const JSONLoader: LoaderWithParser<Table, TableBatch, JSONLoaderOptions> = {
   name: 'JSON',
   id: 'json',
   module: 'json',
@@ -26,5 +29,3 @@ export const JSONLoader = {
 function parseTextSync(text) {
   return JSON.parse(text);
 }
-
-export const _typecheckJSONLoader: LoaderWithParser = JSONLoader;

package/src/lib/binary-utils/array-buffer-utils.ts

@@ -1,46 +1,10 @@
 import {TypedArray} from '../../types';
-import {isBuffer, bufferToArrayBuffer} from './buffer-utils';
-
-/**
- * Convert an object to an array buffer
- */
-export function toArrayBuffer(data: any): ArrayBuffer {
-  // Note: Should be called first, Buffers can trigger other detections below
-  if (isBuffer(data)) {
-    return bufferToArrayBuffer(data);
-  }
-
-  if (data instanceof ArrayBuffer) {
-    return data;
-  }
-
-  // Careful - Node Buffers look like Uint8Arrays (keep after isBuffer)
-  if (ArrayBuffer.isView(data)) {
-    if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) {
-      return data.buffer;
-    }
-    return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
-  }
-
-  if (typeof data === 'string') {
-    const text = data;
-    const uint8Array = new TextEncoder().encode(text);
-    return uint8Array.buffer;
-  }
-
-  // HACK to support Blob polyfill
-  if (data && typeof data === 'object' && data._toArrayBuffer) {
-    return data._toArrayBuffer();
-  }
-
-  throw new Error('toArrayBuffer');
-}
 
 /**
  * compare two binary arrays for equality
- * @param {ArrayBuffer} a
- * @param {ArrayBuffer} b
- * @param {number} byteLength
+ * @param a
+ * @param b
+ * @param byteLength
  */
 export function compareArrayBuffers(
   arrayBuffer1: ArrayBuffer,
@@ -91,7 +55,7 @@ export function concatenateArrayBuffers(...sources: (ArrayBuffer | Uint8Array)[]
 /**
  * Concatenate arbitrary count of typed arrays
  * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Typed_arrays
- * @param {...*} arrays - list of arrays. All arrays should be the same type
+ * @param - list of arrays. All arrays should be the same type
  * @return A concatenated TypedArray
  */
 export function concatenateTypedArrays<T>(...typedArrays: T[]): T {