@loaders.gl/core 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/iterators/make-stream/make-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-stream.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-stream/make-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,oBAAoB,CAAC;AAM9D,oBAAY,iBAAiB,GAAG,oBAAoB,GAAG,qBAAqB,CAAC;AAE7E;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,EACxD,OAAO,CAAC,EAAE,iBAAiB,GAC1B,kBAAkB,CAEpB"}

package/dist/iterators/make-stream/make-stream.js

@@ -0,0 +1,16 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeStream = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const make_dom_stream_1 = require("./make-dom-stream");
+const make_node_stream_1 = __importDefault(require("./make-node-stream"));
+/**
+ * Returns a stream for an (async) iterator (works in both Node.js and browsers)
+ */
+function makeStream(data, options) {
+    return loader_utils_1.isBrowser ? (0, make_dom_stream_1.makeDOMStream)(data, options) : (0, make_node_stream_1.default)(data, options);
+}
+exports.makeStream = makeStream;

package/dist/javascript-utils/is-type.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+import type { Readable } from 'stream';
+/** A DOM or Node readable stream */
+export declare type ReadableStreamType = ReadableStream | Readable;
+export declare const isObject: (x: any) => boolean;
+export declare const isPureObject: (x: any) => boolean;
+export declare const isPromise: (x: any) => boolean;
+export declare const isIterable: (x: any) => boolean;
+export declare const isAsyncIterable: (x: any) => boolean;
+export declare const isIterator: (x: any) => boolean;
+export declare const isResponse: (x: any) => boolean;
+export declare const isFile: (x: any) => boolean;
+export declare const isBlob: (x: any) => boolean;
+/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
+export declare const isBuffer: (x: any) => boolean;
+export declare const isWritableDOMStream: (x: any) => boolean;
+export declare const isReadableDOMStream: (x: any) => boolean;
+export declare const isWritableNodeStream: (x: any) => boolean;
+export declare const isReadableNodeStream: (x: any) => boolean;
+export declare const isReadableStream: (x: any) => boolean;
+export declare const isWritableStream: (x: any) => boolean;
+//# sourceMappingURL=is-type.d.ts.map

package/dist/javascript-utils/is-type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"is-type.d.ts","sourceRoot":"","sources":["../../src/javascript-utils/is-type.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAErC,oCAAoC;AACpC,oBAAY,kBAAkB,GAAG,cAAc,GAAG,QAAQ,CAAC;AAK3D,eAAO,MAAM,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAoD,CAAC;AACxF,eAAO,MAAM,YAAY,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACU,CAAC;AAClD,eAAO,MAAM,SAAS,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAkD,CAAC;AAEvF,eAAO,MAAM,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAA8D,CAAC;AACpG,eAAO,MAAM,eAAe,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACU,CAAC;AACrD,eAAO,MAAM,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAwC,CAAC;AAE9E,eAAO,MAAM,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAEK,CAAC;AAE3C,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAiE,CAAC;AACnG,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAiE,CAAC;AAEnG,uFAAuF;AACvF,eAAO,MAAM,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAyD,CAAC;AAE7F,eAAO,MAAM,mBAAmB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACiB,CAAC;AAEhE,eAAO,MAAM,mBAAmB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAEyC,CAAC;AAGxF,eAAO,MAAM,oBAAoB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACmC,CAAC;AACnF,eAAO,MAAM,oBAAoB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACmC,CAAC;AACnF,eAAO,MAAM,gBAAgB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACQ,CAAC;AACpD,eAAO,MAAM,gBAAgB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACQ,CAAC"}

package/dist/javascript-utils/is-type.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWritableStream = exports.isReadableStream = exports.isReadableNodeStream = exports.isWritableNodeStream = exports.isReadableDOMStream = exports.isWritableDOMStream = exports.isBuffer = exports.isBlob = exports.isFile = exports.isResponse = exports.isIterator = exports.isAsyncIterable = exports.isIterable = exports.isPromise = exports.isPureObject = exports.isObject = void 0;
+const isBoolean = (x) => typeof x === 'boolean';
+const isFunction = (x) => typeof x === 'function';
+const isObject = (x) => x !== null && typeof x === 'object';
+exports.isObject = isObject;
+const isPureObject = (x) => (0, exports.isObject)(x) && x.constructor === {}.constructor;
+exports.isPureObject = isPureObject;
+const isPromise = (x) => (0, exports.isObject)(x) && isFunction(x.then);
+exports.isPromise = isPromise;
+const isIterable = (x) => x && typeof x[Symbol.iterator] === 'function';
+exports.isIterable = isIterable;
+const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
+exports.isAsyncIterable = isAsyncIterable;
+const isIterator = (x) => x && isFunction(x.next);
+exports.isIterator = isIterator;
+const isResponse = (x) => (typeof Response !== 'undefined' && x instanceof Response) ||
+    (x && x.arrayBuffer && x.text && x.json);
+exports.isResponse = isResponse;
+const isFile = (x) => typeof File !== 'undefined' && x instanceof File;
+exports.isFile = isFile;
+const isBlob = (x) => typeof Blob !== 'undefined' && x instanceof Blob;
+exports.isBlob = isBlob;
+/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
+const isBuffer = (x) => x && typeof x === 'object' && x.isBuffer;
+exports.isBuffer = isBuffer;
+const isWritableDOMStream = (x) => (0, exports.isObject)(x) && isFunction(x.abort) && isFunction(x.getWriter);
+exports.isWritableDOMStream = isWritableDOMStream;
+const isReadableDOMStream = (x) => (typeof ReadableStream !== 'undefined' && x instanceof ReadableStream) ||
+    ((0, exports.isObject)(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader));
+exports.isReadableDOMStream = isReadableDOMStream;
+// Not implemented in Firefox: && isFunction(x.pipeTo)
+const isWritableNodeStream = (x) => (0, exports.isObject)(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
+exports.isWritableNodeStream = isWritableNodeStream;
+const isReadableNodeStream = (x) => (0, exports.isObject)(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+exports.isReadableNodeStream = isReadableNodeStream;
+const isReadableStream = (x) => (0, exports.isReadableDOMStream)(x) || (0, exports.isReadableNodeStream)(x);
+exports.isReadableStream = isReadableStream;
+const isWritableStream = (x) => (0, exports.isWritableDOMStream)(x) || (0, exports.isWritableNodeStream)(x);
+exports.isWritableStream = isWritableStream;

package/dist/lib/api/encode.d.ts

@@ -0,0 +1,26 @@
+import type { Writer, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+export declare function encode(data: any, writer: Writer, options?: LoaderOptions): Promise<ArrayBuffer>;
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+export declare function encodeSync(data: any, writer: Writer, options?: LoaderOptions): ArrayBuffer;
+/**
+ * Encode loaded data to text using the specified Writer
+ * @note This is a convenience function not intended for production use on large input data.
+ * It is not optimized for performance. Data maybe converted from text to binary and back.
+ * @throws if the writer does not generate text output
+ */
+export declare function encodeText(data: any, writer: Writer, options?: LoaderOptions): Promise<string>;
+/**
+ * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
+ */
+export declare function encodeInBatches(data: any, writer: Writer, options?: LoaderOptions): AsyncIterable<ArrayBuffer>;
+/**
+ * Encode data stored in a file (on disk) to another file.
+ * @note Node.js only. This function enables using command-line converters as "writers".
+ */
+export declare function encodeURLtoURL(inputUrl: any, outputUrl: any, writer: Writer, options: any): Promise<string>;
+//# sourceMappingURL=encode.d.ts.map

package/dist/lib/api/encode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode.d.ts","sourceRoot":"","sources":["../../../src/lib/api/encode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE;;GAEG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,WAAW,CAAC,CA+CtB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,WAAW,CAK1F;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,WAAW,CAAC,CAO5B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,QAAQ,KAAA,EACR,SAAS,KAAA,EACT,MAAM,EAAE,MAAM,EACd,OAAO,KAAA,GACN,OAAO,CAAC,MAAM,CAAC,CAQjB"}

package/dist/lib/api/encode.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encodeURLtoURL = exports.encodeInBatches = exports.encodeText = exports.encodeSync = exports.encode = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const loader_utils_2 = require("@loaders.gl/loader-utils");
+const write_file_1 = require("../fetch/write-file");
+const fetch_file_1 = require("../fetch/fetch-file");
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+async function encode(data, writer, options) {
+    // TODO Merge default writer options with options argument like it is done in load module.
+    if (writer.encode) {
+        return await writer.encode(data, options);
+    }
+    if (writer.encodeSync) {
+        return writer.encodeSync(data, options);
+    }
+    if (writer.encodeText) {
+        return new TextEncoder().encode(await writer.encodeText(data, options));
+    }
+    if (writer.encodeInBatches) {
+        // Create an iterator representing the data
+        // TODO - Assumes this is a table
+        const batches = encodeInBatches(data, writer, options);
+        // Concatenate the output
+        const chunks = [];
+        for await (const batch of batches) {
+            chunks.push(batch);
+        }
+        // @ts-ignore
+        return (0, loader_utils_1.concatenateArrayBuffers)(...chunks);
+    }
+    if (!loader_utils_2.isBrowser && writer.encodeURLtoURL) {
+        // TODO - how to generate filenames with correct extensions?
+        const tmpInputFilename = getTemporaryFilename('input');
+        await (0, write_file_1.writeFile)(tmpInputFilename, data);
+        const tmpOutputFilename = getTemporaryFilename('output');
+        const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
+        const response = await (0, fetch_file_1.fetchFile)(outputFilename);
+        return response.arrayBuffer();
+    }
+    throw new Error('Writer could not encode data');
+}
+exports.encode = encode;
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+function encodeSync(data, writer, options) {
+    if (writer.encodeSync) {
+        return writer.encodeSync(data, options);
+    }
+    throw new Error('Writer could not synchronously encode data');
+}
+exports.encodeSync = encodeSync;
+/**
+ * Encode loaded data to text using the specified Writer
+ * @note This is a convenience function not intended for production use on large input data.
+ * It is not optimized for performance. Data maybe converted from text to binary and back.
+ * @throws if the writer does not generate text output
+ */
+async function encodeText(data, writer, options) {
+    if (writer.text && writer.encodeText) {
+        return await writer.encodeText(data, options);
+    }
+    if (writer.text && (writer.encode || writer.encodeInBatches)) {
+        const arrayBuffer = await encode(data, writer, options);
+        return new TextDecoder().decode(arrayBuffer);
+    }
+    throw new Error('Writer could not encode data as text');
+}
+exports.encodeText = encodeText;
+/**
+ * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
+ */
+function encodeInBatches(data, writer, options) {
+    if (writer.encodeInBatches) {
+        const dataIterator = getIterator(data);
+        return writer.encodeInBatches(dataIterator, options);
+    }
+    // TODO -fall back to atomic encode?
+    throw new Error('Writer could not encode data in batches');
+}
+exports.encodeInBatches = encodeInBatches;
+/**
+ * Encode data stored in a file (on disk) to another file.
+ * @note Node.js only. This function enables using command-line converters as "writers".
+ */
+async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
+    inputUrl = (0, loader_utils_1.resolvePath)(inputUrl);
+    outputUrl = (0, loader_utils_1.resolvePath)(outputUrl);
+    if (loader_utils_2.isBrowser || !writer.encodeURLtoURL) {
+        throw new Error();
+    }
+    const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
+    return outputFilename;
+}
+exports.encodeURLtoURL = encodeURLtoURL;
+/**
+ * @todo TODO - this is an unacceptable hack!!!
+ */
+function getIterator(data) {
+    const dataIterator = [{ table: data, start: 0, end: data.length }];
+    return dataIterator;
+}
+/**
+ * @todo Move to utils
+ */
+function getTemporaryFilename(filename) {
+    return `/tmp/${filename}`;
+}

package/dist/lib/api/load-in-batches.d.ts

@@ -0,0 +1,13 @@
+import type { LoaderWithParser, LoaderOptions, LoaderContext } from '@loaders.gl/loader-utils';
+declare type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function loadInBatches(files: FileType, loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<AsyncIterable<any>>;
+export declare function loadInBatches(files: FileType[] | FileList, loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<AsyncIterable<any>>;
+export {};
+//# sourceMappingURL=load-in-batches.d.ts.map

package/dist/lib/api/load-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/api/load-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAM7F,aAAK,QAAQ,GAAG,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,QAAQ,CAAC,EAAE,GAAG,QAAQ,CAAC;AAEjG;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,QAAQ,EACf,OAAO,CAAC,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,GAAG,aAAa,EAC/D,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/B,wBAAgB,aAAa,CAC3B,KAAK,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAC5B,OAAO,CAAC,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,GAAG,aAAa,EAC/D,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC"}

package/dist/lib/api/load-in-batches.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadInBatches = void 0;
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const parse_in_batches_1 = require("./parse-in-batches");
+function loadInBatches(files, loaders, options, context) {
+    // Signature: load(url, options)
+    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = null;
+    }
+    // Select fetch function
+    const fetch = (0, option_utils_1.getFetchFunction)(options || {});
+    // Single url/file
+    if (!Array.isArray(files)) {
+        return loadOneFileInBatches(files, loaders, options, fetch);
+    }
+    // Multiple URLs / files
+    const promises = files.map((file) => loadOneFileInBatches(file, loaders, options, fetch));
+    // No point in waiting here for all responses before starting to stream individual streams?
+    return promises;
+}
+exports.loadInBatches = loadInBatches;
+async function loadOneFileInBatches(file, loaders, options, fetch) {
+    if (typeof file === 'string') {
+        const url = file;
+        const response = await fetch(url);
+        return await (0, parse_in_batches_1.parseInBatches)(response, loaders, options);
+    }
+    return await (0, parse_in_batches_1.parseInBatches)(file, loaders, options);
+}

package/dist/lib/api/load.d.ts

@@ -0,0 +1,12 @@
+import type { DataType, Loader, LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Parses `data` using a specified loader
+ * Note: Load does duplicate a lot of parse.
+ * it can also call fetchFile on string urls, which `parse` won't do.
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function load(url: string | DataType, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<any>;
+//# sourceMappingURL=load.d.ts.map

package/dist/lib/api/load.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../../src/lib/api/load.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAO7F;;;;;;;;GAQG;AAEH,wBAAsB,IAAI,CACxB,GAAG,EAAE,MAAM,GAAG,QAAQ,EACtB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,GAAG,CAAC,CA0Bd"}

package/dist/lib/api/load.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.load = void 0;
+const is_type_1 = require("../../javascript-utils/is-type");
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const parse_1 = require("./parse");
+/**
+ * Parses `data` using a specified loader
+ * Note: Load does duplicate a lot of parse.
+ * it can also call fetchFile on string urls, which `parse` won't do.
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+// implementation signature
+async function load(url, loaders, options, context) {
+    // Signature: load(url, options)
+    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = undefined;
+    }
+    // Select fetch function
+    const fetch = (0, option_utils_1.getFetchFunction)(options);
+    // at this point, `url` could be already loaded binary data
+    let data = url;
+    // url is a string, fetch the url
+    if (typeof url === 'string') {
+        data = await fetch(url);
+        // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
+    }
+    if ((0, is_type_1.isBlob)(url)) {
+        // The fetch response object will contain blob.name
+        data = await fetch(url);
+    }
+    // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
+    return await (0, parse_1.parse)(data, loaders, options);
+}
+exports.load = load;

package/dist/lib/api/parse-in-batches.d.ts

@@ -0,0 +1,10 @@
+import type { BatchableDataType, Loader, LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function parseInBatches(data: BatchableDataType, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<AsyncIterable<any>>;
+//# sourceMappingURL=parse-in-batches.d.ts.map

package/dist/lib/api/parse-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/api/parse-in-batches.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,iBAAiB,EACjB,MAAM,EAEN,aAAa,EACb,aAAa,EACd,MAAM,0BAA0B,CAAC;AAYlC;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,iBAAiB,EACvB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAqC7B"}

package/dist/lib/api/parse-in-batches.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseInBatches = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const loader_context_1 = require("../loader-utils/loader-context");
+const get_data_1 = require("../loader-utils/get-data");
+const resource_utils_1 = require("../utils/resource-utils");
+const select_loader_1 = require("./select-loader");
+// Ensure `parse` is available in context if loader falls back to `parse`
+const parse_1 = require("./parse");
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+async function parseInBatches(data, loaders, options, context) {
+    (0, loader_utils_1.assert)(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
+    // Signature: parseInBatches(data, options, url) - Uses registered loaders
+    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = undefined;
+    }
+    data = await data; // Resolve any promise
+    options = options || {};
+    // Extract a url for auto detection
+    const { url } = (0, resource_utils_1.getResourceUrlAndType)(data);
+    // Chooses a loader and normalizes it
+    // Note - only uses URL and contentType for streams and iterator inputs
+    const loader = await (0, select_loader_1.selectLoader)(data, loaders, options);
+    // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
+    if (!loader) {
+        // @ts-ignore
+        return null;
+    }
+    // Normalize options
+    // @ts-ignore
+    options = (0, option_utils_1.normalizeOptions)(options, loader, loaders, url);
+    // @ts-ignore
+    context = (0, loader_context_1.getLoaderContext)(
+    // @ts-ignore
+    { url, parseInBatches, parse: parse_1.parse, loaders: loaders }, options, context);
+    return await parseWithLoaderInBatches(loader, data, options, context);
+}
+exports.parseInBatches = parseInBatches;
+/**
+ * Loader has been selected and context has been prepared, see if we need to emit a metadata batch
+ */
+async function parseWithLoaderInBatches(loader, data, options, context) {
+    const outputIterator = await parseToOutputIterator(loader, data, options, context);
+    // Generate metadata batch if requested
+    if (!options.metadata) {
+        return outputIterator;
+    }
+    const metadataBatch = {
+        batchType: 'metadata',
+        metadata: {
+            _loader: loader,
+            _context: context
+        },
+        // Populate with some default fields to avoid crashing
+        data: [],
+        bytesUsed: 0
+    };
+    async function* makeMetadataBatchIterator(iterator) {
+        yield metadataBatch;
+        yield* iterator;
+    }
+    return makeMetadataBatchIterator(outputIterator);
+}
+/**
+ * Prep work is done, now it is time to start parsing into an output operator
+ * The approach depends on which parse function the loader exposes
+ * `parseInBatches` (preferred), `parse` (fallback)
+ */
+async function parseToOutputIterator(loader, data, options, context) {
+    // Get an iterator from the input
+    const inputIterator = await (0, get_data_1.getAsyncIterableFromData)(data, options);
+    // Apply any iterator transforms (options.transforms)
+    const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);
+    // If loader supports parseInBatches, we are done
+    if (loader.parseInBatches) {
+        return loader.parseInBatches(transformedIterator, options, context);
+    }
+    // Fallback: load atomically using `parse` concatenating input iterator into single chunk
+    async function* parseChunkInBatches() {
+        const arrayBuffer = await (0, loader_utils_1.concatenateArrayBuffersAsync)(transformedIterator);
+        // Call `parse` instead of `loader.parse` to ensure we can call workers etc.
+        const parsedData = await (0, parse_1.parse)(arrayBuffer, loader, 
+        // TODO - Hack: supply loaders MIME type to ensure we match it
+        { ...options, mimeType: loader.mimeTypes[0] }, context);
+        // yield a single batch, the output from loader.parse()
+        // TODO - run through batch builder to apply options etc...
+        const batch = {
+            mimeType: loader.mimeTypes[0],
+            shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
+            batchType: 'data',
+            data: parsedData,
+            length: Array.isArray(parsedData) ? parsedData.length : 1
+        };
+        yield batch;
+    }
+    return parseChunkInBatches();
+}
+/**
+ * Create an iterator chain with any transform iterators (crypto, decompression)
+ * @param inputIterator
+ * @param options
+ */
+async function applyInputTransforms(inputIterator, transforms = []) {
+    let iteratorChain = inputIterator;
+    for await (const transformBatches of transforms) {
+        iteratorChain = transformBatches(iteratorChain);
+    }
+    return iteratorChain;
+}

package/dist/lib/api/parse-sync.d.ts

@@ -0,0 +1,10 @@
+import type { SyncDataType, Loader, LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Parses `data` synchronously using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function parseSync(data: SyncDataType, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): any;
+//# sourceMappingURL=parse-sync.d.ts.map

package/dist/lib/api/parse-sync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-sync.d.ts","sourceRoot":"","sources":["../../../src/lib/api/parse-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,MAAM,EAEN,aAAa,EACb,aAAa,EACd,MAAM,0BAA0B,CAAC;AASlC;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,GAAG,CAmCL"}

package/dist/lib/api/parse-sync.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSync = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const select_loader_1 = require("./select-loader");
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const get_data_1 = require("../loader-utils/get-data");
+const loader_context_1 = require("../loader-utils/loader-context");
+const resource_utils_1 = require("../utils/resource-utils");
+/**
+ * Parses `data` synchronously using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+function parseSync(data, loaders, options, context) {
+    (0, loader_utils_1.assert)(!context || typeof context === 'object'); // parseSync no longer accepts final url
+    // Signature: parseSync(data, options)
+    // Uses registered loaders
+    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = undefined;
+    }
+    options = options || {};
+    // Chooses a loader (and normalizes it)
+    // Also use any loaders in the context, new loaders take priority
+    const typedLoaders = loaders;
+    const candidateLoaders = (0, loader_context_1.getLoadersFromContext)(typedLoaders, context);
+    const loader = (0, select_loader_1.selectLoaderSync)(data, candidateLoaders, options);
+    // Note: if nothrow option was set, it is possible that no loader was found, if so just return null
+    if (!loader) {
+        return null;
+    }
+    // Normalize options
+    options = (0, option_utils_1.normalizeOptions)(options, loader, candidateLoaders);
+    // Extract a url for auto detection
+    const { url } = (0, resource_utils_1.getResourceUrlAndType)(data);
+    const parse = () => {
+        throw new Error('parseSync called parse');
+    };
+    context = (0, loader_context_1.getLoaderContext)({ url, parseSync, parse, loaders: loaders }, options);
+    return parseWithLoaderSync(loader, data, options, context);
+}
+exports.parseSync = parseSync;
+// TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
+function parseWithLoaderSync(loader, data, options, context) {
+    data = (0, get_data_1.getArrayBufferOrStringFromDataSync)(data, loader, options);
+    if (loader.parseTextSync && typeof data === 'string') {
+        return loader.parseTextSync(data, options); // , context, loader);
+    }
+    if (loader.parseSync && data instanceof ArrayBuffer) {
+        return loader.parseSync(data, options, context); // , loader);
+    }
+    // TBD - If synchronous parser not available, return null
+    throw new Error(`${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${context.url || ''}`);
+}

package/dist/lib/api/parse.d.ts

@@ -0,0 +1,10 @@
+import type { DataType, Loader, LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function parse(data: DataType | Promise<DataType>, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<any>;
+//# sourceMappingURL=parse.d.ts.map

package/dist/lib/api/parse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse.d.ts","sourceRoot":"","sources":["../../../src/lib/api/parse.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAU7F;;;;;;GAMG;AACH,wBAAsB,KAAK,CACzB,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,EAClC,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,GAAG,CAAC,CAmCd"}

package/dist/lib/api/parse.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parse = void 0;
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const get_data_1 = require("../loader-utils/get-data");
+const loader_context_1 = require("../loader-utils/loader-context");
+const resource_utils_1 = require("../utils/resource-utils");
+const select_loader_1 = require("./select-loader");
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+async function parse(data, loaders, options, context) {
+    (0, worker_utils_1.assert)(!context || typeof context === 'object'); // parse no longer accepts final url
+    // Signature: parse(data, options, context | url)
+    // Uses registered loaders
+    if (loaders && !Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = undefined;
+    }
+    data = await data; // Resolve any promise
+    options = options || {};
+    // Extract a url for auto detection
+    const { url } = (0, resource_utils_1.getResourceUrlAndType)(data);
+    // Chooses a loader (and normalizes it)
+    // Also use any loaders in the context, new loaders take priority
+    const typedLoaders = loaders;
+    const candidateLoaders = (0, loader_context_1.getLoadersFromContext)(typedLoaders, context);
+    // todo hacky type cast
+    const loader = await (0, select_loader_1.selectLoader)(data, candidateLoaders, options);
+    // Note: if no loader was found, if so just return null
+    if (!loader) {
+        return null;
+    }
+    // Normalize options
+    options = (0, option_utils_1.normalizeOptions)(options, loader, candidateLoaders, url);
+    // Get a context (if already present, will be unchanged)
+    context = (0, loader_context_1.getLoaderContext)({ url, parse, loaders: candidateLoaders }, options, context);
+    return await parseWithLoader(loader, data, options, context);
+}
+exports.parse = parse;
+// TODO: support progress and abort
+// TODO - should accept loader.parseAsyncIterator and concatenate.
+async function parseWithLoader(loader, data, options, context) {
+    (0, worker_utils_1.validateWorkerVersion)(loader);
+    data = await (0, get_data_1.getArrayBufferOrStringFromData)(data, loader, options);
+    // First check for synchronous text parser, wrap results in promises
+    if (loader.parseTextSync && typeof data === 'string') {
+        options.dataType = 'text';
+        return loader.parseTextSync(data, options, context, loader);
+    }
+    // If we have a workerUrl and the loader can parse the given options efficiently in a worker
+    if ((0, loader_utils_1.canParseWithWorker)(loader, options)) {
+        return await (0, loader_utils_1.parseWithWorker)(loader, data, options, context, parse);
+    }
+    // Check for asynchronous parser
+    if (loader.parseText && typeof data === 'string') {
+        return await loader.parseText(data, options, context, loader);
+    }
+    if (loader.parse) {
+        return await loader.parse(data, options, context, loader);
+    }
+    // This should not happen, all sync loaders should also offer `parse` function
+    (0, worker_utils_1.assert)(!loader.parseSync);
+    // TBD - If asynchronous parser not available, return null
+    throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
+}