@loaders.gl/core 3.1.0-alpha.5 → 3.1.0-beta.1

package/dist/iterators/make-iterator/make-string-iterator.d.ts

@@ -0,0 +1,9 @@
+import type { IteratorOptions } from './make-iterator';
+/**
+ * Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers
+ * @param blob string to iterate over
+ * @param options
+ * @param options.chunkSize
+ */
+export declare function makeStringIterator(string: string, options?: IteratorOptions): Iterable<ArrayBuffer>;
+//# sourceMappingURL=make-string-iterator.d.ts.map

package/dist/iterators/make-iterator/make-string-iterator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-string-iterator.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-iterator/make-string-iterator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,iBAAiB,CAAC;AAIrD;;;;;GAKG;AACH,wBAAiB,kBAAkB,CACjC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,GACxB,QAAQ,CAAC,WAAW,CAAC,CAcvB"}

package/dist/iterators/make-iterator/make-string-iterator.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeStringIterator = void 0;
+const DEFAULT_CHUNK_SIZE = 256 * 1024;
+/**
+ * Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers
+ * @param blob string to iterate over
+ * @param options
+ * @param options.chunkSize
+ */
+function* makeStringIterator(string, options) {
+    const chunkSize = options?.chunkSize || DEFAULT_CHUNK_SIZE;
+    let offset = 0;
+    const textEncoder = new TextEncoder();
+    while (offset < string.length) {
+        // Create a chunk of the right size
+        const chunkLength = Math.min(string.length - offset, chunkSize);
+        const chunk = string.slice(offset, offset + chunkLength);
+        offset += chunkLength;
+        // yield an ArrayBuffer chunk
+        yield textEncoder.encode(chunk);
+    }
+}
+exports.makeStringIterator = makeStringIterator;

package/dist/iterators/make-stream/make-dom-stream.d.ts

@@ -0,0 +1,9 @@
+export declare type MakeDOMStreamOptions = {
+    /** Stream allocates an arrayBuffer. Enables use of a default reader. */
+    autoAllocateChunkSize?: number;
+    /** Total number of chunks in queue before back pressure is applied */
+    highWaterMark?: number;
+};
+/** Builds a DOM stream from an iterator */
+export declare function makeDOMStream<ArrayBuffer>(source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>, options?: MakeDOMStreamOptions): ReadableStream;
+//# sourceMappingURL=make-dom-stream.d.ts.map

package/dist/iterators/make-stream/make-dom-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-dom-stream.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-stream/make-dom-stream.ts"],"names":[],"mappings":"AAAA,oBAAY,oBAAoB,GAAG;IACjC,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sEAAsE;IACtE,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,2CAA2C;AAC3C,wBAAgB,aAAa,CAAC,WAAW,EACvC,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,EAC1D,OAAO,CAAC,EAAE,oBAAoB,GAC7B,cAAc,CAsChB"}

package/dist/iterators/make-stream/make-dom-stream.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeDOMStream = void 0;
+/** Builds a DOM stream from an iterator */
+function makeDOMStream(source, options) {
+    const iterator = source[Symbol.asyncIterator]
+        ? source[Symbol.asyncIterator]()
+        : source[Symbol.iterator]();
+    return new ReadableStream({
+        // Create a byte stream (enables `Response(stream).arrayBuffer()`)
+        // Only supported on Chrome
+        // See: https://developer.mozilla.org/en-US/docs/Web/API/ReadableByteStreamController
+        // @ts-expect-error Only Chrome supports byte streams
+        type: 'bytes',
+        async pull(controller) {
+            try {
+                const { done, value } = await iterator.next();
+                if (done) {
+                    controller.close();
+                }
+                else {
+                    // TODO - ignores controller.desiredSize
+                    controller.enqueue(new Uint8Array(value));
+                }
+            }
+            catch (error) {
+                controller.error(error);
+            }
+        },
+        async cancel() {
+            await iterator?.return?.();
+        }
+    }, 
+    // options: QueingStrategy<Uint8Array>
+    {
+        // This is bytes, not chunks
+        highWaterMark: 2 ** 24,
+        ...options
+    });
+}
+exports.makeDOMStream = makeDOMStream;

package/dist/iterators/make-stream/make-node-stream.d.ts

@@ -0,0 +1,7 @@
+/// <reference types="node" />
+import { Readable, ReadableOptions } from 'stream';
+export declare type MakeNodeStreamOptions = ReadableOptions;
+/** Builds a node stream from an iterator */
+declare function makeNodeStream<ArrayBuffer>(source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>, options?: ReadableOptions): Readable;
+export default makeNodeStream;
+//# sourceMappingURL=make-node-stream.d.ts.map

package/dist/iterators/make-stream/make-node-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-node-stream.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-stream/make-node-stream.ts"],"names":[],"mappings":";AAAA,OAAO,EAAC,QAAQ,EAAE,eAAe,EAAC,MAAM,QAAQ,CAAC;AAEjD,oBAAY,qBAAqB,GAAG,eAAe,CAAC;AAEpD,4CAA4C;AAC5C,iBAAS,cAAc,CAAC,WAAW,EACjC,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,EAC1D,OAAO,CAAC,EAAE,eAAe,GACxB,QAAQ,CAKV;AAwDD,eAAe,cAAc,CAAC"}

package/dist/iterators/make-stream/make-node-stream.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_1 = require("stream");
+/** Builds a node stream from an iterator */
+function makeNodeStream(source, options) {
+    const iterator = source[Symbol.asyncIterator]
+        ? source[Symbol.asyncIterator]()
+        : source[Symbol.iterator]();
+    return new AsyncIterableReadable(iterator, options);
+}
+class AsyncIterableReadable extends stream_1.Readable {
+    constructor(it, options) {
+        super(options);
+        this._iterator = it;
+        this._pulling = false;
+        this._bytesMode = !options || !options.objectMode;
+    }
+    async _read(size) {
+        if (!this._pulling) {
+            this._pulling = true;
+            this._pulling = await this._pull(size, this._iterator);
+        }
+    }
+    async _destroy(error, cb) {
+        if (!this._iterator) {
+            return;
+        }
+        if (error) {
+            await this._iterator?.throw?.(error);
+        }
+        else {
+            await this._iterator?.return?.(error);
+        }
+        cb?.(null);
+    }
+    // eslint-disable-next-line complexity
+    async _pull(size, it) {
+        const bm = this._bytesMode;
+        let r = null;
+        // while (this.readable && !(r = await it.next(bm ? size : null)).done) {
+        while (this.readable && !(r = await it.next()).done) {
+            if (size !== null) {
+                size -= bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1;
+            }
+            if (!this.push(new Uint8Array(r.value)) || size <= 0) {
+                break;
+            }
+        }
+        if ((r?.done || !this.readable) && (this.push(null) || true)) {
+            it?.return?.();
+        }
+        return !this.readable;
+    }
+}
+// This module is marked `false` in the the "browser" field of the `package.json` for
+// `@loaders.gl/core`. We avoid using named exports so that bundlers have an easier
+// time resolving this "empty" module.
+exports.default = makeNodeStream;

package/dist/iterators/make-stream/make-stream.d.ts

@@ -0,0 +1,9 @@
+import type { ReadableStreamType } from '../../javascript-utils/is-type';
+import type { MakeDOMStreamOptions } from './make-dom-stream';
+import type { MakeNodeStreamOptions } from './make-node-stream';
+export declare type MakeStreamOptions = MakeDOMStreamOptions | MakeNodeStreamOptions;
+/**
+ * Returns a stream for an (async) iterator (works in both Node.js and browsers)
+ */
+export declare function makeStream(data: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>, options?: MakeStreamOptions): ReadableStreamType;
+//# sourceMappingURL=make-stream.d.ts.map

package/dist/iterators/make-stream/make-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-stream.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-stream/make-stream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,oBAAoB,CAAC;AAM9D,oBAAY,iBAAiB,GAAG,oBAAoB,GAAG,qBAAqB,CAAC;AAE7E;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,EACxD,OAAO,CAAC,EAAE,iBAAiB,GAC1B,kBAAkB,CAEpB"}

package/dist/iterators/make-stream/make-stream.js

@@ -0,0 +1,16 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.makeStream = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const make_dom_stream_1 = require("./make-dom-stream");
+const make_node_stream_1 = __importDefault(require("./make-node-stream"));
+/**
+ * Returns a stream for an (async) iterator (works in both Node.js and browsers)
+ */
+function makeStream(data, options) {
+    return loader_utils_1.isBrowser ? (0, make_dom_stream_1.makeDOMStream)(data, options) : (0, make_node_stream_1.default)(data, options);
+}
+exports.makeStream = makeStream;

package/dist/javascript-utils/is-type.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+import type { Readable } from 'stream';
+/** A DOM or Node readable stream */
+export declare type ReadableStreamType = ReadableStream | Readable;
+export declare const isObject: (x: any) => boolean;
+export declare const isPureObject: (x: any) => boolean;
+export declare const isPromise: (x: any) => boolean;
+export declare const isIterable: (x: any) => boolean;
+export declare const isAsyncIterable: (x: any) => boolean;
+export declare const isIterator: (x: any) => boolean;
+export declare const isResponse: (x: any) => boolean;
+export declare const isFile: (x: any) => boolean;
+export declare const isBlob: (x: any) => boolean;
+/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
+export declare const isBuffer: (x: any) => boolean;
+export declare const isWritableDOMStream: (x: any) => boolean;
+export declare const isReadableDOMStream: (x: any) => boolean;
+export declare const isWritableNodeStream: (x: any) => boolean;
+export declare const isReadableNodeStream: (x: any) => boolean;
+export declare const isReadableStream: (x: any) => boolean;
+export declare const isWritableStream: (x: any) => boolean;
+//# sourceMappingURL=is-type.d.ts.map

package/dist/javascript-utils/is-type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"is-type.d.ts","sourceRoot":"","sources":["../../src/javascript-utils/is-type.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAErC,oCAAoC;AACpC,oBAAY,kBAAkB,GAAG,cAAc,GAAG,QAAQ,CAAC;AAK3D,eAAO,MAAM,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAoD,CAAC;AACxF,eAAO,MAAM,YAAY,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACU,CAAC;AAClD,eAAO,MAAM,SAAS,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAkD,CAAC;AAEvF,eAAO,MAAM,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAA8D,CAAC;AACpG,eAAO,MAAM,eAAe,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACU,CAAC;AACrD,eAAO,MAAM,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAwC,CAAC;AAE9E,eAAO,MAAM,UAAU,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAEK,CAAC;AAE3C,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAiE,CAAC;AACnG,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAiE,CAAC;AAEnG,uFAAuF;AACvF,eAAO,MAAM,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAAyD,CAAC;AAE7F,eAAO,MAAM,mBAAmB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACiB,CAAC;AAEhE,eAAO,MAAM,mBAAmB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OAEyC,CAAC;AAGxF,eAAO,MAAM,oBAAoB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACmC,CAAC;AACnF,eAAO,MAAM,oBAAoB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACmC,CAAC;AACnF,eAAO,MAAM,gBAAgB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACQ,CAAC;AACpD,eAAO,MAAM,gBAAgB,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,OACQ,CAAC"}

package/dist/javascript-utils/is-type.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWritableStream = exports.isReadableStream = exports.isReadableNodeStream = exports.isWritableNodeStream = exports.isReadableDOMStream = exports.isWritableDOMStream = exports.isBuffer = exports.isBlob = exports.isFile = exports.isResponse = exports.isIterator = exports.isAsyncIterable = exports.isIterable = exports.isPromise = exports.isPureObject = exports.isObject = void 0;
+const isBoolean = (x) => typeof x === 'boolean';
+const isFunction = (x) => typeof x === 'function';
+const isObject = (x) => x !== null && typeof x === 'object';
+exports.isObject = isObject;
+const isPureObject = (x) => (0, exports.isObject)(x) && x.constructor === {}.constructor;
+exports.isPureObject = isPureObject;
+const isPromise = (x) => (0, exports.isObject)(x) && isFunction(x.then);
+exports.isPromise = isPromise;
+const isIterable = (x) => x && typeof x[Symbol.iterator] === 'function';
+exports.isIterable = isIterable;
+const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
+exports.isAsyncIterable = isAsyncIterable;
+const isIterator = (x) => x && isFunction(x.next);
+exports.isIterator = isIterator;
+const isResponse = (x) => (typeof Response !== 'undefined' && x instanceof Response) ||
+    (x && x.arrayBuffer && x.text && x.json);
+exports.isResponse = isResponse;
+const isFile = (x) => typeof File !== 'undefined' && x instanceof File;
+exports.isFile = isFile;
+const isBlob = (x) => typeof Blob !== 'undefined' && x instanceof Blob;
+exports.isBlob = isBlob;
+/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
+const isBuffer = (x) => x && typeof x === 'object' && x.isBuffer;
+exports.isBuffer = isBuffer;
+const isWritableDOMStream = (x) => (0, exports.isObject)(x) && isFunction(x.abort) && isFunction(x.getWriter);
+exports.isWritableDOMStream = isWritableDOMStream;
+const isReadableDOMStream = (x) => (typeof ReadableStream !== 'undefined' && x instanceof ReadableStream) ||
+    ((0, exports.isObject)(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader));
+exports.isReadableDOMStream = isReadableDOMStream;
+// Not implemented in Firefox: && isFunction(x.pipeTo)
+const isWritableNodeStream = (x) => (0, exports.isObject)(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
+exports.isWritableNodeStream = isWritableNodeStream;
+const isReadableNodeStream = (x) => (0, exports.isObject)(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+exports.isReadableNodeStream = isReadableNodeStream;
+const isReadableStream = (x) => (0, exports.isReadableDOMStream)(x) || (0, exports.isReadableNodeStream)(x);
+exports.isReadableStream = isReadableStream;
+const isWritableStream = (x) => (0, exports.isWritableDOMStream)(x) || (0, exports.isWritableNodeStream)(x);
+exports.isWritableStream = isWritableStream;

package/dist/lib/api/encode.d.ts

@@ -0,0 +1,26 @@
+import type { Writer, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+export declare function encode(data: any, writer: Writer, options?: LoaderOptions): Promise<ArrayBuffer>;
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+export declare function encodeSync(data: any, writer: Writer, options?: LoaderOptions): ArrayBuffer;
+/**
+ * Encode loaded data to text using the specified Writer
+ * @note This is a convenience function not intended for production use on large input data.
+ * It is not optimized for performance. Data maybe converted from text to binary and back.
+ * @throws if the writer does not generate text output
+ */
+export declare function encodeText(data: any, writer: Writer, options?: LoaderOptions): Promise<string>;
+/**
+ * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
+ */
+export declare function encodeInBatches(data: any, writer: Writer, options?: LoaderOptions): AsyncIterable<ArrayBuffer>;
+/**
+ * Encode data stored in a file (on disk) to another file.
+ * @note Node.js only. This function enables using command-line converters as "writers".
+ */
+export declare function encodeURLtoURL(inputUrl: any, outputUrl: any, writer: Writer, options: any): Promise<string>;
+//# sourceMappingURL=encode.d.ts.map

package/dist/lib/api/encode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode.d.ts","sourceRoot":"","sources":["../../../src/lib/api/encode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE;;GAEG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,WAAW,CAAC,CA+CtB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,WAAW,CAK1F;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,GAAG,EACT,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,WAAW,CAAC,CAO5B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,QAAQ,KAAA,EACR,SAAS,KAAA,EACT,MAAM,EAAE,MAAM,EACd,OAAO,KAAA,GACN,OAAO,CAAC,MAAM,CAAC,CAQjB"}

package/dist/lib/api/encode.js

@@ -0,0 +1,111 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encodeURLtoURL = exports.encodeInBatches = exports.encodeText = exports.encodeSync = exports.encode = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const loader_utils_2 = require("@loaders.gl/loader-utils");
+const write_file_1 = require("../fetch/write-file");
+const fetch_file_1 = require("../fetch/fetch-file");
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+async function encode(data, writer, options) {
+    // TODO Merge default writer options with options argument like it is done in load module.
+    if (writer.encode) {
+        return await writer.encode(data, options);
+    }
+    if (writer.encodeSync) {
+        return writer.encodeSync(data, options);
+    }
+    if (writer.encodeText) {
+        return new TextEncoder().encode(await writer.encodeText(data, options));
+    }
+    if (writer.encodeInBatches) {
+        // Create an iterator representing the data
+        // TODO - Assumes this is a table
+        const batches = encodeInBatches(data, writer, options);
+        // Concatenate the output
+        const chunks = [];
+        for await (const batch of batches) {
+            chunks.push(batch);
+        }
+        // @ts-ignore
+        return (0, loader_utils_1.concatenateArrayBuffers)(...chunks);
+    }
+    if (!loader_utils_2.isBrowser && writer.encodeURLtoURL) {
+        // TODO - how to generate filenames with correct extensions?
+        const tmpInputFilename = getTemporaryFilename('input');
+        await (0, write_file_1.writeFile)(tmpInputFilename, data);
+        const tmpOutputFilename = getTemporaryFilename('output');
+        const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
+        const response = await (0, fetch_file_1.fetchFile)(outputFilename);
+        return response.arrayBuffer();
+    }
+    throw new Error('Writer could not encode data');
+}
+exports.encode = encode;
+/**
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
+ */
+function encodeSync(data, writer, options) {
+    if (writer.encodeSync) {
+        return writer.encodeSync(data, options);
+    }
+    throw new Error('Writer could not synchronously encode data');
+}
+exports.encodeSync = encodeSync;
+/**
+ * Encode loaded data to text using the specified Writer
+ * @note This is a convenience function not intended for production use on large input data.
+ * It is not optimized for performance. Data maybe converted from text to binary and back.
+ * @throws if the writer does not generate text output
+ */
+async function encodeText(data, writer, options) {
+    if (writer.text && writer.encodeText) {
+        return await writer.encodeText(data, options);
+    }
+    if (writer.text && (writer.encode || writer.encodeInBatches)) {
+        const arrayBuffer = await encode(data, writer, options);
+        return new TextDecoder().decode(arrayBuffer);
+    }
+    throw new Error('Writer could not encode data as text');
+}
+exports.encodeText = encodeText;
+/**
+ * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
+ */
+function encodeInBatches(data, writer, options) {
+    if (writer.encodeInBatches) {
+        const dataIterator = getIterator(data);
+        return writer.encodeInBatches(dataIterator, options);
+    }
+    // TODO -fall back to atomic encode?
+    throw new Error('Writer could not encode data in batches');
+}
+exports.encodeInBatches = encodeInBatches;
+/**
+ * Encode data stored in a file (on disk) to another file.
+ * @note Node.js only. This function enables using command-line converters as "writers".
+ */
+async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
+    inputUrl = (0, loader_utils_1.resolvePath)(inputUrl);
+    outputUrl = (0, loader_utils_1.resolvePath)(outputUrl);
+    if (loader_utils_2.isBrowser || !writer.encodeURLtoURL) {
+        throw new Error();
+    }
+    const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
+    return outputFilename;
+}
+exports.encodeURLtoURL = encodeURLtoURL;
+/**
+ * @todo TODO - this is an unacceptable hack!!!
+ */
+function getIterator(data) {
+    const dataIterator = [{ table: data, start: 0, end: data.length }];
+    return dataIterator;
+}
+/**
+ * @todo Move to utils
+ */
+function getTemporaryFilename(filename) {
+    return `/tmp/${filename}`;
+}

package/dist/lib/api/load-in-batches.d.ts

@@ -0,0 +1,13 @@
+import type { LoaderWithParser, LoaderOptions, LoaderContext } from '@loaders.gl/loader-utils';
+declare type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function loadInBatches(files: FileType, loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<AsyncIterable<any>>;
+export declare function loadInBatches(files: FileType[] | FileList, loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<AsyncIterable<any>>;
+export {};
+//# sourceMappingURL=load-in-batches.d.ts.map

package/dist/lib/api/load-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/api/load-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAM7F,aAAK,QAAQ,GAAG,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,GAAG,QAAQ,CAAC,EAAE,GAAG,QAAQ,CAAC;AAEjG;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,QAAQ,EACf,OAAO,CAAC,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,GAAG,aAAa,EAC/D,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/B,wBAAgB,aAAa,CAC3B,KAAK,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAC5B,OAAO,CAAC,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,GAAG,aAAa,EAC/D,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC"}

package/dist/lib/api/load-in-batches.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadInBatches = void 0;
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const parse_in_batches_1 = require("./parse-in-batches");
+function loadInBatches(files, loaders, options, context) {
+    // Signature: load(url, options)
+    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = null;
+    }
+    // Select fetch function
+    const fetch = (0, option_utils_1.getFetchFunction)(options || {});
+    // Single url/file
+    if (!Array.isArray(files)) {
+        return loadOneFileInBatches(files, loaders, options, fetch);
+    }
+    // Multiple URLs / files
+    const promises = files.map((file) => loadOneFileInBatches(file, loaders, options, fetch));
+    // No point in waiting here for all responses before starting to stream individual streams?
+    return promises;
+}
+exports.loadInBatches = loadInBatches;
+async function loadOneFileInBatches(file, loaders, options, fetch) {
+    if (typeof file === 'string') {
+        const url = file;
+        const response = await fetch(url);
+        return await (0, parse_in_batches_1.parseInBatches)(response, loaders, options);
+    }
+    return await (0, parse_in_batches_1.parseInBatches)(file, loaders, options);
+}

package/dist/lib/api/load.d.ts

@@ -0,0 +1,12 @@
+import type { DataType, Loader, LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Parses `data` using a specified loader
+ * Note: Load does duplicate a lot of parse.
+ * it can also call fetchFile on string urls, which `parse` won't do.
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function load(url: string | DataType, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<any>;
+//# sourceMappingURL=load.d.ts.map

package/dist/lib/api/load.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../../src/lib/api/load.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAO7F;;;;;;;;GAQG;AAEH,wBAAsB,IAAI,CACxB,GAAG,EAAE,MAAM,GAAG,QAAQ,EACtB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,GAAG,CAAC,CA0Bd"}

package/dist/lib/api/load.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.load = void 0;
+const is_type_1 = require("../../javascript-utils/is-type");
+const normalize_loader_1 = require("../loader-utils/normalize-loader");
+const option_utils_1 = require("../loader-utils/option-utils");
+const parse_1 = require("./parse");
+/**
+ * Parses `data` using a specified loader
+ * Note: Load does duplicate a lot of parse.
+ * it can also call fetchFile on string urls, which `parse` won't do.
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+// implementation signature
+async function load(url, loaders, options, context) {
+    // Signature: load(url, options)
+    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
+        context = undefined; // context not supported in short signature
+        options = loaders;
+        loaders = undefined;
+    }
+    // Select fetch function
+    const fetch = (0, option_utils_1.getFetchFunction)(options);
+    // at this point, `url` could be already loaded binary data
+    let data = url;
+    // url is a string, fetch the url
+    if (typeof url === 'string') {
+        data = await fetch(url);
+        // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
+    }
+    if ((0, is_type_1.isBlob)(url)) {
+        // The fetch response object will contain blob.name
+        data = await fetch(url);
+    }
+    // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
+    return await (0, parse_1.parse)(data, loaders, options);
+}
+exports.load = load;

package/dist/lib/api/parse-in-batches.d.ts

@@ -0,0 +1,10 @@
+import type { BatchableDataType, Loader, LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+export declare function parseInBatches(data: BatchableDataType, loaders?: Loader | Loader[] | LoaderOptions, options?: LoaderOptions, context?: LoaderContext): Promise<AsyncIterable<any>>;
+//# sourceMappingURL=parse-in-batches.d.ts.map

package/dist/lib/api/parse-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/api/parse-in-batches.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,iBAAiB,EACjB,MAAM,EAEN,aAAa,EACb,aAAa,EACd,MAAM,0BAA0B,CAAC;AAYlC;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,iBAAiB,EACvB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,aAAa,EAC3C,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAqC7B"}