@loaders.gl/core 4.0.0-alpha.23 → 4.0.0-alpha.24

package/dist/iterators/make-iterator/make-stream-iterator.js

@@ -1,96 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeStreamIterator = void 0;
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-/**
- * Returns an async iterable that reads from a stream (works in both Node.js and browsers)
- * @param stream stream to iterator over
- */
-function makeStreamIterator(stream, options) {
-    return loader_utils_1.isBrowser
-        ? makeBrowserStreamIterator(stream, options)
-        : makeNodeStreamIterator(stream, options);
-}
-exports.makeStreamIterator = makeStreamIterator;
-/**
- * Returns an async iterable that reads from a DOM (browser) stream
- * @param stream stream to iterate from
- * @see https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate
- */
-async function* makeBrowserStreamIterator(stream, options) {
-    // WhatWG: stream is supposed to have a `getIterator` method
-    // if (typeof stream.getIterator === 'function') {
-    //   return stream.getIterator();
-    // }
-    // if (typeof stream[Symbol.asyncIterator] === 'function') {
-    //   return makeToArrayBufferIterator(stream);
-    // }
-    // In the browser, we first need to get a lock on the stream
-    const reader = stream.getReader();
-    let nextBatchPromise;
-    try {
-        // eslint-disable-next-line no-constant-condition
-        while (true) {
-            const currentBatchPromise = nextBatchPromise || reader.read();
-            // Issue a read for an additional batch, while we await the next batch
-            // Idea is to make fetching happen in parallel with processing / parsing
-            if (options?._streamReadAhead) {
-                nextBatchPromise = reader.read();
-            }
-            // Read from the stream
-            // value is a Uint8Array
-            const { done, value } = await currentBatchPromise;
-            // Exit if we're done
-            if (done) {
-                return;
-            }
-            // Else yield the chunk
-            yield (0, loader_utils_1.toArrayBuffer)(value);
-        }
-    }
-    catch (error) {
-        // TODO - examples makes it look like this should always be called,
-        // but that generates exceptions so only call it if we do not reach the end
-        reader.releaseLock();
-    }
-}
-/**
- * Returns an async iterable that reads from a DOM (browser) stream
- * @param stream stream to iterate from
- * @note Requires Node.js >= 10
- */
-async function* makeNodeStreamIterator(stream, options) {
-    // Hacky test for node version to ensure we don't call bad polyfills
-    // NODE 10+: stream is an asyncIterator
-    for await (const chunk of stream) {
-        yield (0, loader_utils_1.toArrayBuffer)(chunk); // Coerce each chunk to ArrayBuffer
-    }
-}
-/* TODO - remove NODE < 10
- * @see https://github.com/bustle/streaming-iterables, MIT license
- *
-  if (typeof stream[Symbol.asyncIterator] === 'function') {
-    return;
-  }
-
-  // TODO - check if is this ever used in Node 10+?
-  // eslint-disable-next-line no-constant-condition
-  while (true) {
-    const data = stream.read();
-    if (data !== null) {
-      yield toArrayBuffer(data);
-      // eslint-disable-next-line no-continue
-      continue;
-    }
-    if (stream._readableState?.ended) {
-      return;
-    }
-    await onceReadable(stream);
-  }
-
-async function onceReadable(stream: Readable): Promise<any> {
-  return new Promise((resolve) => {
-    stream.once('readable', resolve);
-  });
-}
-  */

package/dist/iterators/make-iterator/make-string-iterator.js

@@ -1,24 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeStringIterator = void 0;
-const DEFAULT_CHUNK_SIZE = 256 * 1024;
-/**
- * Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers
- * @param blob string to iterate over
- * @param options
- * @param options.chunkSize
- */
-function* makeStringIterator(string, options) {
-    const chunkSize = options?.chunkSize || DEFAULT_CHUNK_SIZE;
-    let offset = 0;
-    const textEncoder = new TextEncoder();
-    while (offset < string.length) {
-        // Create a chunk of the right size
-        const chunkLength = Math.min(string.length - offset, chunkSize);
-        const chunk = string.slice(offset, offset + chunkLength);
-        offset += chunkLength;
-        // yield an ArrayBuffer chunk
-        yield textEncoder.encode(chunk);
-    }
-}
-exports.makeStringIterator = makeStringIterator;

package/dist/iterators/make-stream/make-dom-stream.js

@@ -1,47 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeStream = void 0;
-/**
- * Builds a DOM stream from an iterator
- * This stream is currently used in browsers only,
- * but note that Web stream support is present in Node from Node 16
- * https://nodejs.org/api/webstreams.html#webstreams_web_streams_api
- */
-function makeStream(source, options) {
-    // TODO - add AsyncGenerator to parameter types?
-    const iterator = source[Symbol.asyncIterator]
-        ? source[Symbol.asyncIterator]()
-        : source[Symbol.iterator]();
-    return new ReadableStream({
-        // Create a byte stream (enables `Response(stream).arrayBuffer()`)
-        // Only supported on Chrome
-        // See: https://developer.mozilla.org/en-US/docs/Web/API/ReadableByteStreamController
-        type: 'bytes',
-        async pull(controller) {
-            try {
-                const { done, value } = await iterator.next();
-                if (done) {
-                    controller.close();
-                }
-                else {
-                    // TODO - ignores controller.desiredSize
-                    // @ts-expect-error Unclear why value is not correctly typed
-                    controller.enqueue(new Uint8Array(value));
-                }
-            }
-            catch (error) {
-                controller.error(error);
-            }
-        },
-        async cancel() {
-            await iterator?.return?.();
-        }
-    }, 
-    // options: QueingStrategy<Uint8Array>
-    {
-        // This is bytes, not chunks
-        highWaterMark: 2 ** 24,
-        ...options
-    });
-}
-exports.makeStream = makeStream;

package/dist/iterators/make-stream/make-node-stream.js

@@ -1,85 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeStream = void 0;
-const Stream = __importStar(require("stream"));
-class _Readable {
-}
-const Readable = Stream.Readable || _Readable;
-/** Builds a node stream from an iterator */
-function makeStream(source, options) {
-    // @ts-ignore AsyncGenerator
-    const iterator = source[Symbol.asyncIterator]
-        ? // @ts-ignore AsyncGenerator
-            source[Symbol.asyncIterator]()
-        : // @ts-ignore AsyncGenerator
-            source[Symbol.iterator]();
-    return new AsyncIterableReadable(iterator, options);
-}
-exports.makeStream = makeStream;
-class AsyncIterableReadable extends Readable {
-    constructor(it, options) {
-        super(options);
-        this._iterator = it;
-        this._pulling = false;
-        this._bytesMode = !options || !options.objectMode;
-    }
-    async _read(size) {
-        if (!this._pulling) {
-            this._pulling = true;
-            this._pulling = await this._pull(size, this._iterator);
-        }
-    }
-    async _destroy(error, cb) {
-        if (!this._iterator) {
-            return;
-        }
-        if (error) {
-            await this._iterator?.throw?.(error);
-        }
-        else {
-            await this._iterator?.return?.(error);
-        }
-        cb?.(null);
-    }
-    // eslint-disable-next-line complexity
-    async _pull(size, it) {
-        const bm = this._bytesMode;
-        let r = null;
-        // while (this.readable && !(r = await it.next(bm ? size : null)).done) {
-        while (this.readable && !(r = await it.next()).done) {
-            if (size !== null) {
-                size -= bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1;
-            }
-            if (!this.push(new Uint8Array(r.value)) || size <= 0) {
-                break;
-            }
-        }
-        if ((r?.done || !this.readable) && (this.push(null) || true)) {
-            it?.return?.();
-        }
-        return !this.readable;
-    }
-}

package/dist/javascript-utils/is-type.js

@@ -1,41 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWritableStream = exports.isReadableStream = exports.isReadableNodeStream = exports.isWritableNodeStream = exports.isReadableDOMStream = exports.isWritableDOMStream = exports.isBuffer = exports.isBlob = exports.isFile = exports.isResponse = exports.isIterator = exports.isAsyncIterable = exports.isIterable = exports.isPromise = exports.isPureObject = exports.isObject = void 0;
-const isBoolean = (x) => typeof x === 'boolean';
-const isFunction = (x) => typeof x === 'function';
-const isObject = (x) => x !== null && typeof x === 'object';
-exports.isObject = isObject;
-const isPureObject = (x) => (0, exports.isObject)(x) && x.constructor === {}.constructor;
-exports.isPureObject = isPureObject;
-const isPromise = (x) => (0, exports.isObject)(x) && isFunction(x.then);
-exports.isPromise = isPromise;
-const isIterable = (x) => Boolean(x) && typeof x[Symbol.iterator] === 'function';
-exports.isIterable = isIterable;
-const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
-exports.isAsyncIterable = isAsyncIterable;
-const isIterator = (x) => x && isFunction(x.next);
-exports.isIterator = isIterator;
-const isResponse = (x) => (typeof Response !== 'undefined' && x instanceof Response) ||
-    (x && x.arrayBuffer && x.text && x.json);
-exports.isResponse = isResponse;
-const isFile = (x) => typeof File !== 'undefined' && x instanceof File;
-exports.isFile = isFile;
-const isBlob = (x) => typeof Blob !== 'undefined' && x instanceof Blob;
-exports.isBlob = isBlob;
-/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
-const isBuffer = (x) => x && typeof x === 'object' && x.isBuffer;
-exports.isBuffer = isBuffer;
-const isWritableDOMStream = (x) => (0, exports.isObject)(x) && isFunction(x.abort) && isFunction(x.getWriter);
-exports.isWritableDOMStream = isWritableDOMStream;
-const isReadableDOMStream = (x) => (typeof ReadableStream !== 'undefined' && x instanceof ReadableStream) ||
-    ((0, exports.isObject)(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader));
-exports.isReadableDOMStream = isReadableDOMStream;
-// Not implemented in Firefox: && isFunction(x.pipeTo)
-const isWritableNodeStream = (x) => (0, exports.isObject)(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
-exports.isWritableNodeStream = isWritableNodeStream;
-const isReadableNodeStream = (x) => (0, exports.isObject)(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
-exports.isReadableNodeStream = isReadableNodeStream;
-const isReadableStream = (x) => (0, exports.isReadableDOMStream)(x) || (0, exports.isReadableNodeStream)(x);
-exports.isReadableStream = isReadableStream;
-const isWritableStream = (x) => (0, exports.isWritableDOMStream)(x) || (0, exports.isWritableNodeStream)(x);
-exports.isWritableStream = isWritableStream;

package/dist/lib/api/encode-table.js

@@ -1,54 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-// Copyright 2022 Foursquare Labs, Inc
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encodeTableInBatches = exports.encodeTableAsText = exports.encodeTable = void 0;
-/* global TextEncoder, TextDecoder */
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-async function encodeTable(data, writer, options) {
-    if (writer.encode) {
-        return await writer.encode(data, options);
-    }
-    if (writer.encodeText) {
-        const text = await writer.encodeText(data, options);
-        return new TextEncoder().encode(text);
-    }
-    if (writer.encodeInBatches) {
-        // Create an iterator representing the data
-        // TODO - Assumes this is a table
-        const batches = encodeTableInBatches(data, writer, options);
-        // Concatenate the output
-        const chunks = [];
-        for await (const batch of batches) {
-            chunks.push(batch);
-        }
-        return (0, loader_utils_1.concatenateArrayBuffers)(...chunks);
-    }
-    throw new Error('Writer could not encode data');
-}
-exports.encodeTable = encodeTable;
-async function encodeTableAsText(data, writer, options) {
-    if (writer.text && writer.encodeText) {
-        return await writer.encodeText(data, options);
-    }
-    if (writer.text && (writer.encode || writer.encodeInBatches)) {
-        const arrayBuffer = await encodeTable(data, writer, options);
-        return new TextDecoder().decode(arrayBuffer);
-    }
-    throw new Error('Writer could not encode data as text');
-}
-exports.encodeTableAsText = encodeTableAsText;
-function encodeTableInBatches(data, writer, options) {
-    if (writer.encodeInBatches) {
-        const dataIterator = getIterator(data);
-        // @ts-expect-error
-        return writer.encodeInBatches(dataIterator, options);
-    }
-    // TODO -fall back to atomic encode?
-    throw new Error('Writer could not encode data in batches');
-}
-exports.encodeTableInBatches = encodeTableInBatches;
-function getIterator(data) {
-    const dataIterator = [{ table: data, start: 0, end: data.length }];
-    return dataIterator;
-}

package/dist/lib/api/encode.js

@@ -1,121 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encodeURLtoURL = exports.encodeInBatches = exports.encodeText = exports.encodeSync = exports.encode = void 0;
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const worker_utils_1 = require("@loaders.gl/worker-utils");
-const loader_utils_2 = require("@loaders.gl/loader-utils");
-const loader_utils_3 = require("@loaders.gl/loader-utils");
-const write_file_1 = require("../fetch/write-file");
-const fetch_file_1 = require("../fetch/fetch-file");
-const loader_options_1 = require("./loader-options");
-/**
- * Encode loaded data into a binary ArrayBuffer using the specified Writer.
- */
-async function encode(data, writer, options) {
-    const globalOptions = (0, loader_options_1.getLoaderOptions)();
-    // const globalOptions: WriterOptions = {}; // getWriterOptions();
-    options = { ...globalOptions, ...options };
-    if ((0, loader_utils_1.canEncodeWithWorker)(writer, options)) {
-        return await (0, worker_utils_1.processOnWorker)(writer, data, options);
-    }
-    // TODO Merge default writer options with options argument like it is done in load module.
-    if (writer.encode) {
-        return await writer.encode(data, options);
-    }
-    if (writer.encodeSync) {
-        return writer.encodeSync(data, options);
-    }
-    if (writer.encodeText) {
-        return new TextEncoder().encode(await writer.encodeText(data, options));
-    }
-    if (writer.encodeInBatches) {
-        // Create an iterator representing the data
-        // TODO - Assumes this is a table
-        const batches = encodeInBatches(data, writer, options);
-        // Concatenate the output
-        const chunks = [];
-        for await (const batch of batches) {
-            chunks.push(batch);
-        }
-        // @ts-ignore
-        return (0, loader_utils_2.concatenateArrayBuffers)(...chunks);
-    }
-    if (!loader_utils_3.isBrowser && writer.encodeURLtoURL) {
-        // TODO - how to generate filenames with correct extensions?
-        const tmpInputFilename = getTemporaryFilename('input');
-        await (0, write_file_1.writeFile)(tmpInputFilename, data);
-        const tmpOutputFilename = getTemporaryFilename('output');
-        const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
-        const response = await (0, fetch_file_1.fetchFile)(outputFilename);
-        return response.arrayBuffer();
-    }
-    throw new Error('Writer could not encode data');
-}
-exports.encode = encode;
-/**
- * Encode loaded data into a binary ArrayBuffer using the specified Writer.
- */
-function encodeSync(data, writer, options) {
-    if (writer.encodeSync) {
-        return writer.encodeSync(data, options);
-    }
-    throw new Error('Writer could not synchronously encode data');
-}
-exports.encodeSync = encodeSync;
-/**
- * Encode loaded data to text using the specified Writer
- * @note This is a convenience function not intended for production use on large input data.
- * It is not optimized for performance. Data maybe converted from text to binary and back.
- * @throws if the writer does not generate text output
- */
-async function encodeText(data, writer, options) {
-    if (writer.text && writer.encodeText) {
-        return await writer.encodeText(data, options);
-    }
-    if (writer.text && (writer.encode || writer.encodeInBatches)) {
-        const arrayBuffer = await encode(data, writer, options);
-        return new TextDecoder().decode(arrayBuffer);
-    }
-    throw new Error('Writer could not encode data as text');
-}
-exports.encodeText = encodeText;
-/**
- * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
- */
-function encodeInBatches(data, writer, options) {
-    if (writer.encodeInBatches) {
-        const dataIterator = getIterator(data);
-        // @ts-expect-error
-        return writer.encodeInBatches(dataIterator, options);
-    }
-    // TODO -fall back to atomic encode?
-    throw new Error('Writer could not encode data in batches');
-}
-exports.encodeInBatches = encodeInBatches;
-/**
- * Encode data stored in a file (on disk) to another file.
- * @note Node.js only. This function enables using command-line converters as "writers".
- */
-async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
-    inputUrl = (0, loader_utils_2.resolvePath)(inputUrl);
-    outputUrl = (0, loader_utils_2.resolvePath)(outputUrl);
-    if (loader_utils_3.isBrowser || !writer.encodeURLtoURL) {
-        throw new Error();
-    }
-    const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
-    return outputFilename;
-}
-exports.encodeURLtoURL = encodeURLtoURL;
-/**
- * @todo TODO - this is an unacceptable hack!!!
- */
-function getIterator(data) {
-    const dataIterator = [{ table: data, start: 0, end: data.length }];
-    return dataIterator;
-}
-/**
- * @todo Move to utils
- */
-function getTemporaryFilename(filename) {
-    return `/tmp/${filename}`;
-}

package/dist/lib/api/load-in-batches.js

@@ -1,40 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadInBatches = void 0;
-const normalize_loader_1 = require("../loader-utils/normalize-loader");
-const get_fetch_function_1 = require("../loader-utils/get-fetch-function");
-const parse_in_batches_1 = require("./parse-in-batches");
-function loadInBatches(files, loaders, options, context) {
-    let loadersArray;
-    // Signature: load(url, options)
-    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
-        context = undefined; // context not supported in short signature
-        options = loaders;
-        loadersArray = undefined;
-    }
-    else {
-        loadersArray = loaders;
-    }
-    // Select fetch function
-    const fetch = (0, get_fetch_function_1.getFetchFunction)(options || {});
-    // Single url/file
-    if (!Array.isArray(files)) {
-        return loadOneFileInBatches(files, loadersArray, options || {}, fetch);
-    }
-    // Multiple URLs / files
-    const promises = files.map((file) => loadOneFileInBatches(file, loadersArray, options || {}, fetch));
-    // No point in waiting here for all responses before starting to stream individual streams?
-    return promises;
-}
-exports.loadInBatches = loadInBatches;
-async function loadOneFileInBatches(file, loaders, options, fetch) {
-    if (typeof file === 'string') {
-        const url = file;
-        const response = await fetch(url);
-        // @ts-expect-error
-        return await (0, parse_in_batches_1.parseInBatches)(response, loaders, options);
-    }
-    // @ts-expect-error TODO
-    return await (0, parse_in_batches_1.parseInBatches)(file, loaders, options);
-}

package/dist/lib/api/load.js

@@ -1,43 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.load = void 0;
-const is_type_1 = require("../../javascript-utils/is-type");
-const normalize_loader_1 = require("../loader-utils/normalize-loader");
-const get_fetch_function_1 = require("../loader-utils/get-fetch-function");
-const parse_1 = require("./parse");
-// implementation signature
-async function load(url, loaders, options, context) {
-    let resolvedLoaders;
-    let resolvedOptions;
-    // Signature: load(url, options)
-    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
-        resolvedLoaders = [];
-        resolvedOptions = loaders;
-        context = undefined; // context not supported in short signature
-    }
-    else {
-        resolvedLoaders = loaders;
-        resolvedOptions = options;
-    }
-    // Select fetch function
-    const fetch = (0, get_fetch_function_1.getFetchFunction)(resolvedOptions);
-    // at this point, `url` could be already loaded binary data
-    let data = url;
-    // url is a string, fetch the url
-    if (typeof url === 'string') {
-        data = await fetch(url);
-        // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
-    }
-    if ((0, is_type_1.isBlob)(url)) {
-        // The fetch response object will contain blob.name
-        // @ts-expect-error TODO - This may not work for overridden fetch functions
-        data = await fetch(url);
-    }
-    // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
-    // return await parse(data, loaders as Loader[], options);
-    return Array.isArray(resolvedLoaders)
-        ? await (0, parse_1.parse)(data, resolvedLoaders, resolvedOptions) // loader array overload
-        : await (0, parse_1.parse)(data, resolvedLoaders, resolvedOptions); // single loader overload
-}
-exports.load = load;

package/dist/lib/api/loader-options.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getLoaderOptions = exports.setLoaderOptions = void 0;
-var option_utils_1 = require("../loader-utils/option-utils");
-Object.defineProperty(exports, "setLoaderOptions", { enumerable: true, get: function () { return option_utils_1.setGlobalOptions; } });
-var option_utils_2 = require("../loader-utils/option-utils");
-Object.defineProperty(exports, "getLoaderOptions", { enumerable: true, get: function () { return option_utils_2.getGlobalLoaderOptions; } });