@loaders.gl/core 3.1.0-alpha.2 → 3.1.0-beta.1

package/dist/es5/iterators/make-iterator/make-stream-iterator.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/iterators/make-iterator/make-stream-iterator.ts"],"names":["makeStreamIterator","stream","options","isBrowser","makeBrowserStreamIterator","makeNodeStreamIterator","reader","getReader","nextBatchPromise","currentBatchPromise","read","_streamReadAhead","done","value","error","releaseLock","chunk"],"mappings":";;;;;;;AACA;;AAUO,SAASA,kBAAT,CACLC,MADK,EAELC,OAFK,EAGuB;AAC5B,SAAOC,yBACHC,yBAAyB,CAACH,MAAD,EAA2BC,OAA3B,CADtB,GAEHG,sBAAsB,CAACJ,MAAD,EAAqBC,OAArB,CAF1B;AAGD;;AAOD,gBAAgBE,yBAAhB,CACEH,MADF,EAEEC,OAFF,EAG8B;AAU5B,QAAMI,MAAM,GAAGL,MAAM,CAACM,SAAP,EAAf;AAEA,MAAIC,gBAAJ;;AAEA,MAAI;AAEF,WAAO,IAAP,EAAa;AACX,YAAMC,mBAAmB,GAAGD,gBAAgB,IAAIF,MAAM,CAACI,IAAP,EAAhD;;AAGA,UAAIR,OAAJ,aAAIA,OAAJ,eAAIA,OAAO,CAAES,gBAAb,EAA+B;AAC7BH,QAAAA,gBAAgB,GAAGF,MAAM,CAACI,IAAP,EAAnB;AACD;;AAGD,YAAM;AAACE,QAAAA,IAAD;AAAOC,QAAAA;AAAP,UAAgB,MAAMJ,mBAA5B;;AAEA,UAAIG,IAAJ,EAAU;AACR;AACD;;AAED,YAAM,gCAAcC,KAAd,CAAN;AACD;AACF,GAnBD,CAmBE,OAAOC,KAAP,EAAc;AAGdR,IAAAA,MAAM,CAACS,WAAP;AACD;AACF;;AAOD,gBAAgBV,sBAAhB,CACEJ,MADF,EAEEC,OAFF,EAG8B;AAG5B,aAAW,MAAMc,KAAjB,IAA0Bf,MAA1B,EAAkC;AAChC,UAAM,gCAAce,KAAd,CAAN;AACD;AACF","sourcesContent":["import type {Readable} from 'stream';\nimport {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils';\n\nexport type StreamIteratorOptions = {\n  _streamReadAhead?: boolean;\n};\n\n/**\n * Returns an async iterable that reads from a stream (works in both Node.js and browsers)\n * @param stream stream to iterator over\n */\nexport function makeStreamIterator(\n  stream: ReadableStream | Readable,\n  options?: StreamIteratorOptions\n): AsyncIterable<ArrayBuffer> {\n  return isBrowser\n    ? makeBrowserStreamIterator(stream as ReadableStream, options)\n    : makeNodeStreamIterator(stream as Readable, options);\n}\n\n/**\n * Returns an async iterable that reads from a DOM (browser) stream\n * @param stream stream to iterate from\n * @see https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate\n */\nasync function* makeBrowserStreamIterator(\n  stream: ReadableStream,\n  options?: StreamIteratorOptions\n): AsyncIterable<ArrayBuffer> {\n  // WhatWG: stream is supposed to have a `getIterator` method\n  // if (typeof stream.getIterator === 'function') {\n  //   return stream.getIterator();\n  // }\n  // if (typeof stream[Symbol.asyncIterator] === 'function') {\n  //   return makeToArrayBufferIterator(stream);\n  // }\n\n  // In the browser, we first need to get a lock on the stream\n  const reader = stream.getReader();\n\n  let nextBatchPromise: Promise<{done?: boolean; value?: Uint8Array}> | undefined;\n\n  try {\n    // eslint-disable-next-line no-constant-condition\n    while (true) {\n      const currentBatchPromise = nextBatchPromise || reader.read();\n      // Issue a read for an additional batch, while we await the next batch\n      // Idea is to make fetching happen in parallel with processing / parsing\n      if (options?._streamReadAhead) {\n        nextBatchPromise = reader.read();\n      }\n      // Read from the stream\n      // value is a Uint8Array\n      const {done, value} = await currentBatchPromise;\n      // Exit if we're done\n      if (done) {\n        return;\n      }\n      // Else yield the chunk\n      yield toArrayBuffer(value);\n    }\n  } catch (error) {\n    // TODO - examples makes it look like this should always be called,\n    // but that generates exceptions so only call it if we do not reach the end\n    reader.releaseLock();\n  }\n}\n\n/**\n * Returns an async iterable that reads from a DOM (browser) stream\n * @param stream stream to iterate from\n * @note Requires Node.js >= 10\n */\nasync function* makeNodeStreamIterator(\n  stream: Readable,\n  options?: StreamIteratorOptions\n): AsyncIterable<ArrayBuffer> {\n  // Hacky test for node version to ensure we don't call bad polyfills\n  // NODE 10+: stream is an asyncIterator\n  for await (const chunk of stream) {\n    yield toArrayBuffer(chunk); // Coerce each chunk to ArrayBuffer\n  }\n}\n/* TODO - remove NODE < 10\n * @see https://github.com/bustle/streaming-iterables, MIT license\n *\n  if (typeof stream[Symbol.asyncIterator] === 'function') {\n    return;\n  }\n\n  // TODO - check if is this ever used in Node 10+?\n  // eslint-disable-next-line no-constant-condition\n  while (true) {\n    const data = stream.read();\n    if (data !== null) {\n      yield toArrayBuffer(data);\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n    if (stream._readableState?.ended) {\n      return;\n    }\n    await onceReadable(stream);\n  }\n\nasync function onceReadable(stream: Readable): Promise<any> {\n  return new Promise((resolve) => {\n    stream.once('readable', resolve);\n  });\n}\n  */\n"],"file":"make-stream-iterator.js"}

package/dist/es5/iterators/make-iterator/make-string-iterator.js

@@ -0,0 +1,21 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.makeStringIterator = makeStringIterator;
+const DEFAULT_CHUNK_SIZE = 256 * 1024;
+
+function* makeStringIterator(string, options) {
+  const chunkSize = (options === null || options === void 0 ? void 0 : options.chunkSize) || DEFAULT_CHUNK_SIZE;
+  let offset = 0;
+  const textEncoder = new TextEncoder();
+
+  while (offset < string.length) {
+    const chunkLength = Math.min(string.length - offset, chunkSize);
+    const chunk = string.slice(offset, offset + chunkLength);
+    offset += chunkLength;
+    yield textEncoder.encode(chunk);
+  }
+}
+//# sourceMappingURL=make-string-iterator.js.map

package/dist/es5/iterators/make-iterator/make-string-iterator.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/iterators/make-iterator/make-string-iterator.ts"],"names":["DEFAULT_CHUNK_SIZE","makeStringIterator","string","options","chunkSize","offset","textEncoder","TextEncoder","length","chunkLength","Math","min","chunk","slice","encode"],"mappings":";;;;;;AAEA,MAAMA,kBAAkB,GAAG,MAAM,IAAjC;;AAQO,UAAUC,kBAAV,CACLC,MADK,EAELC,OAFK,EAGkB;AACvB,QAAMC,SAAS,GAAG,CAAAD,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEC,SAAT,KAAsBJ,kBAAxC;AAEA,MAAIK,MAAM,GAAG,CAAb;AACA,QAAMC,WAAW,GAAG,IAAIC,WAAJ,EAApB;;AACA,SAAOF,MAAM,GAAGH,MAAM,CAACM,MAAvB,EAA+B;AAE7B,UAAMC,WAAW,GAAGC,IAAI,CAACC,GAAL,CAAST,MAAM,CAACM,MAAP,GAAgBH,MAAzB,EAAiCD,SAAjC,CAApB;AACA,UAAMQ,KAAK,GAAGV,MAAM,CAACW,KAAP,CAAaR,MAAb,EAAqBA,MAAM,GAAGI,WAA9B,CAAd;AACAJ,IAAAA,MAAM,IAAII,WAAV;AAGA,UAAMH,WAAW,CAACQ,MAAZ,CAAmBF,KAAnB,CAAN;AACD;AACF","sourcesContent":["import type {IteratorOptions} from './make-iterator';\n\nconst DEFAULT_CHUNK_SIZE = 256 * 1024;\n\n/**\n * Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers\n * @param blob string to iterate over\n * @param options\n * @param options.chunkSize\n */\nexport function* makeStringIterator(\n  string: string,\n  options?: IteratorOptions\n): Iterable<ArrayBuffer> {\n  const chunkSize = options?.chunkSize || DEFAULT_CHUNK_SIZE;\n\n  let offset = 0;\n  const textEncoder = new TextEncoder();\n  while (offset < string.length) {\n    // Create a chunk of the right size\n    const chunkLength = Math.min(string.length - offset, chunkSize);\n    const chunk = string.slice(offset, offset + chunkLength);\n    offset += chunkLength;\n\n    // yield an ArrayBuffer chunk\n    yield textEncoder.encode(chunk);\n  }\n}\n"],"file":"make-string-iterator.js"}

package/dist/es5/iterators/make-stream/make-dom-stream.js

@@ -0,0 +1,41 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.makeDOMStream = makeDOMStream;
+
+function makeDOMStream(source, options) {
+  const iterator = source[Symbol.asyncIterator] ? source[Symbol.asyncIterator]() : source[Symbol.iterator]();
+  return new ReadableStream({
+    type: 'bytes',
+
+    async pull(controller) {
+      try {
+        const {
+          done,
+          value
+        } = await iterator.next();
+
+        if (done) {
+          controller.close();
+        } else {
+          controller.enqueue(new Uint8Array(value));
+        }
+      } catch (error) {
+        controller.error(error);
+      }
+    },
+
+    async cancel() {
+      var _iterator$return;
+
+      await (iterator === null || iterator === void 0 ? void 0 : (_iterator$return = iterator.return) === null || _iterator$return === void 0 ? void 0 : _iterator$return.call(iterator));
+    }
+
+  }, {
+    highWaterMark: 2 ** 24,
+    ...options
+  });
+}
+//# sourceMappingURL=make-dom-stream.js.map

package/dist/es5/iterators/make-stream/make-dom-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/iterators/make-stream/make-dom-stream.ts"],"names":["makeDOMStream","source","options","iterator","Symbol","asyncIterator","ReadableStream","type","pull","controller","done","value","next","close","enqueue","Uint8Array","error","cancel","return","highWaterMark"],"mappings":";;;;;;;AAQO,SAASA,aAAT,CACLC,MADK,EAELC,OAFK,EAGW;AAChB,QAAMC,QAAQ,GAAGF,MAAM,CAACG,MAAM,CAACC,aAAR,CAAN,GACZJ,MAAD,CAAuCG,MAAM,CAACC,aAA9C,GADa,GAEZJ,MAAD,CAAkCG,MAAM,CAACD,QAAzC,GAFJ;AAIA,SAAO,IAAIG,cAAJ,CACL;AAKEC,IAAAA,IAAI,EAAE,OALR;;AAOE,UAAMC,IAAN,CAAWC,UAAX,EAAuB;AACrB,UAAI;AACF,cAAM;AAACC,UAAAA,IAAD;AAAOC,UAAAA;AAAP,YAAgB,MAAMR,QAAQ,CAACS,IAAT,EAA5B;;AACA,YAAIF,IAAJ,EAAU;AACRD,UAAAA,UAAU,CAACI,KAAX;AACD,SAFD,MAEO;AAELJ,UAAAA,UAAU,CAACK,OAAX,CAAmB,IAAIC,UAAJ,CAAeJ,KAAf,CAAnB;AACD;AACF,OARD,CAQE,OAAOK,KAAP,EAAc;AACdP,QAAAA,UAAU,CAACO,KAAX,CAAiBA,KAAjB;AACD;AACF,KAnBH;;AAqBE,UAAMC,MAAN,GAAe;AAAA;;AACb,aAAMd,QAAN,aAAMA,QAAN,2CAAMA,QAAQ,CAAEe,MAAhB,qDAAM,sBAAAf,QAAQ,CAAd;AACD;;AAvBH,GADK,EA2BL;AAEEgB,IAAAA,aAAa,EAAE,KAAK,EAFtB;AAGE,OAAGjB;AAHL,GA3BK,CAAP;AAiCD","sourcesContent":["export type MakeDOMStreamOptions = {\n  /** Stream allocates an arrayBuffer. Enables use of a default reader. */\n  autoAllocateChunkSize?: number;\n  /** Total number of chunks in queue before back pressure is applied */\n  highWaterMark?: number;\n};\n\n/** Builds a DOM stream from an iterator */\nexport function makeDOMStream<ArrayBuffer>(\n  source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,\n  options?: MakeDOMStreamOptions\n): ReadableStream {\n  const iterator = source[Symbol.asyncIterator]\n    ? (source as AsyncIterable<ArrayBuffer>)[Symbol.asyncIterator]()\n    : (source as Iterable<ArrayBuffer>)[Symbol.iterator]();\n\n  return new ReadableStream<Uint8Array>(\n    {\n      // Create a byte stream (enables `Response(stream).arrayBuffer()`)\n      // Only supported on Chrome\n      // See: https://developer.mozilla.org/en-US/docs/Web/API/ReadableByteStreamController\n      // @ts-expect-error Only Chrome supports byte streams\n      type: 'bytes',\n\n      async pull(controller) {\n        try {\n          const {done, value} = await iterator.next();\n          if (done) {\n            controller.close();\n          } else {\n            // TODO - ignores controller.desiredSize\n            controller.enqueue(new Uint8Array(value));\n          }\n        } catch (error) {\n          controller.error(error);\n        }\n      },\n\n      async cancel() {\n        await iterator?.return?.();\n      }\n    },\n    // options: QueingStrategy<Uint8Array>\n    {\n      // This is bytes, not chunks\n      highWaterMark: 2 ** 24,\n      ...options\n    }\n  );\n}\n"],"file":"make-dom-stream.js"}

package/dist/es5/iterators/make-stream/make-node-stream.js

@@ -0,0 +1,84 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _stream = require("stream");
+
+function makeNodeStream(source, options) {
+  const iterator = source[Symbol.asyncIterator] ? source[Symbol.asyncIterator]() : source[Symbol.iterator]();
+  return new AsyncIterableReadable(iterator, options);
+}
+
+class AsyncIterableReadable extends _stream.Readable {
+  constructor(it, options) {
+    super(options);
+    (0, _defineProperty2.default)(this, "_pulling", void 0);
+    (0, _defineProperty2.default)(this, "_bytesMode", void 0);
+    (0, _defineProperty2.default)(this, "_iterator", void 0);
+    this._iterator = it;
+    this._pulling = false;
+    this._bytesMode = !options || !options.objectMode;
+  }
+
+  async _read(size) {
+    if (!this._pulling) {
+      this._pulling = true;
+      this._pulling = await this._pull(size, this._iterator);
+    }
+  }
+
+  async _destroy(error, cb) {
+    if (!this._iterator) {
+      return;
+    }
+
+    if (error) {
+      var _this$_iterator, _this$_iterator$throw;
+
+      await ((_this$_iterator = this._iterator) === null || _this$_iterator === void 0 ? void 0 : (_this$_iterator$throw = _this$_iterator.throw) === null || _this$_iterator$throw === void 0 ? void 0 : _this$_iterator$throw.call(_this$_iterator, error));
+    } else {
+      var _this$_iterator2, _this$_iterator2$retu;
+
+      await ((_this$_iterator2 = this._iterator) === null || _this$_iterator2 === void 0 ? void 0 : (_this$_iterator2$retu = _this$_iterator2.return) === null || _this$_iterator2$retu === void 0 ? void 0 : _this$_iterator2$retu.call(_this$_iterator2, error));
+    }
+
+    cb === null || cb === void 0 ? void 0 : cb(null);
+  }
+
+  async _pull(size, it) {
+    var _r;
+
+    const bm = this._bytesMode;
+    let r = null;
+
+    while (this.readable && !(r = await it.next()).done) {
+      if (size !== null) {
+        size -= bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1;
+      }
+
+      if (!this.push(new Uint8Array(r.value)) || size <= 0) {
+        break;
+      }
+    }
+
+    if (((_r = r) !== null && _r !== void 0 && _r.done || !this.readable) && (this.push(null) || true)) {
+      var _it$return;
+
+      it === null || it === void 0 ? void 0 : (_it$return = it.return) === null || _it$return === void 0 ? void 0 : _it$return.call(it);
+    }
+
+    return !this.readable;
+  }
+
+}
+
+var _default = makeNodeStream;
+exports.default = _default;
+//# sourceMappingURL=make-node-stream.js.map

package/dist/es5/iterators/make-stream/make-node-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/iterators/make-stream/make-node-stream.ts"],"names":["makeNodeStream","source","options","iterator","Symbol","asyncIterator","AsyncIterableReadable","Readable","constructor","it","_iterator","_pulling","_bytesMode","objectMode","_read","size","_pull","_destroy","error","cb","throw","return","bm","r","readable","next","done","ArrayBuffer","isView","value","byteLength","push","Uint8Array"],"mappings":";;;;;;;;;;;AAAA;;AAKA,SAASA,cAAT,CACEC,MADF,EAEEC,OAFF,EAGY;AACV,QAAMC,QAAQ,GAAGF,MAAM,CAACG,MAAM,CAACC,aAAR,CAAN,GACbJ,MAAM,CAACG,MAAM,CAACC,aAAR,CAAN,EADa,GAEbJ,MAAM,CAACG,MAAM,CAACD,QAAR,CAAN,EAFJ;AAGA,SAAO,IAAIG,qBAAJ,CAA0BH,QAA1B,EAAoCD,OAApC,CAAP;AACD;;AAED,MAAMI,qBAAN,SAAoCC,gBAApC,CAA6C;AAK3CC,EAAAA,WAAW,CAACC,EAAD,EAAiCP,OAAjC,EAA4D;AACrE,UAAMA,OAAN;AADqE;AAAA;AAAA;AAErE,SAAKQ,SAAL,GAAiBD,EAAjB;AACA,SAAKE,QAAL,GAAgB,KAAhB;AACA,SAAKC,UAAL,GAAkB,CAACV,OAAD,IAAY,CAACA,OAAO,CAACW,UAAvC;AACD;;AAEU,QAALC,KAAK,CAACC,IAAD,EAA8B;AACvC,QAAI,CAAC,KAAKJ,QAAV,EAAoB;AAClB,WAAKA,QAAL,GAAgB,IAAhB;AACA,WAAKA,QAAL,GAAgB,MAAM,KAAKK,KAAL,CAAWD,IAAX,EAAiB,KAAKL,SAAtB,CAAtB;AACD;AACF;;AAEa,QAARO,QAAQ,CAACC,KAAD,EAAsBC,EAAtB,EAAoE;AAChF,QAAI,CAAC,KAAKT,SAAV,EAAqB;AACnB;AACD;;AACD,QAAIQ,KAAJ,EAAW;AAAA;;AACT,gCAAM,KAAKR,SAAX,6EAAM,gBAAgBU,KAAtB,0DAAM,4CAAwBF,KAAxB,CAAN;AACD,KAFD,MAEO;AAAA;;AACL,iCAAM,KAAKR,SAAX,8EAAM,iBAAgBW,MAAtB,0DAAM,6CAAyBH,KAAzB,CAAN;AACD;;AACDC,IAAAA,EAAE,SAAF,IAAAA,EAAE,WAAF,YAAAA,EAAE,CAAG,IAAH,CAAF;AACD;;AAGkB,QAALH,KAAK,CAACD,IAAD,EAAeN,EAAf,EAAiE;AAAA;;AAClF,UAAMa,EAAE,GAAG,KAAKV,UAAhB;AACA,QAAIW,CAAqC,GAAG,IAA5C;;AAEA,WAAO,KAAKC,QAAL,IAAiB,CAAC,CAACD,CAAC,GAAG,MAAMd,EAAE,CAACgB,IAAH,EAAX,EAAsBC,IAA/C,EAAqD;AACnD,UAAIX,IAAI,KAAK,IAAb,EAAmB;AACjBA,QAAAA,IAAI,IAAIO,EAAE,IAAIK,WAAW,CAACC,MAAZ,CAAmBL,CAAC,CAACM,KAArB,CAAN,GAAoCN,CAAC,CAACM,KAAF,CAAQC,UAA5C,GAAyD,CAAjE;AACD;;AACD,UAAI,CAAC,KAAKC,IAAL,CAAU,IAAIC,UAAJ,CAAeT,CAAC,CAACM,KAAjB,CAAV,CAAD,IAAuCd,IAAI,IAAI,CAAnD,EAAsD;AACpD;AACD;AACF;;AACD,QAAI,CAAC,MAAAQ,CAAC,UAAD,wBAAGG,IAAH,IAAW,CAAC,KAAKF,QAAlB,MAAgC,KAAKO,IAAL,CAAU,IAAV,KAAmB,IAAnD,CAAJ,EAA8D;AAAA;;AAC5DtB,MAAAA,EAAE,SAAF,IAAAA,EAAE,WAAF,0BAAAA,EAAE,CAAEY,MAAJ,+DAAAZ,EAAE;AACH;;AACD,WAAO,CAAC,KAAKe,QAAb;AACD;;AAhD0C;;eAsD9BxB,c","sourcesContent":["import {Readable, ReadableOptions} from 'stream';\n\nexport type MakeNodeStreamOptions = ReadableOptions;\n\n/** Builds a node stream from an iterator */\nfunction makeNodeStream<ArrayBuffer>(\n  source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,\n  options?: ReadableOptions\n): Readable {\n  const iterator = source[Symbol.asyncIterator]\n    ? source[Symbol.asyncIterator]()\n    : source[Symbol.iterator]();\n  return new AsyncIterableReadable(iterator, options);\n}\n\nclass AsyncIterableReadable extends Readable {\n  private _pulling: boolean;\n  private _bytesMode: boolean;\n  private _iterator: AsyncIterator<ArrayBuffer>;\n\n  constructor(it: AsyncIterator<ArrayBuffer>, options?: ReadableOptions) {\n    super(options);\n    this._iterator = it;\n    this._pulling = false;\n    this._bytesMode = !options || !options.objectMode;\n  }\n\n  async _read(size: number): Promise<void> {\n    if (!this._pulling) {\n      this._pulling = true;\n      this._pulling = await this._pull(size, this._iterator);\n    }\n  }\n\n  async _destroy(error: Error | null, cb: (e: Error | null) => void): Promise<void> {\n    if (!this._iterator) {\n      return;\n    }\n    if (error) {\n      await this._iterator?.throw?.(error);\n    } else {\n      await this._iterator?.return?.(error);\n    }\n    cb?.(null);\n  }\n\n  // eslint-disable-next-line complexity\n  private async _pull(size: number, it: AsyncIterator<ArrayBuffer>): Promise<boolean> {\n    const bm = this._bytesMode;\n    let r: IteratorResult<ArrayBuffer> | null = null;\n    // while (this.readable && !(r = await it.next(bm ? size : null)).done) {\n    while (this.readable && !(r = await it.next()).done) {\n      if (size !== null) {\n        size -= bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1;\n      }\n      if (!this.push(new Uint8Array(r.value)) || size <= 0) {\n        break;\n      }\n    }\n    if ((r?.done || !this.readable) && (this.push(null) || true)) {\n      it?.return?.();\n    }\n    return !this.readable;\n  }\n}\n\n// This module is marked `false` in the the \"browser\" field of the `package.json` for\n// `@loaders.gl/core`. We avoid using named exports so that bundlers have an easier\n// time resolving this \"empty\" module.\nexport default makeNodeStream;\n"],"file":"make-node-stream.js"}

package/dist/es5/iterators/make-stream/make-stream.js

@@ -0,0 +1,19 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.makeStream = makeStream;
+
+var _loaderUtils = require("@loaders.gl/loader-utils");
+
+var _makeDomStream = require("./make-dom-stream");
+
+var _makeNodeStream = _interopRequireDefault(require("./make-node-stream"));
+
+function makeStream(data, options) {
+  return _loaderUtils.isBrowser ? (0, _makeDomStream.makeDOMStream)(data, options) : (0, _makeNodeStream.default)(data, options);
+}
+//# sourceMappingURL=make-stream.js.map

package/dist/es5/iterators/make-stream/make-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/iterators/make-stream/make-stream.ts"],"names":["makeStream","data","options","isBrowser"],"mappings":";;;;;;;;;AAIA;;AACA;;AACA;;AAOO,SAASA,UAAT,CACLC,IADK,EAELC,OAFK,EAGe;AACpB,SAAOC,yBAAY,kCAAcF,IAAd,EAAoBC,OAApB,CAAZ,GAA2C,6BAAeD,IAAf,EAAqBC,OAArB,CAAlD;AACD","sourcesContent":["import type {ReadableStreamType} from '../../javascript-utils/is-type';\nimport type {MakeDOMStreamOptions} from './make-dom-stream';\nimport type {MakeNodeStreamOptions} from './make-node-stream';\n\nimport {isBrowser} from '@loaders.gl/loader-utils';\nimport {makeDOMStream} from './make-dom-stream';\nimport makeNodeStream from './make-node-stream';\n\nexport type MakeStreamOptions = MakeDOMStreamOptions | MakeNodeStreamOptions;\n\n/**\n * Returns a stream for an (async) iterator (works in both Node.js and browsers)\n */\nexport function makeStream(\n  data: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,\n  options?: MakeStreamOptions\n): ReadableStreamType {\n  return isBrowser ? makeDOMStream(data, options) : makeNodeStream(data, options);\n}\n"],"file":"make-stream.js"}

package/dist/es5/javascript-utils/is-type.js

@@ -0,0 +1,75 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.isWritableStream = exports.isReadableStream = exports.isReadableNodeStream = exports.isWritableNodeStream = exports.isReadableDOMStream = exports.isWritableDOMStream = exports.isBuffer = exports.isBlob = exports.isFile = exports.isResponse = exports.isIterator = exports.isAsyncIterable = exports.isIterable = exports.isPromise = exports.isPureObject = exports.isObject = void 0;
+
+const isBoolean = x => typeof x === 'boolean';
+
+const isFunction = x => typeof x === 'function';
+
+const isObject = x => x !== null && typeof x === 'object';
+
+exports.isObject = isObject;
+
+const isPureObject = x => isObject(x) && x.constructor === {}.constructor;
+
+exports.isPureObject = isPureObject;
+
+const isPromise = x => isObject(x) && isFunction(x.then);
+
+exports.isPromise = isPromise;
+
+const isIterable = x => x && typeof x[Symbol.iterator] === 'function';
+
+exports.isIterable = isIterable;
+
+const isAsyncIterable = x => x && typeof x[Symbol.asyncIterator] === 'function';
+
+exports.isAsyncIterable = isAsyncIterable;
+
+const isIterator = x => x && isFunction(x.next);
+
+exports.isIterator = isIterator;
+
+const isResponse = x => typeof Response !== 'undefined' && x instanceof Response || x && x.arrayBuffer && x.text && x.json;
+
+exports.isResponse = isResponse;
+
+const isFile = x => typeof File !== 'undefined' && x instanceof File;
+
+exports.isFile = isFile;
+
+const isBlob = x => typeof Blob !== 'undefined' && x instanceof Blob;
+
+exports.isBlob = isBlob;
+
+const isBuffer = x => x && typeof x === 'object' && x.isBuffer;
+
+exports.isBuffer = isBuffer;
+
+const isWritableDOMStream = x => isObject(x) && isFunction(x.abort) && isFunction(x.getWriter);
+
+exports.isWritableDOMStream = isWritableDOMStream;
+
+const isReadableDOMStream = x => typeof ReadableStream !== 'undefined' && x instanceof ReadableStream || isObject(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader);
+
+exports.isReadableDOMStream = isReadableDOMStream;
+
+const isWritableNodeStream = x => isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
+
+exports.isWritableNodeStream = isWritableNodeStream;
+
+const isReadableNodeStream = x => isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+
+exports.isReadableNodeStream = isReadableNodeStream;
+
+const isReadableStream = x => isReadableDOMStream(x) || isReadableNodeStream(x);
+
+exports.isReadableStream = isReadableStream;
+
+const isWritableStream = x => isWritableDOMStream(x) || isWritableNodeStream(x);
+
+exports.isWritableStream = isWritableStream;
+//# sourceMappingURL=is-type.js.map

package/dist/es5/javascript-utils/is-type.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/javascript-utils/is-type.ts"],"names":["isBoolean","x","isFunction","isObject","isPureObject","constructor","isPromise","then","isIterable","Symbol","iterator","isAsyncIterable","asyncIterator","isIterator","next","isResponse","Response","arrayBuffer","text","json","isFile","File","isBlob","Blob","isBuffer","isWritableDOMStream","abort","getWriter","isReadableDOMStream","ReadableStream","tee","cancel","getReader","isWritableNodeStream","end","write","writable","isReadableNodeStream","read","pipe","readable","isReadableStream","isWritableStream"],"mappings":";;;;;;;AAKA,MAAMA,SAA8B,GAAIC,CAAD,IAAO,OAAOA,CAAP,KAAa,SAA3D;;AACA,MAAMC,UAA+B,GAAID,CAAD,IAAO,OAAOA,CAAP,KAAa,UAA5D;;AAEO,MAAME,QAA6B,GAAIF,CAAD,IAAOA,CAAC,KAAK,IAAN,IAAc,OAAOA,CAAP,KAAa,QAAxE;;;;AACA,MAAMG,YAAiC,GAAIH,CAAD,IAC/CE,QAAQ,CAACF,CAAD,CAAR,IAAeA,CAAC,CAACI,WAAF,KAAkB,GAAGA,WAD/B;;;;AAEA,MAAMC,SAA8B,GAAIL,CAAD,IAAOE,QAAQ,CAACF,CAAD,CAAR,IAAeC,UAAU,CAACD,CAAC,CAACM,IAAH,CAAvE;;;;AAEA,MAAMC,UAA+B,GAAIP,CAAD,IAAOA,CAAC,IAAI,OAAOA,CAAC,CAACQ,MAAM,CAACC,QAAR,CAAR,KAA8B,UAAlF;;;;AACA,MAAMC,eAAoC,GAAIV,CAAD,IAClDA,CAAC,IAAI,OAAOA,CAAC,CAACQ,MAAM,CAACG,aAAR,CAAR,KAAmC,UADnC;;;;AAEA,MAAMC,UAA+B,GAAIZ,CAAD,IAAOA,CAAC,IAAIC,UAAU,CAACD,CAAC,CAACa,IAAH,CAA9D;;;;AAEA,MAAMC,UAA+B,GAAId,CAAD,IAC5C,OAAOe,QAAP,KAAoB,WAApB,IAAmCf,CAAC,YAAYe,QAAjD,IACCf,CAAC,IAAIA,CAAC,CAACgB,WAAP,IAAsBhB,CAAC,CAACiB,IAAxB,IAAgCjB,CAAC,CAACkB,IAF9B;;;;AAIA,MAAMC,MAA2B,GAAInB,CAAD,IAAO,OAAOoB,IAAP,KAAgB,WAAhB,IAA+BpB,CAAC,YAAYoB,IAAvF;;;;AACA,MAAMC,MAA2B,GAAIrB,CAAD,IAAO,OAAOsB,IAAP,KAAgB,WAAhB,IAA+BtB,CAAC,YAAYsB,IAAvF;;;;AAGA,MAAMC,QAA6B,GAAIvB,CAAD,IAAOA,CAAC,IAAI,OAAOA,CAAP,KAAa,QAAlB,IAA8BA,CAAC,CAACuB,QAA7E;;;;AAEA,MAAMC,mBAAwC,GAAIxB,CAAD,IACtDE,QAAQ,CAACF,CAAD,CAAR,IAAeC,UAAU,CAACD,CAAC,CAACyB,KAAH,CAAzB,IAAsCxB,UAAU,CAACD,CAAC,CAAC0B,SAAH,CAD3C;;;;AAGA,MAAMC,mBAAwC,GAAI3B,CAAD,IACrD,OAAO4B,cAAP,KAA0B,WAA1B,IAAyC5B,CAAC,YAAY4B,cAAvD,IACC1B,QAAQ,CAACF,CAAD,CAAR,IAAeC,UAAU,CAACD,CAAC,CAAC6B,GAAH,CAAzB,IAAoC5B,UAAU,CAACD,CAAC,CAAC8B,MAAH,CAA9C,IAA4D7B,UAAU,CAACD,CAAC,CAAC+B,SAAH,CAFlE;;;;AAKA,MAAMC,oBAAyC,GAAIhC,CAAD,IACvDE,QAAQ,CAACF,CAAD,CAAR,IAAeC,UAAU,CAACD,CAAC,CAACiC,GAAH,CAAzB,IAAoChC,UAAU,CAACD,CAAC,CAACkC,KAAH,CAA9C,IAA2DnC,SAAS,CAACC,CAAC,CAACmC,QAAH,CAD/D;;;;AAEA,MAAMC,oBAAyC,GAAIpC,CAAD,IACvDE,QAAQ,CAACF,CAAD,CAAR,IAAeC,UAAU,CAACD,CAAC,CAACqC,IAAH,CAAzB,IAAqCpC,UAAU,CAACD,CAAC,CAACsC,IAAH,CAA/C,IAA2DvC,SAAS,CAACC,CAAC,CAACuC,QAAH,CAD/D;;;;AAEA,MAAMC,gBAAqC,GAAIxC,CAAD,IACnD2B,mBAAmB,CAAC3B,CAAD,CAAnB,IAA0BoC,oBAAoB,CAACpC,CAAD,CADzC;;;;AAEA,MAAMyC,gBAAqC,GAAIzC,CAAD,IACnDwB,mBAAmB,CAACxB,CAAD,CAAnB,IAA0BgC,oBAAoB,CAAChC,CAAD,CADzC","sourcesContent":["import type {Readable} from 'stream';\n\n/** A DOM or Node readable stream */\nexport type ReadableStreamType = ReadableStream | Readable;\n\nconst isBoolean: (x: any) => boolean = (x) => typeof x === 'boolean';\nconst isFunction: (x: any) => boolean = (x) => typeof x === 'function';\n\nexport const isObject: (x: any) => boolean = (x) => x !== null && typeof x === 'object';\nexport const isPureObject: (x: any) => boolean = (x) =>\n  isObject(x) && x.constructor === {}.constructor;\nexport const isPromise: (x: any) => boolean = (x) => isObject(x) && isFunction(x.then);\n\nexport const isIterable: (x: any) => boolean = (x) => x && typeof x[Symbol.iterator] === 'function';\nexport const isAsyncIterable: (x: any) => boolean = (x) =>\n  x && typeof x[Symbol.asyncIterator] === 'function';\nexport const isIterator: (x: any) => boolean = (x) => x && isFunction(x.next);\n\nexport const isResponse: (x: any) => boolean = (x) =>\n  (typeof Response !== 'undefined' && x instanceof Response) ||\n  (x && x.arrayBuffer && x.text && x.json);\n\nexport const isFile: (x: any) => boolean = (x) => typeof File !== 'undefined' && x instanceof File;\nexport const isBlob: (x: any) => boolean = (x) => typeof Blob !== 'undefined' && x instanceof Blob;\n\n/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */\nexport const isBuffer: (x: any) => boolean = (x) => x && typeof x === 'object' && x.isBuffer;\n\nexport const isWritableDOMStream: (x: any) => boolean = (x) =>\n  isObject(x) && isFunction(x.abort) && isFunction(x.getWriter);\n\nexport const isReadableDOMStream: (x: any) => boolean = (x) =>\n  (typeof ReadableStream !== 'undefined' && x instanceof ReadableStream) ||\n  (isObject(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader));\n// Not implemented in Firefox: && isFunction(x.pipeTo)\n\nexport const isWritableNodeStream: (x: any) => boolean = (x) =>\n  isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);\nexport const isReadableNodeStream: (x: any) => boolean = (x) =>\n  isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);\nexport const isReadableStream: (x: any) => boolean = (x) =>\n  isReadableDOMStream(x) || isReadableNodeStream(x);\nexport const isWritableStream: (x: any) => boolean = (x) =>\n  isWritableDOMStream(x) || isWritableNodeStream(x);\n"],"file":"is-type.js"}

package/dist/es5/lib/api/encode.js

@@ -0,0 +1,108 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.encode = encode;
+exports.encodeSync = encodeSync;
+exports.encodeText = encodeText;
+exports.encodeInBatches = encodeInBatches;
+exports.encodeURLtoURL = encodeURLtoURL;
+
+var _loaderUtils = require("@loaders.gl/loader-utils");
+
+var _writeFile = require("../fetch/write-file");
+
+var _fetchFile = require("../fetch/fetch-file");
+
+async function encode(data, writer, options) {
+  if (writer.encode) {
+    return await writer.encode(data, options);
+  }
+
+  if (writer.encodeSync) {
+    return writer.encodeSync(data, options);
+  }
+
+  if (writer.encodeText) {
+    return new TextEncoder().encode(await writer.encodeText(data, options));
+  }
+
+  if (writer.encodeInBatches) {
+    const batches = encodeInBatches(data, writer, options);
+    const chunks = [];
+
+    for await (const batch of batches) {
+      chunks.push(batch);
+    }
+
+    return (0, _loaderUtils.concatenateArrayBuffers)(...chunks);
+  }
+
+  if (!_loaderUtils.isBrowser && writer.encodeURLtoURL) {
+    const tmpInputFilename = getTemporaryFilename('input');
+    await (0, _writeFile.writeFile)(tmpInputFilename, data);
+    const tmpOutputFilename = getTemporaryFilename('output');
+    const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
+    const response = await (0, _fetchFile.fetchFile)(outputFilename);
+    return response.arrayBuffer();
+  }
+
+  throw new Error('Writer could not encode data');
+}
+
+function encodeSync(data, writer, options) {
+  if (writer.encodeSync) {
+    return writer.encodeSync(data, options);
+  }
+
+  throw new Error('Writer could not synchronously encode data');
+}
+
+async function encodeText(data, writer, options) {
+  if (writer.text && writer.encodeText) {
+    return await writer.encodeText(data, options);
+  }
+
+  if (writer.text && (writer.encode || writer.encodeInBatches)) {
+    const arrayBuffer = await encode(data, writer, options);
+    return new TextDecoder().decode(arrayBuffer);
+  }
+
+  throw new Error('Writer could not encode data as text');
+}
+
+function encodeInBatches(data, writer, options) {
+  if (writer.encodeInBatches) {
+    const dataIterator = getIterator(data);
+    return writer.encodeInBatches(dataIterator, options);
+  }
+
+  throw new Error('Writer could not encode data in batches');
+}
+
+async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
+  inputUrl = (0, _loaderUtils.resolvePath)(inputUrl);
+  outputUrl = (0, _loaderUtils.resolvePath)(outputUrl);
+
+  if (_loaderUtils.isBrowser || !writer.encodeURLtoURL) {
+    throw new Error();
+  }
+
+  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
+  return outputFilename;
+}
+
+function getIterator(data) {
+  const dataIterator = [{
+    table: data,
+    start: 0,
+    end: data.length
+  }];
+  return dataIterator;
+}
+
+function getTemporaryFilename(filename) {
+  return "/tmp/".concat(filename);
+}
+//# sourceMappingURL=encode.js.map

package/dist/es5/lib/api/encode.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/api/encode.ts"],"names":["encode","data","writer","options","encodeSync","encodeText","TextEncoder","encodeInBatches","batches","chunks","batch","push","isBrowser","encodeURLtoURL","tmpInputFilename","getTemporaryFilename","tmpOutputFilename","outputFilename","response","arrayBuffer","Error","text","TextDecoder","decode","dataIterator","getIterator","inputUrl","outputUrl","table","start","end","length","filename"],"mappings":";;;;;;;;;;;AACA;;AAEA;;AACA;;AAKO,eAAeA,MAAf,CACLC,IADK,EAELC,MAFK,EAGLC,OAHK,EAIiB;AAEtB,MAAID,MAAM,CAACF,MAAX,EAAmB;AACjB,WAAO,MAAME,MAAM,CAACF,MAAP,CAAcC,IAAd,EAAoBE,OAApB,CAAb;AACD;;AAED,MAAID,MAAM,CAACE,UAAX,EAAuB;AACrB,WAAOF,MAAM,CAACE,UAAP,CAAkBH,IAAlB,EAAwBE,OAAxB,CAAP;AACD;;AAED,MAAID,MAAM,CAACG,UAAX,EAAuB;AACrB,WAAO,IAAIC,WAAJ,GAAkBN,MAAlB,CAAyB,MAAME,MAAM,CAACG,UAAP,CAAkBJ,IAAlB,EAAwBE,OAAxB,CAA/B,CAAP;AACD;;AAED,MAAID,MAAM,CAACK,eAAX,EAA4B;AAG1B,UAAMC,OAAO,GAAGD,eAAe,CAACN,IAAD,EAAOC,MAAP,EAAeC,OAAf,CAA/B;AAGA,UAAMM,MAAa,GAAG,EAAtB;;AACA,eAAW,MAAMC,KAAjB,IAA0BF,OAA1B,EAAmC;AACjCC,MAAAA,MAAM,CAACE,IAAP,CAAYD,KAAZ;AACD;;AAED,WAAO,0CAAwB,GAAGD,MAA3B,CAAP;AACD;;AAED,MAAI,CAACG,sBAAD,IAAcV,MAAM,CAACW,cAAzB,EAAyC;AAEvC,UAAMC,gBAAgB,GAAGC,oBAAoB,CAAC,OAAD,CAA7C;AACA,UAAM,0BAAUD,gBAAV,EAA4Bb,IAA5B,CAAN;AAEA,UAAMe,iBAAiB,GAAGD,oBAAoB,CAAC,QAAD,CAA9C;AAEA,UAAME,cAAc,GAAG,MAAMJ,cAAc,CACzCC,gBADyC,EAEzCE,iBAFyC,EAGzCd,MAHyC,EAIzCC,OAJyC,CAA3C;AAOA,UAAMe,QAAQ,GAAG,MAAM,0BAAUD,cAAV,CAAvB;AACA,WAAOC,QAAQ,CAACC,WAAT,EAAP;AACD;;AAED,QAAM,IAAIC,KAAJ,CAAU,8BAAV,CAAN;AACD;;AAKM,SAAShB,UAAT,CAAoBH,IAApB,EAA+BC,MAA/B,EAA+CC,OAA/C,EAAqF;AAC1F,MAAID,MAAM,CAACE,UAAX,EAAuB;AACrB,WAAOF,MAAM,CAACE,UAAP,CAAkBH,IAAlB,EAAwBE,OAAxB,CAAP;AACD;;AACD,QAAM,IAAIiB,KAAJ,CAAU,4CAAV,CAAN;AACD;;AAQM,eAAef,UAAf,CACLJ,IADK,EAELC,MAFK,EAGLC,OAHK,EAIY;AACjB,MAAID,MAAM,CAACmB,IAAP,IAAenB,MAAM,CAACG,UAA1B,EAAsC;AACpC,WAAO,MAAMH,MAAM,CAACG,UAAP,CAAkBJ,IAAlB,EAAwBE,OAAxB,CAAb;AACD;;AAED,MAAID,MAAM,CAACmB,IAAP,KAAgBnB,MAAM,CAACF,MAAP,IAAiBE,MAAM,CAACK,eAAxC,CAAJ,EAA8D;AAC5D,UAAMY,WAAW,GAAG,MAAMnB,MAAM,CAACC,IAAD,EAAOC,MAAP,EAAeC,OAAf,CAAhC;AACA,WAAO,IAAImB,WAAJ,GAAkBC,MAAlB,CAAyBJ,WAAzB,CAAP;AACD;;AAED,QAAM,IAAIC,KAAJ,CAAU,sCAAV,CAAN;AACD;;AAKM,SAASb,eAAT,CACLN,IADK,EAELC,MAFK,EAGLC,OAHK,EAIuB;AAC5B,MAAID,MAAM,CAACK,eAAX,EAA4B;AAC1B,UAAMiB,YAAY,GAAGC,WAAW,CAACxB,IAAD,CAAhC;AACA,WAAOC,MAAM,CAACK,eAAP,CAAuBiB,YAAvB,EAAqCrB,OAArC,CAAP;AACD;;AAED,QAAM,IAAIiB,KAAJ,CAAU,yCAAV,CAAN;AACD;;AAMM,eAAeP,cAAf,CACLa,QADK,EAELC,SAFK,EAGLzB,MAHK,EAILC,OAJK,EAKY;AACjBuB,EAAAA,QAAQ,GAAG,8BAAYA,QAAZ,CAAX;AACAC,EAAAA,SAAS,GAAG,8BAAYA,SAAZ,CAAZ;;AACA,MAAIf,0BAAa,CAACV,MAAM,CAACW,cAAzB,EAAyC;AACvC,UAAM,IAAIO,KAAJ,EAAN;AACD;;AACD,QAAMH,cAAc,GAAG,MAAMf,MAAM,CAACW,cAAP,CAAsBa,QAAtB,EAAgCC,SAAhC,EAA2CxB,OAA3C,CAA7B;AACA,SAAOc,cAAP;AACD;;AAKD,SAASQ,WAAT,CAAqBxB,IAArB,EAA2B;AACzB,QAAMuB,YAAY,GAAG,CAAC;AAACI,IAAAA,KAAK,EAAE3B,IAAR;AAAc4B,IAAAA,KAAK,EAAE,CAArB;AAAwBC,IAAAA,GAAG,EAAE7B,IAAI,CAAC8B;AAAlC,GAAD,CAArB;AACA,SAAOP,YAAP;AACD;;AAKD,SAAST,oBAAT,CAA8BiB,QAA9B,EAAwD;AACtD,wBAAeA,QAAf;AACD","sourcesContent":["import type {Writer, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {concatenateArrayBuffers, resolvePath} from '@loaders.gl/loader-utils';\nimport {isBrowser} from '@loaders.gl/loader-utils';\nimport {writeFile} from '../fetch/write-file';\nimport {fetchFile} from '../fetch/fetch-file';\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport async function encode(\n  data: any,\n  writer: Writer,\n  options?: LoaderOptions\n): Promise<ArrayBuffer> {\n  // TODO Merge default writer options with options argument like it is done in load module.\n  if (writer.encode) {\n    return await writer.encode(data, options);\n  }\n\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n\n  if (writer.encodeText) {\n    return new TextEncoder().encode(await writer.encodeText(data, options));\n  }\n\n  if (writer.encodeInBatches) {\n    // Create an iterator representing the data\n    // TODO - Assumes this is a table\n    const batches = encodeInBatches(data, writer, options);\n\n    // Concatenate the output\n    const chunks: any[] = [];\n    for await (const batch of batches) {\n      chunks.push(batch);\n    }\n    // @ts-ignore\n    return concatenateArrayBuffers(...chunks);\n  }\n\n  if (!isBrowser && writer.encodeURLtoURL) {\n    // TODO - how to generate filenames with correct extensions?\n    const tmpInputFilename = getTemporaryFilename('input');\n    await writeFile(tmpInputFilename, data);\n\n    const tmpOutputFilename = getTemporaryFilename('output');\n\n    const outputFilename = await encodeURLtoURL(\n      tmpInputFilename,\n      tmpOutputFilename,\n      writer,\n      options\n    );\n\n    const response = await fetchFile(outputFilename);\n    return response.arrayBuffer();\n  }\n\n  throw new Error('Writer could not encode data');\n}\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport function encodeSync(data: any, writer: Writer, options?: LoaderOptions): ArrayBuffer {\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n  throw new Error('Writer could not synchronously encode data');\n}\n\n/**\n * Encode loaded data to text using the specified Writer\n * @note This is a convenience function not intended for production use on large input data.\n * It is not optimized for performance. Data maybe converted from text to binary and back.\n * @throws if the writer does not generate text output\n */\nexport async function encodeText(\n  data: any,\n  writer: Writer,\n  options?: LoaderOptions\n): Promise<string> {\n  if (writer.text && writer.encodeText) {\n    return await writer.encodeText(data, options);\n  }\n\n  if (writer.text && (writer.encode || writer.encodeInBatches)) {\n    const arrayBuffer = await encode(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n\n  throw new Error('Writer could not encode data as text');\n}\n\n/**\n * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.\n */\nexport function encodeInBatches(\n  data: any,\n  writer: Writer,\n  options?: LoaderOptions\n): AsyncIterable<ArrayBuffer> {\n  if (writer.encodeInBatches) {\n    const dataIterator = getIterator(data);\n    return writer.encodeInBatches(dataIterator, options);\n  }\n  // TODO -fall back to atomic encode?\n  throw new Error('Writer could not encode data in batches');\n}\n\n/**\n * Encode data stored in a file (on disk) to another file.\n * @note Node.js only. This function enables using command-line converters as \"writers\".\n */\nexport async function encodeURLtoURL(\n  inputUrl,\n  outputUrl,\n  writer: Writer,\n  options\n): Promise<string> {\n  inputUrl = resolvePath(inputUrl);\n  outputUrl = resolvePath(outputUrl);\n  if (isBrowser || !writer.encodeURLtoURL) {\n    throw new Error();\n  }\n  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);\n  return outputFilename;\n}\n\n/**\n * @todo TODO - this is an unacceptable hack!!!\n */\nfunction getIterator(data) {\n  const dataIterator = [{table: data, start: 0, end: data.length}];\n  return dataIterator;\n}\n\n/**\n * @todo Move to utils\n */\nfunction getTemporaryFilename(filename: string): string {\n  return `/tmp/${filename}`;\n}\n"],"file":"encode.js"}

package/dist/es5/lib/api/load-in-batches.js

@@ -0,0 +1,40 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.loadInBatches = loadInBatches;
+
+var _normalizeLoader = require("../loader-utils/normalize-loader");
+
+var _optionUtils = require("../loader-utils/option-utils");
+
+var _parseInBatches = require("./parse-in-batches");
+
+function loadInBatches(files, loaders, options, context) {
+  if (!Array.isArray(loaders) && !(0, _normalizeLoader.isLoaderObject)(loaders)) {
+    context = undefined;
+    options = loaders;
+    loaders = null;
+  }
+
+  const fetch = (0, _optionUtils.getFetchFunction)(options || {});
+
+  if (!Array.isArray(files)) {
+    return loadOneFileInBatches(files, loaders, options, fetch);
+  }
+
+  const promises = files.map(file => loadOneFileInBatches(file, loaders, options, fetch));
+  return promises;
+}
+
+async function loadOneFileInBatches(file, loaders, options, fetch) {
+  if (typeof file === 'string') {
+    const url = file;
+    const response = await fetch(url);
+    return await (0, _parseInBatches.parseInBatches)(response, loaders, options);
+  }
+
+  return await (0, _parseInBatches.parseInBatches)(file, loaders, options);
+}
+//# sourceMappingURL=load-in-batches.js.map

package/dist/es5/lib/api/load-in-batches.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/api/load-in-batches.ts"],"names":["loadInBatches","files","loaders","options","context","Array","isArray","undefined","fetch","loadOneFileInBatches","promises","map","file","url","response"],"mappings":";;;;;;;AACA;;AACA;;AAEA;;AAyBO,SAASA,aAAT,CAAuBC,KAAvB,EAA8BC,OAA9B,EAAuCC,OAAvC,EAAgDC,OAAhD,EAAyD;AAE9D,MAAI,CAACC,KAAK,CAACC,OAAN,CAAcJ,OAAd,CAAD,IAA2B,CAAC,qCAAeA,OAAf,CAAhC,EAAyD;AACvDE,IAAAA,OAAO,GAAGG,SAAV;AACAJ,IAAAA,OAAO,GAAGD,OAAV;AACAA,IAAAA,OAAO,GAAG,IAAV;AACD;;AAGD,QAAMM,KAAK,GAAG,mCAAiBL,OAAO,IAAI,EAA5B,CAAd;;AAGA,MAAI,CAACE,KAAK,CAACC,OAAN,CAAcL,KAAd,CAAL,EAA2B;AACzB,WAAOQ,oBAAoB,CAACR,KAAD,EAAQC,OAAR,EAAiBC,OAAjB,EAA0BK,KAA1B,CAA3B;AACD;;AAGD,QAAME,QAAQ,GAAGT,KAAK,CAACU,GAAN,CAAWC,IAAD,IAAUH,oBAAoB,CAACG,IAAD,EAAOV,OAAP,EAAgBC,OAAhB,EAAyBK,KAAzB,CAAxC,CAAjB;AAGA,SAAOE,QAAP;AACD;;AAED,eAAeD,oBAAf,CAAoCG,IAApC,EAA0CV,OAA1C,EAAmDC,OAAnD,EAA4DK,KAA5D,EAAmE;AACjE,MAAI,OAAOI,IAAP,KAAgB,QAApB,EAA8B;AAC5B,UAAMC,GAAG,GAAGD,IAAZ;AACA,UAAME,QAAQ,GAAG,MAAMN,KAAK,CAACK,GAAD,CAA5B;AACA,WAAO,MAAM,oCAAeC,QAAf,EAAyBZ,OAAzB,EAAkCC,OAAlC,CAAb;AACD;;AACD,SAAO,MAAM,oCAAeS,IAAf,EAAqBV,OAArB,EAA8BC,OAA9B,CAAb;AACD","sourcesContent":["import type {LoaderWithParser, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {getFetchFunction} from '../loader-utils/option-utils';\n\nimport {parseInBatches} from './parse-in-batches';\n\ntype FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport function loadInBatches(\n  files: FileType,\n  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<any>>;\n\nexport function loadInBatches(\n  files: FileType[] | FileList,\n  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<any>>;\n\nexport function loadInBatches(files, loaders, options, context) {\n  // Signature: load(url, options)\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders;\n    loaders = null;\n  }\n\n  // Select fetch function\n  const fetch = getFetchFunction(options || {});\n\n  // Single url/file\n  if (!Array.isArray(files)) {\n    return loadOneFileInBatches(files, loaders, options, fetch);\n  }\n\n  // Multiple URLs / files\n  const promises = files.map((file) => loadOneFileInBatches(file, loaders, options, fetch));\n\n  // No point in waiting here for all responses before starting to stream individual streams?\n  return promises;\n}\n\nasync function loadOneFileInBatches(file, loaders, options, fetch) {\n  if (typeof file === 'string') {\n    const url = file;\n    const response = await fetch(url);\n    return await parseInBatches(response, loaders, options);\n  }\n  return await parseInBatches(file, loaders, options);\n}\n"],"file":"load-in-batches.js"}

package/dist/es5/lib/api/load.js

@@ -0,0 +1,36 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.load = load;
+
+var _isType = require("../../javascript-utils/is-type");
+
+var _normalizeLoader = require("../loader-utils/normalize-loader");
+
+var _optionUtils = require("../loader-utils/option-utils");
+
+var _parse = require("./parse");
+
+async function load(url, loaders, options, context) {
+  if (!Array.isArray(loaders) && !(0, _normalizeLoader.isLoaderObject)(loaders)) {
+    context = undefined;
+    options = loaders;
+    loaders = undefined;
+  }
+
+  const fetch = (0, _optionUtils.getFetchFunction)(options);
+  let data = url;
+
+  if (typeof url === 'string') {
+    data = await fetch(url);
+  }
+
+  if ((0, _isType.isBlob)(url)) {
+    data = await fetch(url);
+  }
+
+  return await (0, _parse.parse)(data, loaders, options);
+}
+//# sourceMappingURL=load.js.map

package/dist/es5/lib/api/load.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/api/load.ts"],"names":["load","url","loaders","options","context","Array","isArray","undefined","fetch","data"],"mappings":";;;;;;;AACA;;AACA;;AACA;;AAEA;;AAYO,eAAeA,IAAf,CACLC,GADK,EAELC,OAFK,EAGLC,OAHK,EAILC,OAJK,EAKS;AAEd,MAAI,CAACC,KAAK,CAACC,OAAN,CAAcJ,OAAd,CAAD,IAA2B,CAAC,qCAAeA,OAAf,CAAhC,EAAyD;AACvDE,IAAAA,OAAO,GAAGG,SAAV;AACAJ,IAAAA,OAAO,GAAGD,OAAV;AACAA,IAAAA,OAAO,GAAGK,SAAV;AACD;;AAGD,QAAMC,KAAK,GAAG,mCAAiBL,OAAjB,CAAd;AAGA,MAAIM,IAAI,GAAGR,GAAX;;AAEA,MAAI,OAAOA,GAAP,KAAe,QAAnB,EAA6B;AAC3BQ,IAAAA,IAAI,GAAG,MAAMD,KAAK,CAACP,GAAD,CAAlB;AAED;;AAED,MAAI,oBAAOA,GAAP,CAAJ,EAAiB;AAEfQ,IAAAA,IAAI,GAAG,MAAMD,KAAK,CAACP,GAAD,CAAlB;AACD;;AAGD,SAAO,MAAM,kBAAMQ,IAAN,EAAYP,OAAZ,EAAqBC,OAArB,CAAb;AACD","sourcesContent":["import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {isBlob} from '../../javascript-utils/is-type';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {getFetchFunction} from '../loader-utils/option-utils';\n\nimport {parse} from './parse';\n\n/**\n * Parses `data` using a specified loader\n * Note: Load does duplicate a lot of parse.\n * it can also call fetchFile on string urls, which `parse` won't do.\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\n// implementation signature\nexport async function load(\n  url: string | DataType,\n  loaders?: Loader | Loader[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<any> {\n  // Signature: load(url, options)\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders as LoaderOptions;\n    loaders = undefined;\n  }\n\n  // Select fetch function\n  const fetch = getFetchFunction(options);\n\n  // at this point, `url` could be already loaded binary data\n  let data = url;\n  // url is a string, fetch the url\n  if (typeof url === 'string') {\n    data = await fetch(url);\n    // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)\n  }\n\n  if (isBlob(url)) {\n    // The fetch response object will contain blob.name\n    data = await fetch(url);\n  }\n\n  // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`\n  return await parse(data, loaders, options);\n}\n"],"file":"load.js"}

package/dist/es5/lib/api/parse-in-batches.js

@@ -0,0 +1,114 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.parseInBatches = parseInBatches;
+
+var _loaderUtils = require("@loaders.gl/loader-utils");
+
+var _normalizeLoader = require("../loader-utils/normalize-loader");
+
+var _optionUtils = require("../loader-utils/option-utils");
+
+var _loaderContext = require("../loader-utils/loader-context");
+
+var _getData = require("../loader-utils/get-data");
+
+var _resourceUtils = require("../utils/resource-utils");
+
+var _selectLoader = require("./select-loader");
+
+var _parse = require("./parse");
+
+async function parseInBatches(data, loaders, options, context) {
+  (0, _loaderUtils.assert)(!context || typeof context === 'object');
+
+  if (!Array.isArray(loaders) && !(0, _normalizeLoader.isLoaderObject)(loaders)) {
+    context = undefined;
+    options = loaders;
+    loaders = undefined;
+  }
+
+  data = await data;
+  options = options || {};
+  const {
+    url
+  } = (0, _resourceUtils.getResourceUrlAndType)(data);
+  const loader = await (0, _selectLoader.selectLoader)(data, loaders, options);
+
+  if (!loader) {
+    return null;
+  }
+
+  options = (0, _optionUtils.normalizeOptions)(options, loader, loaders, url);
+  context = (0, _loaderContext.getLoaderContext)({
+    url,
+    parseInBatches,
+    parse: _parse.parse,
+    loaders: loaders
+  }, options, context);
+  return await parseWithLoaderInBatches(loader, data, options, context);
+}
+
+async function parseWithLoaderInBatches(loader, data, options, context) {
+  const outputIterator = await parseToOutputIterator(loader, data, options, context);
+
+  if (!options.metadata) {
+    return outputIterator;
+  }
+
+  const metadataBatch = {
+    batchType: 'metadata',
+    metadata: {
+      _loader: loader,
+      _context: context
+    },
+    data: [],
+    bytesUsed: 0
+  };
+
+  async function* makeMetadataBatchIterator(iterator) {
+    yield metadataBatch;
+    yield* iterator;
+  }
+
+  return makeMetadataBatchIterator(outputIterator);
+}
+
+async function parseToOutputIterator(loader, data, options, context) {
+  const inputIterator = await (0, _getData.getAsyncIterableFromData)(data, options);
+  const transformedIterator = await applyInputTransforms(inputIterator, (options === null || options === void 0 ? void 0 : options.transforms) || []);
+
+  if (loader.parseInBatches) {
+    return loader.parseInBatches(transformedIterator, options, context);
+  }
+
+  async function* parseChunkInBatches() {
+    const arrayBuffer = await (0, _loaderUtils.concatenateArrayBuffersAsync)(transformedIterator);
+    const parsedData = await (0, _parse.parse)(arrayBuffer, loader, { ...options,
+      mimeType: loader.mimeTypes[0]
+    }, context);
+    const batch = {
+      mimeType: loader.mimeTypes[0],
+      shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
+      batchType: 'data',
+      data: parsedData,
+      length: Array.isArray(parsedData) ? parsedData.length : 1
+    };
+    yield batch;
+  }
+
+  return parseChunkInBatches();
+}
+
+async function applyInputTransforms(inputIterator, transforms = []) {
+  let iteratorChain = inputIterator;
+
+  for await (const transformBatches of transforms) {
+    iteratorChain = transformBatches(iteratorChain);
+  }
+
+  return iteratorChain;
+}
+//# sourceMappingURL=parse-in-batches.js.map