@loaders.gl/core 3.1.3 → 4.0.0-alpha.5

package/dist/lib/api/encode.js

@@ -1,111 +1,91 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encodeURLtoURL = exports.encodeInBatches = exports.encodeText = exports.encodeSync = exports.encode = void 0;
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const loader_utils_2 = require("@loaders.gl/loader-utils");
-const write_file_1 = require("../fetch/write-file");
-const fetch_file_1 = require("../fetch/fetch-file");
-/**
- * Encode loaded data into a binary ArrayBuffer using the specified Writer.
- */
-async function encode(data, writer, options) {
-    // TODO Merge default writer options with options argument like it is done in load module.
-    if (writer.encode) {
-        return await writer.encode(data, options);
+import { concatenateArrayBuffers, resolvePath } from '@loaders.gl/loader-utils';
+import { isBrowser } from '@loaders.gl/loader-utils';
+import { writeFile } from '../fetch/write-file';
+import { fetchFile } from '../fetch/fetch-file';
+export async function encode(data, writer, options) {
+  if (writer.encode) {
+    return await writer.encode(data, options);
+  }
+
+  if (writer.encodeSync) {
+    return writer.encodeSync(data, options);
+  }
+
+  if (writer.encodeText) {
+    return new TextEncoder().encode(await writer.encodeText(data, options));
+  }
+
+  if (writer.encodeInBatches) {
+    const batches = encodeInBatches(data, writer, options);
+    const chunks = [];
+
+    for await (const batch of batches) {
+      chunks.push(batch);
     }
-    if (writer.encodeSync) {
-        return writer.encodeSync(data, options);
-    }
-    if (writer.encodeText) {
-        return new TextEncoder().encode(await writer.encodeText(data, options));
-    }
-    if (writer.encodeInBatches) {
-        // Create an iterator representing the data
-        // TODO - Assumes this is a table
-        const batches = encodeInBatches(data, writer, options);
-        // Concatenate the output
-        const chunks = [];
-        for await (const batch of batches) {
-            chunks.push(batch);
-        }
-        // @ts-ignore
-        return (0, loader_utils_1.concatenateArrayBuffers)(...chunks);
-    }
-    if (!loader_utils_2.isBrowser && writer.encodeURLtoURL) {
-        // TODO - how to generate filenames with correct extensions?
-        const tmpInputFilename = getTemporaryFilename('input');
-        await (0, write_file_1.writeFile)(tmpInputFilename, data);
-        const tmpOutputFilename = getTemporaryFilename('output');
-        const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
-        const response = await (0, fetch_file_1.fetchFile)(outputFilename);
-        return response.arrayBuffer();
-    }
-    throw new Error('Writer could not encode data');
+
+    return concatenateArrayBuffers(...chunks);
+  }
+
+  if (!isBrowser && writer.encodeURLtoURL) {
+    const tmpInputFilename = getTemporaryFilename('input');
+    await writeFile(tmpInputFilename, data);
+    const tmpOutputFilename = getTemporaryFilename('output');
+    const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
+    const response = await fetchFile(outputFilename);
+    return response.arrayBuffer();
+  }
+
+  throw new Error('Writer could not encode data');
 }
-exports.encode = encode;
-/**
- * Encode loaded data into a binary ArrayBuffer using the specified Writer.
- */
-function encodeSync(data, writer, options) {
-    if (writer.encodeSync) {
-        return writer.encodeSync(data, options);
-    }
-    throw new Error('Writer could not synchronously encode data');
+export function encodeSync(data, writer, options) {
+  if (writer.encodeSync) {
+    return writer.encodeSync(data, options);
+  }
+
+  throw new Error('Writer could not synchronously encode data');
 }
-exports.encodeSync = encodeSync;
-/**
- * Encode loaded data to text using the specified Writer
- * @note This is a convenience function not intended for production use on large input data.
- * It is not optimized for performance. Data maybe converted from text to binary and back.
- * @throws if the writer does not generate text output
- */
-async function encodeText(data, writer, options) {
-    if (writer.text && writer.encodeText) {
-        return await writer.encodeText(data, options);
-    }
-    if (writer.text && (writer.encode || writer.encodeInBatches)) {
-        const arrayBuffer = await encode(data, writer, options);
-        return new TextDecoder().decode(arrayBuffer);
-    }
-    throw new Error('Writer could not encode data as text');
+export async function encodeText(data, writer, options) {
+  if (writer.text && writer.encodeText) {
+    return await writer.encodeText(data, options);
+  }
+
+  if (writer.text && (writer.encode || writer.encodeInBatches)) {
+    const arrayBuffer = await encode(data, writer, options);
+    return new TextDecoder().decode(arrayBuffer);
+  }
+
+  throw new Error('Writer could not encode data as text');
 }
-exports.encodeText = encodeText;
-/**
- * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
- */
-function encodeInBatches(data, writer, options) {
-    if (writer.encodeInBatches) {
-        const dataIterator = getIterator(data);
-        return writer.encodeInBatches(dataIterator, options);
-    }
-    // TODO -fall back to atomic encode?
-    throw new Error('Writer could not encode data in batches');
+export function encodeInBatches(data, writer, options) {
+  if (writer.encodeInBatches) {
+    const dataIterator = getIterator(data);
+    return writer.encodeInBatches(dataIterator, options);
+  }
+
+  throw new Error('Writer could not encode data in batches');
 }
-exports.encodeInBatches = encodeInBatches;
-/**
- * Encode data stored in a file (on disk) to another file.
- * @note Node.js only. This function enables using command-line converters as "writers".
- */
-async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
-    inputUrl = (0, loader_utils_1.resolvePath)(inputUrl);
-    outputUrl = (0, loader_utils_1.resolvePath)(outputUrl);
-    if (loader_utils_2.isBrowser || !writer.encodeURLtoURL) {
-        throw new Error();
-    }
-    const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
-    return outputFilename;
+export async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
+  inputUrl = resolvePath(inputUrl);
+  outputUrl = resolvePath(outputUrl);
+
+  if (isBrowser || !writer.encodeURLtoURL) {
+    throw new Error();
+  }
+
+  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
+  return outputFilename;
 }
-exports.encodeURLtoURL = encodeURLtoURL;
-/**
- * @todo TODO - this is an unacceptable hack!!!
- */
+
 function getIterator(data) {
-    const dataIterator = [{ table: data, start: 0, end: data.length }];
-    return dataIterator;
+  const dataIterator = [{
+    table: data,
+    start: 0,
+    end: data.length
+  }];
+  return dataIterator;
 }
-/**
- * @todo Move to utils
- */
+
 function getTemporaryFilename(filename) {
-    return `/tmp/${filename}`;
+  return "/tmp/".concat(filename);
 }
+//# sourceMappingURL=encode.js.map

package/dist/lib/api/encode.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/api/encode.ts"],"names":["concatenateArrayBuffers","resolvePath","isBrowser","writeFile","fetchFile","encode","data","writer","options","encodeSync","encodeText","TextEncoder","encodeInBatches","batches","chunks","batch","push","encodeURLtoURL","tmpInputFilename","getTemporaryFilename","tmpOutputFilename","outputFilename","response","arrayBuffer","Error","text","TextDecoder","decode","dataIterator","getIterator","inputUrl","outputUrl","table","start","end","length","filename"],"mappings":"AACA,SAAQA,uBAAR,EAAiCC,WAAjC,QAAmD,0BAAnD;AACA,SAAQC,SAAR,QAAwB,0BAAxB;AACA,SAAQC,SAAR,QAAwB,qBAAxB;AACA,SAAQC,SAAR,QAAwB,qBAAxB;AAKA,OAAO,eAAeC,MAAf,CACLC,IADK,EAELC,MAFK,EAGLC,OAHK,EAIiB;AAEtB,MAAID,MAAM,CAACF,MAAX,EAAmB;AACjB,WAAO,MAAME,MAAM,CAACF,MAAP,CAAcC,IAAd,EAAoBE,OAApB,CAAb;AACD;;AAED,MAAID,MAAM,CAACE,UAAX,EAAuB;AACrB,WAAOF,MAAM,CAACE,UAAP,CAAkBH,IAAlB,EAAwBE,OAAxB,CAAP;AACD;;AAED,MAAID,MAAM,CAACG,UAAX,EAAuB;AACrB,WAAO,IAAIC,WAAJ,GAAkBN,MAAlB,CAAyB,MAAME,MAAM,CAACG,UAAP,CAAkBJ,IAAlB,EAAwBE,OAAxB,CAA/B,CAAP;AACD;;AAED,MAAID,MAAM,CAACK,eAAX,EAA4B;AAG1B,UAAMC,OAAO,GAAGD,eAAe,CAACN,IAAD,EAAOC,MAAP,EAAeC,OAAf,CAA/B;AAGA,UAAMM,MAAa,GAAG,EAAtB;;AACA,eAAW,MAAMC,KAAjB,IAA0BF,OAA1B,EAAmC;AACjCC,MAAAA,MAAM,CAACE,IAAP,CAAYD,KAAZ;AACD;;AAED,WAAOf,uBAAuB,CAAC,GAAGc,MAAJ,CAA9B;AACD;;AAED,MAAI,CAACZ,SAAD,IAAcK,MAAM,CAACU,cAAzB,EAAyC;AAEvC,UAAMC,gBAAgB,GAAGC,oBAAoB,CAAC,OAAD,CAA7C;AACA,UAAMhB,SAAS,CAACe,gBAAD,EAAmBZ,IAAnB,CAAf;AAEA,UAAMc,iBAAiB,GAAGD,oBAAoB,CAAC,QAAD,CAA9C;AAEA,UAAME,cAAc,GAAG,MAAMJ,cAAc,CACzCC,gBADyC,EAEzCE,iBAFyC,EAGzCb,MAHyC,EAIzCC,OAJyC,CAA3C;AAOA,UAAMc,QAAQ,GAAG,MAAMlB,SAAS,CAACiB,cAAD,CAAhC;AACA,WAAOC,QAAQ,CAACC,WAAT,EAAP;AACD;;AAED,QAAM,IAAIC,KAAJ,CAAU,8BAAV,CAAN;AACD;AAKD,OAAO,SAASf,UAAT,CAAoBH,IAApB,EAA+BC,MAA/B,EAA+CC,OAA/C,EAAqF;AAC1F,MAAID,MAAM,CAACE,UAAX,EAAuB;AACrB,WAAOF,MAAM,CAACE,UAAP,CAAkBH,IAAlB,EAAwBE,OAAxB,CAAP;AACD;;AACD,QAAM,IAAIgB,KAAJ,CAAU,4CAAV,CAAN;AACD;AAQD,OAAO,eAAed,UAAf,CACLJ,IADK,EAELC,MAFK,EAGLC,OAHK,EAIY;AACjB,MAAID,MAAM,CAACkB,IAAP,IAAelB,MAAM,CAACG,UAA1B,EAAsC;AACpC,WAAO,MAAMH,MAAM,CAACG,UAAP,CAAkBJ,IAAlB,EAAwBE,OAAxB,CAAb;AACD;;AAED,MAAID,MAAM,CAACkB,IAAP,KAAgBlB,MAAM,CAACF,MAAP,IAAiBE,MAAM,CAACK,eAAxC,CAAJ,EAA8D;AAC5D,UAAMW,WAAW,GAAG,MAAMlB,MAAM,CAACC,IAAD,EAAOC,MAAP,EAAeC,OAAf,CAAhC;AACA,WAAO,IAAIkB,WAAJ,GAAkBC,MAAlB,CAAyBJ,WAAzB,CAAP;AACD;;AAED,QAAM,IAAIC,KAAJ,CAAU,sCAAV,CAAN;AACD;AAKD,OAAO,SAASZ,eAAT,CACLN,IADK,EAELC,MAFK,EAGLC,OAHK,EAIuB;AAC5B,MAAID,MAAM,CAACK,eAAX,EAA4B;AAC1B,UAAMgB,YAAY,GAAGC,WAAW,CAACvB,IAAD,CAAhC;AACA,WAAOC,MAAM,CAACK,eAAP,CAAuBgB,YAAvB,EAAqCpB,OAArC,CAAP;AACD;;AAED,QAAM,IAAIgB,KAAJ,CAAU,yCAAV,CAAN;AACD;AAMD,OAAO,eAAeP,cAAf,CACLa,QADK,EAELC,SAFK,EAGLxB,MAHK,EAILC,OAJK,EAKY;AACjBsB,EAAAA,QAAQ,GAAG7B,WAAW,CAAC6B,QAAD,CAAtB;AACAC,EAAAA,SAAS,GAAG9B,WAAW,CAAC8B,SAAD,CAAvB;;AACA,MAAI7B,SAAS,IAAI,CAACK,MAAM,CAACU,cAAzB,EAAyC;AACvC,UAAM,IAAIO,KAAJ,EAAN;AACD;;AACD,QAAMH,cAAc,GAAG,MAAMd,MAAM,CAACU,cAAP,CAAsBa,QAAtB,EAAgCC,SAAhC,EAA2CvB,OAA3C,CAA7B;AACA,SAAOa,cAAP;AACD;;AAKD,SAASQ,WAAT,CAAqBvB,IAArB,EAA2B;AACzB,QAAMsB,YAAY,GAAG,CAAC;AAACI,IAAAA,KAAK,EAAE1B,IAAR;AAAc2B,IAAAA,KAAK,EAAE,CAArB;AAAwBC,IAAAA,GAAG,EAAE5B,IAAI,CAAC6B;AAAlC,GAAD,CAArB;AACA,SAAOP,YAAP;AACD;;AAKD,SAAST,oBAAT,CAA8BiB,QAA9B,EAAwD;AACtD,wBAAeA,QAAf;AACD","sourcesContent":["import type {Writer, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {concatenateArrayBuffers, resolvePath} from '@loaders.gl/loader-utils';\nimport {isBrowser} from '@loaders.gl/loader-utils';\nimport {writeFile} from '../fetch/write-file';\nimport {fetchFile} from '../fetch/fetch-file';\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport async function encode(\n  data: any,\n  writer: Writer,\n  options?: LoaderOptions\n): Promise<ArrayBuffer> {\n  // TODO Merge default writer options with options argument like it is done in load module.\n  if (writer.encode) {\n    return await writer.encode(data, options);\n  }\n\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n\n  if (writer.encodeText) {\n    return new TextEncoder().encode(await writer.encodeText(data, options));\n  }\n\n  if (writer.encodeInBatches) {\n    // Create an iterator representing the data\n    // TODO - Assumes this is a table\n    const batches = encodeInBatches(data, writer, options);\n\n    // Concatenate the output\n    const chunks: any[] = [];\n    for await (const batch of batches) {\n      chunks.push(batch);\n    }\n    // @ts-ignore\n    return concatenateArrayBuffers(...chunks);\n  }\n\n  if (!isBrowser && writer.encodeURLtoURL) {\n    // TODO - how to generate filenames with correct extensions?\n    const tmpInputFilename = getTemporaryFilename('input');\n    await writeFile(tmpInputFilename, data);\n\n    const tmpOutputFilename = getTemporaryFilename('output');\n\n    const outputFilename = await encodeURLtoURL(\n      tmpInputFilename,\n      tmpOutputFilename,\n      writer,\n      options\n    );\n\n    const response = await fetchFile(outputFilename);\n    return response.arrayBuffer();\n  }\n\n  throw new Error('Writer could not encode data');\n}\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport function encodeSync(data: any, writer: Writer, options?: LoaderOptions): ArrayBuffer {\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n  throw new Error('Writer could not synchronously encode data');\n}\n\n/**\n * Encode loaded data to text using the specified Writer\n * @note This is a convenience function not intended for production use on large input data.\n * It is not optimized for performance. Data maybe converted from text to binary and back.\n * @throws if the writer does not generate text output\n */\nexport async function encodeText(\n  data: any,\n  writer: Writer,\n  options?: LoaderOptions\n): Promise<string> {\n  if (writer.text && writer.encodeText) {\n    return await writer.encodeText(data, options);\n  }\n\n  if (writer.text && (writer.encode || writer.encodeInBatches)) {\n    const arrayBuffer = await encode(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n\n  throw new Error('Writer could not encode data as text');\n}\n\n/**\n * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.\n */\nexport function encodeInBatches(\n  data: any,\n  writer: Writer,\n  options?: LoaderOptions\n): AsyncIterable<ArrayBuffer> {\n  if (writer.encodeInBatches) {\n    const dataIterator = getIterator(data);\n    return writer.encodeInBatches(dataIterator, options);\n  }\n  // TODO -fall back to atomic encode?\n  throw new Error('Writer could not encode data in batches');\n}\n\n/**\n * Encode data stored in a file (on disk) to another file.\n * @note Node.js only. This function enables using command-line converters as \"writers\".\n */\nexport async function encodeURLtoURL(\n  inputUrl,\n  outputUrl,\n  writer: Writer,\n  options\n): Promise<string> {\n  inputUrl = resolvePath(inputUrl);\n  outputUrl = resolvePath(outputUrl);\n  if (isBrowser || !writer.encodeURLtoURL) {\n    throw new Error();\n  }\n  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);\n  return outputFilename;\n}\n\n/**\n * @todo TODO - this is an unacceptable hack!!!\n */\nfunction getIterator(data) {\n  const dataIterator = [{table: data, start: 0, end: data.length}];\n  return dataIterator;\n}\n\n/**\n * @todo Move to utils\n */\nfunction getTemporaryFilename(filename: string): string {\n  return `/tmp/${filename}`;\n}\n"],"file":"encode.js"}

package/dist/lib/api/load-in-batches.js

@@ -1,33 +1,30 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadInBatches = void 0;
-const normalize_loader_1 = require("../loader-utils/normalize-loader");
-const option_utils_1 = require("../loader-utils/option-utils");
-const parse_in_batches_1 = require("./parse-in-batches");
-function loadInBatches(files, loaders, options, context) {
-    // Signature: load(url, options)
-    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
-        context = undefined; // context not supported in short signature
-        options = loaders;
-        loaders = null;
-    }
-    // Select fetch function
-    const fetch = (0, option_utils_1.getFetchFunction)(options || {});
-    // Single url/file
-    if (!Array.isArray(files)) {
-        return loadOneFileInBatches(files, loaders, options, fetch);
-    }
-    // Multiple URLs / files
-    const promises = files.map((file) => loadOneFileInBatches(file, loaders, options, fetch));
-    // No point in waiting here for all responses before starting to stream individual streams?
-    return promises;
+import { isLoaderObject } from '../loader-utils/normalize-loader';
+import { getFetchFunction } from '../loader-utils/option-utils';
+import { parseInBatches } from './parse-in-batches';
+export function loadInBatches(files, loaders, options, context) {
+  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
+    context = undefined;
+    options = loaders;
+    loaders = null;
+  }
+
+  const fetch = getFetchFunction(options || {});
+
+  if (!Array.isArray(files)) {
+    return loadOneFileInBatches(files, loaders, options, fetch);
+  }
+
+  const promises = files.map(file => loadOneFileInBatches(file, loaders, options, fetch));
+  return promises;
 }
-exports.loadInBatches = loadInBatches;
+
 async function loadOneFileInBatches(file, loaders, options, fetch) {
-    if (typeof file === 'string') {
-        const url = file;
-        const response = await fetch(url);
-        return await (0, parse_in_batches_1.parseInBatches)(response, loaders, options);
-    }
-    return await (0, parse_in_batches_1.parseInBatches)(file, loaders, options);
+  if (typeof file === 'string') {
+    const url = file;
+    const response = await fetch(url);
+    return await parseInBatches(response, loaders, options);
+  }
+
+  return await parseInBatches(file, loaders, options);
 }
+//# sourceMappingURL=load-in-batches.js.map

package/dist/lib/api/load-in-batches.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/api/load-in-batches.ts"],"names":["isLoaderObject","getFetchFunction","parseInBatches","loadInBatches","files","loaders","options","context","Array","isArray","undefined","fetch","loadOneFileInBatches","promises","map","file","url","response"],"mappings":"AACA,SAAQA,cAAR,QAA6B,kCAA7B;AACA,SAAQC,gBAAR,QAA+B,8BAA/B;AAEA,SAAQC,cAAR,QAA6B,oBAA7B;AAyBA,OAAO,SAASC,aAAT,CAAuBC,KAAvB,EAA8BC,OAA9B,EAAuCC,OAAvC,EAAgDC,OAAhD,EAAyD;AAE9D,MAAI,CAACC,KAAK,CAACC,OAAN,CAAcJ,OAAd,CAAD,IAA2B,CAACL,cAAc,CAACK,OAAD,CAA9C,EAAyD;AACvDE,IAAAA,OAAO,GAAGG,SAAV;AACAJ,IAAAA,OAAO,GAAGD,OAAV;AACAA,IAAAA,OAAO,GAAG,IAAV;AACD;;AAGD,QAAMM,KAAK,GAAGV,gBAAgB,CAACK,OAAO,IAAI,EAAZ,CAA9B;;AAGA,MAAI,CAACE,KAAK,CAACC,OAAN,CAAcL,KAAd,CAAL,EAA2B;AACzB,WAAOQ,oBAAoB,CAACR,KAAD,EAAQC,OAAR,EAAiBC,OAAjB,EAA0BK,KAA1B,CAA3B;AACD;;AAGD,QAAME,QAAQ,GAAGT,KAAK,CAACU,GAAN,CAAWC,IAAD,IAAUH,oBAAoB,CAACG,IAAD,EAAOV,OAAP,EAAgBC,OAAhB,EAAyBK,KAAzB,CAAxC,CAAjB;AAGA,SAAOE,QAAP;AACD;;AAED,eAAeD,oBAAf,CAAoCG,IAApC,EAA0CV,OAA1C,EAAmDC,OAAnD,EAA4DK,KAA5D,EAAmE;AACjE,MAAI,OAAOI,IAAP,KAAgB,QAApB,EAA8B;AAC5B,UAAMC,GAAG,GAAGD,IAAZ;AACA,UAAME,QAAQ,GAAG,MAAMN,KAAK,CAACK,GAAD,CAA5B;AACA,WAAO,MAAMd,cAAc,CAACe,QAAD,EAAWZ,OAAX,EAAoBC,OAApB,CAA3B;AACD;;AACD,SAAO,MAAMJ,cAAc,CAACa,IAAD,EAAOV,OAAP,EAAgBC,OAAhB,CAA3B;AACD","sourcesContent":["import type {LoaderWithParser, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {getFetchFunction} from '../loader-utils/option-utils';\n\nimport {parseInBatches} from './parse-in-batches';\n\ntype FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport function loadInBatches(\n  files: FileType,\n  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<any>>;\n\nexport function loadInBatches(\n  files: FileType[] | FileList,\n  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<any>>;\n\nexport function loadInBatches(files, loaders, options, context) {\n  // Signature: load(url, options)\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders;\n    loaders = null;\n  }\n\n  // Select fetch function\n  const fetch = getFetchFunction(options || {});\n\n  // Single url/file\n  if (!Array.isArray(files)) {\n    return loadOneFileInBatches(files, loaders, options, fetch);\n  }\n\n  // Multiple URLs / files\n  const promises = files.map((file) => loadOneFileInBatches(file, loaders, options, fetch));\n\n  // No point in waiting here for all responses before starting to stream individual streams?\n  return promises;\n}\n\nasync function loadOneFileInBatches(file, loaders, options, fetch) {\n  if (typeof file === 'string') {\n    const url = file;\n    const response = await fetch(url);\n    return await parseInBatches(response, loaders, options);\n  }\n  return await parseInBatches(file, loaders, options);\n}\n"],"file":"load-in-batches.js"}

package/dist/lib/api/load.js

@@ -1,41 +1,25 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.load = void 0;
-const is_type_1 = require("../../javascript-utils/is-type");
-const normalize_loader_1 = require("../loader-utils/normalize-loader");
-const option_utils_1 = require("../loader-utils/option-utils");
-const parse_1 = require("./parse");
-/**
- * Parses `data` using a specified loader
- * Note: Load does duplicate a lot of parse.
- * it can also call fetchFile on string urls, which `parse` won't do.
- * @param data
- * @param loaders
- * @param options
- * @param context
- */
-// implementation signature
-async function load(url, loaders, options, context) {
-    // Signature: load(url, options)
-    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
-        context = undefined; // context not supported in short signature
-        options = loaders;
-        loaders = undefined;
-    }
-    // Select fetch function
-    const fetch = (0, option_utils_1.getFetchFunction)(options);
-    // at this point, `url` could be already loaded binary data
-    let data = url;
-    // url is a string, fetch the url
-    if (typeof url === 'string') {
-        data = await fetch(url);
-        // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
-    }
-    if ((0, is_type_1.isBlob)(url)) {
-        // The fetch response object will contain blob.name
-        data = await fetch(url);
-    }
-    // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
-    return await (0, parse_1.parse)(data, loaders, options);
+import { isBlob } from '../../javascript-utils/is-type';
+import { isLoaderObject } from '../loader-utils/normalize-loader';
+import { getFetchFunction } from '../loader-utils/option-utils';
+import { parse } from './parse';
+export async function load(url, loaders, options, context) {
+  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
+    context = undefined;
+    options = loaders;
+    loaders = undefined;
+  }
+
+  const fetch = getFetchFunction(options);
+  let data = url;
+
+  if (typeof url === 'string') {
+    data = await fetch(url);
+  }
+
+  if (isBlob(url)) {
+    data = await fetch(url);
+  }
+
+  return await parse(data, loaders, options);
 }
-exports.load = load;
+//# sourceMappingURL=load.js.map

package/dist/lib/api/load.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/api/load.ts"],"names":["isBlob","isLoaderObject","getFetchFunction","parse","load","url","loaders","options","context","Array","isArray","undefined","fetch","data"],"mappings":"AACA,SAAQA,MAAR,QAAqB,gCAArB;AACA,SAAQC,cAAR,QAA6B,kCAA7B;AACA,SAAQC,gBAAR,QAA+B,8BAA/B;AAEA,SAAQC,KAAR,QAAoB,SAApB;AAYA,OAAO,eAAeC,IAAf,CACLC,GADK,EAELC,OAFK,EAGLC,OAHK,EAILC,OAJK,EAKS;AAEd,MAAI,CAACC,KAAK,CAACC,OAAN,CAAcJ,OAAd,CAAD,IAA2B,CAACL,cAAc,CAACK,OAAD,CAA9C,EAAyD;AACvDE,IAAAA,OAAO,GAAGG,SAAV;AACAJ,IAAAA,OAAO,GAAGD,OAAV;AACAA,IAAAA,OAAO,GAAGK,SAAV;AACD;;AAGD,QAAMC,KAAK,GAAGV,gBAAgB,CAACK,OAAD,CAA9B;AAGA,MAAIM,IAAI,GAAGR,GAAX;;AAEA,MAAI,OAAOA,GAAP,KAAe,QAAnB,EAA6B;AAC3BQ,IAAAA,IAAI,GAAG,MAAMD,KAAK,CAACP,GAAD,CAAlB;AAED;;AAED,MAAIL,MAAM,CAACK,GAAD,CAAV,EAAiB;AAEfQ,IAAAA,IAAI,GAAG,MAAMD,KAAK,CAACP,GAAD,CAAlB;AACD;;AAGD,SAAO,MAAMF,KAAK,CAACU,IAAD,EAAOP,OAAP,EAAgBC,OAAhB,CAAlB;AACD","sourcesContent":["import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {isBlob} from '../../javascript-utils/is-type';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {getFetchFunction} from '../loader-utils/option-utils';\n\nimport {parse} from './parse';\n\n/**\n * Parses `data` using a specified loader\n * Note: Load does duplicate a lot of parse.\n * it can also call fetchFile on string urls, which `parse` won't do.\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\n// implementation signature\nexport async function load(\n  url: string | DataType,\n  loaders?: Loader | Loader[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<any> {\n  // Signature: load(url, options)\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders as LoaderOptions;\n    loaders = undefined;\n  }\n\n  // Select fetch function\n  const fetch = getFetchFunction(options);\n\n  // at this point, `url` could be already loaded binary data\n  let data = url;\n  // url is a string, fetch the url\n  if (typeof url === 'string') {\n    data = await fetch(url);\n    // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)\n  }\n\n  if (isBlob(url)) {\n    // The fetch response object will contain blob.name\n    data = await fetch(url);\n  }\n\n  // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`\n  return await parse(data, loaders, options);\n}\n"],"file":"load.js"}

package/dist/lib/api/parse-in-batches.js

@@ -1,120 +1,99 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseInBatches = void 0;
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const normalize_loader_1 = require("../loader-utils/normalize-loader");
-const option_utils_1 = require("../loader-utils/option-utils");
-const loader_context_1 = require("../loader-utils/loader-context");
-const get_data_1 = require("../loader-utils/get-data");
-const resource_utils_1 = require("../utils/resource-utils");
-const select_loader_1 = require("./select-loader");
-// Ensure `parse` is available in context if loader falls back to `parse`
-const parse_1 = require("./parse");
-/**
- * Parses `data` using a specified loader
- * @param data
- * @param loaders
- * @param options
- * @param context
- */
-async function parseInBatches(data, loaders, options, context) {
-    (0, loader_utils_1.assert)(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
-    // Signature: parseInBatches(data, options, url) - Uses registered loaders
-    if (!Array.isArray(loaders) && !(0, normalize_loader_1.isLoaderObject)(loaders)) {
-        context = undefined; // context not supported in short signature
-        options = loaders;
-        loaders = undefined;
-    }
-    data = await data; // Resolve any promise
-    options = options || {};
-    // Extract a url for auto detection
-    const { url } = (0, resource_utils_1.getResourceUrlAndType)(data);
-    // Chooses a loader and normalizes it
-    // Note - only uses URL and contentType for streams and iterator inputs
-    const loader = await (0, select_loader_1.selectLoader)(data, loaders, options);
-    // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
-    if (!loader) {
-        // @ts-ignore
-        return null;
-    }
-    // Normalize options
-    // @ts-ignore
-    options = (0, option_utils_1.normalizeOptions)(options, loader, loaders, url);
-    // @ts-ignore
-    context = (0, loader_context_1.getLoaderContext)(
-    // @ts-ignore
-    { url, parseInBatches, parse: parse_1.parse, loaders: loaders }, options, context);
-    return await parseWithLoaderInBatches(loader, data, options, context);
+import { assert, concatenateArrayBuffersAsync } from '@loaders.gl/loader-utils';
+import { isLoaderObject } from '../loader-utils/normalize-loader';
+import { normalizeOptions } from '../loader-utils/option-utils';
+import { getLoaderContext } from '../loader-utils/loader-context';
+import { getAsyncIterableFromData } from '../loader-utils/get-data';
+import { getResourceUrlAndType } from '../utils/resource-utils';
+import { selectLoader } from './select-loader';
+import { parse } from './parse';
+export async function parseInBatches(data, loaders, options, context) {
+  assert(!context || typeof context === 'object');
+
+  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
+    context = undefined;
+    options = loaders;
+    loaders = undefined;
+  }
+
+  data = await data;
+  options = options || {};
+  const {
+    url
+  } = getResourceUrlAndType(data);
+  const loader = await selectLoader(data, loaders, options);
+
+  if (!loader) {
+    return null;
+  }
+
+  options = normalizeOptions(options, loader, loaders, url);
+  context = getLoaderContext({
+    url,
+    parseInBatches,
+    parse,
+    loaders: loaders
+  }, options, context);
+  return await parseWithLoaderInBatches(loader, data, options, context);
 }
-exports.parseInBatches = parseInBatches;
-/**
- * Loader has been selected and context has been prepared, see if we need to emit a metadata batch
- */
+
 async function parseWithLoaderInBatches(loader, data, options, context) {
-    const outputIterator = await parseToOutputIterator(loader, data, options, context);
-    // Generate metadata batch if requested
-    if (!options.metadata) {
-        return outputIterator;
-    }
-    const metadataBatch = {
-        batchType: 'metadata',
-        metadata: {
-            _loader: loader,
-            _context: context
-        },
-        // Populate with some default fields to avoid crashing
-        data: [],
-        bytesUsed: 0
-    };
-    async function* makeMetadataBatchIterator(iterator) {
-        yield metadataBatch;
-        yield* iterator;
-    }
-    return makeMetadataBatchIterator(outputIterator);
+  const outputIterator = await parseToOutputIterator(loader, data, options, context);
+
+  if (!options.metadata) {
+    return outputIterator;
+  }
+
+  const metadataBatch = {
+    batchType: 'metadata',
+    metadata: {
+      _loader: loader,
+      _context: context
+    },
+    data: [],
+    bytesUsed: 0
+  };
+
+  async function* makeMetadataBatchIterator(iterator) {
+    yield metadataBatch;
+    yield* iterator;
+  }
+
+  return makeMetadataBatchIterator(outputIterator);
 }
-/**
- * Prep work is done, now it is time to start parsing into an output operator
- * The approach depends on which parse function the loader exposes
- * `parseInBatches` (preferred), `parse` (fallback)
- */
+
 async function parseToOutputIterator(loader, data, options, context) {
-    // Get an iterator from the input
-    const inputIterator = await (0, get_data_1.getAsyncIterableFromData)(data, options);
-    // Apply any iterator transforms (options.transforms)
-    const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);
-    // If loader supports parseInBatches, we are done
-    if (loader.parseInBatches) {
-        return loader.parseInBatches(transformedIterator, options, context);
-    }
-    // Fallback: load atomically using `parse` concatenating input iterator into single chunk
-    async function* parseChunkInBatches() {
-        const arrayBuffer = await (0, loader_utils_1.concatenateArrayBuffersAsync)(transformedIterator);
-        // Call `parse` instead of `loader.parse` to ensure we can call workers etc.
-        const parsedData = await (0, parse_1.parse)(arrayBuffer, loader, 
-        // TODO - Hack: supply loaders MIME type to ensure we match it
-        { ...options, mimeType: loader.mimeTypes[0] }, context);
-        // yield a single batch, the output from loader.parse()
-        // TODO - run through batch builder to apply options etc...
-        const batch = {
-            mimeType: loader.mimeTypes[0],
-            shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
-            batchType: 'data',
-            data: parsedData,
-            length: Array.isArray(parsedData) ? parsedData.length : 1
-        };
-        yield batch;
-    }
-    return parseChunkInBatches();
+  const inputIterator = await getAsyncIterableFromData(data, options);
+  const transformedIterator = await applyInputTransforms(inputIterator, (options === null || options === void 0 ? void 0 : options.transforms) || []);
+
+  if (loader.parseInBatches) {
+    return loader.parseInBatches(transformedIterator, options, context);
+  }
+
+  async function* parseChunkInBatches() {
+    const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
+    const parsedData = await parse(arrayBuffer, loader, { ...options,
+      mimeType: loader.mimeTypes[0]
+    }, context);
+    const batch = {
+      mimeType: loader.mimeTypes[0],
+      shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
+      batchType: 'data',
+      data: parsedData,
+      length: Array.isArray(parsedData) ? parsedData.length : 1
+    };
+    yield batch;
+  }
+
+  return parseChunkInBatches();
 }
-/**
- * Create an iterator chain with any transform iterators (crypto, decompression)
- * @param inputIterator
- * @param options
- */
+
 async function applyInputTransforms(inputIterator, transforms = []) {
-    let iteratorChain = inputIterator;
-    for await (const transformBatches of transforms) {
-        iteratorChain = transformBatches(iteratorChain);
-    }
-    return iteratorChain;
+  let iteratorChain = inputIterator;
+
+  for await (const transformBatches of transforms) {
+    iteratorChain = transformBatches(iteratorChain);
+  }
+
+  return iteratorChain;
 }
+//# sourceMappingURL=parse-in-batches.js.map

package/dist/lib/api/parse-in-batches.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/api/parse-in-batches.ts"],"names":["assert","concatenateArrayBuffersAsync","isLoaderObject","normalizeOptions","getLoaderContext","getAsyncIterableFromData","getResourceUrlAndType","selectLoader","parse","parseInBatches","data","loaders","options","context","Array","isArray","undefined","url","loader","parseWithLoaderInBatches","outputIterator","parseToOutputIterator","metadata","metadataBatch","batchType","_loader","_context","bytesUsed","makeMetadataBatchIterator","iterator","inputIterator","transformedIterator","applyInputTransforms","transforms","parseChunkInBatches","arrayBuffer","parsedData","mimeType","mimeTypes","batch","shape","length","iteratorChain","transformBatches"],"mappings":"AAQA,SAAQA,MAAR,EAAgBC,4BAAhB,QAAmD,0BAAnD;AACA,SAAQC,cAAR,QAA6B,kCAA7B;AACA,SAAQC,gBAAR,QAA+B,8BAA/B;AACA,SAAQC,gBAAR,QAA+B,gCAA/B;AACA,SAAQC,wBAAR,QAAuC,0BAAvC;AACA,SAAQC,qBAAR,QAAoC,yBAApC;AACA,SAAQC,YAAR,QAA2B,iBAA3B;AAGA,SAAQC,KAAR,QAAoB,SAApB;AASA,OAAO,eAAeC,cAAf,CACLC,IADK,EAELC,OAFK,EAGLC,OAHK,EAILC,OAJK,EAKwB;AAC7Bb,EAAAA,MAAM,CAAC,CAACa,OAAD,IAAY,OAAOA,OAAP,KAAmB,QAAhC,CAAN;;AAGA,MAAI,CAACC,KAAK,CAACC,OAAN,CAAcJ,OAAd,CAAD,IAA2B,CAACT,cAAc,CAACS,OAAD,CAA9C,EAAyD;AACvDE,IAAAA,OAAO,GAAGG,SAAV;AACAJ,IAAAA,OAAO,GAAGD,OAAV;AACAA,IAAAA,OAAO,GAAGK,SAAV;AACD;;AAEDN,EAAAA,IAAI,GAAG,MAAMA,IAAb;AACAE,EAAAA,OAAO,GAAGA,OAAO,IAAI,EAArB;AAGA,QAAM;AAACK,IAAAA;AAAD,MAAQX,qBAAqB,CAACI,IAAD,CAAnC;AAIA,QAAMQ,MAAM,GAAG,MAAMX,YAAY,CAACG,IAAD,EAAsBC,OAAtB,EAA2CC,OAA3C,CAAjC;;AAEA,MAAI,CAACM,MAAL,EAAa;AAEX,WAAO,IAAP;AACD;;AAIDN,EAAAA,OAAO,GAAGT,gBAAgB,CAACS,OAAD,EAAUM,MAAV,EAAkBP,OAAlB,EAA2BM,GAA3B,CAA1B;AAEAJ,EAAAA,OAAO,GAAGT,gBAAgB,CAExB;AAACa,IAAAA,GAAD;AAAMR,IAAAA,cAAN;AAAsBD,IAAAA,KAAtB;AAA6BG,IAAAA,OAAO,EAAEA;AAAtC,GAFwB,EAGxBC,OAHwB,EAIxBC,OAJwB,CAA1B;AAOA,SAAO,MAAMM,wBAAwB,CAACD,MAAD,EAA6BR,IAA7B,EAAmCE,OAAnC,EAA4CC,OAA5C,CAArC;AACD;;AAKD,eAAeM,wBAAf,CACED,MADF,EAEER,IAFF,EAGEE,OAHF,EAIEC,OAJF,EAK+B;AAC7B,QAAMO,cAAc,GAAG,MAAMC,qBAAqB,CAACH,MAAD,EAASR,IAAT,EAAeE,OAAf,EAAwBC,OAAxB,CAAlD;;AAGA,MAAI,CAACD,OAAO,CAACU,QAAb,EAAuB;AACrB,WAAOF,cAAP;AACD;;AAED,QAAMG,aAAa,GAAG;AACpBC,IAAAA,SAAS,EAAE,UADS;AAEpBF,IAAAA,QAAQ,EAAE;AACRG,MAAAA,OAAO,EAAEP,MADD;AAERQ,MAAAA,QAAQ,EAAEb;AAFF,KAFU;AAOpBH,IAAAA,IAAI,EAAE,EAPc;AAQpBiB,IAAAA,SAAS,EAAE;AARS,GAAtB;;AAWA,kBAAgBC,yBAAhB,CAA0CC,QAA1C,EAAoD;AAClD,UAAMN,aAAN;AACA,WAAOM,QAAP;AACD;;AAED,SAAOD,yBAAyB,CAACR,cAAD,CAAhC;AACD;;AAOD,eAAeC,qBAAf,CACEH,MADF,EAEER,IAFF,EAGEE,OAHF,EAIEC,OAJF,EAK+B;AAE7B,QAAMiB,aAAa,GAAG,MAAMzB,wBAAwB,CAACK,IAAD,EAAOE,OAAP,CAApD;AAGA,QAAMmB,mBAAmB,GAAG,MAAMC,oBAAoB,CAACF,aAAD,EAAgB,CAAAlB,OAAO,SAAP,IAAAA,OAAO,WAAP,YAAAA,OAAO,CAAEqB,UAAT,KAAuB,EAAvC,CAAtD;;AAGA,MAAIf,MAAM,CAACT,cAAX,EAA2B;AACzB,WAAOS,MAAM,CAACT,cAAP,CAAsBsB,mBAAtB,EAA2CnB,OAA3C,EAAoDC,OAApD,CAAP;AACD;;AAGD,kBAAgBqB,mBAAhB,GAAsC;AACpC,UAAMC,WAAW,GAAG,MAAMlC,4BAA4B,CAAC8B,mBAAD,CAAtD;AAEA,UAAMK,UAAU,GAAG,MAAM5B,KAAK,CAC5B2B,WAD4B,EAE5BjB,MAF4B,EAI5B,EAAC,GAAGN,OAAJ;AAAayB,MAAAA,QAAQ,EAAEnB,MAAM,CAACoB,SAAP,CAAiB,CAAjB;AAAvB,KAJ4B,EAK5BzB,OAL4B,CAA9B;AASA,UAAM0B,KAAY,GAAG;AACnBF,MAAAA,QAAQ,EAAEnB,MAAM,CAACoB,SAAP,CAAiB,CAAjB,CADS;AAEnBE,MAAAA,KAAK,EAAE1B,KAAK,CAACC,OAAN,CAAcqB,UAAd,IAA4B,WAA5B,GAA0C,SAF9B;AAGnBZ,MAAAA,SAAS,EAAE,MAHQ;AAInBd,MAAAA,IAAI,EAAE0B,UAJa;AAKnBK,MAAAA,MAAM,EAAE3B,KAAK,CAACC,OAAN,CAAcqB,UAAd,IAA4BA,UAAU,CAACK,MAAvC,GAAgD;AALrC,KAArB;AAOA,UAAMF,KAAN;AACD;;AAED,SAAOL,mBAAmB,EAA1B;AACD;;AAWD,eAAeF,oBAAf,CACEF,aADF,EAEEG,UAA8B,GAAG,EAFnC,EAG+D;AAC7D,MAAIS,aAAa,GAAGZ,aAApB;;AACA,aAAW,MAAMa,gBAAjB,IAAqCV,UAArC,EAAiD;AAC/CS,IAAAA,aAAa,GAAGC,gBAAgB,CAACD,aAAD,CAAhC;AACD;;AACD,SAAOA,aAAP;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {\n  BatchableDataType,\n  Loader,\n  LoaderWithParser,\n  LoaderContext,\n  LoaderOptions\n} from '@loaders.gl/loader-utils';\nimport {assert, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getLoaderContext} from '../loader-utils/loader-context';\nimport {getAsyncIterableFromData} from '../loader-utils/get-data';\nimport {getResourceUrlAndType} from '../utils/resource-utils';\nimport {selectLoader} from './select-loader';\n\n// Ensure `parse` is available in context if loader falls back to `parse`\nimport {parse} from './parse';\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  loaders?: Loader | Loader[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<any>> {\n  assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url\n\n  // Signature: parseInBatches(data, options, url) - Uses registered loaders\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders as LoaderOptions;\n    loaders = undefined;\n  }\n\n  data = await data; // Resolve any promise\n  options = options || {};\n\n  // Extract a url for auto detection\n  const {url} = getResourceUrlAndType(data);\n\n  // Chooses a loader and normalizes it\n  // Note - only uses URL and contentType for streams and iterator inputs\n  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);\n  // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null\n  if (!loader) {\n    // @ts-ignore\n    return null;\n  }\n\n  // Normalize options\n  // @ts-ignore\n  options = normalizeOptions(options, loader, loaders, url);\n  // @ts-ignore\n  context = getLoaderContext(\n    // @ts-ignore\n    {url, parseInBatches, parse, loaders: loaders as Loader[]},\n    options,\n    context\n  );\n\n  return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);\n}\n\n/**\n * Loader has been selected and context has been prepared, see if we need to emit a metadata batch\n */\nasync function parseWithLoaderInBatches(\n  loader: LoaderWithParser,\n  data: BatchableDataType,\n  options: LoaderOptions,\n  context: LoaderContext\n): Promise<AsyncIterable<any>> {\n  const outputIterator = await parseToOutputIterator(loader, data, options, context);\n\n  // Generate metadata batch if requested\n  if (!options.metadata) {\n    return outputIterator;\n  }\n\n  const metadataBatch = {\n    batchType: 'metadata',\n    metadata: {\n      _loader: loader,\n      _context: context\n    },\n    // Populate with some default fields to avoid crashing\n    data: [],\n    bytesUsed: 0\n  };\n\n  async function* makeMetadataBatchIterator(iterator) {\n    yield metadataBatch;\n    yield* iterator;\n  }\n\n  return makeMetadataBatchIterator(outputIterator);\n}\n\n/**\n * Prep work is done, now it is time to start parsing into an output operator\n * The approach depends on which parse function the loader exposes\n * `parseInBatches` (preferred), `parse` (fallback)\n */\nasync function parseToOutputIterator(\n  loader: LoaderWithParser,\n  data: BatchableDataType,\n  options: LoaderOptions,\n  context: LoaderContext\n): Promise<AsyncIterable<any>> {\n  // Get an iterator from the input\n  const inputIterator = await getAsyncIterableFromData(data, options);\n\n  // Apply any iterator transforms (options.transforms)\n  const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);\n\n  // If loader supports parseInBatches, we are done\n  if (loader.parseInBatches) {\n    return loader.parseInBatches(transformedIterator, options, context);\n  }\n\n  // Fallback: load atomically using `parse` concatenating input iterator into single chunk\n  async function* parseChunkInBatches() {\n    const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);\n    // Call `parse` instead of `loader.parse` to ensure we can call workers etc.\n    const parsedData = await parse(\n      arrayBuffer,\n      loader,\n      // TODO - Hack: supply loaders MIME type to ensure we match it\n      {...options, mimeType: loader.mimeTypes[0]},\n      context\n    );\n    // yield a single batch, the output from loader.parse()\n    // TODO - run through batch builder to apply options etc...\n    const batch: Batch = {\n      mimeType: loader.mimeTypes[0],\n      shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',\n      batchType: 'data',\n      data: parsedData,\n      length: Array.isArray(parsedData) ? parsedData.length : 1\n    };\n    yield batch;\n  }\n\n  return parseChunkInBatches();\n}\n\ntype TransformBatches = (\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n) => AsyncIterable<ArrayBuffer>;\n\n/**\n * Create an iterator chain with any transform iterators (crypto, decompression)\n * @param inputIterator\n * @param options\n */\nasync function applyInputTransforms(\n  inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  transforms: TransformBatches[] = []\n): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {\n  let iteratorChain = inputIterator;\n  for await (const transformBatches of transforms) {\n    iteratorChain = transformBatches(iteratorChain);\n  }\n  return iteratorChain;\n}\n"],"file":"parse-in-batches.js"}