@loaders.gl/core 4.0.2 → 4.0.3

package/dist/dist.dev.js

@@ -125,7 +125,7 @@ var __exports__ = (() => {
   }
 
   // ../worker-utils/src/lib/env-utils/version.ts
-  var NPM_TAG = "beta";
+  var NPM_TAG = "latest";
   function getVersion() {
     if (!globalThis._loadersgl_?.version) {
       globalThis._loadersgl_ = globalThis._loadersgl_ || {};
@@ -2997,6 +2997,52 @@ var __exports__ = (() => {
     throw new Error(`${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${context.url || ""}`);
   }
 
+  // ../schema/src/lib/table/simple-table/table-accessors.ts
+  function isTable(table) {
+    const shape = typeof table === "object" && table?.shape;
+    switch (shape) {
+      case "array-row-table":
+      case "object-row-table":
+        return Array.isArray(table.data);
+      case "geojson-table":
+        return Array.isArray(table.features);
+      case "columnar-table":
+        return table.data && typeof table.data === "object";
+      case "arrow-table":
+        return Boolean(table?.data?.numRows !== void 0);
+      default:
+        return false;
+    }
+  }
+  function getTableLength(table) {
+    switch (table.shape) {
+      case "array-row-table":
+      case "object-row-table":
+        return table.data.length;
+      case "geojson-table":
+        return table.features.length;
+      case "arrow-table":
+        const arrowTable = table.data;
+        return arrowTable.numRows;
+      case "columnar-table":
+        for (const column of Object.values(table.data)) {
+          return column.length || 0;
+        }
+        return 0;
+      default:
+        throw new Error("table");
+    }
+  }
+
+  // ../schema/src/lib/table/simple-table/make-table-from-batches.ts
+  function makeBatchFromTable(table) {
+    return {
+      ...table,
+      length: getTableLength(table),
+      batchType: "data"
+    };
+  }
+
   // src/lib/api/parse-in-batches.ts
   async function parseInBatches(data, loaders, options, context) {
     const loaderArray = Array.isArray(loaders) ? loaders : void 0;
@@ -3047,22 +3093,26 @@ var __exports__ = (() => {
     if (loader.parseInBatches) {
       return loader.parseInBatches(transformedIterator, options, context);
     }
-    async function* parseChunkInBatches() {
-      const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
-      const parsedData = await parse(arrayBuffer, loader, {
-        ...options,
-        mimeType: loader.mimeTypes[0]
-      }, context);
-      const batch = {
-        mimeType: loader.mimeTypes[0],
-        shape: Array.isArray(parsedData) ? "row-table" : "unknown",
-        batchType: "data",
-        data: parsedData,
-        length: Array.isArray(parsedData) ? parsedData.length : 1
-      };
-      yield batch;
-    }
-    return parseChunkInBatches();
+    return parseChunkInBatches(transformedIterator, loader, options, context);
+  }
+  async function* parseChunkInBatches(transformedIterator, loader, options, context) {
+    const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
+    const parsedData = await parse(arrayBuffer, loader, {
+      ...options,
+      mimeType: loader.mimeTypes[0]
+    }, context);
+    const batch = convertDataToBatch(parsedData, loader);
+    yield batch;
+  }
+  function convertDataToBatch(parsedData, loader) {
+    const batch = isTable(parsedData) ? makeBatchFromTable(parsedData) : {
+      shape: "unknown",
+      batchType: "data",
+      data: parsedData,
+      length: Array.isArray(parsedData) ? parsedData.length : 1
+    };
+    batch.mimeType = loader.mimeTypes[0];
+    return batch;
   }
   async function applyInputTransforms(inputIterator, transforms = []) {
     let iteratorChain = inputIterator;

package/dist/index.cjs

@@ -1079,6 +1079,7 @@ function parseWithLoaderSync(loader, data, options, context) {
 }
 
 // src/lib/api/parse-in-batches.ts
+var import_schema = require("@loaders.gl/schema");
 var import_loader_utils10 = require("@loaders.gl/loader-utils");
 async function parseInBatches(data, loaders, options, context) {
   const loaderArray = Array.isArray(loaders) ? loaders : void 0;
@@ -1129,25 +1130,29 @@ async function parseToOutputIterator(loader, data, options, context) {
   if (loader.parseInBatches) {
     return loader.parseInBatches(transformedIterator, options, context);
   }
-  async function* parseChunkInBatches() {
-    const arrayBuffer = await (0, import_loader_utils10.concatenateArrayBuffersAsync)(transformedIterator);
-    const parsedData = await parse(
-      arrayBuffer,
-      loader,
-      // TODO - Hack: supply loaders MIME type to ensure we match it
-      { ...options, mimeType: loader.mimeTypes[0] },
-      context
-    );
-    const batch = {
-      mimeType: loader.mimeTypes[0],
-      shape: Array.isArray(parsedData) ? "row-table" : "unknown",
-      batchType: "data",
-      data: parsedData,
-      length: Array.isArray(parsedData) ? parsedData.length : 1
-    };
-    yield batch;
-  }
-  return parseChunkInBatches();
+  return parseChunkInBatches(transformedIterator, loader, options, context);
+}
+async function* parseChunkInBatches(transformedIterator, loader, options, context) {
+  const arrayBuffer = await (0, import_loader_utils10.concatenateArrayBuffersAsync)(transformedIterator);
+  const parsedData = await parse(
+    arrayBuffer,
+    loader,
+    // TODO - Hack: supply loaders MIME type to ensure we match it
+    { ...options, mimeType: loader.mimeTypes[0] },
+    context
+  );
+  const batch = convertDataToBatch(parsedData, loader);
+  yield batch;
+}
+function convertDataToBatch(parsedData, loader) {
+  const batch = (0, import_schema.isTable)(parsedData) ? (0, import_schema.makeBatchFromTable)(parsedData) : {
+    shape: "unknown",
+    batchType: "data",
+    data: parsedData,
+    length: Array.isArray(parsedData) ? parsedData.length : 1
+  };
+  batch.mimeType = loader.mimeTypes[0];
+  return batch;
 }
 async function applyInputTransforms(inputIterator, transforms = []) {
   let iteratorChain = inputIterator;

package/dist/lib/api/encode-table.d.ts

@@ -1,6 +1,6 @@
-import { Writer, WriterOptionsType } from '@loaders.gl/loader-utils';
+import { WriterOptionsType, WriterWithEncoder } from '@loaders.gl/loader-utils';
 import { Table } from '@loaders.gl/schema';
-export declare function encodeTable<WriterT extends Writer = Writer>(data: Table, writer: WriterT, options?: WriterOptionsType<WriterT>): Promise<ArrayBuffer>;
-export declare function encodeTableAsText<WriterT extends Writer = Writer>(data: Table, writer: WriterT, options?: WriterOptionsType<WriterT>): Promise<string>;
-export declare function encodeTableInBatches<WriterT extends Writer = Writer>(data: Table, writer: WriterT, options?: WriterOptionsType<WriterT>): AsyncIterable<ArrayBuffer>;
+export declare function encodeTable<WriterT extends WriterWithEncoder = WriterWithEncoder>(data: Table, writer: WriterT, options?: WriterOptionsType<WriterT>): Promise<ArrayBuffer>;
+export declare function encodeTableAsText<WriterT extends WriterWithEncoder = WriterWithEncoder>(data: Table, writer: WriterT, options?: WriterOptionsType<WriterT>): Promise<string>;
+export declare function encodeTableInBatches<WriterT extends WriterWithEncoder = WriterWithEncoder>(data: Table, writer: WriterT, options?: WriterOptionsType<WriterT>): AsyncIterable<ArrayBuffer>;
 //# sourceMappingURL=encode-table.d.ts.map

package/dist/lib/api/encode-table.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"encode-table.d.ts","sourceRoot":"","sources":["../../../src/lib/api/encode-table.ts"],"names":[],"mappings":"AAKA,OAAO,EAA0B,MAAM,EAAE,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAC5F,OAAO,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAEzC,wBAAsB,WAAW,CAAC,OAAO,SAAS,MAAM,GAAG,MAAM,EAC/D,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,GACnC,OAAO,CAAC,WAAW,CAAC,CAwBtB;AAED,wBAAsB,iBAAiB,CAAC,OAAO,SAAS,MAAM,GAAG,MAAM,EACrE,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,GACnC,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,wBAAgB,oBAAoB,CAAC,OAAO,SAAS,MAAM,GAAG,MAAM,EAClE,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,GACnC,aAAa,CAAC,WAAW,CAAC,CAQ5B"}
+{"version":3,"file":"encode-table.d.ts","sourceRoot":"","sources":["../../../src/lib/api/encode-table.ts"],"names":[],"mappings":"AAKA,OAAO,EAEL,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAEzC,wBAAsB,WAAW,CAAC,OAAO,SAAS,iBAAiB,GAAG,iBAAiB,EACrF,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,GACnC,OAAO,CAAC,WAAW,CAAC,CAwBtB;AAED,wBAAsB,iBAAiB,CAAC,OAAO,SAAS,iBAAiB,GAAG,iBAAiB,EAC3F,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,GACnC,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,wBAAgB,oBAAoB,CAAC,OAAO,SAAS,iBAAiB,GAAG,iBAAiB,EACxF,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,OAAO,EACf,OAAO,CAAC,EAAE,iBAAiB,CAAC,OAAO,CAAC,GACnC,aAAa,CAAC,WAAW,CAAC,CAQ5B"}

package/dist/lib/api/encode-table.js.map

@@ -1 +1 @@
-{"version":3,"file":"encode-table.js","names":["concatenateArrayBuffers","encodeTable","data","writer","options","encode","encodeText","text","TextEncoder","encodeInBatches","batches","encodeTableInBatches","chunks","batch","push","Error","encodeTableAsText","arrayBuffer","TextDecoder","decode","name","dataIterator","getIterator","start","end","length"],"sources":["../../../src/lib/api/encode-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n// Copyright 2022 Foursquare Labs, Inc\n\n/* global TextEncoder, TextDecoder */\nimport {concatenateArrayBuffers, Writer, WriterOptionsType} from '@loaders.gl/loader-utils';\nimport {Table} from '@loaders.gl/schema';\n\nexport async function encodeTable<WriterT extends Writer = Writer>(\n  data: Table,\n  writer: WriterT,\n  options?: WriterOptionsType<WriterT>\n): Promise<ArrayBuffer> {\n  if (writer.encode) {\n    return await writer.encode(data, options);\n  }\n\n  if (writer.encodeText) {\n    const text = await writer.encodeText(data, options);\n    return new TextEncoder().encode(text);\n  }\n\n  if (writer.encodeInBatches) {\n    // Create an iterator representing the data\n    // TODO - Assumes this is a table\n    const batches = encodeTableInBatches(data, writer, options);\n\n    // Concatenate the output\n    const chunks: ArrayBuffer[] = [];\n    for await (const batch of batches) {\n      chunks.push(batch);\n    }\n    return concatenateArrayBuffers(...chunks);\n  }\n\n  throw new Error('Writer could not encode data');\n}\n\nexport async function encodeTableAsText<WriterT extends Writer = Writer>(\n  data: Table,\n  writer: WriterT,\n  options?: WriterOptionsType<WriterT>\n): Promise<string> {\n  if (writer.text && writer.encodeText) {\n    return await writer.encodeText(data, options);\n  }\n\n  if (writer.text) {\n    const arrayBuffer = await encodeTable(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n  throw new Error(`Writer ${writer.name} could not encode data as text`);\n}\n\nexport function encodeTableInBatches<WriterT extends Writer = Writer>(\n  data: Table,\n  writer: WriterT,\n  options?: WriterOptionsType<WriterT>\n): AsyncIterable<ArrayBuffer> {\n  if (writer.encodeInBatches) {\n    const dataIterator = getIterator(data);\n    // @ts-expect-error\n    return writer.encodeInBatches(dataIterator, options);\n  }\n  // TODO -fall back to atomic encode?\n  throw new Error('Writer could not encode data in batches');\n}\n\nfunction getIterator(data: any): Iterable<{start: number; end: number}> {\n  const dataIterator = [{...data, start: 0, end: data.length}];\n  return dataIterator;\n}\n"],"mappings":"AAKA,SAAQA,uBAAuB,QAAkC,0BAA0B;AAG3F,OAAO,eAAeC,WAAWA,CAC/BC,IAAW,EACXC,MAAe,EACfC,OAAoC,EACd;EACtB,IAAID,MAAM,CAACE,MAAM,EAAE;IACjB,OAAO,MAAMF,MAAM,CAACE,MAAM,CAACH,IAAI,EAAEE,OAAO,CAAC;EAC3C;EAEA,IAAID,MAAM,CAACG,UAAU,EAAE;IACrB,MAAMC,IAAI,GAAG,MAAMJ,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;IACnD,OAAO,IAAII,WAAW,CAAC,CAAC,CAACH,MAAM,CAACE,IAAI,CAAC;EACvC;EAEA,IAAIJ,MAAM,CAACM,eAAe,EAAE;IAG1B,MAAMC,OAAO,GAAGC,oBAAoB,CAACT,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IAG3D,MAAMQ,MAAqB,GAAG,EAAE;IAChC,WAAW,MAAMC,KAAK,IAAIH,OAAO,EAAE;MACjCE,MAAM,CAACE,IAAI,CAACD,KAAK,CAAC;IACpB;IACA,OAAOb,uBAAuB,CAAC,GAAGY,MAAM,CAAC;EAC3C;EAEA,MAAM,IAAIG,KAAK,CAAC,8BAA8B,CAAC;AACjD;AAEA,OAAO,eAAeC,iBAAiBA,CACrCd,IAAW,EACXC,MAAe,EACfC,OAAoC,EACnB;EACjB,IAAID,MAAM,CAACI,IAAI,IAAIJ,MAAM,CAACG,UAAU,EAAE;IACpC,OAAO,MAAMH,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;EAC/C;EAEA,IAAID,MAAM,CAACI,IAAI,EAAE;IACf,MAAMU,WAAW,GAAG,MAAMhB,WAAW,CAACC,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IAC5D,OAAO,IAAIc,WAAW,CAAC,CAAC,CAACC,MAAM,CAACF,WAAW,CAAC;EAC9C;EACA,MAAM,IAAIF,KAAK,CAAE,UAASZ,MAAM,CAACiB,IAAK,gCAA+B,CAAC;AACxE;AAEA,OAAO,SAAST,oBAAoBA,CAClCT,IAAW,EACXC,MAAe,EACfC,OAAoC,EACR;EAC5B,IAAID,MAAM,CAACM,eAAe,EAAE;IAC1B,MAAMY,YAAY,GAAGC,WAAW,CAACpB,IAAI,CAAC;IAEtC,OAAOC,MAAM,CAACM,eAAe,CAACY,YAAY,EAAEjB,OAAO,CAAC;EACtD;EAEA,MAAM,IAAIW,KAAK,CAAC,yCAAyC,CAAC;AAC5D;AAEA,SAASO,WAAWA,CAACpB,IAAS,EAA0C;EACtE,MAAMmB,YAAY,GAAG,CAAC;IAAC,GAAGnB,IAAI;IAAEqB,KAAK,EAAE,CAAC;IAAEC,GAAG,EAAEtB,IAAI,CAACuB;EAAM,CAAC,CAAC;EAC5D,OAAOJ,YAAY;AACrB"}
+{"version":3,"file":"encode-table.js","names":["concatenateArrayBuffers","encodeTable","data","writer","options","encode","encodeText","text","TextEncoder","encodeInBatches","batches","encodeTableInBatches","chunks","batch","push","Error","encodeTableAsText","arrayBuffer","TextDecoder","decode","name","dataIterator","getIterator","start","end","length"],"sources":["../../../src/lib/api/encode-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n// Copyright 2022 Foursquare Labs, Inc\n\n/* global TextEncoder, TextDecoder */\nimport {\n  concatenateArrayBuffers,\n  WriterOptionsType,\n  WriterWithEncoder\n} from '@loaders.gl/loader-utils';\nimport {Table} from '@loaders.gl/schema';\n\nexport async function encodeTable<WriterT extends WriterWithEncoder = WriterWithEncoder>(\n  data: Table,\n  writer: WriterT,\n  options?: WriterOptionsType<WriterT>\n): Promise<ArrayBuffer> {\n  if (writer.encode) {\n    return await writer.encode(data, options);\n  }\n\n  if (writer.encodeText) {\n    const text = await writer.encodeText(data, options);\n    return new TextEncoder().encode(text);\n  }\n\n  if (writer.encodeInBatches) {\n    // Create an iterator representing the data\n    // TODO - Assumes this is a table\n    const batches = encodeTableInBatches(data, writer, options);\n\n    // Concatenate the output\n    const chunks: ArrayBuffer[] = [];\n    for await (const batch of batches) {\n      chunks.push(batch);\n    }\n    return concatenateArrayBuffers(...chunks);\n  }\n\n  throw new Error('Writer could not encode data');\n}\n\nexport async function encodeTableAsText<WriterT extends WriterWithEncoder = WriterWithEncoder>(\n  data: Table,\n  writer: WriterT,\n  options?: WriterOptionsType<WriterT>\n): Promise<string> {\n  if (writer.text && writer.encodeText) {\n    return await writer.encodeText(data, options);\n  }\n\n  if (writer.text) {\n    const arrayBuffer = await encodeTable(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n  throw new Error(`Writer ${writer.name} could not encode data as text`);\n}\n\nexport function encodeTableInBatches<WriterT extends WriterWithEncoder = WriterWithEncoder>(\n  data: Table,\n  writer: WriterT,\n  options?: WriterOptionsType<WriterT>\n): AsyncIterable<ArrayBuffer> {\n  if (writer.encodeInBatches) {\n    const dataIterator = getIterator(data);\n    // @ts-expect-error\n    return writer.encodeInBatches(dataIterator, options);\n  }\n  // TODO -fall back to atomic encode?\n  throw new Error('Writer could not encode data in batches');\n}\n\nfunction getIterator(data: any): Iterable<{start: number; end: number}> {\n  const dataIterator = [{...data, start: 0, end: data.length}];\n  return dataIterator;\n}\n"],"mappings":"AAKA,SACEA,uBAAuB,QAGlB,0BAA0B;AAGjC,OAAO,eAAeC,WAAWA,CAC/BC,IAAW,EACXC,MAAe,EACfC,OAAoC,EACd;EACtB,IAAID,MAAM,CAACE,MAAM,EAAE;IACjB,OAAO,MAAMF,MAAM,CAACE,MAAM,CAACH,IAAI,EAAEE,OAAO,CAAC;EAC3C;EAEA,IAAID,MAAM,CAACG,UAAU,EAAE;IACrB,MAAMC,IAAI,GAAG,MAAMJ,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;IACnD,OAAO,IAAII,WAAW,CAAC,CAAC,CAACH,MAAM,CAACE,IAAI,CAAC;EACvC;EAEA,IAAIJ,MAAM,CAACM,eAAe,EAAE;IAG1B,MAAMC,OAAO,GAAGC,oBAAoB,CAACT,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IAG3D,MAAMQ,MAAqB,GAAG,EAAE;IAChC,WAAW,MAAMC,KAAK,IAAIH,OAAO,EAAE;MACjCE,MAAM,CAACE,IAAI,CAACD,KAAK,CAAC;IACpB;IACA,OAAOb,uBAAuB,CAAC,GAAGY,MAAM,CAAC;EAC3C;EAEA,MAAM,IAAIG,KAAK,CAAC,8BAA8B,CAAC;AACjD;AAEA,OAAO,eAAeC,iBAAiBA,CACrCd,IAAW,EACXC,MAAe,EACfC,OAAoC,EACnB;EACjB,IAAID,MAAM,CAACI,IAAI,IAAIJ,MAAM,CAACG,UAAU,EAAE;IACpC,OAAO,MAAMH,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;EAC/C;EAEA,IAAID,MAAM,CAACI,IAAI,EAAE;IACf,MAAMU,WAAW,GAAG,MAAMhB,WAAW,CAACC,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IAC5D,OAAO,IAAIc,WAAW,CAAC,CAAC,CAACC,MAAM,CAACF,WAAW,CAAC;EAC9C;EACA,MAAM,IAAIF,KAAK,CAAE,UAASZ,MAAM,CAACiB,IAAK,gCAA+B,CAAC;AACxE;AAEA,OAAO,SAAST,oBAAoBA,CAClCT,IAAW,EACXC,MAAe,EACfC,OAAoC,EACR;EAC5B,IAAID,MAAM,CAACM,eAAe,EAAE;IAC1B,MAAMY,YAAY,GAAGC,WAAW,CAACpB,IAAI,CAAC;IAEtC,OAAOC,MAAM,CAACM,eAAe,CAACY,YAAY,EAAEjB,OAAO,CAAC;EACtD;EAEA,MAAM,IAAIW,KAAK,CAAC,yCAAyC,CAAC;AAC5D;AAEA,SAASO,WAAWA,CAACpB,IAAS,EAA0C;EACtE,MAAMmB,YAAY,GAAG,CAAC;IAAC,GAAGnB,IAAI;IAAEqB,KAAK,EAAE,CAAC;IAAEC,GAAG,EAAEtB,IAAI,CAACuB;EAAM,CAAC,CAAC;EAC5D,OAAOJ,YAAY;AACrB"}

package/dist/lib/api/encode.d.ts

@@ -1,33 +1,33 @@
-import { Writer, WriterOptions } from '@loaders.gl/loader-utils';
+import { WriterOptions, WriterWithEncoder } from '@loaders.gl/loader-utils';
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
  */
-export declare function encode(data: unknown, writer: Writer, options?: WriterOptions): Promise<ArrayBuffer>;
+export declare function encode(data: unknown, writer: WriterWithEncoder, options?: WriterOptions): Promise<ArrayBuffer>;
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
  */
-export declare function encodeSync(data: unknown, writer: Writer, options?: WriterOptions): ArrayBuffer;
+export declare function encodeSync(data: unknown, writer: WriterWithEncoder, options?: WriterOptions): ArrayBuffer;
 /**
  * Encode loaded data to text using the specified Writer
  * @note This is a convenience function not intended for production use on large input data.
  * It is not optimized for performance. Data maybe converted from text to binary and back.
  * @throws if the writer does not generate text output
  */
-export declare function encodeText(data: unknown, writer: Writer, options?: WriterOptions): Promise<string>;
+export declare function encodeText(data: unknown, writer: WriterWithEncoder, options?: WriterOptions): Promise<string>;
 /**
  * Encode loaded data to text using the specified Writer
  * @note This is a convenience function not intended for production use on large input data.
  * It is not optimized for performance. Data maybe converted from text to binary and back.
  * @throws if the writer does not generate text output
  */
-export declare function encodeTextSync(data: unknown, writer: Writer, options?: WriterOptions): string;
+export declare function encodeTextSync(data: unknown, writer: WriterWithEncoder, options?: WriterOptions): string;
 /**
  * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
  */
-export declare function encodeInBatches(data: unknown, writer: Writer, options?: WriterOptions): AsyncIterable<ArrayBuffer>;
+export declare function encodeInBatches(data: unknown, writer: WriterWithEncoder, options?: WriterOptions): AsyncIterable<ArrayBuffer>;
 /**
  * Encode data stored in a file (on disk) to another file.
  * @note Node.js only. This function enables using command-line converters as "writers".
  */
-export declare function encodeURLtoURL(inputUrl: string, outputUrl: string, writer: Writer, options?: WriterOptions): Promise<string>;
+export declare function encodeURLtoURL(inputUrl: string, outputUrl: string, writer: WriterWithEncoder, options?: WriterOptions): Promise<string>;
 //# sourceMappingURL=encode.d.ts.map

package/dist/lib/api/encode.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"encode.d.ts","sourceRoot":"","sources":["../../../src/lib/api/encode.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,MAAM,EAAE,aAAa,EAAsB,MAAM,0BAA0B,CAAC;AAOpF;;GAEG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,WAAW,CAAC,CAuDtB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,WAAW,CAK9F;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAW7F;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,WAAW,CAAC,CAQ5B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,CAQjB"}
+{"version":3,"file":"encode.d.ts","sourceRoot":"","sources":["../../../src/lib/api/encode.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,aAAa,EAAE,iBAAiB,EAAsB,MAAM,0BAA0B,CAAC;AAO/F;;GAEG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,WAAW,CAAC,CAuDtB;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,WAAW,CAKb;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM,CAWR;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,OAAO,EACb,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,WAAW,CAAC,CAQ5B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,MAAM,CAAC,CAQjB"}

package/dist/lib/api/encode.js.map

@@ -1 +1 @@
-{"version":3,"file":"encode.js","names":["canEncodeWithWorker","concatenateArrayBuffers","resolvePath","NodeFile","processOnWorker","isBrowser","fetchFile","getLoaderOptions","encode","data","writer","options","globalOptions","encodeSync","encodeText","TextEncoder","encodeInBatches","batches","chunks","batch","push","encodeURLtoURL","tmpInputFilename","getTemporaryFilename","file","write","tmpOutputFilename","outputFilename","response","arrayBuffer","Error","text","TextDecoder","decode","name","encodeTextSync","dataIterator","getIterator","inputUrl","outputUrl","start","end","length","filename"],"sources":["../../../src/lib/api/encode.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils';\nimport {concatenateArrayBuffers, resolvePath, NodeFile} from '@loaders.gl/loader-utils';\nimport {processOnWorker} from '@loaders.gl/worker-utils';\nimport {isBrowser} from '@loaders.gl/loader-utils';\nimport {fetchFile} from '../fetch/fetch-file';\nimport {getLoaderOptions} from './loader-options';\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport async function encode(\n  data: unknown,\n  writer: Writer,\n  options?: WriterOptions\n): Promise<ArrayBuffer> {\n  const globalOptions = getLoaderOptions() as WriterOptions;\n  // const globalOptions: WriterOptions = {}; // getWriterOptions();\n  options = {...globalOptions, ...options};\n  if (canEncodeWithWorker(writer, options)) {\n    return await processOnWorker(writer, data, options);\n  }\n\n  // TODO Merge default writer options with options argument like it is done in load module.\n  if (writer.encode) {\n    return await writer.encode(data, options);\n  }\n\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n\n  if (writer.encodeText) {\n    return new TextEncoder().encode(await writer.encodeText(data, options));\n  }\n\n  if (writer.encodeInBatches) {\n    // Create an iterator representing the data\n    // TODO - Assumes this is a table\n    const batches = encodeInBatches(data, writer, options);\n\n    // Concatenate the output\n    const chunks: unknown[] = [];\n    for await (const batch of batches) {\n      chunks.push(batch);\n    }\n    // @ts-ignore\n    return concatenateArrayBuffers(...chunks);\n  }\n\n  if (!isBrowser && writer.encodeURLtoURL) {\n    // TODO - how to generate filenames with correct extensions?\n    const tmpInputFilename = getTemporaryFilename('input');\n    const file = new NodeFile(tmpInputFilename, 'w');\n    await file.write(data as ArrayBuffer);\n\n    const tmpOutputFilename = getTemporaryFilename('output');\n\n    const outputFilename = await encodeURLtoURL(\n      tmpInputFilename,\n      tmpOutputFilename,\n      writer,\n      options\n    );\n\n    const response = await fetchFile(outputFilename);\n    return response.arrayBuffer();\n  }\n\n  throw new Error('Writer could not encode data');\n}\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport function encodeSync(data: unknown, writer: Writer, options?: WriterOptions): ArrayBuffer {\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n  throw new Error('Writer could not synchronously encode data');\n}\n\n/**\n * Encode loaded data to text using the specified Writer\n * @note This is a convenience function not intended for production use on large input data.\n * It is not optimized for performance. Data maybe converted from text to binary and back.\n * @throws if the writer does not generate text output\n */\nexport async function encodeText(\n  data: unknown,\n  writer: Writer,\n  options?: WriterOptions\n): Promise<string> {\n  if (writer.text && writer.encodeText) {\n    return await writer.encodeText(data, options);\n  }\n\n  if (writer.text) {\n    const arrayBuffer = await encode(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n\n  throw new Error(`Writer ${writer.name} could not encode data as text`);\n}\n\n/**\n * Encode loaded data to text using the specified Writer\n * @note This is a convenience function not intended for production use on large input data.\n * It is not optimized for performance. Data maybe converted from text to binary and back.\n * @throws if the writer does not generate text output\n */\nexport function encodeTextSync(data: unknown, writer: Writer, options?: WriterOptions): string {\n  if (writer.text && writer.encodeTextSync) {\n    return writer.encodeTextSync(data, options);\n  }\n\n  if (writer.text && writer.encodeSync) {\n    const arrayBuffer = encodeSync(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n\n  throw new Error(`Writer ${writer.name} could not encode data as text`);\n}\n\n/**\n * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.\n */\nexport function encodeInBatches(\n  data: unknown,\n  writer: Writer,\n  options?: WriterOptions\n): AsyncIterable<ArrayBuffer> {\n  if (writer.encodeInBatches) {\n    const dataIterator = getIterator(data);\n    // @ts-expect-error\n    return writer.encodeInBatches(dataIterator, options);\n  }\n  // TODO -fall back to atomic encode?\n  throw new Error('Writer could not encode data in batches');\n}\n\n/**\n * Encode data stored in a file (on disk) to another file.\n * @note Node.js only. This function enables using command-line converters as \"writers\".\n */\nexport async function encodeURLtoURL(\n  inputUrl: string,\n  outputUrl: string,\n  writer: Writer,\n  options?: WriterOptions\n): Promise<string> {\n  inputUrl = resolvePath(inputUrl);\n  outputUrl = resolvePath(outputUrl);\n  if (isBrowser || !writer.encodeURLtoURL) {\n    throw new Error();\n  }\n  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);\n  return outputFilename;\n}\n\n/**\n * @todo TODO - this is an unacceptable hack!!!\n */\nfunction getIterator(data: any): Iterable<{table: any; start: number; end: number}> {\n  const dataIterator = [{...data, start: 0, end: data.length}];\n  return dataIterator;\n}\n\n/**\n * @todo Move to utils\n */\nfunction getTemporaryFilename(filename: string): string {\n  return `/tmp/${filename}`;\n}\n"],"mappings":"AAGA,SAA+BA,mBAAmB,QAAO,0BAA0B;AACnF,SAAQC,uBAAuB,EAAEC,WAAW,EAAEC,QAAQ,QAAO,0BAA0B;AACvF,SAAQC,eAAe,QAAO,0BAA0B;AACxD,SAAQC,SAAS,QAAO,0BAA0B;AAAC,SAC3CC,SAAS;AAAA,SACTC,gBAAgB;AAKxB,OAAO,eAAeC,MAAMA,CAC1BC,IAAa,EACbC,MAAc,EACdC,OAAuB,EACD;EACtB,MAAMC,aAAa,GAAGL,gBAAgB,CAAC,CAAkB;EAEzDI,OAAO,GAAG;IAAC,GAAGC,aAAa;IAAE,GAAGD;EAAO,CAAC;EACxC,IAAIX,mBAAmB,CAACU,MAAM,EAAEC,OAAO,CAAC,EAAE;IACxC,OAAO,MAAMP,eAAe,CAACM,MAAM,EAAED,IAAI,EAAEE,OAAO,CAAC;EACrD;EAGA,IAAID,MAAM,CAACF,MAAM,EAAE;IACjB,OAAO,MAAME,MAAM,CAACF,MAAM,CAACC,IAAI,EAAEE,OAAO,CAAC;EAC3C;EAEA,IAAID,MAAM,CAACG,UAAU,EAAE;IACrB,OAAOH,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;EACzC;EAEA,IAAID,MAAM,CAACI,UAAU,EAAE;IACrB,OAAO,IAAIC,WAAW,CAAC,CAAC,CAACP,MAAM,CAAC,MAAME,MAAM,CAACI,UAAU,CAACL,IAAI,EAAEE,OAAO,CAAC,CAAC;EACzE;EAEA,IAAID,MAAM,CAACM,eAAe,EAAE;IAG1B,MAAMC,OAAO,GAAGD,eAAe,CAACP,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IAGtD,MAAMO,MAAiB,GAAG,EAAE;IAC5B,WAAW,MAAMC,KAAK,IAAIF,OAAO,EAAE;MACjCC,MAAM,CAACE,IAAI,CAACD,KAAK,CAAC;IACpB;IAEA,OAAOlB,uBAAuB,CAAC,GAAGiB,MAAM,CAAC;EAC3C;EAEA,IAAI,CAACb,SAAS,IAAIK,MAAM,CAACW,cAAc,EAAE;IAEvC,MAAMC,gBAAgB,GAAGC,oBAAoB,CAAC,OAAO,CAAC;IACtD,MAAMC,IAAI,GAAG,IAAIrB,QAAQ,CAACmB,gBAAgB,EAAE,GAAG,CAAC;IAChD,MAAME,IAAI,CAACC,KAAK,CAAChB,IAAmB,CAAC;IAErC,MAAMiB,iBAAiB,GAAGH,oBAAoB,CAAC,QAAQ,CAAC;IAExD,MAAMI,cAAc,GAAG,MAAMN,cAAc,CACzCC,gBAAgB,EAChBI,iBAAiB,EACjBhB,MAAM,EACNC,OACF,CAAC;IAED,MAAMiB,QAAQ,GAAG,MAAMtB,SAAS,CAACqB,cAAc,CAAC;IAChD,OAAOC,QAAQ,CAACC,WAAW,CAAC,CAAC;EAC/B;EAEA,MAAM,IAAIC,KAAK,CAAC,8BAA8B,CAAC;AACjD;AAKA,OAAO,SAASjB,UAAUA,CAACJ,IAAa,EAAEC,MAAc,EAAEC,OAAuB,EAAe;EAC9F,IAAID,MAAM,CAACG,UAAU,EAAE;IACrB,OAAOH,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;EACzC;EACA,MAAM,IAAImB,KAAK,CAAC,4CAA4C,CAAC;AAC/D;AAQA,OAAO,eAAehB,UAAUA,CAC9BL,IAAa,EACbC,MAAc,EACdC,OAAuB,EACN;EACjB,IAAID,MAAM,CAACqB,IAAI,IAAIrB,MAAM,CAACI,UAAU,EAAE;IACpC,OAAO,MAAMJ,MAAM,CAACI,UAAU,CAACL,IAAI,EAAEE,OAAO,CAAC;EAC/C;EAEA,IAAID,MAAM,CAACqB,IAAI,EAAE;IACf,MAAMF,WAAW,GAAG,MAAMrB,MAAM,CAACC,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IACvD,OAAO,IAAIqB,WAAW,CAAC,CAAC,CAACC,MAAM,CAACJ,WAAW,CAAC;EAC9C;EAEA,MAAM,IAAIC,KAAK,CAAE,UAASpB,MAAM,CAACwB,IAAK,gCAA+B,CAAC;AACxE;AAQA,OAAO,SAASC,cAAcA,CAAC1B,IAAa,EAAEC,MAAc,EAAEC,OAAuB,EAAU;EAC7F,IAAID,MAAM,CAACqB,IAAI,IAAIrB,MAAM,CAACyB,cAAc,EAAE;IACxC,OAAOzB,MAAM,CAACyB,cAAc,CAAC1B,IAAI,EAAEE,OAAO,CAAC;EAC7C;EAEA,IAAID,MAAM,CAACqB,IAAI,IAAIrB,MAAM,CAACG,UAAU,EAAE;IACpC,MAAMgB,WAAW,GAAGhB,UAAU,CAACJ,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IACrD,OAAO,IAAIqB,WAAW,CAAC,CAAC,CAACC,MAAM,CAACJ,WAAW,CAAC;EAC9C;EAEA,MAAM,IAAIC,KAAK,CAAE,UAASpB,MAAM,CAACwB,IAAK,gCAA+B,CAAC;AACxE;AAKA,OAAO,SAASlB,eAAeA,CAC7BP,IAAa,EACbC,MAAc,EACdC,OAAuB,EACK;EAC5B,IAAID,MAAM,CAACM,eAAe,EAAE;IAC1B,MAAMoB,YAAY,GAAGC,WAAW,CAAC5B,IAAI,CAAC;IAEtC,OAAOC,MAAM,CAACM,eAAe,CAACoB,YAAY,EAAEzB,OAAO,CAAC;EACtD;EAEA,MAAM,IAAImB,KAAK,CAAC,yCAAyC,CAAC;AAC5D;AAMA,OAAO,eAAeT,cAAcA,CAClCiB,QAAgB,EAChBC,SAAiB,EACjB7B,MAAc,EACdC,OAAuB,EACN;EACjB2B,QAAQ,GAAGpC,WAAW,CAACoC,QAAQ,CAAC;EAChCC,SAAS,GAAGrC,WAAW,CAACqC,SAAS,CAAC;EAClC,IAAIlC,SAAS,IAAI,CAACK,MAAM,CAACW,cAAc,EAAE;IACvC,MAAM,IAAIS,KAAK,CAAC,CAAC;EACnB;EACA,MAAMH,cAAc,GAAG,MAAMjB,MAAM,CAACW,cAAc,CAACiB,QAAQ,EAAEC,SAAS,EAAE5B,OAAO,CAAC;EAChF,OAAOgB,cAAc;AACvB;AAKA,SAASU,WAAWA,CAAC5B,IAAS,EAAsD;EAClF,MAAM2B,YAAY,GAAG,CAAC;IAAC,GAAG3B,IAAI;IAAE+B,KAAK,EAAE,CAAC;IAAEC,GAAG,EAAEhC,IAAI,CAACiC;EAAM,CAAC,CAAC;EAC5D,OAAON,YAAY;AACrB;AAKA,SAASb,oBAAoBA,CAACoB,QAAgB,EAAU;EACtD,OAAQ,QAAOA,QAAS,EAAC;AAC3B"}
+{"version":3,"file":"encode.js","names":["canEncodeWithWorker","concatenateArrayBuffers","resolvePath","NodeFile","processOnWorker","isBrowser","fetchFile","getLoaderOptions","encode","data","writer","options","globalOptions","encodeSync","encodeText","TextEncoder","encodeInBatches","batches","chunks","batch","push","encodeURLtoURL","tmpInputFilename","getTemporaryFilename","file","write","tmpOutputFilename","outputFilename","response","arrayBuffer","Error","text","TextDecoder","decode","name","encodeTextSync","dataIterator","getIterator","inputUrl","outputUrl","start","end","length","filename"],"sources":["../../../src/lib/api/encode.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport {WriterOptions, WriterWithEncoder, canEncodeWithWorker} from '@loaders.gl/loader-utils';\nimport {concatenateArrayBuffers, resolvePath, NodeFile} from '@loaders.gl/loader-utils';\nimport {processOnWorker} from '@loaders.gl/worker-utils';\nimport {isBrowser} from '@loaders.gl/loader-utils';\nimport {fetchFile} from '../fetch/fetch-file';\nimport {getLoaderOptions} from './loader-options';\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport async function encode(\n  data: unknown,\n  writer: WriterWithEncoder,\n  options?: WriterOptions\n): Promise<ArrayBuffer> {\n  const globalOptions = getLoaderOptions() as WriterOptions;\n  // const globalOptions: WriterOptions = {}; // getWriterOptions();\n  options = {...globalOptions, ...options};\n  if (canEncodeWithWorker(writer, options)) {\n    return await processOnWorker(writer, data, options);\n  }\n\n  // TODO Merge default writer options with options argument like it is done in load module.\n  if (writer.encode) {\n    return await writer.encode(data, options);\n  }\n\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n\n  if (writer.encodeText) {\n    return new TextEncoder().encode(await writer.encodeText(data, options));\n  }\n\n  if (writer.encodeInBatches) {\n    // Create an iterator representing the data\n    // TODO - Assumes this is a table\n    const batches = encodeInBatches(data, writer, options);\n\n    // Concatenate the output\n    const chunks: unknown[] = [];\n    for await (const batch of batches) {\n      chunks.push(batch);\n    }\n    // @ts-ignore\n    return concatenateArrayBuffers(...chunks);\n  }\n\n  if (!isBrowser && writer.encodeURLtoURL) {\n    // TODO - how to generate filenames with correct extensions?\n    const tmpInputFilename = getTemporaryFilename('input');\n    const file = new NodeFile(tmpInputFilename, 'w');\n    await file.write(data as ArrayBuffer);\n\n    const tmpOutputFilename = getTemporaryFilename('output');\n\n    const outputFilename = await encodeURLtoURL(\n      tmpInputFilename,\n      tmpOutputFilename,\n      writer,\n      options\n    );\n\n    const response = await fetchFile(outputFilename);\n    return response.arrayBuffer();\n  }\n\n  throw new Error('Writer could not encode data');\n}\n\n/**\n * Encode loaded data into a binary ArrayBuffer using the specified Writer.\n */\nexport function encodeSync(\n  data: unknown,\n  writer: WriterWithEncoder,\n  options?: WriterOptions\n): ArrayBuffer {\n  if (writer.encodeSync) {\n    return writer.encodeSync(data, options);\n  }\n  throw new Error('Writer could not synchronously encode data');\n}\n\n/**\n * Encode loaded data to text using the specified Writer\n * @note This is a convenience function not intended for production use on large input data.\n * It is not optimized for performance. Data maybe converted from text to binary and back.\n * @throws if the writer does not generate text output\n */\nexport async function encodeText(\n  data: unknown,\n  writer: WriterWithEncoder,\n  options?: WriterOptions\n): Promise<string> {\n  if (writer.text && writer.encodeText) {\n    return await writer.encodeText(data, options);\n  }\n\n  if (writer.text) {\n    const arrayBuffer = await encode(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n\n  throw new Error(`Writer ${writer.name} could not encode data as text`);\n}\n\n/**\n * Encode loaded data to text using the specified Writer\n * @note This is a convenience function not intended for production use on large input data.\n * It is not optimized for performance. Data maybe converted from text to binary and back.\n * @throws if the writer does not generate text output\n */\nexport function encodeTextSync(\n  data: unknown,\n  writer: WriterWithEncoder,\n  options?: WriterOptions\n): string {\n  if (writer.text && writer.encodeTextSync) {\n    return writer.encodeTextSync(data, options);\n  }\n\n  if (writer.text && writer.encodeSync) {\n    const arrayBuffer = encodeSync(data, writer, options);\n    return new TextDecoder().decode(arrayBuffer);\n  }\n\n  throw new Error(`Writer ${writer.name} could not encode data as text`);\n}\n\n/**\n * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.\n */\nexport function encodeInBatches(\n  data: unknown,\n  writer: WriterWithEncoder,\n  options?: WriterOptions\n): AsyncIterable<ArrayBuffer> {\n  if (writer.encodeInBatches) {\n    const dataIterator = getIterator(data);\n    // @ts-expect-error\n    return writer.encodeInBatches(dataIterator, options);\n  }\n  // TODO -fall back to atomic encode?\n  throw new Error('Writer could not encode data in batches');\n}\n\n/**\n * Encode data stored in a file (on disk) to another file.\n * @note Node.js only. This function enables using command-line converters as \"writers\".\n */\nexport async function encodeURLtoURL(\n  inputUrl: string,\n  outputUrl: string,\n  writer: WriterWithEncoder,\n  options?: WriterOptions\n): Promise<string> {\n  inputUrl = resolvePath(inputUrl);\n  outputUrl = resolvePath(outputUrl);\n  if (isBrowser || !writer.encodeURLtoURL) {\n    throw new Error();\n  }\n  const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);\n  return outputFilename;\n}\n\n/**\n * @todo TODO - this is an unacceptable hack!!!\n */\nfunction getIterator(data: any): Iterable<{table: any; start: number; end: number}> {\n  const dataIterator = [{...data, start: 0, end: data.length}];\n  return dataIterator;\n}\n\n/**\n * @todo Move to utils\n */\nfunction getTemporaryFilename(filename: string): string {\n  return `/tmp/${filename}`;\n}\n"],"mappings":"AAGA,SAA0CA,mBAAmB,QAAO,0BAA0B;AAC9F,SAAQC,uBAAuB,EAAEC,WAAW,EAAEC,QAAQ,QAAO,0BAA0B;AACvF,SAAQC,eAAe,QAAO,0BAA0B;AACxD,SAAQC,SAAS,QAAO,0BAA0B;AAAC,SAC3CC,SAAS;AAAA,SACTC,gBAAgB;AAKxB,OAAO,eAAeC,MAAMA,CAC1BC,IAAa,EACbC,MAAyB,EACzBC,OAAuB,EACD;EACtB,MAAMC,aAAa,GAAGL,gBAAgB,CAAC,CAAkB;EAEzDI,OAAO,GAAG;IAAC,GAAGC,aAAa;IAAE,GAAGD;EAAO,CAAC;EACxC,IAAIX,mBAAmB,CAACU,MAAM,EAAEC,OAAO,CAAC,EAAE;IACxC,OAAO,MAAMP,eAAe,CAACM,MAAM,EAAED,IAAI,EAAEE,OAAO,CAAC;EACrD;EAGA,IAAID,MAAM,CAACF,MAAM,EAAE;IACjB,OAAO,MAAME,MAAM,CAACF,MAAM,CAACC,IAAI,EAAEE,OAAO,CAAC;EAC3C;EAEA,IAAID,MAAM,CAACG,UAAU,EAAE;IACrB,OAAOH,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;EACzC;EAEA,IAAID,MAAM,CAACI,UAAU,EAAE;IACrB,OAAO,IAAIC,WAAW,CAAC,CAAC,CAACP,MAAM,CAAC,MAAME,MAAM,CAACI,UAAU,CAACL,IAAI,EAAEE,OAAO,CAAC,CAAC;EACzE;EAEA,IAAID,MAAM,CAACM,eAAe,EAAE;IAG1B,MAAMC,OAAO,GAAGD,eAAe,CAACP,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IAGtD,MAAMO,MAAiB,GAAG,EAAE;IAC5B,WAAW,MAAMC,KAAK,IAAIF,OAAO,EAAE;MACjCC,MAAM,CAACE,IAAI,CAACD,KAAK,CAAC;IACpB;IAEA,OAAOlB,uBAAuB,CAAC,GAAGiB,MAAM,CAAC;EAC3C;EAEA,IAAI,CAACb,SAAS,IAAIK,MAAM,CAACW,cAAc,EAAE;IAEvC,MAAMC,gBAAgB,GAAGC,oBAAoB,CAAC,OAAO,CAAC;IACtD,MAAMC,IAAI,GAAG,IAAIrB,QAAQ,CAACmB,gBAAgB,EAAE,GAAG,CAAC;IAChD,MAAME,IAAI,CAACC,KAAK,CAAChB,IAAmB,CAAC;IAErC,MAAMiB,iBAAiB,GAAGH,oBAAoB,CAAC,QAAQ,CAAC;IAExD,MAAMI,cAAc,GAAG,MAAMN,cAAc,CACzCC,gBAAgB,EAChBI,iBAAiB,EACjBhB,MAAM,EACNC,OACF,CAAC;IAED,MAAMiB,QAAQ,GAAG,MAAMtB,SAAS,CAACqB,cAAc,CAAC;IAChD,OAAOC,QAAQ,CAACC,WAAW,CAAC,CAAC;EAC/B;EAEA,MAAM,IAAIC,KAAK,CAAC,8BAA8B,CAAC;AACjD;AAKA,OAAO,SAASjB,UAAUA,CACxBJ,IAAa,EACbC,MAAyB,EACzBC,OAAuB,EACV;EACb,IAAID,MAAM,CAACG,UAAU,EAAE;IACrB,OAAOH,MAAM,CAACG,UAAU,CAACJ,IAAI,EAAEE,OAAO,CAAC;EACzC;EACA,MAAM,IAAImB,KAAK,CAAC,4CAA4C,CAAC;AAC/D;AAQA,OAAO,eAAehB,UAAUA,CAC9BL,IAAa,EACbC,MAAyB,EACzBC,OAAuB,EACN;EACjB,IAAID,MAAM,CAACqB,IAAI,IAAIrB,MAAM,CAACI,UAAU,EAAE;IACpC,OAAO,MAAMJ,MAAM,CAACI,UAAU,CAACL,IAAI,EAAEE,OAAO,CAAC;EAC/C;EAEA,IAAID,MAAM,CAACqB,IAAI,EAAE;IACf,MAAMF,WAAW,GAAG,MAAMrB,MAAM,CAACC,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IACvD,OAAO,IAAIqB,WAAW,CAAC,CAAC,CAACC,MAAM,CAACJ,WAAW,CAAC;EAC9C;EAEA,MAAM,IAAIC,KAAK,CAAE,UAASpB,MAAM,CAACwB,IAAK,gCAA+B,CAAC;AACxE;AAQA,OAAO,SAASC,cAAcA,CAC5B1B,IAAa,EACbC,MAAyB,EACzBC,OAAuB,EACf;EACR,IAAID,MAAM,CAACqB,IAAI,IAAIrB,MAAM,CAACyB,cAAc,EAAE;IACxC,OAAOzB,MAAM,CAACyB,cAAc,CAAC1B,IAAI,EAAEE,OAAO,CAAC;EAC7C;EAEA,IAAID,MAAM,CAACqB,IAAI,IAAIrB,MAAM,CAACG,UAAU,EAAE;IACpC,MAAMgB,WAAW,GAAGhB,UAAU,CAACJ,IAAI,EAAEC,MAAM,EAAEC,OAAO,CAAC;IACrD,OAAO,IAAIqB,WAAW,CAAC,CAAC,CAACC,MAAM,CAACJ,WAAW,CAAC;EAC9C;EAEA,MAAM,IAAIC,KAAK,CAAE,UAASpB,MAAM,CAACwB,IAAK,gCAA+B,CAAC;AACxE;AAKA,OAAO,SAASlB,eAAeA,CAC7BP,IAAa,EACbC,MAAyB,EACzBC,OAAuB,EACK;EAC5B,IAAID,MAAM,CAACM,eAAe,EAAE;IAC1B,MAAMoB,YAAY,GAAGC,WAAW,CAAC5B,IAAI,CAAC;IAEtC,OAAOC,MAAM,CAACM,eAAe,CAACoB,YAAY,EAAEzB,OAAO,CAAC;EACtD;EAEA,MAAM,IAAImB,KAAK,CAAC,yCAAyC,CAAC;AAC5D;AAMA,OAAO,eAAeT,cAAcA,CAClCiB,QAAgB,EAChBC,SAAiB,EACjB7B,MAAyB,EACzBC,OAAuB,EACN;EACjB2B,QAAQ,GAAGpC,WAAW,CAACoC,QAAQ,CAAC;EAChCC,SAAS,GAAGrC,WAAW,CAACqC,SAAS,CAAC;EAClC,IAAIlC,SAAS,IAAI,CAACK,MAAM,CAACW,cAAc,EAAE;IACvC,MAAM,IAAIS,KAAK,CAAC,CAAC;EACnB;EACA,MAAMH,cAAc,GAAG,MAAMjB,MAAM,CAACW,cAAc,CAACiB,QAAQ,EAAEC,SAAS,EAAE5B,OAAO,CAAC;EAChF,OAAOgB,cAAc;AACvB;AAKA,SAASU,WAAWA,CAAC5B,IAAS,EAAsD;EAClF,MAAM2B,YAAY,GAAG,CAAC;IAAC,GAAG3B,IAAI;IAAE+B,KAAK,EAAE,CAAC;IAAEC,GAAG,EAAEhC,IAAI,CAACiC;EAAM,CAAC,CAAC;EAC5D,OAAON,YAAY;AACrB;AAKA,SAASb,oBAAoBA,CAACoB,QAAgB,EAAU;EACtD,OAAQ,QAAOA,QAAS,EAAC;AAC3B"}

package/dist/lib/api/parse-in-batches.js

@@ -1,3 +1,4 @@
+import { isTable, makeBatchFromTable } from '@loaders.gl/schema';
 import { concatenateArrayBuffersAsync } from '@loaders.gl/loader-utils';
 import { isLoaderObject } from "../loader-utils/normalize-loader.js";
 import { normalizeOptions } from "../loader-utils/option-utils.js";
@@ -55,22 +56,26 @@ async function parseToOutputIterator(loader, data, options, context) {
   if (loader.parseInBatches) {
     return loader.parseInBatches(transformedIterator, options, context);
   }
-  async function* parseChunkInBatches() {
-    const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
-    const parsedData = await parse(arrayBuffer, loader, {
-      ...options,
-      mimeType: loader.mimeTypes[0]
-    }, context);
-    const batch = {
-      mimeType: loader.mimeTypes[0],
-      shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
-      batchType: 'data',
-      data: parsedData,
-      length: Array.isArray(parsedData) ? parsedData.length : 1
-    };
-    yield batch;
-  }
-  return parseChunkInBatches();
+  return parseChunkInBatches(transformedIterator, loader, options, context);
+}
+async function* parseChunkInBatches(transformedIterator, loader, options, context) {
+  const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
+  const parsedData = await parse(arrayBuffer, loader, {
+    ...options,
+    mimeType: loader.mimeTypes[0]
+  }, context);
+  const batch = convertDataToBatch(parsedData, loader);
+  yield batch;
+}
+function convertDataToBatch(parsedData, loader) {
+  const batch = isTable(parsedData) ? makeBatchFromTable(parsedData) : {
+    shape: 'unknown',
+    batchType: 'data',
+    data: parsedData,
+    length: Array.isArray(parsedData) ? parsedData.length : 1
+  };
+  batch.mimeType = loader.mimeTypes[0];
+  return batch;
 }
 async function applyInputTransforms(inputIterator) {
   let transforms = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : [];

package/dist/lib/api/parse-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-in-batches.js","names":["concatenateArrayBuffersAsync","isLoaderObject","normalizeOptions","getLoaderContext","getAsyncIterableFromData","getResourceUrl","selectLoader","parse","parseInBatches","data","loaders","options","context","loaderArray","Array","isArray","undefined","url","loader","_parseInBatches","_parse","parseWithLoaderInBatches","outputIterator","parseToOutputIterator","metadata","metadataBatch","batchType","_loader","_context","bytesUsed","makeMetadataBatchIterator","iterator","inputIterator","transformedIterator","applyInputTransforms","transforms","parseChunkInBatches","arrayBuffer","parsedData","mimeType","mimeTypes","batch","shape","length","arguments","iteratorChain","transformBatches"],"sources":["../../../src/lib/api/parse-in-batches.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {Batch} from '@loaders.gl/schema';\nimport type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {LoaderContext, BatchableDataType} from '@loaders.gl/loader-utils';\nimport type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';\nimport {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getLoaderContext} from '../loader-utils/loader-context';\nimport {getAsyncIterableFromData} from '../loader-utils/get-data';\nimport {getResourceUrl} from '../utils/resource-utils';\nimport {selectLoader} from './select-loader';\n\n// Ensure `parse` is available in context if loader falls back to `parse`\nimport {parse} from './parse';\n\n/**\n * Parses `data` synchronously using a specified loader\n */\nexport async function parseInBatches<\n  LoaderT extends Loader,\n  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>\n>(\n  data: BatchableDataType,\n  loader: LoaderT,\n  options?: OptionsT,\n  context?: LoaderContext\n): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;\n\n/**\n * Parses `data` using one of the supplied loaders\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  loaders: Loader[],\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<unknown>>;\n\n/**\n * Parses `data` in batches by selecting a pre-registered loader\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  options?: LoaderOptions\n): Promise<AsyncIterable<unknown>>;\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  loaders?: Loader | Loader[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<unknown> | Iterable<unknown>> {\n  const loaderArray = Array.isArray(loaders) ? loaders : undefined;\n\n  // Signature: parseInBatches(data, options, url) - Uses registered loaders\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders as LoaderOptions;\n    loaders = undefined;\n  }\n\n  data = await data; // Resolve any promise\n  options = options || {};\n\n  // Extract a url for auto detection\n  const url = getResourceUrl(data);\n\n  // Chooses a loader and normalizes it\n  // Note - only uses URL and contentType for streams and iterator inputs\n  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader | Loader[], options);\n  // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null\n  if (!loader) {\n    return [];\n  }\n\n  // Normalize options\n  options = normalizeOptions(options, loader, loaderArray, url);\n  context = getLoaderContext(\n    {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},\n    options,\n    context || null\n  );\n\n  return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);\n}\n\n/**\n * Loader has been selected and context has been prepared, see if we need to emit a metadata batch\n */\nasync function parseWithLoaderInBatches(\n  loader: LoaderWithParser,\n  data: BatchableDataType,\n  options: LoaderOptions,\n  context: LoaderContext\n): Promise<AsyncIterable<unknown>> {\n  const outputIterator = await parseToOutputIterator(loader, data, options, context);\n\n  // Generate metadata batch if requested\n  if (!options.metadata) {\n    return outputIterator;\n  }\n\n  const metadataBatch = {\n    batchType: 'metadata',\n    metadata: {\n      _loader: loader,\n      _context: context\n    },\n    // Populate with some default fields to avoid crashing\n    data: [],\n    bytesUsed: 0\n  };\n\n  async function* makeMetadataBatchIterator(\n    iterator: Iterable<unknown> | AsyncIterable<unknown>\n  ): AsyncIterable<unknown> {\n    yield metadataBatch;\n    yield* iterator;\n  }\n\n  return makeMetadataBatchIterator(outputIterator);\n}\n\n/**\n * Prep work is done, now it is time to start parsing into an output operator\n * The approach depends on which parse function the loader exposes\n * `parseInBatches` (preferred), `parse` (fallback)\n */\nasync function parseToOutputIterator(\n  loader: LoaderWithParser,\n  data: BatchableDataType,\n  options: LoaderOptions,\n  context: LoaderContext\n): Promise<AsyncIterable<unknown>> {\n  // Get an iterator from the input\n  const inputIterator = await getAsyncIterableFromData(data, options);\n\n  // Apply any iterator transforms (options.transforms)\n  const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);\n\n  // If loader supports parseInBatches, we are done\n  if (loader.parseInBatches) {\n    return loader.parseInBatches(transformedIterator, options, context);\n  }\n\n  // Fallback: load atomically using `parse` concatenating input iterator into single chunk\n  async function* parseChunkInBatches() {\n    const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);\n    // Call `parse` instead of `loader.parse` to ensure we can call workers etc.\n    const parsedData = await parse(\n      arrayBuffer,\n      loader,\n      // TODO - Hack: supply loaders MIME type to ensure we match it\n      {...options, mimeType: loader.mimeTypes[0]},\n      context\n    );\n    // yield a single batch, the output from loader.parse()\n    // TODO - run through batch builder to apply options etc...\n    const batch: Batch = {\n      mimeType: loader.mimeTypes[0],\n      shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',\n      batchType: 'data',\n      data: parsedData,\n      length: Array.isArray(parsedData) ? parsedData.length : 1\n    };\n    yield batch;\n  }\n\n  return parseChunkInBatches();\n}\n\ntype TransformBatches = (\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n) => AsyncIterable<ArrayBuffer>;\n\n/**\n * Create an iterator chain with any transform iterators (crypto, decompression)\n * @param inputIterator\n * @param options\n */\nasync function applyInputTransforms(\n  inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  transforms: TransformBatches[] = []\n): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {\n  let iteratorChain = inputIterator;\n  for await (const transformBatches of transforms) {\n    iteratorChain = transformBatches(iteratorChain);\n  }\n  return iteratorChain;\n}\n"],"mappings":"AAOA,SAAQA,4BAA4B,QAAO,0BAA0B;AAAC,SAC9DC,cAAc;AAAA,SACdC,gBAAgB;AAAA,SAChBC,gBAAgB;AAAA,SAChBC,wBAAwB;AAAA,SACxBC,cAAc;AAAA,SACdC,YAAY;AAAA,SAGZC,KAAK;AAwCb,OAAO,eAAeC,cAAcA,CAClCC,IAAuB,EACvBC,OAA2C,EAC3CC,OAAuB,EACvBC,OAAuB,EAC8B;EACrD,MAAMC,WAAW,GAAGC,KAAK,CAACC,OAAO,CAACL,OAAO,CAAC,GAAGA,OAAO,GAAGM,SAAS;EAGhE,IAAI,CAACF,KAAK,CAACC,OAAO,CAACL,OAAO,CAAC,IAAI,CAACT,cAAc,CAACS,OAAO,CAAC,EAAE;IACvDE,OAAO,GAAGI,SAAS;IACnBL,OAAO,GAAGD,OAAwB;IAClCA,OAAO,GAAGM,SAAS;EACrB;EAEAP,IAAI,GAAG,MAAMA,IAAI;EACjBE,OAAO,GAAGA,OAAO,IAAI,CAAC,CAAC;EAGvB,MAAMM,GAAG,GAAGZ,cAAc,CAACI,IAAI,CAAC;EAIhC,MAAMS,MAAM,GAAG,MAAMZ,YAAY,CAACG,IAAI,EAAiBC,OAAO,EAAuBC,OAAO,CAAC;EAE7F,IAAI,CAACO,MAAM,EAAE;IACX,OAAO,EAAE;EACX;EAGAP,OAAO,GAAGT,gBAAgB,CAACS,OAAO,EAAEO,MAAM,EAAEL,WAAW,EAAEI,GAAG,CAAC;EAC7DL,OAAO,GAAGT,gBAAgB,CACxB;IAACc,GAAG;IAAEE,eAAe,EAAEX,cAAc;IAAEY,MAAM,EAAEb,KAAK;IAAEG,OAAO,EAAEG;EAAW,CAAC,EAC3EF,OAAO,EACPC,OAAO,IAAI,IACb,CAAC;EAED,OAAO,MAAMS,wBAAwB,CAACH,MAAM,EAAsBT,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;AAC3F;AAKA,eAAeS,wBAAwBA,CACrCH,MAAwB,EACxBT,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB,EACW;EACjC,MAAMU,cAAc,GAAG,MAAMC,qBAAqB,CAACL,MAAM,EAAET,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;EAGlF,IAAI,CAACD,OAAO,CAACa,QAAQ,EAAE;IACrB,OAAOF,cAAc;EACvB;EAEA,MAAMG,aAAa,GAAG;IACpBC,SAAS,EAAE,UAAU;IACrBF,QAAQ,EAAE;MACRG,OAAO,EAAET,MAAM;MACfU,QAAQ,EAAEhB;IACZ,CAAC;IAEDH,IAAI,EAAE,EAAE;IACRoB,SAAS,EAAE;EACb,CAAC;EAED,gBAAgBC,yBAAyBA,CACvCC,QAAoD,EAC5B;IACxB,MAAMN,aAAa;IACnB,OAAOM,QAAQ;EACjB;EAEA,OAAOD,yBAAyB,CAACR,cAAc,CAAC;AAClD;AAOA,eAAeC,qBAAqBA,CAClCL,MAAwB,EACxBT,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB,EACW;EAEjC,MAAMoB,aAAa,GAAG,MAAM5B,wBAAwB,CAACK,IAAI,EAAEE,OAAO,CAAC;EAGnE,MAAMsB,mBAAmB,GAAG,MAAMC,oBAAoB,CAACF,aAAa,EAAE,CAAArB,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEwB,UAAU,KAAI,EAAE,CAAC;EAGhG,IAAIjB,MAAM,CAACV,cAAc,EAAE;IACzB,OAAOU,MAAM,CAACV,cAAc,CAACyB,mBAAmB,EAAEtB,OAAO,EAAEC,OAAO,CAAC;EACrE;EAGA,gBAAgBwB,mBAAmBA,CAAA,EAAG;IACpC,MAAMC,WAAW,GAAG,MAAMrC,4BAA4B,CAACiC,mBAAmB,CAAC;IAE3E,MAAMK,UAAU,GAAG,MAAM/B,KAAK,CAC5B8B,WAAW,EACXnB,MAAM,EAEN;MAAC,GAAGP,OAAO;MAAE4B,QAAQ,EAAErB,MAAM,CAACsB,SAAS,CAAC,CAAC;IAAC,CAAC,EAC3C5B,OACF,CAAC;IAGD,MAAM6B,KAAY,GAAG;MACnBF,QAAQ,EAAErB,MAAM,CAACsB,SAAS,CAAC,CAAC,CAAC;MAC7BE,KAAK,EAAE5B,KAAK,CAACC,OAAO,CAACuB,UAAU,CAAC,GAAG,WAAW,GAAG,SAAS;MAC1DZ,SAAS,EAAE,MAAM;MACjBjB,IAAI,EAAE6B,UAAU;MAChBK,MAAM,EAAE7B,KAAK,CAACC,OAAO,CAACuB,UAAU,CAAC,GAAGA,UAAU,CAACK,MAAM,GAAG;IAC1D,CAAC;IACD,MAAMF,KAAK;EACb;EAEA,OAAOL,mBAAmB,CAAC,CAAC;AAC9B;AAWA,eAAeF,oBAAoBA,CACjCF,aAAiE,EAEJ;EAAA,IAD7DG,UAA8B,GAAAS,SAAA,CAAAD,MAAA,QAAAC,SAAA,QAAA5B,SAAA,GAAA4B,SAAA,MAAG,EAAE;EAEnC,IAAIC,aAAa,GAAGb,aAAa;EACjC,WAAW,MAAMc,gBAAgB,IAAIX,UAAU,EAAE;IAC/CU,aAAa,GAAGC,gBAAgB,CAACD,aAAa,CAAC;EACjD;EACA,OAAOA,aAAa;AACtB"}
+{"version":3,"file":"parse-in-batches.js","names":["isTable","makeBatchFromTable","concatenateArrayBuffersAsync","isLoaderObject","normalizeOptions","getLoaderContext","getAsyncIterableFromData","getResourceUrl","selectLoader","parse","parseInBatches","data","loaders","options","context","loaderArray","Array","isArray","undefined","url","loader","_parseInBatches","_parse","parseWithLoaderInBatches","outputIterator","parseToOutputIterator","metadata","metadataBatch","batchType","_loader","_context","bytesUsed","makeMetadataBatchIterator","iterator","inputIterator","transformedIterator","applyInputTransforms","transforms","parseChunkInBatches","arrayBuffer","parsedData","mimeType","mimeTypes","batch","convertDataToBatch","shape","length","arguments","iteratorChain","transformBatches"],"sources":["../../../src/lib/api/parse-in-batches.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport {isTable, makeBatchFromTable, type Batch} from '@loaders.gl/schema';\nimport type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {LoaderContext, BatchableDataType} from '@loaders.gl/loader-utils';\nimport type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';\nimport {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';\nimport {isLoaderObject} from '../loader-utils/normalize-loader';\nimport {normalizeOptions} from '../loader-utils/option-utils';\nimport {getLoaderContext} from '../loader-utils/loader-context';\nimport {getAsyncIterableFromData} from '../loader-utils/get-data';\nimport {getResourceUrl} from '../utils/resource-utils';\nimport {selectLoader} from './select-loader';\n\n// Ensure `parse` is available in context if loader falls back to `parse`\nimport {parse} from './parse';\n\n/**\n * Parses `data` synchronously using a specified loader\n */\nexport async function parseInBatches<\n  LoaderT extends Loader,\n  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>\n>(\n  data: BatchableDataType,\n  loader: LoaderT,\n  options?: OptionsT,\n  context?: LoaderContext\n): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;\n\n/**\n * Parses `data` using one of the supplied loaders\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  loaders: Loader[],\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<unknown>>;\n\n/**\n * Parses `data` in batches by selecting a pre-registered loader\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  options?: LoaderOptions\n): Promise<AsyncIterable<unknown>>;\n\n/**\n * Parses `data` using a specified loader\n * @param data\n * @param loaders\n * @param options\n * @param context\n */\nexport async function parseInBatches(\n  data: BatchableDataType,\n  loaders?: Loader | Loader[] | LoaderOptions,\n  options?: LoaderOptions,\n  context?: LoaderContext\n): Promise<AsyncIterable<unknown> | Iterable<unknown>> {\n  const loaderArray = Array.isArray(loaders) ? loaders : undefined;\n\n  // Signature: parseInBatches(data, options, url) - Uses registered loaders\n  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {\n    context = undefined; // context not supported in short signature\n    options = loaders as LoaderOptions;\n    loaders = undefined;\n  }\n\n  data = await data; // Resolve any promise\n  options = options || {};\n\n  // Extract a url for auto detection\n  const url = getResourceUrl(data);\n\n  // Chooses a loader and normalizes it\n  // Note - only uses URL and contentType for streams and iterator inputs\n  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader | Loader[], options);\n  // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null\n  if (!loader) {\n    return [];\n  }\n\n  // Normalize options\n  options = normalizeOptions(options, loader, loaderArray, url);\n  context = getLoaderContext(\n    {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},\n    options,\n    context || null\n  );\n\n  return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);\n}\n\n/**\n * Loader has been selected and context has been prepared, see if we need to emit a metadata batch\n */\nasync function parseWithLoaderInBatches(\n  loader: LoaderWithParser,\n  data: BatchableDataType,\n  options: LoaderOptions,\n  context: LoaderContext\n): Promise<AsyncIterable<unknown>> {\n  const outputIterator = await parseToOutputIterator(loader, data, options, context);\n\n  // Generate metadata batch if requested\n  if (!options.metadata) {\n    return outputIterator;\n  }\n\n  const metadataBatch = {\n    batchType: 'metadata',\n    metadata: {\n      _loader: loader,\n      _context: context\n    },\n    // Populate with some default fields to avoid crashing\n    data: [],\n    bytesUsed: 0\n  };\n\n  async function* makeMetadataBatchIterator(\n    iterator: Iterable<unknown> | AsyncIterable<unknown>\n  ): AsyncIterable<unknown> {\n    yield metadataBatch;\n    yield* iterator;\n  }\n\n  return makeMetadataBatchIterator(outputIterator);\n}\n\n/**\n * Prep work is done, now it is time to start parsing into an output operator\n * The approach depends on which parse function the loader exposes\n * `parseInBatches` (preferred), `parse` (fallback)\n */\nasync function parseToOutputIterator(\n  loader: LoaderWithParser,\n  data: BatchableDataType,\n  options: LoaderOptions,\n  context: LoaderContext\n): Promise<AsyncIterable<unknown>> {\n  // Get an iterator from the input\n  const inputIterator = await getAsyncIterableFromData(data, options);\n\n  // Apply any iterator transforms (options.transforms)\n  const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);\n\n  // If loader supports parseInBatches, we are done\n  if (loader.parseInBatches) {\n    return loader.parseInBatches(transformedIterator, options, context);\n  }\n\n  return parseChunkInBatches(transformedIterator, loader, options, context);\n}\n\n// Fallback: load atomically using `parse` concatenating input iterator into single chunk\nasync function* parseChunkInBatches(\n  transformedIterator: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,\n  loader: Loader,\n  options: LoaderOptions,\n  context: LoaderContext\n): AsyncIterable<Batch> {\n  const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);\n  // Call `parse` instead of `loader.parse` to ensure we can call workers etc.\n  const parsedData = await parse(\n    arrayBuffer,\n    loader,\n    // TODO - Hack: supply loaders MIME type to ensure we match it\n    {...options, mimeType: loader.mimeTypes[0]},\n    context\n  );\n\n  // yield a single batch, the output from loader.parse() repackaged as a batch\n  const batch = convertDataToBatch(parsedData, loader);\n\n  yield batch;\n}\n\n/**\n * Convert parsed data into a single batch\n * @todo run through batch builder to apply options etc...\n */\nfunction convertDataToBatch(parsedData: unknown, loader: Loader): Batch {\n  const batch: Batch = isTable(parsedData)\n    ? makeBatchFromTable(parsedData)\n    : {\n        shape: 'unknown',\n        batchType: 'data',\n        data: parsedData,\n        length: Array.isArray(parsedData) ? parsedData.length : 1\n      };\n\n  batch.mimeType = loader.mimeTypes[0];\n\n  return batch;\n}\n\ntype TransformBatches = (\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n) => AsyncIterable<ArrayBuffer>;\n\n/**\n * Create an iterator chain with any transform iterators (crypto, decompression)\n * @param inputIterator\n * @param options\n */\nasync function applyInputTransforms(\n  inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  transforms: TransformBatches[] = []\n): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {\n  let iteratorChain = inputIterator;\n  for await (const transformBatches of transforms) {\n    iteratorChain = transformBatches(iteratorChain);\n  }\n  return iteratorChain;\n}\n"],"mappings":"AAGA,SAAQA,OAAO,EAAEC,kBAAkB,QAAmB,oBAAoB;AAI1E,SAAQC,4BAA4B,QAAO,0BAA0B;AAAC,SAC9DC,cAAc;AAAA,SACdC,gBAAgB;AAAA,SAChBC,gBAAgB;AAAA,SAChBC,wBAAwB;AAAA,SACxBC,cAAc;AAAA,SACdC,YAAY;AAAA,SAGZC,KAAK;AAwCb,OAAO,eAAeC,cAAcA,CAClCC,IAAuB,EACvBC,OAA2C,EAC3CC,OAAuB,EACvBC,OAAuB,EAC8B;EACrD,MAAMC,WAAW,GAAGC,KAAK,CAACC,OAAO,CAACL,OAAO,CAAC,GAAGA,OAAO,GAAGM,SAAS;EAGhE,IAAI,CAACF,KAAK,CAACC,OAAO,CAACL,OAAO,CAAC,IAAI,CAACT,cAAc,CAACS,OAAO,CAAC,EAAE;IACvDE,OAAO,GAAGI,SAAS;IACnBL,OAAO,GAAGD,OAAwB;IAClCA,OAAO,GAAGM,SAAS;EACrB;EAEAP,IAAI,GAAG,MAAMA,IAAI;EACjBE,OAAO,GAAGA,OAAO,IAAI,CAAC,CAAC;EAGvB,MAAMM,GAAG,GAAGZ,cAAc,CAACI,IAAI,CAAC;EAIhC,MAAMS,MAAM,GAAG,MAAMZ,YAAY,CAACG,IAAI,EAAiBC,OAAO,EAAuBC,OAAO,CAAC;EAE7F,IAAI,CAACO,MAAM,EAAE;IACX,OAAO,EAAE;EACX;EAGAP,OAAO,GAAGT,gBAAgB,CAACS,OAAO,EAAEO,MAAM,EAAEL,WAAW,EAAEI,GAAG,CAAC;EAC7DL,OAAO,GAAGT,gBAAgB,CACxB;IAACc,GAAG;IAAEE,eAAe,EAAEX,cAAc;IAAEY,MAAM,EAAEb,KAAK;IAAEG,OAAO,EAAEG;EAAW,CAAC,EAC3EF,OAAO,EACPC,OAAO,IAAI,IACb,CAAC;EAED,OAAO,MAAMS,wBAAwB,CAACH,MAAM,EAAsBT,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;AAC3F;AAKA,eAAeS,wBAAwBA,CACrCH,MAAwB,EACxBT,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB,EACW;EACjC,MAAMU,cAAc,GAAG,MAAMC,qBAAqB,CAACL,MAAM,EAAET,IAAI,EAAEE,OAAO,EAAEC,OAAO,CAAC;EAGlF,IAAI,CAACD,OAAO,CAACa,QAAQ,EAAE;IACrB,OAAOF,cAAc;EACvB;EAEA,MAAMG,aAAa,GAAG;IACpBC,SAAS,EAAE,UAAU;IACrBF,QAAQ,EAAE;MACRG,OAAO,EAAET,MAAM;MACfU,QAAQ,EAAEhB;IACZ,CAAC;IAEDH,IAAI,EAAE,EAAE;IACRoB,SAAS,EAAE;EACb,CAAC;EAED,gBAAgBC,yBAAyBA,CACvCC,QAAoD,EAC5B;IACxB,MAAMN,aAAa;IACnB,OAAOM,QAAQ;EACjB;EAEA,OAAOD,yBAAyB,CAACR,cAAc,CAAC;AAClD;AAOA,eAAeC,qBAAqBA,CAClCL,MAAwB,EACxBT,IAAuB,EACvBE,OAAsB,EACtBC,OAAsB,EACW;EAEjC,MAAMoB,aAAa,GAAG,MAAM5B,wBAAwB,CAACK,IAAI,EAAEE,OAAO,CAAC;EAGnE,MAAMsB,mBAAmB,GAAG,MAAMC,oBAAoB,CAACF,aAAa,EAAE,CAAArB,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEwB,UAAU,KAAI,EAAE,CAAC;EAGhG,IAAIjB,MAAM,CAACV,cAAc,EAAE;IACzB,OAAOU,MAAM,CAACV,cAAc,CAACyB,mBAAmB,EAAEtB,OAAO,EAAEC,OAAO,CAAC;EACrE;EAEA,OAAOwB,mBAAmB,CAACH,mBAAmB,EAAEf,MAAM,EAAEP,OAAO,EAAEC,OAAO,CAAC;AAC3E;AAGA,gBAAgBwB,mBAAmBA,CACjCH,mBAAuE,EACvEf,MAAc,EACdP,OAAsB,EACtBC,OAAsB,EACA;EACtB,MAAMyB,WAAW,GAAG,MAAMrC,4BAA4B,CAACiC,mBAAmB,CAAC;EAE3E,MAAMK,UAAU,GAAG,MAAM/B,KAAK,CAC5B8B,WAAW,EACXnB,MAAM,EAEN;IAAC,GAAGP,OAAO;IAAE4B,QAAQ,EAAErB,MAAM,CAACsB,SAAS,CAAC,CAAC;EAAC,CAAC,EAC3C5B,OACF,CAAC;EAGD,MAAM6B,KAAK,GAAGC,kBAAkB,CAACJ,UAAU,EAAEpB,MAAM,CAAC;EAEpD,MAAMuB,KAAK;AACb;AAMA,SAASC,kBAAkBA,CAACJ,UAAmB,EAAEpB,MAAc,EAAS;EACtE,MAAMuB,KAAY,GAAG3C,OAAO,CAACwC,UAAU,CAAC,GACpCvC,kBAAkB,CAACuC,UAAU,CAAC,GAC9B;IACEK,KAAK,EAAE,SAAS;IAChBjB,SAAS,EAAE,MAAM;IACjBjB,IAAI,EAAE6B,UAAU;IAChBM,MAAM,EAAE9B,KAAK,CAACC,OAAO,CAACuB,UAAU,CAAC,GAAGA,UAAU,CAACM,MAAM,GAAG;EAC1D,CAAC;EAELH,KAAK,CAACF,QAAQ,GAAGrB,MAAM,CAACsB,SAAS,CAAC,CAAC,CAAC;EAEpC,OAAOC,KAAK;AACd;AAWA,eAAeP,oBAAoBA,CACjCF,aAAiE,EAEJ;EAAA,IAD7DG,UAA8B,GAAAU,SAAA,CAAAD,MAAA,QAAAC,SAAA,QAAA7B,SAAA,GAAA6B,SAAA,MAAG,EAAE;EAEnC,IAAIC,aAAa,GAAGd,aAAa;EACjC,WAAW,MAAMe,gBAAgB,IAAIZ,UAAU,EAAE;IAC/CW,aAAa,GAAGC,gBAAgB,CAACD,aAAa,CAAC;EACjD;EACA,OAAOA,aAAa;AACtB"}

package/dist/null-worker-node.js

@@ -208,7 +208,7 @@ async function parseData({
 }
 
 // src/null-loader.ts
-var VERSION = true ? "4.0.2" : "latest";
+var VERSION = true ? "4.0.3" : "latest";
 var NullLoader = {
   name: "Null loader",
   id: "null",

package/dist/null-worker.js

@@ -208,7 +208,7 @@
   }
 
   // src/null-loader.ts
-  var VERSION = true ? "4.0.2" : "latest";
+  var VERSION = true ? "4.0.3" : "latest";
   var NullLoader = {
     name: "Null loader",
     id: "null",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/core",
-  "version": "4.0.2",
+  "version": "4.0.3",
   "description": "The core API for working with loaders.gl loaders and writers",
   "license": "MIT",
   "type": "module",
@@ -49,9 +49,9 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/loader-utils": "4.0.2",
-    "@loaders.gl/worker-utils": "4.0.2",
+    "@loaders.gl/loader-utils": "4.0.3",
+    "@loaders.gl/worker-utils": "4.0.3",
     "@probe.gl/log": "^4.0.2"
   },
-  "gitHead": "471058d109d5652f28c32c1f296fd632f9a5c806"
+  "gitHead": "03c871839b36c997249dabae1844df53a35d3760"
 }

package/src/lib/api/encode-table.ts

@@ -3,10 +3,14 @@
 // Copyright 2022 Foursquare Labs, Inc
 
 /* global TextEncoder, TextDecoder */
-import {concatenateArrayBuffers, Writer, WriterOptionsType} from '@loaders.gl/loader-utils';
+import {
+  concatenateArrayBuffers,
+  WriterOptionsType,
+  WriterWithEncoder
+} from '@loaders.gl/loader-utils';
 import {Table} from '@loaders.gl/schema';
 
-export async function encodeTable<WriterT extends Writer = Writer>(
+export async function encodeTable<WriterT extends WriterWithEncoder = WriterWithEncoder>(
   data: Table,
   writer: WriterT,
   options?: WriterOptionsType<WriterT>
@@ -36,7 +40,7 @@ export async function encodeTable<WriterT extends Writer = Writer>(
   throw new Error('Writer could not encode data');
 }
 
-export async function encodeTableAsText<WriterT extends Writer = Writer>(
+export async function encodeTableAsText<WriterT extends WriterWithEncoder = WriterWithEncoder>(
   data: Table,
   writer: WriterT,
   options?: WriterOptionsType<WriterT>
@@ -52,7 +56,7 @@ export async function encodeTableAsText<WriterT extends Writer = Writer>(
   throw new Error(`Writer ${writer.name} could not encode data as text`);
 }
 
-export function encodeTableInBatches<WriterT extends Writer = Writer>(
+export function encodeTableInBatches<WriterT extends WriterWithEncoder = WriterWithEncoder>(
   data: Table,
   writer: WriterT,
   options?: WriterOptionsType<WriterT>

package/src/lib/api/encode.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils';
+import {WriterOptions, WriterWithEncoder, canEncodeWithWorker} from '@loaders.gl/loader-utils';
 import {concatenateArrayBuffers, resolvePath, NodeFile} from '@loaders.gl/loader-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
 import {isBrowser} from '@loaders.gl/loader-utils';
@@ -13,7 +13,7 @@ import {getLoaderOptions} from './loader-options';
  */
 export async function encode(
   data: unknown,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): Promise<ArrayBuffer> {
   const globalOptions = getLoaderOptions() as WriterOptions;
@@ -75,7 +75,11 @@ export async function encode(
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
  */
-export function encodeSync(data: unknown, writer: Writer, options?: WriterOptions): ArrayBuffer {
+export function encodeSync(
+  data: unknown,
+  writer: WriterWithEncoder,
+  options?: WriterOptions
+): ArrayBuffer {
   if (writer.encodeSync) {
     return writer.encodeSync(data, options);
   }
@@ -90,7 +94,7 @@ export function encodeSync(data: unknown, writer: Writer, options?: WriterOption
  */
 export async function encodeText(
   data: unknown,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): Promise<string> {
   if (writer.text && writer.encodeText) {
@@ -111,7 +115,11 @@ export async function encodeText(
  * It is not optimized for performance. Data maybe converted from text to binary and back.
  * @throws if the writer does not generate text output
  */
-export function encodeTextSync(data: unknown, writer: Writer, options?: WriterOptions): string {
+export function encodeTextSync(
+  data: unknown,
+  writer: WriterWithEncoder,
+  options?: WriterOptions
+): string {
   if (writer.text && writer.encodeTextSync) {
     return writer.encodeTextSync(data, options);
   }
@@ -129,7 +137,7 @@ export function encodeTextSync(data: unknown, writer: Writer, options?: WriterOp
  */
 export function encodeInBatches(
   data: unknown,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): AsyncIterable<ArrayBuffer> {
   if (writer.encodeInBatches) {
@@ -148,7 +156,7 @@ export function encodeInBatches(
 export async function encodeURLtoURL(
   inputUrl: string,
   outputUrl: string,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): Promise<string> {
   inputUrl = resolvePath(inputUrl);

package/src/lib/api/parse-in-batches.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Batch} from '@loaders.gl/schema';
+import {isTable, makeBatchFromTable, type Batch} from '@loaders.gl/schema';
 import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
 import type {LoaderContext, BatchableDataType} from '@loaders.gl/loader-utils';
 import type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';
@@ -153,30 +153,49 @@ async function parseToOutputIterator(
     return loader.parseInBatches(transformedIterator, options, context);
   }
 
-  // Fallback: load atomically using `parse` concatenating input iterator into single chunk
-  async function* parseChunkInBatches() {
-    const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
-    // Call `parse` instead of `loader.parse` to ensure we can call workers etc.
-    const parsedData = await parse(
-      arrayBuffer,
-      loader,
-      // TODO - Hack: supply loaders MIME type to ensure we match it
-      {...options, mimeType: loader.mimeTypes[0]},
-      context
-    );
-    // yield a single batch, the output from loader.parse()
-    // TODO - run through batch builder to apply options etc...
-    const batch: Batch = {
-      mimeType: loader.mimeTypes[0],
-      shape: Array.isArray(parsedData) ? 'row-table' : 'unknown',
-      batchType: 'data',
-      data: parsedData,
-      length: Array.isArray(parsedData) ? parsedData.length : 1
-    };
-    yield batch;
-  }
+  return parseChunkInBatches(transformedIterator, loader, options, context);
+}
+
+// Fallback: load atomically using `parse` concatenating input iterator into single chunk
+async function* parseChunkInBatches(
+  transformedIterator: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
+  loader: Loader,
+  options: LoaderOptions,
+  context: LoaderContext
+): AsyncIterable<Batch> {
+  const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
+  // Call `parse` instead of `loader.parse` to ensure we can call workers etc.
+  const parsedData = await parse(
+    arrayBuffer,
+    loader,
+    // TODO - Hack: supply loaders MIME type to ensure we match it
+    {...options, mimeType: loader.mimeTypes[0]},
+    context
+  );
 
-  return parseChunkInBatches();
+  // yield a single batch, the output from loader.parse() repackaged as a batch
+  const batch = convertDataToBatch(parsedData, loader);
+
+  yield batch;
+}
+
+/**
+ * Convert parsed data into a single batch
+ * @todo run through batch builder to apply options etc...
+ */
+function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
+  const batch: Batch = isTable(parsedData)
+    ? makeBatchFromTable(parsedData)
+    : {
+        shape: 'unknown',
+        batchType: 'data',
+        data: parsedData,
+        length: Array.isArray(parsedData) ? parsedData.length : 1
+      };
+
+  batch.mimeType = loader.mimeTypes[0];
+
+  return batch;
 }
 
 type TransformBatches = (