@loaders.gl/csv 4.4.0-alpha.1 → 4.4.0-alpha.10

package/dist/csv-arrow-loader.d.ts

@@ -32,6 +32,6 @@ export declare const CSVArrowLoader: {
     readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
     readonly category: "table";
 };
-declare function parseCSVToArrowBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: CSVArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
+declare function parseCSVToArrowBatches(asyncIterator: AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>, options?: CSVArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
 export {};
 //# sourceMappingURL=csv-arrow-loader.d.ts.map

package/dist/csv-arrow-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-arrow-loader.d.ts","sourceRoot":"","sources":["../src/csv-arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAGpE,OAAO,KAAK,EAAC,gBAAgB,EAAC,wBAAqB;AAGnD,MAAM,MAAM,qBAAqB,GAAG,aAAa,GAAG;IAClD,GAAG,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;CAC9C,CAAC;AAEF,eAAO,MAAM,cAAc;uBAGI,UAAU;wBACT,eAAe;kCAElB,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;CAEmC,CAAC;AAS1F,iBAAS,sBAAsB,CAC7B,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,qBAAqB,GAC9B,aAAa,CAAC,eAAe,CAAC,CAGhC"}
+{"version":3,"file":"csv-arrow-loader.d.ts","sourceRoot":"","sources":["../src/csv-arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAGpE,OAAO,KAAK,EAAC,gBAAgB,EAAC,wBAAqB;AAGnD,MAAM,MAAM,qBAAqB,GAAG,aAAa,GAAG;IAClD,GAAG,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;CAC9C,CAAC;AAEF,eAAO,MAAM,cAAc;uBAGI,UAAU;wBACT,eAAe;kCAElB,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;CAEmC,CAAC;AAS1F,iBAAS,sBAAsB,CAC7B,aAAa,EACT,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,CAAC,EAAE,qBAAqB,GAC9B,aAAa,CAAC,eAAe,CAAC,CAGhC"}

package/dist/csv-arrow-loader.js

@@ -21,3 +21,4 @@ function parseCSVToArrowBatches(asyncIterator, options) {
     const tableIterator = CSVLoader.parseInBatches(asyncIterator, options);
     return convertBatches(tableIterator, 'arrow-table');
 }
+//# sourceMappingURL=csv-arrow-loader.js.map

package/dist/csv-arrow-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-arrow-loader.js","sourceRoot":"","sources":["../src/csv-arrow-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,0BAA0B,CAAC;AAGtE,OAAO,EAAC,SAAS,EAAC,wBAAqB;AAMvC,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,GAAG,SAAS;IAEZ,QAAQ,EAAE,IAA6B;IACvC,SAAS,EAAE,IAAkC;IAE7C,KAAK,EAAE,KAAK,EAAE,WAAwB,EAAE,OAA0B,EAAE,EAAE,CACpE,eAAe,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IACjE,SAAS,EAAE,CAAC,IAAY,EAAE,OAA0B,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC;IACvF,cAAc,EAAE,sBAAsB;CACiD,CAAC;AAE1F,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,OAA0B;IACxE,4EAA4E;IAC5E,uEAAuE;IACvE,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1D,OAAO,YAAY,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,sBAAsB,CAC7B,aAE+C,EAC/C,OAA+B;IAE/B,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IACvE,OAAO,cAAc,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;AACtD,CAAC"}

package/dist/csv-format.js

@@ -10,3 +10,4 @@ export const CSVFormat = {
     mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
     category: 'table'
 };
+//# sourceMappingURL=csv-format.js.map

package/dist/csv-format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-format.js","sourceRoot":"","sources":["../src/csv-format.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,6BAA6B;AAC7B,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,EAAE,EAAE,KAAK;IACT,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,KAAK;IACX,UAAU,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;IACjC,SAAS,EAAE,CAAC,UAAU,EAAE,2BAA2B,EAAE,UAAU,CAAC;IAChE,QAAQ,EAAE,OAAO;CACQ,CAAC"}

package/dist/csv-loader.d.ts

@@ -43,6 +43,6 @@ export declare const CSVLoader: {
     readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
     readonly category: "table";
 };
-declare function parseCSVInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: CSVLoaderOptions): AsyncIterable<TableBatch>;
+declare function parseCSVInBatches(asyncIterator: AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>, options?: CSVLoaderOptions): AsyncIterable<TableBatch>;
 export {};
 //# sourceMappingURL=csv-loader.d.ts.map

package/dist/csv-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../src/csv-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAS,aAAa,EAAE,cAAc,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAoB1F,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QAEJ,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;QAC/C,yDAAyD;QACzD,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;QAKhB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;QAEpC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAE9B,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,SAAS;uBAGS,cAAc,GAAG,aAAa;wBAC7B,UAAU;;kCAEb,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;;;;;;;CAuB6C,CAAC;AAsDpG,iBAAS,iBAAiB,CACxB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,gBAAgB,GACzB,aAAa,CAAC,UAAU,CAAC,CAyH3B"}
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../src/csv-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAS,aAAa,EAAE,cAAc,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAoB1F,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QAEJ,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;QAC/C,yDAAyD;QACzD,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;QAKhB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;QAEpC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAE9B,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,SAAS;uBAGS,cAAc,GAAG,aAAa;wBAC7B,UAAU;;kCAEb,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;;;;;;;CAuB6C,CAAC;AAsDpG,iBAAS,iBAAiB,CACxB,aAAa,EACT,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,CAAC,EAAE,gBAAgB,GACzB,aAAa,CAAC,UAAU,CAAC,CAyH3B"}

package/dist/csv-loader.js

@@ -1,14 +1,14 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import { log } from '@loaders.gl/loader-utils';
+import { log, toArrayBufferIterator } from '@loaders.gl/loader-utils';
 import { AsyncQueue, deduceTableSchema, TableBatchBuilder, convertToArrayRow, convertToObjectRow } from '@loaders.gl/schema-utils';
 import Papa from "./papaparse/papaparse.js";
 import AsyncIteratorStreamer from "./papaparse/async-iterator-streamer.js";
 import { CSVFormat } from "./csv-format.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.4.0-alpha.0" !== 'undefined' ? "4.4.0-alpha.0" : 'latest';
+const VERSION = typeof "4.4.0-alpha.10" !== 'undefined' ? "4.4.0-alpha.10" : 'latest';
 const DEFAULT_CSV_SHAPE = 'object-row-table';
 export const CSVLoader = {
     ...CSVFormat,
@@ -85,8 +85,8 @@ function parseCSVInBatches(asyncIterator, options) {
     // Papaparse does not support standard batch size handling
     // TODO - investigate papaparse chunks mode
     options = { ...options };
-    if (options.batchSize === 'auto') {
-        options.batchSize = 4000;
+    if (options?.core?.batchSize === 'auto') {
+        options.core.batchSize = 4000;
     }
     // Apps can call the parse method directly, we so apply default options here
     const csvOptions = { ...CSVLoader.options.csv, ...options?.csv };
@@ -144,7 +144,7 @@ function parseCSVInBatches(asyncIterator, options) {
                 // https://bugs.chromium.org/p/v8/issues/detail?id=2869
                 row = JSON.parse(JSON.stringify(row));
             }
-            const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
+            const shape = options?.shape || csvOptions.shape || DEFAULT_CSV_SHAPE;
             // Add the row
             tableBatchBuilder =
                 tableBatchBuilder ||
@@ -152,7 +152,7 @@ function parseCSVInBatches(asyncIterator, options) {
                     // @ts-expect-error TODO this is not a proper schema
                     schema, {
                         shape,
-                        ...options
+                        ...(options?.core || {})
                     });
             try {
                 tableBatchBuilder.addRow(row);
@@ -182,7 +182,7 @@ function parseCSVInBatches(asyncIterator, options) {
             asyncQueue.close();
         }
     };
-    Papa.parse(asyncIterator, config, AsyncIteratorStreamer);
+    Papa.parse(toArrayBufferIterator(asyncIterator), config, AsyncIteratorStreamer);
     // TODO - Does it matter if we return asyncIterable or asyncIterator
     // return asyncQueue[Symbol.asyncIterator]();
     return asyncQueue;
@@ -290,3 +290,4 @@ function deduceCSVSchema(row, headerRow) {
 //   }
 //   return schema;
 // }
+//# sourceMappingURL=csv-loader.js.map

package/dist/csv-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../src/csv-loader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,OAAO,EAAC,GAAG,EAAE,qBAAqB,EAAC,MAAM,0BAA0B,CAAC;AACpE,OAAO,EACL,UAAU,EACV,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,IAAI,iCAA8B;AACzC,OAAO,qBAAqB,+CAA4C;AACxE,OAAO,EAAC,SAAS,EAAC,wBAAqB;AAEvC,yDAAyD;AACzD,qDAAqD;AACrD,MAAM,OAAO,GAAG,uBAAkB,KAAK,WAAW,CAAC,CAAC,kBAAa,CAAC,CAAC,QAAQ,CAAC;AAE5E,MAAM,iBAAiB,GAAG,kBAAkB,CAAC;AA0B7C,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,GAAG,SAAS;IAEZ,QAAQ,EAAE,IAAiD;IAC3D,SAAS,EAAE,IAA6B;IACxC,OAAO,EAAE,OAAO;IAChB,KAAK,EAAE,KAAK,EAAE,WAAwB,EAAE,OAA0B,EAAE,EAAE,CACpE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;IAC1D,SAAS,EAAE,CAAC,IAAY,EAAE,OAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAChF,cAAc,EAAE,iBAAiB;IACjC,aAAa;IACb,kBAAkB;IAClB,OAAO,EAAE;QACP,GAAG,EAAE;YACH,KAAK,EAAE,iBAAiB,EAAE,qBAAqB;YAC/C,mBAAmB,EAAE,KAAK;YAC1B,cAAc;YACd,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,QAAQ;YACtB,kBAAkB;YAClB,gBAAgB;YAChB,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI;YACnB,QAAQ,EAAE,KAAK;YACf,cAAc,EAAE,IAAI;YACpB,mBAAmB;YACnB,iBAAiB,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC;YACxC,iBAAiB;SAClB;KACF;CACgG,CAAC;AAEpG,KAAK,UAAU,QAAQ,CACrB,OAAe,EACf,OAA0B;IAE1B,4EAA4E;IAC5E,MAAM,UAAU,GAAG,EAAC,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,EAAC,CAAC;IAE/D,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GACV,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpF,MAAM,eAAe,GAAG,MAAM,CAAC;IAE/B,MAAM,eAAe,GAAG;QACtB,uBAAuB;QACvB,GAAG,UAAU;QACb,MAAM,EAAE,eAAe;QACvB,QAAQ,EAAE,KAAK,EAAE,2DAA2D;QAC5E,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC,SAAS;QAC3E,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAa,CAAC;IAElC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEjG,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,iBAAiB,CAAC;IACpD,IAAI,KAAqC,CAAC;IAC1C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,kBAAkB;YACrB,KAAK,GAAG;gBACN,KAAK,EAAE,kBAAkB;gBACzB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;aACzF,CAAC;YACF,MAAM;QACR,KAAK,iBAAiB;YACpB,KAAK,GAAG;gBACN,KAAK,EAAE,iBAAiB;gBACxB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;aACxF,CAAC;YACF,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IACD,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,KAAM,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0DAA0D;AAC1D,SAAS,iBAAiB,CACxB,aAE+C,EAC/C,OAA0B;IAE1B,0DAA0D;IAC1D,2CAA2C;IAC3C,OAAO,GAAG,EAAC,GAAG,OAAO,EAAC,CAAC;IACvB,IAAI,OAAO,EAAE,IAAI,EAAE,SAAS,KAAK,MAAM,EAAE,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IAChC,CAAC;IAED,4EAA4E;IAC5E,MAAM,UAAU,GAAG,EAAC,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,EAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,UAAU,EAAc,CAAC;IAEhD,IAAI,UAAU,GAAY,IAAI,CAAC;IAC/B,IAAI,SAAS,GAAoB,IAAI,CAAC;IACtC,IAAI,iBAAiB,GAA6B,IAAI,CAAC;IACvD,IAAI,MAAM,GAAkB,IAAI,CAAC;IAEjC,MAAM,MAAM,GAAG;QACb,mFAAmF;QACnF,GAAG,UAAU;QACb,MAAM,EAAE,KAAK,EAAE,6EAA6E;QAC5F,QAAQ,EAAE,KAAK,EAAE,2DAA2D;QAC5E,wFAAwF;QACxF,uFAAuF;QACvF,oDAAoD;QACpD,SAAS,EAAE,IAAI,GAAG,IAAI,GAAG,CAAC;QAC1B,8EAA8E;QAC9E,6FAA6F;QAC7F,qEAAqE;QACrE,6FAA6F;QAC7F,uEAAuE;QACvE,cAAc,EAAE,KAAK;QAErB,8BAA8B;QAC9B,sDAAsD;QACtD,IAAI,CAAC,OAAO;YACV,IAAI,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;YAEvB,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;gBAC9B,uCAAuC;gBACvC,MAAM,YAAY,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;oBACxB,OAAO;gBACT,CAAC;YACH,CAAC;YACD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;YAEtC,wCAAwC;YACxC,IAAI,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7B,uDAAuD;gBACvD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBAC5F,IAAI,MAAM,EAAE,CAAC;oBACX,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,EAAE,CAAC,CAAC;oBAClD,OAAO;gBACT,CAAC;YACH,CAAC;YAED,8CAA8C;YAC9C,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,GAAG,KAAK,CAAC;gBACnB,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,SAAS,GAAG,cAAc,CAAC,UAAU,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBAClE,CAAC;gBACD,MAAM,GAAG,eAAe,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YAC3C,CAAC;YAED,IAAI,UAAU,CAAC,mBAAmB,EAAE,CAAC;gBACnC,sFAAsF;gBACtF,uDAAuD;gBACvD,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,KAAK,GAAI,OAAe,EAAE,KAAK,IAAI,UAAU,CAAC,KAAK,IAAI,iBAAiB,CAAC;YAE/E,cAAc;YACd,iBAAiB;gBACf,iBAAiB;oBACjB,IAAI,iBAAiB;oBACnB,oDAAoD;oBACpD,MAAM,EACN;wBACE,KAAK;wBACL,GAAG,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;qBACzB,CACF,CAAC;YAEJ,IAAI,CAAC;gBACH,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC9B,yCAAyC;gBACzC,MAAM,KAAK,GAAG,iBAAiB,IAAI,iBAAiB,CAAC,YAAY,CAAC,EAAC,SAAS,EAAC,CAAC,CAAC;gBAC/E,IAAI,KAAK,EAAE,CAAC;oBACV,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,CAAC,OAAO,CAAC,KAAc,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,kDAAkD;QAClD,QAAQ,CAAC,OAAO;YACd,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;gBACtC,gDAAgD;gBAChD,MAAM,KAAK,GAAG,iBAAiB,IAAI,iBAAiB,CAAC,aAAa,CAAC,EAAC,SAAS,EAAC,CAAC,CAAC;gBAChF,IAAI,KAAK,EAAE,CAAC;oBACV,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,CAAC,OAAO,CAAC,KAAc,CAAC,CAAC;YACrC,CAAC;YAED,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;KACF,CAAC;IAEF,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAEhF,oEAAoE;IACpE,6CAA6C;IAC7C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAa;IAChC,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC;AAChE,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE;QACjC,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,CAAC;KACX,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACxB,CAAC;AAED;;;;;GAKG;AACH,SAAS,0BAA0B;IACjC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,EAAE;QACb,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,OAAO,GAAG,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC;YAC9B,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CAAC,YAAoB,EAAE,QAAgB,CAAC;IAC7D,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,GAAG,EAAE,SAAS;IACrC,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,CAAC,SAAS,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,QAAQ,OAAO,KAAK,EAAE,CAAC;YACrB,KAAK,QAAQ;gBACX,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,SAAS;gBACZ,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC,CAAC;gBACtE,MAAM;YACR,KAAK,QAAQ;gBACX,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC,CAAC;gBACtE,MAAM;YACR;gBACE,GAAG,CAAC,IAAI,CAAC,6BAA6B,OAAO,KAAK,EAAE,CAAC,EAAE,CAAC;gBACxD,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,OAAO;QACL,MAAM;QACN,QAAQ,EAAE;YACR,mBAAmB,EAAE,KAAK;YAC1B,mBAAmB,EAAE,WAAW;SACjC;KACF,CAAC;AACJ,CAAC;AAED,gBAAgB;AAChB,+DAA+D;AAC/D,oEAAoE;AAEpE,8DAA8D;AAC9D,sDAAsD;AACtD,2CAA2C;AAC3C,2DAA2D;AAC3D,4BAA4B;AAC5B,8BAA8B;AAC9B,uBAAuB;AACvB,wBAAwB;AACxB,6DAA6D;AAC7D,yFAAyF;AACzF,iBAAiB;AACjB,uBAAuB;AACvB,iBAAiB;AACjB,kFAAkF;AAClF,iDAAiD;AACjD,qEAAqE;AACrE,QAAQ;AACR,MAAM;AACN,mBAAmB;AACnB,IAAI"}

package/dist/csv-writer.js

@@ -15,3 +15,4 @@ export const CSVWriter = {
     encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
     encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
 };
+//# sourceMappingURL=csv-writer.js.map

package/dist/csv-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-writer.js","sourceRoot":"","sources":["../src/csv-writer.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,OAAO,EAAC,gBAAgB,EAAC,qCAAkC;AAC3D,OAAO,EAAC,SAAS,EAAC,wBAAqB;AAQvC,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,GAAG,SAAS;IACZ,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE;QACP,GAAG,EAAE;YACH,eAAe,EAAE,KAAK;SACvB;KACF;IACD,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAC/B,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM;IACnE,cAAc,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC;CACK,CAAC"}

package/dist/dist.dev.js

@@ -504,6 +504,29 @@ var __exports__ = (() => {
   }
   var log = createLog();
 
+  // ../loader-utils/src/lib/iterators/async-iteration.ts
+  async function* toArrayBufferIterator(asyncIterator) {
+    for await (const chunk of asyncIterator) {
+      yield copyToArrayBuffer(chunk);
+    }
+  }
+  function copyToArrayBuffer(chunk) {
+    if (chunk instanceof ArrayBuffer) {
+      return chunk;
+    }
+    if (ArrayBuffer.isView(chunk)) {
+      const { buffer, byteOffset, byteLength } = chunk;
+      return copyFromBuffer(buffer, byteOffset, byteLength);
+    }
+    return copyFromBuffer(chunk);
+  }
+  function copyFromBuffer(buffer, byteOffset = 0, byteLength = buffer.byteLength - byteOffset) {
+    const view = new Uint8Array(buffer, byteOffset, byteLength);
+    const copy2 = new Uint8Array(view.length);
+    copy2.set(view);
+    return copy2.buffer;
+  }
+
   // ../schema-utils/src/lib/schema/data-type.ts
   function getDataTypeFromValue(value, defaultNumberType = "float32") {
     if (value instanceof Date) {
@@ -14373,8 +14396,8 @@ return true;`);
   }
   function parseCSVInBatches(asyncIterator, options) {
     options = { ...options };
-    if (options.batchSize === "auto") {
-      options.batchSize = 4e3;
+    if (options?.core?.batchSize === "auto") {
+      options.core.batchSize = 4e3;
     }
     const csvOptions = { ...CSVLoader.options.csv, ...options?.csv };
     const asyncQueue = new AsyncQueue2();
@@ -14427,13 +14450,13 @@ return true;`);
         if (csvOptions.optimizeMemoryUsage) {
           row = JSON.parse(JSON.stringify(row));
         }
-        const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
+        const shape = options?.shape || csvOptions.shape || DEFAULT_CSV_SHAPE;
         tableBatchBuilder = tableBatchBuilder || new TableBatchBuilder(
           // @ts-expect-error TODO this is not a proper schema
           schema,
           {
             shape,
-            ...options
+            ...options?.core || {}
           }
         );
         try {
@@ -14460,7 +14483,7 @@ return true;`);
         asyncQueue.close();
       }
     };
-    papaparse_default.parse(asyncIterator, config, AsyncIteratorStreamer);
+    papaparse_default.parse(toArrayBufferIterator(asyncIterator), config, AsyncIteratorStreamer);
     return asyncQueue;
   }
   function isHeaderRow(row) {