@loaders.gl/csv 4.0.0-alpha.1 → 4.0.0-alpha.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/csv",
-  "version": "4.0.0-alpha.1",
+  "version": "4.0.0-alpha.11",
   "description": "Framework-independent loader for CSV and DSV table formats",
   "license": "MIT",
   "publishConfig": {
@@ -16,9 +16,9 @@
     "table",
     "CSV"
   ],
-  "types": "src/index.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "files": [
     "src",
@@ -27,14 +27,14 @@
   ],
   "scripts": {
     "pre-build": "npm run build-bundle",
-    "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/bundle.js"
+    "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js"
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "4.0.0-alpha.1",
-    "@loaders.gl/schema": "4.0.0-alpha.1"
+    "@loaders.gl/loader-utils": "4.0.0-alpha.11",
+    "@loaders.gl/schema": "4.0.0-alpha.11"
   },
   "devDependencies": {
     "d3-dsv": "^1.2.0"
   },
-  "gitHead": "e48f5534fe7188c810f23f965a440c144b3cf6d0"
+  "gitHead": "bc680098cfea790c67b7fb95bab96e8d9288d34f"
 }

package/src/csv-loader.ts

@@ -1,15 +1,20 @@
+// loaders.gl, MIT license
+
 import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import type {Batch} from '@loaders.gl/schema';
-type Schema = any;
+import type {Batch, TableBatch} from '@loaders.gl/schema';
 
 import {
   AsyncQueue,
+  Table,
   TableBatchBuilder,
   convertToArrayRow,
   convertToObjectRow
 } from '@loaders.gl/schema';
-import Papa from './libs/papaparse';
-import AsyncIteratorStreamer from './lib/async-iterator-streamer';
+import Papa from './papaparse/papaparse';
+import AsyncIteratorStreamer from './papaparse/async-iterator-streamer';
+
+type ObjectField = {name: string; index: number; type: any};
+type ObjectSchema = {[key: string]: ObjectField} | ObjectField[];
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -59,7 +64,7 @@ const DEFAULT_CSV_LOADER_OPTIONS = {
   }
 };
 
-export const CSVLoader = {
+export const CSVLoader: LoaderWithParser<Table, TableBatch, CSVLoaderOptions> = {
   id: 'csv',
   module: 'csv',
   name: 'CSV',
@@ -147,7 +152,7 @@ function parseCSVInBatches(
   let isFirstRow: boolean = true;
   let headerRow: string[] | null = null;
   let tableBatchBuilder: TableBatchBuilder | null = null;
-  let schema: Schema | null = null;
+  let schema: ObjectSchema | null = null;
 
   const config = {
     // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
@@ -207,11 +212,14 @@ function parseCSVInBatches(
       // Add the row
       tableBatchBuilder =
         tableBatchBuilder ||
-        new TableBatchBuilder(schema, {
-          // @ts-expect-error
-          shape: csvOptions.shape || 'array-row-table',
-          ...options
-        });
+        new TableBatchBuilder(
+          // @ts-expect-error TODO this is not a proper schema
+          schema,
+          {
+            shape: csvOptions.shape || 'array-row-table',
+            ...options
+          }
+        );
 
       try {
         tableBatchBuilder.addRow(row);
@@ -306,8 +314,8 @@ function generateHeader(columnPrefix: string, count: number = 0): string[] {
   return headers;
 }
 
-function deduceSchema(row, headerRow) {
-  const schema = headerRow ? {} : [];
+function deduceSchema(row, headerRow): ObjectSchema {
+  const schema: ObjectSchema = headerRow ? {} : [];
   for (let i = 0; i < row.length; i++) {
     const columnName = (headerRow && headerRow[i]) || i;
     const value = row[i];
@@ -326,5 +334,3 @@ function deduceSchema(row, headerRow) {
   }
   return schema;
 }
-
-export const _typecheckCSVLoader: LoaderWithParser = CSVLoader;

package/src/csv-writer.ts

@@ -1,6 +1,30 @@
-/* @typedef {import('@loaders.gl/loader-utils').Writer} Writer */
+// loaders.gl, MIT license
 
-/*
- * CSV exporter
- * type {Writer}
- */
+/* global TextEncoder */
+import type {Writer} from '@loaders.gl/loader-utils';
+import type {Table, TableBatch} from '@loaders.gl/schema';
+import type {CSVWriterOptions} from './lib/encoders/encode-csv';
+import {encodeTableAsCSV} from './lib/encoders/encode-csv';
+
+export type {CSVWriterOptions};
+
+const DEFAULT_WRITER_OPTIONS: Required<CSVWriterOptions> = {
+  csv: {
+    useDisplayNames: false
+  },
+  useDisplayNames: false
+};
+
+export const CSVWriter: Writer<Table, TableBatch, CSVWriterOptions> = {
+  id: 'csv',
+  version: 'latest',
+  module: 'csv',
+  name: 'CSV',
+  extensions: ['csv'],
+  mimeTypes: ['text/csv'],
+  options: DEFAULT_WRITER_OPTIONS,
+  text: true,
+  encode: async (table, options) =>
+    new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
+  encodeText: (table, options) => encodeTableAsCSV(table, options)
+};

package/src/index.ts

@@ -1,2 +1,7 @@
+// loaders.gl, MIT license
+
 export type {CSVLoaderOptions} from './csv-loader';
 export {CSVLoader} from './csv-loader';
+
+export type {CSVWriterOptions} from './csv-writer';
+export {CSVWriter} from './csv-writer';

package/src/lib/encoders/encode-csv.ts

@@ -0,0 +1,66 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc.
+
+import {Table, makeArrayRowIterator, getTableNumCols} from '@loaders.gl/schema';
+import {csvFormatRows} from 'd3-dsv';
+
+type EncodableData = string | null;
+
+export type CSVWriterOptions = {
+  csv?: {
+    useDisplayNames?: boolean;
+  };
+  /** @deprecated */
+  useDisplayNames?: boolean;
+};
+
+/**
+ * Encode a Table object as CSV
+ */
+export function encodeTableAsCSV(
+  table: Table,
+  options: CSVWriterOptions = {csv: {useDisplayNames: true}}
+): string {
+  const useDisplayNames = options.useDisplayNames || options.csv?.useDisplayNames;
+
+  const fields = table.schema?.fields || [];
+
+  const columnNames = fields.map((f) => {
+    // This is a leaky abstraction, assuming Kepler metadata
+    const displayName = f.metadata?.displayName;
+    return useDisplayNames && typeof displayName === 'string' ? displayName : f.name;
+  });
+  const formattedData: EncodableData[][] = [columnNames];
+
+  for (const row of makeArrayRowIterator(table)) {
+    const formattedRow: EncodableData[] = [];
+    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
+      const value = row[columnIndex];
+      formattedRow[columnIndex] = preformatFieldValue(value);
+    }
+    formattedData.push(formattedRow);
+  }
+
+  return csvFormatRows(formattedData);
+}
+
+/**
+ * Stringifies a value
+ * @todo Why is it called parse?
+ */
+const preformatFieldValue = (value: unknown): EncodableData => {
+  if (value === null || value === undefined) {
+    // TODO: It would be nice to distinguish between missing values and the empty string
+    // https://github.com/d3/d3-dsv/issues/84
+    return null;
+  }
+  if (value instanceof Date) {
+    // d3-dsv formats dates without timezones if they don't have time info;
+    // this forces them to always use fully-qualified ISO time strings
+    return value.toISOString();
+  }
+  if (typeof value === 'object') {
+    return JSON.stringify(value);
+  }
+  return String(value);
+};

package/src/{lib → papaparse}/async-iterator-streamer.ts

@@ -1,12 +1,12 @@
+// @ts-nocheck
 // A custom papaparse `Streamer` for async iterators
 // Ideally this can be contributed back to papaparse
 // Or papaparse can expose Streamer API so we can extend without forking.
 
-// @ts-nocheck
 /* eslint-disable no-invalid-this */
 
 // Note: papaparse is not an ES6 module
-import Papa from '../libs/papaparse';
+import Papa from './papaparse';
 const {ChunkStreamer} = Papa;
 
 export default function AsyncIteratorStreamer(config) {

package/{dist/libs/papaparse.js → src/papaparse/papaparse.ts}

@@ -1,3 +1,4 @@
+// @ts-nocheck
 // This is a fork of papaparse
 // https://github.com/mholt/PapaParse
 /* @license
@@ -14,52 +15,34 @@ License: MIT
 // - Remove unused jQuery plugin support
 
 /* eslint-disable */
-// @ts-nocheck
-var global = (function() {
-  // alternative method, similar to `Function('return this')()`
-  // but without using `eval` (which is disabled when
-  // using Content Security Policy).
-
-  if (typeof self !== 'undefined') {
-    return self;
-  }
-  if (typeof window !== 'undefined') {
-    return window;
-  }
-  if (typeof global !== 'undefined') {
-    return global;
-  }
+const BYTE_ORDER_MARK = '\ufeff';
 
-  // When running tests none of the above have been defined
-  return {};
-})();
+const Papa = {
+  parse: CsvToJson,
+  unparse: JsonToCsv,
 
-var IS_PAPA_WORKER = false;
+  RECORD_SEP: String.fromCharCode(30),
+  UNIT_SEP: String.fromCharCode(31),
+  BYTE_ORDER_MARK,
+  BAD_DELIMITERS: ['\r', '\n', '"', BYTE_ORDER_MARK],
+  WORKERS_SUPPORTED: false, // !IS_WORKER && !!globalThis.Worker
+  NODE_STREAM_INPUT: 1,
 
-var Papa = {};
-module.exports = Papa;
-Papa.parse = CsvToJson;
-Papa.unparse = JsonToCsv;
+  // Configurable chunk sizes for local and remote files, respectively
+  LocalChunkSize: 1024 * 1024 * 10, // 10 M,
+  RemoteChunkSize: 1024 * 1024 * 5, // 5 M,
+  DefaultDelimiter: ',', // Used if not specified and detection fail,
 
-Papa.RECORD_SEP = String.fromCharCode(30);
-Papa.UNIT_SEP = String.fromCharCode(31);
-Papa.BYTE_ORDER_MARK = '\ufeff';
-Papa.BAD_DELIMITERS = ['\r', '\n', '"', Papa.BYTE_ORDER_MARK];
-Papa.WORKERS_SUPPORTED = false; // !IS_WORKER && !!global.Worker;
-Papa.NODE_STREAM_INPUT = 1;
+  // Exposed for testing and development only
+  Parser: Parser,
+  ParserHandle: ParserHandle,
 
-// Configurable chunk sizes for local and remote files, respectively
-Papa.LocalChunkSize = 1024 * 1024 * 10; // 10 MB
-Papa.RemoteChunkSize = 1024 * 1024 * 5; // 5 MB
-Papa.DefaultDelimiter = ','; // Used if not specified and detection fails
-
-// Exposed for testing and development only
-Papa.Parser = Parser;
-Papa.ParserHandle = ParserHandle;
+  // BEGIN FORK
+  ChunkStreamer: ChunkStreamer,
+  StringStreamer: StringStreamer
+};
+export default Papa;
 
-// BEGIN FORK
-Papa.ChunkStreamer = ChunkStreamer;
-Papa.StringStreamer = StringStreamer;
 /*
 Papa.NetworkStreamer = NetworkStreamer;
 Papa.FileStreamer = FileStreamer;
@@ -76,7 +59,7 @@ if (typeof PAPA_BROWSER_CONTEXT === 'undefined') {
 function CsvToJson(
   _input,
   _config,
-  UserDefinedStreamer // BEGIN FORK
+  UserDefinedStreamer? // BEGIN FORK
 ) {
   _config = _config || {};
   var dynamicTyping = _config.dynamicTyping || false;
@@ -129,7 +112,7 @@ function CsvToJson(
   /*
   else if (_input.readable === true && isFunction(_input.read) && isFunction(_input.on)) {
     streamer = new ReadableStreamStreamer(_config);
-  } else if ((global.File && _input instanceof File) || _input instanceof Object)
+  } else if ((globalThis.File && _input instanceof File) || _input instanceof Object)
     // ...Safari. (see issue #106)
     streamer = new FileStreamer(_config);
   */
@@ -204,7 +187,7 @@ function JsonToCsv(_input, _config) {
 
     if (
       typeof _config.delimiter === 'string' &&
-      !Papa.BAD_DELIMITERS.filter(function(value) {
+      !Papa.BAD_DELIMITERS.filter(function (value) {
         return _config.delimiter.indexOf(value) !== -1;
       }).length
     ) {
@@ -339,7 +322,7 @@ function ChunkStreamer(config) {
   };
   replaceConfig.call(this, config);
 
-  this.parseChunk = function(chunk, isFakeChunk) {
+  this.parseChunk = function (chunk, isFakeChunk) {
     // First chunk pre-processing
     if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) {
       var modifiedChunk = this._config.beforeFirstChunk(chunk);
@@ -367,13 +350,7 @@ function ChunkStreamer(config) {
     var finishedIncludingPreview =
       this._finished || (this._config.preview && this._rowCount >= this._config.preview);
 
-    if (IS_PAPA_WORKER) {
-      global.postMessage({
-        results: results,
-        workerId: Papa.WORKER_ID,
-        finished: finishedIncludingPreview
-      });
-    } else if (isFunction(this._config.chunk) && !isFakeChunk) {
+    if (isFunction(this._config.chunk) && !isFakeChunk) {
       this._config.chunk(results, this._handle);
       if (this._handle.paused() || this._handle.aborted()) return;
       results = undefined;
@@ -401,15 +378,8 @@ function ChunkStreamer(config) {
     return results;
   };
 
-  this._sendError = function(error) {
+  this._sendError = function (error) {
     if (isFunction(this._config.error)) this._config.error(error);
-    else if (IS_PAPA_WORKER && this._config.error) {
-      global.postMessage({
-        workerId: Papa.WORKER_ID,
-        error: error,
-        finished: false
-      });
-    }
   };
 
   function replaceConfig(config) {
@@ -427,11 +397,11 @@ function StringStreamer(config) {
   ChunkStreamer.call(this, config);
 
   var remaining;
-  this.stream = function(s) {
+  this.stream = function (s) {
     remaining = s;
     return this._nextChunk();
   };
-  this._nextChunk = function() {
+  this._nextChunk = function () {
     if (this._finished) return;
     var size = this._config.chunkSize;
     var chunk = size ? remaining.substr(0, size) : remaining;
@@ -447,7 +417,8 @@ StringStreamer.prototype.constructor = StringStreamer;
 function ParserHandle(_config) {
   // One goal is to minimize the use of regular expressions...
   var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
-  var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
+  var ISO_DATE =
+    /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
 
   var self = this;
   var _stepCounter = 0; // Number of times step was called (number of rows parsed)
@@ -467,7 +438,7 @@ function ParserHandle(_config) {
 
   if (isFunction(_config.step)) {
     var userStep = _config.step;
-    _config.step = function(results) {
+    _config.step = function (results) {
       _results = results;
 
       if (needsHeaderRow()) processResults();
@@ -490,7 +461,7 @@ function ParserHandle(_config) {
    * and ignoreLastRow parameters. They are used by streamers (wrapper functions)
    * when an input comes in multiple chunks, like from a file.
    */
-  this.parse = function(input, baseIndex, ignoreLastRow) {
+  this.parse = function (input, baseIndex, ignoreLastRow) {
     var quoteChar = _config.quoteChar || '"';
     if (!_config.newline) _config.newline = guessLineEndings(input, quoteChar);
 
@@ -524,26 +495,26 @@ function ParserHandle(_config) {
     return _paused ? {meta: {paused: true}} : _results || {meta: {paused: false}};
   };
 
-  this.paused = function() {
+  this.paused = function () {
     return _paused;
   };
 
-  this.pause = function() {
+  this.pause = function () {
     _paused = true;
     _parser.abort();
     _input = _input.substr(_parser.getCharIndex());
   };
 
-  this.resume = function() {
+  this.resume = function () {
     _paused = false;
     self.streamer.parseChunk(_input, true);
   };
 
-  this.aborted = function() {
+  this.aborted = function () {
     return _aborted;
   };
 
-  this.abort = function() {
+  this.abort = function () {
     _aborted = true;
     _parser.abort();
     _results.meta.aborted = true;
@@ -620,7 +591,11 @@ function ParserHandle(_config) {
   }
 
   function applyHeaderAndDynamicTypingAndTransformation() {
-    if (!_results || !_results.data || (!_config.header && !_config.dynamicTyping && !_config.transform))
+    if (
+      !_results ||
+      !_results.data ||
+      (!_config.header && !_config.dynamicTyping && !_config.transform)
+    )
       return _results;
 
     function processRow(rowSource, i) {
@@ -802,7 +777,7 @@ function Parser(config) {
   var cursor = 0;
   var aborted = false;
 
-  this.parse = function(input, baseIndex, ignoreLastRow) {
+  this.parse = function (input, baseIndex, ignoreLastRow) {
     // For some reason, in Chrome, this speeds things up (!?)
     if (typeof input !== 'string') throw new Error('Input must be a string');
 
@@ -1075,12 +1050,12 @@ function Parser(config) {
   };
 
   /** Sets the abort flag */
-  this.abort = function() {
+  this.abort = function () {
     aborted = true;
   };
 
   /** Gets the cursor position */
-  this.getCharIndex = function() {
+  this.getCharIndex = function () {
     return cursor;
   };
 }

package/dist/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/csv-loader.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/csv-loader.ts"],"names":["AsyncQueue","TableBatchBuilder","convertToArrayRow","convertToObjectRow","Papa","AsyncIteratorStreamer","VERSION","DEFAULT_CSV_LOADER_OPTIONS","csv","shape","optimizeMemoryUsage","header","columnPrefix","quoteChar","escapeChar","dynamicTyping","comments","skipEmptyLines","delimitersToGuess","CSVLoader","id","module","name","version","extensions","mimeTypes","category","parse","arrayBuffer","options","parseCSV","TextDecoder","decode","parseText","text","parseInBatches","parseCSVInBatches","csvText","csvOptions","firstRow","readFirstRow","isHeaderRow","Boolean","parseWithHeader","papaparseConfig","download","transformHeader","duplicateColumnTransformer","undefined","error","e","Error","result","data","rows","headerRow","meta","fields","generateHeader","length","map","row","Array","isArray","asyncIterator","batchSize","asyncQueue","isFirstRow","tableBatchBuilder","schema","config","chunkSize","step","results","collapsedRow","flat","join","trim","bytesUsed","cursor","deduceSchema","JSON","stringify","addRow","batch","getFullBatch","enqueue","complete","getFinalBatch","close","every","value","preview","observedColumns","Set","col","colName","counter","has","add","count","headers","i","push","columnName","String","index","type","Float32Array","_typecheckCSVLoader"],"mappings":"AAIA,SACEA,UADF,EAEEC,iBAFF,EAGEC,iBAHF,EAIEC,kBAJF,QAKO,oBALP;AAMA,OAAOC,IAAP,MAAiB,kBAAjB;AACA,OAAOC,qBAAP,MAAkC,+BAAlC;AAIA,MAAMC,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AA0BA,MAAMC,0BAA0B,GAAG;AACjCC,EAAAA,GAAG,EAAE;AACHC,IAAAA,KAAK,EAAE,kBADJ;AAEHC,IAAAA,mBAAmB,EAAE,KAFlB;AAIHC,IAAAA,MAAM,EAAE,MAJL;AAKHC,IAAAA,YAAY,EAAE,QALX;AAQHC,IAAAA,SAAS,EAAE,GARR;AASHC,IAAAA,UAAU,EAAE,GATT;AAUHC,IAAAA,aAAa,EAAE,IAVZ;AAWHC,IAAAA,QAAQ,EAAE,KAXP;AAYHC,IAAAA,cAAc,EAAE,IAZb;AAcHC,IAAAA,iBAAiB,EAAE,CAAC,GAAD,EAAM,IAAN,EAAY,GAAZ,EAAiB,GAAjB;AAdhB;AAD4B,CAAnC;AAoBA,OAAO,MAAMC,SAAS,GAAG;AACvBC,EAAAA,EAAE,EAAE,KADmB;AAEvBC,EAAAA,MAAM,EAAE,KAFe;AAGvBC,EAAAA,IAAI,EAAE,KAHiB;AAIvBC,EAAAA,OAAO,EAAEjB,OAJc;AAKvBkB,EAAAA,UAAU,EAAE,CAAC,KAAD,EAAQ,KAAR,EAAe,KAAf,CALW;AAMvBC,EAAAA,SAAS,EAAE,CAAC,UAAD,EAAa,2BAAb,EAA0C,UAA1C,CANY;AAOvBC,EAAAA,QAAQ,EAAE,OAPa;AAQvBC,EAAAA,KAAK,EAAE,OAAOC,WAAP,EAAiCC,OAAjC,KACLC,QAAQ,CAAC,IAAIC,WAAJ,GAAkBC,MAAlB,CAAyBJ,WAAzB,CAAD,EAAwCC,OAAxC,CATa;AAUvBI,EAAAA,SAAS,EAAE,CAACC,IAAD,EAAeL,OAAf,KAA8CC,QAAQ,CAACI,IAAD,EAAOL,OAAP,CAV1C;AAWvBM,EAAAA,cAAc,EAAEC,iBAXO;AAcvBP,EAAAA,OAAO,EAAEtB;AAdc,CAAlB;;AAiBP,eAAeuB,QAAf,CAAwBO,OAAxB,EAAyCR,OAAzC,EAAqE;AAEnE,QAAMS,UAAU,GAAG,EAAC,GAAG/B,0BAA0B,CAACC,GAA/B;AAAoC,QAAGqB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAErB,GAAZ;AAApC,GAAnB;AAEA,QAAM+B,QAAQ,GAAGC,YAAY,CAACH,OAAD,CAA7B;AACA,QAAM1B,MAAe,GACnB2B,UAAU,CAAC3B,MAAX,KAAsB,MAAtB,GAA+B8B,WAAW,CAACF,QAAD,CAA1C,GAAuDG,OAAO,CAACJ,UAAU,CAAC3B,MAAZ,CADhE;AAGA,QAAMgC,eAAe,GAAGhC,MAAxB;AAEA,QAAMiC,eAAe,GAAG,EAEtB,GAAGN,UAFmB;AAGtB3B,IAAAA,MAAM,EAAEgC,eAHc;AAItBE,IAAAA,QAAQ,EAAE,KAJY;AAKtBC,IAAAA,eAAe,EAAEH,eAAe,GAAGI,0BAA0B,EAA7B,GAAkCC,SAL5C;AAMtBC,IAAAA,KAAK,EAAGC,CAAD,IAAO;AACZ,YAAM,IAAIC,KAAJ,CAAUD,CAAV,CAAN;AACD;AARqB,GAAxB;AAWA,QAAME,MAAM,GAAGhD,IAAI,CAACuB,KAAL,CAAWU,OAAX,EAAoBO,eAApB,CAAf;AACA,MAAI;AAACS,IAAAA,IAAI,EAAEC;AAAP,MAAeF,MAAnB;AAEA,QAAMG,SAAS,GAAGH,MAAM,CAACI,IAAP,CAAYC,MAAZ,IAAsBC,cAAc,CAACpB,UAAU,CAAC1B,YAAZ,EAA0B2B,QAAQ,CAACoB,MAAnC,CAAtD;;AAEA,UAAQrB,UAAU,CAAC7B,KAAnB;AACE,SAAK,kBAAL;AACE6C,MAAAA,IAAI,GAAGA,IAAI,CAACM,GAAL,CAAUC,GAAD,IAAUC,KAAK,CAACC,OAAN,CAAcF,GAAd,IAAqB1D,kBAAkB,CAAC0D,GAAD,EAAMN,SAAN,CAAvC,GAA0DM,GAA7E,CAAP;AACA;;AACF,SAAK,iBAAL;AACEP,MAAAA,IAAI,GAAGA,IAAI,CAACM,GAAL,CAAUC,GAAD,IAAUC,KAAK,CAACC,OAAN,CAAcF,GAAd,IAAqBA,GAArB,GAA2B3D,iBAAiB,CAAC2D,GAAD,EAAMN,SAAN,CAA/D,CAAP;AACA;;AACF;AAPF;;AAsBA,SAAOD,IAAP;AACD;;AAGD,SAASlB,iBAAT,CACE4B,aADF,EAEEnC,OAFF,EAGwB;AAAA;;AAGtBA,EAAAA,OAAO,GAAG,EAAC,GAAGA;AAAJ,GAAV;;AACA,MAAIA,OAAO,CAACoC,SAAR,KAAsB,MAA1B,EAAkC;AAChCpC,IAAAA,OAAO,CAACoC,SAAR,GAAoB,IAApB;AACD;;AAGD,QAAM3B,UAAU,GAAG,EAAC,GAAG/B,0BAA0B,CAACC,GAA/B;AAAoC,oBAAGqB,OAAH,6CAAG,SAASrB,GAAZ;AAApC,GAAnB;AAEA,QAAM0D,UAAU,GAAG,IAAIlE,UAAJ,EAAnB;AAEA,MAAImE,UAAmB,GAAG,IAA1B;AACA,MAAIZ,SAA0B,GAAG,IAAjC;AACA,MAAIa,iBAA2C,GAAG,IAAlD;AACA,MAAIC,MAAqB,GAAG,IAA5B;AAEA,QAAMC,MAAM,GAAG,EAEb,GAAGhC,UAFU;AAGb3B,IAAAA,MAAM,EAAE,KAHK;AAIbkC,IAAAA,QAAQ,EAAE,KAJG;AAQb0B,IAAAA,SAAS,EAAE,OAAO,IAAP,GAAc,CARZ;AAcbtD,IAAAA,cAAc,EAAE,KAdH;;AAkBbuD,IAAAA,IAAI,CAACC,OAAD,EAAU;AACZ,UAAIZ,GAAG,GAAGY,OAAO,CAACpB,IAAlB;;AAEA,UAAIf,UAAU,CAACrB,cAAf,EAA+B;AAE7B,cAAMyD,YAAY,GAAGb,GAAG,CAACc,IAAJ,GAAWC,IAAX,CAAgB,EAAhB,EAAoBC,IAApB,EAArB;;AACA,YAAIH,YAAY,KAAK,EAArB,EAAyB;AACvB;AACD;AACF;;AACD,YAAMI,SAAS,GAAGL,OAAO,CAACjB,IAAR,CAAauB,MAA/B;;AAGA,UAAIZ,UAAU,IAAI,CAACZ,SAAnB,EAA8B;AAE5B,cAAM5C,MAAM,GAAG2B,UAAU,CAAC3B,MAAX,KAAsB,MAAtB,GAA+B8B,WAAW,CAACoB,GAAD,CAA1C,GAAkDnB,OAAO,CAACJ,UAAU,CAAC3B,MAAZ,CAAxE;;AACA,YAAIA,MAAJ,EAAY;AACV4C,UAAAA,SAAS,GAAGM,GAAG,CAACD,GAAJ,CAAQb,0BAA0B,EAAlC,CAAZ;AACA;AACD;AACF;;AAGD,UAAIoB,UAAJ,EAAgB;AACdA,QAAAA,UAAU,GAAG,KAAb;;AACA,YAAI,CAACZ,SAAL,EAAgB;AACdA,UAAAA,SAAS,GAAGG,cAAc,CAACpB,UAAU,CAAC1B,YAAZ,EAA0BiD,GAAG,CAACF,MAA9B,CAA1B;AACD;;AACDU,QAAAA,MAAM,GAAGW,YAAY,CAACnB,GAAD,EAAMN,SAAN,CAArB;AACD;;AAED,UAAIjB,UAAU,CAAC5B,mBAAf,EAAoC;AAGlCmD,QAAAA,GAAG,GAAGoB,IAAI,CAACtD,KAAL,CAAWsD,IAAI,CAACC,SAAL,CAAerB,GAAf,CAAX,CAAN;AACD;;AAGDO,MAAAA,iBAAiB,GACfA,iBAAiB,IACjB,IAAInE,iBAAJ,CAAsBoE,MAAtB,EAA8B;AAE5B5D,QAAAA,KAAK,EAAE6B,UAAU,CAAC7B,KAAX,IAAoB,iBAFC;AAG5B,WAAGoB;AAHyB,OAA9B,CAFF;;AAQA,UAAI;AACFuC,QAAAA,iBAAiB,CAACe,MAAlB,CAAyBtB,GAAzB;AAEA,cAAMuB,KAAK,GAAGhB,iBAAiB,IAAIA,iBAAiB,CAACiB,YAAlB,CAA+B;AAACP,UAAAA;AAAD,SAA/B,CAAnC;;AACA,YAAIM,KAAJ,EAAW;AACTlB,UAAAA,UAAU,CAACoB,OAAX,CAAmBF,KAAnB;AACD;AACF,OAPD,CAOE,OAAOnC,KAAP,EAAc;AACdiB,QAAAA,UAAU,CAACoB,OAAX,CAAmBrC,KAAnB;AACD;AACF,KA1EY;;AA6EbsC,IAAAA,QAAQ,CAACd,OAAD,EAAU;AAChB,UAAI;AACF,cAAMK,SAAS,GAAGL,OAAO,CAACjB,IAAR,CAAauB,MAA/B;AAEA,cAAMK,KAAK,GAAGhB,iBAAiB,IAAIA,iBAAiB,CAACoB,aAAlB,CAAgC;AAACV,UAAAA;AAAD,SAAhC,CAAnC;;AACA,YAAIM,KAAJ,EAAW;AACTlB,UAAAA,UAAU,CAACoB,OAAX,CAAmBF,KAAnB;AACD;AACF,OAPD,CAOE,OAAOnC,KAAP,EAAc;AACdiB,QAAAA,UAAU,CAACoB,OAAX,CAAmBrC,KAAnB;AACD;;AAEDiB,MAAAA,UAAU,CAACuB,KAAX;AACD;;AA1FY,GAAf;AA6FArF,EAAAA,IAAI,CAACuB,KAAL,CAAWqC,aAAX,EAA0BM,MAA1B,EAAkCjE,qBAAlC;AAIA,SAAO6D,UAAP;AACD;;AAOD,SAASzB,WAAT,CAAqBoB,GAArB,EAA6C;AAC3C,SAAOA,GAAG,IAAIA,GAAG,CAAC6B,KAAJ,CAAWC,KAAD,IAAW,OAAOA,KAAP,KAAiB,QAAtC,CAAd;AACD;;AAOD,SAASnD,YAAT,CAAsBH,OAAtB,EAA8C;AAC5C,QAAMe,MAAM,GAAGhD,IAAI,CAACuB,KAAL,CAAWU,OAAX,EAAoB;AACjCQ,IAAAA,QAAQ,EAAE,KADuB;AAEjC9B,IAAAA,aAAa,EAAE,IAFkB;AAGjC6E,IAAAA,OAAO,EAAE;AAHwB,GAApB,CAAf;AAKA,SAAOxC,MAAM,CAACC,IAAP,CAAY,CAAZ,CAAP;AACD;;AAQD,SAASN,0BAAT,GAAsC;AACpC,QAAM8C,eAAe,GAAG,IAAIC,GAAJ,EAAxB;AACA,SAAQC,GAAD,IAAS;AACd,QAAIC,OAAO,GAAGD,GAAd;AACA,QAAIE,OAAO,GAAG,CAAd;;AACA,WAAOJ,eAAe,CAACK,GAAhB,CAAoBF,OAApB,CAAP,EAAqC;AACnCA,MAAAA,OAAO,aAAMD,GAAN,cAAaE,OAAb,CAAP;AACAA,MAAAA,OAAO;AACR;;AACDJ,IAAAA,eAAe,CAACM,GAAhB,CAAoBH,OAApB;AACA,WAAOA,OAAP;AACD,GATD;AAUD;;AAQD,SAAStC,cAAT,CAAwB9C,YAAxB,EAA8CwF,KAAa,GAAG,CAA9D,EAA2E;AACzE,QAAMC,OAAiB,GAAG,EAA1B;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGF,KAApB,EAA2BE,CAAC,EAA5B,EAAgC;AAC9BD,IAAAA,OAAO,CAACE,IAAR,WAAgB3F,YAAhB,SAA+B0F,CAAC,GAAG,CAAnC;AACD;;AACD,SAAOD,OAAP;AACD;;AAED,SAASrB,YAAT,CAAsBnB,GAAtB,EAA2BN,SAA3B,EAAsC;AACpC,QAAMc,MAAM,GAAGd,SAAS,GAAG,EAAH,GAAQ,EAAhC;;AACA,OAAK,IAAI+C,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGzC,GAAG,CAACF,MAAxB,EAAgC2C,CAAC,EAAjC,EAAqC;AACnC,UAAME,UAAU,GAAIjD,SAAS,IAAIA,SAAS,CAAC+C,CAAD,CAAvB,IAA+BA,CAAlD;AACA,UAAMX,KAAK,GAAG9B,GAAG,CAACyC,CAAD,CAAjB;;AACA,YAAQ,OAAOX,KAAf;AACE,WAAK,QAAL;AACA,WAAK,SAAL;AAEEtB,QAAAA,MAAM,CAACmC,UAAD,CAAN,GAAqB;AAAClF,UAAAA,IAAI,EAAEmF,MAAM,CAACD,UAAD,CAAb;AAA2BE,UAAAA,KAAK,EAAEJ,CAAlC;AAAqCK,UAAAA,IAAI,EAAEC;AAA3C,SAArB;AACA;;AACF,WAAK,QAAL;AACA;AACEvC,QAAAA,MAAM,CAACmC,UAAD,CAAN,GAAqB;AAAClF,UAAAA,IAAI,EAAEmF,MAAM,CAACD,UAAD,CAAb;AAA2BE,UAAAA,KAAK,EAAEJ,CAAlC;AAAqCK,UAAAA,IAAI,EAAE7C;AAA3C,SAArB;AARJ;AAYD;;AACD,SAAOO,MAAP;AACD;;AAED,OAAO,MAAMwC,mBAAqC,GAAG1F,SAA9C","sourcesContent":["import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {Batch} from '@loaders.gl/schema';\ntype Schema = any;\n\nimport {\n  AsyncQueue,\n  TableBatchBuilder,\n  convertToArrayRow,\n  convertToObjectRow\n} from '@loaders.gl/schema';\nimport Papa from './libs/papaparse';\nimport AsyncIteratorStreamer from './lib/async-iterator-streamer';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type CSVLoaderOptions = LoaderOptions & {\n  csv?: {\n    // loaders.gl options\n    shape?: 'array-row-table' | 'object-row-table' | 'columnar-table';\n    /** optimizes memory usage but increases parsing time. */\n    optimizeMemoryUsage?: boolean;\n    columnPrefix?: string;\n    header?: 'auto';\n\n    // CSV options (papaparse)\n    // delimiter: auto\n    // newline: auto\n    quoteChar?: string;\n    escapeChar?: string;\n    // Convert numbers and boolean values in rows from strings\n    dynamicTyping?: boolean;\n    comments?: boolean;\n    skipEmptyLines?: boolean | 'greedy';\n    // transform: null?\n    delimitersToGuess?: string[];\n    // fastMode: auto\n  };\n};\n\nconst DEFAULT_CSV_LOADER_OPTIONS = {\n  csv: {\n    shape: 'object-row-table',\n    optimizeMemoryUsage: false,\n    // CSV options\n    header: 'auto',\n    columnPrefix: 'column',\n    // delimiter: auto\n    // newline: auto\n    quoteChar: '\"',\n    escapeChar: '\"',\n    dynamicTyping: true,\n    comments: false,\n    skipEmptyLines: true,\n    // transform: null?\n    delimitersToGuess: [',', '\\t', '|', ';']\n    // fastMode: auto\n  }\n};\n\nexport const CSVLoader = {\n  id: 'csv',\n  module: 'csv',\n  name: 'CSV',\n  version: VERSION,\n  extensions: ['csv', 'tsv', 'dsv'],\n  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],\n  category: 'table',\n  parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) =>\n    parseCSV(new TextDecoder().decode(arrayBuffer), options),\n  parseText: (text: string, options?: CSVLoaderOptions) => parseCSV(text, options),\n  parseInBatches: parseCSVInBatches,\n  // @ts-ignore\n  // testText: null,\n  options: DEFAULT_CSV_LOADER_OPTIONS as CSVLoaderOptions\n};\n\nasync function parseCSV(csvText: string, options?: CSVLoaderOptions) {\n  // Apps can call the parse method directly, we so apply default options here\n  const csvOptions = {...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv};\n\n  const firstRow = readFirstRow(csvText);\n  const header: boolean =\n    csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);\n\n  const parseWithHeader = header;\n\n  const papaparseConfig = {\n    // dynamicTyping: true,\n    ...csvOptions,\n    header: parseWithHeader,\n    download: false, // We handle loading, no need for papaparse to do it for us\n    transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,\n    error: (e) => {\n      throw new Error(e);\n    }\n  };\n\n  const result = Papa.parse(csvText, papaparseConfig);\n  let {data: rows} = result;\n\n  const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);\n\n  switch (csvOptions.shape) {\n    case 'object-row-table':\n      rows = rows.map((row) => (Array.isArray(row) ? convertToObjectRow(row, headerRow) : row));\n      break;\n    case 'array-row-table':\n      rows = rows.map((row) => (Array.isArray(row) ? row : convertToArrayRow(row, headerRow)));\n      break;\n    default:\n  }\n\n  /*\n  if (!header && shape === 'object-row-table') {\n    // If the dataset has no header, transform the array result into an object shape with an\n    // autogenerated header\n    return result.data.map((row) =>\n      row.reduce((acc, value, i) => {\n        acc[headerRow[i]] = value;\n        return acc;\n      }, {})\n    );\n  }\n  */\n  return rows;\n}\n\n// TODO - support batch size 0 = no batching/single batch?\nfunction parseCSVInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: CSVLoaderOptions\n): AsyncIterable<Batch> {\n  // Papaparse does not support standard batch size handling\n  // TODO - investigate papaparse chunks mode\n  options = {...options};\n  if (options.batchSize === 'auto') {\n    options.batchSize = 4000;\n  }\n\n  // Apps can call the parse method directly, we so apply default options here\n  const csvOptions = {...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv};\n\n  const asyncQueue = new AsyncQueue<Batch>();\n\n  let isFirstRow: boolean = true;\n  let headerRow: string[] | null = null;\n  let tableBatchBuilder: TableBatchBuilder | null = null;\n  let schema: Schema | null = null;\n\n  const config = {\n    // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,\n    ...csvOptions,\n    header: false, // Unfortunately, header detection is not automatic and does not infer shapes\n    download: false, // We handle loading, no need for papaparse to do it for us\n    // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the\n    // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.\n    // See https://github.com/mholt/PapaParse/issues/465\n    chunkSize: 1024 * 1024 * 5,\n    // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true\n    // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if\n    // both of the skipEmptyLines and step callback options are provided:\n    // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465\n    // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825\n    skipEmptyLines: false,\n\n    // step is called on every row\n    // eslint-disable-next-line complexity\n    step(results) {\n      let row = results.data;\n\n      if (csvOptions.skipEmptyLines) {\n        // Manually reject lines that are empty\n        const collapsedRow = row.flat().join('').trim();\n        if (collapsedRow === '') {\n          return;\n        }\n      }\n      const bytesUsed = results.meta.cursor;\n\n      // Check if we need to save a header row\n      if (isFirstRow && !headerRow) {\n        // Auto detects or can be forced with csvOptions.header\n        const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);\n        if (header) {\n          headerRow = row.map(duplicateColumnTransformer());\n          return;\n        }\n      }\n\n      // If first data row, we can deduce the schema\n      if (isFirstRow) {\n        isFirstRow = false;\n        if (!headerRow) {\n          headerRow = generateHeader(csvOptions.columnPrefix, row.length);\n        }\n        schema = deduceSchema(row, headerRow);\n      }\n\n      if (csvOptions.optimizeMemoryUsage) {\n        // A workaround to allocate new strings and don't retain pointers to original strings.\n        // https://bugs.chromium.org/p/v8/issues/detail?id=2869\n        row = JSON.parse(JSON.stringify(row));\n      }\n\n      // Add the row\n      tableBatchBuilder =\n        tableBatchBuilder ||\n        new TableBatchBuilder(schema, {\n          // @ts-expect-error\n          shape: csvOptions.shape || 'array-row-table',\n          ...options\n        });\n\n      try {\n        tableBatchBuilder.addRow(row);\n        // If a batch has been completed, emit it\n        const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({bytesUsed});\n        if (batch) {\n          asyncQueue.enqueue(batch);\n        }\n      } catch (error) {\n        asyncQueue.enqueue(error as Error);\n      }\n    },\n\n    // complete is called when all rows have been read\n    complete(results) {\n      try {\n        const bytesUsed = results.meta.cursor;\n        // Ensure any final (partial) batch gets emitted\n        const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({bytesUsed});\n        if (batch) {\n          asyncQueue.enqueue(batch);\n        }\n      } catch (error) {\n        asyncQueue.enqueue(error as Error);\n      }\n\n      asyncQueue.close();\n    }\n  };\n\n  Papa.parse(asyncIterator, config, AsyncIteratorStreamer);\n\n  // TODO - Does it matter if we return asyncIterable or asyncIterator\n  // return asyncQueue[Symbol.asyncIterator]();\n  return asyncQueue;\n}\n\n/**\n * Checks if a certain row is a header row\n * @param row the row to check\n * @returns true if the row looks like a header\n */\nfunction isHeaderRow(row: string[]): boolean {\n  return row && row.every((value) => typeof value === 'string');\n}\n\n/**\n * Reads, parses, and returns the first row of a CSV text\n * @param csvText the csv text to parse\n * @returns the first row\n */\nfunction readFirstRow(csvText: string): any[] {\n  const result = Papa.parse(csvText, {\n    download: false,\n    dynamicTyping: true,\n    preview: 1\n  });\n  return result.data[0];\n}\n\n/**\n * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle\n * duplicate header columns and would use the latest occurrence by default.\n * See the header option in https://www.papaparse.com/docs#config\n * @returns a transform function that returns sanitized names for duplicate fields\n */\nfunction duplicateColumnTransformer() {\n  const observedColumns = new Set();\n  return (col) => {\n    let colName = col;\n    let counter = 1;\n    while (observedColumns.has(colName)) {\n      colName = `${col}.${counter}`;\n      counter++;\n    }\n    observedColumns.add(colName);\n    return colName;\n  };\n}\n\n/**\n * Generates the header of a CSV given a prefix and a column count\n * @param columnPrefix the columnPrefix to use\n * @param count the count of column names to generate\n * @returns an array of column names\n */\nfunction generateHeader(columnPrefix: string, count: number = 0): string[] {\n  const headers: string[] = [];\n  for (let i = 0; i < count; i++) {\n    headers.push(`${columnPrefix}${i + 1}`);\n  }\n  return headers;\n}\n\nfunction deduceSchema(row, headerRow) {\n  const schema = headerRow ? {} : [];\n  for (let i = 0; i < row.length; i++) {\n    const columnName = (headerRow && headerRow[i]) || i;\n    const value = row[i];\n    switch (typeof value) {\n      case 'number':\n      case 'boolean':\n        // TODO - booleans could be handled differently...\n        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};\n        break;\n      case 'string':\n      default:\n        schema[columnName] = {name: String(columnName), index: i, type: Array};\n      // We currently only handle numeric rows\n      // TODO we could offer a function to map strings to numbers?\n    }\n  }\n  return schema;\n}\n\nexport const _typecheckCSVLoader: LoaderWithParser = CSVLoader;\n"],"file":"csv-loader.js"}

package/dist/csv-writer.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":[],"names":[],"mappings":"","sourcesContent":[],"file":"csv-writer.js"}

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/index.ts"],"names":["CSVLoader"],"mappings":"AACA,SAAQA,SAAR,QAAwB,cAAxB","sourcesContent":["export type {CSVLoaderOptions} from './csv-loader';\nexport {CSVLoader} from './csv-loader';\n"],"file":"index.js"}

package/dist/lib/async-iterator-streamer.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/lib/async-iterator-streamer.ts"],"names":["Papa","ChunkStreamer","AsyncIteratorStreamer","config","call","textDecoder","TextDecoder","_config","encoding","stream","asyncIterator","_input","chunk","parseChunk","getStringChunk","_finished","error","_sendError","_nextChunk","nextChunk","decode","prototype","Object","create","constructor"],"mappings":"AAQA,OAAOA,IAAP,MAAiB,mBAAjB;AACA,MAAM;AAACC,EAAAA;AAAD,IAAkBD,IAAxB;AAEA,eAAe,SAASE,qBAAT,CAA+BC,MAA/B,EAAuC;AACpDA,EAAAA,MAAM,GAAGA,MAAM,IAAI,EAAnB;AAEAF,EAAAA,aAAa,CAACG,IAAd,CAAmB,IAAnB,EAAyBD,MAAzB;AAEA,OAAKE,WAAL,GAAmB,IAAIC,WAAJ,CAAgB,KAAKC,OAAL,CAAaC,QAA7B,CAAnB;;AAaA,OAAKC,MAAL,GAAc,gBAAgBC,aAAhB,EAA+B;AAC3C,SAAKC,MAAL,GAAcD,aAAd;;AAEA,QAAI;AAGF,iBAAW,MAAME,KAAjB,IAA0BF,aAA1B,EAAyC;AACvC,aAAKG,UAAL,CAAgB,KAAKC,cAAL,CAAoBF,KAApB,CAAhB;AACD;;AAcD,WAAKG,SAAL,GAAiB,IAAjB;AACA,WAAKF,UAAL,CAAgB,EAAhB;AACD,KArBD,CAqBE,OAAOG,KAAP,EAAc;AAEd,WAAKC,UAAL,CAAgBD,KAAhB;AACD;AACF,GA5BD;;AA8BA,OAAKE,UAAL,GAAkB,SAASC,SAAT,GAAqB,CAEtC,CAFD;;AAKA,OAAKL,cAAL,GAAsB,UAAUF,KAAV,EAAiB;AACrC,WAAO,OAAOA,KAAP,KAAiB,QAAjB,GAA4BA,KAA5B,GAAoC,KAAKP,WAAL,CAAiBe,MAAjB,CAAwBR,KAAxB,EAA+B;AAACH,MAAAA,MAAM,EAAE;AAAT,KAA/B,CAA3C;AACD,GAFD;AAGD;AAEDP,qBAAqB,CAACmB,SAAtB,GAAkCC,MAAM,CAACC,MAAP,CAActB,aAAa,CAACoB,SAA5B,CAAlC;AACAnB,qBAAqB,CAACmB,SAAtB,CAAgCG,WAAhC,GAA8CtB,qBAA9C","sourcesContent":["// A custom papaparse `Streamer` for async iterators\n// Ideally this can be contributed back to papaparse\n// Or papaparse can expose Streamer API so we can extend without forking.\n\n// @ts-nocheck\n/* eslint-disable no-invalid-this */\n\n// Note: papaparse is not an ES6 module\nimport Papa from '../libs/papaparse';\nconst {ChunkStreamer} = Papa;\n\nexport default function AsyncIteratorStreamer(config) {\n  config = config || {};\n\n  ChunkStreamer.call(this, config);\n\n  this.textDecoder = new TextDecoder(this._config.encoding);\n\n  // Implement ChunkStreamer base class methods\n\n  // this.pause = function() {\n  //   ChunkStreamer.prototype.pause.apply(this, arguments);\n  // };\n\n  // this.resume = function() {\n  //   ChunkStreamer.prototype.resume.apply(this, arguments);\n  //   this._input.resume();\n  // };\n\n  this.stream = async function (asyncIterator) {\n    this._input = asyncIterator;\n\n    try {\n      // ES2018 version\n      // TODO - check for pause and abort flags?\n      for await (const chunk of asyncIterator) {\n        this.parseChunk(this.getStringChunk(chunk));\n      }\n\n      // ES5 VERSION\n      // while (true) {\n      //   asyncIterator.next().then(function(value) {\n      //     if (value.done) {\n      //       // finalize iterator?\n      //     }\n      //   }\n      //   const  = await ;\n      //   if (done) return total;\n      //   total += value.length;\n      // }\n\n      this._finished = true;\n      this.parseChunk('');\n    } catch (error) {\n      // Inform ChunkStreamer base class of error\n      this._sendError(error);\n    }\n  };\n\n  this._nextChunk = function nextChunk() {\n    // Left empty, as async iterator automatically pulls next chunk\n  };\n\n  // HELPER METHODS\n  this.getStringChunk = function (chunk) {\n    return typeof chunk === 'string' ? chunk : this.textDecoder.decode(chunk, {stream: true});\n  };\n}\n\nAsyncIteratorStreamer.prototype = Object.create(ChunkStreamer.prototype);\nAsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;\n"],"file":"async-iterator-streamer.js"}

package/dist/papaparse/async-iterator-streamer.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/papaparse/async-iterator-streamer.js"],"names":["Papa","ChunkStreamer","AsyncIteratorStreamer","config","call","textDecoder","TextDecoder","_config","encoding","stream","asyncIterator","_input","chunk","parseChunk","getStringChunk","_finished","error","_sendError","_nextChunk","nextChunk","decode","prototype","Object","create","constructor"],"mappings":"AAQA,OAAOA,IAAP,MAAiB,mBAAjB;AACA,MAAM;AAACC,EAAAA;AAAD,IAAkBD,IAAxB;AAEA,eAAe,SAASE,qBAAT,CAA+BC,MAA/B,EAAuC;AACpDA,EAAAA,MAAM,GAAGA,MAAM,IAAI,EAAnB;AAEAF,EAAAA,aAAa,CAACG,IAAd,CAAmB,IAAnB,EAAyBD,MAAzB;AAEA,OAAKE,WAAL,GAAmB,IAAIC,WAAJ,CAAgB,KAAKC,OAAL,CAAaC,QAA7B,CAAnB;;AAaA,OAAKC,MAAL,GAAc,gBAAgBC,aAAhB,EAA+B;AAC3C,SAAKC,MAAL,GAAcD,aAAd;;AAEA,QAAI;AAGF,iBAAW,MAAME,KAAjB,IAA0BF,aAA1B,EAAyC;AACvC,aAAKG,UAAL,CAAgB,KAAKC,cAAL,CAAoBF,KAApB,CAAhB;AACD;;AAcD,WAAKG,SAAL,GAAiB,IAAjB;AACA,WAAKF,UAAL,CAAgB,EAAhB;AACD,KArBD,CAqBE,OAAOG,KAAP,EAAc;AAEd,WAAKC,UAAL,CAAgBD,KAAhB;AACD;AACF,GA5BD;;AA8BA,OAAKE,UAAL,GAAkB,SAASC,SAAT,GAAqB,CAEtC,CAFD;;AAKA,OAAKL,cAAL,GAAsB,UAAUF,KAAV,EAAiB;AACrC,WAAO,OAAOA,KAAP,KAAiB,QAAjB,GAA4BA,KAA5B,GAAoC,KAAKP,WAAL,CAAiBe,MAAjB,CAAwBR,KAAxB,EAA+B;AAACH,MAAAA,MAAM,EAAE;AAAT,KAA/B,CAA3C;AACD,GAFD;AAGD;AAEDP,qBAAqB,CAACmB,SAAtB,GAAkCC,MAAM,CAACC,MAAP,CAActB,aAAa,CAACoB,SAA5B,CAAlC;AACAnB,qBAAqB,CAACmB,SAAtB,CAAgCG,WAAhC,GAA8CtB,qBAA9C","sourcesContent":["// A custom papaparse `Streamer` for async iterators\n// Ideally this can be contributed back to papaparse\n// Or papaparse can expose Streamer API so we can extend without forking.\n\n// @ts-nocheck\n/* eslint-disable no-invalid-this */\n\n// Note: papaparse is not an ES6 module\nimport Papa from '../libs/papaparse';\nconst {ChunkStreamer} = Papa;\n\nexport default function AsyncIteratorStreamer(config) {\n  config = config || {};\n\n  ChunkStreamer.call(this, config);\n\n  this.textDecoder = new TextDecoder(this._config.encoding);\n\n  // Implement ChunkStreamer base class methods\n\n  // this.pause = function() {\n  //   ChunkStreamer.prototype.pause.apply(this, arguments);\n  // };\n\n  // this.resume = function() {\n  //   ChunkStreamer.prototype.resume.apply(this, arguments);\n  //   this._input.resume();\n  // };\n\n  this.stream = async function (asyncIterator) {\n    this._input = asyncIterator;\n\n    try {\n      // ES2018 version\n      // TODO - check for pause and abort flags?\n      for await (const chunk of asyncIterator) {\n        this.parseChunk(this.getStringChunk(chunk));\n      }\n\n      // ES5 VERSION\n      // while (true) {\n      //   asyncIterator.next().then(function(value) {\n      //     if (value.done) {\n      //       // finalize iterator?\n      //     }\n      //   }\n      //   const  = await ;\n      //   if (done) return total;\n      //   total += value.length;\n      // }\n\n      this._finished = true;\n      this.parseChunk('');\n    } catch (error) {\n      // Inform ChunkStreamer base class of error\n      this._sendError(error);\n    }\n  };\n\n  this._nextChunk = function nextChunk() {\n    // Left empty, as async iterator automatically pulls next chunk\n  };\n\n  // HELPER METHODS\n  this.getStringChunk = function (chunk) {\n    return typeof chunk === 'string' ? chunk : this.textDecoder.decode(chunk, {stream: true});\n  };\n}\n\nAsyncIteratorStreamer.prototype = Object.create(ChunkStreamer.prototype);\nAsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;\n"],"file":"async-iterator-streamer.js"}

package/src/papaparse/async-iterator-streamer.js

@@ -1,71 +0,0 @@
-// A custom papaparse `Streamer` for async iterators
-// Ideally this can be contributed back to papaparse
-// Or papaparse can expose Streamer API so we can extend without forking.
-
-// @ts-nocheck
-/* eslint-disable no-invalid-this */
-
-// Note: papaparse is not an ES6 module
-import Papa from '../libs/papaparse';
-const {ChunkStreamer} = Papa;
-
-export default function AsyncIteratorStreamer(config) {
-  config = config || {};
-
-  ChunkStreamer.call(this, config);
-
-  this.textDecoder = new TextDecoder(this._config.encoding);
-
-  // Implement ChunkStreamer base class methods
-
-  // this.pause = function() {
-  //   ChunkStreamer.prototype.pause.apply(this, arguments);
-  // };
-
-  // this.resume = function() {
-  //   ChunkStreamer.prototype.resume.apply(this, arguments);
-  //   this._input.resume();
-  // };
-
-  this.stream = async function (asyncIterator) {
-    this._input = asyncIterator;
-
-    try {
-      // ES2018 version
-      // TODO - check for pause and abort flags?
-      for await (const chunk of asyncIterator) {
-        this.parseChunk(this.getStringChunk(chunk));
-      }
-
-      // ES5 VERSION
-      // while (true) {
-      //   asyncIterator.next().then(function(value) {
-      //     if (value.done) {
-      //       // finalize iterator?
-      //     }
-      //   }
-      //   const  = await ;
-      //   if (done) return total;
-      //   total += value.length;
-      // }
-
-      this._finished = true;
-      this.parseChunk('');
-    } catch (error) {
-      // Inform ChunkStreamer base class of error
-      this._sendError(error);
-    }
-  };
-
-  this._nextChunk = function nextChunk() {
-    // Left empty, as async iterator automatically pulls next chunk
-  };
-
-  // HELPER METHODS
-  this.getStringChunk = function (chunk) {
-    return typeof chunk === 'string' ? chunk : this.textDecoder.decode(chunk, {stream: true});
-  };
-}
-
-AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer.prototype);
-AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;