@loaders.gl/csv 3.1.3 → 4.0.0-alpha.5

package/dist/bundle.js

@@ -1,5 +1,5 @@
-"use strict";
-// @ts-nocheck
 const moduleExports = require('./index');
+
 globalThis.loaders = globalThis.loaders || {};
 module.exports = Object.assign(globalThis.loaders, moduleExports);
+//# sourceMappingURL=bundle.js.map

package/dist/bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/csv-loader.js

@@ -1,267 +1,240 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports._typecheckCSVLoader = exports.CSVLoader = void 0;
-const schema_1 = require("@loaders.gl/schema");
-const papaparse_1 = __importDefault(require("./papaparse/papaparse"));
-const async_iterator_streamer_1 = __importDefault(require("./papaparse/async-iterator-streamer"));
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+import { AsyncQueue, TableBatchBuilder, convertToArrayRow, convertToObjectRow } from '@loaders.gl/schema';
+import Papa from './papaparse/papaparse';
+import AsyncIteratorStreamer from './papaparse/async-iterator-streamer';
+const VERSION = typeof "4.0.0-alpha.5" !== 'undefined' ? "4.0.0-alpha.5" : 'latest';
 const DEFAULT_CSV_LOADER_OPTIONS = {
-    csv: {
-        shape: 'object-row-table',
-        optimizeMemoryUsage: false,
-        // CSV options
-        header: 'auto',
-        columnPrefix: 'column',
-        // delimiter: auto
-        // newline: auto
-        quoteChar: '"',
-        escapeChar: '"',
-        dynamicTyping: true,
-        comments: false,
-        skipEmptyLines: true,
-        // transform: null?
-        delimitersToGuess: [',', '\t', '|', ';']
-        // fastMode: auto
-    }
+  csv: {
+    shape: 'object-row-table',
+    optimizeMemoryUsage: false,
+    header: 'auto',
+    columnPrefix: 'column',
+    quoteChar: '"',
+    escapeChar: '"',
+    dynamicTyping: true,
+    comments: false,
+    skipEmptyLines: true,
+    delimitersToGuess: [',', '\t', '|', ';']
+  }
 };
-exports.CSVLoader = {
-    id: 'csv',
-    module: 'csv',
-    name: 'CSV',
-    version: VERSION,
-    extensions: ['csv', 'tsv', 'dsv'],
-    mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
-    category: 'table',
-    parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
-    parseText: (text, options) => parseCSV(text, options),
-    parseInBatches: parseCSVInBatches,
-    // @ts-ignore
-    // testText: null,
-    options: DEFAULT_CSV_LOADER_OPTIONS
+export const CSVLoader = {
+  id: 'csv',
+  module: 'csv',
+  name: 'CSV',
+  version: VERSION,
+  extensions: ['csv', 'tsv', 'dsv'],
+  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+  category: 'table',
+  parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
+  parseText: (text, options) => parseCSV(text, options),
+  parseInBatches: parseCSVInBatches,
+  options: DEFAULT_CSV_LOADER_OPTIONS
 };
+
 async function parseCSV(csvText, options) {
-    // Apps can call the parse method directly, we so apply default options here
-    const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv };
-    const firstRow = readFirstRow(csvText);
-    const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
-    const parseWithHeader = header;
-    const papaparseConfig = {
-        // dynamicTyping: true,
-        ...csvOptions,
-        header: parseWithHeader,
-        download: false,
-        transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
-        error: (e) => {
-            throw new Error(e);
-        }
-    };
-    const result = papaparse_1.default.parse(csvText, papaparseConfig);
-    let { data: rows } = result;
-    const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
-    switch (csvOptions.shape) {
-        case 'object-row-table':
-            rows = rows.map((row) => (Array.isArray(row) ? (0, schema_1.convertToObjectRow)(row, headerRow) : row));
-            break;
-        case 'array-row-table':
-            rows = rows.map((row) => (Array.isArray(row) ? row : (0, schema_1.convertToArrayRow)(row, headerRow)));
-            break;
-        default:
-    }
-    /*
-    if (!header && shape === 'object-row-table') {
-      // If the dataset has no header, transform the array result into an object shape with an
-      // autogenerated header
-      return result.data.map((row) =>
-        row.reduce((acc, value, i) => {
-          acc[headerRow[i]] = value;
-          return acc;
-        }, {})
-      );
+  const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv,
+    ...(options === null || options === void 0 ? void 0 : options.csv)
+  };
+  const firstRow = readFirstRow(csvText);
+  const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
+  const parseWithHeader = header;
+  const papaparseConfig = { ...csvOptions,
+    header: parseWithHeader,
+    download: false,
+    transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
+    error: e => {
+      throw new Error(e);
     }
-    */
-    return rows;
+  };
+  const result = Papa.parse(csvText, papaparseConfig);
+  let {
+    data: rows
+  } = result;
+  const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
+
+  switch (csvOptions.shape) {
+    case 'object-row-table':
+      rows = rows.map(row => Array.isArray(row) ? convertToObjectRow(row, headerRow) : row);
+      break;
+
+    case 'array-row-table':
+      rows = rows.map(row => Array.isArray(row) ? row : convertToArrayRow(row, headerRow));
+      break;
+
+    default:
+  }
+
+  return rows;
 }
-// TODO - support batch size 0 = no batching/single batch?
+
 function parseCSVInBatches(asyncIterator, options) {
-    // Papaparse does not support standard batch size handling
-    // TODO - investigate papaparse chunks mode
-    options = { ...options };
-    if (options.batchSize === 'auto') {
-        options.batchSize = 4000;
-    }
-    // Apps can call the parse method directly, we so apply default options here
-    const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv };
-    const asyncQueue = new schema_1.AsyncQueue();
-    let isFirstRow = true;
-    let headerRow = null;
-    let tableBatchBuilder = null;
-    let schema = null;
-    const config = {
-        // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
-        ...csvOptions,
-        header: false,
-        download: false,
-        // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the
-        // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.
-        // See https://github.com/mholt/PapaParse/issues/465
-        chunkSize: 1024 * 1024 * 5,
-        // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true
-        // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if
-        // both of the skipEmptyLines and step callback options are provided:
-        // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465
-        // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825
-        skipEmptyLines: false,
-        // step is called on every row
-        // eslint-disable-next-line complexity
-        step(results) {
-            let row = results.data;
-            if (csvOptions.skipEmptyLines) {
-                // Manually reject lines that are empty
-                const collapsedRow = row.flat().join('').trim();
-                if (collapsedRow === '') {
-                    return;
-                }
-            }
-            const bytesUsed = results.meta.cursor;
-            // Check if we need to save a header row
-            if (isFirstRow && !headerRow) {
-                // Auto detects or can be forced with csvOptions.header
-                const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
-                if (header) {
-                    headerRow = row.map(duplicateColumnTransformer());
-                    return;
-                }
-            }
-            // If first data row, we can deduce the schema
-            if (isFirstRow) {
-                isFirstRow = false;
-                if (!headerRow) {
-                    headerRow = generateHeader(csvOptions.columnPrefix, row.length);
-                }
-                schema = deduceSchema(row, headerRow);
-            }
-            if (csvOptions.optimizeMemoryUsage) {
-                // A workaround to allocate new strings and don't retain pointers to original strings.
-                // https://bugs.chromium.org/p/v8/issues/detail?id=2869
-                row = JSON.parse(JSON.stringify(row));
-            }
-            // Add the row
-            tableBatchBuilder =
-                tableBatchBuilder ||
-                    new schema_1.TableBatchBuilder(schema, {
-                        // @ts-expect-error
-                        shape: csvOptions.shape || 'array-row-table',
-                        ...options
-                    });
-            try {
-                tableBatchBuilder.addRow(row);
-                // If a batch has been completed, emit it
-                const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({ bytesUsed });
-                if (batch) {
-                    asyncQueue.enqueue(batch);
-                }
-            }
-            catch (error) {
-                asyncQueue.enqueue(error);
-            }
-        },
-        // complete is called when all rows have been read
-        complete(results) {
-            try {
-                const bytesUsed = results.meta.cursor;
-                // Ensure any final (partial) batch gets emitted
-                const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({ bytesUsed });
-                if (batch) {
-                    asyncQueue.enqueue(batch);
-                }
-            }
-            catch (error) {
-                asyncQueue.enqueue(error);
-            }
-            asyncQueue.close();
+  var _options;
+
+  options = { ...options
+  };
+
+  if (options.batchSize === 'auto') {
+    options.batchSize = 4000;
+  }
+
+  const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv,
+    ...((_options = options) === null || _options === void 0 ? void 0 : _options.csv)
+  };
+  const asyncQueue = new AsyncQueue();
+  let isFirstRow = true;
+  let headerRow = null;
+  let tableBatchBuilder = null;
+  let schema = null;
+  const config = { ...csvOptions,
+    header: false,
+    download: false,
+    chunkSize: 1024 * 1024 * 5,
+    skipEmptyLines: false,
+
+    step(results) {
+      let row = results.data;
+
+      if (csvOptions.skipEmptyLines) {
+        const collapsedRow = row.flat().join('').trim();
+
+        if (collapsedRow === '') {
+          return;
+        }
+      }
+
+      const bytesUsed = results.meta.cursor;
+
+      if (isFirstRow && !headerRow) {
+        const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
+
+        if (header) {
+          headerRow = row.map(duplicateColumnTransformer());
+          return;
+        }
+      }
+
+      if (isFirstRow) {
+        isFirstRow = false;
+
+        if (!headerRow) {
+          headerRow = generateHeader(csvOptions.columnPrefix, row.length);
         }
-    };
-    papaparse_1.default.parse(asyncIterator, config, async_iterator_streamer_1.default);
-    // TODO - Does it matter if we return asyncIterable or asyncIterator
-    // return asyncQueue[Symbol.asyncIterator]();
-    return asyncQueue;
+
+        schema = deduceSchema(row, headerRow);
+      }
+
+      if (csvOptions.optimizeMemoryUsage) {
+        row = JSON.parse(JSON.stringify(row));
+      }
+
+      tableBatchBuilder = tableBatchBuilder || new TableBatchBuilder(schema, {
+        shape: csvOptions.shape || 'array-row-table',
+        ...options
+      });
+
+      try {
+        tableBatchBuilder.addRow(row);
+        const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
+          bytesUsed
+        });
+
+        if (batch) {
+          asyncQueue.enqueue(batch);
+        }
+      } catch (error) {
+        asyncQueue.enqueue(error);
+      }
+    },
+
+    complete(results) {
+      try {
+        const bytesUsed = results.meta.cursor;
+        const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
+          bytesUsed
+        });
+
+        if (batch) {
+          asyncQueue.enqueue(batch);
+        }
+      } catch (error) {
+        asyncQueue.enqueue(error);
+      }
+
+      asyncQueue.close();
+    }
+
+  };
+  Papa.parse(asyncIterator, config, AsyncIteratorStreamer);
+  return asyncQueue;
 }
-/**
- * Checks if a certain row is a header row
- * @param row the row to check
- * @returns true if the row looks like a header
- */
+
 function isHeaderRow(row) {
-    return row && row.every((value) => typeof value === 'string');
+  return row && row.every(value => typeof value === 'string');
 }
-/**
- * Reads, parses, and returns the first row of a CSV text
- * @param csvText the csv text to parse
- * @returns the first row
- */
+
 function readFirstRow(csvText) {
-    const result = papaparse_1.default.parse(csvText, {
-        download: false,
-        dynamicTyping: true,
-        preview: 1
-    });
-    return result.data[0];
+  const result = Papa.parse(csvText, {
+    download: false,
+    dynamicTyping: true,
+    preview: 1
+  });
+  return result.data[0];
 }
-/**
- * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle
- * duplicate header columns and would use the latest occurrence by default.
- * See the header option in https://www.papaparse.com/docs#config
- * @returns a transform function that returns sanitized names for duplicate fields
- */
+
 function duplicateColumnTransformer() {
-    const observedColumns = new Set();
-    return (col) => {
-        let colName = col;
-        let counter = 1;
-        while (observedColumns.has(colName)) {
-            colName = `${col}.${counter}`;
-            counter++;
-        }
-        observedColumns.add(colName);
-        return colName;
-    };
+  const observedColumns = new Set();
+  return col => {
+    let colName = col;
+    let counter = 1;
+
+    while (observedColumns.has(colName)) {
+      colName = "".concat(col, ".").concat(counter);
+      counter++;
+    }
+
+    observedColumns.add(colName);
+    return colName;
+  };
 }
-/**
- * Generates the header of a CSV given a prefix and a column count
- * @param columnPrefix the columnPrefix to use
- * @param count the count of column names to generate
- * @returns an array of column names
- */
+
 function generateHeader(columnPrefix, count = 0) {
-    const headers = [];
-    for (let i = 0; i < count; i++) {
-        headers.push(`${columnPrefix}${i + 1}`);
-    }
-    return headers;
+  const headers = [];
+
+  for (let i = 0; i < count; i++) {
+    headers.push("".concat(columnPrefix).concat(i + 1));
+  }
+
+  return headers;
 }
+
 function deduceSchema(row, headerRow) {
-    const schema = headerRow ? {} : [];
-    for (let i = 0; i < row.length; i++) {
-        const columnName = (headerRow && headerRow[i]) || i;
-        const value = row[i];
-        switch (typeof value) {
-            case 'number':
-            case 'boolean':
-                // TODO - booleans could be handled differently...
-                schema[columnName] = { name: String(columnName), index: i, type: Float32Array };
-                break;
-            case 'string':
-            default:
-                schema[columnName] = { name: String(columnName), index: i, type: Array };
-            // We currently only handle numeric rows
-            // TODO we could offer a function to map strings to numbers?
-        }
+  const schema = headerRow ? {} : [];
+
+  for (let i = 0; i < row.length; i++) {
+    const columnName = headerRow && headerRow[i] || i;
+    const value = row[i];
+
+    switch (typeof value) {
+      case 'number':
+      case 'boolean':
+        schema[columnName] = {
+          name: String(columnName),
+          index: i,
+          type: Float32Array
+        };
+        break;
+
+      case 'string':
+      default:
+        schema[columnName] = {
+          name: String(columnName),
+          index: i,
+          type: Array
+        };
     }
-    return schema;
+  }
+
+  return schema;
 }
-exports._typecheckCSVLoader = exports.CSVLoader;
+
+export const _typecheckCSVLoader = CSVLoader;
+//# sourceMappingURL=csv-loader.js.map

package/dist/csv-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/csv-loader.ts"],"names":["AsyncQueue","TableBatchBuilder","convertToArrayRow","convertToObjectRow","Papa","AsyncIteratorStreamer","VERSION","DEFAULT_CSV_LOADER_OPTIONS","csv","shape","optimizeMemoryUsage","header","columnPrefix","quoteChar","escapeChar","dynamicTyping","comments","skipEmptyLines","delimitersToGuess","CSVLoader","id","module","name","version","extensions","mimeTypes","category","parse","arrayBuffer","options","parseCSV","TextDecoder","decode","parseText","text","parseInBatches","parseCSVInBatches","csvText","csvOptions","firstRow","readFirstRow","isHeaderRow","Boolean","parseWithHeader","papaparseConfig","download","transformHeader","duplicateColumnTransformer","undefined","error","e","Error","result","data","rows","headerRow","meta","fields","generateHeader","length","map","row","Array","isArray","asyncIterator","batchSize","asyncQueue","isFirstRow","tableBatchBuilder","schema","config","chunkSize","step","results","collapsedRow","flat","join","trim","bytesUsed","cursor","deduceSchema","JSON","stringify","addRow","batch","getFullBatch","enqueue","complete","getFinalBatch","close","every","value","preview","observedColumns","Set","col","colName","counter","has","add","count","headers","i","push","columnName","String","index","type","Float32Array","_typecheckCSVLoader"],"mappings":"AAIA,SACEA,UADF,EAEEC,iBAFF,EAGEC,iBAHF,EAIEC,kBAJF,QAKO,oBALP;AAMA,OAAOC,IAAP,MAAiB,uBAAjB;AACA,OAAOC,qBAAP,MAAkC,qCAAlC;AAIA,MAAMC,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AA0BA,MAAMC,0BAA0B,GAAG;AACjCC,EAAAA,GAAG,EAAE;AACHC,IAAAA,KAAK,EAAE,kBADJ;AAEHC,IAAAA,mBAAmB,EAAE,KAFlB;AAIHC,IAAAA,MAAM,EAAE,MAJL;AAKHC,IAAAA,YAAY,EAAE,QALX;AAQHC,IAAAA,SAAS,EAAE,GARR;AASHC,IAAAA,UAAU,EAAE,GATT;AAUHC,IAAAA,aAAa,EAAE,IAVZ;AAWHC,IAAAA,QAAQ,EAAE,KAXP;AAYHC,IAAAA,cAAc,EAAE,IAZb;AAcHC,IAAAA,iBAAiB,EAAE,CAAC,GAAD,EAAM,IAAN,EAAY,GAAZ,EAAiB,GAAjB;AAdhB;AAD4B,CAAnC;AAoBA,OAAO,MAAMC,SAAS,GAAG;AACvBC,EAAAA,EAAE,EAAE,KADmB;AAEvBC,EAAAA,MAAM,EAAE,KAFe;AAGvBC,EAAAA,IAAI,EAAE,KAHiB;AAIvBC,EAAAA,OAAO,EAAEjB,OAJc;AAKvBkB,EAAAA,UAAU,EAAE,CAAC,KAAD,EAAQ,KAAR,EAAe,KAAf,CALW;AAMvBC,EAAAA,SAAS,EAAE,CAAC,UAAD,EAAa,2BAAb,EAA0C,UAA1C,CANY;AAOvBC,EAAAA,QAAQ,EAAE,OAPa;AAQvBC,EAAAA,KAAK,EAAE,OAAOC,WAAP,EAAiCC,OAAjC,KACLC,QAAQ,CAAC,IAAIC,WAAJ,GAAkBC,MAAlB,CAAyBJ,WAAzB,CAAD,EAAwCC,OAAxC,CATa;AAUvBI,EAAAA,SAAS,EAAE,CAACC,IAAD,EAAeL,OAAf,KAA8CC,QAAQ,CAACI,IAAD,EAAOL,OAAP,CAV1C;AAWvBM,EAAAA,cAAc,EAAEC,iBAXO;AAcvBP,EAAAA,OAAO,EAAEtB;AAdc,CAAlB;;AAiBP,eAAeuB,QAAf,CAAwBO,OAAxB,EAAyCR,OAAzC,EAAqE;AAEnE,QAAMS,UAAU,GAAG,EAAC,GAAG/B,0BAA0B,CAACC,GAA/B;AAAoC,QAAGqB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAErB,GAAZ;AAApC,GAAnB;AAEA,QAAM+B,QAAQ,GAAGC,YAAY,CAACH,OAAD,CAA7B;AACA,QAAM1B,MAAe,GACnB2B,UAAU,CAAC3B,MAAX,KAAsB,MAAtB,GAA+B8B,WAAW,CAACF,QAAD,CAA1C,GAAuDG,OAAO,CAACJ,UAAU,CAAC3B,MAAZ,CADhE;AAGA,QAAMgC,eAAe,GAAGhC,MAAxB;AAEA,QAAMiC,eAAe,GAAG,EAEtB,GAAGN,UAFmB;AAGtB3B,IAAAA,MAAM,EAAEgC,eAHc;AAItBE,IAAAA,QAAQ,EAAE,KAJY;AAKtBC,IAAAA,eAAe,EAAEH,eAAe,GAAGI,0BAA0B,EAA7B,GAAkCC,SAL5C;AAMtBC,IAAAA,KAAK,EAAGC,CAAD,IAAO;AACZ,YAAM,IAAIC,KAAJ,CAAUD,CAAV,CAAN;AACD;AARqB,GAAxB;AAWA,QAAME,MAAM,GAAGhD,IAAI,CAACuB,KAAL,CAAWU,OAAX,EAAoBO,eAApB,CAAf;AACA,MAAI;AAACS,IAAAA,IAAI,EAAEC;AAAP,MAAeF,MAAnB;AAEA,QAAMG,SAAS,GAAGH,MAAM,CAACI,IAAP,CAAYC,MAAZ,IAAsBC,cAAc,CAACpB,UAAU,CAAC1B,YAAZ,EAA0B2B,QAAQ,CAACoB,MAAnC,CAAtD;;AAEA,UAAQrB,UAAU,CAAC7B,KAAnB;AACE,SAAK,kBAAL;AACE6C,MAAAA,IAAI,GAAGA,IAAI,CAACM,GAAL,CAAUC,GAAD,IAAUC,KAAK,CAACC,OAAN,CAAcF,GAAd,IAAqB1D,kBAAkB,CAAC0D,GAAD,EAAMN,SAAN,CAAvC,GAA0DM,GAA7E,CAAP;AACA;;AACF,SAAK,iBAAL;AACEP,MAAAA,IAAI,GAAGA,IAAI,CAACM,GAAL,CAAUC,GAAD,IAAUC,KAAK,CAACC,OAAN,CAAcF,GAAd,IAAqBA,GAArB,GAA2B3D,iBAAiB,CAAC2D,GAAD,EAAMN,SAAN,CAA/D,CAAP;AACA;;AACF;AAPF;;AAsBA,SAAOD,IAAP;AACD;;AAGD,SAASlB,iBAAT,CACE4B,aADF,EAEEnC,OAFF,EAGwB;AAAA;;AAGtBA,EAAAA,OAAO,GAAG,EAAC,GAAGA;AAAJ,GAAV;;AACA,MAAIA,OAAO,CAACoC,SAAR,KAAsB,MAA1B,EAAkC;AAChCpC,IAAAA,OAAO,CAACoC,SAAR,GAAoB,IAApB;AACD;;AAGD,QAAM3B,UAAU,GAAG,EAAC,GAAG/B,0BAA0B,CAACC,GAA/B;AAAoC,oBAAGqB,OAAH,6CAAG,SAASrB,GAAZ;AAApC,GAAnB;AAEA,QAAM0D,UAAU,GAAG,IAAIlE,UAAJ,EAAnB;AAEA,MAAImE,UAAmB,GAAG,IAA1B;AACA,MAAIZ,SAA0B,GAAG,IAAjC;AACA,MAAIa,iBAA2C,GAAG,IAAlD;AACA,MAAIC,MAAqB,GAAG,IAA5B;AAEA,QAAMC,MAAM,GAAG,EAEb,GAAGhC,UAFU;AAGb3B,IAAAA,MAAM,EAAE,KAHK;AAIbkC,IAAAA,QAAQ,EAAE,KAJG;AAQb0B,IAAAA,SAAS,EAAE,OAAO,IAAP,GAAc,CARZ;AAcbtD,IAAAA,cAAc,EAAE,KAdH;;AAkBbuD,IAAAA,IAAI,CAACC,OAAD,EAAU;AACZ,UAAIZ,GAAG,GAAGY,OAAO,CAACpB,IAAlB;;AAEA,UAAIf,UAAU,CAACrB,cAAf,EAA+B;AAE7B,cAAMyD,YAAY,GAAGb,GAAG,CAACc,IAAJ,GAAWC,IAAX,CAAgB,EAAhB,EAAoBC,IAApB,EAArB;;AACA,YAAIH,YAAY,KAAK,EAArB,EAAyB;AACvB;AACD;AACF;;AACD,YAAMI,SAAS,GAAGL,OAAO,CAACjB,IAAR,CAAauB,MAA/B;;AAGA,UAAIZ,UAAU,IAAI,CAACZ,SAAnB,EAA8B;AAE5B,cAAM5C,MAAM,GAAG2B,UAAU,CAAC3B,MAAX,KAAsB,MAAtB,GAA+B8B,WAAW,CAACoB,GAAD,CAA1C,GAAkDnB,OAAO,CAACJ,UAAU,CAAC3B,MAAZ,CAAxE;;AACA,YAAIA,MAAJ,EAAY;AACV4C,UAAAA,SAAS,GAAGM,GAAG,CAACD,GAAJ,CAAQb,0BAA0B,EAAlC,CAAZ;AACA;AACD;AACF;;AAGD,UAAIoB,UAAJ,EAAgB;AACdA,QAAAA,UAAU,GAAG,KAAb;;AACA,YAAI,CAACZ,SAAL,EAAgB;AACdA,UAAAA,SAAS,GAAGG,cAAc,CAACpB,UAAU,CAAC1B,YAAZ,EAA0BiD,GAAG,CAACF,MAA9B,CAA1B;AACD;;AACDU,QAAAA,MAAM,GAAGW,YAAY,CAACnB,GAAD,EAAMN,SAAN,CAArB;AACD;;AAED,UAAIjB,UAAU,CAAC5B,mBAAf,EAAoC;AAGlCmD,QAAAA,GAAG,GAAGoB,IAAI,CAACtD,KAAL,CAAWsD,IAAI,CAACC,SAAL,CAAerB,GAAf,CAAX,CAAN;AACD;;AAGDO,MAAAA,iBAAiB,GACfA,iBAAiB,IACjB,IAAInE,iBAAJ,CAAsBoE,MAAtB,EAA8B;AAE5B5D,QAAAA,KAAK,EAAE6B,UAAU,CAAC7B,KAAX,IAAoB,iBAFC;AAG5B,WAAGoB;AAHyB,OAA9B,CAFF;;AAQA,UAAI;AACFuC,QAAAA,iBAAiB,CAACe,MAAlB,CAAyBtB,GAAzB;AAEA,cAAMuB,KAAK,GAAGhB,iBAAiB,IAAIA,iBAAiB,CAACiB,YAAlB,CAA+B;AAACP,UAAAA;AAAD,SAA/B,CAAnC;;AACA,YAAIM,KAAJ,EAAW;AACTlB,UAAAA,UAAU,CAACoB,OAAX,CAAmBF,KAAnB;AACD;AACF,OAPD,CAOE,OAAOnC,KAAP,EAAc;AACdiB,QAAAA,UAAU,CAACoB,OAAX,CAAmBrC,KAAnB;AACD;AACF,KA1EY;;AA6EbsC,IAAAA,QAAQ,CAACd,OAAD,EAAU;AAChB,UAAI;AACF,cAAMK,SAAS,GAAGL,OAAO,CAACjB,IAAR,CAAauB,MAA/B;AAEA,cAAMK,KAAK,GAAGhB,iBAAiB,IAAIA,iBAAiB,CAACoB,aAAlB,CAAgC;AAACV,UAAAA;AAAD,SAAhC,CAAnC;;AACA,YAAIM,KAAJ,EAAW;AACTlB,UAAAA,UAAU,CAACoB,OAAX,CAAmBF,KAAnB;AACD;AACF,OAPD,CAOE,OAAOnC,KAAP,EAAc;AACdiB,QAAAA,UAAU,CAACoB,OAAX,CAAmBrC,KAAnB;AACD;;AAEDiB,MAAAA,UAAU,CAACuB,KAAX;AACD;;AA1FY,GAAf;AA6FArF,EAAAA,IAAI,CAACuB,KAAL,CAAWqC,aAAX,EAA0BM,MAA1B,EAAkCjE,qBAAlC;AAIA,SAAO6D,UAAP;AACD;;AAOD,SAASzB,WAAT,CAAqBoB,GAArB,EAA6C;AAC3C,SAAOA,GAAG,IAAIA,GAAG,CAAC6B,KAAJ,CAAWC,KAAD,IAAW,OAAOA,KAAP,KAAiB,QAAtC,CAAd;AACD;;AAOD,SAASnD,YAAT,CAAsBH,OAAtB,EAA8C;AAC5C,QAAMe,MAAM,GAAGhD,IAAI,CAACuB,KAAL,CAAWU,OAAX,EAAoB;AACjCQ,IAAAA,QAAQ,EAAE,KADuB;AAEjC9B,IAAAA,aAAa,EAAE,IAFkB;AAGjC6E,IAAAA,OAAO,EAAE;AAHwB,GAApB,CAAf;AAKA,SAAOxC,MAAM,CAACC,IAAP,CAAY,CAAZ,CAAP;AACD;;AAQD,SAASN,0BAAT,GAAsC;AACpC,QAAM8C,eAAe,GAAG,IAAIC,GAAJ,EAAxB;AACA,SAAQC,GAAD,IAAS;AACd,QAAIC,OAAO,GAAGD,GAAd;AACA,QAAIE,OAAO,GAAG,CAAd;;AACA,WAAOJ,eAAe,CAACK,GAAhB,CAAoBF,OAApB,CAAP,EAAqC;AACnCA,MAAAA,OAAO,aAAMD,GAAN,cAAaE,OAAb,CAAP;AACAA,MAAAA,OAAO;AACR;;AACDJ,IAAAA,eAAe,CAACM,GAAhB,CAAoBH,OAApB;AACA,WAAOA,OAAP;AACD,GATD;AAUD;;AAQD,SAAStC,cAAT,CAAwB9C,YAAxB,EAA8CwF,KAAa,GAAG,CAA9D,EAA2E;AACzE,QAAMC,OAAiB,GAAG,EAA1B;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGF,KAApB,EAA2BE,CAAC,EAA5B,EAAgC;AAC9BD,IAAAA,OAAO,CAACE,IAAR,WAAgB3F,YAAhB,SAA+B0F,CAAC,GAAG,CAAnC;AACD;;AACD,SAAOD,OAAP;AACD;;AAED,SAASrB,YAAT,CAAsBnB,GAAtB,EAA2BN,SAA3B,EAAsC;AACpC,QAAMc,MAAM,GAAGd,SAAS,GAAG,EAAH,GAAQ,EAAhC;;AACA,OAAK,IAAI+C,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGzC,GAAG,CAACF,MAAxB,EAAgC2C,CAAC,EAAjC,EAAqC;AACnC,UAAME,UAAU,GAAIjD,SAAS,IAAIA,SAAS,CAAC+C,CAAD,CAAvB,IAA+BA,CAAlD;AACA,UAAMX,KAAK,GAAG9B,GAAG,CAACyC,CAAD,CAAjB;;AACA,YAAQ,OAAOX,KAAf;AACE,WAAK,QAAL;AACA,WAAK,SAAL;AAEEtB,QAAAA,MAAM,CAACmC,UAAD,CAAN,GAAqB;AAAClF,UAAAA,IAAI,EAAEmF,MAAM,CAACD,UAAD,CAAb;AAA2BE,UAAAA,KAAK,EAAEJ,CAAlC;AAAqCK,UAAAA,IAAI,EAAEC;AAA3C,SAArB;AACA;;AACF,WAAK,QAAL;AACA;AACEvC,QAAAA,MAAM,CAACmC,UAAD,CAAN,GAAqB;AAAClF,UAAAA,IAAI,EAAEmF,MAAM,CAACD,UAAD,CAAb;AAA2BE,UAAAA,KAAK,EAAEJ,CAAlC;AAAqCK,UAAAA,IAAI,EAAE7C;AAA3C,SAArB;AARJ;AAYD;;AACD,SAAOO,MAAP;AACD;;AAED,OAAO,MAAMwC,mBAAqC,GAAG1F,SAA9C","sourcesContent":["import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {Batch} from '@loaders.gl/schema';\ntype Schema = any;\n\nimport {\n  AsyncQueue,\n  TableBatchBuilder,\n  convertToArrayRow,\n  convertToObjectRow\n} from '@loaders.gl/schema';\nimport Papa from './papaparse/papaparse';\nimport AsyncIteratorStreamer from './papaparse/async-iterator-streamer';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type CSVLoaderOptions = LoaderOptions & {\n  csv?: {\n    // loaders.gl options\n    shape?: 'array-row-table' | 'object-row-table' | 'columnar-table';\n    /** optimizes memory usage but increases parsing time. */\n    optimizeMemoryUsage?: boolean;\n    columnPrefix?: string;\n    header?: 'auto';\n\n    // CSV options (papaparse)\n    // delimiter: auto\n    // newline: auto\n    quoteChar?: string;\n    escapeChar?: string;\n    // Convert numbers and boolean values in rows from strings\n    dynamicTyping?: boolean;\n    comments?: boolean;\n    skipEmptyLines?: boolean | 'greedy';\n    // transform: null?\n    delimitersToGuess?: string[];\n    // fastMode: auto\n  };\n};\n\nconst DEFAULT_CSV_LOADER_OPTIONS = {\n  csv: {\n    shape: 'object-row-table',\n    optimizeMemoryUsage: false,\n    // CSV options\n    header: 'auto',\n    columnPrefix: 'column',\n    // delimiter: auto\n    // newline: auto\n    quoteChar: '\"',\n    escapeChar: '\"',\n    dynamicTyping: true,\n    comments: false,\n    skipEmptyLines: true,\n    // transform: null?\n    delimitersToGuess: [',', '\\t', '|', ';']\n    // fastMode: auto\n  }\n};\n\nexport const CSVLoader = {\n  id: 'csv',\n  module: 'csv',\n  name: 'CSV',\n  version: VERSION,\n  extensions: ['csv', 'tsv', 'dsv'],\n  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],\n  category: 'table',\n  parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) =>\n    parseCSV(new TextDecoder().decode(arrayBuffer), options),\n  parseText: (text: string, options?: CSVLoaderOptions) => parseCSV(text, options),\n  parseInBatches: parseCSVInBatches,\n  // @ts-ignore\n  // testText: null,\n  options: DEFAULT_CSV_LOADER_OPTIONS as CSVLoaderOptions\n};\n\nasync function parseCSV(csvText: string, options?: CSVLoaderOptions) {\n  // Apps can call the parse method directly, we so apply default options here\n  const csvOptions = {...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv};\n\n  const firstRow = readFirstRow(csvText);\n  const header: boolean =\n    csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);\n\n  const parseWithHeader = header;\n\n  const papaparseConfig = {\n    // dynamicTyping: true,\n    ...csvOptions,\n    header: parseWithHeader,\n    download: false, // We handle loading, no need for papaparse to do it for us\n    transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,\n    error: (e) => {\n      throw new Error(e);\n    }\n  };\n\n  const result = Papa.parse(csvText, papaparseConfig);\n  let {data: rows} = result;\n\n  const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);\n\n  switch (csvOptions.shape) {\n    case 'object-row-table':\n      rows = rows.map((row) => (Array.isArray(row) ? convertToObjectRow(row, headerRow) : row));\n      break;\n    case 'array-row-table':\n      rows = rows.map((row) => (Array.isArray(row) ? row : convertToArrayRow(row, headerRow)));\n      break;\n    default:\n  }\n\n  /*\n  if (!header && shape === 'object-row-table') {\n    // If the dataset has no header, transform the array result into an object shape with an\n    // autogenerated header\n    return result.data.map((row) =>\n      row.reduce((acc, value, i) => {\n        acc[headerRow[i]] = value;\n        return acc;\n      }, {})\n    );\n  }\n  */\n  return rows;\n}\n\n// TODO - support batch size 0 = no batching/single batch?\nfunction parseCSVInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: CSVLoaderOptions\n): AsyncIterable<Batch> {\n  // Papaparse does not support standard batch size handling\n  // TODO - investigate papaparse chunks mode\n  options = {...options};\n  if (options.batchSize === 'auto') {\n    options.batchSize = 4000;\n  }\n\n  // Apps can call the parse method directly, we so apply default options here\n  const csvOptions = {...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv};\n\n  const asyncQueue = new AsyncQueue<Batch>();\n\n  let isFirstRow: boolean = true;\n  let headerRow: string[] | null = null;\n  let tableBatchBuilder: TableBatchBuilder | null = null;\n  let schema: Schema | null = null;\n\n  const config = {\n    // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,\n    ...csvOptions,\n    header: false, // Unfortunately, header detection is not automatic and does not infer shapes\n    download: false, // We handle loading, no need for papaparse to do it for us\n    // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the\n    // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.\n    // See https://github.com/mholt/PapaParse/issues/465\n    chunkSize: 1024 * 1024 * 5,\n    // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true\n    // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if\n    // both of the skipEmptyLines and step callback options are provided:\n    // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465\n    // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825\n    skipEmptyLines: false,\n\n    // step is called on every row\n    // eslint-disable-next-line complexity\n    step(results) {\n      let row = results.data;\n\n      if (csvOptions.skipEmptyLines) {\n        // Manually reject lines that are empty\n        const collapsedRow = row.flat().join('').trim();\n        if (collapsedRow === '') {\n          return;\n        }\n      }\n      const bytesUsed = results.meta.cursor;\n\n      // Check if we need to save a header row\n      if (isFirstRow && !headerRow) {\n        // Auto detects or can be forced with csvOptions.header\n        const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);\n        if (header) {\n          headerRow = row.map(duplicateColumnTransformer());\n          return;\n        }\n      }\n\n      // If first data row, we can deduce the schema\n      if (isFirstRow) {\n        isFirstRow = false;\n        if (!headerRow) {\n          headerRow = generateHeader(csvOptions.columnPrefix, row.length);\n        }\n        schema = deduceSchema(row, headerRow);\n      }\n\n      if (csvOptions.optimizeMemoryUsage) {\n        // A workaround to allocate new strings and don't retain pointers to original strings.\n        // https://bugs.chromium.org/p/v8/issues/detail?id=2869\n        row = JSON.parse(JSON.stringify(row));\n      }\n\n      // Add the row\n      tableBatchBuilder =\n        tableBatchBuilder ||\n        new TableBatchBuilder(schema, {\n          // @ts-expect-error\n          shape: csvOptions.shape || 'array-row-table',\n          ...options\n        });\n\n      try {\n        tableBatchBuilder.addRow(row);\n        // If a batch has been completed, emit it\n        const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({bytesUsed});\n        if (batch) {\n          asyncQueue.enqueue(batch);\n        }\n      } catch (error) {\n        asyncQueue.enqueue(error as Error);\n      }\n    },\n\n    // complete is called when all rows have been read\n    complete(results) {\n      try {\n        const bytesUsed = results.meta.cursor;\n        // Ensure any final (partial) batch gets emitted\n        const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({bytesUsed});\n        if (batch) {\n          asyncQueue.enqueue(batch);\n        }\n      } catch (error) {\n        asyncQueue.enqueue(error as Error);\n      }\n\n      asyncQueue.close();\n    }\n  };\n\n  Papa.parse(asyncIterator, config, AsyncIteratorStreamer);\n\n  // TODO - Does it matter if we return asyncIterable or asyncIterator\n  // return asyncQueue[Symbol.asyncIterator]();\n  return asyncQueue;\n}\n\n/**\n * Checks if a certain row is a header row\n * @param row the row to check\n * @returns true if the row looks like a header\n */\nfunction isHeaderRow(row: string[]): boolean {\n  return row && row.every((value) => typeof value === 'string');\n}\n\n/**\n * Reads, parses, and returns the first row of a CSV text\n * @param csvText the csv text to parse\n * @returns the first row\n */\nfunction readFirstRow(csvText: string): any[] {\n  const result = Papa.parse(csvText, {\n    download: false,\n    dynamicTyping: true,\n    preview: 1\n  });\n  return result.data[0];\n}\n\n/**\n * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle\n * duplicate header columns and would use the latest occurrence by default.\n * See the header option in https://www.papaparse.com/docs#config\n * @returns a transform function that returns sanitized names for duplicate fields\n */\nfunction duplicateColumnTransformer() {\n  const observedColumns = new Set();\n  return (col) => {\n    let colName = col;\n    let counter = 1;\n    while (observedColumns.has(colName)) {\n      colName = `${col}.${counter}`;\n      counter++;\n    }\n    observedColumns.add(colName);\n    return colName;\n  };\n}\n\n/**\n * Generates the header of a CSV given a prefix and a column count\n * @param columnPrefix the columnPrefix to use\n * @param count the count of column names to generate\n * @returns an array of column names\n */\nfunction generateHeader(columnPrefix: string, count: number = 0): string[] {\n  const headers: string[] = [];\n  for (let i = 0; i < count; i++) {\n    headers.push(`${columnPrefix}${i + 1}`);\n  }\n  return headers;\n}\n\nfunction deduceSchema(row, headerRow) {\n  const schema = headerRow ? {} : [];\n  for (let i = 0; i < row.length; i++) {\n    const columnName = (headerRow && headerRow[i]) || i;\n    const value = row[i];\n    switch (typeof value) {\n      case 'number':\n      case 'boolean':\n        // TODO - booleans could be handled differently...\n        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};\n        break;\n      case 'string':\n      default:\n        schema[columnName] = {name: String(columnName), index: i, type: Array};\n      // We currently only handle numeric rows\n      // TODO we could offer a function to map strings to numbers?\n    }\n  }\n  return schema;\n}\n\nexport const _typecheckCSVLoader: LoaderWithParser = CSVLoader;\n"],"file":"csv-loader.js"}

package/dist/csv-writer.js

@@ -1,6 +1,2 @@
-"use strict";
-/* @typedef {import('@loaders.gl/loader-utils').Writer} Writer */
-/*
- * CSV exporter
- * type {Writer}
- */
+
+//# sourceMappingURL=csv-writer.js.map

package/dist/index.js

@@ -1,5 +1,2 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CSVLoader = void 0;
-var csv_loader_1 = require("./csv-loader");
-Object.defineProperty(exports, "CSVLoader", { enumerable: true, get: function () { return csv_loader_1.CSVLoader; } });
+export { CSVLoader } from './csv-loader';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"names":["CSVLoader"],"mappings":"AACA,SAAQA,SAAR,QAAwB,cAAxB","sourcesContent":["export type {CSVLoaderOptions} from './csv-loader';\nexport {CSVLoader} from './csv-loader';\n"],"file":"index.js"}