@loaders.gl/csv 4.2.0-alpha.3 → 4.2.0-alpha.5

package/dist/csv-loader.js

@@ -1,209 +1,258 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 import { AsyncQueue, TableBatchBuilder, convertToArrayRow, convertToObjectRow } from '@loaders.gl/schema';
 import Papa from "./papaparse/papaparse.js";
 import AsyncIteratorStreamer from "./papaparse/async-iterator-streamer.js";
-const VERSION = typeof "4.2.0-alpha.3" !== 'undefined' ? "4.2.0-alpha.3" : 'latest';
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof "4.2.0-alpha.4" !== 'undefined' ? "4.2.0-alpha.4" : 'latest';
 const DEFAULT_CSV_SHAPE = 'object-row-table';
 export const CSVLoader = {
-  id: 'csv',
-  module: 'csv',
-  name: 'CSV',
-  version: VERSION,
-  extensions: ['csv', 'tsv', 'dsv'],
-  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
-  category: 'table',
-  parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
-  parseText: (text, options) => parseCSV(text, options),
-  parseInBatches: parseCSVInBatches,
-  options: {
-    csv: {
-      shape: DEFAULT_CSV_SHAPE,
-      optimizeMemoryUsage: false,
-      header: 'auto',
-      columnPrefix: 'column',
-      quoteChar: '"',
-      escapeChar: '"',
-      dynamicTyping: true,
-      comments: false,
-      skipEmptyLines: true,
-      delimitersToGuess: [',', '\t', '|', ';']
+    id: 'csv',
+    module: 'csv',
+    name: 'CSV',
+    version: VERSION,
+    extensions: ['csv', 'tsv', 'dsv'],
+    mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+    category: 'table',
+    parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
+    parseText: (text, options) => parseCSV(text, options),
+    parseInBatches: parseCSVInBatches,
+    // @ts-ignore
+    // testText: null,
+    options: {
+        csv: {
+            shape: DEFAULT_CSV_SHAPE, // 'object-row-table'
+            optimizeMemoryUsage: false,
+            // CSV options
+            header: 'auto',
+            columnPrefix: 'column',
+            // delimiter: auto
+            // newline: auto
+            quoteChar: '"',
+            escapeChar: '"',
+            dynamicTyping: true,
+            comments: false,
+            skipEmptyLines: true,
+            // transform: null?
+            delimitersToGuess: [',', '\t', '|', ';']
+            // fastMode: auto
+        }
     }
-  }
 };
 async function parseCSV(csvText, options) {
-  const csvOptions = {
-    ...CSVLoader.options.csv,
-    ...(options === null || options === void 0 ? void 0 : options.csv)
-  };
-  const firstRow = readFirstRow(csvText);
-  const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
-  const parseWithHeader = header;
-  const papaparseConfig = {
-    ...csvOptions,
-    header: parseWithHeader,
-    download: false,
-    transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
-    error: e => {
-      throw new Error(e);
+    // Apps can call the parse method directly, we so apply default options here
+    const csvOptions = { ...CSVLoader.options.csv, ...options?.csv };
+    const firstRow = readFirstRow(csvText);
+    const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
+    const parseWithHeader = header;
+    const papaparseConfig = {
+        // dynamicTyping: true,
+        ...csvOptions,
+        header: parseWithHeader,
+        download: false, // We handle loading, no need for papaparse to do it for us
+        transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
+        error: (e) => {
+            throw new Error(e);
+        }
+    };
+    const result = Papa.parse(csvText, papaparseConfig);
+    const rows = result.data;
+    const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
+    const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
+    switch (shape) {
+        case 'object-row-table':
+            return {
+                shape: 'object-row-table',
+                data: rows.map((row) => (Array.isArray(row) ? convertToObjectRow(row, headerRow) : row))
+            };
+        case 'array-row-table':
+            return {
+                shape: 'array-row-table',
+                data: rows.map((row) => (Array.isArray(row) ? row : convertToArrayRow(row, headerRow)))
+            };
+        default:
+            throw new Error(shape);
     }
-  };
-  const result = Papa.parse(csvText, papaparseConfig);
-  const rows = result.data;
-  const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
-  const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
-  switch (shape) {
-    case 'object-row-table':
-      return {
-        shape: 'object-row-table',
-        data: rows.map(row => Array.isArray(row) ? convertToObjectRow(row, headerRow) : row)
-      };
-    case 'array-row-table':
-      return {
-        shape: 'array-row-table',
-        data: rows.map(row => Array.isArray(row) ? row : convertToArrayRow(row, headerRow))
-      };
-    default:
-      throw new Error(shape);
-  }
 }
+// TODO - support batch size 0 = no batching/single batch?
 function parseCSVInBatches(asyncIterator, options) {
-  var _options;
-  options = {
-    ...options
-  };
-  if (options.batchSize === 'auto') {
-    options.batchSize = 4000;
-  }
-  const csvOptions = {
-    ...CSVLoader.options.csv,
-    ...((_options = options) === null || _options === void 0 ? void 0 : _options.csv)
-  };
-  const asyncQueue = new AsyncQueue();
-  let isFirstRow = true;
-  let headerRow = null;
-  let tableBatchBuilder = null;
-  let schema = null;
-  const config = {
-    ...csvOptions,
-    header: false,
-    download: false,
-    chunkSize: 1024 * 1024 * 5,
-    skipEmptyLines: false,
-    step(results) {
-      let row = results.data;
-      if (csvOptions.skipEmptyLines) {
-        const collapsedRow = row.flat().join('').trim();
-        if (collapsedRow === '') {
-          return;
-        }
-      }
-      const bytesUsed = results.meta.cursor;
-      if (isFirstRow && !headerRow) {
-        const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
-        if (header) {
-          headerRow = row.map(duplicateColumnTransformer());
-          return;
-        }
-      }
-      if (isFirstRow) {
-        isFirstRow = false;
-        if (!headerRow) {
-          headerRow = generateHeader(csvOptions.columnPrefix, row.length);
-        }
-        schema = deduceSchema(row, headerRow);
-      }
-      if (csvOptions.optimizeMemoryUsage) {
-        row = JSON.parse(JSON.stringify(row));
-      }
-      const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
-      tableBatchBuilder = tableBatchBuilder || new TableBatchBuilder(schema, {
-        shape,
-        ...options
-      });
-      try {
-        tableBatchBuilder.addRow(row);
-        const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
-          bytesUsed
-        });
-        if (batch) {
-          asyncQueue.enqueue(batch);
-        }
-      } catch (error) {
-        asyncQueue.enqueue(error);
-      }
-    },
-    complete(results) {
-      try {
-        const bytesUsed = results.meta.cursor;
-        const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
-          bytesUsed
-        });
-        if (batch) {
-          asyncQueue.enqueue(batch);
-        }
-      } catch (error) {
-        asyncQueue.enqueue(error);
-      }
-      asyncQueue.close();
+    // Papaparse does not support standard batch size handling
+    // TODO - investigate papaparse chunks mode
+    options = { ...options };
+    if (options.batchSize === 'auto') {
+        options.batchSize = 4000;
     }
-  };
-  Papa.parse(asyncIterator, config, AsyncIteratorStreamer);
-  return asyncQueue;
+    // Apps can call the parse method directly, we so apply default options here
+    const csvOptions = { ...CSVLoader.options.csv, ...options?.csv };
+    const asyncQueue = new AsyncQueue();
+    let isFirstRow = true;
+    let headerRow = null;
+    let tableBatchBuilder = null;
+    let schema = null;
+    const config = {
+        // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
+        ...csvOptions,
+        header: false, // Unfortunately, header detection is not automatic and does not infer shapes
+        download: false, // We handle loading, no need for papaparse to do it for us
+        // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the
+        // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.
+        // See https://github.com/mholt/PapaParse/issues/465
+        chunkSize: 1024 * 1024 * 5,
+        // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true
+        // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if
+        // both of the skipEmptyLines and step callback options are provided:
+        // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465
+        // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825
+        skipEmptyLines: false,
+        // step is called on every row
+        // eslint-disable-next-line complexity, max-statements
+        step(results) {
+            let row = results.data;
+            if (csvOptions.skipEmptyLines) {
+                // Manually reject lines that are empty
+                const collapsedRow = row.flat().join('').trim();
+                if (collapsedRow === '') {
+                    return;
+                }
+            }
+            const bytesUsed = results.meta.cursor;
+            // Check if we need to save a header row
+            if (isFirstRow && !headerRow) {
+                // Auto detects or can be forced with csvOptions.header
+                const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
+                if (header) {
+                    headerRow = row.map(duplicateColumnTransformer());
+                    return;
+                }
+            }
+            // If first data row, we can deduce the schema
+            if (isFirstRow) {
+                isFirstRow = false;
+                if (!headerRow) {
+                    headerRow = generateHeader(csvOptions.columnPrefix, row.length);
+                }
+                schema = deduceSchema(row, headerRow);
+            }
+            if (csvOptions.optimizeMemoryUsage) {
+                // A workaround to allocate new strings and don't retain pointers to original strings.
+                // https://bugs.chromium.org/p/v8/issues/detail?id=2869
+                row = JSON.parse(JSON.stringify(row));
+            }
+            const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
+            // Add the row
+            tableBatchBuilder =
+                tableBatchBuilder ||
+                    new TableBatchBuilder(
+                    // @ts-expect-error TODO this is not a proper schema
+                    schema, {
+                        shape,
+                        ...options
+                    });
+            try {
+                tableBatchBuilder.addRow(row);
+                // If a batch has been completed, emit it
+                const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({ bytesUsed });
+                if (batch) {
+                    asyncQueue.enqueue(batch);
+                }
+            }
+            catch (error) {
+                asyncQueue.enqueue(error);
+            }
+        },
+        // complete is called when all rows have been read
+        complete(results) {
+            try {
+                const bytesUsed = results.meta.cursor;
+                // Ensure any final (partial) batch gets emitted
+                const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({ bytesUsed });
+                if (batch) {
+                    asyncQueue.enqueue(batch);
+                }
+            }
+            catch (error) {
+                asyncQueue.enqueue(error);
+            }
+            asyncQueue.close();
+        }
+    };
+    Papa.parse(asyncIterator, config, AsyncIteratorStreamer);
+    // TODO - Does it matter if we return asyncIterable or asyncIterator
+    // return asyncQueue[Symbol.asyncIterator]();
+    return asyncQueue;
 }
+/**
+ * Checks if a certain row is a header row
+ * @param row the row to check
+ * @returns true if the row looks like a header
+ */
 function isHeaderRow(row) {
-  return row && row.every(value => typeof value === 'string');
+    return row && row.every((value) => typeof value === 'string');
 }
+/**
+ * Reads, parses, and returns the first row of a CSV text
+ * @param csvText the csv text to parse
+ * @returns the first row
+ */
 function readFirstRow(csvText) {
-  const result = Papa.parse(csvText, {
-    download: false,
-    dynamicTyping: true,
-    preview: 1
-  });
-  return result.data[0];
+    const result = Papa.parse(csvText, {
+        download: false,
+        dynamicTyping: true,
+        preview: 1
+    });
+    return result.data[0];
 }
+/**
+ * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle
+ * duplicate header columns and would use the latest occurrence by default.
+ * See the header option in https://www.papaparse.com/docs#config
+ * @returns a transform function that returns sanitized names for duplicate fields
+ */
 function duplicateColumnTransformer() {
-  const observedColumns = new Set();
-  return col => {
-    let colName = col;
-    let counter = 1;
-    while (observedColumns.has(colName)) {
-      colName = `${col}.${counter}`;
-      counter++;
-    }
-    observedColumns.add(colName);
-    return colName;
-  };
+    const observedColumns = new Set();
+    return (col) => {
+        let colName = col;
+        let counter = 1;
+        while (observedColumns.has(colName)) {
+            colName = `${col}.${counter}`;
+            counter++;
+        }
+        observedColumns.add(colName);
+        return colName;
+    };
 }
-function generateHeader(columnPrefix) {
-  let count = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;
-  const headers = [];
-  for (let i = 0; i < count; i++) {
-    headers.push(`${columnPrefix}${i + 1}`);
-  }
-  return headers;
+/**
+ * Generates the header of a CSV given a prefix and a column count
+ * @param columnPrefix the columnPrefix to use
+ * @param count the count of column names to generate
+ * @returns an array of column names
+ */
+function generateHeader(columnPrefix, count = 0) {
+    const headers = [];
+    for (let i = 0; i < count; i++) {
+        headers.push(`${columnPrefix}${i + 1}`);
+    }
+    return headers;
 }
 function deduceSchema(row, headerRow) {
-  const schema = headerRow ? {} : [];
-  for (let i = 0; i < row.length; i++) {
-    const columnName = headerRow && headerRow[i] || i;
-    const value = row[i];
-    switch (typeof value) {
-      case 'number':
-      case 'boolean':
-        schema[columnName] = {
-          name: String(columnName),
-          index: i,
-          type: Float32Array
-        };
-        break;
-      case 'string':
-      default:
-        schema[columnName] = {
-          name: String(columnName),
-          index: i,
-          type: Array
-        };
+    const schema = headerRow ? {} : [];
+    for (let i = 0; i < row.length; i++) {
+        const columnName = (headerRow && headerRow[i]) || i;
+        const value = row[i];
+        switch (typeof value) {
+            case 'number':
+            case 'boolean':
+                // TODO - booleans could be handled differently...
+                schema[columnName] = { name: String(columnName), index: i, type: Float32Array };
+                break;
+            case 'string':
+            default:
+                schema[columnName] = { name: String(columnName), index: i, type: Array };
+            // We currently only handle numeric rows
+            // TODO we could offer a function to map strings to numbers?
+        }
     }
-  }
-  return schema;
+    return schema;
 }
-//# sourceMappingURL=csv-loader.js.map

package/dist/csv-writer.js

@@ -1,18 +1,20 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 import { encodeTableAsCSV } from "./lib/encoders/encode-csv.js";
 export const CSVWriter = {
-  id: 'csv',
-  version: 'latest',
-  module: 'csv',
-  name: 'CSV',
-  extensions: ['csv'],
-  mimeTypes: ['text/csv'],
-  options: {
-    csv: {
-      useDisplayNames: false
-    }
-  },
-  text: true,
-  encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
-  encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
+    id: 'csv',
+    version: 'latest',
+    module: 'csv',
+    name: 'CSV',
+    extensions: ['csv'],
+    mimeTypes: ['text/csv'],
+    options: {
+        csv: {
+            useDisplayNames: false
+        }
+    },
+    text: true,
+    encode: async (table, options) => new TextEncoder().encode(encodeTableAsCSV(table, options)).buffer,
+    encodeTextSync: (table, options) => encodeTableAsCSV(table, options)
 };
-//# sourceMappingURL=csv-writer.js.map