@loaders.gl/csv 3.1.0-alpha.2 → 3.1.0-beta.1

package/dist/csv-loader.d.ts

@@ -0,0 +1,34 @@
+import type { LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
+import type { Batch } from '@loaders.gl/schema';
+export declare type CSVLoaderOptions = LoaderOptions & {
+    csv?: {
+        shape?: 'array-row-table' | 'object-row-table' | 'columnar-table';
+        /** optimizes memory usage but increases parsing time. */
+        optimizeMemoryUsage?: boolean;
+        columnPrefix?: string;
+        header?: 'auto';
+        quoteChar?: string;
+        escapeChar?: string;
+        dynamicTyping?: boolean;
+        comments?: boolean;
+        skipEmptyLines?: boolean | 'greedy';
+        delimitersToGuess?: string[];
+    };
+};
+export declare const CSVLoader: {
+    id: string;
+    module: string;
+    name: string;
+    version: any;
+    extensions: string[];
+    mimeTypes: string[];
+    category: string;
+    parse: (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions | undefined) => Promise<any>;
+    parseText: (text: string, options?: CSVLoaderOptions | undefined) => Promise<any>;
+    parseInBatches: typeof parseCSVInBatches;
+    options: CSVLoaderOptions;
+};
+declare function parseCSVInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: CSVLoaderOptions): AsyncIterable<Batch>;
+export declare const _typecheckCSVLoader: LoaderWithParser;
+export {};
+//# sourceMappingURL=csv-loader.d.ts.map

package/dist/csv-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../src/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAgB9C,oBAAY,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QAEJ,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,GAAG,gBAAgB,CAAC;QAClE,yDAAyD;QACzD,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;QAKhB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;QAEpC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAE9B,CAAC;CACH,CAAC;AAsBF,eAAO,MAAM,SAAS;;;;;;;;yBAQO,WAAW;sBAEpB,MAAM;;;CAKzB,CAAC;AAsDF,iBAAS,iBAAiB,CACxB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,gBAAgB,GACzB,aAAa,CAAC,KAAK,CAAC,CAoHtB;AAgFD,eAAO,MAAM,mBAAmB,EAAE,gBAA4B,CAAC"}

package/dist/csv-loader.js

@@ -1,240 +1,267 @@
-import { AsyncQueue, TableBatchBuilder, convertToArrayRow, convertToObjectRow } from '@loaders.gl/schema';
-import Papa from './libs/papaparse';
-import AsyncIteratorStreamer from './lib/async-iterator-streamer';
-const VERSION = typeof "3.1.0-alpha.2" !== 'undefined' ? "3.1.0-alpha.2" : 'latest';
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._typecheckCSVLoader = exports.CSVLoader = void 0;
+const schema_1 = require("@loaders.gl/schema");
+const papaparse_1 = __importDefault(require("./papaparse/papaparse"));
+const async_iterator_streamer_1 = __importDefault(require("./papaparse/async-iterator-streamer"));
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 const DEFAULT_CSV_LOADER_OPTIONS = {
-  csv: {
-    shape: 'object-row-table',
-    optimizeMemoryUsage: false,
-    header: 'auto',
-    columnPrefix: 'column',
-    quoteChar: '"',
-    escapeChar: '"',
-    dynamicTyping: true,
-    comments: false,
-    skipEmptyLines: true,
-    delimitersToGuess: [',', '\t', '|', ';']
-  }
+    csv: {
+        shape: 'object-row-table',
+        optimizeMemoryUsage: false,
+        // CSV options
+        header: 'auto',
+        columnPrefix: 'column',
+        // delimiter: auto
+        // newline: auto
+        quoteChar: '"',
+        escapeChar: '"',
+        dynamicTyping: true,
+        comments: false,
+        skipEmptyLines: true,
+        // transform: null?
+        delimitersToGuess: [',', '\t', '|', ';']
+        // fastMode: auto
+    }
 };
-export const CSVLoader = {
-  id: 'csv',
-  module: 'csv',
-  name: 'CSV',
-  version: VERSION,
-  extensions: ['csv'],
-  mimeTypes: ['text/csv'],
-  category: 'table',
-  parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
-  parseText: (text, options) => parseCSV(text, options),
-  parseInBatches: parseCSVInBatches,
-  options: DEFAULT_CSV_LOADER_OPTIONS
+exports.CSVLoader = {
+    id: 'csv',
+    module: 'csv',
+    name: 'CSV',
+    version: VERSION,
+    extensions: ['csv', 'tsv', 'dsv'],
+    mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+    category: 'table',
+    parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
+    parseText: (text, options) => parseCSV(text, options),
+    parseInBatches: parseCSVInBatches,
+    // @ts-ignore
+    // testText: null,
+    options: DEFAULT_CSV_LOADER_OPTIONS
 };
-
 async function parseCSV(csvText, options) {
-  const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv,
-    ...(options === null || options === void 0 ? void 0 : options.csv)
-  };
-  const firstRow = readFirstRow(csvText);
-  const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
-  const parseWithHeader = header;
-  const papaparseConfig = { ...csvOptions,
-    header: parseWithHeader,
-    download: false,
-    transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
-    error: e => {
-      throw new Error(e);
+    // Apps can call the parse method directly, we so apply default options here
+    const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv };
+    const firstRow = readFirstRow(csvText);
+    const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
+    const parseWithHeader = header;
+    const papaparseConfig = {
+        // dynamicTyping: true,
+        ...csvOptions,
+        header: parseWithHeader,
+        download: false,
+        transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
+        error: (e) => {
+            throw new Error(e);
+        }
+    };
+    const result = papaparse_1.default.parse(csvText, papaparseConfig);
+    let { data: rows } = result;
+    const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
+    switch (csvOptions.shape) {
+        case 'object-row-table':
+            rows = rows.map((row) => (Array.isArray(row) ? (0, schema_1.convertToObjectRow)(row, headerRow) : row));
+            break;
+        case 'array-row-table':
+            rows = rows.map((row) => (Array.isArray(row) ? row : (0, schema_1.convertToArrayRow)(row, headerRow)));
+            break;
+        default:
+    }
+    /*
+    if (!header && shape === 'object-row-table') {
+      // If the dataset has no header, transform the array result into an object shape with an
+      // autogenerated header
+      return result.data.map((row) =>
+        row.reduce((acc, value, i) => {
+          acc[headerRow[i]] = value;
+          return acc;
+        }, {})
+      );
     }
-  };
-  const result = Papa.parse(csvText, papaparseConfig);
-  let {
-    data: rows
-  } = result;
-  const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
-
-  switch (csvOptions.shape) {
-    case 'object-row-table':
-      rows = rows.map(row => Array.isArray(row) ? convertToObjectRow(row, headerRow) : row);
-      break;
-
-    case 'array-row-table':
-      rows = rows.map(row => Array.isArray(row) ? row : convertToArrayRow(row, headerRow));
-      break;
-
-    default:
-  }
-
-  return rows;
+    */
+    return rows;
 }
-
+// TODO - support batch size 0 = no batching/single batch?
 function parseCSVInBatches(asyncIterator, options) {
-  var _options;
-
-  options = { ...options
-  };
-
-  if (options.batchSize === 'auto') {
-    options.batchSize = 4000;
-  }
-
-  const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv,
-    ...((_options = options) === null || _options === void 0 ? void 0 : _options.csv)
-  };
-  const asyncQueue = new AsyncQueue();
-  let isFirstRow = true;
-  let headerRow = null;
-  let tableBatchBuilder = null;
-  let schema = null;
-  const config = { ...csvOptions,
-    header: false,
-    download: false,
-    chunkSize: 1024 * 1024 * 5,
-    skipEmptyLines: false,
-
-    step(results) {
-      let row = results.data;
-
-      if (csvOptions.skipEmptyLines) {
-        const collapsedRow = row.flat().join('').trim();
-
-        if (collapsedRow === '') {
-          return;
-        }
-      }
-
-      const bytesUsed = results.meta.cursor;
-
-      if (isFirstRow && !headerRow) {
-        const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
-
-        if (header) {
-          headerRow = row.map(duplicateColumnTransformer());
-          return;
-        }
-      }
-
-      if (isFirstRow) {
-        isFirstRow = false;
-
-        if (!headerRow) {
-          headerRow = generateHeader(csvOptions.columnPrefix, row.length);
-        }
-
-        schema = deduceSchema(row, headerRow);
-      }
-
-      if (csvOptions.optimizeMemoryUsage) {
-        row = JSON.parse(JSON.stringify(row));
-      }
-
-      tableBatchBuilder = tableBatchBuilder || new TableBatchBuilder(schema, {
-        shape: csvOptions.shape || 'array-row-table',
-        ...options
-      });
-
-      try {
-        tableBatchBuilder.addRow(row);
-        const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({
-          bytesUsed
-        });
-
-        if (batch) {
-          asyncQueue.enqueue(batch);
-        }
-      } catch (error) {
-        asyncQueue.enqueue(error);
-      }
-    },
-
-    complete(results) {
-      try {
-        const bytesUsed = results.meta.cursor;
-        const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({
-          bytesUsed
-        });
-
-        if (batch) {
-          asyncQueue.enqueue(batch);
-        }
-      } catch (error) {
-        asyncQueue.enqueue(error);
-      }
-
-      asyncQueue.close();
+    // Papaparse does not support standard batch size handling
+    // TODO - investigate papaparse chunks mode
+    options = { ...options };
+    if (options.batchSize === 'auto') {
+        options.batchSize = 4000;
     }
-
-  };
-  Papa.parse(asyncIterator, config, AsyncIteratorStreamer);
-  return asyncQueue;
+    // Apps can call the parse method directly, we so apply default options here
+    const csvOptions = { ...DEFAULT_CSV_LOADER_OPTIONS.csv, ...options?.csv };
+    const asyncQueue = new schema_1.AsyncQueue();
+    let isFirstRow = true;
+    let headerRow = null;
+    let tableBatchBuilder = null;
+    let schema = null;
+    const config = {
+        // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
+        ...csvOptions,
+        header: false,
+        download: false,
+        // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the
+        // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.
+        // See https://github.com/mholt/PapaParse/issues/465
+        chunkSize: 1024 * 1024 * 5,
+        // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true
+        // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if
+        // both of the skipEmptyLines and step callback options are provided:
+        // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465
+        // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825
+        skipEmptyLines: false,
+        // step is called on every row
+        // eslint-disable-next-line complexity
+        step(results) {
+            let row = results.data;
+            if (csvOptions.skipEmptyLines) {
+                // Manually reject lines that are empty
+                const collapsedRow = row.flat().join('').trim();
+                if (collapsedRow === '') {
+                    return;
+                }
+            }
+            const bytesUsed = results.meta.cursor;
+            // Check if we need to save a header row
+            if (isFirstRow && !headerRow) {
+                // Auto detects or can be forced with csvOptions.header
+                const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
+                if (header) {
+                    headerRow = row.map(duplicateColumnTransformer());
+                    return;
+                }
+            }
+            // If first data row, we can deduce the schema
+            if (isFirstRow) {
+                isFirstRow = false;
+                if (!headerRow) {
+                    headerRow = generateHeader(csvOptions.columnPrefix, row.length);
+                }
+                schema = deduceSchema(row, headerRow);
+            }
+            if (csvOptions.optimizeMemoryUsage) {
+                // A workaround to allocate new strings and don't retain pointers to original strings.
+                // https://bugs.chromium.org/p/v8/issues/detail?id=2869
+                row = JSON.parse(JSON.stringify(row));
+            }
+            // Add the row
+            tableBatchBuilder =
+                tableBatchBuilder ||
+                    new schema_1.TableBatchBuilder(schema, {
+                        // @ts-expect-error
+                        shape: csvOptions.shape || 'array-row-table',
+                        ...options
+                    });
+            try {
+                tableBatchBuilder.addRow(row);
+                // If a batch has been completed, emit it
+                const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({ bytesUsed });
+                if (batch) {
+                    asyncQueue.enqueue(batch);
+                }
+            }
+            catch (error) {
+                asyncQueue.enqueue(error);
+            }
+        },
+        // complete is called when all rows have been read
+        complete(results) {
+            try {
+                const bytesUsed = results.meta.cursor;
+                // Ensure any final (partial) batch gets emitted
+                const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({ bytesUsed });
+                if (batch) {
+                    asyncQueue.enqueue(batch);
+                }
+            }
+            catch (error) {
+                asyncQueue.enqueue(error);
+            }
+            asyncQueue.close();
+        }
+    };
+    papaparse_1.default.parse(asyncIterator, config, async_iterator_streamer_1.default);
+    // TODO - Does it matter if we return asyncIterable or asyncIterator
+    // return asyncQueue[Symbol.asyncIterator]();
+    return asyncQueue;
 }
-
+/**
+ * Checks if a certain row is a header row
+ * @param row the row to check
+ * @returns true if the row looks like a header
+ */
 function isHeaderRow(row) {
-  return row && row.every(value => typeof value === 'string');
+    return row && row.every((value) => typeof value === 'string');
 }
-
+/**
+ * Reads, parses, and returns the first row of a CSV text
+ * @param csvText the csv text to parse
+ * @returns the first row
+ */
 function readFirstRow(csvText) {
-  const result = Papa.parse(csvText, {
-    download: false,
-    dynamicTyping: true,
-    preview: 1
-  });
-  return result.data[0];
+    const result = papaparse_1.default.parse(csvText, {
+        download: false,
+        dynamicTyping: true,
+        preview: 1
+    });
+    return result.data[0];
 }
-
+/**
+ * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle
+ * duplicate header columns and would use the latest occurrence by default.
+ * See the header option in https://www.papaparse.com/docs#config
+ * @returns a transform function that returns sanitized names for duplicate fields
+ */
 function duplicateColumnTransformer() {
-  const observedColumns = new Set();
-  return col => {
-    let colName = col;
-    let counter = 1;
-
-    while (observedColumns.has(colName)) {
-      colName = `${col}.${counter}`;
-      counter++;
-    }
-
-    observedColumns.add(colName);
-    return colName;
-  };
+    const observedColumns = new Set();
+    return (col) => {
+        let colName = col;
+        let counter = 1;
+        while (observedColumns.has(colName)) {
+            colName = `${col}.${counter}`;
+            counter++;
+        }
+        observedColumns.add(colName);
+        return colName;
+    };
 }
-
+/**
+ * Generates the header of a CSV given a prefix and a column count
+ * @param columnPrefix the columnPrefix to use
+ * @param count the count of column names to generate
+ * @returns an array of column names
+ */
 function generateHeader(columnPrefix, count = 0) {
-  const headers = [];
-
-  for (let i = 0; i < count; i++) {
-    headers.push(`${columnPrefix}${i + 1}`);
-  }
-
-  return headers;
+    const headers = [];
+    for (let i = 0; i < count; i++) {
+        headers.push(`${columnPrefix}${i + 1}`);
+    }
+    return headers;
 }
-
 function deduceSchema(row, headerRow) {
-  const schema = headerRow ? {} : [];
-
-  for (let i = 0; i < row.length; i++) {
-    const columnName = headerRow && headerRow[i] || i;
-    const value = row[i];
-
-    switch (typeof value) {
-      case 'number':
-      case 'boolean':
-        schema[columnName] = {
-          name: String(columnName),
-          index: i,
-          type: Float32Array
-        };
-        break;
-
-      case 'string':
-      default:
-        schema[columnName] = {
-          name: String(columnName),
-          index: i,
-          type: Array
-        };
+    const schema = headerRow ? {} : [];
+    for (let i = 0; i < row.length; i++) {
+        const columnName = (headerRow && headerRow[i]) || i;
+        const value = row[i];
+        switch (typeof value) {
+            case 'number':
+            case 'boolean':
+                // TODO - booleans could be handled differently...
+                schema[columnName] = { name: String(columnName), index: i, type: Float32Array };
+                break;
+            case 'string':
+            default:
+                schema[columnName] = { name: String(columnName), index: i, type: Array };
+            // We currently only handle numeric rows
+            // TODO we could offer a function to map strings to numbers?
+        }
     }
-  }
-
-  return schema;
+    return schema;
 }
-
-export const _typecheckCSVLoader = CSVLoader;
-//# sourceMappingURL=csv-loader.js.map
+exports._typecheckCSVLoader = exports.CSVLoader;

package/dist/csv-writer.d.ts

@@ -0,0 +1 @@
+//# sourceMappingURL=csv-writer.d.ts.map

package/dist/csv-writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../src/csv-writer.ts"],"names":[],"mappings":""}

package/dist/csv-writer.js

@@ -1,2 +1,6 @@
-
-//# sourceMappingURL=csv-writer.js.map
+"use strict";
+/* @typedef {import('@loaders.gl/loader-utils').Writer} Writer */
+/*
+ * CSV exporter
+ * type {Writer}
+ */

package/dist/es5/bundle.js

@@ -0,0 +1,7 @@
+"use strict";
+
+const moduleExports = require('./index');
+
+globalThis.loaders = globalThis.loaders || {};
+module.exports = Object.assign(globalThis.loaders, moduleExports);
+//# sourceMappingURL=bundle.js.map

package/dist/es5/bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":";;AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}