@loaders.gl/csv 4.0.0-alpha.23 → 4.0.0-alpha.24

package/dist/src/csv-loader.js

@@ -1,258 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CSVLoader = void 0;
-const schema_1 = require("@loaders.gl/schema");
-const papaparse_1 = __importDefault(require("./papaparse/papaparse"));
-const async_iterator_streamer_1 = __importDefault(require("./papaparse/async-iterator-streamer"));
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-exports.CSVLoader = {
-    id: 'csv',
-    module: 'csv',
-    name: 'CSV',
-    version: VERSION,
-    extensions: ['csv', 'tsv', 'dsv'],
-    mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
-    category: 'table',
-    parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
-    parseText: (text, options) => parseCSV(text, options),
-    parseInBatches: parseCSVInBatches,
-    // @ts-ignore
-    // testText: null,
-    options: {
-        csv: {
-            shape: 'object-row-table',
-            optimizeMemoryUsage: false,
-            // CSV options
-            header: 'auto',
-            columnPrefix: 'column',
-            // delimiter: auto
-            // newline: auto
-            quoteChar: '"',
-            escapeChar: '"',
-            dynamicTyping: true,
-            comments: false,
-            skipEmptyLines: true,
-            // transform: null?
-            delimitersToGuess: [',', '\t', '|', ';']
-            // fastMode: auto
-        }
-    }
-};
-async function parseCSV(csvText, options) {
-    // Apps can call the parse method directly, we so apply default options here
-    const csvOptions = { ...exports.CSVLoader.options.csv, ...options?.csv };
-    const firstRow = readFirstRow(csvText);
-    const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
-    const parseWithHeader = header;
-    const papaparseConfig = {
-        // dynamicTyping: true,
-        ...csvOptions,
-        header: parseWithHeader,
-        download: false,
-        transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,
-        error: (e) => {
-            throw new Error(e);
-        }
-    };
-    const result = papaparse_1.default.parse(csvText, papaparseConfig);
-    const rows = result.data;
-    const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
-    switch (csvOptions.shape || 'object-row-table') {
-        case 'object-row-table':
-            return {
-                shape: 'object-row-table',
-                data: rows.map((row) => (Array.isArray(row) ? (0, schema_1.convertToObjectRow)(row, headerRow) : row))
-            };
-        case 'array-row-table':
-            return {
-                shape: 'array-row-table',
-                data: rows.map((row) => (Array.isArray(row) ? row : (0, schema_1.convertToArrayRow)(row, headerRow)))
-            };
-    }
-    throw new Error(csvOptions.shape);
-}
-// TODO - support batch size 0 = no batching/single batch?
-function parseCSVInBatches(asyncIterator, options) {
-    // Papaparse does not support standard batch size handling
-    // TODO - investigate papaparse chunks mode
-    options = { ...options };
-    if (options.batchSize === 'auto') {
-        options.batchSize = 4000;
-    }
-    // Apps can call the parse method directly, we so apply default options here
-    const csvOptions = { ...exports.CSVLoader.options.csv, ...options?.csv };
-    const asyncQueue = new schema_1.AsyncQueue();
-    let isFirstRow = true;
-    let headerRow = null;
-    let tableBatchBuilder = null;
-    let schema = null;
-    const config = {
-        // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
-        ...csvOptions,
-        header: false,
-        download: false,
-        // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the
-        // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.
-        // See https://github.com/mholt/PapaParse/issues/465
-        chunkSize: 1024 * 1024 * 5,
-        // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true
-        // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if
-        // both of the skipEmptyLines and step callback options are provided:
-        // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465
-        // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825
-        skipEmptyLines: false,
-        // step is called on every row
-        // eslint-disable-next-line complexity
-        step(results) {
-            let row = results.data;
-            if (csvOptions.skipEmptyLines) {
-                // Manually reject lines that are empty
-                const collapsedRow = row.flat().join('').trim();
-                if (collapsedRow === '') {
-                    return;
-                }
-            }
-            const bytesUsed = results.meta.cursor;
-            // Check if we need to save a header row
-            if (isFirstRow && !headerRow) {
-                // Auto detects or can be forced with csvOptions.header
-                const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);
-                if (header) {
-                    headerRow = row.map(duplicateColumnTransformer());
-                    return;
-                }
-            }
-            // If first data row, we can deduce the schema
-            if (isFirstRow) {
-                isFirstRow = false;
-                if (!headerRow) {
-                    headerRow = generateHeader(csvOptions.columnPrefix, row.length);
-                }
-                schema = deduceSchema(row, headerRow);
-            }
-            if (csvOptions.optimizeMemoryUsage) {
-                // A workaround to allocate new strings and don't retain pointers to original strings.
-                // https://bugs.chromium.org/p/v8/issues/detail?id=2869
-                row = JSON.parse(JSON.stringify(row));
-            }
-            // Add the row
-            tableBatchBuilder =
-                tableBatchBuilder ||
-                    new schema_1.TableBatchBuilder(
-                    // @ts-expect-error TODO this is not a proper schema
-                    schema, {
-                        shape: csvOptions.shape || 'array-row-table',
-                        ...options
-                    });
-            try {
-                tableBatchBuilder.addRow(row);
-                // If a batch has been completed, emit it
-                const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({ bytesUsed });
-                if (batch) {
-                    asyncQueue.enqueue(batch);
-                }
-            }
-            catch (error) {
-                asyncQueue.enqueue(error);
-            }
-        },
-        // complete is called when all rows have been read
-        complete(results) {
-            try {
-                const bytesUsed = results.meta.cursor;
-                // Ensure any final (partial) batch gets emitted
-                const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({ bytesUsed });
-                if (batch) {
-                    asyncQueue.enqueue(batch);
-                }
-            }
-            catch (error) {
-                asyncQueue.enqueue(error);
-            }
-            asyncQueue.close();
-        }
-    };
-    papaparse_1.default.parse(asyncIterator, config, async_iterator_streamer_1.default);
-    // TODO - Does it matter if we return asyncIterable or asyncIterator
-    // return asyncQueue[Symbol.asyncIterator]();
-    return asyncQueue;
-}
-/**
- * Checks if a certain row is a header row
- * @param row the row to check
- * @returns true if the row looks like a header
- */
-function isHeaderRow(row) {
-    return row && row.every((value) => typeof value === 'string');
-}
-/**
- * Reads, parses, and returns the first row of a CSV text
- * @param csvText the csv text to parse
- * @returns the first row
- */
-function readFirstRow(csvText) {
-    const result = papaparse_1.default.parse(csvText, {
-        download: false,
-        dynamicTyping: true,
-        preview: 1
-    });
-    return result.data[0];
-}
-/**
- * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle
- * duplicate header columns and would use the latest occurrence by default.
- * See the header option in https://www.papaparse.com/docs#config
- * @returns a transform function that returns sanitized names for duplicate fields
- */
-function duplicateColumnTransformer() {
-    const observedColumns = new Set();
-    return (col) => {
-        let colName = col;
-        let counter = 1;
-        while (observedColumns.has(colName)) {
-            colName = `${col}.${counter}`;
-            counter++;
-        }
-        observedColumns.add(colName);
-        return colName;
-    };
-}
-/**
- * Generates the header of a CSV given a prefix and a column count
- * @param columnPrefix the columnPrefix to use
- * @param count the count of column names to generate
- * @returns an array of column names
- */
-function generateHeader(columnPrefix, count = 0) {
-    const headers = [];
-    for (let i = 0; i < count; i++) {
-        headers.push(`${columnPrefix}${i + 1}`);
-    }
-    return headers;
-}
-function deduceSchema(row, headerRow) {
-    const schema = headerRow ? {} : [];
-    for (let i = 0; i < row.length; i++) {
-        const columnName = (headerRow && headerRow[i]) || i;
-        const value = row[i];
-        switch (typeof value) {
-            case 'number':
-            case 'boolean':
-                // TODO - booleans could be handled differently...
-                schema[columnName] = { name: String(columnName), index: i, type: Float32Array };
-                break;
-            case 'string':
-            default:
-                schema[columnName] = { name: String(columnName), index: i, type: Array };
-            // We currently only handle numeric rows
-            // TODO we could offer a function to map strings to numbers?
-        }
-    }
-    return schema;
-}

package/dist/src/csv-writer.js

@@ -1,23 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CSVWriter = void 0;
-const encode_csv_1 = require("./lib/encoders/encode-csv");
-const DEFAULT_WRITER_OPTIONS = {
-    csv: {
-        useDisplayNames: false
-    },
-    useDisplayNames: false
-};
-exports.CSVWriter = {
-    id: 'csv',
-    version: 'latest',
-    module: 'csv',
-    name: 'CSV',
-    extensions: ['csv'],
-    mimeTypes: ['text/csv'],
-    options: DEFAULT_WRITER_OPTIONS,
-    text: true,
-    encode: async (table, options) => new TextEncoder().encode((0, encode_csv_1.encodeTableAsCSV)(table, options)).buffer,
-    encodeText: (table, options) => (0, encode_csv_1.encodeTableAsCSV)(table, options)
-};

package/dist/src/index.js

@@ -1,8 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CSVWriter = exports.CSVLoader = void 0;
-var csv_loader_1 = require("./csv-loader");
-Object.defineProperty(exports, "CSVLoader", { enumerable: true, get: function () { return csv_loader_1.CSVLoader; } });
-var csv_writer_1 = require("./csv-writer");
-Object.defineProperty(exports, "CSVWriter", { enumerable: true, get: function () { return csv_writer_1.CSVWriter; } });

package/dist/src/lib/encoders/encode-csv.js

@@ -1,50 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-// Copyright 2022 Foursquare Labs, Inc.
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encodeTableAsCSV = void 0;
-const schema_1 = require("@loaders.gl/schema");
-const d3_dsv_1 = require("d3-dsv");
-/**
- * Encode a Table object as CSV
- */
-function encodeTableAsCSV(table, options = { csv: { useDisplayNames: true } }) {
-    const useDisplayNames = options.useDisplayNames || options.csv?.useDisplayNames;
-    const fields = table.schema?.fields || [];
-    const columnNames = fields.map((f) => {
-        // This is a leaky abstraction, assuming Kepler metadata
-        const displayName = f.metadata?.displayName;
-        return useDisplayNames && typeof displayName === 'string' ? displayName : f.name;
-    });
-    const formattedData = [columnNames];
-    for (const row of (0, schema_1.makeArrayRowIterator)(table)) {
-        const formattedRow = [];
-        for (let columnIndex = 0; columnIndex < (0, schema_1.getTableNumCols)(table); ++columnIndex) {
-            const value = row[columnIndex];
-            formattedRow[columnIndex] = preformatFieldValue(value);
-        }
-        formattedData.push(formattedRow);
-    }
-    return (0, d3_dsv_1.csvFormatRows)(formattedData);
-}
-exports.encodeTableAsCSV = encodeTableAsCSV;
-/**
- * Stringifies a value
- * @todo Why is it called parse?
- */
-const preformatFieldValue = (value) => {
-    if (value === null || value === undefined) {
-        // TODO: It would be nice to distinguish between missing values and the empty string
-        // https://github.com/d3/d3-dsv/issues/84
-        return null;
-    }
-    if (value instanceof Date) {
-        // d3-dsv formats dates without timezones if they don't have time info;
-        // this forces them to always use fully-qualified ISO time strings
-        return value.toISOString();
-    }
-    if (typeof value === 'object') {
-        return JSON.stringify(value);
-    }
-    return String(value);
-};

package/dist/src/papaparse/async-iterator-streamer.js

@@ -1,63 +0,0 @@
-"use strict";
-// @ts-nocheck
-// A custom papaparse `Streamer` for async iterators
-// Ideally this can be contributed back to papaparse
-// Or papaparse can expose Streamer API so we can extend without forking.
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-/* eslint-disable no-invalid-this */
-// Note: papaparse is not an ES6 module
-const papaparse_1 = __importDefault(require("./papaparse"));
-const { ChunkStreamer } = papaparse_1.default;
-function AsyncIteratorStreamer(config) {
-    config = config || {};
-    ChunkStreamer.call(this, config);
-    this.textDecoder = new TextDecoder(this._config.encoding);
-    // Implement ChunkStreamer base class methods
-    // this.pause = function() {
-    //   ChunkStreamer.prototype.pause.apply(this, arguments);
-    // };
-    // this.resume = function() {
-    //   ChunkStreamer.prototype.resume.apply(this, arguments);
-    //   this._input.resume();
-    // };
-    this.stream = async function (asyncIterator) {
-        this._input = asyncIterator;
-        try {
-            // ES2018 version
-            // TODO - check for pause and abort flags?
-            for await (const chunk of asyncIterator) {
-                this.parseChunk(this.getStringChunk(chunk));
-            }
-            // ES5 VERSION
-            // while (true) {
-            //   asyncIterator.next().then(function(value) {
-            //     if (value.done) {
-            //       // finalize iterator?
-            //     }
-            //   }
-            //   const  = await ;
-            //   if (done) return total;
-            //   total += value.length;
-            // }
-            this._finished = true;
-            this.parseChunk('');
-        }
-        catch (error) {
-            // Inform ChunkStreamer base class of error
-            this._sendError(error);
-        }
-    };
-    this._nextChunk = function nextChunk() {
-        // Left empty, as async iterator automatically pulls next chunk
-    };
-    // HELPER METHODS
-    this.getStringChunk = function (chunk) {
-        return typeof chunk === 'string' ? chunk : this.textDecoder.decode(chunk, { stream: true });
-    };
-}
-exports.default = AsyncIteratorStreamer;
-AsyncIteratorStreamer.prototype = Object.create(ChunkStreamer.prototype);
-AsyncIteratorStreamer.prototype.constructor = AsyncIteratorStreamer;