@loaders.gl/csv 4.3.2 → 4.4.0-alpha.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/csv",
-  "version": "4.3.2",
+  "version": "4.4.0-alpha.1",
   "description": "Framework-independent loader for CSV and DSV table formats",
   "license": "MIT",
   "type": "module",
@@ -44,12 +44,12 @@
     "build-bundle-dev": "ocular-bundle ./bundle.ts --env=dev --output=dist/dist.dev.js"
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "4.3.2",
-    "@loaders.gl/schema": "4.3.2",
+    "@loaders.gl/loader-utils": "4.4.0-alpha.1",
+    "@loaders.gl/schema": "4.4.0-alpha.1",
     "d3-dsv": "^1.2.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "^4.3.0"
+    "@loaders.gl/core": "4.4.0-alpha.0"
   },
-  "gitHead": "b4203b8703f64a38d6f79a3113bc7bb51d45c93a"
+  "gitHead": "f1732de45907bd500bf4eedb4803beca8bf4bfb0"
 }

package/src/csv-arrow-loader.ts

@@ -0,0 +1,41 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
+import {convertTable, convertBatches} from '@loaders.gl/schema-utils';
+
+import type {CSVLoaderOptions} from './csv-loader';
+import {CSVLoader} from './csv-loader';
+
+export type CSVArrowLoaderOptions = LoaderOptions & {
+  csv?: Omit<CSVLoaderOptions['csv'], 'shape'>;
+};
+
+export const CSVArrowLoader = {
+  ...CSVLoader,
+
+  dataType: null as unknown as ArrowTable,
+  batchType: null as unknown as ArrowTableBatch,
+
+  parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) =>
+    parseCSVToArrow(new TextDecoder().decode(arrayBuffer), options),
+  parseText: (text: string, options?: CSVLoaderOptions) => parseCSVToArrow(text, options),
+  parseInBatches: parseCSVToArrowBatches
+} as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, CSVArrowLoaderOptions>;
+
+async function parseCSVToArrow(csvText: string, options?: CSVLoaderOptions): Promise<ArrowTable> {
+  // Apps can call the parse method directly, we so apply default options here
+  // const csvOptions = {...CSVArrowLoader.options.csv, ...options?.csv};
+  const table = await CSVLoader.parseText(csvText, options);
+  return convertTable(table, 'arrow-table');
+}
+
+function parseCSVToArrowBatches(
+  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  options?: CSVArrowLoaderOptions
+): AsyncIterable<ArrowTableBatch> {
+  const tableIterator = CSVLoader.parseInBatches(asyncIterator, options);
+  return convertBatches(tableIterator, 'arrow-table');
+}

package/src/csv-format.ts

@@ -0,0 +1,15 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/loader-utils';
+
+/** Comma-Separated Values */
+export const CSVFormat = {
+  id: 'csv',
+  module: 'csv',
+  name: 'CSV',
+  extensions: ['csv', 'tsv', 'dsv'],
+  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+  category: 'table'
+} as const satisfies Format;

package/src/csv-loader.ts

@@ -3,19 +3,19 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import type {ArrayRowTable, ObjectRowTable, TableBatch} from '@loaders.gl/schema';
+import type {Schema, ArrayRowTable, ObjectRowTable, TableBatch} from '@loaders.gl/schema';
 
+import {log} from '@loaders.gl/loader-utils';
 import {
   AsyncQueue,
+  deduceTableSchema,
   TableBatchBuilder,
   convertToArrayRow,
   convertToObjectRow
-} from '@loaders.gl/schema';
+} from '@loaders.gl/schema-utils';
 import Papa from './papaparse/papaparse';
 import AsyncIteratorStreamer from './papaparse/async-iterator-streamer';
-
-type ObjectField = {name: string; index: number; type: any};
-type ObjectSchema = {[key: string]: ObjectField} | ObjectField[];
+import {CSVFormat} from './csv-format';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -48,16 +48,11 @@ export type CSVLoaderOptions = LoaderOptions & {
 };
 
 export const CSVLoader = {
+  ...CSVFormat,
+
   dataType: null as unknown as ObjectRowTable | ArrayRowTable,
   batchType: null as unknown as TableBatch,
-
-  id: 'csv',
-  module: 'csv',
-  name: 'CSV',
   version: VERSION,
-  extensions: ['csv', 'tsv', 'dsv'],
-  mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
-  category: 'table',
   parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) =>
     parseCSV(new TextDecoder().decode(arrayBuffer), options),
   parseText: (text: string, options?: CSVLoaderOptions) => parseCSV(text, options),
@@ -89,7 +84,7 @@ async function parseCSV(
   csvText: string,
   options?: CSVLoaderOptions
 ): Promise<ObjectRowTable | ArrayRowTable> {
-  // Apps can call the parse method directly, we so apply default options here
+  // Apps can call the parse method directly, so we apply default options here
   const csvOptions = {...CSVLoader.options.csv, ...options?.csv};
 
   const firstRow = readFirstRow(csvText);
@@ -115,20 +110,25 @@ async function parseCSV(
   const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
 
   const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
+  let table: ArrayRowTable | ObjectRowTable;
   switch (shape) {
     case 'object-row-table':
-      return {
+      table = {
         shape: 'object-row-table',
         data: rows.map((row) => (Array.isArray(row) ? convertToObjectRow(row, headerRow) : row))
       };
+      break;
     case 'array-row-table':
-      return {
+      table = {
         shape: 'array-row-table',
         data: rows.map((row) => (Array.isArray(row) ? row : convertToArrayRow(row, headerRow)))
       };
+      break;
     default:
       throw new Error(shape);
   }
+  table.schema = deduceTableSchema(table!);
+  return table;
 }
 
 // TODO - support batch size 0 = no batching/single batch?
@@ -151,7 +151,7 @@ function parseCSVInBatches(
   let isFirstRow: boolean = true;
   let headerRow: string[] | null = null;
   let tableBatchBuilder: TableBatchBuilder | null = null;
-  let schema: ObjectSchema | null = null;
+  let schema: Schema | null = null;
 
   const config = {
     // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,
@@ -199,7 +199,7 @@ function parseCSVInBatches(
         if (!headerRow) {
           headerRow = generateHeader(csvOptions.columnPrefix, row.length);
         }
-        schema = deduceSchema(row, headerRow);
+        schema = deduceCSVSchema(row, headerRow);
       }
 
       if (csvOptions.optimizeMemoryUsage) {
@@ -314,23 +314,56 @@ function generateHeader(columnPrefix: string, count: number = 0): string[] {
   return headers;
 }
 
-function deduceSchema(row, headerRow): ObjectSchema {
-  const schema: ObjectSchema = headerRow ? {} : [];
+function deduceCSVSchema(row, headerRow): Schema {
+  const fields: Schema['fields'] = [];
   for (let i = 0; i < row.length; i++) {
     const columnName = (headerRow && headerRow[i]) || i;
     const value = row[i];
     switch (typeof value) {
       case 'number':
+        fields.push({name: String(columnName), type: 'float64', nullable: true});
+        break;
       case 'boolean':
-        // TODO - booleans could be handled differently...
-        schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
+        fields.push({name: String(columnName), type: 'bool', nullable: true});
         break;
       case 'string':
+        fields.push({name: String(columnName), type: 'utf8', nullable: true});
+        break;
       default:
-        schema[columnName] = {name: String(columnName), index: i, type: Array};
-      // We currently only handle numeric rows
-      // TODO we could offer a function to map strings to numbers?
+        log.warn(`CSV: Unknown column type: ${typeof value}`)();
+        fields.push({name: String(columnName), type: 'utf8', nullable: true});
     }
   }
-  return schema;
+  return {
+    fields,
+    metadata: {
+      'loaders.gl#format': 'csv',
+      'loaders.gl#loader': 'CSVLoader'
+    }
+  };
 }
+
+// TODO - remove
+// type ObjectField = {name: string; index: number; type: any};
+// type ObjectSchema = {[key: string]: ObjectField} | ObjectField[];
+
+// function deduceObjectSchema(row, headerRow): ObjectSchema {
+//   const schema: ObjectSchema = headerRow ? {} : [];
+//   for (let i = 0; i < row.length; i++) {
+//     const columnName = (headerRow && headerRow[i]) || i;
+//     const value = row[i];
+//     switch (typeof value) {
+//       case 'number':
+//       case 'boolean':
+//         // TODO - booleans could be handled differently...
+//         schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
+//         break;
+//       case 'string':
+//       default:
+//         schema[columnName] = {name: String(columnName), index: i, type: Array};
+//       // We currently only handle numeric rows
+//       // TODO we could offer a function to map strings to numbers?
+//     }
+//   }
+//   return schema;
+// }

package/src/csv-writer.ts

@@ -6,6 +6,7 @@
 import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 import {encodeTableAsCSV} from './lib/encoders/encode-csv';
+import {CSVFormat} from './csv-format';
 
 export type CSVWriterOptions = WriterOptions & {
   csv?: {
@@ -14,12 +15,8 @@ export type CSVWriterOptions = WriterOptions & {
 };
 
 export const CSVWriter = {
-  id: 'csv',
+  ...CSVFormat,
   version: 'latest',
-  module: 'csv',
-  name: 'CSV',
-  extensions: ['csv'],
-  mimeTypes: ['text/csv'],
   options: {
     csv: {
       useDisplayNames: false

package/src/index.ts

@@ -7,3 +7,6 @@ export {CSVLoader} from './csv-loader';
 
 export type {CSVWriterOptions} from './csv-writer';
 export {CSVWriter} from './csv-writer';
+
+export type {CSVArrowLoaderOptions} from './csv-arrow-loader';
+export {CSVArrowLoader} from './csv-arrow-loader';

package/src/lib/encoders/encode-csv.ts

@@ -3,7 +3,8 @@
 // Copyright (c) vis.gl contributors
 // Copyright 2022 Foursquare Labs, Inc.
 
-import {Table, makeArrayRowIterator, getTableNumCols} from '@loaders.gl/schema';
+import type {Table} from '@loaders.gl/schema';
+import {makeArrayRowIterator, getTableNumCols} from '@loaders.gl/schema-utils';
 import {csvFormatRows} from 'd3-dsv';
 import type {CSVWriterOptions} from '../../csv-writer';
 

package/src/papaparse/async-iterator-streamer.ts

@@ -1,11 +1,11 @@
-// @ts-nocheck
-// A custom papaparse `Streamer` for async iterators
-// Ideally this can be contributed back to papaparse
-// Or papaparse can expose Streamer API so we can extend without forking.
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+// Copyright (c) 2015 Matthew Holt
 
-/* eslint-disable no-invalid-this */
+// This is a fork of papaparse v5.0.0-beta.0 under MIT license
+// https://github.com/mholt/PapaParse
 
-// Note: papaparse is not an ES6 module
 import Papa from './papaparse';
 const {ChunkStreamer} = Papa;
 

package/src/papaparse/papa-constants.ts

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+// Copyright (c) 2015 Matthew Holt
+
+// This is a fork of papaparse v5.0.0-beta.0 under MIT license
+// https://github.com/mholt/PapaParse
+
+const BYTE_ORDER_MARK = '\ufeff';
+
+export const Papa = {
+  RECORD_SEP: String.fromCharCode(30),
+  UNIT_SEP: String.fromCharCode(31),
+  BYTE_ORDER_MARK,
+  BAD_DELIMITERS: ['\r', '\n', '"', BYTE_ORDER_MARK],
+  WORKERS_SUPPORTED: false, // !IS_WORKER && !!globalThis.Worker
+  NODE_STREAM_INPUT: 1,
+
+  // Configurable chunk sizes for local and remote files, respectively
+  LocalChunkSize: 1024 * 1024 * 10, // 10 M,
+  RemoteChunkSize: 1024 * 1024 * 5, // 5 M,
+  DefaultDelimiter: ',' // Used if not specified and detection fail,
+};