@loaders.gl/csv 4.3.1 → 4.4.0-alpha.1

package/dist/csv-arrow-loader.d.ts

@@ -0,0 +1,37 @@
+import type { LoaderOptions } from '@loaders.gl/loader-utils';
+import type { ArrowTable, ArrowTableBatch } from '@loaders.gl/schema';
+import type { CSVLoaderOptions } from "./csv-loader.js";
+export type CSVArrowLoaderOptions = LoaderOptions & {
+    csv?: Omit<CSVLoaderOptions['csv'], 'shape'>;
+};
+export declare const CSVArrowLoader: {
+    readonly dataType: ArrowTable;
+    readonly batchType: ArrowTableBatch;
+    readonly parse: (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) => Promise<ArrowTable>;
+    readonly parseText: (text: string, options?: CSVLoaderOptions) => Promise<ArrowTable>;
+    readonly parseInBatches: typeof parseCSVToArrowBatches;
+    readonly version: any;
+    readonly options: {
+        readonly csv: {
+            readonly shape: "object-row-table";
+            readonly optimizeMemoryUsage: false;
+            readonly header: "auto";
+            readonly columnPrefix: "column";
+            readonly quoteChar: "\"";
+            readonly escapeChar: "\"";
+            readonly dynamicTyping: true;
+            readonly comments: false;
+            readonly skipEmptyLines: true;
+            readonly delimitersToGuess: [",", "\t", "|", ";"];
+        };
+    };
+    readonly id: "csv";
+    readonly module: "csv";
+    readonly name: "CSV";
+    readonly extensions: ["csv", "tsv", "dsv"];
+    readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
+    readonly category: "table";
+};
+declare function parseCSVToArrowBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: CSVArrowLoaderOptions): AsyncIterable<ArrowTableBatch>;
+export {};
+//# sourceMappingURL=csv-arrow-loader.d.ts.map

package/dist/csv-arrow-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-arrow-loader.d.ts","sourceRoot":"","sources":["../src/csv-arrow-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAGpE,OAAO,KAAK,EAAC,gBAAgB,EAAC,wBAAqB;AAGnD,MAAM,MAAM,qBAAqB,GAAG,aAAa,GAAG;IAClD,GAAG,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;CAC9C,CAAC;AAEF,eAAO,MAAM,cAAc;uBAGI,UAAU;wBACT,eAAe;kCAElB,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;CAEmC,CAAC;AAS1F,iBAAS,sBAAsB,CAC7B,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,qBAAqB,GAC9B,aAAa,CAAC,eAAe,CAAC,CAGhC"}

package/dist/csv-arrow-loader.js

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { convertTable, convertBatches } from '@loaders.gl/schema-utils';
+import { CSVLoader } from "./csv-loader.js";
+export const CSVArrowLoader = {
+    ...CSVLoader,
+    dataType: null,
+    batchType: null,
+    parse: async (arrayBuffer, options) => parseCSVToArrow(new TextDecoder().decode(arrayBuffer), options),
+    parseText: (text, options) => parseCSVToArrow(text, options),
+    parseInBatches: parseCSVToArrowBatches
+};
+async function parseCSVToArrow(csvText, options) {
+    // Apps can call the parse method directly, we so apply default options here
+    // const csvOptions = {...CSVArrowLoader.options.csv, ...options?.csv};
+    const table = await CSVLoader.parseText(csvText, options);
+    return convertTable(table, 'arrow-table');
+}
+function parseCSVToArrowBatches(asyncIterator, options) {
+    const tableIterator = CSVLoader.parseInBatches(asyncIterator, options);
+    return convertBatches(tableIterator, 'arrow-table');
+}

package/dist/csv-format.d.ts

@@ -0,0 +1,10 @@
+/** Comma-Separated Values */
+export declare const CSVFormat: {
+    readonly id: "csv";
+    readonly module: "csv";
+    readonly name: "CSV";
+    readonly extensions: ["csv", "tsv", "dsv"];
+    readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
+    readonly category: "table";
+};
+//# sourceMappingURL=csv-format.d.ts.map

package/dist/csv-format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-format.d.ts","sourceRoot":"","sources":["../src/csv-format.ts"],"names":[],"mappings":"AAMA,6BAA6B;AAC7B,eAAO,MAAM,SAAS;;;;;;;CAOK,CAAC"}

package/dist/csv-format.js

@@ -0,0 +1,12 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+/** Comma-Separated Values */
+export const CSVFormat = {
+    id: 'csv',
+    module: 'csv',
+    name: 'CSV',
+    extensions: ['csv', 'tsv', 'dsv'],
+    mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
+    category: 'table'
+};

package/dist/csv-loader.d.ts

@@ -18,13 +18,7 @@ export type CSVLoaderOptions = LoaderOptions & {
 export declare const CSVLoader: {
     readonly dataType: ObjectRowTable | ArrayRowTable;
     readonly batchType: TableBatch;
-    readonly id: "csv";
-    readonly module: "csv";
-    readonly name: "CSV";
     readonly version: any;
-    readonly extensions: ["csv", "tsv", "dsv"];
-    readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
-    readonly category: "table";
     readonly parse: (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) => Promise<ObjectRowTable | ArrayRowTable>;
     readonly parseText: (text: string, options?: CSVLoaderOptions) => Promise<ObjectRowTable | ArrayRowTable>;
     readonly parseInBatches: typeof parseCSVInBatches;
@@ -42,6 +36,12 @@ export declare const CSVLoader: {
             readonly delimitersToGuess: [",", "\t", "|", ";"];
         };
     };
+    readonly id: "csv";
+    readonly module: "csv";
+    readonly name: "CSV";
+    readonly extensions: ["csv", "tsv", "dsv"];
+    readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
+    readonly category: "table";
 };
 declare function parseCSVInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: CSVLoaderOptions): AsyncIterable<TableBatch>;
 export {};

package/dist/csv-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../src/csv-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAoBlF,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QAEJ,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;QAC/C,yDAAyD;QACzD,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;QAKhB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;QAEpC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAE9B,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,SAAS;uBACS,cAAc,GAAG,aAAa;wBAC7B,UAAU;;;;;;;;kCASb,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;CAuB6C,CAAC;AAiDpG,iBAAS,iBAAiB,CACxB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,gBAAgB,GACzB,aAAa,CAAC,UAAU,CAAC,CAyH3B"}
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../src/csv-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,KAAK,EAAS,aAAa,EAAE,cAAc,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAoB1F,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QAEJ,KAAK,CAAC,EAAE,iBAAiB,GAAG,kBAAkB,CAAC;QAC/C,yDAAyD;QACzD,mBAAmB,CAAC,EAAE,OAAO,CAAC;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;QAKhB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,cAAc,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;QAEpC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;KAE9B,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,SAAS;uBAGS,cAAc,GAAG,aAAa;wBAC7B,UAAU;;kCAEb,WAAW,YAAY,gBAAgB;+BAEhD,MAAM,YAAY,gBAAgB;;;;;;;;;;;;;;;;;;;;;;CAuB6C,CAAC;AAsDpG,iBAAS,iBAAiB,CACxB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,gBAAgB,GACzB,aAAa,CAAC,UAAU,CAAC,CAyH3B"}

package/dist/csv-loader.js

@@ -1,23 +1,20 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import { AsyncQueue, TableBatchBuilder, convertToArrayRow, convertToObjectRow } from '@loaders.gl/schema';
+import { log } from '@loaders.gl/loader-utils';
+import { AsyncQueue, deduceTableSchema, TableBatchBuilder, convertToArrayRow, convertToObjectRow } from '@loaders.gl/schema-utils';
 import Papa from "./papaparse/papaparse.js";
 import AsyncIteratorStreamer from "./papaparse/async-iterator-streamer.js";
+import { CSVFormat } from "./csv-format.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.3.0" !== 'undefined' ? "4.3.0" : 'latest';
+const VERSION = typeof "4.4.0-alpha.0" !== 'undefined' ? "4.4.0-alpha.0" : 'latest';
 const DEFAULT_CSV_SHAPE = 'object-row-table';
 export const CSVLoader = {
+    ...CSVFormat,
     dataType: null,
     batchType: null,
-    id: 'csv',
-    module: 'csv',
-    name: 'CSV',
     version: VERSION,
-    extensions: ['csv', 'tsv', 'dsv'],
-    mimeTypes: ['text/csv', 'text/tab-separated-values', 'text/dsv'],
-    category: 'table',
     parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
     parseText: (text, options) => parseCSV(text, options),
     parseInBatches: parseCSVInBatches,
@@ -44,7 +41,7 @@ export const CSVLoader = {
     }
 };
 async function parseCSV(csvText, options) {
-    // Apps can call the parse method directly, we so apply default options here
+    // Apps can call the parse method directly, so we apply default options here
     const csvOptions = { ...CSVLoader.options.csv, ...options?.csv };
     const firstRow = readFirstRow(csvText);
     const header = csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);
@@ -63,20 +60,25 @@ async function parseCSV(csvText, options) {
     const rows = result.data;
     const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);
     const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;
+    let table;
     switch (shape) {
         case 'object-row-table':
-            return {
+            table = {
                 shape: 'object-row-table',
                 data: rows.map((row) => (Array.isArray(row) ? convertToObjectRow(row, headerRow) : row))
             };
+            break;
         case 'array-row-table':
-            return {
+            table = {
                 shape: 'array-row-table',
                 data: rows.map((row) => (Array.isArray(row) ? row : convertToArrayRow(row, headerRow)))
             };
+            break;
         default:
             throw new Error(shape);
     }
+    table.schema = deduceTableSchema(table);
+    return table;
 }
 // TODO - support batch size 0 = no batching/single batch?
 function parseCSVInBatches(asyncIterator, options) {
@@ -135,7 +137,7 @@ function parseCSVInBatches(asyncIterator, options) {
                 if (!headerRow) {
                     headerRow = generateHeader(csvOptions.columnPrefix, row.length);
                 }
-                schema = deduceSchema(row, headerRow);
+                schema = deduceCSVSchema(row, headerRow);
             }
             if (csvOptions.optimizeMemoryUsage) {
                 // A workaround to allocate new strings and don't retain pointers to original strings.
@@ -237,23 +239,54 @@ function generateHeader(columnPrefix, count = 0) {
     }
     return headers;
 }
-function deduceSchema(row, headerRow) {
-    const schema = headerRow ? {} : [];
+function deduceCSVSchema(row, headerRow) {
+    const fields = [];
     for (let i = 0; i < row.length; i++) {
         const columnName = (headerRow && headerRow[i]) || i;
         const value = row[i];
         switch (typeof value) {
             case 'number':
+                fields.push({ name: String(columnName), type: 'float64', nullable: true });
+                break;
             case 'boolean':
-                // TODO - booleans could be handled differently...
-                schema[columnName] = { name: String(columnName), index: i, type: Float32Array };
+                fields.push({ name: String(columnName), type: 'bool', nullable: true });
                 break;
             case 'string':
+                fields.push({ name: String(columnName), type: 'utf8', nullable: true });
+                break;
             default:
-                schema[columnName] = { name: String(columnName), index: i, type: Array };
-            // We currently only handle numeric rows
-            // TODO we could offer a function to map strings to numbers?
+                log.warn(`CSV: Unknown column type: ${typeof value}`)();
+                fields.push({ name: String(columnName), type: 'utf8', nullable: true });
         }
     }
-    return schema;
+    return {
+        fields,
+        metadata: {
+            'loaders.gl#format': 'csv',
+            'loaders.gl#loader': 'CSVLoader'
+        }
+    };
 }
+// TODO - remove
+// type ObjectField = {name: string; index: number; type: any};
+// type ObjectSchema = {[key: string]: ObjectField} | ObjectField[];
+// function deduceObjectSchema(row, headerRow): ObjectSchema {
+//   const schema: ObjectSchema = headerRow ? {} : [];
+//   for (let i = 0; i < row.length; i++) {
+//     const columnName = (headerRow && headerRow[i]) || i;
+//     const value = row[i];
+//     switch (typeof value) {
+//       case 'number':
+//       case 'boolean':
+//         // TODO - booleans could be handled differently...
+//         schema[columnName] = {name: String(columnName), index: i, type: Float32Array};
+//         break;
+//       case 'string':
+//       default:
+//         schema[columnName] = {name: String(columnName), index: i, type: Array};
+//       // We currently only handle numeric rows
+//       // TODO we could offer a function to map strings to numbers?
+//     }
+//   }
+//   return schema;
+// }

package/dist/csv-writer.d.ts

@@ -6,12 +6,7 @@ export type CSVWriterOptions = WriterOptions & {
     };
 };
 export declare const CSVWriter: {
-    readonly id: "csv";
     readonly version: "latest";
-    readonly module: "csv";
-    readonly name: "CSV";
-    readonly extensions: ["csv"];
-    readonly mimeTypes: ["text/csv"];
     readonly options: {
         readonly csv: {
             readonly useDisplayNames: false;
@@ -20,5 +15,11 @@ export declare const CSVWriter: {
     readonly text: true;
     readonly encode: (table: Table, options: CSVWriterOptions | undefined) => Promise<ArrayBufferLike>;
     readonly encodeTextSync: (table: Table, options: CSVWriterOptions | undefined) => string;
+    readonly id: "csv";
+    readonly module: "csv";
+    readonly name: "CSV";
+    readonly extensions: ["csv", "tsv", "dsv"];
+    readonly mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"];
+    readonly category: "table";
 };
 //# sourceMappingURL=csv-writer.d.ts.map

package/dist/csv-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../src/csv-writer.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAoB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC/E,OAAO,KAAK,EAAC,KAAK,EAAa,MAAM,oBAAoB,CAAC;AAG1D,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QACJ,eAAe,CAAC,EAAE,OAAO,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;CAgBqD,CAAC"}
+{"version":3,"file":"csv-writer.d.ts","sourceRoot":"","sources":["../src/csv-writer.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAoB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC/E,OAAO,KAAK,EAAC,KAAK,EAAa,MAAM,oBAAoB,CAAC;AAI1D,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QACJ,eAAe,CAAC,EAAE,OAAO,CAAC;KAC3B,CAAC;CACH,CAAC;AAEF,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;CAYqD,CAAC"}

package/dist/csv-writer.js

@@ -2,13 +2,10 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 import { encodeTableAsCSV } from "./lib/encoders/encode-csv.js";
+import { CSVFormat } from "./csv-format.js";
 export const CSVWriter = {
-    id: 'csv',
+    ...CSVFormat,
     version: 'latest',
-    module: 'csv',
-    name: 'CSV',
-    extensions: ['csv'],
-    mimeTypes: ['text/csv'],
     options: {
         csv: {
             useDisplayNames: false