@loaders.gl/schema 4.0.0-alpha.24 → 4.0.0-alpha.25

package/src/lib/table/batches/row-table-batch-aggregator.ts

@@ -13,8 +13,8 @@ export class RowTableBatchAggregator implements TableBatchAggregator {
   options: TableBatchOptions;
 
   length: number = 0;
-  objectRows: {[columnName: string]: any} | null = null;
-  arrayRows: any[] | null = null;
+  objectRows: {[columnName: string]: unknown}[] | null = null;
+  arrayRows: unknown[][] | null = null;
   cursor: number = 0;
   private _headers: string[] = [];
 

package/src/lib/table/simple-table/table-accessors.ts

@@ -11,9 +11,11 @@ export function getTableLength(table: Table): number {
   switch (table.shape) {
     case 'array-row-table':
     case 'object-row-table':
-    case 'geojson-row-table':
       return table.data.length;
 
+    case 'geojson-table':
+      return table.features.length;
+
     case 'arrow-table':
       return table.data.numRows;
 
@@ -42,12 +44,11 @@ export function getTableNumCols(table: Table): number {
     case 'array-row-table':
       return table.data[0].length;
     case 'object-row-table':
-    case 'geojson-row-table':
       return Object.keys(table.data[0]).length;
-
+    case 'geojson-table':
+      return Object.keys(table.features[0]).length;
     case 'columnar-table':
       return Object.keys(table.data).length;
-
     case 'arrow-table':
       return table.data.numCols;
     default:
@@ -63,9 +64,11 @@ export function getTableCell(table: Table, rowIndex: number, columnName: string)
       return table.data[rowIndex][columnIndex];
 
     case 'object-row-table':
-    case 'geojson-row-table':
       return table.data[rowIndex][columnName];
 
+    case 'geojson-table':
+      return table.features[rowIndex][columnName];
+
     case 'columnar-table':
       const column = table.data[columnName];
       return column[rowIndex];
@@ -88,13 +91,16 @@ export function getTableCellAt(table: Table, rowIndex: number, columnIndex: numb
       return table.data[rowIndex][columnIndex];
 
     case 'object-row-table':
-    case 'geojson-row-table':
-      let columnName = getTableColumnName(table, columnIndex);
-      return table.data[rowIndex][columnName];
+      const columnName1 = getTableColumnName(table, columnIndex);
+      return table.data[rowIndex][columnName1];
+
+    case 'geojson-table':
+      const columnName2 = getTableColumnName(table, columnIndex);
+      return table.features[rowIndex][columnName2];
 
     case 'columnar-table':
-      columnName = getTableColumnName(table, columnIndex);
-      const column = table.data[columnName];
+      const columnName3 = getTableColumnName(table, columnIndex);
+      const column = table.data[columnName3];
       return column[rowIndex];
 
     case 'arrow-table':
@@ -112,7 +118,8 @@ export function getTableRowShape(table: Table): 'array-row-table' | 'object-row-
     case 'object-row-table':
       return table.shape;
 
-    case 'geojson-row-table':
+    case 'geojson-table':
+      // TODO - this is not correct, geojson-table is not a row table
       return 'object-row-table';
 
     case 'columnar-table':
@@ -156,7 +163,6 @@ export function getTableRowAsObject(
       return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
 
     case 'array-row-table':
-    case 'geojson-row-table':
       if (table.schema) {
         const objectRow: {[columnName: string]: unknown} = target || {};
         for (let i = 0; i < table.schema.fields.length; i++) {
@@ -166,6 +172,17 @@ export function getTableRowAsObject(
       }
       throw new Error('no schema');
 
+    case 'geojson-table':
+      if (table.schema) {
+        const objectRow: {[columnName: string]: unknown} = target || {};
+        // TODO - should lift properties to top level
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          objectRow[table.schema.fields[i].name] = table.features[rowIndex][i];
+        }
+        return objectRow;
+      }
+      throw new Error('no schema');
+
     case 'columnar-table':
       if (table.schema) {
         const objectRow: {[columnName: string]: unknown} = target || {};
@@ -214,7 +231,6 @@ export function getTableRowAsArray(
       return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
 
     case 'object-row-table':
-    case 'geojson-row-table':
       if (table.schema) {
         const arrayRow: unknown[] = target || [];
         for (let i = 0; i < table.schema.fields.length; i++) {
@@ -225,6 +241,18 @@ export function getTableRowAsArray(
       // Warning: just slap on the values, this risks mismatches between rows
       return Object.values(table.data[rowIndex]);
 
+    case 'geojson-table':
+      if (table.schema) {
+        const arrayRow: unknown[] = target || [];
+        // TODO - should lift properties to top level
+        for (let i = 0; i < table.schema.fields.length; i++) {
+          arrayRow[i] = table.features[rowIndex][table.schema.fields[i].name];
+        }
+        return arrayRow;
+      }
+      // Warning: just slap on the values, this risks mismatches between rows
+      return Object.values(table.features[rowIndex]);
+
     case 'columnar-table':
       if (table.schema) {
         const arrayRow: unknown[] = target || [];

package/src/types/batch.ts

@@ -0,0 +1,41 @@
+// loaders.gl, MIT license
+
+import {Schema} from './schema';
+import type {RecordBatch} from 'apache-arrow';
+
+/**
+ * A batch of data (or metadata/schema), from a streaming loader
+ * @see parseInBatches()
+ * @see loadInBatches()
+ */
+export type Batch = {
+  /** A batch can contain metadata, data, or in case of unstructured data (JSON) */
+  batchType: 'data' | 'metadata' | 'partial-result' | 'final-result';
+  /** A string identifying the shape of data in this batch (table, etc) */
+  shape: string;
+  /** Schema of the data in this batch */
+  schema?: Schema;
+  /** Data in this batch */
+  data: unknown;
+  /**  If this is an arrow table. @deprecated Use `data` */
+  recordBatch?: RecordBatch;
+  /** Length of data in this batch */
+  length: number;
+
+  /** A count of batches received */
+  batch?: number;
+
+  /** A count of batches received */
+  count?: number;
+
+  /** Bytes used so far */
+  bytesUsed?: number;
+  /** cursor is the  */
+  cursor?: number;
+
+  /** MIME type of the data generating this batch */
+  mimeType?: string;
+
+  /** Any other data */
+  [key: string]: unknown;
+};

package/src/types/category-table.ts

@@ -1,7 +1,8 @@
 // loaders.gl, MIT license
 
-import type {Table as ApacheArrowTable, RecordBatch} from 'apache-arrow';
-import type {Batch, Schema} from './schema';
+import type {Table as ApacheArrowTable} from 'apache-arrow';
+import type {Schema} from './schema';
+import type {Batch} from './batch';
 import type {Feature} from './category-gis';
 
 // Idea was to just import types, but it seems
@@ -15,12 +16,12 @@ export type Table =
   | RowTable
   | ArrayRowTable
   | ObjectRowTable
-  | GeoJSONRowTable
+  | GeoJSONTable
   | ColumnarTable
   | ArrowTable;
 
 /** A table organized as an array of rows */
-export type RowTable = ArrayRowTable | ObjectRowTable | GeoJSONRowTable;
+export type RowTable = ArrayRowTable | ObjectRowTable | GeoJSONTable;
 
 /** A table organized as an array of rows, each row is an array of values */
 export type ArrayRowTable = {
@@ -36,11 +37,17 @@ export type ObjectRowTable = {
   data: {[columnName: string]: any}[];
 };
 
-/** A table organized as an array of rows, each row is a GeoJSON Feature */
-export type GeoJSONRowTable = {
-  shape: 'geojson-row-table';
+/**
+ * A table organized as an array of rows, each row is a GeoJSON Feature
+ * @note For compatibility with GeoJSON, rows are stored in `table.features` instead of `table.data`
+ */
+export type GeoJSONTable = {
+  shape: 'geojson-table';
   schema?: Schema;
-  data: Feature[];
+  /** For compatibility with GeoJSON, the type field must always be set to `FeatureCollection` */
+  type: 'FeatureCollection';
+  /** For compatibility with GeoJSON, rows are stored in `table.features` instead of `table.data` */
+  features: Feature[];
 };
 
 /** A table organized as a map of columns, each column is an array of value */
@@ -69,7 +76,7 @@ export type Tables<TableType = Table> = {
 export type TableBatch =
   | ArrayRowTableBatch
   | ObjectRowTableBatch
-  | GeoJSONRowTableBatch
+  | GeoJSONTableBatch
   | ColumnarTableBatch
   | ArrowTableBatch;
 
@@ -92,11 +99,12 @@ export type ObjectRowTableBatch = Batch & {
 };
 
 /** Batch for a table organized as an array of rows, each row is an array of values */
-export type GeoJSONRowTableBatch = Batch & {
-  shape: 'geojson-row-table';
+export type GeoJSONTableBatch = Batch & {
+  shape: 'geojson-table';
   schema?: Schema;
   schemaType?: 'explicit' | 'deduced';
-  data: Feature[];
+  type: 'FeatureCollection';
+  features: Feature[];
   length: number;
 };
 
@@ -114,6 +122,7 @@ export type ArrowTableBatch = Batch & {
   shape: 'arrow-table';
   schemaType?: 'explicit' | 'deduced';
   schema?: Schema;
-  data: RecordBatch;
+  data: ApacheArrowTable;
+  // recordBatch: RecordBatch;
   length: number;
 };

package/src/types/schema.ts

@@ -1,7 +1,5 @@
 // loaders.gl, MIT license
 
-import type {RecordBatch} from 'apache-arrow';
-
 /** For dictionary type */
 export type KeyType = 'int8' | 'int16' | 'int32' | 'uint8' | 'uint16' | 'uint32';
 
@@ -80,18 +78,3 @@ export type Schema = {
   fields: Field[];
   metadata: SchemaMetadata;
 };
-
-export type Batch = {
-  batchType: 'data' | 'metadata' | 'partial-result' | 'final-result';
-  batch?: number;
-  mimeType?: string;
-  shape: string;
-  data: any;
-  recordBatch?: RecordBatch;
-  length: number;
-  schema?: Schema;
-  bytesUsed?: number;
-  count?: number;
-  cursor?: number;
-  [key: string]: any;
-};