@loaders.gl/schema 4.0.0-alpha.4 → 4.0.0-alpha.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/schema",
-  "version": "4.0.0-alpha.4",
+  "version": "4.0.0-alpha.6",
   "description": "Table format APIs for JSON, CSV, etc...",
   "license": "MIT",
   "publishConfig": {
@@ -18,9 +18,9 @@
     "point cloud",
     "PLY"
   ],
-  "types": "src/index.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "files": [
     "src",
@@ -29,11 +29,10 @@
   ],
   "scripts": {
     "pre-build": "npm run build-bundle",
-    "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/bundle.js"
+    "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js"
   },
   "dependencies": {
-    "@types/geojson": "^7946.0.7",
-    "apache-arrow": "^4.0.0"
+    "@types/geojson": "^7946.0.7"
   },
-  "gitHead": "53026061b3c8871f7e96d3a5826125cc6613bddc"
+  "gitHead": "acc1985050dfaa0f1f0c066f8da5bce7454a046c"
 }

package/src/index.ts

@@ -1,32 +1,60 @@
 // COMMON CATEGORY
-export type {TypedArray, NumberArray, AnyArray} from './types';
+export type {
+  TypedArray,
+  TypedArrayConstructor,
+  NumberArray,
+  ArrayType,
+  AnyArray
+} from './types/types';
 
-export type {Batch} from './category/common';
+export type {Schema, Field, DataType, Batch, SchemaMetadata, FieldMetadata} from './types/schema';
 
 // TABLE CATEGORY TYPES
-
 export type {
   Table,
+  RowTable,
   ArrayRowTable,
   ObjectRowTable,
+  GeoJSONRowTable,
   ColumnarTable,
-  ArrowTable
-} from './category/table/table-types';
+  ArrowTable,
+  Tables
+} from './types/category-table';
 export type {
   TableBatch,
-  RowArrayTableBatch,
-  RowObjectTableBatch,
+  ArrayRowTableBatch,
+  ObjectRowTableBatch,
+  GeoJSONRowTableBatch,
   ColumnarTableBatch,
   ArrowTableBatch
-} from './category/table/table-types';
+} from './types/category-table';
 
 // TABLE CATEGORY UTILS
-export {default as TableBatchBuilder} from './lib/batches/table-batch-builder';
-export type {TableBatchAggregator} from './lib/batches/table-batch-aggregator';
-export {default as RowTableBatchAggregator} from './lib/batches/row-table-batch-aggregator';
-export {default as ColumnarTableBatchAggregator} from './lib/batches/columnar-table-batch-aggregator';
+export {TableBatchBuilder} from './lib/table/batches/table-batch-builder';
+export type {TableBatchAggregator} from './lib/table/batches/table-batch-aggregator';
+export {RowTableBatchAggregator} from './lib/table/batches/row-table-batch-aggregator';
+export {ColumnarTableBatchAggregator} from './lib/table/batches/columnar-table-batch-aggregator';
+
+export {
+  getTableLength,
+  getTableNumCols,
+  getTableCell,
+  getTableRowShape,
+  getTableColumnIndex,
+  getTableColumnName,
+  getTableRowAsObject,
+  getTableRowAsArray,
+  makeRowIterator,
+  makeArrayRowIterator,
+  makeObjectRowIterator
+} from './lib/table/simple-table/table-accessors';
+
+export {ArrowLikeTable} from './lib/table/arrow-api/arrow-like-table';
 
-export {convertToObjectRow, convertToArrayRow} from './lib/utils/row-utils';
+export {makeTableFromData} from './lib/table/simple-table/make-table';
+export {deduceTableSchema} from './lib/table/simple-table/table-schema';
+export {convertToObjectRow, convertToArrayRow} from './lib/table/simple-table/row-utils';
+export {getDataTypeFromArray} from './lib/table/simple-table/data-type';
 
 // MESH CATEGORY
 export type {
@@ -36,27 +64,53 @@ export type {
   MeshGeometry,
   MeshAttribute,
   MeshAttributes
-} from './category/mesh/mesh-types';
+} from './types/category-mesh';
 
-export {getMeshSize, getMeshBoundingBox} from './category/mesh/mesh-utils';
-export {convertMesh} from './category/mesh/convert-mesh';
+export {getMeshSize, getMeshBoundingBox} from './lib/mesh/mesh-utils';
+// Commented out due to https://github.com/visgl/deck.gl/issues/6906 and https://github.com/visgl/loaders.gl/issues/2177
+// export {convertMesh} from './category/mesh/convert-mesh';
 export {
   deduceMeshSchema,
   deduceMeshField,
   makeMeshAttributeMetadata
-} from './category/mesh/deduce-mesh-schema';
+} from './lib/mesh/deduce-mesh-schema';
+
+// TEXTURES
+export type {TextureLevel, GPUTextureFormat} from './types/category-texture';
+
+// IMAGES
+export type {ImageDataType, ImageType, ImageTypeEnum} from './types/category-image';
 
 // TYPES
 // GIS CATEGORY - GEOJSON
-export type {GeoJSON, Feature, Geometry, Position, GeoJsonProperties} from './category/gis';
 export type {
+  GeoJSON,
+  Feature,
+  FeatureCollection,
+  Geometry,
+  Position,
+  GeoJsonProperties,
   Point,
   MultiPoint,
   LineString,
   MultiLineString,
   Polygon,
-  MultiPolygon
-} from './category/gis';
+  MultiPolygon,
+  GeometryCollection
+} from './types/category-gis';
+
+export type {GeojsonGeometryInfo} from './types/category-gis';
+
+// GIS CATEGORY - FLAT GEOJSON
+export type {
+  FlatFeature,
+  FlatIndexedGeometry,
+  FlatGeometry,
+  FlatGeometryType,
+  FlatPoint,
+  FlatLineString,
+  FlatPolygon
+} from './types/category-gis';
 
 // GIS CATEGORY - BINARY
 export type {
@@ -66,19 +120,19 @@ export type {
   BinaryLineGeometry,
   BinaryPolygonGeometry,
   BinaryAttribute
-} from './category/gis';
+} from './types/category-gis';
 export type {
   BinaryFeatures,
   BinaryPointFeatures,
   BinaryLineFeatures,
   BinaryPolygonFeatures
-} from './category/gis';
+} from './types/category-gis';
 
 // SCHEMA
 export {
-  Schema,
-  Field,
-  DataType,
+  Schema as ArrowLikeSchema,
+  Field as ArrowLikeField,
+  DataType as ArrowLikeDataType,
   Null,
   Binary,
   Bool,
@@ -112,13 +166,12 @@ export {
   IntervalYearMonth,
   FixedSizeList,
   Struct
-} from './lib/schema';
+} from './lib/table/arrow-api';
 
 // EXPERIMENTAL APIs
 
 // SCHEMA UTILS
-export {deduceTypeFromColumn, deduceTypeFromValue} from './lib/schema-utils/deduce-column-type';
-export {getTypeInfo} from './lib/arrow/get-type-info';
-export {getArrowTypeFromTypedArray} from './lib/arrow/arrow-like-type-utils';
+export {getTypeInfo} from './lib/table/arrow-api/get-type-info';
+export {getArrowType} from './lib/table/arrow/arrow-type-utils';
 
 export {default as AsyncQueue} from './lib/utils/async-queue';

package/src/{category → lib}/mesh/convert-mesh.ts

@@ -1,6 +1,8 @@
-import type {Mesh} from './mesh-types';
-import type {ColumnarTable, ArrowTable} from '../table/table-types';
-import {convertMeshToArrowTable} from './mesh-to-arrow-table';
+// loaders.gl, MIT license
+
+import type {Mesh} from '../../types/category-mesh';
+import type {ColumnarTable, ArrowTable} from '../../types/category-table';
+// import {convertMeshToArrowTable} from './mesh-to-arrow-table';
 
 type TargetShape = 'mesh' | 'columnar-table' | 'arrow-table';
 
@@ -17,11 +19,11 @@ export function convertMesh(
       return mesh;
     case 'columnar-table':
       return convertMeshToColumnarTable(mesh);
-    case 'arrow-table':
-      return {
-        shape: 'arrow-table',
-        data: convertMeshToArrowTable(mesh)
-      };
+    // case 'arrow-table':
+    //   return {
+    //     shape: 'arrow-table',
+    //     data: convertMeshToArrowTable(mesh)
+    //   };
     default:
       throw new Error(`Unsupported shape ${options?.shape}`);
   }

package/src/{category → lib}/mesh/deduce-mesh-schema.ts

@@ -1,6 +1,8 @@
-import {MeshAttribute, MeshAttributes} from './mesh-types';
-import {Schema, Field, FixedSizeList} from '../../lib/schema';
-import {getArrowTypeFromTypedArray} from '../../lib/arrow/arrow-like-type-utils';
+// loaders.gl, MIT license
+
+import {MeshAttribute, MeshAttributes} from '../../types/category-mesh';
+import {Schema, Field} from '../../types/schema';
+import {getDataTypeFromTypedArray} from '../table/simple-table/data-type';
 
 /**
  * Create a schema for mesh attributes data
@@ -10,10 +12,10 @@ import {getArrowTypeFromTypedArray} from '../../lib/arrow/arrow-like-type-utils'
  */
 export function deduceMeshSchema(
   attributes: MeshAttributes,
-  metadata?: Map<string, string>
+  metadata: Record<string, string> = {}
 ): Schema {
   const fields = deduceMeshFields(attributes);
-  return new Schema(fields, metadata);
+  return {fields, metadata};
 }
 
 /**
@@ -24,19 +26,18 @@ export function deduceMeshSchema(
  * @returns
  */
 export function deduceMeshField(
-  attributeName: string,
+  name: string,
   attribute: MeshAttribute,
-  optionalMetadata?: Map<string, string>
+  optionalMetadata?: Record<string, string>
 ): Field {
-  const type = getArrowTypeFromTypedArray(attribute.value);
+  const type = getDataTypeFromTypedArray(attribute.value);
   const metadata = optionalMetadata ? optionalMetadata : makeMeshAttributeMetadata(attribute);
-  const field = new Field(
-    attributeName,
-    new FixedSizeList(attribute.size, new Field('value', type)),
-    false,
+  return {
+    name,
+    type: {type: 'fixed-size-list', listSize: attribute.size, children: [{name: 'value', type}]},
+    nullable: false,
     metadata
-  );
-  return field;
+  };
 }
 
 /**
@@ -58,16 +59,16 @@ function deduceMeshFields(attributes: MeshAttributes): Field[] {
  * @param attribute
  * @returns
  */
-export function makeMeshAttributeMetadata(attribute: MeshAttribute): Map<string, string> {
-  const result = new Map();
+export function makeMeshAttributeMetadata(attribute: MeshAttribute): Record<string, string> {
+  const result: Record<string, string> = {};
   if ('byteOffset' in attribute) {
-    result.set('byteOffset', attribute.byteOffset!.toString(10));
+    result.byteOffset = attribute.byteOffset!.toString(10);
   }
   if ('byteStride' in attribute) {
-    result.set('byteStride', attribute.byteStride!.toString(10));
+    result.byteStride = attribute.byteStride!.toString(10);
   }
   if ('normalized' in attribute) {
-    result.set('normalized', attribute.normalized!.toString());
+    result.normalized = attribute.normalized!.toString();
   }
   return result;
 }

package/src/{category → lib}/mesh/mesh-to-arrow-table.ts

@@ -1,3 +1,4 @@
+/* Problem with arrow dependency...
 import {
   Table,
   Schema,
@@ -6,9 +7,9 @@ import {
   Field,
   Data,
   FixedSizeListVector
-} from 'apache-arrow/Arrow.dom';
+} from 'apache-arrow';
 import {AbstractVector} from 'apache-arrow/vector';
-import {getArrowType, getArrowVector} from '../../lib/arrow/arrow-type-utils';
+import {getArrowType, getArrowVector} from '../table/arrow/arrow-type-utils';
 import type {Mesh} from './mesh-types';
 import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
 
@@ -18,7 +19,7 @@ import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
  * @param metadata
  * @param batchSize
  * @returns
- */
+ *
 export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): Table {
   const vectors: AbstractVector[] = [];
   const fields: Field[] = [];
@@ -39,3 +40,4 @@ export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): Table {
   const table = new Table(schema, recordBatch);
   return table;
 }
+*/

package/src/{category → lib}/mesh/mesh-utils.ts

@@ -1,7 +1,7 @@
 // Mesh category utilities
 // TODO - move to mesh category module, or to math.gl/geometry module
-import {TypedArray} from '../../types';
-import {MeshAttributes} from './mesh-types';
+import {TypedArray} from '../../types/types';
+import {MeshAttributes} from '../../types/category-mesh';
 
 type TypedArrays = {[key: string]: TypedArray};
 

package/src/lib/{arrow → table/arrow}/arrow-type-utils.ts

@@ -1,4 +1,6 @@
-import type {TypedArray} from '../../types';
+// loaders.gl, MIT license
+
+import type {TypedArray} from '../../../types/types';
 import {
   DataType,
   Float32,
@@ -8,17 +10,17 @@ import {
   Int8,
   Uint16,
   Uint32,
-  Uint8,
-  Int8Vector,
-  Uint8Vector,
-  Int16Vector,
-  Uint16Vector,
-  Int32Vector,
-  Uint32Vector,
-  Float32Vector,
-  Float64Vector
-} from 'apache-arrow/Arrow.dom';
-import {AbstractVector} from 'apache-arrow/vector';
+  Uint8
+  // Int8Vector,
+  // Uint8Vector,
+  // Int16Vector,
+  // Uint16Vector,
+  // Int32Vector,
+  // Uint32Vector,
+  // Float32Vector,
+  // Float64Vector
+} from 'apache-arrow';
+// import {AbstractVector} from 'apache-arrow/vector';
 
 export function getArrowType(array: TypedArray): DataType {
   switch (array.constructor) {
@@ -43,6 +45,7 @@ export function getArrowType(array: TypedArray): DataType {
   }
 }
 
+/*
 export function getArrowVector(array: TypedArray): AbstractVector {
   switch (array.constructor) {
     case Int8Array:
@@ -65,3 +68,4 @@ export function getArrowVector(array: TypedArray): AbstractVector {
       throw new Error('array type not supported');
   }
 }
+*/

package/src/lib/table/arrow/convert-schema-arrow.ts

@@ -0,0 +1,232 @@
+// loaders.gl, MIT license
+
+import type {DataType, Field, Schema, SchemaMetadata} from '../../../types/schema';
+import {
+  Field as ArrowField,
+  Schema as ArrowSchema,
+  DataType as ArrowDataType,
+  Null,
+  Binary,
+  Bool,
+  // Int,
+  Int8,
+  Int16,
+  Int32,
+  Int64,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  // Float,
+  Float16,
+  Float32,
+  Float64,
+  Utf8,
+  // Date,
+  DateDay,
+  DateMillisecond,
+  // Time,
+  TimeMillisecond,
+  TimeSecond,
+  // Timestamp,
+  TimestampSecond,
+  TimestampMillisecond,
+  TimestampMicrosecond,
+  TimestampNanosecond,
+  // Interval,
+  IntervalDayTime,
+  IntervalYearMonth,
+  FixedSizeList,
+  Struct
+} from 'apache-arrow/Arrow.dom';
+
+export function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {
+  return {
+    fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
+    metadata: serializeArrowMetadata(arrowSchema.metadata)
+  };
+}
+
+export function deserializeArrowSchema(schema: Schema): ArrowSchema {
+  return new ArrowSchema(
+    schema.fields.map((field) => deserializeArrowField(field)),
+    deserializeArrowMetadata(schema.metadata)
+  );
+}
+
+export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
+  return Object.fromEntries(arrowMetadata);
+}
+
+export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
+  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
+}
+
+export function serializeArrowField(field: ArrowField): Field {
+  return {
+    name: field.name,
+    type: serializeArrowType(field.type),
+    nullable: field.nullable,
+    metadata: serializeArrowMetadata(field.metadata)
+  };
+}
+
+export function deserializeArrowField(field: Field): ArrowField {
+  return new ArrowField(
+    field.name,
+    deserializeArrowType(field.type),
+    field.nullable,
+    deserializeArrowMetadata(field.metadata)
+  );
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function serializeArrowType(arrowType: ArrowDataType): DataType {
+  switch (arrowType.constructor) {
+    case Null:
+      return 'null';
+    case Binary:
+      return 'binary';
+    case Bool:
+      return 'bool';
+    // case Int: return 'int';
+    case Int8:
+      return 'int8';
+    case Int16:
+      return 'int16';
+    case Int32:
+      return 'int32';
+    case Int64:
+      return 'int64';
+    case Uint8:
+      return 'uint8';
+    case Uint16:
+      return 'uint16';
+    case Uint32:
+      return 'uint32';
+    case Uint64:
+      return 'uint64';
+    // case Float: return 'float';
+    case Float16:
+      return 'float16';
+    case Float32:
+      return 'float32';
+    case Float64:
+      return 'float64';
+    case Utf8:
+      return 'utf8';
+    // case Date: return 'date';
+    case DateDay:
+      return 'date-day';
+    case DateMillisecond:
+      return 'date-millisecond';
+    // case Time: return 'time';
+    case TimeMillisecond:
+      return 'time-millisecond';
+    case TimeSecond:
+      return 'time-second';
+    // case Timestamp: return 'timestamp';
+    case TimestampSecond:
+      return 'timestamp-second';
+    case TimestampMillisecond:
+      return 'timestamp-millisecond';
+    case TimestampMicrosecond:
+      return 'timestamp-microsecond';
+    case TimestampNanosecond:
+      return 'timestamp-nanosecond';
+    // case Interval: return 'interval';
+    case IntervalDayTime:
+      return 'interval-daytime';
+    case IntervalYearMonth:
+      return 'interval-yearmonth';
+    case FixedSizeList:
+      return {
+        type: 'fixed-size-list',
+        listSize: (arrowType as FixedSizeList).listSize,
+        children: [serializeArrowField((arrowType as FixedSizeList).children[0])]
+      };
+    // case Struct:
+    //   return {type: 'struct', children: (arrowType as Struct).children};
+    default:
+      throw new Error('array type not supported');
+  }
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function deserializeArrowType(dataType: DataType): ArrowDataType {
+  if (typeof dataType === 'object') {
+    switch (dataType.type) {
+      case 'fixed-size-list':
+        const child = deserializeArrowField(dataType.children[0]);
+        return new FixedSizeList(dataType.listSize, child);
+      case 'struct':
+        const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        return new Struct(children);
+      default:
+        throw new Error('array type not supported');
+    }
+  }
+
+  switch (dataType) {
+    case 'null':
+      return new Null();
+    case 'binary':
+      return new Binary();
+    case 'bool':
+      return new Bool();
+    // case 'int': return new Int();
+    case 'int8':
+      return new Int8();
+    case 'int16':
+      return new Int16();
+    case 'int32':
+      return new Int32();
+    case 'int64':
+      return new Int64();
+    case 'uint8':
+      return new Uint8();
+    case 'uint16':
+      return new Uint16();
+    case 'uint32':
+      return new Uint32();
+    case 'uint64':
+      return new Uint64();
+    // case 'float': return new Float();
+    case 'float16':
+      return new Float16();
+    case 'float32':
+      return new Float32();
+    case 'float64':
+      return new Float64();
+    case 'utf8':
+      return new Utf8();
+    // case 'date': return new Date();
+    case 'date-day':
+      return new DateDay();
+    case 'date-millisecond':
+      return new DateMillisecond();
+    // case 'time': return new Time();
+    case 'time-millisecond':
+      return new TimeMillisecond();
+    case 'time-second':
+      return new TimeSecond();
+    // case 'timestamp': return new Timestamp();
+    case 'timestamp-second':
+      return new TimestampSecond();
+    case 'timestamp-millisecond':
+      return new TimestampMillisecond();
+    case 'timestamp-microsecond':
+      return new TimestampMicrosecond();
+    case 'timestamp-nanosecond':
+      return new TimestampNanosecond();
+    // case 'interval': return new Interval();
+    case 'interval-daytime':
+      return new IntervalDayTime();
+    case 'interval-yearmonth':
+      return new IntervalYearMonth();
+    default:
+      throw new Error('array type not supported');
+  }
+}