@loaders.gl/parquet 5.0.0-alpha.1 → 5.0.0-alpha.2

package/src/lib/encoders/encode-table-to-parquet-js.ts

@@ -16,7 +16,7 @@ export async function encodeTableToParquetJs(
   options: ParquetJSWriterOptions
 ): Promise<ArrayBuffer> {
   const parquetSchema = new ParquetSchema(
-    convertSchemaToParquetSchema(table.schema!, objectRowTable)
+    convertSchemaToParquetSchema(table.schema!, objectRowTable, options)
   );
   const chunks: Uint8Array[] = [];
   const outputStream = {
@@ -56,12 +56,29 @@ export async function encodeTableToParquetJs(
  */
 function convertSchemaToParquetSchema(
   schema: Schema,
-  objectRowTable: ObjectRowTable
+  objectRowTable: ObjectRowTable,
+  options: ParquetJSWriterOptions
 ): SchemaDefinition {
   const parquetFields: SchemaDefinition = {};
+  const columnEncodings = options.parquet?.columnEncodings || {};
+  const columnDictionaries = options.parquet?.columnDictionaries || {};
+  const fieldNames = new Set(schema.fields.map(field => field.name));
+  for (const columnName of Object.keys(columnEncodings)) {
+    if (!fieldNames.has(columnName)) {
+      throw new Error(`ParquetJSWriter: Unknown column encoding override "${columnName}"`);
+    }
+  }
+  for (const columnName of Object.keys(columnDictionaries)) {
+    if (!fieldNames.has(columnName)) {
+      throw new Error(`ParquetJSWriter: Unknown column dictionary override "${columnName}"`);
+    }
+  }
 
   for (const field of schema.fields) {
-    parquetFields[field.name] = convertFieldToParquetFieldDefinition(field, objectRowTable.data);
+    parquetFields[field.name] = {
+      ...convertFieldToParquetFieldDefinition(field, objectRowTable.data),
+      encoding: columnEncodings[field.name]
+    };
   }
 
   return parquetFields;
@@ -86,23 +103,37 @@ function convertFieldToParquetFieldDefinition(
       return {type: 'BOOLEAN', optional: nullable};
 
     case 'int':
+    case 'int32':
+      return {type: 'INT_32', optional: nullable};
+
     case 'int8':
+      return {type: 'INT_8', optional: nullable};
+
     case 'int16':
-    case 'int32':
+      return {type: 'INT_16', optional: nullable};
+
     case 'uint8':
+      return {type: 'UINT_8', optional: nullable};
+
     case 'uint16':
+      return {type: 'UINT_16', optional: nullable};
+
     case 'uint32':
-      return {type: 'INT32', optional: nullable};
+      return {type: 'UINT_32', optional: nullable};
 
     case 'int64':
+      return {type: 'INT_64', optional: nullable};
+
     case 'uint64':
-      return {type: 'INT64', optional: nullable};
+      return {type: 'UINT_64', optional: nullable};
 
     case 'float':
-    case 'float16':
     case 'float32':
       return {type: 'FLOAT', optional: nullable};
 
+    case 'float16':
+      return {type: 'FLOAT16', optional: nullable};
+
     case 'float64':
       return {type: 'DOUBLE', optional: nullable};
 
@@ -121,18 +152,40 @@ function convertFieldToParquetFieldDefinition(
       return {type: 'TIMESTAMP_MILLIS', optional: nullable};
 
     case 'timestamp-microsecond':
-    case 'timestamp-nanosecond':
       return {type: 'TIMESTAMP_MICROS', optional: nullable};
 
+    case 'timestamp-nanosecond':
+      return {type: 'TIMESTAMP_NANOS', optional: nullable};
+
     case 'time-second':
     case 'time-millisecond':
       return {type: 'TIME_MILLIS', optional: nullable};
 
     case 'time-microsecond':
-    case 'time-nanosecond':
       return {type: 'TIME_MICROS', optional: nullable};
 
+    case 'time-nanosecond':
+      return {type: 'TIME_NANOS', optional: nullable};
+
     default:
+      if (typeof dataType === 'object' && dataType.type === 'decimal') {
+        const physicalType =
+          dataType.precision <= 9
+            ? 'DECIMAL_INT32'
+            : dataType.precision <= 18
+              ? 'DECIMAL_INT64'
+              : 'DECIMAL_FIXED_LEN_BYTE_ARRAY';
+        return {
+          type: physicalType,
+          typeLength:
+            physicalType === 'DECIMAL_FIXED_LEN_BYTE_ARRAY'
+              ? getDecimalByteWidth(dataType.precision)
+              : undefined,
+          precision: dataType.precision,
+          scale: dataType.scale,
+          optional: nullable
+        };
+      }
       if (typeof dataType === 'object' && dataType.type === 'fixed-size-binary') {
         return {
           type: 'FIXED_LEN_BYTE_ARRAY',
@@ -147,6 +200,15 @@ function convertFieldToParquetFieldDefinition(
   }
 }
 
+/** Returns the minimum fixed byte width required by one decimal precision. */
+function getDecimalByteWidth(precision: number): number {
+  for (let byteWidth = 1; byteWidth <= 32; byteWidth++) {
+    const maximumPrecision = Math.floor(Math.log10(2 ** (8 * byteWidth - 1) - 1));
+    if (precision <= maximumPrecision) return byteWidth;
+  }
+  throw new Error(`ParquetJSWriter: Decimal precision ${precision} exceeds 32 bytes`);
+}
+
 /**
  * Finds the first non-nullish value in a column.
  * @param rows table rows

package/src/lib/geo/geospatial-metadata.ts

@@ -10,35 +10,22 @@ import {
   getMetadataValue,
   setGeoMetadata,
   setMetadataValue,
+  type GeoArrowEncoding,
+  type GeoArrowMetadata,
   type GeoColumnMetadata,
   type GeoMetadata,
   type GeoParquetGeometryType
 } from '@loaders.gl/gis';
+import {
+  GeoArrowMetadataSchema,
+  GeoParquetMetadataSchema
+} from '@loaders.gl/gis/geospatial-metadata-zod-schema';
 import {convertArrowToSchema} from '@loaders.gl/schema-utils';
 
 const GEOARROW_EXTENSION_NAME_KEY = 'ARROW:extension:name';
 const GEOARROW_EXTENSION_METADATA_KEY = 'ARROW:extension:metadata';
 const GEOPARQUET_VERSION = '1.1.0';
 
-type GeoArrowEncoding =
-  | 'geoarrow.geometry'
-  | 'geoarrow.geometrycollection'
-  | 'geoarrow.multipolygon'
-  | 'geoarrow.polygon'
-  | 'geoarrow.multilinestring'
-  | 'geoarrow.linestring'
-  | 'geoarrow.multipoint'
-  | 'geoarrow.point'
-  | 'geoarrow.wkb'
-  | 'geoarrow.wkt';
-
-type GeoArrowMetadata = {
-  encoding?: GeoArrowEncoding;
-  crs?: Record<string, unknown>;
-  edges?: 'spherical';
-  [key: string]: unknown;
-};
-
 const GEOARROW_ENCODINGS = [
   'geoarrow.geometry',
   'geoarrow.geometrycollection',
@@ -48,6 +35,7 @@ const GEOARROW_ENCODINGS = [
   'geoarrow.linestring',
   'geoarrow.multipoint',
   'geoarrow.point',
+  'geoarrow.box',
   'geoarrow.wkb',
   'geoarrow.wkt'
 ] as const satisfies GeoArrowEncoding[];
@@ -116,9 +104,10 @@ function getGeometryMetadataForField(
   const columnMetadata = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_METADATA_KEY);
   if (columnMetadata) {
     try {
+      const parsedMetadata = GeoArrowMetadataSchema.parse(JSON.parse(columnMetadata));
       metadata = {
         ...(metadata || {}),
-        ...(JSON.parse(columnMetadata) as GeoArrowMetadata)
+        ...parsedMetadata
       };
     } catch {
       return metadata;
@@ -354,51 +343,12 @@ function getGeoArrowMetadataFromGeoParquetField(
 }
 
 function isValidGeoParquetMetadata(geoMetadata: GeoMetadata | null): boolean {
-  if (!geoMetadata || typeof geoMetadata !== 'object') {
-    return false;
-  }
-
-  if (
-    typeof geoMetadata.version !== 'string' ||
-    typeof geoMetadata.primary_column !== 'string' ||
-    !geoMetadata.columns ||
-    typeof geoMetadata.columns !== 'object'
-  ) {
-    return false;
-  }
-
-  const primaryColumnMetadata = geoMetadata.columns[geoMetadata.primary_column];
-  if (!primaryColumnMetadata) {
-    return false;
-  }
-
-  return Object.values(geoMetadata.columns).every(isValidGeoParquetColumnMetadata);
-}
-
-function isValidGeoParquetColumnMetadata(columnMetadata: unknown): boolean {
-  if (!columnMetadata || typeof columnMetadata !== 'object') {
-    return false;
-  }
-
-  const {encoding, geometry_types} = columnMetadata as GeoColumnMetadata;
-  return (
-    typeof encoding === 'string' &&
-    Boolean(
-      GEOPARQUET_TO_GEOARROW_ENCODINGS[
-        encoding.toLowerCase() as keyof typeof GEOPARQUET_TO_GEOARROW_ENCODINGS
-      ]
-    ) &&
-    Array.isArray(geometry_types)
-  );
+  return GeoParquetMetadataSchema.safeParse(geoMetadata).success;
 }
 
-function synthesizeGeoParquetColumnMetadata(geometryMetadata: {
-  encoding?: GeoArrowEncoding;
-  crs?: Record<string, unknown>;
-  crs_type?: 'projjson' | 'wkt2:2019';
-  edges?: 'spherical';
-  geometry_types?: GeoParquetGeometryType[];
-}): GeoColumnMetadata | null {
+function synthesizeGeoParquetColumnMetadata(
+  geometryMetadata: GeoArrowMetadata
+): GeoColumnMetadata | null {
   const encoding = geometryMetadata.encoding
     ? GEOARROW_TO_GEOPARQUET_ENCODINGS[
         geometryMetadata.encoding as keyof typeof GEOARROW_TO_GEOPARQUET_ENCODINGS
@@ -418,10 +368,10 @@ function synthesizeGeoParquetColumnMetadata(geometryMetadata: {
     geometry_types: geometryTypes
   };
 
-  if (geometryMetadata.crs !== undefined) {
+  if (geometryMetadata.crs !== undefined && typeof geometryMetadata.crs !== 'string') {
     columnMetadata.crs = geometryMetadata.crs;
   }
-  if (geometryMetadata.crs_type !== undefined) {
+  if (geometryMetadata.crs_type === 'projjson' || geometryMetadata.crs_type === 'wkt2:2019') {
     columnMetadata.crs_type = geometryMetadata.crs_type;
   }
   if (geometryMetadata.edges === 'spherical') {