@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/src/index.ts

@@ -2,33 +2,62 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-export {Buffer} from './polyfills/buffer/install-buffer-polyfill';
-
 // import {ArrowTable, ArrowTableBatch} from '@loaders.gl/arrow';
 
 export {ParquetFormat} from './parquet-format';
 
-export {
-  ParquetJSONWorkerLoader,
-  ParquetJSONLoader,
-  GeoParquetWorkerLoader,
-  GeoParquetLoader,
-  // deprecated
-  ParquetJSONWorkerLoader as ParquetWorkerLoader,
-  ParquetJSONLoader as ParquetLoader
-} from './parquet-json-loader';
+export type {ParquetLoaderOptions, ParquetJSLoaderOptions} from './parquet-loader-options';
+export {ParquetLoader} from './parquet-loader-types';
+export {GeoParquetLoader} from './geoparquet-loader';
+export {ParquetJSLoader} from './parquet-js-loader-types';
 
+export {ParquetSourceLoader} from './parquet-source-loader-types';
+export type {
+  ParquetBatch,
+  ParquetBatchMetadata,
+  ParquetBatchProvenance,
+  ParquetColumnChunkMetadata,
+  ParquetColumnChunkStatistics,
+  ParquetDatasetBatch,
+  ParquetDatasetBatchProvenance,
+  ParquetDatasetBoundingBox,
+  ParquetDatasetFile,
+  ParquetDatasetFileCollection,
+  ParquetDatasetFileProvider,
+  ParquetDatasetFileQuery,
+  ParquetDatasetFiles,
+  ParquetDatasetPartitionValue,
+  ParquetDatasetReadOptions,
+  ParquetDatasetSourceOptions,
+  ParquetDatasetTelemetry,
+  ParquetMetadataRequestOptions,
+  ParquetObjectVersion,
+  ParquetComparisonPredicate,
+  ParquetInPredicate,
+  ParquetLogicalPredicate,
+  ParquetNotPredicate,
+  ParquetNullPredicate,
+  ParquetPredicate,
+  ParquetPredicateProperty,
+  ParquetPredicateValue,
+  ParquetRangeRequestOptions,
+  ParquetReadOptions,
+  ParquetRowGroupMetadata,
+  ParquetSourceBatch,
+  ParquetSourceLoaderOptions,
+  ParquetSourceMetadata,
+  ParquetSourceReadOptions,
+  ParquetTelemetry,
+  ParquetTelemetryEvent
+} from './parquet-source-types';
 export {
-  ParquetJSONWriter as _ParquetJSONWriter,
-  // deprecated
-  ParquetJSONWriter as ParquetWriter
-} from './parquet-json-writer';
+  PARQUET_SOURCE_CAPABILITIES,
+  type ParquetSourceCapabilities
+} from './parquet-source-capabilities';
 
-// EXPERIMENTAL - expose Parquet WASM loaders/writer
-
-export type {ParquetArrowLoaderOptions} from './parquet-arrow-loader';
-export {ParquetArrowLoader, ParquetArrowWorkerLoader} from './parquet-arrow-loader';
-export {ParquetArrowWriter} from './parquet-arrow-writer';
+export {ParquetWriter} from './parquet-writer';
+export type {ParquetJSWriterEncoding, ParquetJSWriterOptions} from './parquet-js-writer';
+export {ParquetJSWriter} from './parquet-js-writer';
 
 // EXPERIMENTAL - expose the internal parquetjs API
 
@@ -42,6 +71,3 @@ export {
   convertParquetSchema,
   convertParquetSchema as convertParquetToArrowSchema
 } from './lib/arrow/convert-schema-from-parquet';
-
-// Experimental
-export {BufferPolyfill, installBufferPolyfill} from './polyfills/buffer/index';

package/src/lib/arrow/convert-columns-to-row-group.ts

@@ -0,0 +1,3 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors

package/src/lib/arrow/convert-schema-from-parquet.ts

@@ -5,25 +5,28 @@
 import {Schema, Field, DataType} from '@loaders.gl/schema';
 
 import type {ParquetSchema} from '../../parquetjs/schema/schema';
-import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';
+import type {FieldDefinition, ParquetType, SchemaDefinition} from '../../parquetjs/schema/declare';
 import {FileMetaData} from '../../parquetjs/parquet-thrift';
 
 export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
   BOOLEAN: 'bool',
   INT32: 'int32',
-  INT64: 'float64',
+  INT64: 'int64',
   INT96: 'float64',
   FLOAT: 'float32',
   DOUBLE: 'float64',
   BYTE_ARRAY: 'binary',
   FIXED_LEN_BYTE_ARRAY: 'binary',
   UTF8: 'utf8',
-  DATE: 'int32',
-  TIME_MILLIS: 'int64',
-  TIME_MICROS: 'int64',
-  TIMESTAMP_MILLIS: 'int64',
-  TIMESTAMP_MICROS: 'int64',
-  UINT_8: 'int32',
+  ENUM: 'utf8',
+  DATE: 'date-day',
+  TIME_MILLIS: 'time-millisecond',
+  TIME_MICROS: 'time-microsecond',
+  TIME_NANOS: 'time-nanosecond',
+  TIMESTAMP_MILLIS: 'timestamp-millisecond',
+  TIMESTAMP_MICROS: 'timestamp-microsecond',
+  TIMESTAMP_NANOS: 'timestamp-nanosecond',
+  UINT_8: 'uint8',
   UINT_16: 'uint16',
   UINT_32: 'uint32',
   UINT_64: 'uint64',
@@ -31,14 +34,20 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
   INT_16: 'int16',
   INT_32: 'int32',
   INT_64: 'int64',
+  UUID: {type: 'fixed-size-binary', byteWidth: 16},
+  FLOAT16: 'float16',
+  UNKNOWN: 'null',
+  VARIANT: 'binary',
+  GEOMETRY: 'binary',
+  GEOGRAPHY: 'binary',
   JSON: 'binary',
   BSON: 'binary',
   // TODO check interal type
   INTERVAL: 'binary',
-  DECIMAL_INT32: 'float32',
-  DECIMAL_INT64: 'float64',
-  DECIMAL_BYTE_ARRAY: 'float64',
-  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+  DECIMAL_INT32: {type: 'decimal', bitWidth: 128, precision: 38, scale: 0},
+  DECIMAL_INT64: {type: 'decimal', bitWidth: 128, precision: 38, scale: 0},
+  DECIMAL_BYTE_ARRAY: {type: 'decimal', bitWidth: 128, precision: 38, scale: 0},
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: {type: 'decimal', bitWidth: 128, precision: 38, scale: 0}
 };
 
 export function convertParquetSchema(
@@ -56,36 +65,94 @@ export function convertParquetSchema(
   return schema;
 }
 
-function getFields(schema: FieldDefinition): Field[] {
+function getFields(schema: SchemaDefinition): Field[] {
   const fields: Field[] = [];
 
   for (const name in schema) {
-    const field = schema[name];
-
-    if (field.fields) {
-      const children = getFields(field.fields);
-      fields.push({name, type: {type: 'struct', children}, nullable: field.optional});
-    } else {
-      const type = PARQUET_TYPE_MAPPING[field.type];
-      const metadata = getFieldMetadata(field);
-      const arrowField = {name, type, nullable: field.optional, metadata};
-      fields.push(arrowField);
-    }
+    fields.push(getField(name, schema[name]));
   }
 
   return fields;
 }
 
-function getFieldMetadata(field: ParquetField): Record<string, string> | undefined {
+/** Converts one Parquet field, preserving repeated values as Arrow lists. */
+function getField(name: string, field: FieldDefinition): Field {
+  const elementField: Field = field.fields
+    ? {
+        name,
+        type: {type: 'struct', children: getFields(field.fields)},
+        nullable: Boolean(field.optional),
+        metadata: getFieldMetadata(field)
+      }
+    : {
+        name,
+        type: getFieldType(field),
+        nullable: Boolean(field.optional),
+        metadata: getFieldMetadata(field)
+      };
+
+  if (!field.repeated) {
+    return elementField;
+  }
+
+  return {
+    name,
+    type: {
+      type: 'list',
+      children: [{...elementField, name: 'element', nullable: false}]
+    },
+    nullable: Boolean(field.optional),
+    metadata: elementField.metadata
+  };
+}
+
+/** Returns the exact serialized Arrow type for one decoded Parquet field. */
+function getFieldType(field: FieldDefinition): DataType {
+  if (field.logicalType?.type === 'DECIMAL') {
+    const precision = field.precision ?? field.presision;
+    const scale = field.scale;
+    if (!precision || scale === undefined) {
+      throw new Error('Parquet DECIMAL logical type requires precision and scale');
+    }
+    if (precision > 76) {
+      throw new Error(`Parquet DECIMAL precision ${precision} exceeds Arrow Decimal256`);
+    }
+    return {type: 'decimal', bitWidth: precision <= 38 ? 128 : 256, precision, scale};
+  }
+  if (field.logicalType?.type === 'UUID') {
+    return {type: 'fixed-size-binary', byteWidth: field.typeLength || 16};
+  }
+  return PARQUET_TYPE_MAPPING[field.type!];
+}
+
+/** Returns defined physical field properties as Arrow-compatible string metadata. */
+function getFieldMetadata(field: FieldDefinition): Record<string, string> | undefined {
   let metadata: Record<string, string> | undefined;
 
   for (const key in field) {
-    if (key !== 'name') {
-      let value = field[key] || '';
-      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-      metadata = metadata || {};
-      metadata[key] = value;
+    const fieldValue = field[key];
+    if (
+      key === 'name' ||
+      key === 'fields' ||
+      (key === 'physicalType' && fieldValue === field.type) ||
+      (key === 'presision' && field.precision !== undefined) ||
+      fieldValue === undefined
+    ) {
+      continue;
+    }
+    const metadataValue =
+      typeof fieldValue === 'string'
+        ? fieldValue
+        : typeof fieldValue === 'boolean'
+          ? fieldValue
+            ? 'true'
+            : 'false'
+          : JSON.stringify(fieldValue);
+    if (metadataValue === undefined) {
+      continue;
     }
+    metadata ||= {};
+    metadata[key] = metadataValue;
   }
 
   return metadata;

package/src/lib/arrow/convert-schema-to-parquet.ts

@@ -18,19 +18,22 @@ import {
 export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
   BOOLEAN: 'bool',
   INT32: 'int32',
-  INT64: 'float64',
+  INT64: 'int64',
   INT96: 'float64',
   FLOAT: 'float32',
   DOUBLE: 'float64',
   BYTE_ARRAY: 'binary',
   FIXED_LEN_BYTE_ARRAY: 'binary',
   UTF8: 'utf8',
-  DATE: 'int32',
-  TIME_MILLIS: 'int64',
-  TIME_MICROS: 'int64',
-  TIMESTAMP_MILLIS: 'int64',
-  TIMESTAMP_MICROS: 'int64',
-  UINT_8: 'int32',
+  ENUM: 'utf8',
+  DATE: 'date-day',
+  TIME_MILLIS: 'time-millisecond',
+  TIME_MICROS: 'time-microsecond',
+  TIME_NANOS: 'time-nanosecond',
+  TIMESTAMP_MILLIS: 'timestamp-millisecond',
+  TIMESTAMP_MICROS: 'timestamp-microsecond',
+  TIMESTAMP_NANOS: 'timestamp-nanosecond',
+  UINT_8: 'uint8',
   UINT_16: 'uint16',
   UINT_32: 'uint32',
   UINT_64: 'uint64',
@@ -38,6 +41,12 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
   INT_16: 'int16',
   INT_32: 'int32',
   INT_64: 'int64',
+  UUID: {type: 'fixed-size-binary', byteWidth: 16},
+  FLOAT16: 'float16',
+  UNKNOWN: 'null',
+  VARIANT: 'binary',
+  GEOMETRY: 'binary',
+  GEOGRAPHY: 'binary',
   JSON: 'binary',
   BSON: 'binary',
   // TODO check interval type