@loaders.gl/parquet 3.3.2 → 3.4.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.3.2",
+  "version": "3.4.0-alpha.2",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -42,9 +42,9 @@
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.3.2",
-    "@loaders.gl/loader-utils": "3.3.2",
-    "@loaders.gl/schema": "3.3.2",
+    "@loaders.gl/compression": "3.4.0-alpha.2",
+    "@loaders.gl/loader-utils": "3.4.0-alpha.2",
+    "@loaders.gl/schema": "3.4.0-alpha.2",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -70,5 +70,5 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "a37b0af509ca05468aec6e5667d6a427b9c15d29"
+  "gitHead": "f1c00c124d8d0c41a138ff40afb0d1a00711bf2e"
 }

package/src/index.ts

@@ -4,7 +4,11 @@ import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 
 import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
 import {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';
-import {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';
+import {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';
+import {
+  parseParquetInColumns,
+  parseParquetFileInColumnarBatches
+} from './lib/parsers/parse-parquet-to-columns';
 import {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
 
 export {ParquetWorkerLoader, ParquetWasmWorkerLoader};
@@ -16,6 +20,13 @@ export const ParquetLoader = {
   parseFileInBatches: parseParquetFileInBatches
 };
 
+/** ParquetJS table loader */
+export const ParquetColumnarLoader = {
+  ...ParquetWorkerLoader,
+  parse: parseParquetInColumns,
+  parseFileInBatches: parseParquetFileInColumnarBatches
+};
+
 export const ParquetWasmLoader = {
   ...ParquetWasmWorkerLoader,
   parse: parseParquetWasm
@@ -34,7 +45,16 @@ export {ParquetSchema} from './parquetjs/schema/schema';
 export {ParquetReader} from './parquetjs/parser/parquet-reader';
 export {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';
 
-export {convertParquetToArrowSchema} from './lib/convert-schema';
+export {
+  convertSchemaFromParquet,
+  convertSchemaFromParquet as convertParquetToArrowSchema
+} from './lib/arrow/convert-schema-from-parquet';
 
 // TESTS
 export const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;
+
+// Geo Metadata
+export {default as geoJSONSchema} from './lib/geo/geoparquet-schema';
+
+export type {GeoMetadata} from './lib/geo/decode-geo-metadata';
+export {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';

package/src/lib/arrow/convert-row-group-to-columns.ts

@@ -0,0 +1,15 @@
+// loaders.gl, MIT license
+
+import {Schema} from '@loaders.gl/schema';
+import {ParquetBuffer} from '@loaders.gl/parquet/parquetjs/schema/declare';
+
+export function convertParquetRowGroupToColumns(
+  schema: Schema,
+  rowGroup: ParquetBuffer
+): Record<string, any[]> {
+  const columns: Record<string, any[]> = {};
+  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
+    columns[columnName] = columns[columnName] || data.values;
+  }
+  return columns;
+}

package/src/lib/{convert-schema.ts → arrow/convert-schema-from-parquet.ts}

@@ -1,5 +1,8 @@
-import type {ParquetSchema} from '../parquetjs/schema/schema';
-import type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';
+// loaders.gl, MIT license
+
+import type {ParquetSchema} from '../../parquetjs/schema/schema';
+import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';
+import {FileMetaData} from '@loaders.gl/parquet/parquetjs/parquet-thrift';
 
 import {
   Schema,
@@ -45,7 +48,7 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
   INT_64: Int64,
   JSON: Binary,
   BSON: Binary,
-  // TODO check interal type
+  // TODO check interval type
   INTERVAL: Binary,
   DECIMAL_INT32: Float32,
   DECIMAL_INT64: Float64,
@@ -53,24 +56,13 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
   DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
 };
 
-export function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {
+export function convertSchemaFromParquet(
+  parquetSchema: ParquetSchema,
+  parquetMetadata?: FileMetaData
+): Schema {
   const fields = getFields(parquetSchema.schema);
-
-  // TODO add metadata if needed.
-  return new Schema(fields);
-}
-
-function getFieldMetadata(field: ParquetField): Map<string, string> {
-  const metadata = new Map();
-
-  for (const key in field) {
-    if (key !== 'name') {
-      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-      metadata.set(key, value);
-    }
-  }
-
-  return metadata;
+  const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
+  return new Schema(fields, metadata);
 }
 
 function getFields(schema: FieldDefinition): Field[] {
@@ -80,8 +72,8 @@ function getFields(schema: FieldDefinition): Field[] {
     const field = schema[name];
 
     if (field.fields) {
-      const childField = getFields(field.fields);
-      const nestedField = new Field(name, new Struct(childField), field.optional);
+      const childFields = getFields(field.fields);
+      const nestedField = new Field(name, new Struct(childFields), field.optional);
       fields.push(nestedField);
     } else {
       const FieldType = PARQUET_TYPE_MAPPING[field.type];
@@ -93,3 +85,30 @@ function getFields(schema: FieldDefinition): Field[] {
 
   return fields;
 }
+
+function getFieldMetadata(field: ParquetField): Map<string, string> {
+  const metadata = new Map();
+
+  for (const key in field) {
+    if (key !== 'name') {
+      let value = field[key] || '';
+      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+      metadata.set(key, value);
+    }
+  }
+
+  return metadata;
+}
+
+function getSchemaMetadata(parquetMetadata: FileMetaData): Map<string, string> {
+  const metadata = new Map();
+
+  const keyValueList = parquetMetadata.key_value_metadata || [];
+  for (const {key, value} of keyValueList) {
+    if (typeof value === 'string') {
+      metadata.set(key, value);
+    }
+  }
+
+  return metadata;
+}

package/src/lib/arrow/convert-schema-to-parquet.ts

@@ -0,0 +1,102 @@
+// loaders.gl, MIT license
+
+// import type {ParquetSchema} from '../../parquetjs/schema/schema';
+import type {
+  // FieldDefinition, ParquetField,
+  ParquetType
+} from '../../parquetjs/schema/declare';
+
+import {
+  Schema,
+  // Struct,
+  // Field,
+  DataType,
+  Bool,
+  Float64,
+  Int32,
+  Float32,
+  Binary,
+  Utf8,
+  Int64,
+  Uint16,
+  Uint32,
+  Uint64,
+  Int8,
+  Int16
+} from '@loaders.gl/schema';
+
+export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
+  BOOLEAN: Bool,
+  INT32: Int32,
+  INT64: Float64,
+  INT96: Float64,
+  FLOAT: Float32,
+  DOUBLE: Float64,
+  BYTE_ARRAY: Binary,
+  FIXED_LEN_BYTE_ARRAY: Binary,
+  UTF8: Utf8,
+  DATE: Int32,
+  TIME_MILLIS: Int64,
+  TIME_MICROS: Int64,
+  TIMESTAMP_MILLIS: Int64,
+  TIMESTAMP_MICROS: Int64,
+  UINT_8: Int32,
+  UINT_16: Uint16,
+  UINT_32: Uint32,
+  UINT_64: Uint64,
+  INT_8: Int8,
+  INT_16: Int16,
+  INT_32: Int32,
+  INT_64: Int64,
+  JSON: Binary,
+  BSON: Binary,
+  // TODO check interval type
+  INTERVAL: Binary,
+  DECIMAL_INT32: Float32,
+  DECIMAL_INT64: Float64,
+  DECIMAL_BYTE_ARRAY: Float64,
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
+};
+
+export function convertToParquetSchema(schema: Schema): Schema {
+  const fields = []; // getFields(schema.fields);
+
+  // TODO add metadata if needed.
+  return new Schema(fields);
+}
+
+// function getFields(schema: Field[]): Definition[] {
+//   const fields: Field[] = [];
+
+//   for (const name in schema) {
+//     const field = schema[name];
+
+//     // @ts-ignore
+//     const children = field.children as DataType[];
+//     if (children) {
+//       const childField = getFields(field.fields);
+//       const nestedField = new Field(name, new Struct(childField), field.optional);
+//       fields.push(nestedField);
+//     } else {
+//       const FieldType = PARQUET_TYPE_MAPPING[field.type];
+//       const metadata = getFieldMetadata(field);
+//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+//       fields.push(arrowField);
+//     }
+//   }
+
+//   return fields;
+// }
+
+// function getFieldMetadata(field: ParquetField): Map<string, string> {
+//   const metadata = new Map();
+
+//   for (const key in field) {
+//     if (key !== 'name') {
+//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+//       metadata.set(key, value);
+//     }
+//   }
+
+//   return metadata;
+// }

package/src/lib/geo/decode-geo-metadata.ts

@@ -0,0 +1,99 @@
+// loaders.gl, MIT license
+import {Schema, Field} from '@loaders.gl/schema';
+
+/* eslint-disable camelcase */
+
+/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */
+export type GeoMetadata = {
+  version?: string;
+  primary_column?: string;
+  columns: Record<string, GeoColumnMetadata>;
+  [key: string]: unknown;
+};
+
+/** A geoarrow / geoparquet geo metadata for one geometry column  */
+export type GeoColumnMetadata = {
+  bounding_box?:
+    | [number, number, number, number]
+    | [number, number, number, number, number, number];
+  crs?: string;
+  geometry_type?: string[];
+  edges?: string;
+  [key: string]: unknown;
+};
+
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
+export function getGeoMetadata(schema: Schema): GeoMetadata | null {
+  const stringifiedGeoMetadata = schema.metadata.get('geo');
+  if (!stringifiedGeoMetadata) {
+    return null;
+  }
+
+  try {
+    const geoMetadata = JSON.parse(stringifiedGeoMetadata) as GeoMetadata;
+    return geoMetadata;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+export function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {
+  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
+  schema.metadata.set('geo', stringifiedGeoMetadata);
+}
+
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export function unpackGeoMetadata(schema: Schema): void {
+  const geoMetadata = getGeoMetadata(schema);
+  if (!geoMetadata) {
+    return;
+  }
+
+  // Store Parquet Schema Level Metadata
+
+  const {version, primary_column, columns} = geoMetadata;
+  if (version) {
+    schema.metadata.set('geo.version', version);
+  }
+
+  if (primary_column) {
+    schema.metadata.set('geo.primary_column', primary_column);
+  }
+
+  // store column names as comma separated list
+  schema.metadata.set('geo.columns', Object.keys(columns || {}).join(''));
+
+  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
+    const field = schema.fields.find((field) => field.name === columnName);
+    if (field) {
+      if (field.name === primary_column) {
+        field.metadata.set('geo.primary_field', 'true');
+      }
+      unpackGeoFieldMetadata(field, columnMetadata);
+    }
+  }
+}
+
+function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
+  for (const [key, value] of Object.entries(columnMetadata || {})) {
+    switch (key) {
+      case 'geometry_type':
+        field.metadata.set(`geo.${key}`, (value as string[]).join(','));
+        break;
+      case 'bbox':
+      case 'crs':
+      case 'edges':
+      default:
+        field.metadata.set(`geo.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
+    }
+  }
+}

package/src/lib/geo/geoparquet-schema.ts

@@ -0,0 +1,69 @@
+// loaders.gl, MIT license
+
+/* eslint-disable camelcase */
+
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+export default {
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  title: 'GeoParquet',
+  description: 'Parquet metadata included in the geo field.',
+  type: 'object',
+  required: ['version', 'primary_column', 'columns'],
+  properties: {
+    version: {type: 'string', const: '1.0.0-beta.1'},
+    primary_column: {type: 'string', minLength: 1},
+    columns: {
+      type: 'object',
+      minProperties: 1,
+      patternProperties: {
+        '.+': {
+          type: 'object',
+          required: ['encoding', 'geometry_types'],
+          properties: {
+            encoding: {type: 'string', const: 'WKB'},
+            geometry_types: {
+              type: 'array',
+              uniqueItems: true,
+              items: {
+                type: 'string',
+                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+              }
+            },
+            crs: {
+              oneOf: [
+                {
+                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+                },
+                {type: 'null'}
+              ]
+            },
+            edges: {type: 'string', enum: ['planar', 'spherical']},
+            orientation: {type: 'string', const: 'counterclockwise'},
+            bbox: {
+              type: 'array',
+              items: {type: 'number'},
+              oneOf: [
+                {
+                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                  minItems: 4,
+                  maxItems: 4
+                },
+                {
+                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                  minItems: 6,
+                  maxItems: 6
+                }
+              ]
+            },
+            epoch: {type: 'number'}
+          }
+        }
+      },
+      additionalProperties: false
+    }
+  }
+};

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -0,0 +1,49 @@
+// loaders.gl, MIT license
+
+// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import {ColumnarTableBatch, Schema} from '@loaders.gl/schema';
+import {makeReadableFile} from '@loaders.gl/loader-utils';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+import {ParquetBuffer} from '../../parquetjs/schema/declare';
+import {convertSchemaFromParquet} from '../arrow/convert-schema-from-parquet';
+import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
+import {unpackGeoMetadata} from '../geo/decode-geo-metadata';
+
+export async function parseParquetInColumns(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetLoaderOptions
+) {
+  const blob = new Blob([arrayBuffer]);
+  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+    return batch;
+  }
+  return null;
+}
+
+export async function* parseParquetFileInColumnarBatches(
+  blob: Blob,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ColumnarTableBatch> {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const parquetSchema = await reader.getSchema();
+  const parquetMetadata = await reader.getFileMetadata();
+  const schema = convertSchemaFromParquet(parquetSchema, parquetMetadata);
+  unpackGeoMetadata(schema);
+  const rowGroups = reader.rowGroupIterator(options?.parquet);
+  for await (const rowGroup of rowGroups) {
+    yield convertRowGroupToTableBatch(schema, rowGroup);
+  }
+}
+
+function convertRowGroupToTableBatch(schema: Schema, rowGroup: ParquetBuffer): ColumnarTableBatch {
+  const data = convertParquetRowGroupToColumns(schema, rowGroup);
+  return {
+    shape: 'columnar-table',
+    batchType: 'data',
+    schema,
+    data,
+    length: rowGroup.rowCount
+  };
+}

package/src/lib/{parse-parquet.ts → parsers/parse-parquet-to-rows.ts}

@@ -1,8 +1,8 @@
 // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
 // import {ColumnarTableBatch} from '@loaders.gl/schema';
 import {makeReadableFile} from '@loaders.gl/loader-utils';
-import type {ParquetLoaderOptions} from '../parquet-loader';
-import {ParquetReader} from '../parquetjs/parser/parquet-reader';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 
 export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
   const blob = new Blob([arrayBuffer]);