@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.2

package/src/lib/geo/decode-geo-metadata.ts

@@ -0,0 +1,99 @@
+// loaders.gl, MIT license
+import {Schema, Field} from '@loaders.gl/schema';
+
+/* eslint-disable camelcase */
+
+/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */
+export type GeoMetadata = {
+  version?: string;
+  primary_column?: string;
+  columns: Record<string, GeoColumnMetadata>;
+  [key: string]: unknown;
+};
+
+/** A geoarrow / geoparquet geo metadata for one geometry column  */
+export type GeoColumnMetadata = {
+  bounding_box?:
+    | [number, number, number, number]
+    | [number, number, number, number, number, number];
+  crs?: string;
+  geometry_type?: string[];
+  edges?: string;
+  [key: string]: unknown;
+};
+
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
+export function getGeoMetadata(schema: Schema): GeoMetadata | null {
+  const stringifiedGeoMetadata = schema.metadata.get('geo');
+  if (!stringifiedGeoMetadata) {
+    return null;
+  }
+
+  try {
+    const geoMetadata = JSON.parse(stringifiedGeoMetadata) as GeoMetadata;
+    return geoMetadata;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+export function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {
+  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
+  schema.metadata.set('geo', stringifiedGeoMetadata);
+}
+
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export function unpackGeoMetadata(schema: Schema): void {
+  const geoMetadata = getGeoMetadata(schema);
+  if (!geoMetadata) {
+    return;
+  }
+
+  // Store Parquet Schema Level Metadata
+
+  const {version, primary_column, columns} = geoMetadata;
+  if (version) {
+    schema.metadata.set('geo.version', version);
+  }
+
+  if (primary_column) {
+    schema.metadata.set('geo.primary_column', primary_column);
+  }
+
+  // store column names as comma separated list
+  schema.metadata.set('geo.columns', Object.keys(columns || {}).join(''));
+
+  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
+    const field = schema.fields.find((field) => field.name === columnName);
+    if (field) {
+      if (field.name === primary_column) {
+        field.metadata.set('geo.primary_field', 'true');
+      }
+      unpackGeoFieldMetadata(field, columnMetadata);
+    }
+  }
+}
+
+function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
+  for (const [key, value] of Object.entries(columnMetadata || {})) {
+    switch (key) {
+      case 'geometry_type':
+        field.metadata.set(`geo.${key}`, (value as string[]).join(','));
+        break;
+      case 'bbox':
+      case 'crs':
+      case 'edges':
+      default:
+        field.metadata.set(`geo.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
+    }
+  }
+}

package/src/lib/geo/geoparquet-schema.ts

@@ -0,0 +1,69 @@
+// loaders.gl, MIT license
+
+/* eslint-disable camelcase */
+
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+export default {
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  title: 'GeoParquet',
+  description: 'Parquet metadata included in the geo field.',
+  type: 'object',
+  required: ['version', 'primary_column', 'columns'],
+  properties: {
+    version: {type: 'string', const: '1.0.0-beta.1'},
+    primary_column: {type: 'string', minLength: 1},
+    columns: {
+      type: 'object',
+      minProperties: 1,
+      patternProperties: {
+        '.+': {
+          type: 'object',
+          required: ['encoding', 'geometry_types'],
+          properties: {
+            encoding: {type: 'string', const: 'WKB'},
+            geometry_types: {
+              type: 'array',
+              uniqueItems: true,
+              items: {
+                type: 'string',
+                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+              }
+            },
+            crs: {
+              oneOf: [
+                {
+                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+                },
+                {type: 'null'}
+              ]
+            },
+            edges: {type: 'string', enum: ['planar', 'spherical']},
+            orientation: {type: 'string', const: 'counterclockwise'},
+            bbox: {
+              type: 'array',
+              items: {type: 'number'},
+              oneOf: [
+                {
+                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                  minItems: 4,
+                  maxItems: 4
+                },
+                {
+                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                  minItems: 6,
+                  maxItems: 6
+                }
+              ]
+            },
+            epoch: {type: 'number'}
+          }
+        }
+      },
+      additionalProperties: false
+    }
+  }
+};

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -0,0 +1,49 @@
+// loaders.gl, MIT license
+
+// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import {ColumnarTableBatch, Schema} from '@loaders.gl/schema';
+import {makeReadableFile} from '@loaders.gl/loader-utils';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+import {ParquetBuffer} from '../../parquetjs/schema/declare';
+import {convertSchemaFromParquet} from '../arrow/convert-schema-from-parquet';
+import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
+import {unpackGeoMetadata} from '../geo/decode-geo-metadata';
+
+export async function parseParquetInColumns(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetLoaderOptions
+) {
+  const blob = new Blob([arrayBuffer]);
+  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+    return batch;
+  }
+  return null;
+}
+
+export async function* parseParquetFileInColumnarBatches(
+  blob: Blob,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ColumnarTableBatch> {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const parquetSchema = await reader.getSchema();
+  const parquetMetadata = await reader.getFileMetadata();
+  const schema = convertSchemaFromParquet(parquetSchema, parquetMetadata);
+  unpackGeoMetadata(schema);
+  const rowGroups = reader.rowGroupIterator(options?.parquet);
+  for await (const rowGroup of rowGroups) {
+    yield convertRowGroupToTableBatch(schema, rowGroup);
+  }
+}
+
+function convertRowGroupToTableBatch(schema: Schema, rowGroup: ParquetBuffer): ColumnarTableBatch {
+  const data = convertParquetRowGroupToColumns(schema, rowGroup);
+  return {
+    shape: 'columnar-table',
+    batchType: 'data',
+    schema,
+    data,
+    length: rowGroup.rowCount
+  };
+}

package/src/lib/parsers/parse-parquet-to-rows.ts

@@ -0,0 +1,40 @@
+// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+// import {ColumnarTableBatch} from '@loaders.gl/schema';
+import {makeReadableFile} from '@loaders.gl/loader-utils';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+
+export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
+  const blob = new Blob([arrayBuffer]);
+  for await (const batch of parseParquetFileInBatches(blob, options)) {
+    return batch;
+  }
+  return null;
+}
+
+export async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const rowBatches = reader.rowBatchIterator(options?.parquet);
+  for await (const rows of rowBatches) {
+    yield rows;
+  }
+}
+
+// export async function* parseParquetFileInColumnarBatches(blob: Blob, options?: {columnList?: string[][]}): AsyncIterable<ColumnarTableBatch> {
+//   const rowGroupReader = new ParquetRowGroupReader({data: blob, columnList: options?.columnList});
+//   try {
+//     for await (const rowGroup of rowGroupReader) {
+//       yield convertRowGroupToTableBatch(rowGroup);
+//     }
+//   } finally {
+//     await rowGroupReader.close();
+//   }
+// }
+
+// function convertRowGroupToTableBatch(rowGroup): ColumnarTableBatch {
+//   // @ts-expect-error
+//   return {
+//     data: rowGroup
+//   };
+// }