@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/src/lib/arrow/convert-schema-to-parquet.ts

@@ -0,0 +1,90 @@
+// loaders.gl, MIT license
+
+// import type {ParquetSchema} from '../../parquetjs/schema/schema';
+import type {
+  // FieldDefinition, ParquetField,
+  ParquetType
+} from '../../parquetjs/schema/declare';
+
+import {
+  Schema,
+  // Struct,
+  // Field,
+  DataType
+} from '@loaders.gl/schema';
+
+export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
+  // TODO check interval type
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+};
+
+export function convertToParquetSchema(schema: Schema): Schema {
+  const fields = []; // getFields(schema.fields);
+
+  // TODO add metadata if needed.
+  return {fields, metadata: {}};
+}
+
+// function getFields(schema: Field[]): Definition[] {
+//   const fields: Field[] = [];
+
+//   for (const name in schema) {
+//     const field = schema[name];
+
+//     // @ts-ignore
+//     const children = field.children as DataType[];
+//     if (children) {
+//       const childField = getFields(field.fields);
+//       const nestedField = new Field(name, new Struct(childField), field.optional);
+//       fields.push(nestedField);
+//     } else {
+//       const FieldType = PARQUET_TYPE_MAPPING[field.type];
+//       const metadata = getFieldMetadata(field);
+//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+//       fields.push(arrowField);
+//     }
+//   }
+
+//   return fields;
+// }
+
+// function getFieldMetadata(field: ParquetField): Map<string, string> {
+//   const metadata = new Map();
+
+//   for (const key in field) {
+//     if (key !== 'name') {
+//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+//       metadata.set(key, value);
+//     }
+//   }
+
+//   return metadata;
+// }

package/src/lib/geo/decode-geo-metadata.ts

@@ -0,0 +1,108 @@
+// loaders.gl, MIT license
+import {Schema, Field} from '@loaders.gl/schema';
+
+/* eslint-disable camelcase */
+
+/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */
+export type GeoMetadata = {
+  version?: string;
+  primary_column?: string;
+  columns: Record<string, GeoColumnMetadata>;
+  [key: string]: unknown;
+};
+
+/** A geoarrow / geoparquet geo metadata for one geometry column  */
+export type GeoColumnMetadata = {
+  bounding_box?:
+    | [number, number, number, number]
+    | [number, number, number, number, number, number];
+  crs?: string;
+  geometry_type?: string[];
+  edges?: string;
+  [key: string]: unknown;
+};
+
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
+export function getGeoMetadata(schema: Schema): GeoMetadata | null {
+  const stringifiedGeoMetadata = schema.metadata.geo;
+  if (!stringifiedGeoMetadata) {
+    return null;
+  }
+
+  try {
+    const geoMetadata = JSON.parse(stringifiedGeoMetadata) as GeoMetadata;
+    return geoMetadata;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Stores a geoarrow / geoparquet geo metadata object in the schema
+ * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
+ */
+export function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {
+  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
+  schema.metadata.geo = stringifiedGeoMetadata;
+}
+
+/**
+ * Unpacks geo metadata into separate metadata fields (parses the long JSON string)
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
+ */
+export function unpackGeoMetadata(schema: Schema): void {
+  const geoMetadata = getGeoMetadata(schema);
+  if (!geoMetadata) {
+    return;
+  }
+
+  // Store Parquet Schema Level Metadata
+
+  const {version, primary_column, columns} = geoMetadata;
+  if (version) {
+    schema.metadata['geo.version'] = version;
+  }
+
+  if (primary_column) {
+    schema.metadata['geo.primary_column'] = primary_column;
+  }
+
+  // store column names as comma separated list
+  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
+
+  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
+    const field = schema.fields.find((field) => field.name === columnName);
+    if (field) {
+      if (field.name === primary_column) {
+        setFieldMetadata(field, 'geo.primary_field', 'true');
+      }
+      unpackGeoFieldMetadata(field, columnMetadata);
+    }
+  }
+}
+
+function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
+  for (const [key, value] of Object.entries(columnMetadata || {})) {
+    switch (key) {
+      case 'geometry_type':
+        setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));
+        break;
+      case 'bbox':
+      case 'crs':
+      case 'edges':
+      default:
+        setFieldMetadata(
+          field,
+          `geo.${key}`,
+          typeof value === 'string' ? value : JSON.stringify(value)
+        );
+    }
+  }
+}
+
+function setFieldMetadata(field: Field, key: string, value: string): void {
+  field.metadata = field.metadata || {};
+  field.metadata[key] = value;
+}

package/src/lib/geo/geoparquet-schema.ts

@@ -0,0 +1,69 @@
+// loaders.gl, MIT license
+
+/* eslint-disable camelcase */
+
+/**
+ * Geoparquet JSON schema for geo metadata
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ */
+export default {
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  title: 'GeoParquet',
+  description: 'Parquet metadata included in the geo field.',
+  type: 'object',
+  required: ['version', 'primary_column', 'columns'],
+  properties: {
+    version: {type: 'string', const: '1.0.0-beta.1'},
+    primary_column: {type: 'string', minLength: 1},
+    columns: {
+      type: 'object',
+      minProperties: 1,
+      patternProperties: {
+        '.+': {
+          type: 'object',
+          required: ['encoding', 'geometry_types'],
+          properties: {
+            encoding: {type: 'string', const: 'WKB'},
+            geometry_types: {
+              type: 'array',
+              uniqueItems: true,
+              items: {
+                type: 'string',
+                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+              }
+            },
+            crs: {
+              oneOf: [
+                {
+                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+                },
+                {type: 'null'}
+              ]
+            },
+            edges: {type: 'string', enum: ['planar', 'spherical']},
+            orientation: {type: 'string', const: 'counterclockwise'},
+            bbox: {
+              type: 'array',
+              items: {type: 'number'},
+              oneOf: [
+                {
+                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                  minItems: 4,
+                  maxItems: 4
+                },
+                {
+                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                  minItems: 6,
+                  maxItems: 6
+                }
+              ]
+            },
+            epoch: {type: 'number'}
+          }
+        }
+      },
+      additionalProperties: false
+    }
+  }
+};

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -0,0 +1,60 @@
+// loaders.gl, MIT license
+
+// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';
+import {makeReadableFile} from '@loaders.gl/loader-utils';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+import {ParquetRowGroup} from '../../parquetjs/schema/declare';
+import {ParquetSchema} from '../../parquetjs/schema/schema';
+import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
+import {materializeColumns} from '../../parquetjs/schema/shred';
+// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
+import {unpackGeoMetadata} from '../geo/decode-geo-metadata';
+
+export async function parseParquetInColumns(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetLoaderOptions
+): Promise<ColumnarTable> {
+  const blob = new Blob([arrayBuffer]);
+  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+    return {
+      shape: 'columnar-table',
+      schema: batch.schema,
+      data: batch.data
+    };
+  }
+  throw new Error('empty table');
+}
+
+export async function* parseParquetFileInColumnarBatches(
+  blob: Blob,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ColumnarTableBatch> {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const parquetSchema = await reader.getSchema();
+  const parquetMetadata = await reader.getFileMetadata();
+  const schema = convertParquetSchema(parquetSchema, parquetMetadata);
+  unpackGeoMetadata(schema);
+  const rowGroups = reader.rowGroupIterator(options?.parquet);
+  for await (const rowGroup of rowGroups) {
+    yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
+  }
+}
+
+function convertRowGroupToTableBatch(
+  parquetSchema: ParquetSchema,
+  rowGroup: ParquetRowGroup,
+  schema: Schema
+): ColumnarTableBatch {
+  // const data = convertParquetRowGroupToColumns(schema, rowGroup);
+  const data = materializeColumns(parquetSchema, rowGroup);
+  return {
+    shape: 'columnar-table',
+    batchType: 'data',
+    schema,
+    data,
+    length: rowGroup.rowCount
+  };
+}

package/src/lib/parsers/parse-parquet-to-rows.ts

@@ -0,0 +1,45 @@
+// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+// import {ColumnarTableBatch} from '@loaders.gl/schema';
+import {makeReadableFile} from '@loaders.gl/loader-utils';
+import {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
+import type {ParquetRow} from '../../parquetjs/schema/declare';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+
+export async function parseParquet(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetLoaderOptions
+): Promise<ObjectRowTable> {
+  const blob = new Blob([arrayBuffer]);
+
+  const rows: ParquetRow[] = [];
+  for await (const batch of parseParquetFileInBatches(blob, options)) {
+    // we have only one input batch so return
+    for (const row of batch.data) {
+      rows.push(row);
+    }
+  }
+
+  return {
+    shape: 'object-row-table',
+    // TODO - spread can fail for very large number of batches
+    data: rows
+  };
+}
+
+export async function* parseParquetFileInBatches(
+  blob: Blob,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ObjectRowTableBatch> {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const rowBatches = reader.rowBatchIterator(options?.parquet);
+  for await (const rows of rowBatches) {
+    yield {
+      shape: 'object-row-table',
+      data: rows,
+      batchType: 'data',
+      length: rows.length
+    };
+  }
+}

package/src/lib/wasm/encode-parquet-wasm.ts

@@ -0,0 +1,40 @@
+import type {Table} from 'apache-arrow';
+import type {WriterOptions} from '@loaders.gl/loader-utils';
+
+import {RecordBatchStreamWriter} from 'apache-arrow';
+import {loadWasm} from './load-wasm';
+
+export type ParquetWriterOptions = WriterOptions & {
+  parquet?: {
+    wasmUrl?: string;
+  };
+};
+
+/**
+ * Encode Arrow Table to Parquet buffer
+ */
+export async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {
+  const wasmUrl = options?.parquet?.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+
+  const arrowIPCBytes = tableToIPC(table);
+  // TODO: provide options for how to write table.
+  const writerProperties = new wasm.WriterPropertiesBuilder().build();
+  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
+  return parquetBytes.buffer.slice(
+    parquetBytes.byteOffset,
+    parquetBytes.byteLength + parquetBytes.byteOffset
+  );
+}
+
+/**
+ * Serialize a {@link Table} to the IPC format. This function is a convenience
+ * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.
+ * Opposite of {@link tableFromIPC}.
+ *
+ * @param table The Table to serialize.
+ * @param type Whether to serialize the Table as a file or a stream.
+ */
+export function tableToIPC(table: Table): Uint8Array {
+  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
+}

package/src/lib/wasm/load-wasm/index.ts

@@ -0,0 +1 @@
+export {loadWasm} from './load-wasm-node';

package/src/lib/wasm/load-wasm/load-wasm-browser.ts

@@ -0,0 +1,15 @@
+import * as wasmEsm from 'parquet-wasm/esm2/arrow1';
+
+let cached: typeof wasmEsm | null = null;
+
+export async function loadWasm(wasmUrl?: string) {
+  if (cached !== null) {
+    return cached;
+  }
+
+  // For ESM bundles, need to await the default export, which loads the WASM
+  await wasmEsm.default(wasmUrl);
+  cached = wasmEsm;
+
+  return wasmEsm;
+}

package/src/lib/wasm/load-wasm/load-wasm-node.ts

@@ -0,0 +1,5 @@
+import * as wasmNode from 'parquet-wasm/node/arrow1';
+
+export async function loadWasm(wasmUrl?: string) {
+  return wasmNode;
+}

package/src/lib/wasm/parse-parquet-wasm.ts

@@ -0,0 +1,42 @@
+// eslint-disable
+import type {RecordBatch} from 'apache-arrow';
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import {Table as ArrowTable, RecordBatchStreamReader} from 'apache-arrow';
+import {loadWasm} from './load-wasm/load-wasm-node';
+
+export type ParquetWasmLoaderOptions = LoaderOptions & {
+  parquet?: {
+    type?: 'arrow-table';
+    wasmUrl?: string;
+  };
+};
+
+export async function parseParquetWasm(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetWasmLoaderOptions
+): Promise<ArrowTable> {
+  const wasmUrl = options?.parquet?.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+
+  const arr = new Uint8Array(arrayBuffer);
+  const arrowIPCUint8Arr = wasm.readParquet(arr);
+  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(
+    arrowIPCUint8Arr.byteOffset,
+    arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset
+  );
+  const arrowTable = tableFromIPC(arrowIPCBuffer);
+  return arrowTable;
+}
+
+/**
+ * Deserialize the IPC format into a {@link Table}. This function is a
+ * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
+ */
+function tableFromIPC(input: ArrayBuffer): ArrowTable {
+  const reader = RecordBatchStreamReader.from(input);
+  const recordBatches: RecordBatch[] = [];
+  for (const recordBatch of reader) {
+    recordBatches.push(recordBatch);
+  }
+  return new ArrowTable(recordBatches);
+}