@loaders.gl/parquet 3.4.6 → 4.0.0-alpha.10

package/src/lib/arrow/convert-schema-to-parquet.ts

@@ -10,59 +10,47 @@ import {
   Schema,
   // Struct,
   // Field,
-  DataType,
-  Bool,
-  Float64,
-  Int32,
-  Float32,
-  Binary,
-  Utf8,
-  Int64,
-  Uint16,
-  Uint32,
-  Uint64,
-  Int8,
-  Int16
+  DataType
 } from '@loaders.gl/schema';
 
-export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
-  BOOLEAN: Bool,
-  INT32: Int32,
-  INT64: Float64,
-  INT96: Float64,
-  FLOAT: Float32,
-  DOUBLE: Float64,
-  BYTE_ARRAY: Binary,
-  FIXED_LEN_BYTE_ARRAY: Binary,
-  UTF8: Utf8,
-  DATE: Int32,
-  TIME_MILLIS: Int64,
-  TIME_MICROS: Int64,
-  TIMESTAMP_MILLIS: Int64,
-  TIMESTAMP_MICROS: Int64,
-  UINT_8: Int32,
-  UINT_16: Uint16,
-  UINT_32: Uint32,
-  UINT_64: Uint64,
-  INT_8: Int8,
-  INT_16: Int16,
-  INT_32: Int32,
-  INT_64: Int64,
-  JSON: Binary,
-  BSON: Binary,
+export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
   // TODO check interval type
-  INTERVAL: Binary,
-  DECIMAL_INT32: Float32,
-  DECIMAL_INT64: Float64,
-  DECIMAL_BYTE_ARRAY: Float64,
-  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
 };
 
 export function convertToParquetSchema(schema: Schema): Schema {
   const fields = []; // getFields(schema.fields);
 
   // TODO add metadata if needed.
-  return new Schema(fields);
+  return {fields, metadata: {}};
 }
 
 // function getFields(schema: Field[]): Definition[] {

package/src/lib/geo/decode-geo-metadata.ts

@@ -26,7 +26,7 @@ export type GeoColumnMetadata = {
  * Reads the GeoMetadata object from the metadata
  * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
 export function getGeoMetadata(schema: Schema): GeoMetadata | null {
-  const stringifiedGeoMetadata = schema.metadata.get('geo');
+  const stringifiedGeoMetadata = schema.metadata.geo;
   if (!stringifiedGeoMetadata) {
     return null;
   }
@@ -45,7 +45,7 @@ export function getGeoMetadata(schema: Schema): GeoMetadata | null {
  */
 export function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {
   const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
-  schema.metadata.set('geo', stringifiedGeoMetadata);
+  schema.metadata.geo = stringifiedGeoMetadata;
 }
 
 /**
@@ -62,21 +62,21 @@ export function unpackGeoMetadata(schema: Schema): void {
 
   const {version, primary_column, columns} = geoMetadata;
   if (version) {
-    schema.metadata.set('geo.version', version);
+    schema.metadata['geo.version'] = version;
   }
 
   if (primary_column) {
-    schema.metadata.set('geo.primary_column', primary_column);
+    schema.metadata['geo.primary_column'] = primary_column;
   }
 
   // store column names as comma separated list
-  schema.metadata.set('geo.columns', Object.keys(columns || {}).join(''));
+  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
 
   for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
     const field = schema.fields.find((field) => field.name === columnName);
     if (field) {
       if (field.name === primary_column) {
-        field.metadata.set('geo.primary_field', 'true');
+        setFieldMetadata(field, 'geo.primary_field', 'true');
       }
       unpackGeoFieldMetadata(field, columnMetadata);
     }
@@ -87,13 +87,22 @@ function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
   for (const [key, value] of Object.entries(columnMetadata || {})) {
     switch (key) {
       case 'geometry_type':
-        field.metadata.set(`geo.${key}`, (value as string[]).join(','));
+        setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));
         break;
       case 'bbox':
       case 'crs':
       case 'edges':
       default:
-        field.metadata.set(`geo.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
+        setFieldMetadata(
+          field,
+          `geo.${key}`,
+          typeof value === 'string' ? value : JSON.stringify(value)
+        );
     }
   }
 }
+
+function setFieldMetadata(field: Field, key: string, value: string): void {
+  field.metadata = field.metadata || {};
+  field.metadata[key] = value;
+}

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -1,24 +1,30 @@
 // loaders.gl, MIT license
 
 // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
-import {ColumnarTableBatch, Schema} from '@loaders.gl/schema';
+import {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';
 import {makeReadableFile} from '@loaders.gl/loader-utils';
 import type {ParquetLoaderOptions} from '../../parquet-loader';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
-import {ParquetBuffer} from '../../parquetjs/schema/declare';
-import {convertSchemaFromParquet} from '../arrow/convert-schema-from-parquet';
-import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
+import {ParquetRowGroup} from '../../parquetjs/schema/declare';
+import {ParquetSchema} from '../../parquetjs/schema/schema';
+import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
+import {materializeColumns} from '../../parquetjs/schema/shred';
+// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
 import {unpackGeoMetadata} from '../geo/decode-geo-metadata';
 
 export async function parseParquetInColumns(
   arrayBuffer: ArrayBuffer,
   options?: ParquetLoaderOptions
-) {
+): Promise<ColumnarTable> {
   const blob = new Blob([arrayBuffer]);
   for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
-    return batch;
+    return {
+      shape: 'columnar-table',
+      schema: batch.schema,
+      data: batch.data
+    };
   }
-  return null;
+  throw new Error('empty table');
 }
 
 export async function* parseParquetFileInColumnarBatches(
@@ -29,16 +35,21 @@ export async function* parseParquetFileInColumnarBatches(
   const reader = new ParquetReader(file);
   const parquetSchema = await reader.getSchema();
   const parquetMetadata = await reader.getFileMetadata();
-  const schema = convertSchemaFromParquet(parquetSchema, parquetMetadata);
+  const schema = convertParquetSchema(parquetSchema, parquetMetadata);
   unpackGeoMetadata(schema);
   const rowGroups = reader.rowGroupIterator(options?.parquet);
   for await (const rowGroup of rowGroups) {
-    yield convertRowGroupToTableBatch(schema, rowGroup);
+    yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
   }
 }
 
-function convertRowGroupToTableBatch(schema: Schema, rowGroup: ParquetBuffer): ColumnarTableBatch {
-  const data = convertParquetRowGroupToColumns(schema, rowGroup);
+function convertRowGroupToTableBatch(
+  parquetSchema: ParquetSchema,
+  rowGroup: ParquetRowGroup,
+  schema: Schema
+): ColumnarTableBatch {
+  // const data = convertParquetRowGroupToColumns(schema, rowGroup);
+  const data = materializeColumns(parquetSchema, rowGroup);
   return {
     shape: 'columnar-table',
     batchType: 'data',

package/src/lib/parsers/parse-parquet-to-rows.ts

@@ -1,40 +1,45 @@
 // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
 // import {ColumnarTableBatch} from '@loaders.gl/schema';
 import {makeReadableFile} from '@loaders.gl/loader-utils';
+import {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
 import type {ParquetLoaderOptions} from '../../parquet-loader';
+import type {ParquetRow} from '../../parquetjs/schema/declare';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 
-export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
+export async function parseParquet(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetLoaderOptions
+): Promise<ObjectRowTable> {
   const blob = new Blob([arrayBuffer]);
+
+  const rows: ParquetRow[] = [];
   for await (const batch of parseParquetFileInBatches(blob, options)) {
-    return batch;
+    // we have only one input batch so return
+    for (const row of batch.data) {
+      rows.push(row);
+    }
   }
-  return null;
+
+  return {
+    shape: 'object-row-table',
+    // TODO - spread can fail for very large number of batches
+    data: rows
+  };
 }
 
-export async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {
+export async function* parseParquetFileInBatches(
+  blob: Blob,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ObjectRowTableBatch> {
   const file = makeReadableFile(blob);
   const reader = new ParquetReader(file);
   const rowBatches = reader.rowBatchIterator(options?.parquet);
   for await (const rows of rowBatches) {
-    yield rows;
+    yield {
+      shape: 'object-row-table',
+      data: rows,
+      batchType: 'data',
+      length: rows.length
+    };
   }
 }
-
-// export async function* parseParquetFileInColumnarBatches(blob: Blob, options?: {columnList?: string[][]}): AsyncIterable<ColumnarTableBatch> {
-//   const rowGroupReader = new ParquetRowGroupReader({data: blob, columnList: options?.columnList});
-//   try {
-//     for await (const rowGroup of rowGroupReader) {
-//       yield convertRowGroupToTableBatch(rowGroup);
-//     }
-//   } finally {
-//     await rowGroupReader.close();
-//   }
-// }
-
-// function convertRowGroupToTableBatch(rowGroup): ColumnarTableBatch {
-//   // @ts-expect-error
-//   return {
-//     data: rowGroup
-//   };
-// }

package/src/lib/wasm/parse-parquet-wasm.ts

@@ -1,20 +1,20 @@
 // eslint-disable
 import type {RecordBatch} from 'apache-arrow';
 import type {LoaderOptions} from '@loaders.gl/loader-utils';
-import {Table, RecordBatchStreamReader} from 'apache-arrow';
+import {Table as ArrowTable, RecordBatchStreamReader} from 'apache-arrow';
 import {loadWasm} from './load-wasm/load-wasm-node';
 
-export type ParquetLoaderOptions = LoaderOptions & {
+export type ParquetWasmLoaderOptions = LoaderOptions & {
   parquet?: {
     type?: 'arrow-table';
     wasmUrl?: string;
   };
 };
 
-export async function parseParquet(
+export async function parseParquetWasm(
   arrayBuffer: ArrayBuffer,
-  options?: ParquetLoaderOptions
-): Promise<Table> {
+  options?: ParquetWasmLoaderOptions
+): Promise<ArrowTable> {
   const wasmUrl = options?.parquet?.wasmUrl;
   const wasm = await loadWasm(wasmUrl);
 
@@ -32,11 +32,11 @@ export async function parseParquet(
  * Deserialize the IPC format into a {@link Table}. This function is a
  * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
  */
-function tableFromIPC(input: ArrayBuffer): Table {
+function tableFromIPC(input: ArrayBuffer): ArrowTable {
   const reader = RecordBatchStreamReader.from(input);
   const recordBatches: RecordBatch[] = [];
   for (const recordBatch of reader) {
     recordBatches.push(recordBatch);
   }
-  return new Table(recordBatches);
+  return new ArrowTable(recordBatches);
 }

package/src/parquet-loader.ts

@@ -1,4 +1,11 @@
+// loaders.gl, MIT license
 import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ColumnarTable,
+  ColumnarTableBatch
+} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -23,7 +30,7 @@ const DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {
 };
 
 /** ParquetJS table loader */
-export const ParquetLoader = {
+export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -37,4 +44,20 @@ export const ParquetLoader = {
   options: DEFAULT_PARQUET_LOADER_OPTIONS
 };
 
-export const _typecheckParquetLoader: Loader = ParquetLoader;
+export const ParqueColumnnartLoader: Loader<
+  ColumnarTable,
+  ColumnarTableBatch,
+  ParquetLoaderOptions
+> = {
+  name: 'Apache Parquet',
+  id: 'parquet',
+  module: 'parquet',
+  version: VERSION,
+  worker: true,
+  category: 'table',
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  binary: true,
+  tests: ['PAR1', 'PARE'],
+  options: DEFAULT_PARQUET_LOADER_OPTIONS
+};

package/src/parquet-writer.ts

@@ -1,4 +1,7 @@
+// loaders.gl, MIT license
+
 import type {Writer} from '@loaders.gl/loader-utils';
+import {Table, TableBatch} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -8,7 +11,7 @@ export type ParquetWriterOptions = {};
 
 const DEFAULT_PARQUET_LOADER_OPTIONS = {};
 
-export const ParquetWriter: Writer = {
+export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',

package/src/parquetjs/encoder/parquet-encoder.ts

@@ -4,11 +4,12 @@ import {stream} from '@loaders.gl/loader-utils';
 import {ParquetCodecOptions, PARQUET_CODECS} from '../codecs';
 import * as Compression from '../compression';
 import {
-  ParquetBuffer,
+  ParquetRowGroup,
   ParquetCodec,
-  ParquetData,
+  ParquetColumnChunk,
   ParquetField,
-  PrimitiveType
+  PrimitiveType,
+  ParquetRow
 } from '../schema/declare';
 import {ParquetSchema} from '../schema/schema';
 import * as Shred from '../schema/shred';
@@ -105,7 +106,7 @@ export class ParquetEncoder<T> {
 
   public schema: ParquetSchema;
   public envelopeWriter: ParquetEnvelopeWriter;
-  public rowBuffer: ParquetBuffer;
+  public rowBuffer: ParquetRowGroup;
   public rowGroupSize: number;
   public closed: boolean;
   public userMetadata: Record<string, string>;
@@ -144,7 +145,7 @@ export class ParquetEncoder<T> {
    * Append a single row to the parquet file. Rows are buffered in memory until
    * rowGroupSize rows are in the buffer or close() is called
    */
-  async appendRow<T>(row: T): Promise<void> {
+  async appendRow<T extends ParquetRow>(row: T): Promise<void> {
     if (this.closed) {
       throw new Error('writer was closed');
     }
@@ -271,7 +272,7 @@ export class ParquetEnvelopeWriter {
    * Encode a parquet row group. The records object should be created using the
    * shredRecord method
    */
-  async writeRowGroup(records: ParquetBuffer): Promise<void> {
+  async writeRowGroup(records: ParquetRowGroup): Promise<void> {
     const rgroup = await encodeRowGroup(this.schema, records, {
       baseOffset: this.offset,
       pageSize: this.pageSize,
@@ -363,7 +364,7 @@ function encodeValues(
  */
 async function encodeDataPage(
   column: ParquetField,
-  data: ParquetData
+  data: ParquetColumnChunk
 ): Promise<{
   header: PageHeader;
   headerSize: number;
@@ -422,7 +423,7 @@ async function encodeDataPage(
  */
 async function encodeDataPageV2(
   column: ParquetField,
-  data: ParquetData,
+  data: ParquetColumnChunk,
   rowCount: number
 ): Promise<{
   header: PageHeader;
@@ -482,7 +483,7 @@ async function encodeDataPageV2(
  */
 async function encodeColumnChunk(
   column: ParquetField,
-  buffer: ParquetBuffer,
+  buffer: ParquetRowGroup,
   offset: number,
   opts: ParquetEncoderOptions
 ): Promise<{
@@ -539,7 +540,7 @@ async function encodeColumnChunk(
  */
 async function encodeRowGroup(
   schema: ParquetSchema,
-  data: ParquetBuffer,
+  data: ParquetRowGroup,
   opts: ParquetEncoderOptions
 ): Promise<{
   body: Buffer;

package/src/parquetjs/parser/decoders.ts

@@ -1,7 +1,7 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 import {
   ParquetCodec,
-  ParquetData,
+  ParquetColumnChunk,
   ParquetOptions,
   ParquetPageData,
   ParquetType,
@@ -32,14 +32,14 @@ import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils'
 export async function decodeDataPages(
   buffer: Buffer,
   options: ParquetOptions
-): Promise<ParquetData> {
+): Promise<ParquetColumnChunk> {
   const cursor: CursorBuffer = {
     buffer,
     offset: 0,
     size: buffer.length
   };
 
-  const data: ParquetData = {
+  const data: ParquetColumnChunk = {
     rlevels: [],
     dlevels: [],
     values: [],

package/src/parquetjs/parser/parquet-reader.ts

@@ -3,14 +3,14 @@ import type {ReadableFile} from '@loaders.gl/loader-utils';
 
 import {ParquetSchema} from '../schema/schema';
 import {decodeSchema} from './decoders';
-import {materializeRecords} from '../schema/shred';
+import {materializeRows} from '../schema/shred';
 
 import {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';
 import {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';
 import {
-  ParquetBuffer,
+  ParquetRowGroup,
   ParquetCompression,
-  ParquetData,
+  ParquetColumnChunk,
   PrimitiveType,
   ParquetOptions
 } from '../schema/declare';
@@ -68,7 +68,7 @@ export class ParquetReader {
   async *rowBatchIterator(props?: ParquetIterationProps) {
     const schema = await this.getSchema();
     for await (const rowGroup of this.rowGroupIterator(props)) {
-      yield materializeRecords(schema, rowGroup);
+      yield materializeRows(schema, rowGroup);
     }
   }
 
@@ -172,8 +172,8 @@ export class ParquetReader {
     schema: ParquetSchema,
     rowGroup: RowGroup,
     columnList: string[][]
-  ): Promise<ParquetBuffer> {
-    const buffer: ParquetBuffer = {
+  ): Promise<ParquetRowGroup> {
+    const buffer: ParquetRowGroup = {
       rowCount: Number(rowGroup.num_rows),
       columnData: {}
     };
@@ -191,7 +191,7 @@ export class ParquetReader {
   /**
    * Each row group contains column chunks for all the columns.
    */
-  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {
+  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {
     if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
       throw new Error('external references are not supported');
     }

package/src/parquetjs/schema/declare.ts

@@ -120,25 +120,26 @@ export interface ParquetPageData {
   pageHeader: PageHeader;
 }
 
-export interface ParquetRecord {
+export interface ParquetRow {
   [key: string]: any;
 }
 
 /** @
  * Holds data for one row group (column chunks) */
-export class ParquetBuffer {
+export class ParquetRowGroup {
   /** Number of rows in this page */
   rowCount: number;
+  /** Map of Column chunks */
+  columnData: Record<string, ParquetColumnChunk>;
 
-  columnData: Record<string, ParquetData>;
-  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {
+  constructor(rowCount: number = 0, columnData: Record<string, ParquetColumnChunk> = {}) {
     this.rowCount = rowCount;
     this.columnData = columnData;
   }
 }
 
 /** Holds the data for one column chunk */
-export interface ParquetData {
+export interface ParquetColumnChunk {
   dlevels: number[];
   rlevels: number[];
   values: any[];

package/src/parquetjs/schema/schema.ts

@@ -4,14 +4,14 @@ import {PARQUET_CODECS} from '../codecs';
 import {PARQUET_COMPRESSION_METHODS} from '../compression';
 import {
   FieldDefinition,
-  ParquetBuffer,
+  ParquetRowGroup,
   ParquetCompression,
   ParquetField,
-  ParquetRecord,
+  ParquetRow,
   RepetitionType,
   SchemaDefinition
 } from './declare';
-import {materializeRecords, shredBuffer, shredRecord} from './shred';
+import {materializeRows, shredBuffer, shredRecord} from './shred';
 import {PARQUET_LOGICAL_TYPES} from './types';
 
 /**
@@ -70,12 +70,12 @@ export class ParquetSchema {
     return branch;
   }
 
-  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {
-    shredRecord(this, record, buffer);
+  shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void {
+    shredRecord(this, row, rowGroup);
   }
 
-  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {
-    return materializeRecords(this, buffer);
+  materializeRows(rowGroup: ParquetRowGroup): ParquetRow[] {
+    return materializeRows(this, rowGroup);
   }
 
   compress(type: ParquetCompression): this {
@@ -84,7 +84,7 @@ export class ParquetSchema {
     return this;
   }
 
-  buffer(): ParquetBuffer {
+  rowGroup(): ParquetRowGroup {
     return shredBuffer(this);
   }
 }