@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/src/parquetjs/parquet-thrift/FileMetaData.ts

@@ -16,6 +16,7 @@ import * as ColumnOrder from './ColumnOrder';
 import * as KeyValue from './KeyValue';
 import * as RowGroup from './RowGroup';
 import * as SchemaElement from './SchemaElement';
+import * as EncryptionAlgorithm from './EncryptionAlgorithm';
 export interface IFileMetaDataArgs {
   version: number;
   schema: Array<SchemaElement.SchemaElement>;
@@ -24,6 +25,8 @@ export interface IFileMetaDataArgs {
   key_value_metadata?: Array<KeyValue.KeyValue>;
   created_by?: string;
   column_orders?: Array<ColumnOrder.ColumnOrder>;
+  encryption_algorithm?: EncryptionAlgorithm.EncryptionAlgorithm;
+  footer_signing_key_metadata?: Uint8Array;
 }
 export class FileMetaData {
   public version: number;
@@ -33,6 +36,8 @@ export class FileMetaData {
   public key_value_metadata?: Array<KeyValue.KeyValue>;
   public created_by?: string;
   public column_orders?: Array<ColumnOrder.ColumnOrder>;
+  public encryption_algorithm?: EncryptionAlgorithm.EncryptionAlgorithm;
+  public footer_signing_key_metadata?: Uint8Array;
   constructor(args: IFileMetaDataArgs | null = null) {
     if (args != null && args.version != null) {
       this.version = args.version;
@@ -79,6 +84,12 @@ export class FileMetaData {
     if (args != null && args.column_orders != null) {
       this.column_orders = args.column_orders;
     }
+    if (args != null && args.encryption_algorithm != null) {
+      this.encryption_algorithm = args.encryption_algorithm;
+    }
+    if (args != null && args.footer_signing_key_metadata != null) {
+      this.footer_signing_key_metadata = args.footer_signing_key_metadata;
+    }
   }
   public write(output: thrift.TProtocol): void {
     output.writeStructBegin('FileMetaData');
@@ -133,6 +144,16 @@ export class FileMetaData {
       output.writeListEnd();
       output.writeFieldEnd();
     }
+    if (this.encryption_algorithm != null) {
+      output.writeFieldBegin('encryption_algorithm', thrift.Thrift.Type.STRUCT, 8);
+      this.encryption_algorithm.write(output);
+      output.writeFieldEnd();
+    }
+    if (this.footer_signing_key_metadata != null) {
+      output.writeFieldBegin('footer_signing_key_metadata', thrift.Thrift.Type.STRING, 9);
+      output.writeBinary(this.footer_signing_key_metadata);
+      output.writeFieldEnd();
+    }
     output.writeFieldStop();
     output.writeStructEnd();
     return;
@@ -233,6 +254,20 @@ export class FileMetaData {
             input.skip(fieldType);
           }
           break;
+        case 8:
+          if (fieldType === thrift.Thrift.Type.STRUCT) {
+            _args.encryption_algorithm = EncryptionAlgorithm.EncryptionAlgorithm.read(input);
+          } else {
+            input.skip(fieldType);
+          }
+          break;
+        case 9:
+          if (fieldType === thrift.Thrift.Type.STRING) {
+            _args.footer_signing_key_metadata = input.readBinary();
+          } else {
+            input.skip(fieldType);
+          }
+          break;
         default: {
           input.skip(fieldType);
         }

package/src/parquetjs/parquet-thrift/GeospatialStatistics.ts

@@ -0,0 +1,68 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* eslint-disable camelcase */
+import * as thrift from '../utils/thrift-runtime';
+import {BoundingBox} from './BoundingBox';
+
+/** Constructor arguments for Parquet geospatial column statistics. */
+export interface IGeospatialStatisticsArgs {
+  bbox?: BoundingBox;
+  geospatial_types?: number[];
+}
+
+/** Parquet geospatial column statistics stored on a column chunk. */
+export class GeospatialStatistics {
+  public bbox?: BoundingBox;
+  public geospatial_types?: number[];
+
+  constructor(args: IGeospatialStatisticsArgs = {}) {
+    this.bbox = args.bbox;
+    this.geospatial_types = args.geospatial_types;
+  }
+
+  /** Writes these geospatial statistics to a Thrift protocol. */
+  public write(output: thrift.TProtocol): void {
+    output.writeStructBegin('GeospatialStatistics');
+    if (this.bbox) {
+      output.writeFieldBegin('bbox', thrift.Thrift.Type.STRUCT, 1);
+      this.bbox.write(output);
+      output.writeFieldEnd();
+    }
+    if (this.geospatial_types) {
+      output.writeFieldBegin('geospatial_types', thrift.Thrift.Type.LIST, 2);
+      output.writeListBegin(thrift.Thrift.Type.I32, this.geospatial_types.length);
+      for (const geospatialType of this.geospatial_types) output.writeI32(geospatialType);
+      output.writeListEnd();
+      output.writeFieldEnd();
+    }
+    output.writeFieldStop();
+    output.writeStructEnd();
+  }
+
+  /** Reads geospatial statistics from a Thrift protocol. */
+  public static read(input: thrift.TProtocol): GeospatialStatistics {
+    input.readStructBegin();
+    const args: IGeospatialStatisticsArgs = {};
+    while (true) {
+      const field = input.readFieldBegin();
+      if (field.ftype === thrift.Thrift.Type.STOP) break;
+      if (field.fid === 1 && field.ftype === thrift.Thrift.Type.STRUCT) {
+        args.bbox = BoundingBox.read(input);
+      } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.LIST) {
+        const list = input.readListBegin();
+        args.geospatial_types = [];
+        for (let index = 0; index < list.size; index++) {
+          args.geospatial_types.push(input.readI32());
+        }
+        input.readListEnd();
+      } else {
+        input.skip(field.ftype);
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return new GeospatialStatistics(args);
+  }
+}

package/src/parquetjs/parquet-thrift/SizeStatistics.ts

@@ -0,0 +1,98 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {CompactInt64 as Int64} from '../utils/uint8-array-compact-protocol';
+import * as thrift from '../utils/thrift-runtime';
+
+/** Constructor arguments for Parquet size statistics. */
+export interface ISizeStatisticsArgs {
+  unencoded_byte_array_data_bytes?: number | Int64;
+  repetition_level_histogram?: Array<number | Int64>;
+  definition_level_histogram?: Array<number | Int64>;
+}
+
+/**
+ * Optional Parquet metadata used to estimate decoded memory and nested values.
+ *
+ * This is intentionally a faithful representation of the Parquet thrift
+ * structure. Consumers should treat absent histograms as unknown rather than
+ * as zero counts.
+ */
+export class SizeStatistics {
+  public unencoded_byte_array_data_bytes?: Int64;
+  public repetition_level_histogram?: Array<Int64>;
+  public definition_level_histogram?: Array<Int64>;
+
+  constructor(args: ISizeStatisticsArgs = {}) {
+    this.unencoded_byte_array_data_bytes = toInt64(args.unencoded_byte_array_data_bytes);
+    this.repetition_level_histogram = toInt64List(args.repetition_level_histogram);
+    this.definition_level_histogram = toInt64List(args.definition_level_histogram);
+  }
+
+  /** Writes size statistics to a Thrift protocol. */
+  public write(output: thrift.TProtocol): void {
+    output.writeStructBegin('SizeStatistics');
+    if (this.unencoded_byte_array_data_bytes !== undefined) {
+      output.writeFieldBegin('unencoded_byte_array_data_bytes', thrift.Thrift.Type.I64, 1);
+      output.writeI64(this.unencoded_byte_array_data_bytes);
+      output.writeFieldEnd();
+    }
+    writeInt64List(output, 'repetition_level_histogram', 2, this.repetition_level_histogram);
+    writeInt64List(output, 'definition_level_histogram', 3, this.definition_level_histogram);
+    output.writeFieldStop();
+    output.writeStructEnd();
+  }
+
+  /** Reads size statistics from a Thrift protocol. */
+  public static read(input: thrift.TProtocol): SizeStatistics {
+    input.readStructBegin();
+    const args: ISizeStatisticsArgs = {};
+    while (true) {
+      const field = input.readFieldBegin();
+      if (field.ftype === thrift.Thrift.Type.STOP) break;
+      if (field.fid === 1 && field.ftype === thrift.Thrift.Type.I64) {
+        args.unencoded_byte_array_data_bytes = input.readI64();
+      } else if (field.fid === 2 && field.ftype === thrift.Thrift.Type.LIST) {
+        args.repetition_level_histogram = readInt64List(input);
+      } else if (field.fid === 3 && field.ftype === thrift.Thrift.Type.LIST) {
+        args.definition_level_histogram = readInt64List(input);
+      } else {
+        input.skip(field.ftype);
+      }
+      input.readFieldEnd();
+    }
+    input.readStructEnd();
+    return new SizeStatistics(args);
+  }
+}
+
+function toInt64(value: number | Int64 | undefined): Int64 | undefined {
+  return value === undefined ? undefined : typeof value === 'number' ? new Int64(value) : value;
+}
+
+function toInt64List(values: Array<number | Int64> | undefined): Array<Int64> | undefined {
+  return values?.map(value => toInt64(value)!);
+}
+
+function writeInt64List(
+  output: thrift.TProtocol,
+  name: string,
+  fieldId: number,
+  values: Array<Int64> | undefined
+): void {
+  if (!values) return;
+  output.writeFieldBegin(name, thrift.Thrift.Type.LIST, fieldId);
+  output.writeListBegin(thrift.Thrift.Type.I64, values.length);
+  for (const value of values) output.writeI64(value);
+  output.writeListEnd();
+  output.writeFieldEnd();
+}
+
+function readInt64List(input: thrift.TProtocol): Array<Int64> {
+  const list = input.readListBegin();
+  const values: Array<Int64> = [];
+  for (let index = 0; index < list.size; index++) values.push(input.readI64());
+  input.readListEnd();
+  return values;
+}

package/src/parquetjs/parquet-thrift/index.ts

@@ -48,6 +48,12 @@ export * from './PageHeader';
 export * from './KeyValue';
 export * from './SortingColumn';
 export * from './PageEncodingStats';
+export * from './BoundingBox';
+export * from './GeospatialStatistics';
+export * from './SizeStatistics';
+export * from './EncryptionAlgorithm';
+export * from './ColumnCryptoMetaData';
+export * from './FileCryptoMetaData';
 export * from './ColumnMetaData';
 export * from './ColumnChunk';
 export * from './RowGroup';