@loaders.gl/parquet 4.0.0-alpha.24 → 4.0.0-alpha.26

package/src/buffer-polyfill/index.ts

@@ -0,0 +1,9 @@
+// loaders.gl, MIT license
+export {Buffer} from './buffer-polyfill.node';
+export {BufferPolyfill} from './buffer-polyfill.node';
+
+export {installBufferPolyfill} from './buffer-polyfill.node';
+
+// import { installBufferPolyfill } from "./buffer-polyfill.node";
+
+// installBufferPolyfill();

package/src/buffer-polyfill/install-buffer-polyfill.ts

@@ -0,0 +1,6 @@
+export {Buffer} from './buffer-polyfill.node';
+export {BufferPolyfill} from './buffer-polyfill.node';
+
+import {installBufferPolyfill} from './buffer-polyfill.node';
+
+installBufferPolyfill();

package/src/index.ts

@@ -5,9 +5,11 @@ import type {
   ObjectRowTable,
   ObjectRowTableBatch,
   ColumnarTable,
-  ColumnarTableBatch
+  ColumnarTableBatch,
+  GeoJSONTable,
+  GeoJSONTableBatch
 } from '@loaders.gl/schema';
-import type {Table as ArrowTable} from 'apache-arrow';
+// import type {Table as ApacheArrowTable} from 'apache-arrow';
 
 // ParquetLoader
 
@@ -22,24 +24,26 @@ import {
   parseParquetFileInColumnarBatches
 } from './lib/parsers/parse-parquet-to-columns';
 
-import {parseParquetWasm, ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';
-import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
+// import type {ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';
+// import {parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
+// import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
 
-export {ParquetWorkerLoader, ParquetWasmWorkerLoader};
+export {ParquetWorkerLoader};
+// export {ParquetWasmWorkerLoader};
 
 /** ParquetJS table loader */
 export const ParquetLoader: LoaderWithParser<
-  ObjectRowTable,
-  ObjectRowTableBatch,
+  ObjectRowTable | GeoJSONTable,
+  ObjectRowTableBatch | GeoJSONTableBatch,
   ParquetLoaderOptions
 > = {
   ...ParquetWorkerLoader,
   parse: parseParquet,
+  // @ts-expect-error
   parseFileInBatches: parseParquetFileInBatches
 };
 
 /** ParquetJS table loader */
-// @ts-expect-error
 export const ParquetColumnarLoader: LoaderWithParser<
   ColumnarTable,
   ColumnarTableBatch,
@@ -47,18 +51,24 @@ export const ParquetColumnarLoader: LoaderWithParser<
 > = {
   ...ParquetColumnarWorkerLoader,
   parse: parseParquetInColumns,
+  // @ts-expect-error
   parseFileInBatches: parseParquetFileInColumnarBatches
 };
 
-export const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {
-  ...ParquetWasmWorkerLoader,
-  parse: parseParquetWasm
-};
+// export const ParquetWasmLoader: LoaderWithParser<
+//   ApacheArrowTable,
+//   never,
+//   ParquetWasmLoaderOptions
+// > = {
+//   ...ParquetWasmWorkerLoader,
+//   // @ts-expect-error Getting strange errors in wasm
+//   parse: () => {} // parseParquetWasm
+// };
 
 // ParquetWriter
 
 export {ParquetWriter as _ParquetWriter} from './parquet-writer';
-export {ParquetWasmWriter} from './parquet-wasm-writer';
+// export {ParquetWasmWriter} from './parquet-wasm-writer';
 
 // EXPERIMENTAL - expose the internal parquetjs API
 
@@ -74,7 +84,12 @@ export {
 } from './lib/arrow/convert-schema-from-parquet';
 
 // Geo Metadata
-export {default as geoJSONSchema} from './lib/geo/geoparquet-schema';
+// import {default as GEOPARQUET_METADATA_SCHEMA} from './lib/geo/geoparquet-metadata-schema.json';
+// export {GEOPARQUET_METADATA_SCHEMA};
+export {GEOPARQUET_METADATA_JSON_SCHEMA} from './lib/geo/geoparquet-metadata-schema';
 
 export type {GeoMetadata} from './lib/geo/decode-geo-metadata';
 export {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';
+
+// Experimental
+export {BufferPolyfill, installBufferPolyfill} from './buffer-polyfill';

package/src/lib/geo/decode-geo-column.ts

@@ -0,0 +1,51 @@
+// loaders.gl, MIT license
+
+import type {ArrayRowTable, GeoJSONTable, ObjectRowTable, Schema} from '@loaders.gl/schema';
+import type {Feature, Geometry} from '@loaders.gl/schema';
+import {getTableLength, getTableRowAsObject} from '@loaders.gl/schema';
+// import {binaryToGeometry} from '@loaders.gl/gis';
+import {HexWKBLoader, WKTLoader} from '@loaders.gl/wkt';
+
+import {GeoColumnMetadata, getGeoMetadata} from './decode-geo-metadata';
+
+/** TODO - move to loaders.gl/gis? */
+export function convertWKBTableToGeoJSON(
+  table: ArrayRowTable | ObjectRowTable,
+  schema: Schema
+): GeoJSONTable {
+  const geoMetadata = getGeoMetadata(schema);
+  const primaryColumn = geoMetadata?.primary_column;
+  if (!primaryColumn) {
+    throw new Error('no geometry column');
+  }
+  const columnMetadata = geoMetadata.columns[primaryColumn];
+
+  const features: Feature[] = [];
+
+  const length = getTableLength(table);
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    const row = getTableRowAsObject(table, rowIndex);
+    const geometry = parseGeometry(row[primaryColumn] as string, columnMetadata);
+    delete row[primaryColumn];
+    const feature: Feature = {type: 'Feature', geometry: geometry!, properties: row};
+    features.push(feature);
+  }
+
+  return {shape: 'geojson-table', schema, type: 'FeatureCollection', features};
+}
+
+function parseGeometry(geometry: string, columnMetadata: GeoColumnMetadata): Geometry | null {
+  switch (columnMetadata.encoding) {
+    case 'wkt':
+      return WKTLoader.parseTextSync?.(geometry) || null;
+    case 'wkb':
+    default:
+      const geojson = HexWKBLoader.parseTextSync?.(geometry, {
+        wkb: {shape: 'geometry'}
+      }) as unknown as Geometry;
+      return geojson; // binaryGeometry ? binaryToGeometry(binaryGeometry) : null;
+    // const binaryGeometry = WKBLoader.parseSync?.(geometry);
+    // ts-ignore
+    // return binaryGeometry ? binaryToGeometry(binaryGeometry) : null;
+  }
+}

package/src/lib/geo/decode-geo-metadata.ts

@@ -3,7 +3,28 @@ import {Schema, Field} from '@loaders.gl/schema';
 
 /* eslint-disable camelcase */
 
-/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */
+type GeometryType =
+  | 'Point'
+  | 'LineString'
+  | 'Polygon'
+  | 'MultiPoint'
+  | 'MultiLineString'
+  | 'MultiPolygon'
+  | 'GeometryCollection'
+  | 'Point Z'
+  | 'LineString Z'
+  | 'Polygon Z'
+  | 'MultiPoint Z'
+  | 'MultiLineString Z'
+  | 'MultiPolygon Z'
+  | 'GeometryCollection Z';
+
+/**
+ * A geoarrow / geoparquet geo metadata object
+ * (stored in stringified form in the top level metadata 'geo' key)
+ * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+ * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
+ * */
 export type GeoMetadata = {
   version?: string;
   primary_column?: string;
@@ -13,32 +34,55 @@ export type GeoMetadata = {
 
 /** A geoarrow / geoparquet geo metadata for one geometry column  */
 export type GeoColumnMetadata = {
-  bounding_box?:
-    | [number, number, number, number]
-    | [number, number, number, number, number, number];
-  crs?: string;
-  geometry_type?: string[];
-  edges?: string;
+  encoding: 'wkb' | 'wkt';
+  geometry_types: GeometryType[];
+  crs?: object | null;
+  orientation?: 'counterclockwise';
+  bbox?: [number, number, number, number] | [number, number, number, number, number, number];
+  edges?: 'planar' | 'spherical';
+  epoch?: number;
   [key: string]: unknown;
 };
 
-/**
- * Reads the GeoMetadata object from the metadata
- * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
-export function getGeoMetadata(schema: Schema): GeoMetadata | null {
-  const stringifiedGeoMetadata = schema.metadata.geo;
-  if (!stringifiedGeoMetadata) {
+/** Parse a key with stringified arrow metadata */
+export function parseJSONStringMetadata(
+  schema: Schema,
+  metadataKey: string
+): Record<string, unknown> | null {
+  const stringifiedMetadata = schema.metadata[metadataKey];
+  if (!stringifiedMetadata) {
     return null;
   }
 
   try {
-    const geoMetadata = JSON.parse(stringifiedGeoMetadata) as GeoMetadata;
-    return geoMetadata;
+    const metadata = JSON.parse(stringifiedMetadata);
+    if (!metadata || typeof metadata !== 'object') {
+      return null;
+    }
+    return metadata;
   } catch {
     return null;
   }
 }
 
+export function unpackJSONStringMetadata(schema: Schema, metadataKey: string): void {
+  const json = parseJSONStringMetadata(schema, metadataKey);
+  for (const [key, value] of Object.entries(json || {})) {
+    schema.metadata[`${metadataKey}.${key}`] =
+      typeof value === 'string' ? value : JSON.stringify(value);
+  }
+}
+
+// GEO METADATA
+
+/**
+ * Reads the GeoMetadata object from the metadata
+ * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
+export function getGeoMetadata(schema: Schema): GeoMetadata | null {
+  const geoMetadata = parseJSONStringMetadata(schema, 'geo') as GeoMetadata;
+  return geoMetadata;
+}
+
 /**
  * Stores a geoarrow / geoparquet geo metadata object in the schema
  * @note geoarrow / geoparquet geo metadata is a single stringified JSON field
@@ -83,6 +127,7 @@ export function unpackGeoMetadata(schema: Schema): void {
   }
 }
 
+// eslint-disable-next-line complexity
 function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
   for (const [key, value] of Object.entries(columnMetadata || {})) {
     switch (key) {
@@ -90,7 +135,28 @@ function unpackGeoFieldMetadata(field: Field, columnMetadata): void {
         setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));
         break;
       case 'bbox':
+        setFieldMetadata(field, `geo.crs.${key}`, JSON.stringify(value));
+        break;
       case 'crs':
+        for (const [crsKey, crsValue] of Object.entries(value || {})) {
+          switch (crsKey) {
+            case 'id':
+              const crsId =
+                typeof crsValue === 'object'
+                  ? `${crsValue?.authority}:${crsValue?.code}`
+                  : JSON.stringify(crsValue);
+              setFieldMetadata(field, `geo.crs.${crsKey}`, crsId);
+              break;
+            default:
+              setFieldMetadata(
+                field,
+                `geo.crs.${crsKey}`,
+                typeof crsValue === 'string' ? crsValue : JSON.stringify(crsValue)
+              );
+              break;
+          }
+        }
+        break;
       case 'edges':
       default:
         setFieldMetadata(

package/src/lib/geo/geoparquet-metadata-schema.json

@@ -0,0 +1,60 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "GeoParquet",
+  "description": "Parquet metadata included in the geo field.",
+  "type": "object",
+  "required": ["version", "primary_column", "columns"],
+  "properties": {
+    "version": {"type": "string", "const": "1.0.0-beta.1"},
+    "primary_column": {"type": "string", "minLength": 1},
+    "columns": {
+      "type": "object",
+      "minProperties": 1,
+      "patternProperties": {
+        ".+": {
+          "type": "object",
+          "required": ["encoding", "geometry_types"],
+          "properties": {
+            "encoding": {"type": "string", "const": "WKB"},
+            "geometry_types": {
+              "type": "array",
+              "uniqueItems": true,
+              "items": {
+                "type": "string",
+                "pattern": "^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$"
+              }
+            },
+            "crs": {
+              "oneOf": [
+                {
+                  "$ref": "https://proj.org/schemas/v0.5/projjson.schema.json"
+                },
+                {"type": "null"}
+              ]
+            },
+            "edges": {"type": "string", "enum": ["planar", "spherical"]},
+            "orientation": {"type": "string", "const": "counterclockwise"},
+            "bbox": {
+              "type": "array",
+              "items": {"type": "number"},
+              "oneOf": [
+                {
+                  "description": "2D bbox consisting of (xmin, ymin, xmax, ymax)",
+                  "minItems": 4,
+                  "maxItems": 4
+                },
+                {
+                  "description": "3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)",
+                  "minItems": 6,
+                  "maxItems": 6
+                }
+              ]
+            },
+            "epoch": {"type": "number"}
+          }
+        }
+      },
+      "additionalProperties": false
+    }
+  }
+}

package/src/lib/geo/{geoparquet-schema.ts → geoparquet-metadata-schema.ts}

@@ -7,7 +7,7 @@
  * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
  * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
  */
-export default {
+export const GEOPARQUET_METADATA_JSON_SCHEMA = {
   $schema: 'http://json-schema.org/draft-07/schema#',
   title: 'GeoParquet',
   description: 'Parquet metadata included in the geo field.',

package/src/lib/parsers/get-parquet-schema.ts

@@ -0,0 +1,14 @@
+// loaders.gl
+import {Schema} from '@loaders.gl/schema';
+import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
+import {unpackGeoMetadata, unpackJSONStringMetadata} from '../geo/decode-geo-metadata';
+
+export async function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema> {
+  const parquetSchema = await reader.getSchema();
+  const parquetMetadata = await reader.getFileMetadata();
+  const schema = convertParquetSchema(parquetSchema, parquetMetadata);
+  unpackGeoMetadata(schema);
+  unpackJSONStringMetadata(schema, 'pandas');
+  return schema;
+}

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -1,23 +1,25 @@
 // loaders.gl, MIT license
 
-// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
 import {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';
 import {makeReadableFile} from '@loaders.gl/loader-utils';
 import type {ParquetLoaderOptions} from '../../parquet-loader';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {ParquetRowGroup} from '../../parquetjs/schema/declare';
 import {ParquetSchema} from '../../parquetjs/schema/schema';
-import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
 import {materializeColumns} from '../../parquetjs/schema/shred';
-// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';
-import {unpackGeoMetadata} from '../geo/decode-geo-metadata';
+import {getSchemaFromParquetReader} from './get-parquet-schema';
+import {installBufferPolyfill} from '../../buffer-polyfill';
 
 export async function parseParquetInColumns(
   arrayBuffer: ArrayBuffer,
   options?: ParquetLoaderOptions
 ): Promise<ColumnarTable> {
+  installBufferPolyfill();
   const blob = new Blob([arrayBuffer]);
-  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+
+  for await (const batch of parseParquetFileInColumnarBatches(reader, options)) {
     return {
       shape: 'columnar-table',
       schema: batch.schema,
@@ -28,24 +30,24 @@ export async function parseParquetInColumns(
 }
 
 export async function* parseParquetFileInColumnarBatches(
-  blob: Blob,
+  reader: ParquetReader,
   options?: ParquetLoaderOptions
 ): AsyncIterable<ColumnarTableBatch> {
-  const file = makeReadableFile(blob);
-  const reader = new ParquetReader(file);
+  // Extract schema and geo metadata
+  const schema = await getSchemaFromParquetReader(reader);
+
   const parquetSchema = await reader.getSchema();
-  const parquetMetadata = await reader.getFileMetadata();
-  const schema = convertParquetSchema(parquetSchema, parquetMetadata);
-  unpackGeoMetadata(schema);
+
+  // Iterate over row batches
   const rowGroups = reader.rowGroupIterator(options?.parquet);
   for await (const rowGroup of rowGroups) {
-    yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
+    yield convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);
   }
 }
 
 function convertRowGroupToTableBatch(
-  parquetSchema: ParquetSchema,
   rowGroup: ParquetRowGroup,
+  parquetSchema: ParquetSchema,
   schema: Schema
 ): ColumnarTableBatch {
   // const data = convertParquetRowGroupToColumns(schema, rowGroup);

package/src/lib/parsers/parse-parquet-to-rows.ts

@@ -1,44 +1,71 @@
 // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
 // import {ColumnarTableBatch} from '@loaders.gl/schema';
 import {makeReadableFile} from '@loaders.gl/loader-utils';
-import {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
+import {GeoJSONTable, ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
 import type {ParquetLoaderOptions} from '../../parquet-loader';
 import type {ParquetRow} from '../../parquetjs/schema/declare';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
+import {getSchemaFromParquetReader} from './get-parquet-schema';
+import {installBufferPolyfill} from '../../buffer-polyfill';
+import {convertWKBTableToGeoJSON} from '../geo/decode-geo-column';
 
 export async function parseParquet(
   arrayBuffer: ArrayBuffer,
   options?: ParquetLoaderOptions
-): Promise<ObjectRowTable> {
+): Promise<ObjectRowTable | GeoJSONTable> {
+  installBufferPolyfill();
   const blob = new Blob([arrayBuffer]);
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file, {
+    preserveBinary: options?.parquet?.preserveBinary
+  });
+
+  const schema = await getSchemaFromParquetReader(reader);
 
   const rows: ParquetRow[] = [];
-  for await (const batch of parseParquetFileInBatches(blob, options)) {
+
+  const rowBatches = reader.rowBatchIterator(options?.parquet);
+  for await (const rowBatch of rowBatches) {
     // we have only one input batch so return
-    for (const row of batch.data) {
+    for (const row of rowBatch) {
       rows.push(row);
     }
   }
-
-  return {
+  const objectRowTable: ObjectRowTable = {
     shape: 'object-row-table',
-    // TODO - spread can fail for very large number of batches
+    schema,
     data: rows
   };
+
+  const shape = options?.parquet?.shape;
+  switch (shape) {
+    case 'object-row-table':
+      return objectRowTable;
+
+    case 'geojson-table':
+      try {
+        return convertWKBTableToGeoJSON(objectRowTable, schema);
+      } catch (error) {
+        return objectRowTable;
+      }
+
+    default:
+      throw new Error(shape);
+  }
 }
 
 export async function* parseParquetFileInBatches(
-  blob: Blob,
+  reader: ParquetReader,
   options?: ParquetLoaderOptions
 ): AsyncIterable<ObjectRowTableBatch> {
-  const file = makeReadableFile(blob);
-  const reader = new ParquetReader(file);
+  const schema = await getSchemaFromParquetReader(reader);
   const rowBatches = reader.rowBatchIterator(options?.parquet);
   for await (const rows of rowBatches) {
     yield {
+      batchType: 'data',
       shape: 'object-row-table',
+      schema,
       data: rows,
-      batchType: 'data',
       length: rows.length
     };
   }

package/src/parquet-loader.ts

@@ -4,33 +4,38 @@ import type {
   ObjectRowTable,
   ObjectRowTableBatch,
   ColumnarTable,
-  ColumnarTableBatch
+  ColumnarTableBatch,
+  GeoJSONTable,
+  GeoJSONTableBatch
 } from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+/** Options for the parquet loader */
 export type ParquetLoaderOptions = LoaderOptions & {
+  /** Options for the parquet loader */
   parquet?: {
-    type?: 'object-row-table';
-    url?: string;
+    /** Format of returned parsed data */
+    shape?: 'object-row-table' | 'geojson-table';
+    /** Restrict which columns that are parsed from the table. Can save significant memory. */
     columnList?: string[] | string[][];
+    /** If true, binary values are not converted to strings */
+    preserveBinary?: boolean;
+    /**  @deprecated not used? Set to true to indicate that this is a geoparquet file. */
     geoparquet?: boolean;
+    /** @deprecated URL to override loaders.gl/core parser system */
+    url?: string;
   };
 };
 
-const DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {
-  parquet: {
-    type: 'object-row-table',
-    url: undefined,
-    columnList: [],
-    geoparquet: true
-  }
-};
-
 /** ParquetJS table loader */
-export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions> = {
+export const ParquetLoader: Loader<
+  ObjectRowTable | GeoJSONTable,
+  ObjectRowTableBatch | GeoJSONTableBatch,
+  ParquetLoaderOptions
+> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -41,10 +46,18 @@ export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetL
   mimeTypes: ['application/octet-stream'],
   binary: true,
   tests: ['PAR1', 'PARE'],
-  options: DEFAULT_PARQUET_LOADER_OPTIONS
+  options: {
+    parquet: {
+      shape: 'object-row-table',
+      columnList: [],
+      geoparquet: true,
+      url: undefined,
+      preserveBinary: false
+    }
+  }
 };
 
-export const ParqueColumnnartLoader: Loader<
+export const ParquetColumnnarLoader: Loader<
   ColumnarTable,
   ColumnarTableBatch,
   ParquetLoaderOptions
@@ -59,5 +72,5 @@ export const ParqueColumnnartLoader: Loader<
   mimeTypes: ['application/octet-stream'],
   binary: true,
   tests: ['PAR1', 'PARE'],
-  options: DEFAULT_PARQUET_LOADER_OPTIONS
+  options: ParquetLoader.options
 };

package/src/parquetjs/codecs/plain.ts

@@ -200,6 +200,7 @@ function decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[]
   for (let i = 0; i < count; i++) {
     const len = cursor.buffer.readUInt32LE(cursor.offset);
     cursor.offset += 4;
+    // values.push(cursor.buffer.buffer.slice(cursor.offset, cursor.offset + len));
     values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
     cursor.offset += len;
   }

package/src/parquetjs/codecs/rle.ts

@@ -63,6 +63,8 @@ export function encodeValues(
   }
 
   const envelope = Buffer.alloc(buf.length + 4);
+
+  // @ts-ignore buffer polyfill
   envelope.writeUInt32LE(buf.length, undefined);
   buf.copy(envelope, 4);
 

package/src/parquetjs/encoder/parquet-encoder.ts

@@ -632,6 +632,7 @@ function encodeFooter(
 
   const metadataEncoded = serializeThrift(metadata);
   const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
+
   metadataEncoded.copy(footerEncoded);
   footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
   footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);

package/src/parquetjs/parquet-thrift/index.ts

@@ -4,6 +4,10 @@
  * Autogenerated by @creditkarma/thrift-typescript v3.7.2
  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
  */
+export {Buffer} from '../../buffer-polyfill/install-buffer-polyfill';
+
+export {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';
+
 export * from './Type';
 export * from './ConvertedType';
 export * from './FieldRepetitionType';