@loaders.gl/parquet 4.0.0-alpha.9 → 4.0.0-beta.1

package/src/parquet-wasm-writer.ts

@@ -1,17 +1,15 @@
+// loaders.gl, MIT license
+
 import type {Writer} from '@loaders.gl/loader-utils';
 import {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';
+import type {Table as ArrowTable} from 'apache-arrow';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-const DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {
-  parquet: {
-    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
-  }
-};
-
-export const ParquetWasmWriter: Writer = {
+/** Parquet WASM writer */
+export const ParquetWasmWriter: Writer<ArrowTable, never, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet-wasm',
   module: 'parquet',
@@ -20,5 +18,9 @@ export const ParquetWasmWriter: Writer = {
   mimeTypes: ['application/octet-stream'],
   encode,
   binary: true,
-  options: DEFAULT_PARQUET_WRITER_OPTIONS
+  options: {
+    parquet: {
+      wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+    }
+  }
 };

package/src/parquet-writer.ts

@@ -9,8 +9,6 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type ParquetWriterOptions = {};
 
-const DEFAULT_PARQUET_LOADER_OPTIONS = {};
-
 export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
@@ -20,7 +18,7 @@ export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
   mimeTypes: ['application/octet-stream'],
   encodeSync,
   binary: true,
-  options: DEFAULT_PARQUET_LOADER_OPTIONS
+  options: {}
 };
 
 function encodeSync(data, options?: ParquetWriterOptions) {

package/src/parquetjs/codecs/plain.ts

@@ -200,6 +200,7 @@ function decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[]
   for (let i = 0; i < count; i++) {
     const len = cursor.buffer.readUInt32LE(cursor.offset);
     cursor.offset += 4;
+    // values.push(cursor.buffer.buffer.slice(cursor.offset, cursor.offset + len));
     values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
     cursor.offset += len;
   }

package/src/parquetjs/codecs/rle.ts

@@ -63,6 +63,8 @@ export function encodeValues(
   }
 
   const envelope = Buffer.alloc(buf.length + 4);
+
+  // @ts-ignore buffer polyfill
   envelope.writeUInt32LE(buf.length, undefined);
   buf.copy(envelope, 4);
 

package/src/parquetjs/encoder/parquet-encoder.ts

@@ -632,6 +632,7 @@ function encodeFooter(
 
   const metadataEncoded = serializeThrift(metadata);
   const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
+
   metadataEncoded.copy(footerEncoded);
   footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
   footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);

package/src/parquetjs/parquet-thrift/index.ts

@@ -4,6 +4,10 @@
  * Autogenerated by @creditkarma/thrift-typescript v3.7.2
  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
  */
+export {Buffer} from '../../buffer-polyfill/install-buffer-polyfill';
+
+export {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';
+
 export * from './Type';
 export * from './ConvertedType';
 export * from './FieldRepetitionType';

package/src/parquetjs/parser/decoders.ts

@@ -2,7 +2,7 @@
 import {
   ParquetCodec,
   ParquetColumnChunk,
-  ParquetOptions,
+  ParquetReaderContext,
   ParquetPageData,
   ParquetType,
   PrimitiveType,
@@ -31,7 +31,7 @@ import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils'
  */
 export async function decodeDataPages(
   buffer: Buffer,
-  options: ParquetOptions
+  context: ParquetReaderContext
 ): Promise<ParquetColumnChunk> {
   const cursor: CursorBuffer = {
     buffer,
@@ -47,15 +47,15 @@ export async function decodeDataPages(
     count: 0
   };
 
-  let dictionary = options.dictionary || [];
+  let dictionary = context.dictionary || [];
 
   while (
     // @ts-ignore size can be undefined
     cursor.offset < cursor.size &&
-    (!options.numValues || data.dlevels.length < Number(options.numValues))
+    (!context.numValues || data.dlevels.length < Number(context.numValues))
   ) {
     // Looks like we have to decode these in sequence due to cursor updates?
-    const page = await decodePage(cursor, options);
+    const page = await decodePage(cursor, context);
 
     if (page.dictionary) {
       dictionary = page.dictionary;
@@ -88,13 +88,14 @@ export async function decodeDataPages(
 /**
  * Decode parquet page based on page type
  * @param cursor
- * @param options
+ * @param context
  */
 export async function decodePage(
   cursor: CursorBuffer,
-  options: ParquetOptions
+  context: ParquetReaderContext
 ): Promise<ParquetPageData> {
   let page;
+
   const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);
   cursor.offset += length;
 
@@ -102,14 +103,14 @@ export async function decodePage(
 
   switch (pageType) {
     case 'DATA_PAGE':
-      page = await decodeDataPage(cursor, pageHeader, options);
+      page = await decodeDataPage(cursor, pageHeader, context);
       break;
     case 'DATA_PAGE_V2':
-      page = await decodeDataPageV2(cursor, pageHeader, options);
+      page = await decodeDataPageV2(cursor, pageHeader, context);
       break;
     case 'DICTIONARY_PAGE':
       page = {
-        dictionary: await decodeDictionaryPage(cursor, pageHeader, options),
+        dictionary: await decodeDictionaryPage(cursor, pageHeader, context),
         pageHeader
       };
       break;
@@ -225,7 +226,7 @@ function decodeValues(
 async function decodeDataPage(
   cursor: CursorBuffer,
   header: PageHeader,
-  options: ParquetOptions
+  context: ParquetReaderContext
 ): Promise<ParquetPageData> {
   const cursorEnd = cursor.offset + header.compressed_page_size;
   const valueCount = header.data_page_header?.num_values;
@@ -233,9 +234,9 @@ async function decodeDataPage(
   /* uncompress page */
   let dataCursor = cursor;
 
-  if (options.compression !== 'UNCOMPRESSED') {
+  if (context.compression !== 'UNCOMPRESSED') {
     const valuesBuf = await decompress(
-      options.compression,
+      context.compression,
       cursor.buffer.slice(cursor.offset, cursorEnd),
       header.uncompressed_page_size
     );
@@ -255,9 +256,9 @@ async function decodeDataPage(
   // tslint:disable-next-line:prefer-array-literal
   let rLevels = new Array(valueCount);
 
-  if (options.column.rLevelMax > 0) {
+  if (context.column.rLevelMax > 0) {
     rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {
-      bitWidth: getBitWidth(options.column.rLevelMax),
+      bitWidth: getBitWidth(context.column.rLevelMax),
       disableEnvelope: false
       // column: opts.column
     });
@@ -272,9 +273,9 @@ async function decodeDataPage(
   ) as ParquetCodec;
   // tslint:disable-next-line:prefer-array-literal
   let dLevels = new Array(valueCount);
-  if (options.column.dLevelMax > 0) {
+  if (context.column.dLevelMax > 0) {
     dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {
-      bitWidth: getBitWidth(options.column.dLevelMax),
+      bitWidth: getBitWidth(context.column.dLevelMax),
       disableEnvelope: false
       // column: opts.column
     });
@@ -283,20 +284,20 @@ async function decodeDataPage(
   }
   let valueCountNonNull = 0;
   for (const dlvl of dLevels) {
-    if (dlvl === options.column.dLevelMax) {
+    if (dlvl === context.column.dLevelMax) {
       valueCountNonNull++;
     }
   }
 
   /* read values */
   const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;
-  const decodeOptions = {
-    typeLength: options.column.typeLength,
-    bitWidth: options.column.typeLength
+  const decodeOptions: ParquetCodecOptions = {
+    typeLength: context.column.typeLength,
+    bitWidth: context.column.typeLength
   };
 
   const values = decodeValues(
-    options.column.primitiveType!,
+    context.column.primitiveType!,
     valueEncoding,
     dataCursor,
     valueCountNonNull,
@@ -322,7 +323,7 @@ async function decodeDataPage(
 async function decodeDataPageV2(
   cursor: CursorBuffer,
   header: PageHeader,
-  opts: any
+  context: ParquetReaderContext
 ): Promise<ParquetPageData> {
   const cursorEnd = cursor.offset + header.compressed_page_size;
 
@@ -337,9 +338,9 @@ async function decodeDataPageV2(
   /* read repetition levels */
   // tslint:disable-next-line:prefer-array-literal
   let rLevels = new Array(valueCount);
-  if (opts.column.rLevelMax > 0) {
+  if (context.column.rLevelMax > 0) {
     rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {
-      bitWidth: getBitWidth(opts.column.rLevelMax),
+      bitWidth: getBitWidth(context.column.rLevelMax),
       disableEnvelope: true
     });
   } else {
@@ -349,9 +350,9 @@ async function decodeDataPageV2(
   /* read definition levels */
   // tslint:disable-next-line:prefer-array-literal
   let dLevels = new Array(valueCount);
-  if (opts.column.dLevelMax > 0) {
+  if (context.column.dLevelMax > 0) {
     dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {
-      bitWidth: getBitWidth(opts.column.dLevelMax),
+      bitWidth: getBitWidth(context.column.dLevelMax),
       disableEnvelope: true
     });
   } else {
@@ -363,7 +364,7 @@ async function decodeDataPageV2(
 
   if (header.data_page_header_v2?.is_compressed) {
     const valuesBuf = await decompress(
-      opts.compression,
+      context.compression,
       cursor.buffer.slice(cursor.offset, cursorEnd),
       header.uncompressed_page_size
     );
@@ -378,12 +379,12 @@ async function decodeDataPageV2(
   }
 
   const decodeOptions = {
-    typeLength: opts.column.typeLength,
-    bitWidth: opts.column.typeLength
+    typeLength: context.column.typeLength,
+    bitWidth: context.column.typeLength
   };
 
   const values = decodeValues(
-    opts.column.primitiveType!,
+    context.column.primitiveType!,
     valueEncoding,
     valuesBufCursor,
     valueCountNonNull,
@@ -403,13 +404,13 @@ async function decodeDataPageV2(
  * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.
  * @param cursor
  * @param pageHeader
- * @param options
+ * @param context
  */
 async function decodeDictionaryPage(
   cursor: CursorBuffer,
   pageHeader: PageHeader,
-  options: ParquetOptions
-): Promise<string[]> {
+  context: ParquetReaderContext
+): Promise<(string | ArrayBuffer)[]> {
   const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
 
   let dictCursor = {
@@ -420,9 +421,9 @@ async function decodeDictionaryPage(
 
   cursor.offset = cursorEnd;
 
-  if (options.compression !== 'UNCOMPRESSED') {
+  if (context.compression !== 'UNCOMPRESSED') {
     const valuesBuf = await decompress(
-      options.compression,
+      context.compression,
       dictCursor.buffer.slice(dictCursor.offset, cursorEnd),
       pageHeader.uncompressed_page_size
     );
@@ -438,11 +439,32 @@ async function decodeDictionaryPage(
 
   const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
 
-  return decodeValues(
-    options.column.primitiveType!,
-    options.column.encoding!,
+  const decodedDictionaryValues = decodeValues(
+    context.column.primitiveType!,
+    context.column.encoding!,
     dictCursor,
     numValues,
-    options as ParquetCodecOptions
-  ).map((d) => d.toString());
+    // TODO - this looks wrong?
+    context as ParquetCodecOptions
+  );
+
+  // Makes it look a little easier
+  let values: any[];
+  if (context?.preserveBinary) {
+    values = decodedDictionaryValues.map((d) => preserveBinary(d));
+  } else {
+    values = decodedDictionaryValues.map((d) => d.toString());
+  }
+  return values;
+}
+
+function preserveBinary(d: any): ArrayBuffer | ArrayBufferView | string {
+  if (ArrayBuffer.isView(d)) {
+    return d;
+  }
+  // Convert to ArrayBuffer
+  if (Buffer.isBuffer(d)) {
+    return d.buffer.slice(d.byteOffset, d.byteLength);
+  }
+  return d.toString();
 }

package/src/parquetjs/parser/parquet-reader.ts

@@ -12,13 +12,14 @@ import {
   ParquetCompression,
   ParquetColumnChunk,
   PrimitiveType,
-  ParquetOptions
+  ParquetReaderContext
 } from '../schema/declare';
 import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
 import {decodeDataPages, decodePage} from './decoders';
 
 export type ParquetReaderProps = {
   defaultDictionarySize?: number;
+  preserveBinary?: boolean;
 };
 
 /** Properties for initializing a ParquetRowGroupReader */
@@ -27,10 +28,6 @@ export type ParquetIterationProps = {
   columnList?: string[] | string[][];
 };
 
-const DEFAULT_PROPS: Required<ParquetReaderProps> = {
-  defaultDictionarySize: 1e6
-};
-
 /**
  * The parquet envelope reader allows direct, unbuffered access to the individual
  * sections of the parquet file, namely the header, footer and the row groups.
@@ -38,13 +35,18 @@ const DEFAULT_PROPS: Required<ParquetReaderProps> = {
  * rows from a parquet file use the ParquetReader instead
  */
 export class ParquetReader {
+  static defaultProps: Required<ParquetReaderProps> = {
+    defaultDictionarySize: 1e6,
+    preserveBinary: false
+  };
+
   props: Required<ParquetReaderProps>;
   file: ReadableFile;
   metadata: Promise<FileMetaData> | null = null;
 
   constructor(file: ReadableFile, props?: ParquetReaderProps) {
     this.file = file;
-    this.props = {...DEFAULT_PROPS, ...props};
+    this.props = {...ParquetReader.defaultProps, ...props};
   }
 
   close(): void {
@@ -163,6 +165,7 @@ export class ParquetReader {
     const metadataBuf = await this.file.read(metadataOffset, metadataSize);
     // let metadata = new parquet_thrift.FileMetaData();
     // parquet_util.decodeThrift(metadata, metadataBuf);
+
     const {metadata} = decodeFileMetadata(metadataBuf);
     return metadata;
   }
@@ -218,14 +221,16 @@ export class ParquetReader {
       );
     }
 
-    const options: ParquetOptions = {
+    const context: ParquetReaderContext = {
       type,
       rLevelMax: field.rLevelMax,
       dLevelMax: field.dLevelMax,
       compression,
       column: field,
       numValues: colChunk.meta_data?.num_values,
-      dictionary: []
+      dictionary: [],
+      // Options - TBD is this the right place for these?
+      preserveBinary: this.props.preserveBinary
     };
 
     let dictionary;
@@ -235,24 +240,24 @@ export class ParquetReader {
     if (dictionaryPageOffset) {
       const dictionaryOffset = Number(dictionaryPageOffset);
       // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
-      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+      dictionary = await this.getDictionary(dictionaryOffset, context, pagesOffset);
     }
 
-    dictionary = options.dictionary?.length ? options.dictionary : dictionary;
+    dictionary = context.dictionary?.length ? context.dictionary : dictionary;
     const pagesBuf = await this.file.read(pagesOffset, pagesSize);
-    return await decodeDataPages(pagesBuf, {...options, dictionary});
+    return await decodeDataPages(pagesBuf, {...context, dictionary});
   }
 
   /**
    * Getting dictionary for allows to flatten values by indices.
    * @param dictionaryPageOffset
-   * @param options
+   * @param context
    * @param pagesOffset
    * @returns
    */
   async getDictionary(
     dictionaryPageOffset: number,
-    options: ParquetOptions,
+    context: ParquetReaderContext,
     pagesOffset: number
   ): Promise<string[]> {
     if (dictionaryPageOffset === 0) {
@@ -273,7 +278,7 @@ export class ParquetReader {
     const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
 
     const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};
-    const decodedPage = await decodePage(cursor, options);
+    const decodedPage = await decodePage(cursor, context);
 
     return decodedPage.dictionary!;
   }

package/src/parquetjs/schema/declare.ts

@@ -99,7 +99,7 @@ export interface ParquetField {
 }
 
 /** @todo better name, this is an internal type? */
-export interface ParquetOptions {
+export interface ParquetReaderContext {
   type: ParquetType;
   rLevelMax: number;
   dLevelMax: number;
@@ -107,6 +107,8 @@ export interface ParquetOptions {
   column: ParquetField;
   numValues?: Int64;
   dictionary?: ParquetDictionary;
+  /** If true, binary values are not converted to strings */
+  preserveBinary?: boolean;
 }
 
 export interface ParquetPageData {

package/src/parquetjs/utils/read-utils.ts

@@ -1,4 +1,4 @@
-import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';
+import {TBufferedTransport, TCompactProtocol, TFramedTransport} from '../parquet-thrift';
 import {FileMetaData, PageHeader} from '../parquet-thrift';
 
 class UFramedTransport extends TFramedTransport {
@@ -12,7 +12,7 @@ export function serializeThrift(obj: any): Buffer {
   const output: Buffer[] = [];
 
   const transport = new TBufferedTransport(undefined, (buf) => {
-    output.push(buf as Buffer);
+    output.push(buf as unknown as Buffer);
   });
 
   const protocol = new TCompactProtocol(transport);

package/dist/bundle.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"bundle.d.ts","sourceRoot":"","sources":["../src/bundle.ts"],"names":[],"mappings":"AACA,QAAA,MAAM,aAAa,KAAqB,CAAC"}

package/dist/bundle.js

@@ -1,5 +0,0 @@
-"use strict";
-// @ts-nocheck
-const moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);

package/dist/constants.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AACA;;GAEG;AACH,eAAO,MAAM,aAAa,SAAS,CAAC;AACpC,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;GAEG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC;;GAEG;AACH,eAAO,MAAM,kBAAkB,UAAU,CAAC;AAC1C,eAAO,MAAM,sBAAsB,QAAQ,CAAC"}

package/dist/constants.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PARQUET_RDLVL_ENCODING = exports.PARQUET_RDLVL_TYPE = exports.PARQUET_VERSION = exports.PARQUET_MAGIC_ENCRYPTED = exports.PARQUET_MAGIC = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-/**
- * Parquet File Magic String
- */
-exports.PARQUET_MAGIC = 'PAR1';
-exports.PARQUET_MAGIC_ENCRYPTED = 'PARE';
-/**
- * Parquet File Format Version
- */
-exports.PARQUET_VERSION = 1;
-/**
- * Internal type used for repetition/definition levels
- */
-exports.PARQUET_RDLVL_TYPE = 'INT32';
-exports.PARQUET_RDLVL_ENCODING = 'RLE';

package/dist/es5/lib/geo/geoparquet-schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"geoparquet-schema.js","names":["$schema","title","description","type","required","properties","version","const","primary_column","minLength","columns","minProperties","patternProperties","encoding","geometry_types","uniqueItems","items","pattern","crs","oneOf","$ref","edges","enum","orientation","bbox","minItems","maxItems","epoch","additionalProperties","exports","default","_default"],"sources":["../../../../src/lib/geo/geoparquet-schema.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n/* eslint-disable camelcase */\n\n/**\n * Geoparquet JSON schema for geo metadata\n * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md\n * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md\n */\nexport default {\n  $schema: 'http://json-schema.org/draft-07/schema#',\n  title: 'GeoParquet',\n  description: 'Parquet metadata included in the geo field.',\n  type: 'object',\n  required: ['version', 'primary_column', 'columns'],\n  properties: {\n    version: {type: 'string', const: '1.0.0-beta.1'},\n    primary_column: {type: 'string', minLength: 1},\n    columns: {\n      type: 'object',\n      minProperties: 1,\n      patternProperties: {\n        '.+': {\n          type: 'object',\n          required: ['encoding', 'geometry_types'],\n          properties: {\n            encoding: {type: 'string', const: 'WKB'},\n            geometry_types: {\n              type: 'array',\n              uniqueItems: true,\n              items: {\n                type: 'string',\n                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'\n              }\n            },\n            crs: {\n              oneOf: [\n                {\n                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'\n                },\n                {type: 'null'}\n              ]\n            },\n            edges: {type: 'string', enum: ['planar', 'spherical']},\n            orientation: {type: 'string', const: 'counterclockwise'},\n            bbox: {\n              type: 'array',\n              items: {type: 'number'},\n              oneOf: [\n                {\n                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',\n                  minItems: 4,\n                  maxItems: 4\n                },\n                {\n                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',\n                  minItems: 6,\n                  maxItems: 6\n                }\n              ]\n            },\n            epoch: {type: 'number'}\n          }\n        }\n      },\n      additionalProperties: false\n    }\n  }\n};\n"],"mappings":";;;;;;eASe;EACbA,OAAO,EAAE,yCAAyC;EAClDC,KAAK,EAAE,YAAY;EACnBC,WAAW,EAAE,6CAA6C;EAC1DC,IAAI,EAAE,QAAQ;EACdC,QAAQ,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,SAAS,CAAC;EAClDC,UAAU,EAAE;IACVC,OAAO,EAAE;MAACH,IAAI,EAAE,QAAQ;MAAEI,KAAK,EAAE;IAAc,CAAC;IAChDC,cAAc,EAAE;MAACL,IAAI,EAAE,QAAQ;MAAEM,SAAS,EAAE;IAAC,CAAC;IAC9CC,OAAO,EAAE;MACPP,IAAI,EAAE,QAAQ;MACdQ,aAAa,EAAE,CAAC;MAChBC,iBAAiB,EAAE;QACjB,IAAI,EAAE;UACJT,IAAI,EAAE,QAAQ;UACdC,QAAQ,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAC;UACxCC,UAAU,EAAE;YACVQ,QAAQ,EAAE;cAACV,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAK,CAAC;YACxCO,cAAc,EAAE;cACdX,IAAI,EAAE,OAAO;cACbY,WAAW,EAAE,IAAI;cACjBC,KAAK,EAAE;gBACLb,IAAI,EAAE,QAAQ;gBACdc,OAAO,EAAE;cACX;YACF,CAAC;YACDC,GAAG,EAAE;cACHC,KAAK,EAAE,CACL;gBACEC,IAAI,EAAE;cACR,CAAC,EACD;gBAACjB,IAAI,EAAE;cAAM,CAAC;YAElB,CAAC;YACDkB,KAAK,EAAE;cAAClB,IAAI,EAAE,QAAQ;cAAEmB,IAAI,EAAE,CAAC,QAAQ,EAAE,WAAW;YAAC,CAAC;YACtDC,WAAW,EAAE;cAACpB,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAkB,CAAC;YACxDiB,IAAI,EAAE;cACJrB,IAAI,EAAE,OAAO;cACba,KAAK,EAAE;gBAACb,IAAI,EAAE;cAAQ,CAAC;cACvBgB,KAAK,EAAE,CACL;gBACEjB,WAAW,EAAE,gDAAgD;gBAC7DuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC,EACD;gBACExB,WAAW,EAAE,4DAA4D;gBACzEuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC;YAEL,CAAC;YACDC,KAAK,EAAE;cAACxB,IAAI,EAAE;YAAQ;UACxB;QACF;MACF,CAAC;MACDyB,oBAAoB,EAAE;IACxB;EACF;AACF,CAAC;AAAAC,OAAA,CAAAC,OAAA,GAAAC,QAAA"}

package/dist/esm/lib/geo/geoparquet-schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"geoparquet-schema.js","names":["$schema","title","description","type","required","properties","version","const","primary_column","minLength","columns","minProperties","patternProperties","encoding","geometry_types","uniqueItems","items","pattern","crs","oneOf","$ref","edges","enum","orientation","bbox","minItems","maxItems","epoch","additionalProperties"],"sources":["../../../../src/lib/geo/geoparquet-schema.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n/* eslint-disable camelcase */\n\n/**\n * Geoparquet JSON schema for geo metadata\n * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md\n * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md\n */\nexport default {\n  $schema: 'http://json-schema.org/draft-07/schema#',\n  title: 'GeoParquet',\n  description: 'Parquet metadata included in the geo field.',\n  type: 'object',\n  required: ['version', 'primary_column', 'columns'],\n  properties: {\n    version: {type: 'string', const: '1.0.0-beta.1'},\n    primary_column: {type: 'string', minLength: 1},\n    columns: {\n      type: 'object',\n      minProperties: 1,\n      patternProperties: {\n        '.+': {\n          type: 'object',\n          required: ['encoding', 'geometry_types'],\n          properties: {\n            encoding: {type: 'string', const: 'WKB'},\n            geometry_types: {\n              type: 'array',\n              uniqueItems: true,\n              items: {\n                type: 'string',\n                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'\n              }\n            },\n            crs: {\n              oneOf: [\n                {\n                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'\n                },\n                {type: 'null'}\n              ]\n            },\n            edges: {type: 'string', enum: ['planar', 'spherical']},\n            orientation: {type: 'string', const: 'counterclockwise'},\n            bbox: {\n              type: 'array',\n              items: {type: 'number'},\n              oneOf: [\n                {\n                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',\n                  minItems: 4,\n                  maxItems: 4\n                },\n                {\n                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',\n                  minItems: 6,\n                  maxItems: 6\n                }\n              ]\n            },\n            epoch: {type: 'number'}\n          }\n        }\n      },\n      additionalProperties: false\n    }\n  }\n};\n"],"mappings":"AASA,eAAe;EACbA,OAAO,EAAE,yCAAyC;EAClDC,KAAK,EAAE,YAAY;EACnBC,WAAW,EAAE,6CAA6C;EAC1DC,IAAI,EAAE,QAAQ;EACdC,QAAQ,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,SAAS,CAAC;EAClDC,UAAU,EAAE;IACVC,OAAO,EAAE;MAACH,IAAI,EAAE,QAAQ;MAAEI,KAAK,EAAE;IAAc,CAAC;IAChDC,cAAc,EAAE;MAACL,IAAI,EAAE,QAAQ;MAAEM,SAAS,EAAE;IAAC,CAAC;IAC9CC,OAAO,EAAE;MACPP,IAAI,EAAE,QAAQ;MACdQ,aAAa,EAAE,CAAC;MAChBC,iBAAiB,EAAE;QACjB,IAAI,EAAE;UACJT,IAAI,EAAE,QAAQ;UACdC,QAAQ,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAC;UACxCC,UAAU,EAAE;YACVQ,QAAQ,EAAE;cAACV,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAK,CAAC;YACxCO,cAAc,EAAE;cACdX,IAAI,EAAE,OAAO;cACbY,WAAW,EAAE,IAAI;cACjBC,KAAK,EAAE;gBACLb,IAAI,EAAE,QAAQ;gBACdc,OAAO,EAAE;cACX;YACF,CAAC;YACDC,GAAG,EAAE;cACHC,KAAK,EAAE,CACL;gBACEC,IAAI,EAAE;cACR,CAAC,EACD;gBAACjB,IAAI,EAAE;cAAM,CAAC;YAElB,CAAC;YACDkB,KAAK,EAAE;cAAClB,IAAI,EAAE,QAAQ;cAAEmB,IAAI,EAAE,CAAC,QAAQ,EAAE,WAAW;YAAC,CAAC;YACtDC,WAAW,EAAE;cAACpB,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAkB,CAAC;YACxDiB,IAAI,EAAE;cACJrB,IAAI,EAAE,OAAO;cACba,KAAK,EAAE;gBAACb,IAAI,EAAE;cAAQ,CAAC;cACvBgB,KAAK,EAAE,CACL;gBACEjB,WAAW,EAAE,gDAAgD;gBAC7DuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC,EACD;gBACExB,WAAW,EAAE,4DAA4D;gBACzEuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC;YAEL,CAAC;YACDC,KAAK,EAAE;cAACxB,IAAI,EAAE;YAAQ;UACxB;QACF;MACF,CAAC;MACDyB,oBAAoB,EAAE;IACxB;EACF;AACF,CAAC"}

package/dist/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,kBAAkB,EACnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,KAAK,IAAI,UAAU,EAAC,MAAM,cAAc,CAAC;AAItD,OAAO,EACL,aAAa,IAAI,mBAAmB,EAEpC,oBAAoB,EACrB,MAAM,kBAAkB,CAAC;AAO1B,OAAO,EAAmB,wBAAwB,EAAC,MAAM,+BAA+B,CAAC;AACzF,OAAO,EAAC,iBAAiB,IAAI,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AAEnF,OAAO,EAAC,mBAAmB,EAAE,uBAAuB,EAAC,CAAC;AAEtD,6BAA6B;AAC7B,eAAO,MAAM,aAAa,EAAE,gBAAgB,CAC1C,cAAc,EACd,mBAAmB,EACnB,oBAAoB,CAKrB,CAAC;AAEF,6BAA6B;AAE7B,eAAO,MAAM,qBAAqB,EAAE,gBAAgB,CAClD,aAAa,EACb,kBAAkB,EAClB,oBAAoB,CAKrB,CAAC;AAEF,eAAO,MAAM,iBAAiB,EAAE,gBAAgB,CAAC,UAAU,EAAE,KAAK,EAAE,wBAAwB,CAG3F,CAAC;AAIF,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAC,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,MAAM,yCAAyC,CAAC;AAGjD,eAAO,MAAM,uBAAuB,EAAE,gBAAgC,CAAC;AAGvE,OAAO,EAAC,OAAO,IAAI,aAAa,EAAC,MAAM,6BAA6B,CAAC;AAErE,YAAY,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAC,cAAc,EAAE,cAAc,EAAE,iBAAiB,EAAC,MAAM,+BAA+B,CAAC"}

package/dist/index.js

@@ -1,58 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = exports.geoJSONSchema = exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.convertParquetSchema = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetColumnarLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
-// ParquetLoader
-const parquet_loader_1 = require("./parquet-loader");
-Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
-const parse_parquet_to_rows_1 = require("./lib/parsers/parse-parquet-to-rows");
-const parse_parquet_to_columns_1 = require("./lib/parsers/parse-parquet-to-columns");
-const parse_parquet_wasm_1 = require("./lib/wasm/parse-parquet-wasm");
-const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
-Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
-/** ParquetJS table loader */
-exports.ParquetLoader = {
-    ...parquet_loader_1.ParquetLoader,
-    parse: parse_parquet_to_rows_1.parseParquet,
-    parseFileInBatches: parse_parquet_to_rows_1.parseParquetFileInBatches
-};
-/** ParquetJS table loader */
-// @ts-expect-error
-exports.ParquetColumnarLoader = {
-    ...parquet_loader_1.ParquetLoader,
-    parse: parse_parquet_to_columns_1.parseParquetInColumns,
-    parseFileInBatches: parse_parquet_to_columns_1.parseParquetFileInColumnarBatches
-};
-exports.ParquetWasmLoader = {
-    ...parquet_wasm_loader_1.ParquetWasmLoader,
-    parse: parse_parquet_wasm_1.parseParquetWasm
-};
-// ParquetWriter
-var parquet_writer_1 = require("./parquet-writer");
-Object.defineProperty(exports, "_ParquetWriter", { enumerable: true, get: function () { return parquet_writer_1.ParquetWriter; } });
-var parquet_wasm_writer_1 = require("./parquet-wasm-writer");
-Object.defineProperty(exports, "ParquetWasmWriter", { enumerable: true, get: function () { return parquet_wasm_writer_1.ParquetWasmWriter; } });
-// EXPERIMENTAL - expose the internal parquetjs API
-var compression_1 = require("./parquetjs/compression");
-Object.defineProperty(exports, "preloadCompressions", { enumerable: true, get: function () { return compression_1.preloadCompressions; } });
-var schema_1 = require("./parquetjs/schema/schema");
-Object.defineProperty(exports, "ParquetSchema", { enumerable: true, get: function () { return schema_1.ParquetSchema; } });
-var parquet_reader_1 = require("./parquetjs/parser/parquet-reader");
-Object.defineProperty(exports, "ParquetReader", { enumerable: true, get: function () { return parquet_reader_1.ParquetReader; } });
-var parquet_encoder_1 = require("./parquetjs/encoder/parquet-encoder");
-Object.defineProperty(exports, "ParquetEncoder", { enumerable: true, get: function () { return parquet_encoder_1.ParquetEncoder; } });
-var convert_schema_from_parquet_1 = require("./lib/arrow/convert-schema-from-parquet");
-Object.defineProperty(exports, "convertParquetSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
-Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
-// TESTS
-exports._typecheckParquetLoader = exports.ParquetLoader;
-// Geo Metadata
-var geoparquet_schema_1 = require("./lib/geo/geoparquet-schema");
-Object.defineProperty(exports, "geoJSONSchema", { enumerable: true, get: function () { return __importDefault(geoparquet_schema_1).default; } });
-var decode_geo_metadata_1 = require("./lib/geo/decode-geo-metadata");
-Object.defineProperty(exports, "getGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.getGeoMetadata; } });
-Object.defineProperty(exports, "setGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.setGeoMetadata; } });
-Object.defineProperty(exports, "unpackGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.unpackGeoMetadata; } });

package/dist/lib/arrow/convert-columns-to-row-group.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-columns-to-row-group.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-columns-to-row-group.ts"],"names":[],"mappings":""}

package/dist/lib/arrow/convert-columns-to-row-group.js

@@ -1 +0,0 @@
-"use strict";

package/dist/lib/arrow/convert-row-group-to-columns.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-row-group-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-row-group-to-columns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,eAAe,EAAC,MAAM,8CAA8C,CAAC;AAE7E,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAMvB"}

package/dist/lib/arrow/convert-row-group-to-columns.js

@@ -1,12 +0,0 @@
-"use strict";
-// loaders.gl, MIT license
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertParquetRowGroupToColumns = void 0;
-function convertParquetRowGroupToColumns(schema, rowGroup) {
-    const columns = {};
-    for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
-        columns[columnName] = columns[columnName] || data.values;
-    }
-    return columns;
-}
-exports.convertParquetRowGroupToColumns = convertParquetRowGroupToColumns;

package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAC,YAAY,EAAC,MAAM,gCAAgC,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,YAAY,GAAG,IAAI,GACnC,MAAM,CAUR"}