@loaders.gl/parquet 4.2.0-alpha.6 → 4.2.0-beta.2

package/dist/lib/constants.js

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-export const VERSION = typeof "4.2.0-alpha.5" !== 'undefined' ? "4.2.0-alpha.5" : 'latest';
+export const VERSION = typeof "4.2.0-beta.1" !== 'undefined' ? "4.2.0-beta.1" : 'latest';
 export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';
 /**
  * Parquet File Magic String

package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAQ/D;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAUxB;AAED;;GAEG;AACH,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,kBAAkB,CAAC,CAanC"}
+{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAS/D;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAYxB;AAED;;GAEG;AACH,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,kBAAkB,CAAC,CAgBnC"}

package/dist/lib/parsers/parse-parquet-to-columns.js

@@ -5,11 +5,13 @@ import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
 import { materializeColumns } from "../../parquetjs/schema/shred.js";
 import { getSchemaFromParquetReader } from "./get-parquet-schema.js";
 import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
+import { preloadCompressions } from "../../parquetjs/compression.js";
 /**
  * @deprecated
  */
 export async function parseParquetFileInColumns(file, options) {
     installBufferPolyfill();
+    await preloadCompressions(options);
     for await (const batch of parseParquetFileInColumnarBatches(file, options)) {
         return {
             shape: 'columnar-table',
@@ -23,6 +25,8 @@ export async function parseParquetFileInColumns(file, options) {
  * @deprecated
  */
 export async function* parseParquetFileInColumnarBatches(file, options) {
+    installBufferPolyfill();
+    await preloadCompressions(options);
     const reader = new ParquetReader(file);
     // Extract schema and geo metadata
     const schema = await getSchemaFromParquetReader(reader);

package/dist/lib/parsers/parse-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAM/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CA0BzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CAuBpC"}
+{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAO/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CA0BpC"}

package/dist/lib/parsers/parse-parquet.js

@@ -4,6 +4,7 @@
 import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
 import { getSchemaFromParquetReader } from "./get-parquet-schema.js";
 import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
+import { preloadCompressions } from "../../parquetjs/compression.js";
 /**
  *  * Parse a parquet file using parquetjs
  * @param file
@@ -12,6 +13,7 @@ import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
  */
 export async function parseParquetFile(file, options) {
     installBufferPolyfill();
+    await preloadCompressions(options);
     const reader = new ParquetReader(file, {
         preserveBinary: options?.parquet?.preserveBinary
     });
@@ -38,6 +40,8 @@ export async function parseParquetFile(file, options) {
  * @param options
  */
 export async function* parseParquetFileInBatches(file, options) {
+    installBufferPolyfill();
+    await preloadCompressions(options);
     const reader = new ParquetReader(file, {
         preserveBinary: options?.parquet?.preserveBinary
     });

package/dist/parquet-loader.d.ts

@@ -1,5 +1,8 @@
-import type { Loader, LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
+import type { LoaderOptions } from '@loaders.gl/loader-utils';
 import type { ObjectRowTable, ObjectRowTableBatch, GeoJSONTable, GeoJSONTableBatch, ColumnarTable, ColumnarTableBatch } from '@loaders.gl/schema';
+import { parseParquetFile, parseParquetFileInBatches } from "./lib/parsers/parse-parquet.js";
+import { parseGeoParquetFile, parseGeoParquetFileInBatches } from "./lib/parsers/parse-geoparquet.js";
+import { parseParquetFileInColumns, parseParquetFileInColumnarBatches } from "./lib/parsers/parse-parquet-to-columns.js";
 export { Buffer } from "./polyfills/buffer/install-buffer-polyfill.js";
 /** Options for the parquet loader */
 export type ParquetLoaderOptions = LoaderOptions & {
@@ -20,14 +23,155 @@ export type ParquetLoaderOptions = LoaderOptions & {
 /**
  * ParquetJS table loader
  */
-export declare const ParquetWorkerLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions>;
+export declare const ParquetWorkerLoader: {
+    readonly dataType: ObjectRowTable;
+    readonly batchType: ObjectRowTableBatch;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: true;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly shape: "object-row-table";
+            readonly columnList: [];
+            readonly geoparquet: true;
+            readonly url: undefined;
+            readonly preserveBinary: false;
+        };
+    };
+};
 /** ParquetJS table loader */
-export declare const ParquetLoader: LoaderWithParser<ObjectRowTable | GeoJSONTable, ObjectRowTableBatch | GeoJSONTableBatch, ParquetLoaderOptions>;
-export declare const GeoParquetWorkerLoader: Loader<GeoJSONTable, GeoJSONTableBatch, ParquetLoaderOptions>;
+export declare const ParquetLoader: {
+    readonly dataType: ObjectRowTable | GeoJSONTable;
+    readonly batchType: ObjectRowTableBatch | GeoJSONTableBatch;
+    readonly parse: (arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) => Promise<ObjectRowTable>;
+    readonly parseFile: typeof parseParquetFile;
+    readonly parseFileInBatches: typeof parseParquetFileInBatches;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: true;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly shape: "object-row-table";
+            readonly columnList: [];
+            readonly geoparquet: true;
+            readonly url: undefined;
+            readonly preserveBinary: false;
+        };
+    };
+};
+export declare const GeoParquetWorkerLoader: {
+    readonly dataType: GeoJSONTable;
+    readonly batchType: GeoJSONTableBatch;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: true;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly shape: "geojson-table";
+            readonly columnList: [];
+            readonly geoparquet: true;
+            readonly url: undefined;
+            readonly preserveBinary: false;
+        };
+    };
+};
 /** ParquetJS table loader */
-export declare const GeoParquetLoader: LoaderWithParser<ObjectRowTable | GeoJSONTable, ObjectRowTableBatch | GeoJSONTableBatch, ParquetLoaderOptions>;
+export declare const GeoParquetLoader: {
+    readonly parse: (arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) => Promise<ObjectRowTable | GeoJSONTable>;
+    readonly parseFile: typeof parseGeoParquetFile;
+    readonly parseFileInBatches: typeof parseGeoParquetFileInBatches;
+    readonly dataType: GeoJSONTable;
+    readonly batchType: GeoJSONTableBatch;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: true;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly shape: "geojson-table";
+            readonly columnList: [];
+            readonly geoparquet: true;
+            readonly url: undefined;
+            readonly preserveBinary: false;
+        };
+    };
+};
 /** @deprecated Test to see if we can improve perf of parquetjs loader */
-export declare const ParquetColumnarWorkerLoader: Loader<ColumnarTable, ColumnarTableBatch, ParquetLoaderOptions>;
+export declare const ParquetColumnarWorkerLoader: {
+    readonly dataType: ColumnarTable;
+    readonly batchType: ColumnarTableBatch;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: true;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly shape: "object-row-table";
+            readonly columnList: [];
+            readonly geoparquet: true;
+            readonly url: undefined;
+            readonly preserveBinary: false;
+        };
+    };
+};
 /** @deprecated Test to see if we can improve perf of parquetjs loader */
-export declare const ParquetColumnarLoader: LoaderWithParser<ColumnarTable, ColumnarTableBatch, ParquetLoaderOptions>;
+export declare const ParquetColumnarLoader: {
+    readonly parse: (arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) => Promise<ColumnarTable>;
+    readonly parseFile: typeof parseParquetFileInColumns;
+    readonly parseFileInBatches: typeof parseParquetFileInColumnarBatches;
+    readonly dataType: ColumnarTable;
+    readonly batchType: ColumnarTableBatch;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: true;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly shape: "object-row-table";
+            readonly columnList: [];
+            readonly geoparquet: true;
+            readonly url: undefined;
+            readonly preserveBinary: false;
+        };
+    };
+};
 //# sourceMappingURL=parquet-loader.d.ts.map

package/dist/parquet-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EACnB,MAAM,oBAAoB,CAAC;AAU5B,OAAO,EAAC,MAAM,EAAC,sDAAmD;AAMlE,qCAAqC;AACrC,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,qCAAqC;IACrC,OAAO,CAAC,EAAE;QACR,qCAAqC;QACrC,KAAK,CAAC,EAAE,kBAAkB,GAAG,eAAe,CAAC;QAC7C,0FAA0F;QAC1F,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;QACnC,0DAA0D;QAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,qFAAqF;QACrF,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,gEAAgE;QAChE,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,EAAE,MAAM,CACtC,cAAc,EACd,mBAAmB,EACnB,oBAAoB,CAqBrB,CAAC;AAEF,6BAA6B;AAC7B,eAAO,MAAM,aAAa,EAAE,gBAAgB,CAC1C,cAAc,GAAG,YAAY,EAC7B,mBAAmB,GAAG,iBAAiB,EACvC,oBAAoB,CAQrB,CAAC;AAMF,eAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC,YAAY,EAAE,iBAAiB,EAAE,oBAAoB,CAqB9F,CAAC;AAEJ,6BAA6B;AAC7B,eAAO,MAAM,gBAAgB,EAAE,gBAAgB,CAC7C,cAAc,GAAG,YAAY,EAC7B,mBAAmB,GAAG,iBAAiB,EACvC,oBAAoB,CAQrB,CAAC;AAEF,yEAAyE;AACzE,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAC9C,aAAa,EACb,kBAAkB,EAClB,oBAAoB,CAarB,CAAC;AAEF,yEAAyE;AACzE,eAAO,MAAM,qBAAqB,EAAE,gBAAgB,CAClD,aAAa,EACb,kBAAkB,EAClB,oBAAoB,CAQrB,CAAC"}
+{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EACnB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAC,gBAAgB,EAAE,yBAAyB,EAAC,uCAAoC;AACxF,OAAO,EAAC,mBAAmB,EAAE,4BAA4B,EAAC,0CAAuC;AACjG,OAAO,EACL,yBAAyB,EACzB,iCAAiC,EAClC,kDAA+C;AAEhD,OAAO,EAAC,MAAM,EAAC,sDAAmD;AAMlE,qCAAqC;AACrC,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,qCAAqC;IACrC,OAAO,CAAC,EAAE;QACR,qCAAqC;QACrC,KAAK,CAAC,EAAE,kBAAkB,GAAG,eAAe,CAAC;QAC7C,0FAA0F;QAC1F,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;QACnC,0DAA0D;QAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,qFAAqF;QACrF,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,gEAAgE;QAChE,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;CAuBsD,CAAC;AAEvF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;kCAMH,WAAW,YAAY,oBAAoB;;;;;;;;;;;;;;;;;;;;;;CASjE,CAAC;AAMF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;CAuB+C,CAAC;AAEnF,6BAA6B;AAC7B,eAAO,MAAM,gBAAgB;kCAGR,WAAW,YAAY,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;CAS/D,CAAC;AAEF,yEAAyE;AACzE,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;CAe4C,CAAC;AAErF,yEAAyE;AACzE,eAAO,MAAM,qBAAqB;kCAEb,WAAW,YAAY,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;CAK8B,CAAC"}

package/dist/parquet-loader.js

@@ -8,11 +8,13 @@ import { parseParquetFileInColumns, parseParquetFileInColumnarBatches } from "./
 export { Buffer } from "./polyfills/buffer/install-buffer-polyfill.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0-alpha.5" !== 'undefined' ? "4.2.0-alpha.5" : 'latest';
+const VERSION = typeof "4.2.0-beta.1" !== 'undefined' ? "4.2.0-beta.1" : 'latest';
 /**
  * ParquetJS table loader
  */
 export const ParquetWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: 'Apache Parquet',
     id: 'parquet',
     module: 'parquet',
@@ -36,6 +38,8 @@ export const ParquetWorkerLoader = {
 /** ParquetJS table loader */
 export const ParquetLoader = {
     ...ParquetWorkerLoader,
+    dataType: null,
+    batchType: null,
     parse: (arrayBuffer, options) => parseParquetFile(new BlobFile(arrayBuffer), options),
     parseFile: parseParquetFile,
     parseFileInBatches: parseParquetFileInBatches
@@ -44,6 +48,8 @@ export const ParquetLoader = {
 // @ts-ignore
 ParquetLoader.Buffer = Buffer;
 export const GeoParquetWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: 'Apache Parquet',
     id: 'parquet',
     module: 'parquet',
@@ -75,6 +81,8 @@ export const GeoParquetLoader = {
 };
 /** @deprecated Test to see if we can improve perf of parquetjs loader */
 export const ParquetColumnarWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: 'Apache Parquet',
     id: 'parquet',
     module: 'parquet',

package/dist/parquet-wasm-loader.d.ts

@@ -1,4 +1,4 @@
-import type { Loader, LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
+import type { LoaderOptions } from '@loaders.gl/loader-utils';
 import type { ArrowTable } from '@loaders.gl/arrow';
 /** Parquet WASM loader options */
 export type ParquetWasmLoaderOptions = LoaderOptions & {
@@ -8,7 +8,26 @@ export type ParquetWasmLoaderOptions = LoaderOptions & {
     };
 };
 /** Parquet WASM table loader */
-export declare const ParquetWasmWorkerLoader: Loader<ArrowTable, never, ParquetWasmLoaderOptions>;
+export declare const ParquetWasmWorkerLoader: {
+    readonly dataType: ArrowTable;
+    readonly batchType: never;
+    readonly name: "Apache Parquet";
+    readonly id: "parquet-wasm";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly worker: false;
+    readonly category: "table";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+    readonly options: {
+        readonly parquet: {
+            readonly type: "arrow-table";
+            readonly wasmUrl: "https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm";
+        };
+    };
+};
 /** Parquet WASM table loader */
-export declare const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions>;
+export declare const ParquetWasmLoader: any;
 //# sourceMappingURL=parquet-wasm-loader.d.ts.map

package/dist/parquet-wasm-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-wasm-loader.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAKlD,kCAAkC;AAClC,MAAM,MAAM,wBAAwB,GAAG,aAAa,GAAG;IACrD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,uBAAuB,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,wBAAwB,CAiBvF,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,iBAAiB,EAAE,gBAAgB,CAAC,UAAU,EAAE,KAAK,EAAE,wBAAwB,CAM3F,CAAC"}
+{"version":3,"file":"parquet-wasm-loader.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAKlD,kCAAkC;AAClC,MAAM,MAAM,wBAAwB,GAAG,aAAa,GAAG;IACrD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;CAoBoC,CAAC;AAEzE,gCAAgC;AAChC,eAAO,MAAM,iBAAiB,KAMoD,CAAC"}

package/dist/parquet-wasm-loader.js

@@ -5,6 +5,8 @@ import { parseParquetWasm } from "./lib/wasm/parse-parquet-wasm.js";
 import { VERSION, PARQUET_WASM_URL } from "./lib/constants.js";
 /** Parquet WASM table loader */
 export const ParquetWasmWorkerLoader = {
+    dataType: null,
+    batchType: null,
     name: 'Apache Parquet',
     id: 'parquet-wasm',
     module: 'parquet',

package/dist/parquet-wasm-writer.d.ts

@@ -1,5 +1,3 @@
-import type { WriterWithEncoder } from '@loaders.gl/loader-utils';
-import type { ArrowTable } from '@loaders.gl/arrow';
 import type { WriterOptions } from '@loaders.gl/loader-utils';
 export type ParquetWriterOptions = WriterOptions & {
     parquet?: {
@@ -7,5 +5,5 @@ export type ParquetWriterOptions = WriterOptions & {
     };
 };
 /** Parquet WASM writer */
-export declare const ParquetWasmWriter: WriterWithEncoder<ArrowTable, never, ParquetWriterOptions>;
+export declare const ParquetWasmWriter: any;
 //# sourceMappingURL=parquet-wasm-writer.d.ts.map

package/dist/parquet-wasm-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-wasm-writer.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-writer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAI5D,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,0BAA0B;AAC1B,eAAO,MAAM,iBAAiB,EAAE,iBAAiB,CAAC,UAAU,EAAE,KAAK,EAAE,oBAAoB,CAiBxF,CAAC"}
+{"version":3,"file":"parquet-wasm-writer.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-writer.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAI5D,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,0BAA0B;AAC1B,eAAO,MAAM,iBAAiB,KAiBiD,CAAC"}

package/dist/parquet-writer.d.ts

@@ -1,5 +1,17 @@
-import type { WriterWithEncoder } from '@loaders.gl/loader-utils';
-import { Table, TableBatch } from '@loaders.gl/schema';
+import { Table } from '@loaders.gl/schema';
 export type ParquetWriterOptions = {};
-export declare const ParquetWriter: WriterWithEncoder<Table, TableBatch, ParquetWriterOptions>;
+export declare const ParquetWriter: {
+    readonly name: "Apache Parquet";
+    readonly id: "parquet";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly options: {};
+    readonly encode: (data: Table, options: ParquetWriterOptions | undefined) => Promise<ArrayBuffer>;
+    readonly encodeSync: typeof encodeSync;
+};
+declare function encodeSync(data: any, options?: ParquetWriterOptions): ArrayBuffer;
+export {};
 //# sourceMappingURL=parquet-writer.d.ts.map

package/dist/parquet-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAC,KAAK,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEtC,eAAO,MAAM,aAAa,EAAE,iBAAiB,CAAC,KAAK,EAAE,UAAU,EAAE,oBAAoB,CAWpF,CAAC"}
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAKA,OAAO,EAAC,KAAK,EAAa,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEtC,eAAO,MAAM,aAAa;;;;;;;;;;;CAWqD,CAAC;AAEhF,iBAAS,UAAU,CAAC,IAAI,KAAA,EAAE,OAAO,CAAC,EAAE,oBAAoB,eAEvD"}

package/dist/parquet-writer.js

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0-alpha.5" !== 'undefined' ? "4.2.0-alpha.5" : 'latest';
+const VERSION = typeof "4.2.0-beta.1" !== 'undefined' ? "4.2.0-beta.1" : 'latest';
 export const ParquetWriter = {
     name: 'Apache Parquet',
     id: 'parquet',

package/dist/parquetjs/codecs/rle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,qBAAkB;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA4DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CA4BV"}
+{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,qBAAkB;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA4DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CAgCV"}

package/dist/parquetjs/codecs/rle.js

@@ -67,13 +67,17 @@ export function decodeValues(type, cursor, count, opts) {
     while (values.length < count) {
         const header = varint.decode(cursor.buffer, cursor.offset);
         cursor.offset += varint.encodingLength(header);
+        let decodedValues;
         if (header & 1) {
             const count = (header >> 1) * 8;
-            values.push(...decodeRunBitpacked(cursor, count, opts));
+            decodedValues = decodeRunBitpacked(cursor, count, opts);
         }
         else {
             const count = header >> 1;
-            values.push(...decodeRunRepeated(cursor, count, opts));
+            decodedValues = decodeRunRepeated(cursor, count, opts);
+        }
+        for (const value of decodedValues) {
+            values.push(value);
         }
     }
     values = values.slice(0, count);

package/dist/parquetjs/compression.d.ts

@@ -10,7 +10,7 @@ export declare const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Com
  * @param options.modules External library dependencies
  */
 export declare function preloadCompressions(options?: {
-    modules: {
+    modules?: {
         [key: string]: any;
     };
 }): Promise<void[]>;

package/dist/parquetjs/compression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,4BAAyB;AAsCpD;;GAEG;AAEH,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAW/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}
+{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAC,kBAAkB,EAAC,4BAAyB;AAsCpD;;GAEG;AAEH,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAW/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,CAAC,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAMnF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}

package/dist/parquetjs/compression.js

@@ -4,6 +4,7 @@
 import { NoCompression, GZipCompression, SnappyCompression, BrotliCompression, 
 // LZOCompression,
 LZ4Compression, ZstdCompression } from '@loaders.gl/compression';
+import { registerJSModules } from '@loaders.gl/loader-utils';
 /** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
 function toBuffer(arrayBuffer) {
     return Buffer.from(arrayBuffer);
@@ -56,8 +57,9 @@ export const PARQUET_COMPRESSION_METHODS = {
  * @param options.modules External library dependencies
  */
 export async function preloadCompressions(options) {
+    registerJSModules(options?.modules);
     const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-    return await Promise.all(compressions.map((compression) => compression.preload()));
+    return await Promise.all(compressions.map((compression) => compression.preload(options?.modules)));
 }
 /**
  * Deflate a value using compression method `method`

package/dist/parquetjs/parser/decoders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAmD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}
+{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAyD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}

package/dist/parquetjs/parser/decoders.js

@@ -35,7 +35,10 @@ export async function decodeDataPages(buffer, context) {
             // eslint-disable-next-line no-continue
             continue;
         }
-        if (dictionary.length) {
+        const valueEncoding = getThriftEnum(Encoding, page.pageHeader.data_page_header?.encoding);
+        // Pages might be in different encodings. We don't need to decode in case
+        // of 'PLAIN' encoding because all values are already in place
+        if (dictionary.length && valueEncoding !== 'PLAIN') {
             // eslint-disable-next-line no-loop-func
             page.values = page.values.map((value) => dictionary[value]);
         }

package/dist/parquetjs/schema/types.js

@@ -296,6 +296,7 @@ function fromPrimitive_JSON(value) {
     return JSON.parse(value);
 }
 function toPrimitive_BSON(value) {
+    // @ts-ignore
     const arrayBuffer = BSONWriter.encodeSync?.(value);
     return Buffer.from(arrayBuffer);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.2.0-alpha.6",
+  "version": "4.2.0-beta.2",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -60,13 +60,13 @@
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.2.0-alpha.6",
-    "@loaders.gl/bson": "4.2.0-alpha.6",
-    "@loaders.gl/compression": "4.2.0-alpha.6",
-    "@loaders.gl/gis": "4.2.0-alpha.6",
-    "@loaders.gl/loader-utils": "4.2.0-alpha.6",
-    "@loaders.gl/schema": "4.2.0-alpha.6",
-    "@loaders.gl/wkt": "4.2.0-alpha.6",
+    "@loaders.gl/arrow": "4.2.0-beta.2",
+    "@loaders.gl/bson": "4.2.0-beta.2",
+    "@loaders.gl/compression": "4.2.0-beta.2",
+    "@loaders.gl/gis": "4.2.0-beta.2",
+    "@loaders.gl/loader-utils": "4.2.0-beta.2",
+    "@loaders.gl/schema": "4.2.0-beta.2",
+    "@loaders.gl/wkt": "4.2.0-beta.2",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
@@ -93,5 +93,5 @@
     "@loaders.gl/core": "^4.0.0",
     "apache-arrow": ">= 15.0.0"
   },
-  "gitHead": "37bd8ca71763529f18727ee4bf29dd176aa914ca"
+  "gitHead": "13c03663fec365a4d24c06377d50035cebd7ba9f"
 }

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -11,6 +11,7 @@ import {ParquetSchema} from '../../parquetjs/schema/schema';
 import {materializeColumns} from '../../parquetjs/schema/shred';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
 import {installBufferPolyfill} from '../../polyfills/buffer/index';
+import {preloadCompressions} from '../../parquetjs/compression';
 
 /**
  * @deprecated
@@ -20,6 +21,8 @@ export async function parseParquetFileInColumns(
   options?: ParquetLoaderOptions
 ): Promise<ColumnarTable> {
   installBufferPolyfill();
+  await preloadCompressions(options);
+
   for await (const batch of parseParquetFileInColumnarBatches(file, options)) {
     return {
       shape: 'columnar-table',
@@ -37,6 +40,9 @@ export async function* parseParquetFileInColumnarBatches(
   file: ReadableFile,
   options?: ParquetLoaderOptions
 ): AsyncIterable<ColumnarTableBatch> {
+  installBufferPolyfill();
+  await preloadCompressions(options);
+
   const reader = new ParquetReader(file);
 
   // Extract schema and geo metadata

package/src/lib/parsers/parse-parquet.ts

@@ -10,6 +10,7 @@ import type {ParquetRow} from '../../parquetjs/schema/declare';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
 import {installBufferPolyfill} from '../../polyfills/buffer/index';
+import {preloadCompressions} from '../../parquetjs/compression';
 
 /**
  *  * Parse a parquet file using parquetjs
@@ -22,6 +23,7 @@ export async function parseParquetFile(
   options?: ParquetLoaderOptions
 ): Promise<ObjectRowTable> {
   installBufferPolyfill();
+  await preloadCompressions(options);
 
   const reader = new ParquetReader(file, {
     preserveBinary: options?.parquet?.preserveBinary
@@ -57,6 +59,9 @@ export async function* parseParquetFileInBatches(
   file: ReadableFile,
   options?: ParquetLoaderOptions
 ): AsyncIterable<ObjectRowTableBatch> {
+  installBufferPolyfill();
+  await preloadCompressions(options);
+
   const reader = new ParquetReader(file, {
     preserveBinary: options?.parquet?.preserveBinary
   });