@loaders.gl/parquet 4.2.1 → 4.3.0-alpha.1

package/dist/lib/parsers/parse-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAO/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CA0BpC"}
+{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAO/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CAoCzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CA0BpC"}

package/dist/lib/parsers/parse-parquet.js

@@ -1,6 +1,7 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
+import { default as log } from '@probe.gl/log';
 import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
 import { getSchemaFromParquetReader } from "./get-parquet-schema.js";
 import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
@@ -21,10 +22,19 @@ export async function parseParquetFile(file, options) {
     const rows = [];
     const rowBatches = reader.rowBatchIterator(options?.parquet);
     for await (const rowBatch of rowBatches) {
+        let limitHasReached = false;
         // we have only one input batch so return
         for (const row of rowBatch) {
+            if (options?.limit && rows.length >= options?.limit) {
+                limitHasReached = true;
+                break;
+            }
             rows.push(row);
         }
+        if (limitHasReached) {
+            log.warn(`Rows number limit has been reached. Only first ${options?.limit} are loaded`)();
+            break;
+        }
     }
     const objectRowTable = {
         shape: 'object-row-table',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.2.1",
+  "version": "4.3.0-alpha.1",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -60,13 +60,14 @@
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.2.1",
-    "@loaders.gl/bson": "4.2.1",
-    "@loaders.gl/compression": "4.2.1",
-    "@loaders.gl/gis": "4.2.1",
-    "@loaders.gl/loader-utils": "4.2.1",
-    "@loaders.gl/schema": "4.2.1",
-    "@loaders.gl/wkt": "4.2.1",
+    "@loaders.gl/arrow": "4.3.0-alpha.1",
+    "@loaders.gl/bson": "4.3.0-alpha.1",
+    "@loaders.gl/compression": "4.3.0-alpha.1",
+    "@loaders.gl/gis": "4.3.0-alpha.1",
+    "@loaders.gl/loader-utils": "4.3.0-alpha.1",
+    "@loaders.gl/schema": "4.3.0-alpha.1",
+    "@loaders.gl/wkt": "4.3.0-alpha.1",
+    "@probe.gl/log": "^4.0.9",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
@@ -93,5 +94,5 @@
     "@loaders.gl/core": "^4.0.0",
     "apache-arrow": ">= 15.0.0"
   },
-  "gitHead": "b2ea799ff9f233ab8af0a6c3696e28ddf6e3c5a3"
+  "gitHead": "568446ea69eb590f0c42d965459b636216cda74e"
 }

package/src/lib/parsers/parse-parquet.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import {default as log} from '@probe.gl/log';
 import type {ReadableFile} from '@loaders.gl/loader-utils';
 import type {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
 
@@ -35,10 +36,19 @@ export async function parseParquetFile(
 
   const rowBatches = reader.rowBatchIterator(options?.parquet);
   for await (const rowBatch of rowBatches) {
+    let limitHasReached = false;
     // we have only one input batch so return
     for (const row of rowBatch) {
+      if (options?.limit && rows.length >= options?.limit) {
+        limitHasReached = true;
+        break;
+      }
       rows.push(row);
     }
+    if (limitHasReached) {
+      log.warn(`Rows number limit has been reached. Only first ${options?.limit} are loaded`)();
+      break;
+    }
   }
   const objectRowTable: ObjectRowTable = {
     shape: 'object-row-table',