@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.3

package/dist/lib/parsers/parse-parquet-to-columns.js

@@ -0,0 +1,40 @@
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseParquetFileInColumnarBatches = exports.parseParquetInColumns = void 0;
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
+const convert_schema_from_parquet_1 = require("../arrow/convert-schema-from-parquet");
+const convert_row_group_to_columns_1 = require("../arrow/convert-row-group-to-columns");
+const decode_geo_metadata_1 = require("../geo/decode-geo-metadata");
+async function parseParquetInColumns(arrayBuffer, options) {
+    const blob = new Blob([arrayBuffer]);
+    for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+        return batch;
+    }
+    return null;
+}
+exports.parseParquetInColumns = parseParquetInColumns;
+async function* parseParquetFileInColumnarBatches(blob, options) {
+    const file = (0, loader_utils_1.makeReadableFile)(blob);
+    const reader = new parquet_reader_1.ParquetReader(file);
+    const parquetSchema = await reader.getSchema();
+    const parquetMetadata = await reader.getFileMetadata();
+    const schema = (0, convert_schema_from_parquet_1.convertSchemaFromParquet)(parquetSchema, parquetMetadata);
+    (0, decode_geo_metadata_1.unpackGeoMetadata)(schema);
+    const rowGroups = reader.rowGroupIterator(options?.parquet);
+    for await (const rowGroup of rowGroups) {
+        yield convertRowGroupToTableBatch(schema, rowGroup);
+    }
+}
+exports.parseParquetFileInColumnarBatches = parseParquetFileInColumnarBatches;
+function convertRowGroupToTableBatch(schema, rowGroup) {
+    const data = (0, convert_row_group_to_columns_1.convertParquetRowGroupToColumns)(schema, rowGroup);
+    return {
+        shape: 'columnar-table',
+        batchType: 'data',
+        schema,
+        data,
+        length: rowGroup.rowCount
+    };
+}

package/dist/lib/parsers/parse-parquet-to-rows.d.ts

@@ -0,0 +1,4 @@
+import type { ParquetLoaderOptions } from '../../parquet-loader';
+export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<import("../../parquetjs/schema/declare").ParquetRecord[] | null>;
+export declare function parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncGenerator<import("../../parquetjs/schema/declare").ParquetRecord[], void, unknown>;
+//# sourceMappingURL=parse-parquet-to-rows.d.ts.map

package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-rows.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAG/D,wBAAsB,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,oBAAoB,4EAM1F;AAED,wBAAuB,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,oBAAoB,2FAO1F"}

package/dist/lib/parsers/parse-parquet-to-rows.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseParquetFileInBatches = exports.parseParquet = void 0;
+// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+// import {ColumnarTableBatch} from '@loaders.gl/schema';
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const parquet_reader_1 = require("../../parquetjs/parser/parquet-reader");
+async function parseParquet(arrayBuffer, options) {
+    const blob = new Blob([arrayBuffer]);
+    for await (const batch of parseParquetFileInBatches(blob, options)) {
+        return batch;
+    }
+    return null;
+}
+exports.parseParquet = parseParquet;
+async function* parseParquetFileInBatches(blob, options) {
+    const file = (0, loader_utils_1.makeReadableFile)(blob);
+    const reader = new parquet_reader_1.ParquetReader(file);
+    const rowBatches = reader.rowBatchIterator(options?.parquet);
+    for await (const rows of rowBatches) {
+        yield rows;
+    }
+}
+exports.parseParquetFileInBatches = parseParquetFileInBatches;
+// export async function* parseParquetFileInColumnarBatches(blob: Blob, options?: {columnList?: string[][]}): AsyncIterable<ColumnarTableBatch> {
+//   const rowGroupReader = new ParquetRowGroupReader({data: blob, columnList: options?.columnList});
+//   try {
+//     for await (const rowGroup of rowGroupReader) {
+//       yield convertRowGroupToTableBatch(rowGroup);
+//     }
+//   } finally {
+//     await rowGroupReader.close();
+//   }
+// }
+// function convertRowGroupToTableBatch(rowGroup): ColumnarTableBatch {
+//   // @ts-expect-error
+//   return {
+//     data: rowGroup
+//   };
+// }

package/dist/parquet-loader.d.ts

@@ -3,6 +3,8 @@ export type ParquetLoaderOptions = LoaderOptions & {
     parquet?: {
         type?: 'object-row-table';
         url?: string;
+        columnList?: string[] | string[][];
+        geoparquet?: boolean;
     };
 };
 /** ParquetJS table loader */

package/dist/parquet-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,kBAAkB,CAAC;QAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC;AASF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYzB,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAsB,CAAC"}
+{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,MAAM,MAAM,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,kBAAkB,CAAC;QAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;QACnC,UAAU,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;CACH,CAAC;AAWF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYzB,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAsB,CAAC"}

package/dist/parquet-loader.js

@@ -7,7 +7,9 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {
     parquet: {
         type: 'object-row-table',
-        url: undefined
+        url: undefined,
+        columnList: [],
+        geoparquet: true
     }
 };
 /** ParquetJS table loader */