@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/lib/convert-schema.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertParquetToArrowSchema = exports.PARQUET_TYPE_MAPPING = void 0;
+const schema_1 = require("@loaders.gl/schema");
+exports.PARQUET_TYPE_MAPPING = {
+    BOOLEAN: schema_1.Bool,
+    INT32: schema_1.Int32,
+    INT64: schema_1.Float64,
+    INT96: schema_1.Float64,
+    FLOAT: schema_1.Float32,
+    DOUBLE: schema_1.Float64,
+    BYTE_ARRAY: schema_1.Binary,
+    FIXED_LEN_BYTE_ARRAY: schema_1.Binary,
+    UTF8: schema_1.Utf8,
+    DATE: schema_1.Int32,
+    TIME_MILLIS: schema_1.Int64,
+    TIME_MICROS: schema_1.Int64,
+    TIMESTAMP_MILLIS: schema_1.Int64,
+    TIMESTAMP_MICROS: schema_1.Int64,
+    UINT_8: schema_1.Int32,
+    UINT_16: schema_1.Uint16,
+    UINT_32: schema_1.Uint32,
+    UINT_64: schema_1.Uint64,
+    INT_8: schema_1.Int8,
+    INT_16: schema_1.Int16,
+    INT_32: schema_1.Int32,
+    INT_64: schema_1.Int64,
+    JSON: schema_1.Binary,
+    BSON: schema_1.Binary,
+    // TODO check interal type
+    INTERVAL: schema_1.Binary,
+    DECIMAL_INT32: schema_1.Float32,
+    DECIMAL_INT64: schema_1.Float64,
+    DECIMAL_BYTE_ARRAY: schema_1.Float64,
+    DECIMAL_FIXED_LEN_BYTE_ARRAY: schema_1.Float64
+};
+function convertParquetToArrowSchema(parquetSchema) {
+    const fields = getFields(parquetSchema.schema);
+    // TODO add metadata if needed.
+    return new schema_1.Schema(fields);
+}
+exports.convertParquetToArrowSchema = convertParquetToArrowSchema;
+function getFieldMetadata(field) {
+    const metadata = new Map();
+    for (const key in field) {
+        if (key !== 'name') {
+            const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+            metadata.set(key, value);
+        }
+    }
+    return metadata;
+}
+function getFields(schema) {
+    const fields = [];
+    for (const name in schema) {
+        const field = schema[name];
+        if (field.fields) {
+            const childField = getFields(field.fields);
+            const nestedField = new schema_1.Field(name, new schema_1.Struct(childField), field.optional);
+            fields.push(nestedField);
+        }
+        else {
+            const FieldType = exports.PARQUET_TYPE_MAPPING[field.type];
+            const metadata = getFieldMetadata(field);
+            const arrowField = new schema_1.Field(name, new FieldType(), field.optional, metadata);
+            fields.push(arrowField);
+        }
+    }
+    return fields;
+}

package/dist/lib/parse-parquet.d.ts

@@ -0,0 +1,4 @@
+import type { ParquetLoaderOptions } from '../parquet-loader';
+export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<any[][] | null>;
+export declare function parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncGenerator<any[][], void, unknown>;
+//# sourceMappingURL=parse-parquet.d.ts.map

package/dist/lib/parse-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../src/lib/parse-parquet.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAI5D,wBAAsB,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,oBAAoB,2BAM1F;AAED,wBAAuB,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,oBAAoB,0CAa1F"}

package/dist/lib/parse-parquet.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseParquetFileInBatches = exports.parseParquet = void 0;
+const parquet_reader_1 = require("../parquetjs/parser/parquet-reader");
+async function parseParquet(arrayBuffer, options) {
+    const blob = new Blob([arrayBuffer]);
+    for await (const batch of parseParquetFileInBatches(blob, options)) {
+        return batch;
+    }
+    return null;
+}
+exports.parseParquet = parseParquet;
+async function* parseParquetFileInBatches(blob, options) {
+    const reader = await parquet_reader_1.ParquetReader.openBlob(blob);
+    const rows = [];
+    try {
+        const cursor = reader.getCursor();
+        let record;
+        while ((record = await cursor.next())) {
+            rows.push(record);
+        }
+    }
+    finally {
+        await reader.close();
+    }
+    yield rows;
+}
+exports.parseParquetFileInBatches = parseParquetFileInBatches;

package/dist/lib/read-array-buffer.d.ts

@@ -0,0 +1,19 @@
+export declare function readArrayBuffer(file: Blob | ArrayBuffer | any, start: number, length: number): Promise<ArrayBuffer>;
+/**
+ * Read a slice of a Blob or File, without loading the entire file into memory
+ * The trick when reading File objects is to read successive "slices" of the File
+ * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
+ * Actually reading from file happens in `readAsArrayBuffer`
+ * @param blob to read
+ export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
+  return await new Promise((resolve, reject) => {
+    const fileReader = new FileReader();
+    fileReader.onload = (event: ProgressEvent<FileReader>) =>
+      resolve(event?.target?.result as ArrayBuffer);
+    // TODO - reject with a proper Error
+    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
+    fileReader.readAsArrayBuffer(blob);
+  });
+}
+*/
+//# sourceMappingURL=read-array-buffer.d.ts.map

package/dist/lib/read-array-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-array-buffer.d.ts","sourceRoot":"","sources":["../../src/lib/read-array-buffer.ts"],"names":[],"mappings":"AAEA,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,GAAG,WAAW,GAAG,GAAG,EAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,CAAC,CAMtB;AAED;;;;;;;;;;;;;;;;EAgBE"}

package/dist/lib/read-array-buffer.js

@@ -0,0 +1,29 @@
+"use strict";
+// Random-Access read
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readArrayBuffer = void 0;
+async function readArrayBuffer(file, start, length) {
+    if (file instanceof Blob) {
+        const slice = file.slice(start, start + length);
+        return await slice.arrayBuffer();
+    }
+    return await file.read(start, start + length);
+}
+exports.readArrayBuffer = readArrayBuffer;
+/**
+ * Read a slice of a Blob or File, without loading the entire file into memory
+ * The trick when reading File objects is to read successive "slices" of the File
+ * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
+ * Actually reading from file happens in `readAsArrayBuffer`
+ * @param blob to read
+ export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
+  return await new Promise((resolve, reject) => {
+    const fileReader = new FileReader();
+    fileReader.onload = (event: ProgressEvent<FileReader>) =>
+      resolve(event?.target?.result as ArrayBuffer);
+    // TODO - reject with a proper Error
+    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
+    fileReader.readAsArrayBuffer(blob);
+  });
+}
+*/

package/dist/parquet-loader.d.ts

@@ -0,0 +1,23 @@
+import type { Loader, LoaderOptions } from '@loaders.gl/loader-utils';
+export declare type ParquetLoaderOptions = LoaderOptions & {
+    parquet?: {
+        type?: 'object-row-table';
+        url?: string;
+    };
+};
+/** ParquetJS table loader */
+export declare const ParquetLoader: {
+    name: string;
+    id: string;
+    module: string;
+    version: any;
+    worker: boolean;
+    category: string;
+    extensions: string[];
+    mimeTypes: string[];
+    binary: boolean;
+    tests: string[];
+    options: ParquetLoaderOptions;
+};
+export declare const _typecheckParquetLoader: Loader;
+//# sourceMappingURL=parquet-loader.d.ts.map

package/dist/parquet-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,oBAAY,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,kBAAkB,CAAC;QAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC;AASF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYzB,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAsB,CAAC"}

package/dist/parquet-loader.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+const DEFAULT_PARQUET_LOADER_OPTIONS = {
+    parquet: {
+        type: 'object-row-table',
+        url: undefined
+    }
+};
+/** ParquetJS table loader */
+exports.ParquetLoader = {
+    name: 'Apache Parquet',
+    id: 'parquet',
+    module: 'parquet',
+    version: VERSION,
+    worker: true,
+    category: 'table',
+    extensions: ['parquet'],
+    mimeTypes: ['application/octet-stream'],
+    binary: true,
+    tests: ['PAR1', 'PARE'],
+    options: DEFAULT_PARQUET_LOADER_OPTIONS
+};
+exports._typecheckParquetLoader = exports.ParquetLoader;