@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/parquetjs/schema/schema.js

@@ -0,0 +1,162 @@
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetSchema = void 0;
+const codecs_1 = require("../codecs");
+const compression_1 = require("../compression");
+const shred_1 = require("./shred");
+const types_1 = require("./types");
+/**
+ * A parquet file schema
+ */
+class ParquetSchema {
+    /**
+     * Create a new schema from a JSON schema definition
+     */
+    constructor(schema) {
+        this.schema = schema;
+        this.fields = buildFields(schema, 0, 0, []);
+        this.fieldList = listFields(this.fields);
+    }
+    /**
+     * Retrieve a field definition
+     */
+    findField(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        else {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.slice(0); // clone array
+        }
+        let n = this.fields;
+        for (; path.length > 1; path.shift()) {
+            n = n[path[0]].fields;
+        }
+        return n[path[0]];
+    }
+    /**
+     * Retrieve a field definition and all the field's ancestors
+     */
+    findFieldBranch(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        const branch = [];
+        let n = this.fields;
+        for (; path.length > 0; path.shift()) {
+            branch.push(n[path[0]]);
+            if (path.length > 1) {
+                n = n[path[0]].fields;
+            }
+        }
+        return branch;
+    }
+    shredRecord(record, buffer) {
+        (0, shred_1.shredRecord)(this, record, buffer);
+    }
+    materializeRecords(buffer) {
+        return (0, shred_1.materializeRecords)(this, buffer);
+    }
+    compress(type) {
+        setCompress(this.schema, type);
+        setCompress(this.fields, type);
+        return this;
+    }
+    buffer() {
+        return (0, shred_1.shredBuffer)(this);
+    }
+}
+exports.ParquetSchema = ParquetSchema;
+function setCompress(schema, type) {
+    for (const name in schema) {
+        const node = schema[name];
+        if (node.fields) {
+            setCompress(node.fields, type);
+        }
+        else {
+            node.compression = type;
+        }
+    }
+}
+// eslint-disable-next-line max-statements, complexity
+function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
+    const fieldList = {};
+    for (const name in schema) {
+        const opts = schema[name];
+        /* field repetition type */
+        const required = !opts.optional;
+        const repeated = Boolean(opts.repeated);
+        let rLevelMax = rLevelParentMax;
+        let dLevelMax = dLevelParentMax;
+        let repetitionType = 'REQUIRED';
+        if (!required) {
+            repetitionType = 'OPTIONAL';
+            dLevelMax++;
+        }
+        if (repeated) {
+            repetitionType = 'REPEATED';
+            rLevelMax++;
+            if (required)
+                dLevelMax++;
+        }
+        /* nested field */
+        if (opts.fields) {
+            const cpath = path.concat([name]);
+            fieldList[name] = {
+                name,
+                path: cpath,
+                key: cpath.join(),
+                repetitionType,
+                rLevelMax,
+                dLevelMax,
+                isNested: true,
+                fieldCount: Object.keys(opts.fields).length,
+                fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+            };
+            continue; // eslint-disable-line no-continue
+        }
+        const typeDef = types_1.PARQUET_LOGICAL_TYPES[opts.type];
+        if (!typeDef) {
+            throw new Error(`invalid parquet type: ${opts.type}`);
+        }
+        opts.encoding = opts.encoding || 'PLAIN';
+        if (!(opts.encoding in codecs_1.PARQUET_CODECS)) {
+            throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
+        }
+        opts.compression = opts.compression || 'UNCOMPRESSED';
+        if (!(opts.compression in compression_1.PARQUET_COMPRESSION_METHODS)) {
+            throw new Error(`unsupported compression method: ${opts.compression}`);
+        }
+        /* add to schema */
+        const cpath = path.concat([name]);
+        fieldList[name] = {
+            name,
+            primitiveType: typeDef.primitiveType,
+            originalType: typeDef.originalType,
+            path: cpath,
+            key: cpath.join(),
+            repetitionType,
+            encoding: opts.encoding,
+            compression: opts.compression,
+            typeLength: opts.typeLength || typeDef.typeLength,
+            presision: opts.presision,
+            scale: opts.scale,
+            rLevelMax,
+            dLevelMax
+        };
+    }
+    return fieldList;
+}
+function listFields(fields) {
+    let list = [];
+    for (const k in fields) {
+        list.push(fields[k]);
+        if (fields[k].isNested) {
+            list = list.concat(listFields(fields[k].fields));
+        }
+    }
+    return list;
+}

package/dist/parquetjs/schema/shred.d.ts

@@ -0,0 +1,48 @@
+import { ParquetBuffer, ParquetRecord } from './declare';
+import { ParquetSchema } from './schema';
+export { ParquetBuffer };
+export declare function shredBuffer(schema: ParquetSchema): ParquetBuffer;
+/**
+ * 'Shred' a record into a list of <value, repetition_level, definition_level>
+ * tuples per column using the Google Dremel Algorithm..
+ *
+ * The buffer argument must point to an object into which the shredded record
+ * will be returned. You may re-use the buffer for repeated calls to this function
+ * to append to an existing buffer, as long as the schema is unchanged.
+ *
+ * The format in which the shredded records will be stored in the buffer is as
+ * follows:
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ */
+export declare function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void;
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The buffer argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ */
+export declare function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[];
+//# sourceMappingURL=shred.d.ts.map

package/dist/parquetjs/schema/shred.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAOhG"}

package/dist/parquetjs/schema/shred.js

@@ -0,0 +1,225 @@
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.materializeRecords = exports.shredRecord = exports.shredBuffer = exports.ParquetBuffer = void 0;
+const declare_1 = require("./declare");
+Object.defineProperty(exports, "ParquetBuffer", { enumerable: true, get: function () { return declare_1.ParquetBuffer; } });
+const Types = __importStar(require("./types"));
+function shredBuffer(schema) {
+    const columnData = {};
+    for (const field of schema.fieldList) {
+        columnData[field.key] = {
+            dlevels: [],
+            rlevels: [],
+            values: [],
+            pageHeaders: [],
+            count: 0
+        };
+    }
+    return { rowCount: 0, columnData };
+}
+exports.shredBuffer = shredBuffer;
+/**
+ * 'Shred' a record into a list of <value, repetition_level, definition_level>
+ * tuples per column using the Google Dremel Algorithm..
+ *
+ * The buffer argument must point to an object into which the shredded record
+ * will be returned. You may re-use the buffer for repeated calls to this function
+ * to append to an existing buffer, as long as the schema is unchanged.
+ *
+ * The format in which the shredded records will be stored in the buffer is as
+ * follows:
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ */
+function shredRecord(schema, record, buffer) {
+    /* shred the record, this may raise an exception */
+    const data = shredBuffer(schema).columnData;
+    shredRecordFields(schema.fields, record, data, 0, 0);
+    /* if no error during shredding, add the shredded record to the buffer */
+    if (buffer.rowCount === 0) {
+        buffer.rowCount = 1;
+        buffer.columnData = data;
+        return;
+    }
+    buffer.rowCount += 1;
+    for (const field of schema.fieldList) {
+        Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
+        Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
+        Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
+        buffer.columnData[field.key].count += data[field.key].count;
+    }
+}
+exports.shredRecord = shredRecord;
+// eslint-disable-next-line max-statements, complexity
+function shredRecordFields(fields, record, data, rLevel, dLevel) {
+    for (const name in fields) {
+        const field = fields[name];
+        // fetch values
+        let values = [];
+        if (record &&
+            field.name in record &&
+            record[field.name] !== undefined &&
+            record[field.name] !== null) {
+            if (record[field.name].constructor === Array) {
+                values = record[field.name];
+            }
+            else {
+                values.push(record[field.name]);
+            }
+        }
+        // check values
+        if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {
+            throw new Error(`missing required field: ${field.name}`);
+        }
+        if (values.length > 1 && field.repetitionType !== 'REPEATED') {
+            throw new Error(`too many values for field: ${field.name}`);
+        }
+        // push null
+        if (values.length === 0) {
+            if (field.isNested) {
+                shredRecordFields(field.fields, null, data, rLevel, dLevel);
+            }
+            else {
+                data[field.key].count += 1;
+                data[field.key].rlevels.push(rLevel);
+                data[field.key].dlevels.push(dLevel);
+            }
+            continue; // eslint-disable-line no-continue
+        }
+        // push values
+        for (let i = 0; i < values.length; i++) {
+            const rlvl = i === 0 ? rLevel : field.rLevelMax;
+            if (field.isNested) {
+                shredRecordFields(field.fields, values[i], data, rlvl, field.dLevelMax);
+            }
+            else {
+                data[field.key].count += 1;
+                data[field.key].rlevels.push(rlvl);
+                data[field.key].dlevels.push(field.dLevelMax);
+                data[field.key].values.push(Types.toPrimitive((field.originalType || field.primitiveType), values[i]));
+            }
+        }
+    }
+}
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The buffer argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ */
+function materializeRecords(schema, buffer) {
+    const records = [];
+    for (let i = 0; i < buffer.rowCount; i++)
+        records.push({});
+    for (const key in buffer.columnData) {
+        materializeColumn(schema, buffer, key, records);
+    }
+    return records;
+}
+exports.materializeRecords = materializeRecords;
+// eslint-disable-next-line max-statements, complexity
+function materializeColumn(schema, buffer, key, records) {
+    const data = buffer.columnData[key];
+    if (!data.count)
+        return;
+    const field = schema.findField(key);
+    const branch = schema.findFieldBranch(key);
+    // tslint:disable-next-line:prefer-array-literal
+    const rLevels = new Array(field.rLevelMax + 1).fill(0);
+    let vIndex = 0;
+    for (let i = 0; i < data.count; i++) {
+        const dLevel = data.dlevels[i];
+        const rLevel = data.rlevels[i];
+        rLevels[rLevel]++;
+        rLevels.fill(0, rLevel + 1);
+        let rIndex = 0;
+        let record = records[rLevels[rIndex++] - 1];
+        // Internal nodes
+        for (const step of branch) {
+            if (step === field)
+                break;
+            if (dLevel < step.dLevelMax)
+                break;
+            if (step.repetitionType === 'REPEATED') {
+                if (!(step.name in record)) {
+                    // eslint-disable max-depth
+                    record[step.name] = [];
+                }
+                const ix = rLevels[rIndex++];
+                while (record[step.name].length <= ix) {
+                    // eslint-disable max-depth
+                    record[step.name].push({});
+                }
+                record = record[step.name][ix];
+            }
+            else {
+                record[step.name] = record[step.name] || {};
+                record = record[step.name];
+            }
+        }
+        // Leaf node
+        if (dLevel === field.dLevelMax) {
+            const value = Types.fromPrimitive(
+            // @ts-ignore
+            field.originalType || field.primitiveType, data.values[vIndex], field);
+            vIndex++;
+            if (field.repetitionType === 'REPEATED') {
+                if (!(field.name in record)) {
+                    // eslint-disable max-depth
+                    record[field.name] = [];
+                }
+                const ix = rLevels[rIndex];
+                while (record[field.name].length <= ix) {
+                    // eslint-disable max-depth
+                    record[field.name].push(null);
+                }
+                record[field.name][ix] = value;
+            }
+            else {
+                record[field.name] = value;
+            }
+        }
+    }
+}

package/dist/parquetjs/schema/types.d.ts

@@ -0,0 +1,20 @@
+import { OriginalType, ParquetField, ParquetType, PrimitiveType } from './declare';
+export interface ParquetTypeKit {
+    primitiveType: PrimitiveType;
+    originalType?: OriginalType;
+    typeLength?: number;
+    toPrimitive: Function;
+    fromPrimitive?: Function;
+}
+export declare const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit>;
+/**
+ * Convert a value from it's native representation to the internal/underlying
+ * primitive type
+ */
+export declare function toPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+/**
+ * Convert a value from it's internal/underlying primitive representation to
+ * the native representation
+ */
+export declare function fromPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+//# sourceMappingURL=types.d.ts.map

package/dist/parquetjs/schema/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAM9E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAUhF"}