@loaders.gl/parquet 4.0.0-alpha.22 → 4.0.0-alpha.24

package/dist/parquetjs/schema/types.js

@@ -1,416 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fromPrimitive = exports.toPrimitive = exports.PARQUET_LOGICAL_TYPES = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-/* eslint-disable camelcase */
-const bson_1 = require("@loaders.gl/bson");
-exports.PARQUET_LOGICAL_TYPES = {
-    BOOLEAN: {
-        primitiveType: 'BOOLEAN',
-        toPrimitive: toPrimitive_BOOLEAN,
-        fromPrimitive: fromPrimitive_BOOLEAN
-    },
-    INT32: {
-        primitiveType: 'INT32',
-        toPrimitive: toPrimitive_INT32
-    },
-    INT64: {
-        primitiveType: 'INT64',
-        toPrimitive: toPrimitive_INT64
-    },
-    INT96: {
-        primitiveType: 'INT96',
-        toPrimitive: toPrimitive_INT96
-    },
-    FLOAT: {
-        primitiveType: 'FLOAT',
-        toPrimitive: toPrimitive_FLOAT
-    },
-    DOUBLE: {
-        primitiveType: 'DOUBLE',
-        toPrimitive: toPrimitive_DOUBLE
-    },
-    BYTE_ARRAY: {
-        primitiveType: 'BYTE_ARRAY',
-        toPrimitive: toPrimitive_BYTE_ARRAY
-    },
-    FIXED_LEN_BYTE_ARRAY: {
-        primitiveType: 'FIXED_LEN_BYTE_ARRAY',
-        toPrimitive: toPrimitive_BYTE_ARRAY
-    },
-    UTF8: {
-        primitiveType: 'BYTE_ARRAY',
-        originalType: 'UTF8',
-        toPrimitive: toPrimitive_UTF8,
-        fromPrimitive: fromPrimitive_UTF8
-    },
-    TIME_MILLIS: {
-        primitiveType: 'INT32',
-        originalType: 'TIME_MILLIS',
-        toPrimitive: toPrimitive_TIME_MILLIS
-    },
-    TIME_MICROS: {
-        primitiveType: 'INT64',
-        originalType: 'TIME_MICROS',
-        toPrimitive: toPrimitive_TIME_MICROS
-    },
-    DATE: {
-        primitiveType: 'INT32',
-        originalType: 'DATE',
-        toPrimitive: toPrimitive_DATE,
-        fromPrimitive: fromPrimitive_DATE
-    },
-    TIMESTAMP_MILLIS: {
-        primitiveType: 'INT64',
-        originalType: 'TIMESTAMP_MILLIS',
-        toPrimitive: toPrimitive_TIMESTAMP_MILLIS,
-        fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS
-    },
-    TIMESTAMP_MICROS: {
-        primitiveType: 'INT64',
-        originalType: 'TIMESTAMP_MICROS',
-        toPrimitive: toPrimitive_TIMESTAMP_MICROS,
-        fromPrimitive: fromPrimitive_TIMESTAMP_MICROS
-    },
-    UINT_8: {
-        primitiveType: 'INT32',
-        originalType: 'UINT_8',
-        toPrimitive: toPrimitive_UINT8
-    },
-    UINT_16: {
-        primitiveType: 'INT32',
-        originalType: 'UINT_16',
-        toPrimitive: toPrimitive_UINT16
-    },
-    UINT_32: {
-        primitiveType: 'INT32',
-        originalType: 'UINT_32',
-        toPrimitive: toPrimitive_UINT32
-    },
-    UINT_64: {
-        primitiveType: 'INT64',
-        originalType: 'UINT_64',
-        toPrimitive: toPrimitive_UINT64
-    },
-    INT_8: {
-        primitiveType: 'INT32',
-        originalType: 'INT_8',
-        toPrimitive: toPrimitive_INT8
-    },
-    INT_16: {
-        primitiveType: 'INT32',
-        originalType: 'INT_16',
-        toPrimitive: toPrimitive_INT16
-    },
-    INT_32: {
-        primitiveType: 'INT32',
-        originalType: 'INT_32',
-        toPrimitive: toPrimitive_INT32
-    },
-    INT_64: {
-        primitiveType: 'INT64',
-        originalType: 'INT_64',
-        toPrimitive: toPrimitive_INT64
-    },
-    JSON: {
-        primitiveType: 'BYTE_ARRAY',
-        originalType: 'JSON',
-        toPrimitive: toPrimitive_JSON,
-        fromPrimitive: fromPrimitive_JSON
-    },
-    BSON: {
-        primitiveType: 'BYTE_ARRAY',
-        originalType: 'BSON',
-        toPrimitive: toPrimitive_BSON,
-        fromPrimitive: fromPrimitive_BSON
-    },
-    INTERVAL: {
-        primitiveType: 'FIXED_LEN_BYTE_ARRAY',
-        originalType: 'INTERVAL',
-        typeLength: 12,
-        toPrimitive: toPrimitive_INTERVAL,
-        fromPrimitive: fromPrimitive_INTERVAL
-    },
-    DECIMAL_INT32: {
-        primitiveType: 'INT32',
-        originalType: 'DECIMAL_INT32',
-        toPrimitive: decimalToPrimitive_INT32,
-        fromPrimitive: decimalFromPrimitive_INT
-    },
-    DECIMAL_INT64: {
-        primitiveType: 'INT64',
-        originalType: 'DECIMAL_INT64',
-        toPrimitive: decimalToPrimitive_INT64,
-        fromPrimitive: decimalFromPrimitive_INT
-    },
-    DECIMAL_BYTE_ARRAY: {
-        primitiveType: 'BYTE_ARRAY',
-        originalType: 'DECIMAL_BYTE_ARRAY',
-        toPrimitive: decimalToPrimitive_BYTE_ARRAY,
-        fromPrimitive: decimalFromPrimitive_BYTE_ARRAY
-    },
-    DECIMAL_FIXED_LEN_BYTE_ARRAY: {
-        primitiveType: 'FIXED_LEN_BYTE_ARRAY',
-        originalType: 'DECIMAL_FIXED_LEN_BYTE_ARRAY',
-        toPrimitive: decimalToPrimitive_BYTE_ARRAY,
-        fromPrimitive: decimalFromPrimitive_BYTE_ARRAY
-    }
-};
-/**
- * Convert a value from it's native representation to the internal/underlying
- * primitive type
- */
-function toPrimitive(type, value, field) {
-    if (!(type in exports.PARQUET_LOGICAL_TYPES)) {
-        throw new Error(`invalid type: ${type}`);
-    }
-    return exports.PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);
-}
-exports.toPrimitive = toPrimitive;
-/**
- * Convert a value from it's internal/underlying primitive representation to
- * the native representation
- */
-function fromPrimitive(type, value, field) {
-    if (!(type in exports.PARQUET_LOGICAL_TYPES)) {
-        throw new Error(`invalid type: ${type}`);
-    }
-    if ('fromPrimitive' in exports.PARQUET_LOGICAL_TYPES[type]) {
-        return exports.PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value, field);
-        // tslint:disable-next-line:no-else-after-return
-    }
-    return value;
-}
-exports.fromPrimitive = fromPrimitive;
-function toPrimitive_BOOLEAN(value) {
-    return Boolean(value);
-}
-function fromPrimitive_BOOLEAN(value) {
-    return Boolean(value);
-}
-function toPrimitive_FLOAT(value) {
-    const v = parseFloat(value);
-    if (isNaN(v)) {
-        throw new Error(`invalid value for FLOAT: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_DOUBLE(value) {
-    const v = parseFloat(value);
-    if (isNaN(v)) {
-        throw new Error(`invalid value for DOUBLE: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_INT8(value) {
-    const v = parseInt(value, 10);
-    if (v < -0x80 || v > 0x7f || isNaN(v)) {
-        throw new Error(`invalid value for INT8: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_UINT8(value) {
-    const v = parseInt(value, 10);
-    if (v < 0 || v > 0xff || isNaN(v)) {
-        throw new Error(`invalid value for UINT8: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_INT16(value) {
-    const v = parseInt(value, 10);
-    if (v < -0x8000 || v > 0x7fff || isNaN(v)) {
-        throw new Error(`invalid value for INT16: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_UINT16(value) {
-    const v = parseInt(value, 10);
-    if (v < 0 || v > 0xffff || isNaN(v)) {
-        throw new Error(`invalid value for UINT16: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_INT32(value) {
-    const v = parseInt(value, 10);
-    if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
-        throw new Error(`invalid value for INT32: ${value}`);
-    }
-    return v;
-}
-function decimalToPrimitive_INT32(value, field) {
-    const primitiveValue = value * 10 ** (field.scale || 0);
-    const v = Math.round(((primitiveValue * 10 ** -field.presision) % 1) * 10 ** field.presision);
-    if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
-        throw new Error(`invalid value for INT32: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_UINT32(value) {
-    const v = parseInt(value, 10);
-    if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
-        throw new Error(`invalid value for UINT32: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_INT64(value) {
-    const v = parseInt(value, 10);
-    if (isNaN(v)) {
-        throw new Error(`invalid value for INT64: ${value}`);
-    }
-    return v;
-}
-function decimalToPrimitive_INT64(value, field) {
-    const primitiveValue = value * 10 ** (field.scale || 0);
-    const v = Math.round(((primitiveValue * 10 ** -field.presision) % 1) * 10 ** field.presision);
-    if (isNaN(v)) {
-        throw new Error(`invalid value for INT64: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_UINT64(value) {
-    const v = parseInt(value, 10);
-    if (v < 0 || isNaN(v)) {
-        throw new Error(`invalid value for UINT64: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_INT96(value) {
-    const v = parseInt(value, 10);
-    if (isNaN(v)) {
-        throw new Error(`invalid value for INT96: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_BYTE_ARRAY(value) {
-    return Buffer.from(value);
-}
-function decimalToPrimitive_BYTE_ARRAY(value) {
-    // TBD
-    return Buffer.from(value);
-}
-function toPrimitive_UTF8(value) {
-    return Buffer.from(value, 'utf8');
-}
-function fromPrimitive_UTF8(value) {
-    return value.toString();
-}
-function toPrimitive_JSON(value) {
-    return Buffer.from(JSON.stringify(value));
-}
-function fromPrimitive_JSON(value) {
-    return JSON.parse(value);
-}
-function toPrimitive_BSON(value) {
-    const arrayBuffer = bson_1.BSONWriter.encodeSync?.(value);
-    return Buffer.from(arrayBuffer);
-}
-function fromPrimitive_BSON(value) {
-    return bson_1.BSONLoader.parseSync?.(value);
-}
-function toPrimitive_TIME_MILLIS(value) {
-    const v = parseInt(value, 10);
-    if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {
-        throw new Error(`invalid value for TIME_MILLIS: ${value}`);
-    }
-    return v;
-}
-function toPrimitive_TIME_MICROS(value) {
-    const v = parseInt(value, 10);
-    if (v < 0 || isNaN(v)) {
-        throw new Error(`invalid value for TIME_MICROS: ${value}`);
-    }
-    return v;
-}
-const kMillisPerDay = 86400000;
-function toPrimitive_DATE(value) {
-    /* convert from date */
-    if (value instanceof Date) {
-        return value.getTime() / kMillisPerDay;
-    }
-    /* convert from integer */
-    {
-        const v = parseInt(value, 10);
-        if (v < 0 || isNaN(v)) {
-            throw new Error(`invalid value for DATE: ${value}`);
-        }
-        return v;
-    }
-}
-function fromPrimitive_DATE(value) {
-    return new Date(value * kMillisPerDay);
-}
-function toPrimitive_TIMESTAMP_MILLIS(value) {
-    /* convert from date */
-    if (value instanceof Date) {
-        return value.getTime();
-    }
-    /* convert from integer */
-    {
-        const v = parseInt(value, 10);
-        if (v < 0 || isNaN(v)) {
-            throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);
-        }
-        return v;
-    }
-}
-function fromPrimitive_TIMESTAMP_MILLIS(value) {
-    return new Date(value);
-}
-function toPrimitive_TIMESTAMP_MICROS(value) {
-    /* convert from date */
-    if (value instanceof Date) {
-        return value.getTime() * 1000;
-    }
-    /* convert from integer */
-    {
-        const v = parseInt(value, 10);
-        if (v < 0 || isNaN(v)) {
-            throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);
-        }
-        return v;
-    }
-}
-function fromPrimitive_TIMESTAMP_MICROS(value) {
-    return new Date(value / 1000);
-}
-function toPrimitive_INTERVAL(value) {
-    if (!value.months || !value.days || !value.milliseconds) {
-        throw new Error('value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }');
-    }
-    const buf = Buffer.alloc(12);
-    buf.writeUInt32LE(value.months, 0);
-    buf.writeUInt32LE(value.days, 4);
-    buf.writeUInt32LE(value.milliseconds, 8);
-    return buf;
-}
-function fromPrimitive_INTERVAL(value) {
-    const buf = Buffer.from(value);
-    const months = buf.readUInt32LE(0);
-    const days = buf.readUInt32LE(4);
-    const millis = buf.readUInt32LE(8);
-    return { months, days, milliseconds: millis };
-}
-function decimalFromPrimitive_INT(value, field) {
-    const presisionInt = Math.round(((value * 10 ** -field.presision) % 1) * 10 ** field.presision);
-    return presisionInt * 10 ** -(field.scale || 0);
-}
-function decimalFromPrimitive_BYTE_ARRAY(value, field) {
-    let number = 0;
-    if (value.length <= 4) {
-        // Bytewise operators faster. Use them if it is possible
-        for (let i = 0; i < value.length; i++) {
-            // `value.length - i - 1` bytes have reverse order (big-endian)
-            const component = value[i] << (8 * (value.length - i - 1));
-            number += component;
-        }
-    }
-    else {
-        for (let i = 0; i < value.length; i++) {
-            // `value.length - i - 1` bytes have reverse order (big-endian)
-            const component = value[i] * 2 ** (8 * (value.length - 1 - i));
-            number += component;
-        }
-    }
-    const presisionInt = Math.round(((number * 10 ** -field.presision) % 1) * 10 ** field.presision);
-    return presisionInt * 10 ** -(field.scale || 0);
-}

package/dist/parquetjs/utils/file-utils.js

@@ -1,43 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.osopen = exports.osclose = exports.oswrite = exports.load = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-function load(name) {
-    return (module || global).require(name);
-}
-exports.load = load;
-function oswrite(os, buf) {
-    return new Promise((resolve, reject) => {
-        os.write(buf, (err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve();
-            }
-        });
-    });
-}
-exports.oswrite = oswrite;
-function osclose(os) {
-    return new Promise((resolve, reject) => {
-        os.close((err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve();
-            }
-        });
-    });
-}
-exports.osclose = osclose;
-function osopen(path, opts) {
-    return new Promise((resolve, reject) => {
-        const outputStream = loader_utils_1.fs.createWriteStream(path, opts);
-        outputStream.once('open', (fd) => resolve(outputStream));
-        outputStream.once('error', (err) => reject(err));
-    });
-}
-exports.osopen = osopen;

package/dist/parquetjs/utils/read-utils.js

@@ -1,109 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fieldIndexOf = exports.getBitWidth = exports.decodePageHeader = exports.decodeFileMetadata = exports.getThriftEnum = exports.decodeThrift = exports.serializeThrift = void 0;
-const thrift_1 = require("thrift");
-const parquet_thrift_1 = require("../parquet-thrift");
-class UFramedTransport extends thrift_1.TFramedTransport {
-    constructor() {
-        super(...arguments);
-        this.readPos = 0;
-    }
-}
-/**
- * Helper function that serializes a thrift object into a buffer
- */
-function serializeThrift(obj) {
-    const output = [];
-    const transport = new thrift_1.TBufferedTransport(undefined, (buf) => {
-        output.push(buf);
-    });
-    const protocol = new thrift_1.TCompactProtocol(transport);
-    obj.write(protocol);
-    transport.flush();
-    return Buffer.concat(output);
-}
-exports.serializeThrift = serializeThrift;
-function decodeThrift(obj, buf, offset) {
-    if (!offset) {
-        // tslint:disable-next-line:no-parameter-reassignment
-        offset = 0;
-    }
-    const transport = new UFramedTransport(buf);
-    transport.readPos = offset;
-    const protocol = new thrift_1.TCompactProtocol(transport);
-    obj.read(protocol);
-    return transport.readPos - offset;
-}
-exports.decodeThrift = decodeThrift;
-/**
- * FIXME not ideal that this is linear
- */
-function getThriftEnum(klass, value) {
-    for (const k in klass) {
-        if (klass[k] === value) {
-            return k;
-        }
-    }
-    throw new Error('Invalid ENUM value');
-}
-exports.getThriftEnum = getThriftEnum;
-function decodeFileMetadata(buf, offset) {
-    if (!offset) {
-        // tslint:disable-next-line:no-parameter-reassignment
-        offset = 0;
-    }
-    const transport = new UFramedTransport(buf);
-    transport.readPos = offset;
-    const protocol = new thrift_1.TCompactProtocol(transport);
-    const metadata = parquet_thrift_1.FileMetaData.read(protocol);
-    return { length: transport.readPos - offset, metadata };
-}
-exports.decodeFileMetadata = decodeFileMetadata;
-function decodePageHeader(buf, offset) {
-    if (!offset) {
-        // tslint:disable-next-line:no-parameter-reassignment
-        offset = 0;
-    }
-    const transport = new UFramedTransport(buf);
-    transport.readPos = offset;
-    const protocol = new thrift_1.TCompactProtocol(transport);
-    const pageHeader = parquet_thrift_1.PageHeader.read(protocol);
-    return { length: transport.readPos - offset, pageHeader };
-}
-exports.decodePageHeader = decodePageHeader;
-/**
- * Get the number of bits required to store a given value
- */
-function getBitWidth(val) {
-    if (val === 0) {
-        return 0;
-        // tslint:disable-next-line:no-else-after-return
-    }
-    return Math.ceil(Math.log2(val + 1));
-}
-exports.getBitWidth = getBitWidth;
-// Supports MQTT path wildcards
-// + all immediate children
-// # all descendents
-function fieldIndexOf(arr, elem) {
-    for (let j = 0; j < arr.length; j++) {
-        if (arr[j].length > elem.length) {
-            continue; // eslint-disable-line no-continue
-        }
-        let m = true;
-        for (let i = 0; i < elem.length; i++) {
-            if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
-                continue; // eslint-disable-line no-continue
-            }
-            if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
-                continue; // eslint-disable-line no-continue
-            }
-            m = false;
-            break;
-        }
-        if (m)
-            return j;
-    }
-    return -1;
-}
-exports.fieldIndexOf = fieldIndexOf;

package/dist/workers/parquet-worker.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const index_1 = require("../index");
-(0, loader_utils_1.createLoaderWorker)(index_1.ParquetLoader);