@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/parquet-writer.d.ts

@@ -1,4 +1,5 @@
 import type { Writer } from '@loaders.gl/loader-utils';
-export declare type ParquetWriterOptions = {};
-export declare const ParquetWriter: Writer;
+import { Table, TableBatch } from '@loaders.gl/schema';
+export type ParquetWriterOptions = {};
+export declare const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions>;
 //# sourceMappingURL=parquet-writer.d.ts.map

package/dist/parquet-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,oBAAY,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAU3B,CAAC"}
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,oBAAoB,CAUzE,CAAC"}

package/dist/parquet-writer.js

@@ -1,18 +1,22 @@
-const VERSION = typeof "4.0.0-alpha.5" !== 'undefined' ? "4.0.0-alpha.5" : 'latest';
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetWriter = void 0;
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {};
-export const ParquetWriter = {
-  name: 'Apache Parquet',
-  id: 'parquet',
-  module: 'parquet',
-  version: VERSION,
-  extensions: ['parquet'],
-  mimeTypes: ['application/octet-stream'],
-  encodeSync,
-  binary: true,
-  options: DEFAULT_PARQUET_LOADER_OPTIONS
+exports.ParquetWriter = {
+    name: 'Apache Parquet',
+    id: 'parquet',
+    module: 'parquet',
+    version: VERSION,
+    extensions: ['parquet'],
+    mimeTypes: ['application/octet-stream'],
+    encodeSync,
+    binary: true,
+    options: DEFAULT_PARQUET_LOADER_OPTIONS
 };
-
 function encodeSync(data, options) {
-  return new ArrayBuffer(0);
+    return new ArrayBuffer(0);
 }
-//# sourceMappingURL=parquet-writer.js.map

package/dist/parquetjs/codecs/declare.js

@@ -1,2 +1,2 @@
-export {};
-//# sourceMappingURL=declare.js.map
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/parquetjs/codecs/dictionary.js

@@ -1,12 +1,14 @@
-import { decodeValues as decodeRleValues } from './rle';
-export function decodeValues(type, cursor, count, opts) {
-  opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
-  cursor.offset += 1;
-  return decodeRleValues(type, cursor, count, { ...opts,
-    disableEnvelope: true
-  });
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encodeValues = exports.decodeValues = void 0;
+const rle_1 = require("./rle");
+function decodeValues(type, cursor, count, opts) {
+    opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
+    cursor.offset += 1;
+    return (0, rle_1.decodeValues)(type, cursor, count, { ...opts, disableEnvelope: true });
 }
-export function encodeValues(type, cursor, count, opts) {
-  throw new Error('Encode dictionary functionality is not supported');
+exports.decodeValues = decodeValues;
+function encodeValues(type, cursor, count, opts) {
+    throw new Error('Encode dictionary functionality is not supported');
 }
-//# sourceMappingURL=dictionary.js.map
+exports.encodeValues = encodeValues;

package/dist/parquetjs/codecs/index.js

@@ -1,23 +1,55 @@
-import * as PLAIN from './plain';
-import * as RLE from './rle';
-import * as DICTIONARY from './dictionary';
-export * from './declare';
-export const PARQUET_CODECS = {
-  PLAIN: {
-    encodeValues: PLAIN.encodeValues,
-    decodeValues: PLAIN.decodeValues
-  },
-  RLE: {
-    encodeValues: RLE.encodeValues,
-    decodeValues: RLE.decodeValues
-  },
-  PLAIN_DICTIONARY: {
-    encodeValues: DICTIONARY.encodeValues,
-    decodeValues: DICTIONARY.decodeValues
-  },
-  RLE_DICTIONARY: {
-    encodeValues: DICTIONARY.encodeValues,
-    decodeValues: DICTIONARY.decodeValues
-  }
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PARQUET_CODECS = void 0;
+const PLAIN = __importStar(require("./plain"));
+const RLE = __importStar(require("./rle"));
+const DICTIONARY = __importStar(require("./dictionary"));
+__exportStar(require("./declare"), exports);
+exports.PARQUET_CODECS = {
+    PLAIN: {
+        encodeValues: PLAIN.encodeValues,
+        decodeValues: PLAIN.decodeValues
+    },
+    RLE: {
+        encodeValues: RLE.encodeValues,
+        decodeValues: RLE.decodeValues
+    },
+    // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.
+    PLAIN_DICTIONARY: {
+        // @ts-ignore
+        encodeValues: DICTIONARY.encodeValues,
+        decodeValues: DICTIONARY.decodeValues
+    },
+    // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.
+    RLE_DICTIONARY: {
+        // @ts-ignore
+        encodeValues: DICTIONARY.encodeValues,
+        decodeValues: DICTIONARY.decodeValues
+    }
 };
-//# sourceMappingURL=index.js.map

package/dist/parquetjs/codecs/plain.js

@@ -1,270 +1,211 @@
-import INT53 from 'int53';
-export function encodeValues(type, values, opts) {
-  switch (type) {
-    case 'BOOLEAN':
-      return encodeValues_BOOLEAN(values);
-
-    case 'INT32':
-      return encodeValues_INT32(values);
-
-    case 'INT64':
-      return encodeValues_INT64(values);
-
-    case 'INT96':
-      return encodeValues_INT96(values);
-
-    case 'FLOAT':
-      return encodeValues_FLOAT(values);
-
-    case 'DOUBLE':
-      return encodeValues_DOUBLE(values);
-
-    case 'BYTE_ARRAY':
-      return encodeValues_BYTE_ARRAY(values);
-
-    case 'FIXED_LEN_BYTE_ARRAY':
-      return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
-
-    default:
-      throw new Error("unsupported type: ".concat(type));
-  }
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeValues = exports.encodeValues = void 0;
+const int53_1 = __importDefault(require("int53"));
+function encodeValues(type, values, opts) {
+    switch (type) {
+        case 'BOOLEAN':
+            return encodeValues_BOOLEAN(values);
+        case 'INT32':
+            return encodeValues_INT32(values);
+        case 'INT64':
+            return encodeValues_INT64(values);
+        case 'INT96':
+            return encodeValues_INT96(values);
+        case 'FLOAT':
+            return encodeValues_FLOAT(values);
+        case 'DOUBLE':
+            return encodeValues_DOUBLE(values);
+        case 'BYTE_ARRAY':
+            return encodeValues_BYTE_ARRAY(values);
+        case 'FIXED_LEN_BYTE_ARRAY':
+            return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
+        default:
+            throw new Error(`unsupported type: ${type}`);
+    }
 }
-export function decodeValues(type, cursor, count, opts) {
-  switch (type) {
-    case 'BOOLEAN':
-      return decodeValues_BOOLEAN(cursor, count);
-
-    case 'INT32':
-      return decodeValues_INT32(cursor, count);
-
-    case 'INT64':
-      return decodeValues_INT64(cursor, count);
-
-    case 'INT96':
-      return decodeValues_INT96(cursor, count);
-
-    case 'FLOAT':
-      return decodeValues_FLOAT(cursor, count);
-
-    case 'DOUBLE':
-      return decodeValues_DOUBLE(cursor, count);
-
-    case 'BYTE_ARRAY':
-      return decodeValues_BYTE_ARRAY(cursor, count);
-
-    case 'FIXED_LEN_BYTE_ARRAY':
-      return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
-
-    default:
-      throw new Error("unsupported type: ".concat(type));
-  }
+exports.encodeValues = encodeValues;
+function decodeValues(type, cursor, count, opts) {
+    switch (type) {
+        case 'BOOLEAN':
+            return decodeValues_BOOLEAN(cursor, count);
+        case 'INT32':
+            return decodeValues_INT32(cursor, count);
+        case 'INT64':
+            return decodeValues_INT64(cursor, count);
+        case 'INT96':
+            return decodeValues_INT96(cursor, count);
+        case 'FLOAT':
+            return decodeValues_FLOAT(cursor, count);
+        case 'DOUBLE':
+            return decodeValues_DOUBLE(cursor, count);
+        case 'BYTE_ARRAY':
+            return decodeValues_BYTE_ARRAY(cursor, count);
+        case 'FIXED_LEN_BYTE_ARRAY':
+            return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
+        default:
+            throw new Error(`unsupported type: ${type}`);
+    }
 }
-
+exports.decodeValues = decodeValues;
 function encodeValues_BOOLEAN(values) {
-  const buf = Buffer.alloc(Math.ceil(values.length / 8));
-  buf.fill(0);
-
-  for (let i = 0; i < values.length; i++) {
-    if (values[i]) {
-      buf[Math.floor(i / 8)] |= 1 << i % 8;
+    const buf = Buffer.alloc(Math.ceil(values.length / 8));
+    buf.fill(0);
+    for (let i = 0; i < values.length; i++) {
+        if (values[i]) {
+            buf[Math.floor(i / 8)] |= 1 << i % 8;
+        }
     }
-  }
-
-  return buf;
+    return buf;
 }
-
 function decodeValues_BOOLEAN(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
-    values.push((b & 1 << i % 8) > 0);
-  }
-
-  cursor.offset += Math.ceil(count / 8);
-  return values;
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
+        values.push((b & (1 << i % 8)) > 0);
+    }
+    cursor.offset += Math.ceil(count / 8);
+    return values;
 }
-
 function encodeValues_INT32(values) {
-  const buf = Buffer.alloc(4 * values.length);
-
-  for (let i = 0; i < values.length; i++) {
-    buf.writeInt32LE(values[i], i * 4);
-  }
-
-  return buf;
+    const buf = Buffer.alloc(4 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        buf.writeInt32LE(values[i], i * 4);
+    }
+    return buf;
 }
-
 function decodeValues_INT32(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.readInt32LE(cursor.offset));
-    cursor.offset += 4;
-  }
-
-  return values;
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.readInt32LE(cursor.offset));
+        cursor.offset += 4;
+    }
+    return values;
 }
-
 function encodeValues_INT64(values) {
-  const buf = Buffer.alloc(8 * values.length);
-
-  for (let i = 0; i < values.length; i++) {
-    INT53.writeInt64LE(values[i], buf, i * 8);
-  }
-
-  return buf;
+    const buf = Buffer.alloc(8 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        int53_1.default.writeInt64LE(values[i], buf, i * 8);
+    }
+    return buf;
 }
-
 function decodeValues_INT64(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));
-    cursor.offset += 8;
-  }
-
-  return values;
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(int53_1.default.readInt64LE(cursor.buffer, cursor.offset));
+        cursor.offset += 8;
+    }
+    return values;
 }
-
 function encodeValues_INT96(values) {
-  const buf = Buffer.alloc(12 * values.length);
-
-  for (let i = 0; i < values.length; i++) {
-    if (values[i] >= 0) {
-      INT53.writeInt64LE(values[i], buf, i * 12);
-      buf.writeUInt32LE(0, i * 12 + 8);
-    } else {
-      INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);
-      buf.writeUInt32LE(0xffffffff, i * 12 + 8);
+    const buf = Buffer.alloc(12 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        if (values[i] >= 0) {
+            int53_1.default.writeInt64LE(values[i], buf, i * 12);
+            buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision
+        }
+        else {
+            int53_1.default.writeInt64LE(~-values[i] + 1, buf, i * 12);
+            buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision
+        }
     }
-  }
-
-  return buf;
+    return buf;
 }
-
 function decodeValues_INT96(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    const low = INT53.readInt64LE(cursor.buffer, cursor.offset);
-    const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
-
-    if (high === 0xffffffff) {
-      values.push(~-low + 1);
-    } else {
-      values.push(low);
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        const low = int53_1.default.readInt64LE(cursor.buffer, cursor.offset);
+        const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
+        if (high === 0xffffffff) {
+            values.push(~-low + 1); // truncate to 64 actual precision
+        }
+        else {
+            values.push(low); // truncate to 64 actual precision
+        }
+        cursor.offset += 12;
     }
-
-    cursor.offset += 12;
-  }
-
-  return values;
+    return values;
 }
-
 function encodeValues_FLOAT(values) {
-  const buf = Buffer.alloc(4 * values.length);
-
-  for (let i = 0; i < values.length; i++) {
-    buf.writeFloatLE(values[i], i * 4);
-  }
-
-  return buf;
+    const buf = Buffer.alloc(4 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        buf.writeFloatLE(values[i], i * 4);
+    }
+    return buf;
 }
-
 function decodeValues_FLOAT(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.readFloatLE(cursor.offset));
-    cursor.offset += 4;
-  }
-
-  return values;
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.readFloatLE(cursor.offset));
+        cursor.offset += 4;
+    }
+    return values;
 }
-
 function encodeValues_DOUBLE(values) {
-  const buf = Buffer.alloc(8 * values.length);
-
-  for (let i = 0; i < values.length; i++) {
-    buf.writeDoubleLE(values[i], i * 8);
-  }
-
-  return buf;
+    const buf = Buffer.alloc(8 * values.length);
+    for (let i = 0; i < values.length; i++) {
+        buf.writeDoubleLE(values[i], i * 8);
+    }
+    return buf;
 }
-
 function decodeValues_DOUBLE(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.readDoubleLE(cursor.offset));
-    cursor.offset += 8;
-  }
-
-  return values;
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.readDoubleLE(cursor.offset));
+        cursor.offset += 8;
+    }
+    return values;
 }
-
 function encodeValues_BYTE_ARRAY(values) {
-  let buf_len = 0;
-
-  for (let i = 0; i < values.length; i++) {
-    values[i] = Buffer.from(values[i]);
-    buf_len += 4 + values[i].length;
-  }
-
-  const buf = Buffer.alloc(buf_len);
-  let buf_pos = 0;
-
-  for (let i = 0; i < values.length; i++) {
-    buf.writeUInt32LE(values[i].length, buf_pos);
-    values[i].copy(buf, buf_pos + 4);
-    buf_pos += 4 + values[i].length;
-  }
-
-  return buf;
+    // tslint:disable-next-line:variable-name
+    let buf_len = 0;
+    for (let i = 0; i < values.length; i++) {
+        values[i] = Buffer.from(values[i]);
+        buf_len += 4 + values[i].length;
+    }
+    const buf = Buffer.alloc(buf_len);
+    // tslint:disable-next-line:variable-name
+    let buf_pos = 0;
+    for (let i = 0; i < values.length; i++) {
+        buf.writeUInt32LE(values[i].length, buf_pos);
+        values[i].copy(buf, buf_pos + 4);
+        buf_pos += 4 + values[i].length;
+    }
+    return buf;
 }
-
 function decodeValues_BYTE_ARRAY(cursor, count) {
-  const values = [];
-
-  for (let i = 0; i < count; i++) {
-    const len = cursor.buffer.readUInt32LE(cursor.offset);
-    cursor.offset += 4;
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
-    cursor.offset += len;
-  }
-
-  return values;
+    const values = [];
+    for (let i = 0; i < count; i++) {
+        const len = cursor.buffer.readUInt32LE(cursor.offset);
+        cursor.offset += 4;
+        values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
+        cursor.offset += len;
+    }
+    return values;
 }
-
 function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
-  if (!opts.typeLength) {
-    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
-  }
-
-  for (let i = 0; i < values.length; i++) {
-    values[i] = Buffer.from(values[i]);
-
-    if (values[i].length !== opts.typeLength) {
-      throw new Error("invalid value for FIXED_LEN_BYTE_ARRAY: ".concat(values[i]));
+    if (!opts.typeLength) {
+        throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
+    }
+    for (let i = 0; i < values.length; i++) {
+        values[i] = Buffer.from(values[i]);
+        if (values[i].length !== opts.typeLength) {
+            throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
+        }
     }
-  }
-
-  return Buffer.concat(values);
+    return Buffer.concat(values);
 }
-
 function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
-  const values = [];
-
-  if (!opts.typeLength) {
-    throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
-  }
-
-  for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
-    cursor.offset += opts.typeLength;
-  }
-
-  return values;
+    const values = [];
+    if (!opts.typeLength) {
+        throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
+    }
+    for (let i = 0; i < count; i++) {
+        values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
+        cursor.offset += opts.typeLength;
+    }
+    return values;
 }
-//# sourceMappingURL=plain.js.map