@loaders.gl/parquet 3.3.0-alpha.7 → 3.3.0-alpha.8

package/dist/parquet-writer.d.ts

@@ -1,4 +1,4 @@
 import type { Writer } from '@loaders.gl/loader-utils';
-export declare type ParquetWriterOptions = {};
+export type ParquetWriterOptions = {};
 export declare const ParquetWriter: Writer;
 //# sourceMappingURL=parquet-writer.d.ts.map

package/dist/parquet-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,oBAAY,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAU3B,CAAC"}
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAU3B,CAAC"}

package/dist/parquetjs/encoder/writer.js

@@ -62,20 +62,6 @@ const PARQUET_RDLVL_ENCODING = 'RLE';
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 class ParquetWriter {
-    /**
-     * Create a new buffered parquet writer for a given envelope writer
-     */
-    constructor(schema, envelopeWriter, opts) {
-        this.schema = schema;
-        this.envelopeWriter = envelopeWriter;
-        // @ts-ignore Row buffer typings...
-        this.rowBuffer = {};
-        this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
-        this.closed = false;
-        this.userMetadata = {};
-        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        this.writeHeader();
-    }
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
@@ -96,6 +82,20 @@ class ParquetWriter {
         const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
         return new ParquetWriter(schema, envelopeWriter, opts);
     }
+    /**
+     * Create a new buffered parquet writer for a given envelope writer
+     */
+    constructor(schema, envelopeWriter, opts) {
+        this.schema = schema;
+        this.envelopeWriter = envelopeWriter;
+        // @ts-ignore Row buffer typings...
+        this.rowBuffer = {};
+        this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
+        this.closed = false;
+        this.userMetadata = {};
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this.writeHeader();
+    }
     async writeHeader() {
         // TODO - better not mess with promises in the constructor
         try {
@@ -174,6 +174,14 @@ exports.ParquetWriter = ParquetWriter;
  * called in the correct order to produce a valid file.
  */
 class ParquetEnvelopeWriter {
+    /**
+     * Create a new parquet envelope writer that writes to the specified stream
+     */
+    static async openStream(schema, outputStream, opts) {
+        const writeFn = file_utils_1.oswrite.bind(undefined, outputStream);
+        const closeFn = file_utils_1.osclose.bind(undefined, outputStream);
+        return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);
+    }
     constructor(schema, writeFn, closeFn, fileOffset, opts) {
         this.schema = schema;
         this.write = writeFn;
@@ -184,14 +192,6 @@ class ParquetEnvelopeWriter {
         this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
         this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
     }
-    /**
-     * Create a new parquet envelope writer that writes to the specified stream
-     */
-    static async openStream(schema, outputStream, opts) {
-        const writeFn = file_utils_1.oswrite.bind(undefined, outputStream);
-        const closeFn = file_utils_1.osclose.bind(undefined, outputStream);
-        return new ParquetEnvelopeWriter(schema, writeFn, closeFn, 0, opts);
-    }
     writeSection(buf) {
         this.offset += buf.length;
         return this.write(buf);

package/dist/parquetjs/parser/parquet-envelope-reader.js

@@ -13,17 +13,17 @@ const DEFAULT_DICTIONARY_SIZE = 1e6;
  * rows from a parquet file use the ParquetReader instead
  */
 class ParquetEnvelopeReader {
+    static async openBuffer(buffer) {
+        const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
+        const closeFn = () => Promise.resolve();
+        return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
+    }
     constructor(read, close, fileSize, options) {
         this.read = read;
         this.close = close;
         this.fileSize = fileSize;
         this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;
     }
-    static async openBuffer(buffer) {
-        const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
-        const closeFn = () => Promise.resolve();
-        return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
-    }
     async readHeader() {
         const buffer = await this.read(0, constants_1.PARQUET_MAGIC.length);
         const magic = buffer.toString();

package/dist/parquetjs/parser/parquet-reader.js

@@ -15,22 +15,6 @@ const decoders_1 = require("./decoders");
  * avoid leaking file descriptors.
  */
 class ParquetReader {
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is not recommended to call this constructor directly except for advanced
-     * and internal use cases. Consider using one of the open{File,Buffer} methods
-     * instead
-     */
-    constructor(metadata, envelopeReader) {
-        if (metadata.version !== constants_1.PARQUET_VERSION) {
-            throw new Error('invalid parquet version');
-        }
-        this.metadata = metadata;
-        this.envelopeReader = envelopeReader;
-        const root = this.metadata.schema[0];
-        const { schema } = (0, decoders_1.decodeSchema)(this.metadata.schema, 1, root.num_children);
-        this.schema = new schema_1.ParquetSchema(schema);
-    }
     /**
      * return a new parquet reader initialized with a read function
      */
@@ -82,6 +66,22 @@ class ParquetReader {
             throw err;
         }
     }
+    /**
+     * Create a new parquet reader from the file metadata and an envelope reader.
+     * It is not recommended to call this constructor directly except for advanced
+     * and internal use cases. Consider using one of the open{File,Buffer} methods
+     * instead
+     */
+    constructor(metadata, envelopeReader) {
+        if (metadata.version !== constants_1.PARQUET_VERSION) {
+            throw new Error('invalid parquet version');
+        }
+        this.metadata = metadata;
+        this.envelopeReader = envelopeReader;
+        const root = this.metadata.schema[0];
+        const { schema } = (0, decoders_1.decodeSchema)(this.metadata.schema, 1, root.num_children);
+        this.schema = new schema_1.ParquetSchema(schema);
+    }
     /**
      * Close this parquet reader. You MUST call this method once you're finished
      * reading rows

package/dist/parquetjs/schema/declare.d.ts

@@ -1,18 +1,18 @@
 import Int64 from 'node-int64';
 import type { PageHeader } from '../parquet-thrift';
-export declare type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
-export declare type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4' | 'LZ4_RAW' | 'ZSTD';
-export declare type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';
-export declare type ParquetType = PrimitiveType | OriginalType;
+export type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
+export type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4' | 'LZ4_RAW' | 'ZSTD';
+export type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';
+export type ParquetType = PrimitiveType | OriginalType;
 /**
  * Physical type
  */
-export declare type PrimitiveType = 'BOOLEAN' | 'INT32' | 'INT64' | 'INT96' | 'FLOAT' | 'DOUBLE' | 'BYTE_ARRAY' | 'FIXED_LEN_BYTE_ARRAY';
+export type PrimitiveType = 'BOOLEAN' | 'INT32' | 'INT64' | 'INT96' | 'FLOAT' | 'DOUBLE' | 'BYTE_ARRAY' | 'FIXED_LEN_BYTE_ARRAY';
 /**
  * Logical type
  */
-export declare type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'JSON' | 'BSON' | 'INTERVAL';
-export declare type ParquetDictionary = string[];
+export type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'JSON' | 'BSON' | 'INTERVAL';
+export type ParquetDictionary = string[];
 export interface SchemaDefinition {
     [string: string]: FieldDefinition;
 }

package/dist/parquetjs/schema/declare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,oBAAY,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,oBAAY,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,oBAAY,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,oBAAY,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,oBAAY,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,oBAAY,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,oBAAY,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,qBAAa,aAAa;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC5B,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAM;CAI/E"}
+{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,qBAAa,aAAa;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC5B,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAM;CAI/E"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.3.0-alpha.7",
+  "version": "3.3.0-alpha.8",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -40,9 +40,9 @@
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.3.0-alpha.7",
-    "@loaders.gl/loader-utils": "3.3.0-alpha.7",
-    "@loaders.gl/schema": "3.3.0-alpha.7",
+    "@loaders.gl/compression": "3.3.0-alpha.8",
+    "@loaders.gl/loader-utils": "3.3.0-alpha.8",
+    "@loaders.gl/schema": "3.3.0-alpha.8",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -68,5 +68,5 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "29b08f3519c50984e84bf4234e607cab7c7d1c3e"
+  "gitHead": "69cfde0340328dd800c7c90151b56b406f47e9ae"
 }