@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundled.d.ts +6 -0
- package/dist/bundled.d.ts.map +1 -0
- package/dist/bundled.js +7 -0
- package/dist/bundled.js.map +1 -0
- package/dist/geoparquet-loader-with-parser.d.ts +34 -0
- package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
- package/dist/geoparquet-loader-with-parser.js +52 -0
- package/dist/geoparquet-loader-with-parser.js.map +1 -0
- package/dist/geoparquet-loader.d.ts +61 -0
- package/dist/geoparquet-loader.d.ts.map +1 -0
- package/dist/geoparquet-loader.js +30 -0
- package/dist/geoparquet-loader.js.map +1 -0
- package/dist/index.cjs +5879 -5643
- package/dist/index.cjs.map +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -11
- package/dist/index.js.map +1 -1
- package/dist/lib/constants.d.ts +1 -1
- package/dist/lib/constants.js +2 -2
- package/dist/lib/constants.js.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js +149 -0
- package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
- package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
- package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
- package/dist/lib/geo/geospatial-metadata.js +357 -0
- package/dist/lib/geo/geospatial-metadata.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
- package/dist/lib/parsers/get-parquet-schema.js +3 -1
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
- package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-tables.d.ts +50 -0
- package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-tables.js +101 -0
- package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +6 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js +23 -0
- package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
- package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -4
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +77 -26
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
- package/dist/lib/utils/load-wasm.d.ts +2 -2
- package/dist/lib/utils/load-wasm.d.ts.map +1 -1
- package/dist/lib/utils/load-wasm.js +7 -7
- package/dist/lib/utils/load-wasm.js.map +1 -1
- package/dist/lib/utils/make-stream-iterator.js +1 -1
- package/dist/lib/utils/make-stream-iterator.js.map +1 -1
- package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
- package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
- package/dist/lib/utils/normalize-parquet-options.js +11 -0
- package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
- package/dist/parquet-format.d.ts +2 -0
- package/dist/parquet-format.d.ts.map +1 -1
- package/dist/parquet-format.js +2 -0
- package/dist/parquet-format.js.map +1 -1
- package/dist/parquet-js-loader.d.ts +57 -0
- package/dist/parquet-js-loader.d.ts.map +1 -0
- package/dist/parquet-js-loader.js +33 -0
- package/dist/parquet-js-loader.js.map +1 -0
- package/dist/parquet-js-writer.d.ts +34 -0
- package/dist/parquet-js-writer.d.ts.map +1 -0
- package/dist/parquet-js-writer.js +28 -0
- package/dist/parquet-js-writer.js.map +1 -0
- package/dist/parquet-loader-options.d.ts +56 -0
- package/dist/parquet-loader-options.d.ts.map +1 -0
- package/dist/parquet-loader-options.js +11 -0
- package/dist/parquet-loader-options.js.map +1 -0
- package/dist/parquet-loader-with-parser.d.ts +33 -0
- package/dist/parquet-loader-with-parser.d.ts.map +1 -0
- package/dist/parquet-loader-with-parser.js +40 -0
- package/dist/parquet-loader-with-parser.js.map +1 -0
- package/dist/parquet-loader.d.ts +62 -0
- package/dist/parquet-loader.d.ts.map +1 -0
- package/dist/parquet-loader.js +53 -0
- package/dist/parquet-loader.js.map +1 -0
- package/dist/parquet-wasm-loader-with-parser.d.ts +40 -0
- package/dist/parquet-wasm-loader-with-parser.d.ts.map +1 -0
- package/dist/parquet-wasm-loader-with-parser.js +68 -0
- package/dist/parquet-wasm-loader-with-parser.js.map +1 -0
- package/dist/parquet-writer.d.ts +10 -0
- package/dist/parquet-writer.d.ts.map +1 -0
- package/dist/parquet-writer.js +29 -0
- package/dist/parquet-writer.js.map +1 -0
- package/dist/parquet_wasm_bg.wasm +0 -0
- package/dist/parquetjs/codecs/declare.d.ts +2 -2
- package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
- package/dist/parquetjs/codecs/dictionary.js +1 -1
- package/dist/parquetjs/codecs/dictionary.js.map +1 -1
- package/dist/parquetjs/codecs/plain.d.ts +1 -1
- package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
- package/dist/parquetjs/codecs/plain.js +36 -41
- package/dist/parquetjs/codecs/plain.js.map +1 -1
- package/dist/parquetjs/codecs/rle.d.ts +1 -1
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +16 -21
- package/dist/parquetjs/codecs/rle.js.map +1 -1
- package/dist/parquetjs/compression.d.ts +2 -3
- package/dist/parquetjs/compression.d.ts.map +1 -1
- package/dist/parquetjs/compression.js +4 -105
- package/dist/parquetjs/compression.js.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.d.ts +4 -4
- package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
- package/dist/parquetjs/encoder/parquet-encoder.js +38 -33
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +1 -1
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.d.ts +0 -1
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +0 -1
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/parquetjs/parser/decoders.d.ts +1 -1
- package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
- package/dist/parquetjs/parser/decoders.js +4 -22
- package/dist/parquetjs/parser/decoders.js.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts +1 -1
- package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
- package/dist/parquetjs/parser/parquet-reader.js +9 -9
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
- package/dist/parquetjs/schema/declare.d.ts +1 -1
- package/dist/parquetjs/schema/declare.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.d.ts.map +1 -1
- package/dist/parquetjs/schema/types.js +36 -35
- package/dist/parquetjs/schema/types.js.map +1 -1
- package/dist/parquetjs/utils/binary-utils.d.ts +39 -0
- package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
- package/dist/parquetjs/utils/binary-utils.js +102 -0
- package/dist/parquetjs/utils/binary-utils.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.d.ts +1 -1
- package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/file-utils.js +3 -3
- package/dist/parquetjs/utils/file-utils.js.map +1 -1
- package/dist/parquetjs/utils/read-utils.d.ts +4 -4
- package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
- package/dist/parquetjs/utils/read-utils.js +12 -17
- package/dist/parquetjs/utils/read-utils.js.map +1 -1
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +78 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
- package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
- package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
- package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
- package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
- package/dist/unbundled.d.ts +6 -0
- package/dist/unbundled.d.ts.map +1 -0
- package/dist/unbundled.js +7 -0
- package/dist/unbundled.js.map +1 -0
- package/package.json +21 -25
- package/src/bundled.ts +9 -0
- package/src/geoparquet-loader-with-parser.ts +95 -0
- package/src/geoparquet-loader.ts +54 -0
- package/src/index.ts +7 -25
- package/src/lib/constants.ts +1 -1
- package/src/lib/encoders/encode-arrow-to-parquet.ts +9 -15
- package/src/lib/encoders/encode-table-to-parquet-js.ts +199 -0
- package/src/lib/geo/geospatial-metadata.ts +507 -0
- package/src/lib/parsers/get-parquet-schema.ts +3 -1
- package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
- package/src/lib/parsers/parse-parquet-tables.ts +137 -0
- package/src/lib/parsers/parse-parquet-to-arrow-js.ts +36 -0
- package/src/lib/parsers/parse-parquet-to-arrow.ts +63 -36
- package/src/lib/parsers/parse-parquet-to-columns.ts +6 -7
- package/src/lib/parsers/parse-parquet-to-json.ts +101 -33
- package/src/lib/utils/load-wasm.ts +16 -9
- package/src/lib/utils/make-stream-iterator.ts +1 -1
- package/src/lib/utils/normalize-parquet-options.ts +14 -0
- package/src/parquet-format.ts +2 -0
- package/src/parquet-js-loader.ts +43 -0
- package/src/parquet-js-writer.ts +52 -0
- package/src/parquet-loader-options.ts +63 -0
- package/src/parquet-loader-with-parser.ts +68 -0
- package/src/parquet-loader.ts +79 -0
- package/src/parquet-wasm-loader-with-parser.ts +115 -0
- package/src/parquet-writer.ts +44 -0
- package/src/parquetjs/codecs/declare.ts +2 -2
- package/src/parquetjs/codecs/dictionary.ts +1 -1
- package/src/parquetjs/codecs/plain.ts +67 -53
- package/src/parquetjs/codecs/rle.ts +20 -26
- package/src/parquetjs/compression.ts +7 -114
- package/src/parquetjs/encoder/parquet-encoder.ts +49 -43
- package/src/parquetjs/parquet-thrift/ColumnIndex.ts +1 -1
- package/src/parquetjs/parquet-thrift/index.ts +0 -2
- package/src/parquetjs/parser/decoders.ts +5 -23
- package/src/parquetjs/parser/parquet-reader.ts +10 -12
- package/src/parquetjs/schema/declare.ts +1 -1
- package/src/parquetjs/schema/types.ts +48 -40
- package/src/parquetjs/utils/binary-utils.ts +127 -0
- package/src/parquetjs/utils/file-utils.ts +4 -4
- package/src/parquetjs/utils/read-utils.ts +19 -34
- package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
- package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
- package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
- package/src/unbundled.ts +9 -0
- package/dist/parquet-arrow-loader.d.ts +0 -43
- package/dist/parquet-arrow-loader.d.ts.map +0 -1
- package/dist/parquet-arrow-loader.js +0 -49
- package/dist/parquet-arrow-loader.js.map +0 -1
- package/dist/parquet-arrow-writer.d.ts +0 -9
- package/dist/parquet-arrow-writer.d.ts.map +0 -1
- package/dist/parquet-arrow-writer.js +0 -23
- package/dist/parquet-arrow-writer.js.map +0 -1
- package/dist/parquet-json-loader.d.ts +0 -148
- package/dist/parquet-json-loader.d.ts.map +0 -1
- package/dist/parquet-json-loader.js +0 -106
- package/dist/parquet-json-loader.js.map +0 -1
- package/dist/parquet-json-writer.d.ts +0 -19
- package/dist/parquet-json-writer.d.ts.map +0 -1
- package/dist/parquet-json-writer.js +0 -20
- package/dist/parquet-json-writer.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
- package/dist/polyfills/buffer/buffer.d.ts +0 -222
- package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
- package/dist/polyfills/buffer/buffer.js +0 -1868
- package/dist/polyfills/buffer/buffer.js.map +0 -1
- package/dist/polyfills/buffer/index.d.ts +0 -4
- package/dist/polyfills/buffer/index.d.ts.map +0 -1
- package/dist/polyfills/buffer/index.js +0 -9
- package/dist/polyfills/buffer/index.js.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
- package/dist/workers/parquet-json-worker.d.ts +0 -2
- package/dist/workers/parquet-json-worker.d.ts.map +0 -1
- package/dist/workers/parquet-json-worker.js +0 -7
- package/dist/workers/parquet-json-worker.js.map +0 -1
- package/src/parquet-arrow-loader.ts +0 -74
- package/src/parquet-arrow-writer.ts +0 -33
- package/src/parquet-json-loader.ts +0 -160
- package/src/parquet-json-writer.ts +0 -27
- package/src/parquetjs/modules.d.ts +0 -25
- package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
- package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
- package/src/polyfills/buffer/buffer.ts +0 -2205
- package/src/polyfills/buffer/index.ts +0 -11
- package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
- package/src/workers/parquet-json-worker.ts +0 -8
|
@@ -9,11 +9,13 @@ import * as Shred from "../schema/shred.js";
|
|
|
9
9
|
import { ColumnChunk, ColumnMetaData, CompressionCodec, ConvertedType, DataPageHeader, DataPageHeaderV2, Encoding, FieldRepetitionType, FileMetaData, KeyValue, PageHeader, PageType, RowGroup, SchemaElement, Type } from "../parquet-thrift/index.js";
|
|
10
10
|
import { osopen, oswrite, osclose } from "../utils/file-utils.js";
|
|
11
11
|
import { getBitWidth, serializeThrift } from "../utils/read-utils.js";
|
|
12
|
-
import
|
|
12
|
+
import { concatUint8Arrays, encodeUtf8, writeUInt32LE } from "../utils/binary-utils.js";
|
|
13
|
+
import { CompactInt64 } from "../utils/uint8-array-compact-protocol.js";
|
|
13
14
|
/**
|
|
14
15
|
* Parquet File Magic String
|
|
15
16
|
*/
|
|
16
17
|
const PARQUET_MAGIC = 'PAR1';
|
|
18
|
+
const PARQUET_MAGIC_BYTES = encodeUtf8(PARQUET_MAGIC);
|
|
17
19
|
/**
|
|
18
20
|
* Parquet File Format Version
|
|
19
21
|
*/
|
|
@@ -63,8 +65,7 @@ export class ParquetEncoder {
|
|
|
63
65
|
constructor(schema, envelopeWriter, opts) {
|
|
64
66
|
this.schema = schema;
|
|
65
67
|
this.envelopeWriter = envelopeWriter;
|
|
66
|
-
|
|
67
|
-
this.rowBuffer = {};
|
|
68
|
+
this.rowBuffer = schema.rowGroup();
|
|
68
69
|
this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
|
|
69
70
|
this.closed = false;
|
|
70
71
|
this.userMetadata = {};
|
|
@@ -91,8 +92,8 @@ export class ParquetEncoder {
|
|
|
91
92
|
}
|
|
92
93
|
Shred.shredRecord(this.schema, row, this.rowBuffer);
|
|
93
94
|
if (this.rowBuffer.rowCount >= this.rowGroupSize) {
|
|
94
|
-
|
|
95
|
-
this.rowBuffer =
|
|
95
|
+
await this.envelopeWriter.writeRowGroup(this.rowBuffer);
|
|
96
|
+
this.rowBuffer = this.schema.rowGroup();
|
|
96
97
|
}
|
|
97
98
|
}
|
|
98
99
|
/**
|
|
@@ -107,8 +108,8 @@ export class ParquetEncoder {
|
|
|
107
108
|
}
|
|
108
109
|
this.closed = true;
|
|
109
110
|
if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
|
|
110
|
-
|
|
111
|
-
this.rowBuffer =
|
|
111
|
+
await this.envelopeWriter.writeRowGroup(this.rowBuffer);
|
|
112
|
+
this.rowBuffer = this.schema.rowGroup();
|
|
112
113
|
}
|
|
113
114
|
await this.envelopeWriter.writeFooter(this.userMetadata);
|
|
114
115
|
await this.envelopeWriter.close();
|
|
@@ -182,7 +183,7 @@ export class ParquetEnvelopeWriter {
|
|
|
182
183
|
* Encode the parquet file header
|
|
183
184
|
*/
|
|
184
185
|
writeHeader() {
|
|
185
|
-
return this.writeSection(
|
|
186
|
+
return this.writeSection(PARQUET_MAGIC_BYTES);
|
|
186
187
|
}
|
|
187
188
|
/**
|
|
188
189
|
* Encode a parquet row group. The records object should be created using the
|
|
@@ -266,14 +267,14 @@ function encodeValues(type, encoding, values, opts) {
|
|
|
266
267
|
*/
|
|
267
268
|
async function encodeDataPage(column, data) {
|
|
268
269
|
/* encode repetition and definition levels */
|
|
269
|
-
let rLevelsBuf =
|
|
270
|
+
let rLevelsBuf = new Uint8Array(0);
|
|
270
271
|
if (column.rLevelMax > 0) {
|
|
271
272
|
rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
|
|
272
273
|
bitWidth: getBitWidth(column.rLevelMax)
|
|
273
274
|
// disableEnvelope: false
|
|
274
275
|
});
|
|
275
276
|
}
|
|
276
|
-
let dLevelsBuf =
|
|
277
|
+
let dLevelsBuf = new Uint8Array(0);
|
|
277
278
|
if (column.dLevelMax > 0) {
|
|
278
279
|
dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
|
|
279
280
|
bitWidth: getBitWidth(column.dLevelMax)
|
|
@@ -285,7 +286,7 @@ async function encodeDataPage(column, data) {
|
|
|
285
286
|
typeLength: column.typeLength,
|
|
286
287
|
bitWidth: column.typeLength
|
|
287
288
|
});
|
|
288
|
-
const dataBuf =
|
|
289
|
+
const dataBuf = concatUint8Arrays([rLevelsBuf, dLevelsBuf, valuesBuf]);
|
|
289
290
|
// compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
|
|
290
291
|
const compressedBuf = await Compression.deflate(column.compression, dataBuf);
|
|
291
292
|
/* build page header */
|
|
@@ -302,7 +303,7 @@ async function encodeDataPage(column, data) {
|
|
|
302
303
|
});
|
|
303
304
|
/* concat page header, repetition and definition levels and values */
|
|
304
305
|
const headerBuf = serializeThrift(header);
|
|
305
|
-
const page =
|
|
306
|
+
const page = concatUint8Arrays([headerBuf, compressedBuf]);
|
|
306
307
|
return { header, headerSize: headerBuf.length, page };
|
|
307
308
|
}
|
|
308
309
|
/**
|
|
@@ -317,14 +318,14 @@ async function encodeDataPageV2(column, data, rowCount) {
|
|
|
317
318
|
// compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
|
|
318
319
|
const compressedBuf = await Compression.deflate(column.compression, valuesBuf);
|
|
319
320
|
/* encode repetition and definition levels */
|
|
320
|
-
let rLevelsBuf =
|
|
321
|
+
let rLevelsBuf = new Uint8Array(0);
|
|
321
322
|
if (column.rLevelMax > 0) {
|
|
322
323
|
rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
|
|
323
324
|
bitWidth: getBitWidth(column.rLevelMax),
|
|
324
325
|
disableEnvelope: true
|
|
325
326
|
});
|
|
326
327
|
}
|
|
327
|
-
let dLevelsBuf =
|
|
328
|
+
let dLevelsBuf = new Uint8Array(0);
|
|
328
329
|
if (column.dLevelMax > 0) {
|
|
329
330
|
dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
|
|
330
331
|
bitWidth: getBitWidth(column.dLevelMax),
|
|
@@ -348,7 +349,7 @@ async function encodeDataPageV2(column, data, rowCount) {
|
|
|
348
349
|
});
|
|
349
350
|
/* concat page header, repetition and definition levels and values */
|
|
350
351
|
const headerBuf = serializeThrift(header);
|
|
351
|
-
const page =
|
|
352
|
+
const page = concatUint8Arrays([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
|
|
352
353
|
return { header, headerSize: headerBuf.length, page };
|
|
353
354
|
}
|
|
354
355
|
/**
|
|
@@ -358,7 +359,7 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
|
|
|
358
359
|
const data = buffer.columnData[column.path.join()];
|
|
359
360
|
const baseOffset = (opts.baseOffset || 0) + offset;
|
|
360
361
|
/* encode data page(s) */
|
|
361
|
-
// const pages:
|
|
362
|
+
// const pages: Uint8Array[] = [];
|
|
362
363
|
let pageBuf;
|
|
363
364
|
// tslint:disable-next-line:variable-name
|
|
364
365
|
let total_uncompressed_size = 0;
|
|
@@ -373,16 +374,16 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
|
|
|
373
374
|
total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
|
|
374
375
|
total_compressed_size += result.header.compressed_page_size + result.headerSize;
|
|
375
376
|
}
|
|
376
|
-
// const pagesBuf =
|
|
377
|
+
// const pagesBuf = concatUint8Arrays(pages);
|
|
377
378
|
// const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;
|
|
378
379
|
/* prepare metadata header */
|
|
379
380
|
const metadata = new ColumnMetaData({
|
|
380
381
|
path_in_schema: column.path,
|
|
381
|
-
num_values: data.count,
|
|
382
|
-
data_page_offset: baseOffset,
|
|
382
|
+
num_values: int64(data.count),
|
|
383
|
+
data_page_offset: int64(baseOffset),
|
|
383
384
|
encodings: [],
|
|
384
|
-
total_uncompressed_size, // : pagesBuf.length,
|
|
385
|
-
total_compressed_size,
|
|
385
|
+
total_uncompressed_size: int64(total_uncompressed_size), // : pagesBuf.length,
|
|
386
|
+
total_compressed_size: int64(total_compressed_size),
|
|
386
387
|
type: Type[column.primitiveType],
|
|
387
388
|
codec: CompressionCodec[column.compression]
|
|
388
389
|
});
|
|
@@ -391,7 +392,7 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
|
|
|
391
392
|
metadata.encodings.push(Encoding[column.encoding]);
|
|
392
393
|
/* concat metadata header and data pages */
|
|
393
394
|
const metadataOffset = baseOffset + pageBuf.length;
|
|
394
|
-
const body =
|
|
395
|
+
const body = concatUint8Arrays([pageBuf, serializeThrift(metadata)]);
|
|
395
396
|
return { body, metadata, metadataOffset };
|
|
396
397
|
}
|
|
397
398
|
/**
|
|
@@ -399,23 +400,23 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
|
|
|
399
400
|
*/
|
|
400
401
|
async function encodeRowGroup(schema, data, opts) {
|
|
401
402
|
const metadata = new RowGroup({
|
|
402
|
-
num_rows: data.rowCount,
|
|
403
|
+
num_rows: int64(data.rowCount),
|
|
403
404
|
columns: [],
|
|
404
|
-
total_byte_size: 0
|
|
405
|
+
total_byte_size: int64(0)
|
|
405
406
|
});
|
|
406
|
-
let body =
|
|
407
|
+
let body = new Uint8Array(0);
|
|
407
408
|
for (const field of schema.fieldList) {
|
|
408
409
|
if (field.isNested) {
|
|
409
410
|
continue; // eslint-disable-line no-continue
|
|
410
411
|
}
|
|
411
412
|
const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
|
|
412
413
|
const cchunk = new ColumnChunk({
|
|
413
|
-
file_offset: cchunkData.metadataOffset,
|
|
414
|
+
file_offset: int64(cchunkData.metadataOffset),
|
|
414
415
|
meta_data: cchunkData.metadata
|
|
415
416
|
});
|
|
416
417
|
metadata.columns.push(cchunk);
|
|
417
|
-
metadata.total_byte_size =
|
|
418
|
-
body =
|
|
418
|
+
metadata.total_byte_size = int64(Number(metadata.total_byte_size) + cchunkData.body.length);
|
|
419
|
+
body = concatUint8Arrays([body, cchunkData.body]);
|
|
419
420
|
}
|
|
420
421
|
return { body, metadata };
|
|
421
422
|
}
|
|
@@ -426,7 +427,7 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
|
426
427
|
const metadata = new FileMetaData({
|
|
427
428
|
version: PARQUET_VERSION,
|
|
428
429
|
created_by: 'parquets',
|
|
429
|
-
num_rows: rowCount,
|
|
430
|
+
num_rows: int64(rowCount),
|
|
430
431
|
row_groups: rowGroups,
|
|
431
432
|
schema: [],
|
|
432
433
|
key_value_metadata: []
|
|
@@ -464,10 +465,14 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
|
|
|
464
465
|
metadata.schema.push(schemaElem);
|
|
465
466
|
}
|
|
466
467
|
const metadataEncoded = serializeThrift(metadata);
|
|
467
|
-
const footerEncoded =
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
footerEncoded.
|
|
468
|
+
const footerEncoded = new Uint8Array(metadataEncoded.length + 8);
|
|
469
|
+
footerEncoded.set(metadataEncoded);
|
|
470
|
+
writeUInt32LE(footerEncoded, metadataEncoded.length, metadataEncoded.length);
|
|
471
|
+
footerEncoded.set(PARQUET_MAGIC_BYTES, metadataEncoded.length + 4);
|
|
471
472
|
return footerEncoded;
|
|
472
473
|
}
|
|
474
|
+
/** Wrap a writer metadata number in the thrift int64 compatibility object. */
|
|
475
|
+
function int64(value) {
|
|
476
|
+
return new CompactInt64(BigInt(value));
|
|
477
|
+
}
|
|
473
478
|
//# sourceMappingURL=parquet-encoder.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-encoder.js","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAIpE,OAAO,EAAsB,cAAc,EAAC,2BAAwB;AACpE,OAAO,KAAK,WAAW,0BAAuB;AAU9C,OAAO,KAAK,KAAK,2BAAwB;AACzC,OAAO,EACL,WAAW,EACX,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,IAAI,EACL,mCAAgC;AACjC,OAAO,EAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAC,+BAA4B;AAC7D,OAAO,EAAC,WAAW,EAAE,eAAe,EAAC,+BAA4B;AACjE,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B;;GAEG;AACH,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B;;GAEG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B;;GAEG;AACH,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,8BAA8B,GAAG,IAAI,CAAC;AAE5C;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAiBrC;;;;GAIG;AACH,6DAA6D;AAC7D,MAAM,OAAO,cAAc;IACzB;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CACnB,MAAqB,EACrB,IAAY,EACZ,IAA4B;QAE5B,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9C,OAAO,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,MAAqB,EACrB,YAA6B,EAC7B,OAA8B,EAAE;QAEhC,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC1F,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC;IAEM,MAAM,CAAgB;IACtB,cAAc,CAAwB;IACtC,SAAS,CAAkB;IAC3B,YAAY,CAAS;IACrB,MAAM,CAAU;IAChB,YAAY,CAAyB;IAE5C;;OAEG;IACH,YACE,MAAqB,EACrB,cAAqC,EACrC,IAA2B;QAE3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,mCAAmC;QACnC,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,8BAA8B,CAAC;QACxE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,mEAAmE;QACnE,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,WAAW;QACf,0DAA0D;QAC1D,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAuB,GAAM;QAC1C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,aAAa;YACb,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,QAAqB;QAC/B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAChF,aAAa;YACb,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACtB,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAClC,8BAA8B;QAE9B,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,GAAW,EAAE,KAAa;QACpC,qCAAqC;QACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACzB,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,qBAAqB;IAChC;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,MAAqB,EACrB,YAA6B,EAC7B,IAA2B;QAE3B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACtD,OAAO,IAAI,qBAAqB,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACtE,CAAC;IAEM,MAAM,CAAgB;IACtB,KAAK,CAAiC;IACtC,KAAK,CAAsB;IAC3B,MAAM,CAAS;IACf,QAAQ,CAAS;IACjB,SAAS,CAAa;IACtB,QAAQ,CAAS;IACjB,aAAa,CAAU;IAE9B,YACE,MAAqB,EACrB,OAAuC,EACvC,OAA4B,EAC5B,UAAkB,EAClB,IAA2B;QAE3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,yBAAyB,CAAC;QAC3D,IAAI,CAAC,aAAa,GAAG,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACrF,CAAC;IAED,YAAY,CAAC,GAAW;QACtB,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,OAAwB;QAC1C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;YACxD,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACrC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,YAAoC;QAC9C,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,qDAAqD;YACrD,YAAY,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC,YAAY,CACtB,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CACvE,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH;;GAEG;AACH,SAAS,YAAY,CACnB,IAAmB,EACnB,QAAsB,EACtB,MAAa,EACb,IAAyB;IAEzB,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAC3B,MAAoB,EACpB,IAAwB;IAMxB,6CAA6C;IAC7C,IAAI,UAAU,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,yBAAyB;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,yBAAyB;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,aAAc,EAAE,MAAM,CAAC,QAAS,EAAE,IAAI,CAAC,MAAM,EAAE;QACnF,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,QAAQ,EAAE,MAAM,CAAC,UAAU;KAC5B,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,CAAiB,CAAC,CAAC;IAEnF,8GAA8G;IAC9G,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,WAAY,EAAE,OAAO,CAAC,CAAC;IAE9E,uBAAuB;IACvB,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,SAAS;QACxB,gBAAgB,EAAE,IAAI,cAAc,CAAC;YACnC,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAS,CAAQ;YAC3C,yBAAyB,EAAE,QAAQ,CAAC,sBAAsB,CAAC,EAAE,4BAA4B;YACzF,yBAAyB,EAAE,QAAQ,CAAC,sBAAsB,CAAC,CAAC,2BAA2B;SACxF,CAAC;QACF,sBAAsB,EAAE,OAAO,CAAC,MAAM;QACtC,oBAAoB,EAAE,aAAa,CAAC,MAAM;KAC3C,CAAC,CAAC;IAEH,qEAAqE;IACrE,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,aAAa,CAAiB,CAAC,CAAC;IAEvE,OAAO,EAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,MAAoB,EACpB,IAAwB,EACxB,QAAgB;IAMhB,mBAAmB;IACnB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,aAAc,EAAE,MAAM,CAAC,QAAS,EAAE,IAAI,CAAC,MAAM,EAAE;QACnF,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,QAAQ,EAAE,MAAM,CAAC,UAAU;KAC5B,CAAC,CAAC;IAEH,8GAA8G;IAC9G,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,WAAY,EAAE,SAAS,CAAC,CAAC;IAEhF,6CAA6C;IAC7C,IAAI,UAAU,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;IAED,uBAAuB;IACvB,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,YAAY;QAC3B,mBAAmB,EAAE,IAAI,gBAAgB,CAAC;YACxC,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,SAAS,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1C,QAAQ,EAAE,QAAQ;YAClB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAS,CAAQ;YAC3C,6BAA6B,EAAE,UAAU,CAAC,MAAM;YAChD,6BAA6B,EAAE,UAAU,CAAC,MAAM;YAChD,aAAa,EAAE,MAAM,CAAC,WAAW,KAAK,cAAc;SACrD,CAAC;QACF,sBAAsB,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;QAChF,oBAAoB,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;KACnF,CAAC,CAAC;IAEH,qEAAqE;IACrE,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,CAAiB,CAAC,CAAC;IAC/F,OAAO,EAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,iBAAiB,CAC9B,MAAoB,EACpB,MAAuB,EACvB,MAAc,EACd,IAA2B;IAM3B,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;IACnD,yBAAyB;IACzB,8BAA8B;IAC9B,IAAI,OAAe,CAAC;IACpB,yCAAyC;IACzC,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,yCAAyC;IACzC,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,CAAC;QACC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa;YAC/B,CAAC,CAAC,MAAM,gBAAgB,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC;YACvD,CAAC,CAAC,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACvC,2BAA2B;QAC3B,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;QACtB,uBAAuB,IAAI,MAAM,CAAC,MAAM,CAAC,sBAAsB,GAAG,MAAM,CAAC,UAAU,CAAC;QACpF,qBAAqB,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAM,CAAC,UAAU,CAAC;IAClF,CAAC;IAED,yCAAyC;IACzC,yHAAyH;IAEzH,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;QAClC,cAAc,EAAE,MAAM,CAAC,IAAI;QAC3B,UAAU,EAAE,IAAI,CAAC,KAAK;QACtB,gBAAgB,EAAE,UAAU;QAC5B,SAAS,EAAE,EAAE;QACb,uBAAuB,EAAE,sBAAsB;QAC/C,qBAAqB;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,aAAc,CAAC;QACjC,KAAK,EAAE,gBAAgB,CAAC,MAAM,CAAC,WAAY,CAAC;KAC7C,CAAC,CAAC;IAEH,oBAAoB;IACpB,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAC1D,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAS,CAAC,CAAC,CAAC;IAEpD,2CAA2C;IAC3C,MAAM,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAiB,CAAC,CAAC;IACjF,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAC3B,MAAqB,EACrB,IAAqB,EACrB,IAA2B;IAK3B,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;QAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,EAAE;QACX,eAAe,EAAE,CAAC;KACnB,CAAC,CAAC;IAEH,IAAI,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,SAAS,CAAC,kCAAkC;QAC9C,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3E,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC;YAC7B,WAAW,EAAE,UAAU,CAAC,cAAc;YACtC,SAAS,EAAE,UAAU,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,eAAe,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhG,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAiB,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,MAAqB,EACrB,QAAgB,EAChB,SAAqB,EACrB,YAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC;QAChC,OAAO,EAAE,eAAe;QACxB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,EAAE;QACV,kBAAkB,EAAE,EAAE;KACvB,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC;YACtB,GAAG;YACH,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC;SACzB,CAAC,CAAC;QACH,QAAQ,CAAC,kBAAkB,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,CAAC;QACC,MAAM,UAAU,GAAG,IAAI,aAAa,CAAC;YACnC,IAAI,EAAE,MAAM;YACZ,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM;SAChD,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,IAAI,aAAa,CAAC;YACnC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,eAAe,EAAE,IAAW;SAC7B,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,aAAc,CAAS,CAAC;QACvD,CAAC;QAED,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,UAAU,CAAC,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC,YAAY,CAAkB,CAAC;QACjF,CAAC;QAED,UAAU,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;QAE1C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,eAAe,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/D,eAAe,CAAC,IAAI,CAAC,aAA2B,CAAC,CAAC;IAClD,aAAa,CAAC,aAAa,CAAC,eAAe,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC5E,aAAa,CAAC,KAAK,CAAC,aAAa,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/D,OAAO,aAAa,CAAC;AACvB,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet-encoder.js","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAIpE,OAAO,EAAsB,cAAc,EAAC,2BAAwB;AACpE,OAAO,KAAK,WAAW,0BAAuB;AAU9C,OAAO,KAAK,KAAK,2BAAwB;AACzC,OAAO,EACL,WAAW,EACX,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,IAAI,EACL,mCAAgC;AACjC,OAAO,EAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAC,+BAA4B;AAC7D,OAAO,EAAC,WAAW,EAAE,eAAe,EAAC,+BAA4B;AACjE,OAAO,EAAC,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAC,iCAA8B;AACnF,OAAO,EAAC,YAAY,EAAC,iDAA8C;AAEnE;;GAEG;AACH,MAAM,aAAa,GAAG,MAAM,CAAC;AAC7B,MAAM,mBAAmB,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC;AAEtD;;GAEG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B;;GAEG;AACH,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,8BAA8B,GAAG,IAAI,CAAC;AAE5C;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAiBrC;;;;GAIG;AACH,6DAA6D;AAC7D,MAAM,OAAO,cAAc;IACzB;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CACnB,MAAqB,EACrB,IAAY,EACZ,IAA4B;QAE5B,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9C,OAAO,cAAc,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,MAAqB,EACrB,YAA6B,EAC7B,OAA8B,EAAE;QAEhC,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC1F,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC;IAEM,MAAM,CAAgB;IACtB,cAAc,CAAwB;IACtC,SAAS,CAAkB;IAC3B,YAAY,CAAS;IACrB,MAAM,CAAU;IAChB,YAAY,CAAyB;IAE5C;;OAEG;IACH,YACE,MAAqB,EACrB,cAAqC,EACrC,IAA2B;QAE3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,8BAA8B,CAAC;QACxE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,mEAAmE;QACnE,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,WAAW;QACf,0DAA0D;QAC1D,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAuB,GAAM;QAC1C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,QAAqB;QAC/B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAEnB,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAChF,MAAM,IAAI,CAAC,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAClC,8BAA8B;QAE9B,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,GAAW,EAAE,KAAa;QACpC,qCAAqC;QACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACzB,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,qBAAqB;IAChC;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,UAAU,CACrB,MAAqB,EACrB,YAA6B,EAC7B,IAA2B;QAE3B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACtD,OAAO,IAAI,qBAAqB,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IACtE,CAAC;IAEM,MAAM,CAAgB;IACtB,KAAK,CAAqC;IAC1C,KAAK,CAAsB;IAC3B,MAAM,CAAS;IACf,QAAQ,CAAS;IACjB,SAAS,CAAa;IACtB,QAAQ,CAAS;IACjB,aAAa,CAAU;IAE9B,YACE,MAAqB,EACrB,OAA2C,EAC3C,OAA4B,EAC5B,UAAkB,EAClB,IAA2B;QAE3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,yBAAyB,CAAC;QAC3D,IAAI,CAAC,aAAa,GAAG,eAAe,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACrF,CAAC;IAED,YAAY,CAAC,GAAe;QAC1B,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC;IAChD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,OAAwB;QAC1C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;YACxD,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACrC,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,YAAoC;QAC9C,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,qDAAqD;YACrD,YAAY,GAAG,EAAE,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC,YAAY,CACtB,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CACvE,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH;;GAEG;AACH,SAAS,YAAY,CACnB,IAAmB,EACnB,QAAsB,EACtB,MAAa,EACb,IAAyB;IAEzB,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAC3B,MAAoB,EACpB,IAAwB;IAMxB,6CAA6C;IAC7C,IAAI,UAAU,GAAe,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,yBAAyB;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,GAAe,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,yBAAyB;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,aAAc,EAAE,MAAM,CAAC,QAAS,EAAE,IAAI,CAAC,MAAM,EAAE;QACnF,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,QAAQ,EAAE,MAAM,CAAC,UAAU;KAC5B,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,iBAAiB,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;IAEvE,8GAA8G;IAC9G,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,WAAY,EAAE,OAAO,CAAC,CAAC;IAE9E,uBAAuB;IACvB,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,SAAS;QACxB,gBAAgB,EAAE,IAAI,cAAc,CAAC;YACnC,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAS,CAAQ;YAC3C,yBAAyB,EAAE,QAAQ,CAAC,sBAAsB,CAAC,EAAE,4BAA4B;YACzF,yBAAyB,EAAE,QAAQ,CAAC,sBAAsB,CAAC,CAAC,2BAA2B;SACxF,CAAC;QACF,sBAAsB,EAAE,OAAO,CAAC,MAAM;QACtC,oBAAoB,EAAE,aAAa,CAAC,MAAM;KAC3C,CAAC,CAAC;IAEH,qEAAqE;IACrE,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;IAE3D,OAAO,EAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,MAAoB,EACpB,IAAwB,EACxB,QAAgB;IAMhB,mBAAmB;IACnB,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,aAAc,EAAE,MAAM,CAAC,QAAS,EAAE,IAAI,CAAC,MAAM,EAAE;QACnF,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,QAAQ,EAAE,MAAM,CAAC,UAAU;KAC5B,CAAC,CAAC;IAEH,8GAA8G;IAC9G,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,WAAY,EAAE,SAAS,CAAC,CAAC;IAEhF,6CAA6C;IAC7C,IAAI,UAAU,GAAe,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,GAAe,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACzB,UAAU,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,IAAI,CAAC,OAAO,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC;YACvC,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;IAED,uBAAuB;IACvB,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,YAAY;QAC3B,mBAAmB,EAAE,IAAI,gBAAgB,CAAC;YACxC,UAAU,EAAE,IAAI,CAAC,KAAK;YACtB,SAAS,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1C,QAAQ,EAAE,QAAQ;YAClB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAS,CAAQ;YAC3C,6BAA6B,EAAE,UAAU,CAAC,MAAM;YAChD,6BAA6B,EAAE,UAAU,CAAC,MAAM;YAChD,aAAa,EAAE,MAAM,CAAC,WAAW,KAAK,cAAc;SACrD,CAAC;QACF,sBAAsB,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;QAChF,oBAAoB,EAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;KACnF,CAAC,CAAC;IAEH,qEAAqE;IACrE,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC;IACnF,OAAO,EAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,iBAAiB,CAC9B,MAAoB,EACpB,MAAuB,EACvB,MAAc,EACd,IAA2B;IAM3B,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;IACnD,yBAAyB;IACzB,kCAAkC;IAClC,IAAI,OAAmB,CAAC;IACxB,yCAAyC;IACzC,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,yCAAyC;IACzC,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,CAAC;QACC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa;YAC/B,CAAC,CAAC,MAAM,gBAAgB,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC;YACvD,CAAC,CAAC,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACvC,2BAA2B;QAC3B,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;QACtB,uBAAuB,IAAI,MAAM,CAAC,MAAM,CAAC,sBAAsB,GAAG,MAAM,CAAC,UAAU,CAAC;QACpF,qBAAqB,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAM,CAAC,UAAU,CAAC;IAClF,CAAC;IAED,6CAA6C;IAC7C,yHAAyH;IAEzH,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;QAClC,cAAc,EAAE,MAAM,CAAC,IAAI;QAC3B,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7B,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC;QACnC,SAAS,EAAE,EAAE;QACb,uBAAuB,EAAE,KAAK,CAAC,uBAAuB,CAAC,EAAE,sBAAsB;QAC/E,qBAAqB,EAAE,KAAK,CAAC,qBAAqB,CAAC;QACnD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,aAAc,CAAC;QACjC,KAAK,EAAE,gBAAgB,CAAC,MAAM,CAAC,WAAY,CAAC;KAC7C,CAAC,CAAC;IAEH,oBAAoB;IACpB,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC;IAC1D,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAS,CAAC,CAAC,CAAC;IAEpD,2CAA2C;IAC3C,MAAM,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IACnD,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACrE,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAC3B,MAAqB,EACrB,IAAqB,EACrB,IAA2B;IAK3B,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;QAC5B,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC9B,OAAO,EAAE,EAAE;QACX,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC;KAC1B,CAAC,CAAC;IAEH,IAAI,IAAI,GAAe,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,SAAS,CAAC,kCAAkC;QAC9C,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3E,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC;YAC7B,WAAW,EAAE,KAAK,CAAC,UAAU,CAAC,cAAc,CAAC;YAC7C,SAAS,EAAE,UAAU,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,eAAe,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5F,IAAI,GAAG,iBAAiB,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,EAAC,IAAI,EAAE,QAAQ,EAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,MAAqB,EACrB,QAAgB,EAChB,SAAqB,EACrB,YAAoC;IAEpC,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC;QAChC,OAAO,EAAE,eAAe;QACxB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC;QACzB,UAAU,EAAE,SAAS;QACrB,MAAM,EAAE,EAAE;QACV,kBAAkB,EAAE,EAAE;KACvB,CAAC,CAAC;IAEH,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC;YACtB,GAAG;YACH,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC;SACzB,CAAC,CAAC;QACH,QAAQ,CAAC,kBAAkB,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,CAAC;QACC,MAAM,UAAU,GAAG,IAAI,aAAa,CAAC;YACnC,IAAI,EAAE,MAAM;YACZ,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM;SAChD,CAAC,CAAC;QACH,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,IAAI,aAAa,CAAC;YACnC,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,eAAe,EAAE,IAAW;SAC7B,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,aAAc,CAAS,CAAC;QACvD,CAAC;QAED,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,UAAU,CAAC,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC,YAAY,CAAkB,CAAC;QACjF,CAAC;QAED,UAAU,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;QAE1C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,eAAe,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEjE,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACnC,aAAa,CAAC,aAAa,EAAE,eAAe,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;IAC7E,aAAa,CAAC,GAAG,CAAC,mBAAmB,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,8EAA8E;AAC9E,SAAS,KAAK,CAAC,KAAa;IAC1B,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -43,7 +43,7 @@ export class ColumnIndex {
|
|
|
43
43
|
throw new thrift.Thrift.TProtocolException(thrift.Thrift.TProtocolExceptionType.UNKNOWN, 'Required field[boundary_order] is unset!');
|
|
44
44
|
}
|
|
45
45
|
if (args != null && args.null_counts != null) {
|
|
46
|
-
this.null_counts = args.null_counts.map(
|
|
46
|
+
this.null_counts = args.null_counts.map(c => new Int64(+c));
|
|
47
47
|
}
|
|
48
48
|
}
|
|
49
49
|
write(output) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ColumnIndex.js","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/ColumnIndex.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,oBAAoB;AACpB,oBAAoB;AACpB;;;GAGG;AACH,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AASjC,MAAM,OAAO,WAAW;IACf,UAAU,CAAiB;IAC3B,UAAU,CAAgB;IAC1B,UAAU,CAAgB;IAC1B,cAAc,CAA8B;IAC5C,WAAW,CAAgB;IAClC,YAAY,IAAsB;QAChC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,sCAAsC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,sCAAsC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,sCAAsC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,0CAA0C,CAC3C,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"ColumnIndex.js","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/ColumnIndex.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,oBAAoB;AACpB,oBAAoB;AACpB;;;GAGG;AACH,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AASjC,MAAM,OAAO,WAAW;IACf,UAAU,CAAiB;IAC3B,UAAU,CAAgB;IAC1B,UAAU,CAAgB;IAC1B,cAAc,CAA8B;IAC5C,WAAW,CAAgB;IAClC,YAAY,IAAsB;QAChC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,sCAAsC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,sCAAsC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,sCAAsC,CACvC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YAChD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,0CAA0C,CAC3C,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;YAC7C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IACM,KAAK,CAAC,MAAwB;QACnC,MAAM,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACvE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,OAAgB,EAAQ,EAAE;gBACjD,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC5B,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACzE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,OAAe,EAAQ,EAAE;gBAChD,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;YAC5B,MAAM,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YACzE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,OAAe,EAAQ,EAAE;gBAChD,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC;YAChC,MAAM,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACrC,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;YAC7B,MAAM,CAAC,eAAe,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACvE,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAc,EAAQ,EAAE;gBAChD,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IACM,MAAM,CAAC,IAAI,CAAC,KAAuB;QACxC,KAAK,CAAC,eAAe,EAAE,CAAC;QACxB,IAAI,KAAK,GAAQ,EAAE,CAAC;QACpB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAkB,KAAK,CAAC,cAAc,EAAE,CAAC;YAClD,MAAM,SAAS,GAAuB,GAAG,CAAC,KAAK,CAAC;YAChD,MAAM,OAAO,GAAW,GAAG,CAAC,GAAG,CAAC;YAChC,IAAI,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC1C,MAAM;YACR,CAAC;YACD,QAAQ,OAAO,EAAE,CAAC;gBAChB,KAAK,CAAC;oBACJ,IAAI,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;wBAC1C,MAAM,OAAO,GAAmB,IAAI,KAAK,EAAW,CAAC;wBACrD,MAAM,UAAU,GAAiB,KAAK,CAAC,aAAa,EAAE,CAAC;wBACvD,MAAM,MAAM,GAAW,UAAU,CAAC,IAAI,CAAC;wBACvC,KAAK,IAAI,GAAG,GAAW,CAAC,EAAE,GAAG,GAAG,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;4BAC9C,MAAM,OAAO,GAAY,KAAK,CAAC,QAAQ,EAAE,CAAC;4BAC1C,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACxB,CAAC;wBACD,KAAK,CAAC,WAAW,EAAE,CAAC;wBACpB,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC;oBAC7B,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;oBACD,MAAM;gBACR,KAAK,CAAC;oBACJ,IAAI,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;wBAC1C,MAAM,OAAO,GAAkB,IAAI,KAAK,EAAU,CAAC;wBACnD,MAAM,UAAU,GAAiB,KAAK,CAAC,aAAa,EAAE,CAAC;wBACvD,MAAM,MAAM,GAAW,UAAU,CAAC,IAAI,CAAC;wBACvC,KAAK,IAAI,GAAG,GAAW,CAAC,EAAE,GAAG,GAAG,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;4BAC9C,MAAM,OAAO,GAAW,KAAK,CAAC,UAAU,EAAE,CAAC;4BAC3C,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACxB,CAAC;wBACD,KAAK,CAAC,WAAW,EAAE,CAAC;wBACpB,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC;oBAC7B,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;oBACD,MAAM;gBACR,KAAK,CAAC;oBACJ,IAAI,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;wBAC1C,MAAM,OAAO,GAAkB,IAAI,KAAK,EAAU,CAAC;wBACnD,MAAM,UAAU,GAAiB,KAAK,CAAC,aAAa,EAAE,CAAC;wBACvD,MAAM,MAAM,GAAW,UAAU,CAAC,IAAI,CAAC;wBACvC,KAAK,IAAI,GAAG,GAAW,CAAC,EAAE,GAAG,GAAG,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;4BAC9C,MAAM,QAAQ,GAAW,KAAK,CAAC,UAAU,EAAE,CAAC;4BAC5C,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBACzB,CAAC;wBACD,KAAK,CAAC,WAAW,EAAE,CAAC;wBACpB,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC;oBAC7B,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;oBACD,MAAM;gBACR,KAAK,CAAC;oBACJ,IAAI,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;wBACzC,MAAM,QAAQ,GAAgC,KAAK,CAAC,OAAO,EAAE,CAAC;wBAC9D,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC;oBAClC,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;oBACD,MAAM;gBACR,KAAK,CAAC;oBACJ,IAAI,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;wBAC1C,MAAM,QAAQ,GAAiB,IAAI,KAAK,EAAS,CAAC;wBAClD,MAAM,UAAU,GAAiB,KAAK,CAAC,aAAa,EAAE,CAAC;wBACvD,MAAM,MAAM,GAAW,UAAU,CAAC,IAAI,CAAC;wBACvC,KAAK,IAAI,GAAG,GAAW,CAAC,EAAE,GAAG,GAAG,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;4BAC9C,MAAM,QAAQ,GAAU,KAAK,CAAC,OAAO,EAAE,CAAC;4BACxC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBAC1B,CAAC;wBACD,KAAK,CAAC,WAAW,EAAE,CAAC;wBACpB,KAAK,CAAC,WAAW,GAAG,QAAQ,CAAC;oBAC/B,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACxB,CAAC;oBACD,MAAM;gBACR,OAAO,CAAC,CAAC,CAAC;oBACR,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,aAAa,EAAE,CAAC;QACtB,IACE,KAAK,CAAC,UAAU,KAAK,SAAS;YAC9B,KAAK,CAAC,UAAU,KAAK,SAAS;YAC9B,KAAK,CAAC,UAAU,KAAK,SAAS;YAC9B,KAAK,CAAC,cAAc,KAAK,SAAS,EAClC,CAAC;YACD,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,kBAAkB,CACxC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,OAAO,EAC5C,uCAAuC,CACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/index.ts"],"names":[],"mappings":"AAYA,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/index.ts"],"names":[],"mappings":"AAYA,OAAO,EAAC,kBAAkB,EAAE,gBAAgB,EAAE,gBAAgB,EAAC,MAAM,QAAQ,CAAC;AAE9E,0BAAuB;AACvB,mCAAgC;AAChC,yCAAsC;AACtC,8BAA2B;AAC3B,sCAAmC;AACnC,8BAA2B;AAC3B,mCAAgC;AAChC,gCAA6B;AAC7B,gCAA6B;AAC7B,8BAA2B;AAC3B,6BAA0B;AAC1B,8BAA2B;AAC3B,8BAA2B;AAC3B,8BAA2B;AAC3B,8BAA2B;AAC3B,iCAA8B;AAC9B,kCAA+B;AAC/B,kCAA+B;AAC/B,mCAAgC;AAChC,8BAA2B;AAC3B,6BAA0B;AAC1B,8BAA2B;AAC3B,8BAA2B;AAC3B,mCAAgC;AAChC,oCAAiC;AACjC,qCAAkC;AAClC,0CAAuC;AACvC,sCAAmC;AACnC,gCAA6B;AAC7B,8BAA2B;AAC3B,mCAAgC;AAChC,uCAAoC;AACpC,oCAAiC;AACjC,iCAA8B;AAC9B,8BAA2B;AAC3B,sCAAmC;AACnC,kCAA+B;AAC/B,iCAA8B;AAC9B,iCAA8B;AAC9B,kCAA+B;AAC/B,8BAA2B;AAC3B,iCAA8B;AAC9B,iCAA8B"}
|
|
@@ -9,7 +9,6 @@
|
|
|
9
9
|
* Autogenerated by @creditkarma/thrift-typescript v3.7.2
|
|
10
10
|
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
11
11
|
*/
|
|
12
|
-
export { Buffer } from "../../polyfills/buffer/install-buffer-polyfill.js";
|
|
13
12
|
export { TBufferedTransport, TCompactProtocol, TFramedTransport } from 'thrift';
|
|
14
13
|
export * from "./Type.js";
|
|
15
14
|
export * from "./ConvertedType.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,oBAAoB;AACpB,oBAAoB;AACpB;;;GAGG;AACH,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAEpE,oBAAoB;AACpB,oBAAoB;AACpB;;;GAGG;AACH,OAAO,EAAC,kBAAkB,EAAE,gBAAgB,EAAE,gBAAgB,EAAC,MAAM,QAAQ,CAAC;AAE9E,0BAAuB;AACvB,mCAAgC;AAChC,yCAAsC;AACtC,8BAA2B;AAC3B,sCAAmC;AACnC,8BAA2B;AAC3B,mCAAgC;AAChC,gCAA6B;AAC7B,gCAA6B;AAC7B,8BAA2B;AAC3B,6BAA0B;AAC1B,8BAA2B;AAC3B,8BAA2B;AAC3B,8BAA2B;AAC3B,8BAA2B;AAC3B,iCAA8B;AAC9B,kCAA+B;AAC/B,kCAA+B;AAC/B,mCAAgC;AAChC,8BAA2B;AAC3B,6BAA0B;AAC1B,8BAA2B;AAC3B,8BAA2B;AAC3B,mCAAgC;AAChC,oCAAiC;AACjC,qCAAkC;AAClC,0CAAuC;AACvC,sCAAmC;AACnC,gCAA6B;AAC7B,8BAA2B;AAC3B,mCAAgC;AAChC,uCAAoC;AACpC,oCAAiC;AACjC,iCAA8B;AAC9B,8BAA2B;AAC3B,sCAAmC;AACnC,kCAA+B;AAC/B,iCAA8B;AAC9B,iCAA8B;AAC9B,kCAA+B;AAC/B,8BAA2B;AAC3B,iCAA8B;AAC9B,iCAA8B"}
|
|
@@ -8,7 +8,7 @@ import { SchemaElement } from "../parquet-thrift/index.js";
|
|
|
8
8
|
* @param compression - compression type
|
|
9
9
|
* @returns parquet data page data
|
|
10
10
|
*/
|
|
11
|
-
export declare function decodeDataPages(buffer:
|
|
11
|
+
export declare function decodeDataPages(buffer: Uint8Array, context: ParquetReaderContext): Promise<ParquetColumnChunk>;
|
|
12
12
|
/**
|
|
13
13
|
* Decode parquet page based on page type
|
|
14
14
|
* @param cursor
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":"AAMA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAyD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}
|
|
@@ -45,7 +45,7 @@ export async function decodeDataPages(buffer, context) {
|
|
|
45
45
|
// of 'PLAIN' encoding because all values are already in place
|
|
46
46
|
if (dictionary.length && valueEncoding !== 'PLAIN') {
|
|
47
47
|
// eslint-disable-next-line no-loop-func
|
|
48
|
-
page.values = page.values.map(
|
|
48
|
+
page.values = page.values.map(value => dictionary[value]);
|
|
49
49
|
}
|
|
50
50
|
for (let index = 0; index < page.rlevels.length; index++) {
|
|
51
51
|
data.rlevels.push(page.rlevels[index]);
|
|
@@ -131,7 +131,7 @@ export function decodeSchema(schemaElements, offset, len) {
|
|
|
131
131
|
else {
|
|
132
132
|
const type = getThriftEnum(Type, schemaElement.type);
|
|
133
133
|
let logicalType = type;
|
|
134
|
-
if (schemaElement.converted_type) {
|
|
134
|
+
if (schemaElement.converted_type !== undefined && schemaElement.converted_type !== null) {
|
|
135
135
|
logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);
|
|
136
136
|
}
|
|
137
137
|
switch (logicalType) {
|
|
@@ -307,7 +307,7 @@ async function decodeDictionaryPage(cursor, pageHeader, context) {
|
|
|
307
307
|
};
|
|
308
308
|
cursor.offset = cursorEnd;
|
|
309
309
|
if (context.compression !== 'UNCOMPRESSED') {
|
|
310
|
-
const valuesBuf = await decompress(context.compression, dictCursor.buffer.
|
|
310
|
+
const valuesBuf = await decompress(context.compression, dictCursor.buffer.subarray(dictCursor.offset), pageHeader.uncompressed_page_size);
|
|
311
311
|
dictCursor = {
|
|
312
312
|
buffer: valuesBuf,
|
|
313
313
|
offset: 0,
|
|
@@ -319,24 +319,6 @@ async function decodeDictionaryPage(cursor, pageHeader, context) {
|
|
|
319
319
|
const decodedDictionaryValues = decodeValues(context.column.primitiveType, context.column.encoding, dictCursor, numValues,
|
|
320
320
|
// TODO - this looks wrong?
|
|
321
321
|
context);
|
|
322
|
-
|
|
323
|
-
let values;
|
|
324
|
-
if (context?.preserveBinary) {
|
|
325
|
-
values = decodedDictionaryValues.map((d) => preserveBinary(d));
|
|
326
|
-
}
|
|
327
|
-
else {
|
|
328
|
-
values = decodedDictionaryValues.map((d) => d.toString());
|
|
329
|
-
}
|
|
330
|
-
return values;
|
|
331
|
-
}
|
|
332
|
-
function preserveBinary(d) {
|
|
333
|
-
if (ArrayBuffer.isView(d)) {
|
|
334
|
-
return d;
|
|
335
|
-
}
|
|
336
|
-
// Convert to ArrayBuffer
|
|
337
|
-
if (Buffer.isBuffer(d)) {
|
|
338
|
-
return new Uint8Array(d.buffer, d.byteOffset, d.byteLength).slice().buffer;
|
|
339
|
-
}
|
|
340
|
-
return d.toString();
|
|
322
|
+
return decodedDictionaryValues;
|
|
341
323
|
}
|
|
342
324
|
//# sourceMappingURL=decoders.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"decoders.js","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAWpE,OAAO,EAAoC,cAAc,EAAC,2BAAwB;AAClF,OAAO,EACL,aAAa,EACb,QAAQ,EACR,mBAAmB,EAEnB,QAAQ,EAER,IAAI,EACL,mCAAgC;AACjC,OAAO,EAAC,UAAU,EAAC,0BAAuB;AAC1C,OAAO,EAAC,kBAAkB,EAAE,sBAAsB,EAAC,+BAA4B;AAC/E,OAAO,EAAC,gBAAgB,EAAE,aAAa,EAAE,WAAW,EAAC,+BAA4B;AAEjF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,OAA6B;IAE7B,MAAM,MAAM,GAAiB;QAC3B,MAAM;QACN,MAAM,EAAE,CAAC;QACT,IAAI,EAAE,MAAM,CAAC,MAAM;KACpB,CAAC;IAEF,MAAM,IAAI,GAAuB;QAC/B,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;QACV,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,CAAC;KACT,CAAC;IAEF,IAAI,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAE1C;IACE,mCAAmC;IACnC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI;QAC3B,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EACvE,CAAC;QACD,wEAAwE;QACxE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,uCAAuC;YACvC,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAG,aAAa,CACjC,QAAQ,EACR,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE,QAAS,CAC5B,CAAC;QAClB,yEAAyE;QACzE,8DAA8D;QAC9D,IAAI,UAAU,CAAC,MAAM,IAAI,aAAa,KAAK,OAAO,EAAE,CAAC;YACnD,wCAAwC;YACxC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAoB,EACpB,OAA6B;IAE7B,IAAI,IAAI,CAAC;IAET,MAAM,EAAC,UAAU,EAAE,MAAM,EAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC;IAExB,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAE1D,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,IAAI,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YACzD,MAAM;QACR,KAAK,cAAc;YACjB,IAAI,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,iBAAiB;YACpB,IAAI,GAAG;gBACL,UAAU,EAAE,MAAM,oBAAoB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC;gBACnE,UAAU;aACX,CAAC;YACF,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,YAAY,CAC1B,cAA+B,EAC/B,MAAc,EACd,GAAW;IAMX,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,IAAI,IAAI,GAAG,MAAM,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAE3C,MAAM,cAAc,GAClB,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,eAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAEzF,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,QAAQ,cAAc,EAAE,CAAC;YACvB,KAAK,UAAU;gBACb,MAAM;YACR,KAAK,UAAU;gBACb,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR,KAAK,UAAU;gBACb,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,aAAa,CAAC,YAAa,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,YAAY,CAAC,cAAc,EAAE,IAAI,GAAG,CAAC,EAAE,aAAa,CAAC,YAAa,CAAC,CAAC;YAChF,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YAChB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG;gBAC3B,mBAAmB;gBACnB,QAAQ;gBACR,QAAQ;gBACR,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,aAAa,CAAC,IAAK,CAAC,CAAC;YACtD,IAAI,WAAW,GAAG,IAAI,CAAC;YAEvB,IAAI,aAAa,CAAC,cAAc,EAAE,CAAC;gBACjC,WAAW,GAAG,aAAa,CAAC,aAAa,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;YAC3E,CAAC;YAED,QAAQ,WAAW,EAAE,CAAC;gBACpB,KAAK,SAAS;oBACZ,WAAW,GAAG,GAAG,WAAW,IAAI,IAAI,EAAiB,CAAC;oBACtD,MAAM;gBACR,QAAQ;YACV,CAAC;YAED,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG;gBAC3B,IAAI,EAAE,WAA0B;gBAChC,UAAU,EAAE,aAAa,CAAC,WAAW;gBACrC,SAAS,EAAE,aAAa,CAAC,SAAS;gBAClC,KAAK,EAAE,aAAa,CAAC,KAAK;gBAC1B,QAAQ;gBACR,QAAQ;aACT,CAAC;YACF,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IACD,OAAO,EAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,IAAmB,EACnB,QAAsB,EACtB,MAAoB,EACpB,KAAa,EACb,IAAyB;IAEzB,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC1E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,cAAc,CAC3B,MAAoB,EACpB,MAAkB,EAClB,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,EAAE,UAAU,CAAC;IAEvD,qBAAqB;IACrB,IAAI,UAAU,GAAG,MAAM,CAAC;IAExB,IAAI,OAAO,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC,OAAO,CAAC,WAAW,EACnB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAC7C,MAAM,CAAC,sBAAsB,CAC9B,CAAC;QACF,UAAU,GAAG;YACX,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB,CAAC;QACF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,4BAA4B;IAC5B,MAAM,cAAc,GAAG,aAAa,CAClC,QAAQ,EACR,MAAM,CAAC,gBAAgB,EAAE,yBAA0B,CACpC,CAAC;IAClB,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IAEpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAW,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,KAAK;YACtB,sBAAsB;SACvB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4BAA4B;IAC5B,MAAM,cAAc,GAAG,aAAa,CAClC,QAAQ,EACR,MAAM,CAAC,gBAAgB,EAAE,yBAA0B,CACpC,CAAC;IAClB,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAW,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,KAAK;YACtB,sBAAsB;SACvB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtC,iBAAiB,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,gBAAgB,EAAE,QAAS,CAAiB,CAAC;IAClG,MAAM,aAAa,GAAwB;QACzC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;QACrC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,YAAY,CACzB,OAAO,CAAC,MAAM,CAAC,aAAc,EAC7B,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,CACd,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,OAAO;QAChB,MAAM;QACN,KAAK,EAAE,UAAW;QAClB,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB,CAC7B,MAAoB,EACpB,MAAkB,EAClB,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAE9D,MAAM,UAAU,GAAG,MAAM,CAAC,mBAAmB,EAAE,UAAU,CAAC;IAC1D,aAAa;IACb,MAAM,iBAAiB,GAAG,UAAU,GAAG,MAAM,CAAC,mBAAmB,EAAE,SAAS,CAAC;IAC7E,MAAM,aAAa,GAAG,aAAa,CACjC,QAAQ,EACR,MAAM,CAAC,mBAAmB,EAAE,QAAS,CACtB,CAAC;IAElB,4BAA4B;IAC5B,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,EAAE,UAAW,EAAE;YACtF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4BAA4B;IAC5B,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,EAAE,UAAW,EAAE;YACtF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,iBAAiB;IACjB,IAAI,eAAe,GAAG,MAAM,CAAC;IAE7B,IAAI,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC,OAAO,CAAC,WAAW,EACnB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAC7C,MAAM,CAAC,sBAAsB,CAC9B,CAAC;QAEF,eAAe,GAAG;YAChB,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB,CAAC;QAEF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;QACrC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,YAAY,CACzB,OAAO,CAAC,MAAM,CAAC,aAAc,EAC7B,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,aAAa,CACd,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,OAAO;QAChB,MAAM;QACN,KAAK,EAAE,UAAW;QAClB,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAoB,EACpB,UAAsB,EACtB,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAElE,IAAI,UAAU,GAAiB;QAC7B,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC;QACrD,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM;KAChC,CAAC;IAEF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAE1B,IAAI,OAAO,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC,OAAO,CAAC,WAAW,EACnB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,EACrD,UAAU,CAAC,sBAAsB,CAClC,CAAC;QAEF,UAAU,GAAG;YACX,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB,CAAC;QAEF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,EAAE,sBAAsB,EAAE,UAAU,IAAI,CAAC,CAAC;IAEtE,MAAM,uBAAuB,GAAG,YAAY,CAC1C,OAAO,CAAC,MAAM,CAAC,aAAc,EAC7B,OAAO,CAAC,MAAM,CAAC,QAAS,EACxB,UAAU,EACV,SAAS;IACT,2BAA2B;IAC3B,OAA8B,CAC/B,CAAC;IAEF,gCAAgC;IAChC,IAAI,MAAa,CAAC;IAClB,IAAI,OAAO,EAAE,cAAc,EAAE,CAAC;QAC5B,MAAM,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,CAAM;IAC5B,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,CAAC;IACX,CAAC;IACD,yBAAyB;IACzB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC;IAC7E,CAAC;IACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;AACtB,CAAC"}
|
|
1
|
+
{"version":3,"file":"decoders.js","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAWpE,OAAO,EAAoC,cAAc,EAAC,2BAAwB;AAClF,OAAO,EACL,aAAa,EACb,QAAQ,EACR,mBAAmB,EAEnB,QAAQ,EAER,IAAI,EACL,mCAAgC;AACjC,OAAO,EAAC,UAAU,EAAC,0BAAuB;AAC1C,OAAO,EAAC,kBAAkB,EAAE,sBAAsB,EAAC,+BAA4B;AAC/E,OAAO,EAAC,gBAAgB,EAAE,aAAa,EAAE,WAAW,EAAC,+BAA4B;AAEjF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAkB,EAClB,OAA6B;IAE7B,MAAM,MAAM,GAAiB;QAC3B,MAAM;QACN,MAAM,EAAE,CAAC;QACT,IAAI,EAAE,MAAM,CAAC,MAAM;KACpB,CAAC;IAEF,MAAM,IAAI,GAAuB;QAC/B,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;QACV,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,CAAC;KACT,CAAC;IAEF,IAAI,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAE1C;IACE,mCAAmC;IACnC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI;QAC3B,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EACvE,CAAC;QACD,wEAAwE;QACxE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE/C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;YAC7B,uCAAuC;YACvC,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAG,aAAa,CACjC,QAAQ,EACR,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE,QAAS,CAC5B,CAAC;QAClB,yEAAyE;QACzE,8DAA8D;QAC9D,IAAI,UAAU,CAAC,MAAM,IAAI,aAAa,KAAK,OAAO,EAAE,CAAC;YACnD,wCAAwC;YACxC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5D,CAAC;QAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YACvC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAoB,EACpB,OAA6B;IAE7B,IAAI,IAAI,CAAC;IAET,MAAM,EAAC,UAAU,EAAE,MAAM,EAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC;IAExB,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAE1D,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,WAAW;YACd,IAAI,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YACzD,MAAM;QACR,KAAK,cAAc;YACjB,IAAI,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YAC3D,MAAM;QACR,KAAK,iBAAiB;YACpB,IAAI,GAAG;gBACL,UAAU,EAAE,MAAM,oBAAoB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC;gBACnE,UAAU;aACX,CAAC;YACF,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,YAAY,CAC1B,cAA+B,EAC/B,MAAc,EACd,GAAW;IAMX,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,IAAI,IAAI,GAAG,MAAM,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAE3C,MAAM,cAAc,GAClB,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,eAAgB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAEzF,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,QAAQ,cAAc,EAAE,CAAC;YACvB,KAAK,UAAU;gBACb,MAAM;YACR,KAAK,UAAU;gBACb,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR,KAAK,UAAU;gBACb,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,aAAa,CAAC,YAAa,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,YAAY,CAAC,cAAc,EAAE,IAAI,GAAG,CAAC,EAAE,aAAa,CAAC,YAAa,CAAC,CAAC;YAChF,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YAChB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG;gBAC3B,mBAAmB;gBACnB,QAAQ;gBACR,QAAQ;gBACR,MAAM,EAAE,GAAG,CAAC,MAAM;aACnB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,aAAa,CAAC,IAAK,CAAC,CAAC;YACtD,IAAI,WAAW,GAAG,IAAI,CAAC;YAEvB,IAAI,aAAa,CAAC,cAAc,KAAK,SAAS,IAAI,aAAa,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;gBACxF,WAAW,GAAG,aAAa,CAAC,aAAa,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;YAC3E,CAAC;YAED,QAAQ,WAAW,EAAE,CAAC;gBACpB,KAAK,SAAS;oBACZ,WAAW,GAAG,GAAG,WAAW,IAAI,IAAI,EAAiB,CAAC;oBACtD,MAAM;gBACR,QAAQ;YACV,CAAC;YAED,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG;gBAC3B,IAAI,EAAE,WAA0B;gBAChC,UAAU,EAAE,aAAa,CAAC,WAAW;gBACrC,SAAS,EAAE,aAAa,CAAC,SAAS;gBAClC,KAAK,EAAE,aAAa,CAAC,KAAK;gBAC1B,QAAQ;gBACR,QAAQ;aACT,CAAC;YACF,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IACD,OAAO,EAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,IAAmB,EACnB,QAAsB,EACtB,MAAoB,EACpB,KAAa,EACb,IAAyB;IAEzB,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC1E,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,cAAc,CAC3B,MAAoB,EACpB,MAAkB,EAClB,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,EAAE,UAAU,CAAC;IAEvD,qBAAqB;IACrB,IAAI,UAAU,GAAG,MAAM,CAAC;IAExB,IAAI,OAAO,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC,OAAO,CAAC,WAAW,EACnB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAC7C,MAAM,CAAC,sBAAsB,CAC9B,CAAC;QACF,UAAU,GAAG;YACX,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB,CAAC;QACF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,4BAA4B;IAC5B,MAAM,cAAc,GAAG,aAAa,CAClC,QAAQ,EACR,MAAM,CAAC,gBAAgB,EAAE,yBAA0B,CACpC,CAAC;IAClB,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IAEpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAW,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,KAAK;YACtB,sBAAsB;SACvB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4BAA4B;IAC5B,MAAM,cAAc,GAAG,aAAa,CAClC,QAAQ,EACR,MAAM,CAAC,gBAAgB,EAAE,yBAA0B,CACpC,CAAC;IAClB,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,cAAc,EAAE,UAAU,EAAE,UAAW,EAAE;YAClF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,KAAK;YACtB,sBAAsB;SACvB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,IAAI,KAAK,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtC,iBAAiB,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,gBAAgB,EAAE,QAAS,CAAiB,CAAC;IAClG,MAAM,aAAa,GAAwB;QACzC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;QACrC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,YAAY,CACzB,OAAO,CAAC,MAAM,CAAC,aAAc,EAC7B,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,CACd,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,OAAO;QAChB,MAAM;QACN,KAAK,EAAE,UAAW;QAClB,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB,CAC7B,MAAoB,EACpB,MAAkB,EAClB,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAE9D,MAAM,UAAU,GAAG,MAAM,CAAC,mBAAmB,EAAE,UAAU,CAAC;IAC1D,aAAa;IACb,MAAM,iBAAiB,GAAG,UAAU,GAAG,MAAM,CAAC,mBAAmB,EAAE,SAAS,CAAC;IAC7E,MAAM,aAAa,GAAG,aAAa,CACjC,QAAQ,EACR,MAAM,CAAC,mBAAmB,EAAE,QAAS,CACtB,CAAC;IAElB,4BAA4B;IAC5B,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,EAAE,UAAW,EAAE;YACtF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,4BAA4B;IAC5B,gDAAgD;IAChD,IAAI,OAAO,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,YAAY,CAAC,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,EAAE,UAAW,EAAE;YACtF,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC;YAC/C,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,iBAAiB;IACjB,IAAI,eAAe,GAAG,MAAM,CAAC;IAE7B,IAAI,MAAM,CAAC,mBAAmB,EAAE,aAAa,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC,OAAO,CAAC,WAAW,EACnB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAC7C,MAAM,CAAC,sBAAsB,CAC9B,CAAC;QAEF,eAAe,GAAG;YAChB,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB,CAAC;QAEF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;QACrC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;KACpC,CAAC;IAEF,MAAM,MAAM,GAAG,YAAY,CACzB,OAAO,CAAC,MAAM,CAAC,aAAc,EAC7B,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,aAAa,CACd,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,OAAO;QAChB,OAAO,EAAE,OAAO;QAChB,MAAM;QACN,KAAK,EAAE,UAAW;QAClB,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAoB,EACpB,UAAsB,EACtB,OAA6B;IAE7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAElE,IAAI,UAAU,GAAiB;QAC7B,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC;QACrD,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM;KAChC,CAAC;IAEF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAE1B,IAAI,OAAO,CAAC,WAAW,KAAK,cAAc,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,MAAM,UAAU,CAChC,OAAO,CAAC,WAAW,EACnB,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAC7C,UAAU,CAAC,sBAAsB,CAClC,CAAC;QAEF,UAAU,GAAG;YACX,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,SAAS,CAAC,MAAM;SACvB,CAAC;QAEF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,EAAE,sBAAsB,EAAE,UAAU,IAAI,CAAC,CAAC;IAEtE,MAAM,uBAAuB,GAAG,YAAY,CAC1C,OAAO,CAAC,MAAM,CAAC,aAAc,EAC7B,OAAO,CAAC,MAAM,CAAC,QAAS,EACxB,UAAU,EACV,SAAS;IACT,2BAA2B;IAC3B,OAA8B,CAC/B,CAAC;IAEF,OAAO,uBAAuB,CAAC;AACjC,CAAC"}
|
|
@@ -55,6 +55,6 @@ export declare class ParquetReader {
|
|
|
55
55
|
* @param pagesOffset
|
|
56
56
|
* @returns
|
|
57
57
|
*/
|
|
58
|
-
getDictionary(dictionaryPageOffset: number, context: ParquetReaderContext, pagesOffset: number): Promise<
|
|
58
|
+
getDictionary(dictionaryPageOffset: number, context: ParquetReaderContext, pagesOffset: number): Promise<any[]>;
|
|
59
59
|
}
|
|
60
60
|
//# sourceMappingURL=parquet-reader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,mCAAgC;AACpG,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EACrB,6BAA0B;
|
|
1
|
+
{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,mCAAgC;AACpG,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EACrB,6BAA0B;AAI3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;CACpC,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAI/C;IAEF,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;gBAElC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAmB/C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC;IAQzC;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAU9C,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAyBzC,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC;IAgB3B;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAuDhG;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,oBAAoB,EAC7B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,GAAG,EAAE,CAAC;CAwBlB"}
|
|
@@ -9,6 +9,7 @@ import { materializeRows } from "../schema/shred.js";
|
|
|
9
9
|
import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from "../../lib/constants.js";
|
|
10
10
|
import { CompressionCodec, Type } from "../parquet-thrift/index.js";
|
|
11
11
|
import { decodeFileMetadata, getThriftEnum, fieldIndexOf } from "../utils/read-utils.js";
|
|
12
|
+
import { decodeString, readUInt32LE, toUint8Array } from "../utils/binary-utils.js";
|
|
12
13
|
/**
|
|
13
14
|
* The parquet envelope reader allows direct, unbuffered access to the individual
|
|
14
15
|
* sections of the parquet file, namely the header, footer and the row groups.
|
|
@@ -52,7 +53,7 @@ export class ParquetReader {
|
|
|
52
53
|
/** Iterate over the raw row groups */
|
|
53
54
|
async *rowGroupIterator(props) {
|
|
54
55
|
// Ensure strings are nested in arrays
|
|
55
|
-
const columnList = (props?.columnList || []).map(
|
|
56
|
+
const columnList = (props?.columnList || []).map(x => (Array.isArray(x) ? x : [x]));
|
|
56
57
|
const metadata = await this.getFileMetadata();
|
|
57
58
|
const schema = await this.getSchema();
|
|
58
59
|
const rowGroupCount = metadata?.row_groups.length || 0;
|
|
@@ -95,8 +96,7 @@ export class ParquetReader {
|
|
|
95
96
|
/** Metadata is stored in the footer */
|
|
96
97
|
async readHeader() {
|
|
97
98
|
const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length);
|
|
98
|
-
const
|
|
99
|
-
const magic = buffer.toString();
|
|
99
|
+
const magic = decodeString(toUint8Array(arrayBuffer));
|
|
100
100
|
switch (magic) {
|
|
101
101
|
case PARQUET_MAGIC:
|
|
102
102
|
break;
|
|
@@ -110,18 +110,18 @@ export class ParquetReader {
|
|
|
110
110
|
async readFooter() {
|
|
111
111
|
const trailerLen = PARQUET_MAGIC.length + 4;
|
|
112
112
|
const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);
|
|
113
|
-
const
|
|
114
|
-
const magic =
|
|
113
|
+
const trailer = toUint8Array(arrayBuffer);
|
|
114
|
+
const magic = decodeString(trailer, 4);
|
|
115
115
|
if (magic !== PARQUET_MAGIC) {
|
|
116
116
|
throw new Error(`Not a valid parquet file (magic="${magic})`);
|
|
117
117
|
}
|
|
118
|
-
const metadataSize =
|
|
118
|
+
const metadataSize = readUInt32LE(trailer, 0);
|
|
119
119
|
const metadataOffset = this.file.size - metadataSize - trailerLen;
|
|
120
120
|
if (metadataOffset < PARQUET_MAGIC.length) {
|
|
121
121
|
throw new Error(`Invalid metadata size ${metadataOffset}`);
|
|
122
122
|
}
|
|
123
123
|
const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);
|
|
124
|
-
const metadataBuf =
|
|
124
|
+
const metadataBuf = toUint8Array(arrayBuffer2);
|
|
125
125
|
// let metadata = new parquet_thrift.FileMetaData();
|
|
126
126
|
// parquet_util.decodeThrift(metadata, metadataBuf);
|
|
127
127
|
const { metadata } = decodeFileMetadata(metadataBuf);
|
|
@@ -181,7 +181,7 @@ export class ParquetReader {
|
|
|
181
181
|
}
|
|
182
182
|
dictionary = context.dictionary?.length ? context.dictionary : dictionary;
|
|
183
183
|
const arrayBuffer = await this.file.read(pagesOffset, pagesSize);
|
|
184
|
-
const pagesBuf =
|
|
184
|
+
const pagesBuf = toUint8Array(arrayBuffer);
|
|
185
185
|
return await decodeDataPages(pagesBuf, { ...context, dictionary });
|
|
186
186
|
}
|
|
187
187
|
/**
|
|
@@ -203,7 +203,7 @@ export class ParquetReader {
|
|
|
203
203
|
}
|
|
204
204
|
const dictionarySize = Math.min(this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize);
|
|
205
205
|
const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);
|
|
206
|
-
const pagesBuf =
|
|
206
|
+
const pagesBuf = toUint8Array(arrayBuffer);
|
|
207
207
|
const cursor = { buffer: pagesBuf, offset: 0, size: pagesBuf.length };
|
|
208
208
|
const decodedPage = await decodePage(cursor, context);
|
|
209
209
|
return decodedPage.dictionary;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-reader.js","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAIpE,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EAAC,YAAY,EAAE,eAAe,EAAE,UAAU,EAAC,sBAAmB;AACrE,OAAO,EAAC,eAAe,EAAC,2BAAwB;AAEhD,OAAO,EAAC,aAAa,EAAE,uBAAuB,EAAC,+BAA4B;AAC3E,OAAO,EAAc,gBAAgB,EAA0B,IAAI,EAAC,mCAAgC;AAQpG,OAAO,EAAC,kBAAkB,EAAE,aAAa,EAAE,YAAY,EAAC,+BAA4B;
|
|
1
|
+
{"version":3,"file":"parquet-reader.js","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AACpC,qCAAqC;AACrC,oEAAoE;AAIpE,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EAAC,YAAY,EAAE,eAAe,EAAE,UAAU,EAAC,sBAAmB;AACrE,OAAO,EAAC,eAAe,EAAC,2BAAwB;AAEhD,OAAO,EAAC,aAAa,EAAE,uBAAuB,EAAC,+BAA4B;AAC3E,OAAO,EAAc,gBAAgB,EAA0B,IAAI,EAAC,mCAAgC;AAQpG,OAAO,EAAC,kBAAkB,EAAE,aAAa,EAAE,YAAY,EAAC,+BAA4B;AACpF,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,YAAY,EAAC,iCAA8B;AAa/E;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACxB,MAAM,CAAC,YAAY,GAAiC;QAClD,oCAAoC;QACpC,qBAAqB,EAAE,UAAU;QACjC,cAAc,EAAE,KAAK;KACtB,CAAC;IAEF,KAAK,CAA+B;IACpC,IAAI,CAAe;IACnB,QAAQ,GAAiC,IAAI,CAAC;IAE9C,YAAY,IAAkB,EAAE,KAA0B;QACxD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,EAAC,GAAG,aAAa,CAAC,YAAY,EAAE,GAAG,KAAK,EAAC,CAAC;IACzD,CAAC;IAED,KAAK;QACH,mEAAmE;QACnE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED,qBAAqB;IAErB,8BAA8B;IAC9B,KAAK,CAAC,CAAC,WAAW,CAAC,KAA6B;QAC9C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,cAAc;YACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,KAAK,CAAC,CAAC,gBAAgB,CAAC,KAA6B;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1D,MAAM,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,KAAK,CAAC,CAAC,gBAAgB,CAAC,KAA6B;QACnD,sCAAsC;QACtC,MAAM,UAAU,GAAe,CAAC,KAAK,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhG,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QAEtC,MAAM,aAAa,GAAG,QAAQ,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;QAEvD,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,aAAa,EAAE,aAAa,EAAE,EAAE,CAAC;YAC3E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CACtC,MAAM,EACN,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,EAClC,UAAU,CACX,CAAC;YACF,MAAM,QAAQ,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,EAAC,MAAM,EAAE,gBAAgB,EAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,YAAa,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACnD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC9C,MAAM,EAAE,GAA2B,EAAE,CAAC;QACtC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,kBAAmB,EAAE,CAAC;YAC9C,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAM,CAAC;QACzB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QACpC,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,oBAAoB;IAEpB,uCAAuC;IACvC,KAAK,CAAC,UAAU;QACd,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;QACtD,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,aAAa;gBAChB,MAAM;YACR,KAAK,uBAAuB;gBAC1B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC1D;gBACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,GAAG,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,uCAAuC;IACvC,KAAK,CAAC,UAAU;QACd,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,UAAU,EAAE,UAAU,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;QAE1C,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,YAAY,GAAG,UAAU,CAAC;QAClE,IAAI,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,yBAAyB,cAAc,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QACxE,MAAM,WAAW,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;QAC/C,oDAAoD;QACpD,oDAAoD;QAEpD,MAAM,EAAC,QAAQ,EAAC,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACnD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,4EAA4E;IAC5E,KAAK,CAAC,YAAY,CAChB,MAAqB,EACrB,QAAkB,EAClB,UAAsB;QAEtB,MAAM,MAAM,GAAoB;YAC9B,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACnC,UAAU,EAAE,EAAE;SACf,CAAC;QACF,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,SAAS,CAAC;YACvC,MAAM,MAAM,GAAG,WAAW,EAAE,cAAc,CAAC;YAC3C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,UAAU,EAAE,MAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnE,SAAS,CAAC,kCAAkC;YAC9C,CAAC;YACD,MAAM,CAAC,UAAU,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,MAAqB,EAAE,QAAqB;QAChE,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAe,CAAC,CAAC;QACpE,MAAM,IAAI,GAAkB,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,SAAS,EAAE,IAAK,CAAQ,CAAC;QAElF,IAAI,IAAI,KAAK,KAAK,CAAC,aAAa,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,WAAW,GAAuB,aAAa,CACnD,gBAAgB,EAChB,QAAQ,CAAC,SAAS,EAAE,KAAM,CACpB,CAAC;QAET,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,gBAAiB,CAAC,CAAC;QAClE,IAAI,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,qBAAsB,CAAC,CAAC;QAEnE,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YACxB,SAAS,GAAG,IAAI,CAAC,GAAG,CAClB,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,WAAW,EAC5B,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAClD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAyB;YACpC,IAAI;YACJ,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,WAAW;YACX,MAAM,EAAE,KAAK;YACb,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,UAAU;YACzC,UAAU,EAAE,EAAE;YACd,mDAAmD;YACnD,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc;SAC1C,CAAC;QAEF,IAAI,UAAU,CAAC;QAEf,MAAM,oBAAoB,GAAG,QAAQ,EAAE,SAAS,EAAE,sBAAsB,CAAC;QAEzE,IAAI,oBAAoB,EAAE,CAAC;YACzB,MAAM,gBAAgB,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACtD,2FAA2F;YAC3F,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;QAChF,CAAC;QAED,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;QAC1E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;QAC3C,OAAO,MAAM,eAAe,CAAC,QAAQ,EAAE,EAAC,GAAG,OAAO,EAAE,UAAU,EAAC,CAAC,CAAC;IACnE,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,aAAa,CACjB,oBAA4B,EAC5B,OAA6B,EAC7B,WAAmB;QAEnB,IAAI,oBAAoB,KAAK,CAAC,EAAE,CAAC;YAC/B,sFAAsF;YACtF,2DAA2D;YAE3D,wFAAwF;YACxF,6GAA6G;YAC7G,iEAAiE;YACjE,4FAA4F;YAC5F,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,oBAAoB,EACrC,IAAI,CAAC,KAAK,CAAC,qBAAqB,CACjC,CAAC;QACF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;QAC/E,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,MAAM,GAAG,EAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAC,CAAC;QACpE,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAEtD,OAAO,WAAW,CAAC,UAAW,CAAC;IACjC,CAAC"}
|
|
@@ -12,7 +12,7 @@ export type PrimitiveType = 'BOOLEAN' | 'INT32' | 'INT64' | 'INT96' | 'FLOAT' |
|
|
|
12
12
|
* Logical type
|
|
13
13
|
*/
|
|
14
14
|
export type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'JSON' | 'BSON' | 'INTERVAL';
|
|
15
|
-
export type ParquetDictionary =
|
|
15
|
+
export type ParquetDictionary = any[];
|
|
16
16
|
export interface SchemaDefinition {
|
|
17
17
|
[string: string]: FieldDefinition;
|
|
18
18
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,6BAA0B;AAElD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,
|
|
1
|
+
{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,6BAA0B;AAElD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC;AAEtC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0DAA0D;IAC1D,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2BAA2B;IAC3B,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0CAA0C;IAC1C,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;kDACkD;AAClD,qBAAa,eAAe;IAC1B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;gBAEnC,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAM;CAItF;AAED,0CAA0C;AAC1C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B"}
|