@loaders.gl/parquet 3.0.13 → 3.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.es5.min.js +1 -1
- package/dist/dist.es5.min.js.map +1 -1
- package/dist/dist.min.js +1 -1
- package/dist/dist.min.js.map +1 -1
- package/dist/es5/bundle.js +2 -2
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/index.js +9 -9
- package/dist/es5/parquet-loader.js +70 -19
- package/dist/es5/parquet-loader.js.map +1 -1
- package/dist/es5/parquet-writer.js +4 -4
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/es5/parquetjs/codecs/index.js +6 -4
- package/dist/es5/parquetjs/codecs/index.js.map +1 -1
- package/dist/es5/parquetjs/codecs/plain.js +43 -41
- package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
- package/dist/es5/parquetjs/codecs/rle.js +35 -25
- package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
- package/dist/es5/parquetjs/compression.js +9 -7
- package/dist/es5/parquetjs/compression.js.map +1 -1
- package/dist/es5/parquetjs/file.js +15 -15
- package/dist/es5/parquetjs/file.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +152 -141
- package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +160 -147
- package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +259 -248
- package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +79 -67
- package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +124 -113
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +169 -158
- package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DateType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +79 -68
- package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +94 -83
- package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +182 -170
- package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/IntType.js +79 -68
- package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +79 -68
- package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/ListType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +343 -319
- package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MapType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/NullType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +75 -64
- package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +94 -83
- package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +169 -158
- package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +94 -83
- package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +124 -113
- package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +199 -188
- package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +94 -83
- package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js +135 -124
- package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/StringType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js +79 -68
- package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +101 -88
- package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +79 -68
- package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +45 -31
- package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
- package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
- package/dist/es5/parquetjs/reader.js +813 -276
- package/dist/es5/parquetjs/reader.js.map +1 -1
- package/dist/es5/parquetjs/schema/declare.js +11 -9
- package/dist/es5/parquetjs/schema/declare.js.map +1 -1
- package/dist/es5/parquetjs/schema/schema.js +87 -73
- package/dist/es5/parquetjs/schema/schema.js.map +1 -1
- package/dist/es5/parquetjs/schema/shred.js +95 -55
- package/dist/es5/parquetjs/schema/shred.js.map +1 -1
- package/dist/es5/parquetjs/schema/types.js +25 -25
- package/dist/es5/parquetjs/schema/types.js.map +1 -1
- package/dist/es5/parquetjs/util.js +71 -39
- package/dist/es5/parquetjs/util.js.map +1 -1
- package/dist/es5/parquetjs/writer.js +467 -200
- package/dist/es5/parquetjs/writer.js.map +1 -1
- package/dist/esm/parquet-loader.js +1 -1
- package/dist/esm/parquet-writer.js +1 -1
- package/dist/parquet-worker.js +1 -1
- package/dist/parquet-worker.js.map +1 -1
- package/package.json +4 -4
|
@@ -8,6 +8,8 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
8
8
|
exports.encodeValues = encodeValues;
|
|
9
9
|
exports.decodeValues = decodeValues;
|
|
10
10
|
|
|
11
|
+
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
|
|
12
|
+
|
|
11
13
|
var _varint = _interopRequireDefault(require("varint"));
|
|
12
14
|
|
|
13
15
|
function encodeValues(type, values, opts) {
|
|
@@ -19,18 +21,20 @@ function encodeValues(type, values, opts) {
|
|
|
19
21
|
case 'BOOLEAN':
|
|
20
22
|
case 'INT32':
|
|
21
23
|
case 'INT64':
|
|
22
|
-
values = values.map(
|
|
24
|
+
values = values.map(function (x) {
|
|
25
|
+
return parseInt(x, 10);
|
|
26
|
+
});
|
|
23
27
|
break;
|
|
24
28
|
|
|
25
29
|
default:
|
|
26
30
|
throw new Error("unsupported type: ".concat(type));
|
|
27
31
|
}
|
|
28
32
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
33
|
+
var buf = Buffer.alloc(0);
|
|
34
|
+
var run = [];
|
|
35
|
+
var repeats = 0;
|
|
32
36
|
|
|
33
|
-
for (
|
|
37
|
+
for (var i = 0; i < values.length; i++) {
|
|
34
38
|
if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
|
|
35
39
|
if (run.length) {
|
|
36
40
|
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
@@ -60,7 +64,7 @@ function encodeValues(type, values, opts) {
|
|
|
60
64
|
return buf;
|
|
61
65
|
}
|
|
62
66
|
|
|
63
|
-
|
|
67
|
+
var envelope = Buffer.alloc(buf.length + 4);
|
|
64
68
|
envelope.writeUInt32LE(buf.length, undefined);
|
|
65
69
|
buf.copy(envelope, 4);
|
|
66
70
|
return envelope;
|
|
@@ -75,19 +79,25 @@ function decodeValues(type, cursor, count, opts) {
|
|
|
75
79
|
cursor.offset += 4;
|
|
76
80
|
}
|
|
77
81
|
|
|
78
|
-
|
|
82
|
+
var values = [];
|
|
79
83
|
|
|
80
84
|
while (values.length < count) {
|
|
81
|
-
|
|
85
|
+
var header = _varint.default.decode(cursor.buffer, cursor.offset);
|
|
82
86
|
|
|
83
87
|
cursor.offset += _varint.default.encodingLength(header);
|
|
84
88
|
|
|
85
89
|
if (header & 1) {
|
|
86
|
-
|
|
87
|
-
|
|
90
|
+
var _values;
|
|
91
|
+
|
|
92
|
+
var _count = (header >> 1) * 8;
|
|
93
|
+
|
|
94
|
+
(_values = values).push.apply(_values, (0, _toConsumableArray2.default)(decodeRunBitpacked(cursor, _count, opts)));
|
|
88
95
|
} else {
|
|
89
|
-
|
|
90
|
-
|
|
96
|
+
var _values2;
|
|
97
|
+
|
|
98
|
+
var _count2 = header >> 1;
|
|
99
|
+
|
|
100
|
+
(_values2 = values).push.apply(_values2, (0, _toConsumableArray2.default)(decodeRunRepeated(cursor, _count2, opts)));
|
|
91
101
|
}
|
|
92
102
|
}
|
|
93
103
|
|
|
@@ -101,15 +111,15 @@ function decodeValues(type, cursor, count, opts) {
|
|
|
101
111
|
}
|
|
102
112
|
|
|
103
113
|
function decodeRunBitpacked(cursor, count, opts) {
|
|
104
|
-
|
|
114
|
+
var bitWidth = opts.bitWidth;
|
|
105
115
|
|
|
106
116
|
if (count % 8 !== 0) {
|
|
107
117
|
throw new Error('must be a multiple of 8');
|
|
108
118
|
}
|
|
109
119
|
|
|
110
|
-
|
|
120
|
+
var values = new Array(count).fill(0);
|
|
111
121
|
|
|
112
|
-
for (
|
|
122
|
+
for (var b = 0; b < bitWidth * count; b++) {
|
|
113
123
|
if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
|
|
114
124
|
values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
|
|
115
125
|
}
|
|
@@ -120,10 +130,10 @@ function decodeRunBitpacked(cursor, count, opts) {
|
|
|
120
130
|
}
|
|
121
131
|
|
|
122
132
|
function decodeRunRepeated(cursor, count, opts) {
|
|
123
|
-
|
|
124
|
-
|
|
133
|
+
var bitWidth = opts.bitWidth;
|
|
134
|
+
var value = 0;
|
|
125
135
|
|
|
126
|
-
for (
|
|
136
|
+
for (var i = 0; i < Math.ceil(bitWidth / 8); i++) {
|
|
127
137
|
value <<= 8;
|
|
128
138
|
value += cursor.buffer[cursor.offset];
|
|
129
139
|
cursor.offset += 1;
|
|
@@ -133,15 +143,15 @@ function decodeRunRepeated(cursor, count, opts) {
|
|
|
133
143
|
}
|
|
134
144
|
|
|
135
145
|
function encodeRunBitpacked(values, opts) {
|
|
136
|
-
|
|
146
|
+
var bitWidth = opts.bitWidth;
|
|
137
147
|
|
|
138
|
-
for (
|
|
148
|
+
for (var i = 0; i < values.length % 8; i++) {
|
|
139
149
|
values.push(0);
|
|
140
150
|
}
|
|
141
151
|
|
|
142
|
-
|
|
152
|
+
var buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
|
|
143
153
|
|
|
144
|
-
for (
|
|
154
|
+
for (var b = 0; b < bitWidth * values.length; b++) {
|
|
145
155
|
if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
|
|
146
156
|
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
147
157
|
}
|
|
@@ -151,10 +161,10 @@ function encodeRunBitpacked(values, opts) {
|
|
|
151
161
|
}
|
|
152
162
|
|
|
153
163
|
function encodeRunRepeated(value, count, opts) {
|
|
154
|
-
|
|
155
|
-
|
|
164
|
+
var bitWidth = opts.bitWidth;
|
|
165
|
+
var buf = Buffer.alloc(Math.ceil(bitWidth / 8));
|
|
156
166
|
|
|
157
|
-
for (
|
|
167
|
+
for (var i = 0; i < buf.length; i++) {
|
|
158
168
|
buf.writeUInt8(value & 0xff, i);
|
|
159
169
|
value >>= 8;
|
|
160
170
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,kBAAkB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGgB,iBAAiB,CAACT,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAChD6B,IAAAA,KAAK,KAAK,CAAV;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,QAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AACA6B,IAAAA,KAAK,KAAK,CAAV;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n value <<= 8;\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n value >>= 8;\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
|
|
1
|
+
{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAChD6B,IAAAA,KAAK,KAAK,CAAV;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AACA6B,IAAAA,KAAK,KAAK,CAAV;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n value <<= 8;\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n value >>= 8;\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
4
|
|
|
5
|
+
var _typeof = require("@babel/runtime/helpers/typeof");
|
|
6
|
+
|
|
5
7
|
Object.defineProperty(exports, "__esModule", {
|
|
6
8
|
value: true
|
|
7
9
|
});
|
|
@@ -15,14 +17,14 @@ var _zlib = _interopRequireDefault(require("zlib"));
|
|
|
15
17
|
|
|
16
18
|
var _snappyjs = _interopRequireDefault(require("snappyjs"));
|
|
17
19
|
|
|
18
|
-
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
|
20
|
+
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
|
19
21
|
|
|
20
|
-
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null ||
|
|
22
|
+
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
|
|
21
23
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
var brotli;
|
|
25
|
+
var lzo;
|
|
26
|
+
var lz4js;
|
|
27
|
+
var PARQUET_COMPRESSION_METHODS = {
|
|
26
28
|
UNCOMPRESSED: {
|
|
27
29
|
deflate: deflate_identity,
|
|
28
30
|
inflate: inflate_identity
|
|
@@ -77,7 +79,7 @@ function deflate_lzo(value) {
|
|
|
77
79
|
|
|
78
80
|
function deflate_brotli(value) {
|
|
79
81
|
brotli = brotli || Util.load('brotli');
|
|
80
|
-
|
|
82
|
+
var result = brotli.compress(value, {
|
|
81
83
|
mode: 0,
|
|
82
84
|
quality: 8,
|
|
83
85
|
lgwin: 22
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["brotli","lzo","lz4js","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","deflate","deflate_identity","inflate","inflate_identity","GZIP","deflate_gzip","inflate_gzip","SNAPPY","deflate_snappy","inflate_snappy","LZO","deflate_lzo","inflate_lzo","BROTLI","deflate_brotli","inflate_brotli","LZ4","deflate_lz4","inflate_lz4","method","value","Error","zlib","gzipSync","snappyjs","compress","Util","load","result","mode","quality","lgwin","Buffer","from","alloc","err","size","gunzipSync","uncompress","decompress","length"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["brotli","lzo","lz4js","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","deflate","deflate_identity","inflate","inflate_identity","GZIP","deflate_gzip","inflate_gzip","SNAPPY","deflate_snappy","inflate_snappy","LZO","deflate_lzo","inflate_lzo","BROTLI","deflate_brotli","inflate_brotli","LZ4","deflate_lz4","inflate_lz4","method","value","Error","zlib","gzipSync","snappyjs","compress","Util","load","result","mode","quality","lgwin","Buffer","from","alloc","err","size","gunzipSync","uncompress","decompress","length"],"mappings":";;;;;;;;;;;;;AAGA;;AACA;;AACA;;;;;;AAEA,IAAIA,MAAJ;AACA,IAAIC,GAAJ;AACA,IAAIC,KAAJ;AAOO,IAAMC,2BAA8E,GAAG;AAC5FC,EAAAA,YAAY,EAAE;AACZC,IAAAA,OAAO,EAAEC,gBADG;AAEZC,IAAAA,OAAO,EAAEC;AAFG,GAD8E;AAK5FC,EAAAA,IAAI,EAAE;AACJJ,IAAAA,OAAO,EAAEK,YADL;AAEJH,IAAAA,OAAO,EAAEI;AAFL,GALsF;AAS5FC,EAAAA,MAAM,EAAE;AACNP,IAAAA,OAAO,EAAEQ,cADH;AAENN,IAAAA,OAAO,EAAEO;AAFH,GAToF;AAa5FC,EAAAA,GAAG,EAAE;AACHV,IAAAA,OAAO,EAAEW,WADN;AAEHT,IAAAA,OAAO,EAAEU;AAFN,GAbuF;AAiB5FC,EAAAA,MAAM,EAAE;AACNb,IAAAA,OAAO,EAAEc,cADH;AAENZ,IAAAA,OAAO,EAAEa;AAFH,GAjBoF;AAqB5FC,EAAAA,GAAG,EAAE;AACHhB,IAAAA,OAAO,EAAEiB,WADN;AAEHf,IAAAA,OAAO,EAAEgB;AAFN;AArBuF,CAAvF;;;AA8BA,SAASlB,OAAT,CAAiBmB,MAAjB,EAA6CC,KAA7C,EAAoE;AACzE,MAAI,EAAED,MAAM,IAAIrB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIuB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOrB,2BAA2B,CAACqB,MAAD,CAA3B,CAAoCnB,OAApC,CAA4CoB,KAA5C,CAAP;AACD;;AAED,SAASnB,gBAAT,CAA0BmB,KAA1B,EAAiD;AAC/C,SAAOA,KAAP;AACD;;AAED,SAASf,YAAT,CAAsBe,KAAtB,EAA6C;AAC3C,SAAOE,cAAKC,QAAL,CAAcH,KAAd,CAAP;AACD;;AAED,SAASZ,cAAT,CAAwBY,KAAxB,EAA+C;AAC7C,SAAOI,kBAASC,QAAT,CAAkBL,KAAlB,CAAP;AACD;;AAED,SAAST,WAAT,CAAqBS,KAArB,EAA4C;AAC1CxB,EAAAA,GAAG,GAAGA,GAAG,IAAI8B,IAAI,CAACC,IAAL,CAAU,KAAV,CAAb;AACA,SAAO/B,GAAG,CAAC6B,QAAJ,CAAaL,KAAb,CAAP;AACD;;AAED,SAASN,cAAT,CAAwBM,KAAxB,EAA+C;AAC7CzB,EAAAA,MAAM,GAAGA,MAAM,IAAI+B,IAAI,CAACC,IAAL,CAAU,QAAV,CAAnB;AACA,MAAMC,MAAM,GAAGjC,MAAM,CAAC8B,QAAP,CAAgBL,KAAhB,EAAuB;AACpCS,IAAAA,IAAI,EAAE,CAD8B;AAEpCC,IAAAA,OAAO,EAAE,CAF2B;AAGpCC,IAAAA,KAAK,EAAE;AAH6B,GAAvB,CAAf;AAKA,SAAOH,MAAM,GAAGI,MAAM,CAACC,IAAP,CAAYL,MAAZ,CAAH,GAAyBI,MAAM,CAACE,KAAP,CAAa,CAAb,CAAtC;AACD;;AAED,SAASjB,WAAT,CAAqBG,KAArB,EAA4C;AAC1CvB,EAAAA,KAAK,GAAGA,KAAK,IAAI6B,IAAI,CAACC,IAAL,CAAU,OAAV,CAAjB;;AACA,MAAI;AAMF,WAAOK,MAAM,CAACC,IAAP,CAAYpC,KAAK,CAAC4B,QAAN,CAAeL,KAAf,CAAZ,CAAP;AACD,GAPD,CAOE,OAAOe,GAAP,EAAY;AACZ,UAAMA,GAAN;AACD;AACF;;AAKM,SAASjC,OAAT,CAAiBiB,MAAjB,EAA6CC,KAA7C,EAA4DgB,IAA5D,EAAkF;AACvF,MAAI,EAAEjB,MAAM,IAAIrB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIuB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOrB,2BAA2B,CAACqB,MAAD,CAA3B,CAAoCjB,OAApC,CAA4CkB,KAA5C,EAAmDgB,IAAnD,CAAP;AACD;;AAED,SAASjC,gBAAT,CAA0BiB,KAA1B,EAAiD;AAC/C,SAAOA,KAAP;AACD;;AAED,SAASd,YAAT,CAAsBc,KAAtB,EAA6C;AAC3C,SAAOE,cAAKe,UAAL,CAAgBjB,KAAhB,CAAP;AACD;;AAED,SAASX,cAAT,CAAwBW,KAAxB,EAA+C;AAC7C,SAAOI,kBAASc,UAAT,CAAoBlB,KAApB,CAAP;AACD;;AAED,SAASR,WAAT,CAAqBQ,KAArB,EAAoCgB,IAApC,EAA0D;AACxDxC,EAAAA,GAAG,GAAGA,GAAG,IAAI8B,IAAI,CAACC,IAAL,CAAU,KAAV,CAAb;AACA,SAAO/B,GAAG,CAAC2C,UAAJ,CAAenB,KAAf,EAAsBgB,IAAtB,CAAP;AACD;;AAED,SAASlB,WAAT,CAAqBE,KAArB,EAAoCgB,IAApC,EAA0D;AACxDvC,EAAAA,KAAK,GAAGA,KAAK,IAAI6B,IAAI,CAACC,IAAL,CAAU,OAAV,CAAjB;;AACA,MAAI;AAMF,WAAOK,MAAM,CAACC,IAAP,CAAYpC,KAAK,CAAC0C,UAAN,CAAiBnB,KAAjB,EAAwBgB,IAAxB,CAAZ,CAAP;AACD,GAPD,CAOE,OAAOD,GAAP,EAAY;AACZ,UAAMA,GAAN;AACD;AACF;;AAED,SAASpB,cAAT,CAAwBK,KAAxB,EAA+C;AAC7CzB,EAAAA,MAAM,GAAGA,MAAM,IAAI+B,IAAI,CAACC,IAAL,CAAU,QAAV,CAAnB;;AACA,MAAI,CAACP,KAAK,CAACoB,MAAX,EAAmB;AACjB,WAAOR,MAAM,CAACE,KAAP,CAAa,CAAb,CAAP;AACD;;AACD,SAAOF,MAAM,CAACC,IAAP,CAAYtC,MAAM,CAAC4C,UAAP,CAAkBnB,KAAlB,CAAZ,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {ParquetCompression} from './schema/declare';\nimport * as Util from './util';\nimport zlib from 'zlib';\nimport snappyjs from 'snappyjs';\n\nlet brotli: any;\nlet lzo: any;\nlet lz4js: any;\n\nexport interface ParquetCompressionKit {\n deflate: (value: Buffer) => Buffer;\n inflate: (value: Buffer, size: number) => Buffer;\n}\n\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, ParquetCompressionKit> = {\n UNCOMPRESSED: {\n deflate: deflate_identity,\n inflate: inflate_identity\n },\n GZIP: {\n deflate: deflate_gzip,\n inflate: inflate_gzip\n },\n SNAPPY: {\n deflate: deflate_snappy,\n inflate: inflate_snappy\n },\n LZO: {\n deflate: deflate_lzo,\n inflate: inflate_lzo\n },\n BROTLI: {\n deflate: deflate_brotli,\n inflate: inflate_brotli\n },\n LZ4: {\n deflate: deflate_lz4,\n inflate: inflate_lz4\n }\n};\n\n/**\n * Deflate a value using compression method `method`\n */\nexport function deflate(method: ParquetCompression, value: Buffer): Buffer {\n if (!(method in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`invalid compression method: ${method}`);\n }\n\n return PARQUET_COMPRESSION_METHODS[method].deflate(value);\n}\n\nfunction deflate_identity(value: Buffer): Buffer {\n return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n lzo = lzo || Util.load('lzo');\n return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n brotli = brotli || Util.load('brotli');\n const result = brotli.compress(value, {\n mode: 0,\n quality: 8,\n lgwin: 22\n });\n return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n lz4js = lz4js || Util.load('lz4js');\n try {\n // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n // const compressedSize = lz4.encodeBlock(value, result);\n // // remove unnecessary bytes\n // result = result.slice(0, compressedSize);\n // return result;\n return Buffer.from(lz4js.compress(value));\n } catch (err) {\n throw err;\n }\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n if (!(method in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`invalid compression method: ${method}`);\n }\n\n return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\nfunction inflate_identity(value: Buffer): Buffer {\n return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n lzo = lzo || Util.load('lzo');\n return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n lz4js = lz4js || Util.load('lz4js');\n try {\n // let result = Buffer.alloc(size);\n // const uncompressedSize = lz4js.decodeBlock(value, result);\n // // remove unnecessary bytes\n // result = result.slice(0, uncompressedSize);\n // return result;\n return Buffer.from(lz4js.decompress(value, size));\n } catch (err) {\n throw err;\n }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n brotli = brotli || Util.load('brotli');\n if (!value.length) {\n return Buffer.alloc(0);\n }\n return Buffer.from(brotli.decompress(value));\n}\n"],"file":"compression.js"}
|
|
@@ -16,8 +16,8 @@ exports.osopen = osopen;
|
|
|
16
16
|
var _fs = _interopRequireDefault(require("fs"));
|
|
17
17
|
|
|
18
18
|
function fopen(filePath) {
|
|
19
|
-
return new Promise((resolve, reject)
|
|
20
|
-
_fs.default.open(filePath, 'r', (err, fd)
|
|
19
|
+
return new Promise(function (resolve, reject) {
|
|
20
|
+
_fs.default.open(filePath, 'r', function (err, fd) {
|
|
21
21
|
if (err) {
|
|
22
22
|
reject(err);
|
|
23
23
|
} else {
|
|
@@ -28,8 +28,8 @@ function fopen(filePath) {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
function fstat(filePath) {
|
|
31
|
-
return new Promise((resolve, reject)
|
|
32
|
-
_fs.default.stat(filePath, (err, stat)
|
|
31
|
+
return new Promise(function (resolve, reject) {
|
|
32
|
+
_fs.default.stat(filePath, function (err, stat) {
|
|
33
33
|
if (err) {
|
|
34
34
|
reject(err);
|
|
35
35
|
} else {
|
|
@@ -40,9 +40,9 @@ function fstat(filePath) {
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
function fread(fd, position, length) {
|
|
43
|
-
|
|
44
|
-
return new Promise((resolve, reject)
|
|
45
|
-
_fs.default.read(fd, buffer, 0, length, position, (err, bytesRead, buf)
|
|
43
|
+
var buffer = Buffer.alloc(length);
|
|
44
|
+
return new Promise(function (resolve, reject) {
|
|
45
|
+
_fs.default.read(fd, buffer, 0, length, position, function (err, bytesRead, buf) {
|
|
46
46
|
if (err || bytesRead !== length) {
|
|
47
47
|
reject(err || Error('read failed'));
|
|
48
48
|
} else {
|
|
@@ -53,8 +53,8 @@ function fread(fd, position, length) {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
function fclose(fd) {
|
|
56
|
-
return new Promise((resolve, reject)
|
|
57
|
-
_fs.default.close(fd, err
|
|
56
|
+
return new Promise(function (resolve, reject) {
|
|
57
|
+
_fs.default.close(fd, function (err) {
|
|
58
58
|
if (err) {
|
|
59
59
|
reject(err);
|
|
60
60
|
} else {
|
|
@@ -65,8 +65,8 @@ function fclose(fd) {
|
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
function oswrite(os, buf) {
|
|
68
|
-
return new Promise((resolve, reject)
|
|
69
|
-
os.write(buf, err
|
|
68
|
+
return new Promise(function (resolve, reject) {
|
|
69
|
+
os.write(buf, function (err) {
|
|
70
70
|
if (err) {
|
|
71
71
|
reject(err);
|
|
72
72
|
} else {
|
|
@@ -77,8 +77,8 @@ function oswrite(os, buf) {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
function osclose(os) {
|
|
80
|
-
return new Promise((resolve, reject)
|
|
81
|
-
os.close(err
|
|
80
|
+
return new Promise(function (resolve, reject) {
|
|
81
|
+
os.close(function (err) {
|
|
82
82
|
if (err) {
|
|
83
83
|
reject(err);
|
|
84
84
|
} else {
|
|
@@ -89,8 +89,8 @@ function osclose(os) {
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
function osopen(path, opts) {
|
|
92
|
-
return new Promise((resolve, reject)
|
|
93
|
-
|
|
92
|
+
return new Promise(function (resolve, reject) {
|
|
93
|
+
var outputStream = _fs.default.createWriteStream(path, opts);
|
|
94
94
|
|
|
95
95
|
outputStream.on('open', function (fd) {
|
|
96
96
|
resolve(outputStream);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/parquetjs/file.ts"],"names":["fopen","filePath","Promise","resolve","reject","fs","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","on"],"mappings":";;;;;;;;;;;;;;;AACA;;AAEO,SAASA,KAAT,CAAeC,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAY,
|
|
1
|
+
{"version":3,"sources":["../../../src/parquetjs/file.ts"],"names":["fopen","filePath","Promise","resolve","reject","fs","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","on"],"mappings":";;;;;;;;;;;;;;;AACA;;AAEO,SAASA,KAAT,CAAeC,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCC,gBAAGC,IAAH,CAAQL,QAAR,EAAkB,GAAlB,EAAuB,UAACM,GAAD,EAAMC,EAAN,EAAa;AAClC,UAAID,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACK,EAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASC,KAAT,CAAeR,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAsB,UAACC,OAAD,EAAUC,MAAV,EAAqB;AAChDC,gBAAGK,IAAH,CAAQT,QAAR,EAAkB,UAACM,GAAD,EAAMG,IAAN,EAAe;AAC/B,UAAIH,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACO,IAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASC,KAAT,CAAeH,EAAf,EAAmBI,QAAnB,EAA6BC,MAA7B,EAAqC;AAC1C,MAAMC,MAAM,GAAGC,MAAM,CAACC,KAAP,CAAaH,MAAb,CAAf;AAEA,SAAO,IAAIX,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCC,gBAAGY,IAAH,CAAQT,EAAR,EAAYM,MAAZ,EAAoB,CAApB,EAAuBD,MAAvB,EAA+BD,QAA/B,EAAyC,UAACL,GAAD,EAAMW,SAAN,EAAiBC,GAAjB,EAAyB;AAChE,UAAIZ,GAAG,IAAIW,SAAS,KAAKL,MAAzB,EAAiC;AAC/BT,QAAAA,MAAM,CAACG,GAAG,IAAIa,KAAK,CAAC,aAAD,CAAb,CAAN;AACD,OAFD,MAEO;AACLjB,QAAAA,OAAO,CAACgB,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASE,MAAT,CAAgBb,EAAhB,EAAoB;AACzB,SAAO,IAAIN,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCC,gBAAGiB,KAAH,CAASd,EAAT,EAAa,UAACD,GAAD,EAAS;AACpB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACI,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASgB,OAAT,CAAiBC,EAAjB,EAAqBL,GAArB,EAAyC;AAC9C,SAAO,IAAIjB,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCoB,IAAAA,EAAE,CAACC,KAAH,CAASN,GAAT,EAAc,UAACZ,GAAD,EAAS;AACrB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASuB,OAAT,CAAiBF,EAAjB,EAAoC;AACzC,SAAO,IAAItB,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCoB,IAAAA,EAAE,CAACF,KAAH,CAAS,UAACf,GAAD,EAAS;AAChB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASwB,MAAT,CAAgBC,IAAhB,EAAsBC,IAAtB,EAA4B;AACjC,SAAO,IAAI3B,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtC,QAAM0B,YAAY,GAAGzB,YAAG0B,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;;AAEAC,IAAAA,YAAY,CAACE,EAAb,CAAgB,MAAhB,EAAwB,UAAUxB,EAAV,EAAc;AACpCL,MAAAA,OAAO,CAAC2B,YAAD,CAAP;AACD,KAFD;AAIAA,IAAAA,YAAY,CAACE,EAAb,CAAgB,OAAhB,EAAyB,UAAUzB,GAAV,EAAe;AACtCH,MAAAA,MAAM,CAACG,GAAD,CAAN;AACD,KAFD;AAGD,GAVM,CAAP;AAWD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\n\nexport function fopen(filePath) {\n return new Promise((resolve, reject) => {\n fs.open(filePath, 'r', (err, fd) => {\n if (err) {\n reject(err);\n } else {\n resolve(fd);\n }\n });\n });\n}\n\nexport function fstat(filePath) {\n return new Promise<fs.Stats>((resolve, reject) => {\n fs.stat(filePath, (err, stat) => {\n if (err) {\n reject(err);\n } else {\n resolve(stat);\n }\n });\n });\n}\n\nexport function fread(fd, position, length) {\n const buffer = Buffer.alloc(length);\n\n return new Promise((resolve, reject) => {\n fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {\n if (err || bytesRead !== length) {\n reject(err || Error('read failed'));\n } else {\n resolve(buf);\n }\n });\n });\n}\n\nexport function fclose(fd) {\n return new Promise((resolve, reject) => {\n fs.close(fd, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve(err);\n }\n });\n });\n}\n\nexport function oswrite(os, buf): Promise<void> {\n return new Promise((resolve, reject) => {\n os.write(buf, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osclose(os): Promise<void> {\n return new Promise((resolve, reject) => {\n os.close((err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osopen(path, opts) {\n return new Promise((resolve, reject) => {\n const outputStream = fs.createWriteStream(path, opts);\n\n outputStream.on('open', function (fd) {\n resolve(outputStream);\n });\n\n outputStream.on('error', function (err) {\n reject(err);\n });\n });\n}\n"],"file":"file.js"}
|
|
@@ -1,53 +1,67 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
+
|
|
5
|
+
var _typeof = require("@babel/runtime/helpers/typeof");
|
|
6
|
+
|
|
3
7
|
Object.defineProperty(exports, "__esModule", {
|
|
4
8
|
value: true
|
|
5
9
|
});
|
|
6
10
|
exports.BsonType = void 0;
|
|
7
11
|
|
|
8
|
-
var
|
|
12
|
+
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
9
13
|
|
|
10
|
-
|
|
14
|
+
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
11
15
|
|
|
12
|
-
|
|
16
|
+
var thrift = _interopRequireWildcard(require("thrift"));
|
|
17
|
+
|
|
18
|
+
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
|
|
13
19
|
|
|
14
|
-
|
|
15
|
-
constructor() {}
|
|
20
|
+
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
|
|
16
21
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
output.writeStructEnd();
|
|
21
|
-
return;
|
|
22
|
+
var BsonType = function () {
|
|
23
|
+
function BsonType() {
|
|
24
|
+
(0, _classCallCheck2.default)(this, BsonType);
|
|
22
25
|
}
|
|
23
26
|
|
|
24
|
-
|
|
25
|
-
|
|
27
|
+
(0, _createClass2.default)(BsonType, [{
|
|
28
|
+
key: "write",
|
|
29
|
+
value: function write(output) {
|
|
30
|
+
output.writeStructBegin('BsonType');
|
|
31
|
+
output.writeFieldStop();
|
|
32
|
+
output.writeStructEnd();
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
}], [{
|
|
36
|
+
key: "read",
|
|
37
|
+
value: function read(input) {
|
|
38
|
+
input.readStructBegin();
|
|
26
39
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
40
|
+
while (true) {
|
|
41
|
+
var ret = input.readFieldBegin();
|
|
42
|
+
var fieldType = ret.ftype;
|
|
43
|
+
var fieldId = ret.fid;
|
|
31
44
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
45
|
+
if (fieldType === thrift.Thrift.Type.STOP) {
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
switch (fieldId) {
|
|
50
|
+
default:
|
|
51
|
+
{
|
|
52
|
+
input.skip(fieldType);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
35
55
|
|
|
36
|
-
|
|
37
|
-
default:
|
|
38
|
-
{
|
|
39
|
-
input.skip(fieldType);
|
|
40
|
-
}
|
|
56
|
+
input.readFieldEnd();
|
|
41
57
|
}
|
|
42
58
|
|
|
43
|
-
input.
|
|
59
|
+
input.readStructEnd();
|
|
60
|
+
return new BsonType();
|
|
44
61
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
}
|
|
62
|
+
}]);
|
|
63
|
+
return BsonType;
|
|
64
|
+
}();
|
|
51
65
|
|
|
52
66
|
exports.BsonType = BsonType;
|
|
53
67
|
//# sourceMappingURL=BsonType.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/parquetjs/parquet-thrift/BsonType.ts"],"names":["BsonType","
|
|
1
|
+
{"version":3,"sources":["../../../../src/parquetjs/parquet-thrift/BsonType.ts"],"names":["BsonType","output","writeStructBegin","writeFieldStop","writeStructEnd","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","thrift","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"mappings":";;;;;;;;;;;;;;;AAMA;;;;;;IAEaA,Q;AACX,sBAAc;AAAA;AAAE;;;;WAChB,eAAaC,MAAb,EAA6C;AAC3CA,MAAAA,MAAM,CAACC,gBAAP,CAAwB,UAAxB;AACAD,MAAAA,MAAM,CAACE,cAAP;AACAF,MAAAA,MAAM,CAACG,cAAP;AACA;AACD;;;WACD,cAAmBC,KAAnB,EAAsD;AACpDA,MAAAA,KAAK,CAACC,eAAN;;AACA,aAAO,IAAP,EAAa;AACX,YAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAN,EAA3B;AACA,YAAMC,SAA6B,GAAGF,GAAG,CAACG,KAA1C;AACA,YAAMC,OAAe,GAAGJ,GAAG,CAACK,GAA5B;;AACA,YAAIH,SAAS,KAAKI,MAAM,CAACC,MAAP,CAAcC,IAAd,CAAmBC,IAArC,EAA2C;AACzC;AACD;;AACD,gBAAQL,OAAR;AACE;AAAS;AACPN,cAAAA,KAAK,CAACY,IAAN,CAAWR,SAAX;AACD;AAHH;;AAKAJ,QAAAA,KAAK,CAACa,YAAN;AACD;;AACDb,MAAAA,KAAK,CAACc,aAAN;AACA,aAAO,IAAInB,QAAJ,EAAP;AACD","sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nimport * as thrift from 'thrift';\nexport interface IBsonTypeArgs {}\nexport class BsonType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('BsonType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): BsonType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new BsonType();\n }\n}\n"],"file":"BsonType.js"}
|