@loaders.gl/parquet 3.0.13 → 3.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/dist.es5.min.js +1 -1
  2. package/dist/dist.es5.min.js.map +1 -1
  3. package/dist/dist.min.js +1 -1
  4. package/dist/dist.min.js.map +1 -1
  5. package/dist/es5/bundle.js +2 -2
  6. package/dist/es5/bundle.js.map +1 -1
  7. package/dist/es5/index.js +9 -9
  8. package/dist/es5/parquet-loader.js +70 -19
  9. package/dist/es5/parquet-loader.js.map +1 -1
  10. package/dist/es5/parquet-writer.js +4 -4
  11. package/dist/es5/parquet-writer.js.map +1 -1
  12. package/dist/es5/parquetjs/codecs/index.js +6 -4
  13. package/dist/es5/parquetjs/codecs/index.js.map +1 -1
  14. package/dist/es5/parquetjs/codecs/plain.js +43 -41
  15. package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
  16. package/dist/es5/parquetjs/codecs/rle.js +35 -25
  17. package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
  18. package/dist/es5/parquetjs/compression.js +9 -7
  19. package/dist/es5/parquetjs/compression.js.map +1 -1
  20. package/dist/es5/parquetjs/file.js +15 -15
  21. package/dist/es5/parquetjs/file.js.map +1 -1
  22. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
  23. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +45 -31
  24. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  25. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +152 -141
  26. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  27. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +160 -147
  28. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  29. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +259 -248
  30. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  31. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +79 -67
  32. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  33. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
  34. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
  35. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +124 -113
  36. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  37. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +169 -158
  38. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  39. package/dist/es5/parquetjs/parquet-thrift/DateType.js +45 -31
  40. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
  41. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +79 -68
  42. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  43. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +94 -83
  44. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  45. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
  46. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +45 -31
  47. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  48. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
  49. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +182 -170
  50. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  51. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +45 -31
  52. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  53. package/dist/es5/parquetjs/parquet-thrift/IntType.js +79 -68
  54. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
  55. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +45 -31
  56. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  57. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +79 -68
  58. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  59. package/dist/es5/parquetjs/parquet-thrift/ListType.js +45 -31
  60. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
  61. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +343 -319
  62. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  63. package/dist/es5/parquetjs/parquet-thrift/MapType.js +45 -31
  64. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
  65. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +45 -31
  66. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  67. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +45 -31
  68. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  69. package/dist/es5/parquetjs/parquet-thrift/NullType.js +45 -31
  70. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
  71. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +75 -64
  72. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  73. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +94 -83
  74. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  75. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +169 -158
  76. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  77. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +94 -83
  78. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  79. package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
  80. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +124 -113
  81. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  82. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +199 -188
  83. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  84. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +94 -83
  85. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  86. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +135 -124
  87. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  88. package/dist/es5/parquetjs/parquet-thrift/StringType.js +45 -31
  89. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
  90. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +79 -68
  91. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  92. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +101 -88
  93. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  94. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +79 -68
  95. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  96. package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
  97. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +45 -31
  98. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  99. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +45 -31
  100. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  101. package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
  102. package/dist/es5/parquetjs/reader.js +813 -276
  103. package/dist/es5/parquetjs/reader.js.map +1 -1
  104. package/dist/es5/parquetjs/schema/declare.js +11 -9
  105. package/dist/es5/parquetjs/schema/declare.js.map +1 -1
  106. package/dist/es5/parquetjs/schema/schema.js +87 -73
  107. package/dist/es5/parquetjs/schema/schema.js.map +1 -1
  108. package/dist/es5/parquetjs/schema/shred.js +95 -55
  109. package/dist/es5/parquetjs/schema/shred.js.map +1 -1
  110. package/dist/es5/parquetjs/schema/types.js +25 -25
  111. package/dist/es5/parquetjs/schema/types.js.map +1 -1
  112. package/dist/es5/parquetjs/util.js +71 -39
  113. package/dist/es5/parquetjs/util.js.map +1 -1
  114. package/dist/es5/parquetjs/writer.js +467 -200
  115. package/dist/es5/parquetjs/writer.js.map +1 -1
  116. package/dist/esm/parquet-loader.js +1 -1
  117. package/dist/esm/parquet-writer.js +1 -1
  118. package/dist/parquet-worker.js +1 -1
  119. package/dist/parquet-worker.js.map +1 -1
  120. package/package.json +4 -4
@@ -8,6 +8,8 @@ Object.defineProperty(exports, "__esModule", {
8
8
  exports.encodeValues = encodeValues;
9
9
  exports.decodeValues = decodeValues;
10
10
 
11
+ var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
12
+
11
13
  var _varint = _interopRequireDefault(require("varint"));
12
14
 
13
15
  function encodeValues(type, values, opts) {
@@ -19,18 +21,20 @@ function encodeValues(type, values, opts) {
19
21
  case 'BOOLEAN':
20
22
  case 'INT32':
21
23
  case 'INT64':
22
- values = values.map(x => parseInt(x, 10));
24
+ values = values.map(function (x) {
25
+ return parseInt(x, 10);
26
+ });
23
27
  break;
24
28
 
25
29
  default:
26
30
  throw new Error("unsupported type: ".concat(type));
27
31
  }
28
32
 
29
- let buf = Buffer.alloc(0);
30
- let run = [];
31
- let repeats = 0;
33
+ var buf = Buffer.alloc(0);
34
+ var run = [];
35
+ var repeats = 0;
32
36
 
33
- for (let i = 0; i < values.length; i++) {
37
+ for (var i = 0; i < values.length; i++) {
34
38
  if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
35
39
  if (run.length) {
36
40
  buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
@@ -60,7 +64,7 @@ function encodeValues(type, values, opts) {
60
64
  return buf;
61
65
  }
62
66
 
63
- const envelope = Buffer.alloc(buf.length + 4);
67
+ var envelope = Buffer.alloc(buf.length + 4);
64
68
  envelope.writeUInt32LE(buf.length, undefined);
65
69
  buf.copy(envelope, 4);
66
70
  return envelope;
@@ -75,19 +79,25 @@ function decodeValues(type, cursor, count, opts) {
75
79
  cursor.offset += 4;
76
80
  }
77
81
 
78
- let values = [];
82
+ var values = [];
79
83
 
80
84
  while (values.length < count) {
81
- const header = _varint.default.decode(cursor.buffer, cursor.offset);
85
+ var header = _varint.default.decode(cursor.buffer, cursor.offset);
82
86
 
83
87
  cursor.offset += _varint.default.encodingLength(header);
84
88
 
85
89
  if (header & 1) {
86
- const count = (header >> 1) * 8;
87
- values.push(...decodeRunBitpacked(cursor, count, opts));
90
+ var _values;
91
+
92
+ var _count = (header >> 1) * 8;
93
+
94
+ (_values = values).push.apply(_values, (0, _toConsumableArray2.default)(decodeRunBitpacked(cursor, _count, opts)));
88
95
  } else {
89
- const count = header >> 1;
90
- values.push(...decodeRunRepeated(cursor, count, opts));
96
+ var _values2;
97
+
98
+ var _count2 = header >> 1;
99
+
100
+ (_values2 = values).push.apply(_values2, (0, _toConsumableArray2.default)(decodeRunRepeated(cursor, _count2, opts)));
91
101
  }
92
102
  }
93
103
 
@@ -101,15 +111,15 @@ function decodeValues(type, cursor, count, opts) {
101
111
  }
102
112
 
103
113
  function decodeRunBitpacked(cursor, count, opts) {
104
- const bitWidth = opts.bitWidth;
114
+ var bitWidth = opts.bitWidth;
105
115
 
106
116
  if (count % 8 !== 0) {
107
117
  throw new Error('must be a multiple of 8');
108
118
  }
109
119
 
110
- const values = new Array(count).fill(0);
120
+ var values = new Array(count).fill(0);
111
121
 
112
- for (let b = 0; b < bitWidth * count; b++) {
122
+ for (var b = 0; b < bitWidth * count; b++) {
113
123
  if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & 1 << b % 8) {
114
124
  values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
115
125
  }
@@ -120,10 +130,10 @@ function decodeRunBitpacked(cursor, count, opts) {
120
130
  }
121
131
 
122
132
  function decodeRunRepeated(cursor, count, opts) {
123
- const bitWidth = opts.bitWidth;
124
- let value = 0;
133
+ var bitWidth = opts.bitWidth;
134
+ var value = 0;
125
135
 
126
- for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
136
+ for (var i = 0; i < Math.ceil(bitWidth / 8); i++) {
127
137
  value <<= 8;
128
138
  value += cursor.buffer[cursor.offset];
129
139
  cursor.offset += 1;
@@ -133,15 +143,15 @@ function decodeRunRepeated(cursor, count, opts) {
133
143
  }
134
144
 
135
145
  function encodeRunBitpacked(values, opts) {
136
- const bitWidth = opts.bitWidth;
146
+ var bitWidth = opts.bitWidth;
137
147
 
138
- for (let i = 0; i < values.length % 8; i++) {
148
+ for (var i = 0; i < values.length % 8; i++) {
139
149
  values.push(0);
140
150
  }
141
151
 
142
- const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
152
+ var buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
143
153
 
144
- for (let b = 0; b < bitWidth * values.length; b++) {
154
+ for (var b = 0; b < bitWidth * values.length; b++) {
145
155
  if ((values[Math.floor(b / bitWidth)] & 1 << b % bitWidth) > 0) {
146
156
  buf[Math.floor(b / 8)] |= 1 << b % 8;
147
157
  }
@@ -151,10 +161,10 @@ function encodeRunBitpacked(values, opts) {
151
161
  }
152
162
 
153
163
  function encodeRunRepeated(value, count, opts) {
154
- const bitWidth = opts.bitWidth;
155
- const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
164
+ var bitWidth = opts.bitWidth;
165
+ var buf = Buffer.alloc(Math.ceil(bitWidth / 8));
156
166
 
157
- for (let i = 0; i < buf.length; i++) {
167
+ for (var i = 0; i < buf.length; i++) {
158
168
  buf.writeUInt8(value & 0xff, i);
159
169
  value >>= 8;
160
170
  }
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,kBAAkB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGgB,iBAAiB,CAACT,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAChD6B,IAAAA,KAAK,KAAK,CAAV;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,QAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AACA6B,IAAAA,KAAK,KAAK,CAAV;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n value <<= 8;\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n value >>= 8;\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
1
+ {"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","varint","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":";;;;;;;;;;;;AAIA;;AAGO,SAASA,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAW,UAACC,CAAD;AAAA,eAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAAf;AAAA,OAAX,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,6BAA+BH,IAA/B,EAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,MAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;;AAEM,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,QAAME,MAAM,GAAGC,gBAAOC,MAAP,CAAcL,MAAM,CAACM,MAArB,EAA6BN,MAAM,CAACE,MAApC,CAAf;;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiBE,gBAAOG,cAAP,CAAsBJ,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AAAA;;AACd,UAAMF,MAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;;AACA,iBAAA1B,MAAM,EAACgB,IAAP,iDAAee,kBAAkB,CAACR,MAAD,EAASC,MAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AAAA;;AACL,UAAMuB,OAAK,GAAGE,MAAM,IAAI,CAAxB;;AACA,kBAAA1B,MAAM,EAACgB,IAAP,kDAAegB,iBAAiB,CAACT,MAAD,EAASC,OAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACiC,KAAP,CAAa,CAAb,EAAgBT,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS+B,kBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,MAAIV,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,MAAMF,MAAM,GAAG,IAAImC,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGV,KAA/B,EAAsCa,CAAC,EAAvC,EAA2C;AACzC,QAAId,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAAP,GAAgBa,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnErC,MAAAA,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDX,EAAAA,MAAM,CAACE,MAAP,IAAiBS,QAAQ,IAAIV,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAASgC,iBAAT,CACET,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI7B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG2B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CvB,CAAC,EAA9C,EAAkD;AAChD6B,IAAAA,KAAK,KAAK,CAAV;AACAA,IAAAA,KAAK,IAAIjB,MAAM,CAACM,MAAP,CAAcN,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIU,KAAJ,CAAUX,KAAV,EAAiBY,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAAS1B,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,MAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIlC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIyB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGlC,MAAM,CAACY,MAAtC,EAA8CyB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACrC,MAAM,CAACsC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE5B,MAAAA,GAAG,CAACgC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO9B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAgB3C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2ByB,KAA3B,EAA0ChB,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,MAAMiC,QAAgB,GAAGjC,IAAI,CAACiC,QAA9B;AAEA,MAAM5B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa8B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAIvB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACsC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B7B,CAA7B;AACA6B,IAAAA,KAAK,KAAK,CAAV;AACD;;AAED,SAAOjC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACmC,IAAP,CAAYf,gBAAOgB,MAAP,CAAcnB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n type: PrimitiveType,\n values: any[],\n opts: ParquetCodecOptions\n): Buffer {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n switch (type) {\n case 'BOOLEAN':\n case 'INT32':\n case 'INT64':\n // tslint:disable-next-line:no-parameter-reassignment\n values = values.map((x) => parseInt(x, 10));\n break;\n\n default:\n throw new Error(`unsupported type: ${type}`);\n }\n\n let buf = Buffer.alloc(0);\n let run: any[] = [];\n let repeats = 0;\n\n for (let i = 0; i < values.length; i++) {\n // If we are at the beginning of a run and the next value is same we start\n // collecting repeated values\n if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n // If we have any data in runs we need to encode them\n if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n run = [];\n }\n repeats = 1;\n } else if (repeats > 0 && values[i] === values[i - 1]) {\n repeats += 1;\n } else {\n // If values changes we need to post any previous repeated values\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n repeats = 0;\n }\n run.push(values[i]);\n }\n }\n\n if (repeats) {\n buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n } else if (run.length) {\n buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n }\n\n if (opts.disableEnvelope) {\n return buf;\n }\n\n const envelope = Buffer.alloc(buf.length + 4);\n envelope.writeUInt32LE(buf.length, undefined);\n buf.copy(envelope, 4);\n\n return envelope;\n}\n\nexport function decodeValues(\n type: PrimitiveType,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n if (!('bitWidth' in opts)) {\n throw new Error('bitWidth is required');\n }\n\n if (!opts.disableEnvelope) {\n cursor.offset += 4;\n }\n\n let values: number[] = [];\n while (values.length < count) {\n const header = varint.decode(cursor.buffer, cursor.offset);\n cursor.offset += varint.encodingLength(header);\n if (header & 1) {\n const count = (header >> 1) * 8;\n values.push(...decodeRunBitpacked(cursor, count, opts));\n } else {\n const count = header >> 1;\n values.push(...decodeRunRepeated(cursor, count, opts));\n }\n }\n values = values.slice(0, count);\n\n if (values.length !== count) {\n throw new Error('invalid RLE encoding');\n }\n\n return values;\n}\n\nfunction decodeRunBitpacked(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n if (count % 8 !== 0) {\n throw new Error('must be a multiple of 8');\n }\n\n // tslint:disable-next-line:prefer-array-literal\n const values = new Array(count).fill(0);\n for (let b = 0; b < bitWidth * count; b++) {\n if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n }\n }\n\n cursor.offset += bitWidth * (count / 8);\n return values;\n}\n\nfunction decodeRunRepeated(\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): number[] {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n let value = 0;\n for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n value <<= 8;\n value += cursor.buffer[cursor.offset];\n cursor.offset += 1;\n }\n\n // tslint:disable-next-line:prefer-array-literal\n return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n for (let i = 0; i < values.length % 8; i++) {\n values.push(0);\n }\n\n const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n for (let b = 0; b < bitWidth * values.length; b++) {\n if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n buf[Math.floor(b / 8)] |= 1 << b % 8;\n }\n }\n\n return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n // @ts-ignore\n const bitWidth: number = opts.bitWidth;\n\n const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n for (let i = 0; i < buf.length; i++) {\n buf.writeUInt8(value & 0xff, i);\n value >>= 8;\n }\n\n return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
@@ -2,6 +2,8 @@
2
2
 
3
3
  var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
4
 
5
+ var _typeof = require("@babel/runtime/helpers/typeof");
6
+
5
7
  Object.defineProperty(exports, "__esModule", {
6
8
  value: true
7
9
  });
@@ -15,14 +17,14 @@ var _zlib = _interopRequireDefault(require("zlib"));
15
17
 
16
18
  var _snappyjs = _interopRequireDefault(require("snappyjs"));
17
19
 
18
- function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
20
+ function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
19
21
 
20
- function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
22
+ function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
21
23
 
22
- let brotli;
23
- let lzo;
24
- let lz4js;
25
- const PARQUET_COMPRESSION_METHODS = {
24
+ var brotli;
25
+ var lzo;
26
+ var lz4js;
27
+ var PARQUET_COMPRESSION_METHODS = {
26
28
  UNCOMPRESSED: {
27
29
  deflate: deflate_identity,
28
30
  inflate: inflate_identity
@@ -77,7 +79,7 @@ function deflate_lzo(value) {
77
79
 
78
80
  function deflate_brotli(value) {
79
81
  brotli = brotli || Util.load('brotli');
80
- const result = brotli.compress(value, {
82
+ var result = brotli.compress(value, {
81
83
  mode: 0,
82
84
  quality: 8,
83
85
  lgwin: 22
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["brotli","lzo","lz4js","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","deflate","deflate_identity","inflate","inflate_identity","GZIP","deflate_gzip","inflate_gzip","SNAPPY","deflate_snappy","inflate_snappy","LZO","deflate_lzo","inflate_lzo","BROTLI","deflate_brotli","inflate_brotli","LZ4","deflate_lz4","inflate_lz4","method","value","Error","zlib","gzipSync","snappyjs","compress","Util","load","result","mode","quality","lgwin","Buffer","from","alloc","err","size","gunzipSync","uncompress","decompress","length"],"mappings":";;;;;;;;;;;AAGA;;AACA;;AACA;;;;;;AAEA,IAAIA,MAAJ;AACA,IAAIC,GAAJ;AACA,IAAIC,KAAJ;AAOO,MAAMC,2BAA8E,GAAG;AAC5FC,EAAAA,YAAY,EAAE;AACZC,IAAAA,OAAO,EAAEC,gBADG;AAEZC,IAAAA,OAAO,EAAEC;AAFG,GAD8E;AAK5FC,EAAAA,IAAI,EAAE;AACJJ,IAAAA,OAAO,EAAEK,YADL;AAEJH,IAAAA,OAAO,EAAEI;AAFL,GALsF;AAS5FC,EAAAA,MAAM,EAAE;AACNP,IAAAA,OAAO,EAAEQ,cADH;AAENN,IAAAA,OAAO,EAAEO;AAFH,GAToF;AAa5FC,EAAAA,GAAG,EAAE;AACHV,IAAAA,OAAO,EAAEW,WADN;AAEHT,IAAAA,OAAO,EAAEU;AAFN,GAbuF;AAiB5FC,EAAAA,MAAM,EAAE;AACNb,IAAAA,OAAO,EAAEc,cADH;AAENZ,IAAAA,OAAO,EAAEa;AAFH,GAjBoF;AAqB5FC,EAAAA,GAAG,EAAE;AACHhB,IAAAA,OAAO,EAAEiB,WADN;AAEHf,IAAAA,OAAO,EAAEgB;AAFN;AArBuF,CAAvF;;;AA8BA,SAASlB,OAAT,CAAiBmB,MAAjB,EAA6CC,KAA7C,EAAoE;AACzE,MAAI,EAAED,MAAM,IAAIrB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIuB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOrB,2BAA2B,CAACqB,MAAD,CAA3B,CAAoCnB,OAApC,CAA4CoB,KAA5C,CAAP;AACD;;AAED,SAASnB,gBAAT,CAA0BmB,KAA1B,EAAiD;AAC/C,SAAOA,KAAP;AACD;;AAED,SAASf,YAAT,CAAsBe,KAAtB,EAA6C;AAC3C,SAAOE,cAAKC,QAAL,CAAcH,KAAd,CAAP;AACD;;AAED,SAASZ,cAAT,CAAwBY,KAAxB,EAA+C;AAC7C,SAAOI,kBAASC,QAAT,CAAkBL,KAAlB,CAAP;AACD;;AAED,SAAST,WAAT,CAAqBS,KAArB,EAA4C;AAC1CxB,EAAAA,GAAG,GAAGA,GAAG,IAAI8B,IAAI,CAACC,IAAL,CAAU,KAAV,CAAb;AACA,SAAO/B,GAAG,CAAC6B,QAAJ,CAAaL,KAAb,CAAP;AACD;;AAED,SAASN,cAAT,CAAwBM,KAAxB,EAA+C;AAC7CzB,EAAAA,MAAM,GAAGA,MAAM,IAAI+B,IAAI,CAACC,IAAL,CAAU,QAAV,CAAnB;AACA,QAAMC,MAAM,GAAGjC,MAAM,CAAC8B,QAAP,CAAgBL,KAAhB,EAAuB;AACpCS,IAAAA,IAAI,EAAE,CAD8B;AAEpCC,IAAAA,OAAO,EAAE,CAF2B;AAGpCC,IAAAA,KAAK,EAAE;AAH6B,GAAvB,CAAf;AAKA,SAAOH,MAAM,GAAGI,MAAM,CAACC,IAAP,CAAYL,MAAZ,CAAH,GAAyBI,MAAM,CAACE,KAAP,CAAa,CAAb,CAAtC;AACD;;AAED,SAASjB,WAAT,CAAqBG,KAArB,EAA4C;AAC1CvB,EAAAA,KAAK,GAAGA,KAAK,IAAI6B,IAAI,CAACC,IAAL,CAAU,OAAV,CAAjB;;AACA,MAAI;AAMF,WAAOK,MAAM,CAACC,IAAP,CAAYpC,KAAK,CAAC4B,QAAN,CAAeL,KAAf,CAAZ,CAAP;AACD,GAPD,CAOE,OAAOe,GAAP,EAAY;AACZ,UAAMA,GAAN;AACD;AACF;;AAKM,SAASjC,OAAT,CAAiBiB,MAAjB,EAA6CC,KAA7C,EAA4DgB,IAA5D,EAAkF;AACvF,MAAI,EAAEjB,MAAM,IAAIrB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIuB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOrB,2BAA2B,CAACqB,MAAD,CAA3B,CAAoCjB,OAApC,CAA4CkB,KAA5C,EAAmDgB,IAAnD,CAAP;AACD;;AAED,SAASjC,gBAAT,CAA0BiB,KAA1B,EAAiD;AAC/C,SAAOA,KAAP;AACD;;AAED,SAASd,YAAT,CAAsBc,KAAtB,EAA6C;AAC3C,SAAOE,cAAKe,UAAL,CAAgBjB,KAAhB,CAAP;AACD;;AAED,SAASX,cAAT,CAAwBW,KAAxB,EAA+C;AAC7C,SAAOI,kBAASc,UAAT,CAAoBlB,KAApB,CAAP;AACD;;AAED,SAASR,WAAT,CAAqBQ,KAArB,EAAoCgB,IAApC,EAA0D;AACxDxC,EAAAA,GAAG,GAAGA,GAAG,IAAI8B,IAAI,CAACC,IAAL,CAAU,KAAV,CAAb;AACA,SAAO/B,GAAG,CAAC2C,UAAJ,CAAenB,KAAf,EAAsBgB,IAAtB,CAAP;AACD;;AAED,SAASlB,WAAT,CAAqBE,KAArB,EAAoCgB,IAApC,EAA0D;AACxDvC,EAAAA,KAAK,GAAGA,KAAK,IAAI6B,IAAI,CAACC,IAAL,CAAU,OAAV,CAAjB;;AACA,MAAI;AAMF,WAAOK,MAAM,CAACC,IAAP,CAAYpC,KAAK,CAAC0C,UAAN,CAAiBnB,KAAjB,EAAwBgB,IAAxB,CAAZ,CAAP;AACD,GAPD,CAOE,OAAOD,GAAP,EAAY;AACZ,UAAMA,GAAN;AACD;AACF;;AAED,SAASpB,cAAT,CAAwBK,KAAxB,EAA+C;AAC7CzB,EAAAA,MAAM,GAAGA,MAAM,IAAI+B,IAAI,CAACC,IAAL,CAAU,QAAV,CAAnB;;AACA,MAAI,CAACP,KAAK,CAACoB,MAAX,EAAmB;AACjB,WAAOR,MAAM,CAACE,KAAP,CAAa,CAAb,CAAP;AACD;;AACD,SAAOF,MAAM,CAACC,IAAP,CAAYtC,MAAM,CAAC4C,UAAP,CAAkBnB,KAAlB,CAAZ,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {ParquetCompression} from './schema/declare';\nimport * as Util from './util';\nimport zlib from 'zlib';\nimport snappyjs from 'snappyjs';\n\nlet brotli: any;\nlet lzo: any;\nlet lz4js: any;\n\nexport interface ParquetCompressionKit {\n deflate: (value: Buffer) => Buffer;\n inflate: (value: Buffer, size: number) => Buffer;\n}\n\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, ParquetCompressionKit> = {\n UNCOMPRESSED: {\n deflate: deflate_identity,\n inflate: inflate_identity\n },\n GZIP: {\n deflate: deflate_gzip,\n inflate: inflate_gzip\n },\n SNAPPY: {\n deflate: deflate_snappy,\n inflate: inflate_snappy\n },\n LZO: {\n deflate: deflate_lzo,\n inflate: inflate_lzo\n },\n BROTLI: {\n deflate: deflate_brotli,\n inflate: inflate_brotli\n },\n LZ4: {\n deflate: deflate_lz4,\n inflate: inflate_lz4\n }\n};\n\n/**\n * Deflate a value using compression method `method`\n */\nexport function deflate(method: ParquetCompression, value: Buffer): Buffer {\n if (!(method in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`invalid compression method: ${method}`);\n }\n\n return PARQUET_COMPRESSION_METHODS[method].deflate(value);\n}\n\nfunction deflate_identity(value: Buffer): Buffer {\n return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n lzo = lzo || Util.load('lzo');\n return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n brotli = brotli || Util.load('brotli');\n const result = brotli.compress(value, {\n mode: 0,\n quality: 8,\n lgwin: 22\n });\n return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n lz4js = lz4js || Util.load('lz4js');\n try {\n // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n // const compressedSize = lz4.encodeBlock(value, result);\n // // remove unnecessary bytes\n // result = result.slice(0, compressedSize);\n // return result;\n return Buffer.from(lz4js.compress(value));\n } catch (err) {\n throw err;\n }\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n if (!(method in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`invalid compression method: ${method}`);\n }\n\n return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\nfunction inflate_identity(value: Buffer): Buffer {\n return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n lzo = lzo || Util.load('lzo');\n return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n lz4js = lz4js || Util.load('lz4js');\n try {\n // let result = Buffer.alloc(size);\n // const uncompressedSize = lz4js.decodeBlock(value, result);\n // // remove unnecessary bytes\n // result = result.slice(0, uncompressedSize);\n // return result;\n return Buffer.from(lz4js.decompress(value, size));\n } catch (err) {\n throw err;\n }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n brotli = brotli || Util.load('brotli');\n if (!value.length) {\n return Buffer.alloc(0);\n }\n return Buffer.from(brotli.decompress(value));\n}\n"],"file":"compression.js"}
1
+ {"version":3,"sources":["../../../src/parquetjs/compression.ts"],"names":["brotli","lzo","lz4js","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","deflate","deflate_identity","inflate","inflate_identity","GZIP","deflate_gzip","inflate_gzip","SNAPPY","deflate_snappy","inflate_snappy","LZO","deflate_lzo","inflate_lzo","BROTLI","deflate_brotli","inflate_brotli","LZ4","deflate_lz4","inflate_lz4","method","value","Error","zlib","gzipSync","snappyjs","compress","Util","load","result","mode","quality","lgwin","Buffer","from","alloc","err","size","gunzipSync","uncompress","decompress","length"],"mappings":";;;;;;;;;;;;;AAGA;;AACA;;AACA;;;;;;AAEA,IAAIA,MAAJ;AACA,IAAIC,GAAJ;AACA,IAAIC,KAAJ;AAOO,IAAMC,2BAA8E,GAAG;AAC5FC,EAAAA,YAAY,EAAE;AACZC,IAAAA,OAAO,EAAEC,gBADG;AAEZC,IAAAA,OAAO,EAAEC;AAFG,GAD8E;AAK5FC,EAAAA,IAAI,EAAE;AACJJ,IAAAA,OAAO,EAAEK,YADL;AAEJH,IAAAA,OAAO,EAAEI;AAFL,GALsF;AAS5FC,EAAAA,MAAM,EAAE;AACNP,IAAAA,OAAO,EAAEQ,cADH;AAENN,IAAAA,OAAO,EAAEO;AAFH,GAToF;AAa5FC,EAAAA,GAAG,EAAE;AACHV,IAAAA,OAAO,EAAEW,WADN;AAEHT,IAAAA,OAAO,EAAEU;AAFN,GAbuF;AAiB5FC,EAAAA,MAAM,EAAE;AACNb,IAAAA,OAAO,EAAEc,cADH;AAENZ,IAAAA,OAAO,EAAEa;AAFH,GAjBoF;AAqB5FC,EAAAA,GAAG,EAAE;AACHhB,IAAAA,OAAO,EAAEiB,WADN;AAEHf,IAAAA,OAAO,EAAEgB;AAFN;AArBuF,CAAvF;;;AA8BA,SAASlB,OAAT,CAAiBmB,MAAjB,EAA6CC,KAA7C,EAAoE;AACzE,MAAI,EAAED,MAAM,IAAIrB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIuB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOrB,2BAA2B,CAACqB,MAAD,CAA3B,CAAoCnB,OAApC,CAA4CoB,KAA5C,CAAP;AACD;;AAED,SAASnB,gBAAT,CAA0BmB,KAA1B,EAAiD;AAC/C,SAAOA,KAAP;AACD;;AAED,SAASf,YAAT,CAAsBe,KAAtB,EAA6C;AAC3C,SAAOE,cAAKC,QAAL,CAAcH,KAAd,CAAP;AACD;;AAED,SAASZ,cAAT,CAAwBY,KAAxB,EAA+C;AAC7C,SAAOI,kBAASC,QAAT,CAAkBL,KAAlB,CAAP;AACD;;AAED,SAAST,WAAT,CAAqBS,KAArB,EAA4C;AAC1CxB,EAAAA,GAAG,GAAGA,GAAG,IAAI8B,IAAI,CAACC,IAAL,CAAU,KAAV,CAAb;AACA,SAAO/B,GAAG,CAAC6B,QAAJ,CAAaL,KAAb,CAAP;AACD;;AAED,SAASN,cAAT,CAAwBM,KAAxB,EAA+C;AAC7CzB,EAAAA,MAAM,GAAGA,MAAM,IAAI+B,IAAI,CAACC,IAAL,CAAU,QAAV,CAAnB;AACA,MAAMC,MAAM,GAAGjC,MAAM,CAAC8B,QAAP,CAAgBL,KAAhB,EAAuB;AACpCS,IAAAA,IAAI,EAAE,CAD8B;AAEpCC,IAAAA,OAAO,EAAE,CAF2B;AAGpCC,IAAAA,KAAK,EAAE;AAH6B,GAAvB,CAAf;AAKA,SAAOH,MAAM,GAAGI,MAAM,CAACC,IAAP,CAAYL,MAAZ,CAAH,GAAyBI,MAAM,CAACE,KAAP,CAAa,CAAb,CAAtC;AACD;;AAED,SAASjB,WAAT,CAAqBG,KAArB,EAA4C;AAC1CvB,EAAAA,KAAK,GAAGA,KAAK,IAAI6B,IAAI,CAACC,IAAL,CAAU,OAAV,CAAjB;;AACA,MAAI;AAMF,WAAOK,MAAM,CAACC,IAAP,CAAYpC,KAAK,CAAC4B,QAAN,CAAeL,KAAf,CAAZ,CAAP;AACD,GAPD,CAOE,OAAOe,GAAP,EAAY;AACZ,UAAMA,GAAN;AACD;AACF;;AAKM,SAASjC,OAAT,CAAiBiB,MAAjB,EAA6CC,KAA7C,EAA4DgB,IAA5D,EAAkF;AACvF,MAAI,EAAEjB,MAAM,IAAIrB,2BAAZ,CAAJ,EAA8C;AAC5C,UAAM,IAAIuB,KAAJ,uCAAyCF,MAAzC,EAAN;AACD;;AAED,SAAOrB,2BAA2B,CAACqB,MAAD,CAA3B,CAAoCjB,OAApC,CAA4CkB,KAA5C,EAAmDgB,IAAnD,CAAP;AACD;;AAED,SAASjC,gBAAT,CAA0BiB,KAA1B,EAAiD;AAC/C,SAAOA,KAAP;AACD;;AAED,SAASd,YAAT,CAAsBc,KAAtB,EAA6C;AAC3C,SAAOE,cAAKe,UAAL,CAAgBjB,KAAhB,CAAP;AACD;;AAED,SAASX,cAAT,CAAwBW,KAAxB,EAA+C;AAC7C,SAAOI,kBAASc,UAAT,CAAoBlB,KAApB,CAAP;AACD;;AAED,SAASR,WAAT,CAAqBQ,KAArB,EAAoCgB,IAApC,EAA0D;AACxDxC,EAAAA,GAAG,GAAGA,GAAG,IAAI8B,IAAI,CAACC,IAAL,CAAU,KAAV,CAAb;AACA,SAAO/B,GAAG,CAAC2C,UAAJ,CAAenB,KAAf,EAAsBgB,IAAtB,CAAP;AACD;;AAED,SAASlB,WAAT,CAAqBE,KAArB,EAAoCgB,IAApC,EAA0D;AACxDvC,EAAAA,KAAK,GAAGA,KAAK,IAAI6B,IAAI,CAACC,IAAL,CAAU,OAAV,CAAjB;;AACA,MAAI;AAMF,WAAOK,MAAM,CAACC,IAAP,CAAYpC,KAAK,CAAC0C,UAAN,CAAiBnB,KAAjB,EAAwBgB,IAAxB,CAAZ,CAAP;AACD,GAPD,CAOE,OAAOD,GAAP,EAAY;AACZ,UAAMA,GAAN;AACD;AACF;;AAED,SAASpB,cAAT,CAAwBK,KAAxB,EAA+C;AAC7CzB,EAAAA,MAAM,GAAGA,MAAM,IAAI+B,IAAI,CAACC,IAAL,CAAU,QAAV,CAAnB;;AACA,MAAI,CAACP,KAAK,CAACoB,MAAX,EAAmB;AACjB,WAAOR,MAAM,CAACE,KAAP,CAAa,CAAb,CAAP;AACD;;AACD,SAAOF,MAAM,CAACC,IAAP,CAAYtC,MAAM,CAAC4C,UAAP,CAAkBnB,KAAlB,CAAZ,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {ParquetCompression} from './schema/declare';\nimport * as Util from './util';\nimport zlib from 'zlib';\nimport snappyjs from 'snappyjs';\n\nlet brotli: any;\nlet lzo: any;\nlet lz4js: any;\n\nexport interface ParquetCompressionKit {\n deflate: (value: Buffer) => Buffer;\n inflate: (value: Buffer, size: number) => Buffer;\n}\n\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, ParquetCompressionKit> = {\n UNCOMPRESSED: {\n deflate: deflate_identity,\n inflate: inflate_identity\n },\n GZIP: {\n deflate: deflate_gzip,\n inflate: inflate_gzip\n },\n SNAPPY: {\n deflate: deflate_snappy,\n inflate: inflate_snappy\n },\n LZO: {\n deflate: deflate_lzo,\n inflate: inflate_lzo\n },\n BROTLI: {\n deflate: deflate_brotli,\n inflate: inflate_brotli\n },\n LZ4: {\n deflate: deflate_lz4,\n inflate: inflate_lz4\n }\n};\n\n/**\n * Deflate a value using compression method `method`\n */\nexport function deflate(method: ParquetCompression, value: Buffer): Buffer {\n if (!(method in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`invalid compression method: ${method}`);\n }\n\n return PARQUET_COMPRESSION_METHODS[method].deflate(value);\n}\n\nfunction deflate_identity(value: Buffer): Buffer {\n return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n lzo = lzo || Util.load('lzo');\n return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n brotli = brotli || Util.load('brotli');\n const result = brotli.compress(value, {\n mode: 0,\n quality: 8,\n lgwin: 22\n });\n return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n lz4js = lz4js || Util.load('lz4js');\n try {\n // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n // const compressedSize = lz4.encodeBlock(value, result);\n // // remove unnecessary bytes\n // result = result.slice(0, compressedSize);\n // return result;\n return Buffer.from(lz4js.compress(value));\n } catch (err) {\n throw err;\n }\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n if (!(method in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`invalid compression method: ${method}`);\n }\n\n return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\nfunction inflate_identity(value: Buffer): Buffer {\n return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n lzo = lzo || Util.load('lzo');\n return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n lz4js = lz4js || Util.load('lz4js');\n try {\n // let result = Buffer.alloc(size);\n // const uncompressedSize = lz4js.decodeBlock(value, result);\n // // remove unnecessary bytes\n // result = result.slice(0, uncompressedSize);\n // return result;\n return Buffer.from(lz4js.decompress(value, size));\n } catch (err) {\n throw err;\n }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n brotli = brotli || Util.load('brotli');\n if (!value.length) {\n return Buffer.alloc(0);\n }\n return Buffer.from(brotli.decompress(value));\n}\n"],"file":"compression.js"}
@@ -16,8 +16,8 @@ exports.osopen = osopen;
16
16
  var _fs = _interopRequireDefault(require("fs"));
17
17
 
18
18
  function fopen(filePath) {
19
- return new Promise((resolve, reject) => {
20
- _fs.default.open(filePath, 'r', (err, fd) => {
19
+ return new Promise(function (resolve, reject) {
20
+ _fs.default.open(filePath, 'r', function (err, fd) {
21
21
  if (err) {
22
22
  reject(err);
23
23
  } else {
@@ -28,8 +28,8 @@ function fopen(filePath) {
28
28
  }
29
29
 
30
30
  function fstat(filePath) {
31
- return new Promise((resolve, reject) => {
32
- _fs.default.stat(filePath, (err, stat) => {
31
+ return new Promise(function (resolve, reject) {
32
+ _fs.default.stat(filePath, function (err, stat) {
33
33
  if (err) {
34
34
  reject(err);
35
35
  } else {
@@ -40,9 +40,9 @@ function fstat(filePath) {
40
40
  }
41
41
 
42
42
  function fread(fd, position, length) {
43
- const buffer = Buffer.alloc(length);
44
- return new Promise((resolve, reject) => {
45
- _fs.default.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
43
+ var buffer = Buffer.alloc(length);
44
+ return new Promise(function (resolve, reject) {
45
+ _fs.default.read(fd, buffer, 0, length, position, function (err, bytesRead, buf) {
46
46
  if (err || bytesRead !== length) {
47
47
  reject(err || Error('read failed'));
48
48
  } else {
@@ -53,8 +53,8 @@ function fread(fd, position, length) {
53
53
  }
54
54
 
55
55
  function fclose(fd) {
56
- return new Promise((resolve, reject) => {
57
- _fs.default.close(fd, err => {
56
+ return new Promise(function (resolve, reject) {
57
+ _fs.default.close(fd, function (err) {
58
58
  if (err) {
59
59
  reject(err);
60
60
  } else {
@@ -65,8 +65,8 @@ function fclose(fd) {
65
65
  }
66
66
 
67
67
  function oswrite(os, buf) {
68
- return new Promise((resolve, reject) => {
69
- os.write(buf, err => {
68
+ return new Promise(function (resolve, reject) {
69
+ os.write(buf, function (err) {
70
70
  if (err) {
71
71
  reject(err);
72
72
  } else {
@@ -77,8 +77,8 @@ function oswrite(os, buf) {
77
77
  }
78
78
 
79
79
  function osclose(os) {
80
- return new Promise((resolve, reject) => {
81
- os.close(err => {
80
+ return new Promise(function (resolve, reject) {
81
+ os.close(function (err) {
82
82
  if (err) {
83
83
  reject(err);
84
84
  } else {
@@ -89,8 +89,8 @@ function osclose(os) {
89
89
  }
90
90
 
91
91
  function osopen(path, opts) {
92
- return new Promise((resolve, reject) => {
93
- const outputStream = _fs.default.createWriteStream(path, opts);
92
+ return new Promise(function (resolve, reject) {
93
+ var outputStream = _fs.default.createWriteStream(path, opts);
94
94
 
95
95
  outputStream.on('open', function (fd) {
96
96
  resolve(outputStream);
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/parquetjs/file.ts"],"names":["fopen","filePath","Promise","resolve","reject","fs","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","on"],"mappings":";;;;;;;;;;;;;;;AACA;;AAEO,SAASA,KAAT,CAAeC,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCC,gBAAGC,IAAH,CAAQL,QAAR,EAAkB,GAAlB,EAAuB,CAACM,GAAD,EAAMC,EAAN,KAAa;AAClC,UAAID,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACK,EAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASC,KAAT,CAAeR,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAsB,CAACC,OAAD,EAAUC,MAAV,KAAqB;AAChDC,gBAAGK,IAAH,CAAQT,QAAR,EAAkB,CAACM,GAAD,EAAMG,IAAN,KAAe;AAC/B,UAAIH,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACO,IAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASC,KAAT,CAAeH,EAAf,EAAmBI,QAAnB,EAA6BC,MAA7B,EAAqC;AAC1C,QAAMC,MAAM,GAAGC,MAAM,CAACC,KAAP,CAAaH,MAAb,CAAf;AAEA,SAAO,IAAIX,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCC,gBAAGY,IAAH,CAAQT,EAAR,EAAYM,MAAZ,EAAoB,CAApB,EAAuBD,MAAvB,EAA+BD,QAA/B,EAAyC,CAACL,GAAD,EAAMW,SAAN,EAAiBC,GAAjB,KAAyB;AAChE,UAAIZ,GAAG,IAAIW,SAAS,KAAKL,MAAzB,EAAiC;AAC/BT,QAAAA,MAAM,CAACG,GAAG,IAAIa,KAAK,CAAC,aAAD,CAAb,CAAN;AACD,OAFD,MAEO;AACLjB,QAAAA,OAAO,CAACgB,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASE,MAAT,CAAgBb,EAAhB,EAAoB;AACzB,SAAO,IAAIN,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCC,gBAAGiB,KAAH,CAASd,EAAT,EAAcD,GAAD,IAAS;AACpB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACI,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASgB,OAAT,CAAiBC,EAAjB,EAAqBL,GAArB,EAAyC;AAC9C,SAAO,IAAIjB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCoB,IAAAA,EAAE,CAACC,KAAH,CAASN,GAAT,EAAeZ,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASuB,OAAT,CAAiBF,EAAjB,EAAoC;AACzC,SAAO,IAAItB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCoB,IAAAA,EAAE,CAACF,KAAH,CAAUf,GAAD,IAAS;AAChB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASwB,MAAT,CAAgBC,IAAhB,EAAsBC,IAAtB,EAA4B;AACjC,SAAO,IAAI3B,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAM0B,YAAY,GAAGzB,YAAG0B,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;;AAEAC,IAAAA,YAAY,CAACE,EAAb,CAAgB,MAAhB,EAAwB,UAAUxB,EAAV,EAAc;AACpCL,MAAAA,OAAO,CAAC2B,YAAD,CAAP;AACD,KAFD;AAIAA,IAAAA,YAAY,CAACE,EAAb,CAAgB,OAAhB,EAAyB,UAAUzB,GAAV,EAAe;AACtCH,MAAAA,MAAM,CAACG,GAAD,CAAN;AACD,KAFD;AAGD,GAVM,CAAP;AAWD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\n\nexport function fopen(filePath) {\n return new Promise((resolve, reject) => {\n fs.open(filePath, 'r', (err, fd) => {\n if (err) {\n reject(err);\n } else {\n resolve(fd);\n }\n });\n });\n}\n\nexport function fstat(filePath) {\n return new Promise<fs.Stats>((resolve, reject) => {\n fs.stat(filePath, (err, stat) => {\n if (err) {\n reject(err);\n } else {\n resolve(stat);\n }\n });\n });\n}\n\nexport function fread(fd, position, length) {\n const buffer = Buffer.alloc(length);\n\n return new Promise((resolve, reject) => {\n fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {\n if (err || bytesRead !== length) {\n reject(err || Error('read failed'));\n } else {\n resolve(buf);\n }\n });\n });\n}\n\nexport function fclose(fd) {\n return new Promise((resolve, reject) => {\n fs.close(fd, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve(err);\n }\n });\n });\n}\n\nexport function oswrite(os, buf): Promise<void> {\n return new Promise((resolve, reject) => {\n os.write(buf, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osclose(os): Promise<void> {\n return new Promise((resolve, reject) => {\n os.close((err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osopen(path, opts) {\n return new Promise((resolve, reject) => {\n const outputStream = fs.createWriteStream(path, opts);\n\n outputStream.on('open', function (fd) {\n resolve(outputStream);\n });\n\n outputStream.on('error', function (err) {\n reject(err);\n });\n });\n}\n"],"file":"file.js"}
1
+ {"version":3,"sources":["../../../src/parquetjs/file.ts"],"names":["fopen","filePath","Promise","resolve","reject","fs","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","on"],"mappings":";;;;;;;;;;;;;;;AACA;;AAEO,SAASA,KAAT,CAAeC,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCC,gBAAGC,IAAH,CAAQL,QAAR,EAAkB,GAAlB,EAAuB,UAACM,GAAD,EAAMC,EAAN,EAAa;AAClC,UAAID,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACK,EAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASC,KAAT,CAAeR,QAAf,EAAyB;AAC9B,SAAO,IAAIC,OAAJ,CAAsB,UAACC,OAAD,EAAUC,MAAV,EAAqB;AAChDC,gBAAGK,IAAH,CAAQT,QAAR,EAAkB,UAACM,GAAD,EAAMG,IAAN,EAAe;AAC/B,UAAIH,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACO,IAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASC,KAAT,CAAeH,EAAf,EAAmBI,QAAnB,EAA6BC,MAA7B,EAAqC;AAC1C,MAAMC,MAAM,GAAGC,MAAM,CAACC,KAAP,CAAaH,MAAb,CAAf;AAEA,SAAO,IAAIX,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCC,gBAAGY,IAAH,CAAQT,EAAR,EAAYM,MAAZ,EAAoB,CAApB,EAAuBD,MAAvB,EAA+BD,QAA/B,EAAyC,UAACL,GAAD,EAAMW,SAAN,EAAiBC,GAAjB,EAAyB;AAChE,UAAIZ,GAAG,IAAIW,SAAS,KAAKL,MAAzB,EAAiC;AAC/BT,QAAAA,MAAM,CAACG,GAAG,IAAIa,KAAK,CAAC,aAAD,CAAb,CAAN;AACD,OAFD,MAEO;AACLjB,QAAAA,OAAO,CAACgB,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASE,MAAT,CAAgBb,EAAhB,EAAoB;AACzB,SAAO,IAAIN,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCC,gBAAGiB,KAAH,CAASd,EAAT,EAAa,UAACD,GAAD,EAAS;AACpB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO,CAACI,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASgB,OAAT,CAAiBC,EAAjB,EAAqBL,GAArB,EAAyC;AAC9C,SAAO,IAAIjB,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCoB,IAAAA,EAAE,CAACC,KAAH,CAASN,GAAT,EAAc,UAACZ,GAAD,EAAS;AACrB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASuB,OAAT,CAAiBF,EAAjB,EAAoC;AACzC,SAAO,IAAItB,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtCoB,IAAAA,EAAE,CAACF,KAAH,CAAS,UAACf,GAAD,EAAS;AAChB,UAAIA,GAAJ,EAAS;AACPH,QAAAA,MAAM,CAACG,GAAD,CAAN;AACD,OAFD,MAEO;AACLJ,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;;AAEM,SAASwB,MAAT,CAAgBC,IAAhB,EAAsBC,IAAtB,EAA4B;AACjC,SAAO,IAAI3B,OAAJ,CAAY,UAACC,OAAD,EAAUC,MAAV,EAAqB;AACtC,QAAM0B,YAAY,GAAGzB,YAAG0B,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;;AAEAC,IAAAA,YAAY,CAACE,EAAb,CAAgB,MAAhB,EAAwB,UAAUxB,EAAV,EAAc;AACpCL,MAAAA,OAAO,CAAC2B,YAAD,CAAP;AACD,KAFD;AAIAA,IAAAA,YAAY,CAACE,EAAb,CAAgB,OAAhB,EAAyB,UAAUzB,GAAV,EAAe;AACtCH,MAAAA,MAAM,CAACG,GAAD,CAAN;AACD,KAFD;AAGD,GAVM,CAAP;AAWD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\n\nexport function fopen(filePath) {\n return new Promise((resolve, reject) => {\n fs.open(filePath, 'r', (err, fd) => {\n if (err) {\n reject(err);\n } else {\n resolve(fd);\n }\n });\n });\n}\n\nexport function fstat(filePath) {\n return new Promise<fs.Stats>((resolve, reject) => {\n fs.stat(filePath, (err, stat) => {\n if (err) {\n reject(err);\n } else {\n resolve(stat);\n }\n });\n });\n}\n\nexport function fread(fd, position, length) {\n const buffer = Buffer.alloc(length);\n\n return new Promise((resolve, reject) => {\n fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {\n if (err || bytesRead !== length) {\n reject(err || Error('read failed'));\n } else {\n resolve(buf);\n }\n });\n });\n}\n\nexport function fclose(fd) {\n return new Promise((resolve, reject) => {\n fs.close(fd, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve(err);\n }\n });\n });\n}\n\nexport function oswrite(os, buf): Promise<void> {\n return new Promise((resolve, reject) => {\n os.write(buf, (err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osclose(os): Promise<void> {\n return new Promise((resolve, reject) => {\n os.close((err) => {\n if (err) {\n reject(err);\n } else {\n resolve();\n }\n });\n });\n}\n\nexport function osopen(path, opts) {\n return new Promise((resolve, reject) => {\n const outputStream = fs.createWriteStream(path, opts);\n\n outputStream.on('open', function (fd) {\n resolve(outputStream);\n });\n\n outputStream.on('error', function (err) {\n reject(err);\n });\n });\n}\n"],"file":"file.js"}
@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
4
4
  value: true
5
5
  });
6
6
  exports.BoundaryOrder = void 0;
7
- let BoundaryOrder;
7
+ var BoundaryOrder;
8
8
  exports.BoundaryOrder = BoundaryOrder;
9
9
 
10
10
  (function (BoundaryOrder) {
@@ -1,53 +1,67 @@
1
1
  "use strict";
2
2
 
3
+ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
+
5
+ var _typeof = require("@babel/runtime/helpers/typeof");
6
+
3
7
  Object.defineProperty(exports, "__esModule", {
4
8
  value: true
5
9
  });
6
10
  exports.BsonType = void 0;
7
11
 
8
- var thrift = _interopRequireWildcard(require("thrift"));
12
+ var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
9
13
 
10
- function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
14
+ var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
11
15
 
12
- function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
16
+ var thrift = _interopRequireWildcard(require("thrift"));
17
+
18
+ function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
13
19
 
14
- class BsonType {
15
- constructor() {}
20
+ function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
16
21
 
17
- write(output) {
18
- output.writeStructBegin('BsonType');
19
- output.writeFieldStop();
20
- output.writeStructEnd();
21
- return;
22
+ var BsonType = function () {
23
+ function BsonType() {
24
+ (0, _classCallCheck2.default)(this, BsonType);
22
25
  }
23
26
 
24
- static read(input) {
25
- input.readStructBegin();
27
+ (0, _createClass2.default)(BsonType, [{
28
+ key: "write",
29
+ value: function write(output) {
30
+ output.writeStructBegin('BsonType');
31
+ output.writeFieldStop();
32
+ output.writeStructEnd();
33
+ return;
34
+ }
35
+ }], [{
36
+ key: "read",
37
+ value: function read(input) {
38
+ input.readStructBegin();
26
39
 
27
- while (true) {
28
- const ret = input.readFieldBegin();
29
- const fieldType = ret.ftype;
30
- const fieldId = ret.fid;
40
+ while (true) {
41
+ var ret = input.readFieldBegin();
42
+ var fieldType = ret.ftype;
43
+ var fieldId = ret.fid;
31
44
 
32
- if (fieldType === thrift.Thrift.Type.STOP) {
33
- break;
34
- }
45
+ if (fieldType === thrift.Thrift.Type.STOP) {
46
+ break;
47
+ }
48
+
49
+ switch (fieldId) {
50
+ default:
51
+ {
52
+ input.skip(fieldType);
53
+ }
54
+ }
35
55
 
36
- switch (fieldId) {
37
- default:
38
- {
39
- input.skip(fieldType);
40
- }
56
+ input.readFieldEnd();
41
57
  }
42
58
 
43
- input.readFieldEnd();
59
+ input.readStructEnd();
60
+ return new BsonType();
44
61
  }
45
-
46
- input.readStructEnd();
47
- return new BsonType();
48
- }
49
-
50
- }
62
+ }]);
63
+ return BsonType;
64
+ }();
51
65
 
52
66
  exports.BsonType = BsonType;
53
67
  //# sourceMappingURL=BsonType.js.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../../src/parquetjs/parquet-thrift/BsonType.ts"],"names":["BsonType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","thrift","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"mappings":";;;;;;;AAMA;;;;;;AAEO,MAAMA,QAAN,CAAe;AACpBC,EAAAA,WAAW,GAAG,CAAE;;AACTC,EAAAA,KAAK,CAACC,MAAD,EAAiC;AAC3CA,IAAAA,MAAM,CAACC,gBAAP,CAAwB,UAAxB;AACAD,IAAAA,MAAM,CAACE,cAAP;AACAF,IAAAA,MAAM,CAACG,cAAP;AACA;AACD;;AACiB,SAAJC,IAAI,CAACC,KAAD,EAAoC;AACpDA,IAAAA,KAAK,CAACC,eAAN;;AACA,WAAO,IAAP,EAAa;AACX,YAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAN,EAA3B;AACA,YAAMC,SAA6B,GAAGF,GAAG,CAACG,KAA1C;AACA,YAAMC,OAAe,GAAGJ,GAAG,CAACK,GAA5B;;AACA,UAAIH,SAAS,KAAKI,MAAM,CAACC,MAAP,CAAcC,IAAd,CAAmBC,IAArC,EAA2C;AACzC;AACD;;AACD,cAAQL,OAAR;AACE;AAAS;AACPN,YAAAA,KAAK,CAACY,IAAN,CAAWR,SAAX;AACD;AAHH;;AAKAJ,MAAAA,KAAK,CAACa,YAAN;AACD;;AACDb,IAAAA,KAAK,CAACc,aAAN;AACA,WAAO,IAAItB,QAAJ,EAAP;AACD;;AA1BmB","sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nimport * as thrift from 'thrift';\nexport interface IBsonTypeArgs {}\nexport class BsonType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('BsonType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): BsonType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new BsonType();\n }\n}\n"],"file":"BsonType.js"}
1
+ {"version":3,"sources":["../../../../src/parquetjs/parquet-thrift/BsonType.ts"],"names":["BsonType","output","writeStructBegin","writeFieldStop","writeStructEnd","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","thrift","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"mappings":";;;;;;;;;;;;;;;AAMA;;;;;;IAEaA,Q;AACX,sBAAc;AAAA;AAAE;;;;WAChB,eAAaC,MAAb,EAA6C;AAC3CA,MAAAA,MAAM,CAACC,gBAAP,CAAwB,UAAxB;AACAD,MAAAA,MAAM,CAACE,cAAP;AACAF,MAAAA,MAAM,CAACG,cAAP;AACA;AACD;;;WACD,cAAmBC,KAAnB,EAAsD;AACpDA,MAAAA,KAAK,CAACC,eAAN;;AACA,aAAO,IAAP,EAAa;AACX,YAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAN,EAA3B;AACA,YAAMC,SAA6B,GAAGF,GAAG,CAACG,KAA1C;AACA,YAAMC,OAAe,GAAGJ,GAAG,CAACK,GAA5B;;AACA,YAAIH,SAAS,KAAKI,MAAM,CAACC,MAAP,CAAcC,IAAd,CAAmBC,IAArC,EAA2C;AACzC;AACD;;AACD,gBAAQL,OAAR;AACE;AAAS;AACPN,cAAAA,KAAK,CAACY,IAAN,CAAWR,SAAX;AACD;AAHH;;AAKAJ,QAAAA,KAAK,CAACa,YAAN;AACD;;AACDb,MAAAA,KAAK,CAACc,aAAN;AACA,aAAO,IAAInB,QAAJ,EAAP;AACD","sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nimport * as thrift from 'thrift';\nexport interface IBsonTypeArgs {}\nexport class BsonType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('BsonType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): BsonType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new BsonType();\n }\n}\n"],"file":"BsonType.js"}