@loaders.gl/parquet 3.1.0-beta.7 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/dist/es5/bundle.js +1 -1
  2. package/dist/es5/bundle.js.map +1 -1
  3. package/dist/es5/constants.js +5 -5
  4. package/dist/es5/constants.js.map +1 -1
  5. package/dist/es5/index.js +19 -10
  6. package/dist/es5/index.js.map +1 -1
  7. package/dist/es5/lib/convert-schema.js +13 -13
  8. package/dist/es5/lib/convert-schema.js.map +1 -1
  9. package/dist/es5/lib/parse-parquet.js +154 -19
  10. package/dist/es5/lib/parse-parquet.js.map +1 -1
  11. package/dist/es5/lib/read-array-buffer.js +43 -6
  12. package/dist/es5/lib/read-array-buffer.js.map +1 -1
  13. package/dist/es5/parquet-loader.js +4 -4
  14. package/dist/es5/parquet-loader.js.map +1 -1
  15. package/dist/es5/parquet-writer.js +4 -4
  16. package/dist/es5/parquet-writer.js.map +1 -1
  17. package/dist/es5/parquetjs/codecs/dictionary.js +10 -2
  18. package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
  19. package/dist/es5/parquetjs/codecs/index.js +6 -4
  20. package/dist/es5/parquetjs/codecs/index.js.map +1 -1
  21. package/dist/es5/parquetjs/codecs/plain.js +43 -41
  22. package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
  23. package/dist/es5/parquetjs/codecs/rle.js +35 -25
  24. package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
  25. package/dist/es5/parquetjs/compression.js +110 -27
  26. package/dist/es5/parquetjs/compression.js.map +1 -1
  27. package/dist/es5/parquetjs/encoder/writer.js +737 -301
  28. package/dist/es5/parquetjs/encoder/writer.js.map +1 -1
  29. package/dist/es5/parquetjs/file.js +15 -15
  30. package/dist/es5/parquetjs/file.js.map +1 -1
  31. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
  32. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +45 -31
  33. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  34. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +152 -141
  35. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  36. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +160 -147
  37. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  38. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +259 -248
  39. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  40. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +79 -67
  41. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  42. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
  43. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
  44. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +124 -113
  45. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  46. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +169 -158
  47. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  48. package/dist/es5/parquetjs/parquet-thrift/DateType.js +45 -31
  49. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
  50. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +79 -68
  51. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  52. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +94 -83
  53. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  54. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
  55. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +45 -31
  56. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  57. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
  58. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +182 -170
  59. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  60. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +45 -31
  61. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  62. package/dist/es5/parquetjs/parquet-thrift/IntType.js +79 -68
  63. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
  64. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +45 -31
  65. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  66. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +79 -68
  67. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  68. package/dist/es5/parquetjs/parquet-thrift/ListType.js +45 -31
  69. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
  70. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +343 -319
  71. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  72. package/dist/es5/parquetjs/parquet-thrift/MapType.js +45 -31
  73. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
  74. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +45 -31
  75. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  76. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +45 -31
  77. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  78. package/dist/es5/parquetjs/parquet-thrift/NullType.js +45 -31
  79. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
  80. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +75 -64
  81. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  82. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +94 -83
  83. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  84. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +169 -158
  85. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  86. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +94 -83
  87. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  88. package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
  89. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +124 -113
  90. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  91. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +199 -188
  92. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  93. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +94 -83
  94. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  95. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +135 -124
  96. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  97. package/dist/es5/parquetjs/parquet-thrift/StringType.js +45 -31
  98. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
  99. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +79 -68
  100. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  101. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +101 -88
  102. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  103. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +79 -68
  104. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  105. package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
  106. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +45 -31
  107. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  108. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +45 -31
  109. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  110. package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
  111. package/dist/es5/parquetjs/parser/decoders.js +391 -218
  112. package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
  113. package/dist/es5/parquetjs/parser/parquet-cursor.js +180 -62
  114. package/dist/es5/parquetjs/parser/parquet-cursor.js.map +1 -1
  115. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js +370 -125
  116. package/dist/es5/parquetjs/parser/parquet-envelope-reader.js.map +1 -1
  117. package/dist/es5/parquetjs/parser/parquet-reader.js +320 -91
  118. package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
  119. package/dist/es5/parquetjs/schema/declare.js +11 -9
  120. package/dist/es5/parquetjs/schema/declare.js.map +1 -1
  121. package/dist/es5/parquetjs/schema/schema.js +87 -73
  122. package/dist/es5/parquetjs/schema/schema.js.map +1 -1
  123. package/dist/es5/parquetjs/schema/shred.js +96 -56
  124. package/dist/es5/parquetjs/schema/shred.js.map +1 -1
  125. package/dist/es5/parquetjs/schema/types.js +40 -39
  126. package/dist/es5/parquetjs/schema/types.js.map +1 -1
  127. package/dist/es5/parquetjs/utils/buffer-utils.js +1 -1
  128. package/dist/es5/parquetjs/utils/buffer-utils.js.map +1 -1
  129. package/dist/es5/parquetjs/utils/file-utils.js +12 -8
  130. package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
  131. package/dist/es5/parquetjs/utils/read-utils.js +50 -22
  132. package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
  133. package/dist/esm/parquet-loader.js +1 -1
  134. package/dist/esm/parquet-loader.js.map +1 -1
  135. package/dist/esm/parquet-writer.js +1 -1
  136. package/dist/esm/parquet-writer.js.map +1 -1
  137. package/package.json +5 -5
@@ -1,5 +1,7 @@
1
1
  "use strict";
2
2
 
3
+ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
+
3
5
  Object.defineProperty(exports, "__esModule", {
4
6
  value: true
5
7
  });
@@ -7,6 +9,10 @@ exports.decodeDataPages = decodeDataPages;
7
9
  exports.decodePage = decodePage;
8
10
  exports.decodeSchema = decodeSchema;
9
11
 
12
+ var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
13
+
14
+ var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
15
+
10
16
  var _codecs = require("../codecs");
11
17
 
12
18
  var _parquetThrift = require("../parquet-thrift");
@@ -17,91 +23,165 @@ var _constants = require("../../constants");
17
23
 
18
24
  var _readUtils = require("../utils/read-utils");
19
25
 
20
- async function decodeDataPages(buffer, options) {
21
- const cursor = {
22
- buffer,
23
- offset: 0,
24
- size: buffer.length
25
- };
26
- const data = {
27
- rlevels: [],
28
- dlevels: [],
29
- values: [],
30
- pageHeaders: [],
31
- count: 0
32
- };
33
- let dictionary = options.dictionary || [];
26
+ function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
34
27
 
35
- while (cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
36
- const page = await decodePage(cursor, options);
28
+ function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
37
29
 
38
- if (page.dictionary) {
39
- dictionary = page.dictionary;
40
- continue;
41
- }
30
+ function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
42
31
 
43
- if (dictionary.length) {
44
- page.values = page.values.map(value => dictionary[value]);
45
- }
46
-
47
- for (let index = 0; index < page.rlevels.length; index++) {
48
- data.rlevels.push(page.rlevels[index]);
49
- data.dlevels.push(page.dlevels[index]);
50
- const value = page.values[index];
32
+ function decodeDataPages(_x, _x2) {
33
+ return _decodeDataPages.apply(this, arguments);
34
+ }
51
35
 
52
- if (value !== undefined) {
53
- data.values.push(value);
36
+ function _decodeDataPages() {
37
+ _decodeDataPages = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(buffer, options) {
38
+ var cursor, data, dictionary, page, index, value;
39
+ return _regenerator.default.wrap(function _callee$(_context) {
40
+ while (1) {
41
+ switch (_context.prev = _context.next) {
42
+ case 0:
43
+ cursor = {
44
+ buffer: buffer,
45
+ offset: 0,
46
+ size: buffer.length
47
+ };
48
+ data = {
49
+ rlevels: [],
50
+ dlevels: [],
51
+ values: [],
52
+ pageHeaders: [],
53
+ count: 0
54
+ };
55
+ dictionary = options.dictionary || [];
56
+
57
+ case 3:
58
+ if (!(cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues)))) {
59
+ _context.next = 16;
60
+ break;
61
+ }
62
+
63
+ _context.next = 6;
64
+ return decodePage(cursor, options);
65
+
66
+ case 6:
67
+ page = _context.sent;
68
+
69
+ if (!page.dictionary) {
70
+ _context.next = 10;
71
+ break;
72
+ }
73
+
74
+ dictionary = page.dictionary;
75
+ return _context.abrupt("continue", 3);
76
+
77
+ case 10:
78
+ if (dictionary.length) {
79
+ page.values = page.values.map(function (value) {
80
+ return dictionary[value];
81
+ });
82
+ }
83
+
84
+ for (index = 0; index < page.rlevels.length; index++) {
85
+ data.rlevels.push(page.rlevels[index]);
86
+ data.dlevels.push(page.dlevels[index]);
87
+ value = page.values[index];
88
+
89
+ if (value !== undefined) {
90
+ data.values.push(value);
91
+ }
92
+ }
93
+
94
+ data.count += page.count;
95
+ data.pageHeaders.push(page.pageHeader);
96
+ _context.next = 3;
97
+ break;
98
+
99
+ case 16:
100
+ return _context.abrupt("return", data);
101
+
102
+ case 17:
103
+ case "end":
104
+ return _context.stop();
105
+ }
54
106
  }
55
- }
56
-
57
- data.count += page.count;
58
- data.pageHeaders.push(page.pageHeader);
59
- }
60
-
61
- return data;
107
+ }, _callee);
108
+ }));
109
+ return _decodeDataPages.apply(this, arguments);
62
110
  }
63
111
 
64
- async function decodePage(cursor, options) {
65
- let page;
66
- const {
67
- pageHeader,
68
- length
69
- } = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset);
70
- cursor.offset += length;
71
- const pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
72
-
73
- switch (pageType) {
74
- case 'DATA_PAGE':
75
- page = await decodeDataPage(cursor, pageHeader, options);
76
- break;
77
-
78
- case 'DATA_PAGE_V2':
79
- page = await decodeDataPageV2(cursor, pageHeader, options);
80
- break;
81
-
82
- case 'DICTIONARY_PAGE':
83
- page = {
84
- dictionary: await decodeDictionaryPage(cursor, pageHeader, options),
85
- pageHeader
86
- };
87
- break;
88
-
89
- default:
90
- throw new Error("invalid page type: ".concat(pageType));
91
- }
112
+ function decodePage(_x3, _x4) {
113
+ return _decodePage.apply(this, arguments);
114
+ }
92
115
 
93
- return page;
116
+ function _decodePage() {
117
+ _decodePage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(cursor, options) {
118
+ var page, _decodePageHeader, pageHeader, length, pageType;
119
+
120
+ return _regenerator.default.wrap(function _callee2$(_context2) {
121
+ while (1) {
122
+ switch (_context2.prev = _context2.next) {
123
+ case 0:
124
+ _decodePageHeader = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset), pageHeader = _decodePageHeader.pageHeader, length = _decodePageHeader.length;
125
+ cursor.offset += length;
126
+ pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
127
+ _context2.t0 = pageType;
128
+ _context2.next = _context2.t0 === 'DATA_PAGE' ? 6 : _context2.t0 === 'DATA_PAGE_V2' ? 10 : _context2.t0 === 'DICTIONARY_PAGE' ? 14 : 20;
129
+ break;
130
+
131
+ case 6:
132
+ _context2.next = 8;
133
+ return decodeDataPage(cursor, pageHeader, options);
134
+
135
+ case 8:
136
+ page = _context2.sent;
137
+ return _context2.abrupt("break", 21);
138
+
139
+ case 10:
140
+ _context2.next = 12;
141
+ return decodeDataPageV2(cursor, pageHeader, options);
142
+
143
+ case 12:
144
+ page = _context2.sent;
145
+ return _context2.abrupt("break", 21);
146
+
147
+ case 14:
148
+ _context2.next = 16;
149
+ return decodeDictionaryPage(cursor, pageHeader, options);
150
+
151
+ case 16:
152
+ _context2.t1 = _context2.sent;
153
+ _context2.t2 = pageHeader;
154
+ page = {
155
+ dictionary: _context2.t1,
156
+ pageHeader: _context2.t2
157
+ };
158
+ return _context2.abrupt("break", 21);
159
+
160
+ case 20:
161
+ throw new Error("invalid page type: ".concat(pageType));
162
+
163
+ case 21:
164
+ return _context2.abrupt("return", page);
165
+
166
+ case 22:
167
+ case "end":
168
+ return _context2.stop();
169
+ }
170
+ }
171
+ }, _callee2);
172
+ }));
173
+ return _decodePage.apply(this, arguments);
94
174
  }
95
175
 
96
176
  function decodeSchema(schemaElements, offset, len) {
97
- const schema = {};
98
- let next = offset;
177
+ var schema = {};
178
+ var next = offset;
99
179
 
100
- for (let i = 0; i < len; i++) {
101
- const schemaElement = schemaElements[next];
102
- const repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
103
- let optional = false;
104
- let repeated = false;
180
+ for (var i = 0; i < len; i++) {
181
+ var schemaElement = schemaElements[next];
182
+ var repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
183
+ var optional = false;
184
+ var repeated = false;
105
185
 
106
186
  switch (repetitionType) {
107
187
  case 'REQUIRED':
@@ -120,16 +200,16 @@ function decodeSchema(schemaElements, offset, len) {
120
200
  }
121
201
 
122
202
  if (schemaElement.num_children > 0) {
123
- const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
203
+ var res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
124
204
  next = res.next;
125
205
  schema[schemaElement.name] = {
126
- optional,
127
- repeated,
206
+ optional: optional,
207
+ repeated: repeated,
128
208
  fields: res.schema
129
209
  };
130
210
  } else {
131
- const type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
132
- let logicalType = type;
211
+ var type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
212
+ var logicalType = type;
133
213
 
134
214
  if (schemaElement.converted_type) {
135
215
  logicalType = (0, _readUtils.getThriftEnum)(_parquetThrift.ConvertedType, schemaElement.converted_type);
@@ -148,17 +228,17 @@ function decodeSchema(schemaElements, offset, len) {
148
228
  typeLength: schemaElement.type_length,
149
229
  presision: schemaElement.precision,
150
230
  scale: schemaElement.scale,
151
- optional,
152
- repeated
231
+ optional: optional,
232
+ repeated: repeated
153
233
  };
154
234
  next++;
155
235
  }
156
236
  }
157
237
 
158
238
  return {
159
- schema,
160
- offset,
161
- next
239
+ schema: schema,
240
+ offset: offset,
241
+ next: next
162
242
  };
163
243
  }
164
244
 
@@ -170,147 +250,240 @@ function decodeValues(type, encoding, cursor, count, opts) {
170
250
  return _codecs.PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
171
251
  }
172
252
 
173
- async function decodeDataPage(cursor, header, options) {
174
- var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
175
-
176
- const cursorEnd = cursor.offset + header.compressed_page_size;
177
- const valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
178
- let dataCursor = cursor;
179
-
180
- if (options.compression !== 'UNCOMPRESSED') {
181
- const valuesBuf = await (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
182
- dataCursor = {
183
- buffer: valuesBuf,
184
- offset: 0,
185
- size: valuesBuf.length
186
- };
187
- cursor.offset = cursorEnd;
188
- }
189
-
190
- const rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
191
- let rLevels = new Array(valueCount);
192
-
193
- if (options.column.rLevelMax > 0) {
194
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
195
- bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
196
- disableEnvelope: false
197
- });
198
- } else {
199
- rLevels.fill(0);
200
- }
201
-
202
- const dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
203
- let dLevels = new Array(valueCount);
204
-
205
- if (options.column.dLevelMax > 0) {
206
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
207
- bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
208
- disableEnvelope: false
209
- });
210
- } else {
211
- dLevels.fill(0);
212
- }
213
-
214
- let valueCountNonNull = 0;
215
-
216
- for (const dlvl of dLevels) {
217
- if (dlvl === options.column.dLevelMax) {
218
- valueCountNonNull++;
219
- }
220
- }
221
-
222
- const valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
223
- const decodeOptions = {
224
- typeLength: options.column.typeLength,
225
- bitWidth: options.column.typeLength
226
- };
227
- const values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
228
- return {
229
- dlevels: dLevels,
230
- rlevels: rLevels,
231
- values,
232
- count: valueCount,
233
- pageHeader: header
234
- };
253
+ function decodeDataPage(_x5, _x6, _x7) {
254
+ return _decodeDataPage.apply(this, arguments);
235
255
  }
236
256
 
237
- async function decodeDataPageV2(cursor, header, opts) {
238
- var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
239
-
240
- const cursorEnd = cursor.offset + header.compressed_page_size;
241
- const valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
242
- const valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
243
- const valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
244
- let rLevels = new Array(valueCount);
245
-
246
- if (opts.column.rLevelMax > 0) {
247
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
248
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
249
- disableEnvelope: true
250
- });
251
- } else {
252
- rLevels.fill(0);
253
- }
254
-
255
- let dLevels = new Array(valueCount);
256
-
257
- if (opts.column.dLevelMax > 0) {
258
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
259
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
260
- disableEnvelope: true
261
- });
262
- } else {
263
- dLevels.fill(0);
264
- }
265
-
266
- let valuesBufCursor = cursor;
267
-
268
- if ((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed) {
269
- const valuesBuf = await (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
270
- valuesBufCursor = {
271
- buffer: valuesBuf,
272
- offset: 0,
273
- size: valuesBuf.length
274
- };
275
- cursor.offset = cursorEnd;
276
- }
257
+ function _decodeDataPage() {
258
+ _decodeDataPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(cursor, header, options) {
259
+ var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
260
+
261
+ var cursorEnd, valueCount, dataCursor, valuesBuf, rLevelEncoding, rLevels, dLevelEncoding, dLevels, valueCountNonNull, _iterator, _step, dlvl, valueEncoding, decodeOptions, values;
262
+
263
+ return _regenerator.default.wrap(function _callee3$(_context3) {
264
+ while (1) {
265
+ switch (_context3.prev = _context3.next) {
266
+ case 0:
267
+ cursorEnd = cursor.offset + header.compressed_page_size;
268
+ valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
269
+ dataCursor = cursor;
270
+
271
+ if (!(options.compression !== 'UNCOMPRESSED')) {
272
+ _context3.next = 9;
273
+ break;
274
+ }
275
+
276
+ _context3.next = 6;
277
+ return (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
278
+
279
+ case 6:
280
+ valuesBuf = _context3.sent;
281
+ dataCursor = {
282
+ buffer: valuesBuf,
283
+ offset: 0,
284
+ size: valuesBuf.length
285
+ };
286
+ cursor.offset = cursorEnd;
287
+
288
+ case 9:
289
+ rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
290
+ rLevels = new Array(valueCount);
291
+
292
+ if (options.column.rLevelMax > 0) {
293
+ rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
294
+ bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
295
+ disableEnvelope: false
296
+ });
297
+ } else {
298
+ rLevels.fill(0);
299
+ }
300
+
301
+ dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
302
+ dLevels = new Array(valueCount);
303
+
304
+ if (options.column.dLevelMax > 0) {
305
+ dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
306
+ bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
307
+ disableEnvelope: false
308
+ });
309
+ } else {
310
+ dLevels.fill(0);
311
+ }
312
+
313
+ valueCountNonNull = 0;
314
+ _iterator = _createForOfIteratorHelper(dLevels);
315
+
316
+ try {
317
+ for (_iterator.s(); !(_step = _iterator.n()).done;) {
318
+ dlvl = _step.value;
319
+
320
+ if (dlvl === options.column.dLevelMax) {
321
+ valueCountNonNull++;
322
+ }
323
+ }
324
+ } catch (err) {
325
+ _iterator.e(err);
326
+ } finally {
327
+ _iterator.f();
328
+ }
329
+
330
+ valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
331
+ decodeOptions = {
332
+ typeLength: options.column.typeLength,
333
+ bitWidth: options.column.typeLength
334
+ };
335
+ values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
336
+ return _context3.abrupt("return", {
337
+ dlevels: dLevels,
338
+ rlevels: rLevels,
339
+ values: values,
340
+ count: valueCount,
341
+ pageHeader: header
342
+ });
343
+
344
+ case 22:
345
+ case "end":
346
+ return _context3.stop();
347
+ }
348
+ }
349
+ }, _callee3);
350
+ }));
351
+ return _decodeDataPage.apply(this, arguments);
352
+ }
277
353
 
278
- const decodeOptions = {
279
- typeLength: opts.column.typeLength,
280
- bitWidth: opts.column.typeLength
281
- };
282
- const values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
283
- return {
284
- dlevels: dLevels,
285
- rlevels: rLevels,
286
- values,
287
- count: valueCount,
288
- pageHeader: header
289
- };
354
+ function decodeDataPageV2(_x8, _x9, _x10) {
355
+ return _decodeDataPageV.apply(this, arguments);
290
356
  }
291
357
 
292
- async function decodeDictionaryPage(cursor, pageHeader, options) {
293
- var _pageHeader$dictionar;
358
+ function _decodeDataPageV() {
359
+ _decodeDataPageV = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4(cursor, header, opts) {
360
+ var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
361
+
362
+ var cursorEnd, valueCount, valueCountNonNull, valueEncoding, rLevels, dLevels, valuesBufCursor, valuesBuf, decodeOptions, values;
363
+ return _regenerator.default.wrap(function _callee4$(_context4) {
364
+ while (1) {
365
+ switch (_context4.prev = _context4.next) {
366
+ case 0:
367
+ cursorEnd = cursor.offset + header.compressed_page_size;
368
+ valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
369
+ valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
370
+ valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
371
+ rLevels = new Array(valueCount);
372
+
373
+ if (opts.column.rLevelMax > 0) {
374
+ rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
375
+ bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
376
+ disableEnvelope: true
377
+ });
378
+ } else {
379
+ rLevels.fill(0);
380
+ }
381
+
382
+ dLevels = new Array(valueCount);
383
+
384
+ if (opts.column.dLevelMax > 0) {
385
+ dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
386
+ bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
387
+ disableEnvelope: true
388
+ });
389
+ } else {
390
+ dLevels.fill(0);
391
+ }
392
+
393
+ valuesBufCursor = cursor;
394
+
395
+ if (!((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed)) {
396
+ _context4.next = 15;
397
+ break;
398
+ }
399
+
400
+ _context4.next = 12;
401
+ return (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
402
+
403
+ case 12:
404
+ valuesBuf = _context4.sent;
405
+ valuesBufCursor = {
406
+ buffer: valuesBuf,
407
+ offset: 0,
408
+ size: valuesBuf.length
409
+ };
410
+ cursor.offset = cursorEnd;
411
+
412
+ case 15:
413
+ decodeOptions = {
414
+ typeLength: opts.column.typeLength,
415
+ bitWidth: opts.column.typeLength
416
+ };
417
+ values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
418
+ return _context4.abrupt("return", {
419
+ dlevels: dLevels,
420
+ rlevels: rLevels,
421
+ values: values,
422
+ count: valueCount,
423
+ pageHeader: header
424
+ });
425
+
426
+ case 18:
427
+ case "end":
428
+ return _context4.stop();
429
+ }
430
+ }
431
+ }, _callee4);
432
+ }));
433
+ return _decodeDataPageV.apply(this, arguments);
434
+ }
294
435
 
295
- const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
296
- let dictCursor = {
297
- offset: 0,
298
- buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
299
- size: cursorEnd - cursor.offset
300
- };
301
- cursor.offset = cursorEnd;
302
-
303
- if (options.compression !== 'UNCOMPRESSED') {
304
- const valuesBuf = await (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
305
- dictCursor = {
306
- buffer: valuesBuf,
307
- offset: 0,
308
- size: valuesBuf.length
309
- };
310
- cursor.offset = cursorEnd;
311
- }
436
+ function decodeDictionaryPage(_x11, _x12, _x13) {
437
+ return _decodeDictionaryPage.apply(this, arguments);
438
+ }
312
439
 
313
- const numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
314
- return decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(d => d.toString());
440
+ function _decodeDictionaryPage() {
441
+ _decodeDictionaryPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5(cursor, pageHeader, options) {
442
+ var _pageHeader$dictionar;
443
+
444
+ var cursorEnd, dictCursor, valuesBuf, numValues;
445
+ return _regenerator.default.wrap(function _callee5$(_context5) {
446
+ while (1) {
447
+ switch (_context5.prev = _context5.next) {
448
+ case 0:
449
+ cursorEnd = cursor.offset + pageHeader.compressed_page_size;
450
+ dictCursor = {
451
+ offset: 0,
452
+ buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
453
+ size: cursorEnd - cursor.offset
454
+ };
455
+ cursor.offset = cursorEnd;
456
+
457
+ if (!(options.compression !== 'UNCOMPRESSED')) {
458
+ _context5.next = 9;
459
+ break;
460
+ }
461
+
462
+ _context5.next = 6;
463
+ return (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
464
+
465
+ case 6:
466
+ valuesBuf = _context5.sent;
467
+ dictCursor = {
468
+ buffer: valuesBuf,
469
+ offset: 0,
470
+ size: valuesBuf.length
471
+ };
472
+ cursor.offset = cursorEnd;
473
+
474
+ case 9:
475
+ numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
476
+ return _context5.abrupt("return", decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(function (d) {
477
+ return d.toString();
478
+ }));
479
+
480
+ case 11:
481
+ case "end":
482
+ return _context5.stop();
483
+ }
484
+ }
485
+ }, _callee5);
486
+ }));
487
+ return _decodeDictionaryPage.apply(this, arguments);
315
488
  }
316
489
  //# sourceMappingURL=decoders.js.map