@loaders.gl/parquet 3.4.13 → 3.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/dist/dist.min.js +15 -24
  2. package/dist/dist.min.js.map +3 -3
  3. package/dist/es5/bundle.js +1 -1
  4. package/dist/es5/bundle.js.map +1 -1
  5. package/dist/es5/constants.js +5 -5
  6. package/dist/es5/constants.js.map +1 -1
  7. package/dist/es5/index.js +24 -24
  8. package/dist/es5/index.js.map +1 -1
  9. package/dist/es5/lib/arrow/convert-row-group-to-columns.js +2 -7
  10. package/dist/es5/lib/arrow/convert-row-group-to-columns.js.map +1 -1
  11. package/dist/es5/lib/arrow/convert-schema-from-parquet.js +22 -33
  12. package/dist/es5/lib/arrow/convert-schema-from-parquet.js.map +1 -1
  13. package/dist/es5/lib/arrow/convert-schema-to-parquet.js +2 -2
  14. package/dist/es5/lib/arrow/convert-schema-to-parquet.js.map +1 -1
  15. package/dist/es5/lib/geo/decode-geo-metadata.js +16 -27
  16. package/dist/es5/lib/geo/decode-geo-metadata.js.map +1 -1
  17. package/dist/es5/lib/parsers/parse-parquet-to-columns.js +20 -151
  18. package/dist/es5/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  19. package/dist/es5/lib/parsers/parse-parquet-to-rows.js +13 -138
  20. package/dist/es5/lib/parsers/parse-parquet-to-rows.js.map +1 -1
  21. package/dist/es5/lib/wasm/encode-parquet-wasm.js +8 -29
  22. package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -1
  23. package/dist/es5/lib/wasm/load-wasm/index.js +1 -1
  24. package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +10 -33
  25. package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -1
  26. package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +4 -22
  27. package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -1
  28. package/dist/es5/lib/wasm/parse-parquet-wasm.js +13 -46
  29. package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -1
  30. package/dist/es5/parquet-loader.js +4 -4
  31. package/dist/es5/parquet-loader.js.map +1 -1
  32. package/dist/es5/parquet-wasm-loader.js +4 -4
  33. package/dist/es5/parquet-wasm-loader.js.map +1 -1
  34. package/dist/es5/parquet-wasm-writer.js +3 -3
  35. package/dist/es5/parquet-wasm-writer.js.map +1 -1
  36. package/dist/es5/parquet-writer.js +4 -4
  37. package/dist/es5/parquet-writer.js.map +1 -1
  38. package/dist/es5/parquetjs/codecs/dictionary.js +3 -6
  39. package/dist/es5/parquetjs/codecs/dictionary.js.map +1 -1
  40. package/dist/es5/parquetjs/codecs/index.js +4 -5
  41. package/dist/es5/parquetjs/codecs/index.js.map +1 -1
  42. package/dist/es5/parquetjs/codecs/plain.js +41 -41
  43. package/dist/es5/parquetjs/codecs/plain.js.map +1 -1
  44. package/dist/es5/parquetjs/codecs/rle.js +25 -30
  45. package/dist/es5/parquetjs/codecs/rle.js.map +1 -1
  46. package/dist/es5/parquetjs/compression.js +26 -90
  47. package/dist/es5/parquetjs/compression.js.map +1 -1
  48. package/dist/es5/parquetjs/encoder/parquet-encoder.js +245 -536
  49. package/dist/es5/parquetjs/encoder/parquet-encoder.js.map +1 -1
  50. package/dist/es5/parquetjs/parquet-thrift/BoundaryOrder.js +1 -1
  51. package/dist/es5/parquetjs/parquet-thrift/BsonType.js +28 -40
  52. package/dist/es5/parquetjs/parquet-thrift/BsonType.js.map +1 -1
  53. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js +123 -133
  54. package/dist/es5/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -1
  55. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js +138 -150
  56. package/dist/es5/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  57. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js +241 -251
  58. package/dist/es5/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -1
  59. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js +58 -70
  60. package/dist/es5/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -1
  61. package/dist/es5/parquetjs/parquet-thrift/CompressionCodec.js +1 -1
  62. package/dist/es5/parquetjs/parquet-thrift/ConvertedType.js +1 -1
  63. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js +97 -107
  64. package/dist/es5/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -1
  65. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js +136 -146
  66. package/dist/es5/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -1
  67. package/dist/es5/parquetjs/parquet-thrift/DateType.js +28 -40
  68. package/dist/es5/parquetjs/parquet-thrift/DateType.js.map +1 -1
  69. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js +58 -68
  70. package/dist/es5/parquetjs/parquet-thrift/DecimalType.js.map +1 -1
  71. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js +71 -81
  72. package/dist/es5/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -1
  73. package/dist/es5/parquetjs/parquet-thrift/Encoding.js +1 -1
  74. package/dist/es5/parquetjs/parquet-thrift/EnumType.js +28 -40
  75. package/dist/es5/parquetjs/parquet-thrift/EnumType.js.map +1 -1
  76. package/dist/es5/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -1
  77. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js +164 -174
  78. package/dist/es5/parquetjs/parquet-thrift/FileMetaData.js.map +1 -1
  79. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js +28 -40
  80. package/dist/es5/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -1
  81. package/dist/es5/parquetjs/parquet-thrift/IntType.js +58 -68
  82. package/dist/es5/parquetjs/parquet-thrift/IntType.js.map +1 -1
  83. package/dist/es5/parquetjs/parquet-thrift/JsonType.js +28 -40
  84. package/dist/es5/parquetjs/parquet-thrift/JsonType.js.map +1 -1
  85. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js +58 -68
  86. package/dist/es5/parquetjs/parquet-thrift/KeyValue.js.map +1 -1
  87. package/dist/es5/parquetjs/parquet-thrift/ListType.js +28 -40
  88. package/dist/es5/parquetjs/parquet-thrift/ListType.js.map +1 -1
  89. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js +274 -310
  90. package/dist/es5/parquetjs/parquet-thrift/LogicalType.js.map +1 -1
  91. package/dist/es5/parquetjs/parquet-thrift/MapType.js +28 -40
  92. package/dist/es5/parquetjs/parquet-thrift/MapType.js.map +1 -1
  93. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js +28 -40
  94. package/dist/es5/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -1
  95. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js +28 -40
  96. package/dist/es5/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -1
  97. package/dist/es5/parquetjs/parquet-thrift/NullType.js +28 -40
  98. package/dist/es5/parquetjs/parquet-thrift/NullType.js.map +1 -1
  99. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js +56 -66
  100. package/dist/es5/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -1
  101. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js +71 -81
  102. package/dist/es5/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -1
  103. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js +136 -146
  104. package/dist/es5/parquetjs/parquet-thrift/PageHeader.js.map +1 -1
  105. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js +71 -81
  106. package/dist/es5/parquetjs/parquet-thrift/PageLocation.js.map +1 -1
  107. package/dist/es5/parquetjs/parquet-thrift/PageType.js +1 -1
  108. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js +105 -115
  109. package/dist/es5/parquetjs/parquet-thrift/RowGroup.js.map +1 -1
  110. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js +162 -172
  111. package/dist/es5/parquetjs/parquet-thrift/SchemaElement.js.map +1 -1
  112. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js +71 -81
  113. package/dist/es5/parquetjs/parquet-thrift/SortingColumn.js.map +1 -1
  114. package/dist/es5/parquetjs/parquet-thrift/Statistics.js +106 -116
  115. package/dist/es5/parquetjs/parquet-thrift/Statistics.js.map +1 -1
  116. package/dist/es5/parquetjs/parquet-thrift/StringType.js +28 -40
  117. package/dist/es5/parquetjs/parquet-thrift/StringType.js.map +1 -1
  118. package/dist/es5/parquetjs/parquet-thrift/TimeType.js +58 -68
  119. package/dist/es5/parquetjs/parquet-thrift/TimeType.js.map +1 -1
  120. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js +76 -90
  121. package/dist/es5/parquetjs/parquet-thrift/TimeUnit.js.map +1 -1
  122. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js +58 -68
  123. package/dist/es5/parquetjs/parquet-thrift/TimestampType.js.map +1 -1
  124. package/dist/es5/parquetjs/parquet-thrift/Type.js +1 -1
  125. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js +28 -40
  126. package/dist/es5/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -1
  127. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js +28 -40
  128. package/dist/es5/parquetjs/parquet-thrift/UUIDType.js.map +1 -1
  129. package/dist/es5/parquetjs/parquet-thrift/index.js +43 -43
  130. package/dist/es5/parquetjs/parquet-thrift/index.js.map +1 -1
  131. package/dist/es5/parquetjs/parser/decoders.js +195 -327
  132. package/dist/es5/parquetjs/parser/decoders.js.map +1 -1
  133. package/dist/es5/parquetjs/parser/parquet-reader.js +155 -582
  134. package/dist/es5/parquetjs/parser/parquet-reader.js.map +1 -1
  135. package/dist/es5/parquetjs/schema/declare.js +10 -11
  136. package/dist/es5/parquetjs/schema/declare.js.map +1 -1
  137. package/dist/es5/parquetjs/schema/schema.js +65 -82
  138. package/dist/es5/parquetjs/schema/schema.js.map +1 -1
  139. package/dist/es5/parquetjs/schema/shred.js +56 -87
  140. package/dist/es5/parquetjs/schema/shred.js.map +1 -1
  141. package/dist/es5/parquetjs/schema/types.js +40 -40
  142. package/dist/es5/parquetjs/schema/types.js.map +1 -1
  143. package/dist/es5/parquetjs/utils/file-utils.js +8 -12
  144. package/dist/es5/parquetjs/utils/file-utils.js.map +1 -1
  145. package/dist/es5/parquetjs/utils/read-utils.js +22 -39
  146. package/dist/es5/parquetjs/utils/read-utils.js.map +1 -1
  147. package/dist/esm/parquet-loader.js +1 -1
  148. package/dist/esm/parquet-wasm-loader.js +1 -1
  149. package/dist/esm/parquet-wasm-writer.js +1 -1
  150. package/dist/esm/parquet-writer.js +1 -1
  151. package/dist/parquet-worker.js +15 -24
  152. package/dist/parquet-worker.js.map +3 -3
  153. package/package.json +6 -6
@@ -1,145 +1,86 @@
1
1
  "use strict";
2
2
 
3
- var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
3
  Object.defineProperty(exports, "__esModule", {
5
4
  value: true
6
5
  });
7
6
  exports.decodeDataPages = decodeDataPages;
8
7
  exports.decodePage = decodePage;
9
8
  exports.decodeSchema = decodeSchema;
10
- var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
11
- var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
12
9
  var _codecs = require("../codecs");
13
10
  var _parquetThrift = require("../parquet-thrift");
14
11
  var _compression = require("../compression");
15
12
  var _constants = require("../../constants");
16
13
  var _readUtils = require("../utils/read-utils");
17
- function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
18
- function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
19
- function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
20
- function decodeDataPages(_x, _x2) {
21
- return _decodeDataPages.apply(this, arguments);
22
- }
23
- function _decodeDataPages() {
24
- _decodeDataPages = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(buffer, options) {
25
- var cursor, data, dictionary, page, index, value;
26
- return _regenerator.default.wrap(function _callee$(_context) {
27
- while (1) switch (_context.prev = _context.next) {
28
- case 0:
29
- cursor = {
30
- buffer: buffer,
31
- offset: 0,
32
- size: buffer.length
33
- };
34
- data = {
35
- rlevels: [],
36
- dlevels: [],
37
- values: [],
38
- pageHeaders: [],
39
- count: 0
40
- };
41
- dictionary = options.dictionary || [];
42
- case 3:
43
- if (!(cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues)))) {
44
- _context.next = 16;
45
- break;
46
- }
47
- _context.next = 6;
48
- return decodePage(cursor, options);
49
- case 6:
50
- page = _context.sent;
51
- if (!page.dictionary) {
52
- _context.next = 10;
53
- break;
54
- }
55
- dictionary = page.dictionary;
56
- return _context.abrupt("continue", 3);
57
- case 10:
58
- if (dictionary.length) {
59
- page.values = page.values.map(function (value) {
60
- return dictionary[value];
61
- });
62
- }
63
- for (index = 0; index < page.rlevels.length; index++) {
64
- data.rlevels.push(page.rlevels[index]);
65
- data.dlevels.push(page.dlevels[index]);
66
- value = page.values[index];
67
- if (value !== undefined) {
68
- data.values.push(value);
69
- }
70
- }
71
- data.count += page.count;
72
- data.pageHeaders.push(page.pageHeader);
73
- _context.next = 3;
74
- break;
75
- case 16:
76
- return _context.abrupt("return", data);
77
- case 17:
78
- case "end":
79
- return _context.stop();
14
+ async function decodeDataPages(buffer, options) {
15
+ const cursor = {
16
+ buffer,
17
+ offset: 0,
18
+ size: buffer.length
19
+ };
20
+ const data = {
21
+ rlevels: [],
22
+ dlevels: [],
23
+ values: [],
24
+ pageHeaders: [],
25
+ count: 0
26
+ };
27
+ let dictionary = options.dictionary || [];
28
+ while (cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
29
+ const page = await decodePage(cursor, options);
30
+ if (page.dictionary) {
31
+ dictionary = page.dictionary;
32
+ continue;
33
+ }
34
+ if (dictionary.length) {
35
+ page.values = page.values.map(value => dictionary[value]);
36
+ }
37
+ for (let index = 0; index < page.rlevels.length; index++) {
38
+ data.rlevels.push(page.rlevels[index]);
39
+ data.dlevels.push(page.dlevels[index]);
40
+ const value = page.values[index];
41
+ if (value !== undefined) {
42
+ data.values.push(value);
80
43
  }
81
- }, _callee);
82
- }));
83
- return _decodeDataPages.apply(this, arguments);
84
- }
85
- function decodePage(_x3, _x4) {
86
- return _decodePage.apply(this, arguments);
44
+ }
45
+ data.count += page.count;
46
+ data.pageHeaders.push(page.pageHeader);
47
+ }
48
+ return data;
87
49
  }
88
- function _decodePage() {
89
- _decodePage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(cursor, options) {
90
- var page, _decodePageHeader, pageHeader, length, pageType;
91
- return _regenerator.default.wrap(function _callee2$(_context2) {
92
- while (1) switch (_context2.prev = _context2.next) {
93
- case 0:
94
- _decodePageHeader = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset), pageHeader = _decodePageHeader.pageHeader, length = _decodePageHeader.length;
95
- cursor.offset += length;
96
- pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
97
- _context2.t0 = pageType;
98
- _context2.next = _context2.t0 === 'DATA_PAGE' ? 6 : _context2.t0 === 'DATA_PAGE_V2' ? 10 : _context2.t0 === 'DICTIONARY_PAGE' ? 14 : 20;
99
- break;
100
- case 6:
101
- _context2.next = 8;
102
- return decodeDataPage(cursor, pageHeader, options);
103
- case 8:
104
- page = _context2.sent;
105
- return _context2.abrupt("break", 21);
106
- case 10:
107
- _context2.next = 12;
108
- return decodeDataPageV2(cursor, pageHeader, options);
109
- case 12:
110
- page = _context2.sent;
111
- return _context2.abrupt("break", 21);
112
- case 14:
113
- _context2.next = 16;
114
- return decodeDictionaryPage(cursor, pageHeader, options);
115
- case 16:
116
- _context2.t1 = _context2.sent;
117
- _context2.t2 = pageHeader;
118
- page = {
119
- dictionary: _context2.t1,
120
- pageHeader: _context2.t2
121
- };
122
- return _context2.abrupt("break", 21);
123
- case 20:
124
- throw new Error("invalid page type: ".concat(pageType));
125
- case 21:
126
- return _context2.abrupt("return", page);
127
- case 22:
128
- case "end":
129
- return _context2.stop();
130
- }
131
- }, _callee2);
132
- }));
133
- return _decodePage.apply(this, arguments);
50
+ async function decodePage(cursor, options) {
51
+ let page;
52
+ const {
53
+ pageHeader,
54
+ length
55
+ } = (0, _readUtils.decodePageHeader)(cursor.buffer, cursor.offset);
56
+ cursor.offset += length;
57
+ const pageType = (0, _readUtils.getThriftEnum)(_parquetThrift.PageType, pageHeader.type);
58
+ switch (pageType) {
59
+ case 'DATA_PAGE':
60
+ page = await decodeDataPage(cursor, pageHeader, options);
61
+ break;
62
+ case 'DATA_PAGE_V2':
63
+ page = await decodeDataPageV2(cursor, pageHeader, options);
64
+ break;
65
+ case 'DICTIONARY_PAGE':
66
+ page = {
67
+ dictionary: await decodeDictionaryPage(cursor, pageHeader, options),
68
+ pageHeader
69
+ };
70
+ break;
71
+ default:
72
+ throw new Error("invalid page type: ".concat(pageType));
73
+ }
74
+ return page;
134
75
  }
135
76
  function decodeSchema(schemaElements, offset, len) {
136
- var schema = {};
137
- var next = offset;
138
- for (var i = 0; i < len; i++) {
139
- var schemaElement = schemaElements[next];
140
- var repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
141
- var optional = false;
142
- var repeated = false;
77
+ const schema = {};
78
+ let next = offset;
79
+ for (let i = 0; i < len; i++) {
80
+ const schemaElement = schemaElements[next];
81
+ const repetitionType = next > 0 ? (0, _readUtils.getThriftEnum)(_parquetThrift.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
82
+ let optional = false;
83
+ let repeated = false;
143
84
  switch (repetitionType) {
144
85
  case 'REQUIRED':
145
86
  break;
@@ -153,16 +94,16 @@ function decodeSchema(schemaElements, offset, len) {
153
94
  throw new Error('parquet: unknown repetition type');
154
95
  }
155
96
  if (schemaElement.num_children > 0) {
156
- var res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
97
+ const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
157
98
  next = res.next;
158
99
  schema[schemaElement.name] = {
159
- optional: optional,
160
- repeated: repeated,
100
+ optional,
101
+ repeated,
161
102
  fields: res.schema
162
103
  };
163
104
  } else {
164
- var type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
165
- var logicalType = type;
105
+ const type = (0, _readUtils.getThriftEnum)(_parquetThrift.Type, schemaElement.type);
106
+ let logicalType = type;
166
107
  if (schemaElement.converted_type) {
167
108
  logicalType = (0, _readUtils.getThriftEnum)(_parquetThrift.ConvertedType, schemaElement.converted_type);
168
109
  }
@@ -177,16 +118,16 @@ function decodeSchema(schemaElements, offset, len) {
177
118
  typeLength: schemaElement.type_length,
178
119
  presision: schemaElement.precision,
179
120
  scale: schemaElement.scale,
180
- optional: optional,
181
- repeated: repeated
121
+ optional,
122
+ repeated
182
123
  };
183
124
  next++;
184
125
  }
185
126
  }
186
127
  return {
187
- schema: schema,
188
- offset: offset,
189
- next: next
128
+ schema,
129
+ offset,
130
+ next
190
131
  };
191
132
  }
192
133
  function decodeValues(type, encoding, cursor, count, opts) {
@@ -195,199 +136,126 @@ function decodeValues(type, encoding, cursor, count, opts) {
195
136
  }
196
137
  return _codecs.PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
197
138
  }
198
- function decodeDataPage(_x5, _x6, _x7) {
199
- return _decodeDataPage.apply(this, arguments);
200
- }
201
- function _decodeDataPage() {
202
- _decodeDataPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee3(cursor, header, options) {
203
- var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
204
- var cursorEnd, valueCount, dataCursor, valuesBuf, rLevelEncoding, rLevels, dLevelEncoding, dLevels, valueCountNonNull, _iterator, _step, dlvl, valueEncoding, decodeOptions, values;
205
- return _regenerator.default.wrap(function _callee3$(_context3) {
206
- while (1) switch (_context3.prev = _context3.next) {
207
- case 0:
208
- cursorEnd = cursor.offset + header.compressed_page_size;
209
- valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
210
- dataCursor = cursor;
211
- if (!(options.compression !== 'UNCOMPRESSED')) {
212
- _context3.next = 9;
213
- break;
214
- }
215
- _context3.next = 6;
216
- return (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
217
- case 6:
218
- valuesBuf = _context3.sent;
219
- dataCursor = {
220
- buffer: valuesBuf,
221
- offset: 0,
222
- size: valuesBuf.length
223
- };
224
- cursor.offset = cursorEnd;
225
- case 9:
226
- rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
227
- rLevels = new Array(valueCount);
228
- if (options.column.rLevelMax > 0) {
229
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
230
- bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
231
- disableEnvelope: false
232
- });
233
- } else {
234
- rLevels.fill(0);
235
- }
236
- dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
237
- dLevels = new Array(valueCount);
238
- if (options.column.dLevelMax > 0) {
239
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
240
- bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
241
- disableEnvelope: false
242
- });
243
- } else {
244
- dLevels.fill(0);
245
- }
246
- valueCountNonNull = 0;
247
- _iterator = _createForOfIteratorHelper(dLevels);
248
- try {
249
- for (_iterator.s(); !(_step = _iterator.n()).done;) {
250
- dlvl = _step.value;
251
- if (dlvl === options.column.dLevelMax) {
252
- valueCountNonNull++;
253
- }
254
- }
255
- } catch (err) {
256
- _iterator.e(err);
257
- } finally {
258
- _iterator.f();
259
- }
260
- valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
261
- decodeOptions = {
262
- typeLength: options.column.typeLength,
263
- bitWidth: options.column.typeLength
264
- };
265
- values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
266
- return _context3.abrupt("return", {
267
- dlevels: dLevels,
268
- rlevels: rLevels,
269
- values: values,
270
- count: valueCount,
271
- pageHeader: header
272
- });
273
- case 22:
274
- case "end":
275
- return _context3.stop();
276
- }
277
- }, _callee3);
278
- }));
279
- return _decodeDataPage.apply(this, arguments);
280
- }
281
- function decodeDataPageV2(_x8, _x9, _x10) {
282
- return _decodeDataPageV.apply(this, arguments);
283
- }
284
- function _decodeDataPageV() {
285
- _decodeDataPageV = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee4(cursor, header, opts) {
286
- var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
287
- var cursorEnd, valueCount, valueCountNonNull, valueEncoding, rLevels, dLevels, valuesBufCursor, valuesBuf, decodeOptions, values;
288
- return _regenerator.default.wrap(function _callee4$(_context4) {
289
- while (1) switch (_context4.prev = _context4.next) {
290
- case 0:
291
- cursorEnd = cursor.offset + header.compressed_page_size;
292
- valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
293
- valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
294
- valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
295
- rLevels = new Array(valueCount);
296
- if (opts.column.rLevelMax > 0) {
297
- rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
298
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
299
- disableEnvelope: true
300
- });
301
- } else {
302
- rLevels.fill(0);
303
- }
304
- dLevels = new Array(valueCount);
305
- if (opts.column.dLevelMax > 0) {
306
- dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
307
- bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
308
- disableEnvelope: true
309
- });
310
- } else {
311
- dLevels.fill(0);
312
- }
313
- valuesBufCursor = cursor;
314
- if (!((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed)) {
315
- _context4.next = 15;
316
- break;
317
- }
318
- _context4.next = 12;
319
- return (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
320
- case 12:
321
- valuesBuf = _context4.sent;
322
- valuesBufCursor = {
323
- buffer: valuesBuf,
324
- offset: 0,
325
- size: valuesBuf.length
326
- };
327
- cursor.offset = cursorEnd;
328
- case 15:
329
- decodeOptions = {
330
- typeLength: opts.column.typeLength,
331
- bitWidth: opts.column.typeLength
332
- };
333
- values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
334
- return _context4.abrupt("return", {
335
- dlevels: dLevels,
336
- rlevels: rLevels,
337
- values: values,
338
- count: valueCount,
339
- pageHeader: header
340
- });
341
- case 18:
342
- case "end":
343
- return _context4.stop();
344
- }
345
- }, _callee4);
346
- }));
347
- return _decodeDataPageV.apply(this, arguments);
139
+ async function decodeDataPage(cursor, header, options) {
140
+ var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
141
+ const cursorEnd = cursor.offset + header.compressed_page_size;
142
+ const valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
143
+ let dataCursor = cursor;
144
+ if (options.compression !== 'UNCOMPRESSED') {
145
+ const valuesBuf = await (0, _compression.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
146
+ dataCursor = {
147
+ buffer: valuesBuf,
148
+ offset: 0,
149
+ size: valuesBuf.length
150
+ };
151
+ cursor.offset = cursorEnd;
152
+ }
153
+ const rLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
154
+ let rLevels = new Array(valueCount);
155
+ if (options.column.rLevelMax > 0) {
156
+ rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
157
+ bitWidth: (0, _readUtils.getBitWidth)(options.column.rLevelMax),
158
+ disableEnvelope: false
159
+ });
160
+ } else {
161
+ rLevels.fill(0);
162
+ }
163
+ const dLevelEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
164
+ let dLevels = new Array(valueCount);
165
+ if (options.column.dLevelMax > 0) {
166
+ dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
167
+ bitWidth: (0, _readUtils.getBitWidth)(options.column.dLevelMax),
168
+ disableEnvelope: false
169
+ });
170
+ } else {
171
+ dLevels.fill(0);
172
+ }
173
+ let valueCountNonNull = 0;
174
+ for (const dlvl of dLevels) {
175
+ if (dlvl === options.column.dLevelMax) {
176
+ valueCountNonNull++;
177
+ }
178
+ }
179
+ const valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
180
+ const decodeOptions = {
181
+ typeLength: options.column.typeLength,
182
+ bitWidth: options.column.typeLength
183
+ };
184
+ const values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
185
+ return {
186
+ dlevels: dLevels,
187
+ rlevels: rLevels,
188
+ values,
189
+ count: valueCount,
190
+ pageHeader: header
191
+ };
348
192
  }
349
- function decodeDictionaryPage(_x11, _x12, _x13) {
350
- return _decodeDictionaryPage.apply(this, arguments);
193
+ async function decodeDataPageV2(cursor, header, opts) {
194
+ var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
195
+ const cursorEnd = cursor.offset + header.compressed_page_size;
196
+ const valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
197
+ const valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
198
+ const valueEncoding = (0, _readUtils.getThriftEnum)(_parquetThrift.Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
199
+ let rLevels = new Array(valueCount);
200
+ if (opts.column.rLevelMax > 0) {
201
+ rLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
202
+ bitWidth: (0, _readUtils.getBitWidth)(opts.column.rLevelMax),
203
+ disableEnvelope: true
204
+ });
205
+ } else {
206
+ rLevels.fill(0);
207
+ }
208
+ let dLevels = new Array(valueCount);
209
+ if (opts.column.dLevelMax > 0) {
210
+ dLevels = decodeValues(_constants.PARQUET_RDLVL_TYPE, _constants.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
211
+ bitWidth: (0, _readUtils.getBitWidth)(opts.column.dLevelMax),
212
+ disableEnvelope: true
213
+ });
214
+ } else {
215
+ dLevels.fill(0);
216
+ }
217
+ let valuesBufCursor = cursor;
218
+ if ((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed) {
219
+ const valuesBuf = await (0, _compression.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
220
+ valuesBufCursor = {
221
+ buffer: valuesBuf,
222
+ offset: 0,
223
+ size: valuesBuf.length
224
+ };
225
+ cursor.offset = cursorEnd;
226
+ }
227
+ const decodeOptions = {
228
+ typeLength: opts.column.typeLength,
229
+ bitWidth: opts.column.typeLength
230
+ };
231
+ const values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
232
+ return {
233
+ dlevels: dLevels,
234
+ rlevels: rLevels,
235
+ values,
236
+ count: valueCount,
237
+ pageHeader: header
238
+ };
351
239
  }
352
- function _decodeDictionaryPage() {
353
- _decodeDictionaryPage = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee5(cursor, pageHeader, options) {
354
- var _pageHeader$dictionar;
355
- var cursorEnd, dictCursor, valuesBuf, numValues;
356
- return _regenerator.default.wrap(function _callee5$(_context5) {
357
- while (1) switch (_context5.prev = _context5.next) {
358
- case 0:
359
- cursorEnd = cursor.offset + pageHeader.compressed_page_size;
360
- dictCursor = {
361
- offset: 0,
362
- buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
363
- size: cursorEnd - cursor.offset
364
- };
365
- cursor.offset = cursorEnd;
366
- if (!(options.compression !== 'UNCOMPRESSED')) {
367
- _context5.next = 9;
368
- break;
369
- }
370
- _context5.next = 6;
371
- return (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
372
- case 6:
373
- valuesBuf = _context5.sent;
374
- dictCursor = {
375
- buffer: valuesBuf,
376
- offset: 0,
377
- size: valuesBuf.length
378
- };
379
- cursor.offset = cursorEnd;
380
- case 9:
381
- numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
382
- return _context5.abrupt("return", decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(function (d) {
383
- return d.toString();
384
- }));
385
- case 11:
386
- case "end":
387
- return _context5.stop();
388
- }
389
- }, _callee5);
390
- }));
391
- return _decodeDictionaryPage.apply(this, arguments);
240
+ async function decodeDictionaryPage(cursor, pageHeader, options) {
241
+ var _pageHeader$dictionar;
242
+ const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
243
+ let dictCursor = {
244
+ offset: 0,
245
+ buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
246
+ size: cursorEnd - cursor.offset
247
+ };
248
+ cursor.offset = cursorEnd;
249
+ if (options.compression !== 'UNCOMPRESSED') {
250
+ const valuesBuf = await (0, _compression.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
251
+ dictCursor = {
252
+ buffer: valuesBuf,
253
+ offset: 0,
254
+ size: valuesBuf.length
255
+ };
256
+ cursor.offset = cursorEnd;
257
+ }
258
+ const numValues = (pageHeader === null || pageHeader === void 0 ? void 0 : (_pageHeader$dictionar = pageHeader.dictionary_page_header) === null || _pageHeader$dictionar === void 0 ? void 0 : _pageHeader$dictionar.num_values) || 0;
259
+ return decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map(d => d.toString());
392
260
  }
393
261
  //# sourceMappingURL=decoders.js.map