@loaders.gl/parquet 4.0.0-alpha.22 → 4.0.0-alpha.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/dist.min.js +17 -17
  2. package/dist/dist.min.js.map +3 -3
  3. package/dist/es5/index.js +0 -3
  4. package/dist/es5/index.js.map +1 -1
  5. package/dist/es5/parquet-loader.js +1 -1
  6. package/dist/es5/parquet-wasm-loader.js +1 -1
  7. package/dist/es5/parquet-wasm-writer.js +1 -1
  8. package/dist/es5/parquet-writer.js +2 -3
  9. package/dist/es5/parquet-writer.js.map +1 -1
  10. package/dist/esm/index.js +0 -1
  11. package/dist/esm/index.js.map +1 -1
  12. package/dist/esm/parquet-loader.js +1 -1
  13. package/dist/esm/parquet-wasm-loader.js +1 -1
  14. package/dist/esm/parquet-wasm-writer.js +1 -1
  15. package/dist/esm/parquet-writer.js +2 -3
  16. package/dist/esm/parquet-writer.js.map +1 -1
  17. package/dist/index.d.ts +0 -1
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/parquet-worker.js +17 -17
  20. package/dist/parquet-worker.js.map +3 -3
  21. package/dist/parquet-writer.d.ts.map +1 -1
  22. package/package.json +11 -9
  23. package/src/index.ts +0 -3
  24. package/src/parquet-writer.ts +1 -3
  25. package/dist/bundle.js +0 -5
  26. package/dist/constants.js +0 -18
  27. package/dist/index.js +0 -58
  28. package/dist/lib/arrow/convert-columns-to-row-group.js +0 -1
  29. package/dist/lib/arrow/convert-row-group-to-columns.js +0 -12
  30. package/dist/lib/arrow/convert-schema-from-parquet.js +0 -86
  31. package/dist/lib/arrow/convert-schema-to-parquet.js +0 -71
  32. package/dist/lib/geo/decode-geo-metadata.js +0 -77
  33. package/dist/lib/geo/geoparquet-schema.js +0 -69
  34. package/dist/lib/parsers/parse-parquet-to-columns.js +0 -46
  35. package/dist/lib/parsers/parse-parquet-to-rows.js +0 -37
  36. package/dist/lib/wasm/encode-parquet-wasm.js +0 -30
  37. package/dist/lib/wasm/load-wasm/index.js +0 -5
  38. package/dist/lib/wasm/load-wasm/load-wasm-browser.js +0 -38
  39. package/dist/lib/wasm/load-wasm/load-wasm-node.js +0 -31
  40. package/dist/lib/wasm/parse-parquet-wasm.js +0 -27
  41. package/dist/parquet-loader.js +0 -41
  42. package/dist/parquet-wasm-loader.js +0 -26
  43. package/dist/parquet-wasm-writer.js +0 -24
  44. package/dist/parquet-writer.js +0 -22
  45. package/dist/parquetjs/codecs/declare.js +0 -2
  46. package/dist/parquetjs/codecs/dictionary.js +0 -14
  47. package/dist/parquetjs/codecs/index.js +0 -55
  48. package/dist/parquetjs/codecs/plain.js +0 -211
  49. package/dist/parquetjs/codecs/rle.js +0 -145
  50. package/dist/parquetjs/compression.js +0 -183
  51. package/dist/parquetjs/encoder/parquet-encoder.js +0 -484
  52. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +0 -15
  53. package/dist/parquetjs/parquet-thrift/BsonType.js +0 -62
  54. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +0 -211
  55. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +0 -217
  56. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +0 -402
  57. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +0 -108
  58. package/dist/parquetjs/parquet-thrift/CompressionCodec.js +0 -20
  59. package/dist/parquetjs/parquet-thrift/ConvertedType.js +0 -34
  60. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +0 -170
  61. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -230
  62. package/dist/parquetjs/parquet-thrift/DateType.js +0 -62
  63. package/dist/parquetjs/parquet-thrift/DecimalType.js +0 -109
  64. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -126
  65. package/dist/parquetjs/parquet-thrift/Encoding.js +0 -20
  66. package/dist/parquetjs/parquet-thrift/EnumType.js +0 -62
  67. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -15
  68. package/dist/parquetjs/parquet-thrift/FileMetaData.js +0 -260
  69. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +0 -62
  70. package/dist/parquetjs/parquet-thrift/IntType.js +0 -109
  71. package/dist/parquetjs/parquet-thrift/JsonType.js +0 -62
  72. package/dist/parquetjs/parquet-thrift/KeyValue.js +0 -106
  73. package/dist/parquetjs/parquet-thrift/ListType.js +0 -62
  74. package/dist/parquetjs/parquet-thrift/LogicalType.js +0 -384
  75. package/dist/parquetjs/parquet-thrift/MapType.js +0 -62
  76. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +0 -62
  77. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +0 -62
  78. package/dist/parquetjs/parquet-thrift/NullType.js +0 -62
  79. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +0 -101
  80. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +0 -131
  81. package/dist/parquetjs/parquet-thrift/PageHeader.js +0 -220
  82. package/dist/parquetjs/parquet-thrift/PageLocation.js +0 -145
  83. package/dist/parquetjs/parquet-thrift/PageType.js +0 -16
  84. package/dist/parquetjs/parquet-thrift/RowGroup.js +0 -186
  85. package/dist/parquetjs/parquet-thrift/SchemaElement.js +0 -243
  86. package/dist/parquetjs/parquet-thrift/SortingColumn.js +0 -131
  87. package/dist/parquetjs/parquet-thrift/Statistics.js +0 -180
  88. package/dist/parquetjs/parquet-thrift/StringType.js +0 -62
  89. package/dist/parquetjs/parquet-thrift/TimeType.js +0 -110
  90. package/dist/parquetjs/parquet-thrift/TimeUnit.js +0 -131
  91. package/dist/parquetjs/parquet-thrift/TimestampType.js +0 -110
  92. package/dist/parquetjs/parquet-thrift/Type.js +0 -20
  93. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -62
  94. package/dist/parquetjs/parquet-thrift/UUIDType.js +0 -62
  95. package/dist/parquetjs/parquet-thrift/index.js +0 -65
  96. package/dist/parquetjs/parser/decoders.js +0 -318
  97. package/dist/parquetjs/parser/parquet-reader.js +0 -200
  98. package/dist/parquetjs/schema/declare.js +0 -12
  99. package/dist/parquetjs/schema/schema.js +0 -162
  100. package/dist/parquetjs/schema/shred.js +0 -355
  101. package/dist/parquetjs/schema/types.js +0 -416
  102. package/dist/parquetjs/utils/file-utils.js +0 -43
  103. package/dist/parquetjs/utils/read-utils.js +0 -109
  104. package/dist/workers/parquet-worker.js +0 -5
@@ -1,31 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.loadWasm = void 0;
27
- const wasmNode = __importStar(require("parquet-wasm/node/arrow1"));
28
- async function loadWasm(wasmUrl) {
29
- return wasmNode;
30
- }
31
- exports.loadWasm = loadWasm;
@@ -1,27 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.parseParquetWasm = void 0;
4
- const apache_arrow_1 = require("apache-arrow");
5
- const load_wasm_node_1 = require("./load-wasm/load-wasm-node");
6
- async function parseParquetWasm(arrayBuffer, options) {
7
- const wasmUrl = options?.parquet?.wasmUrl;
8
- const wasm = await (0, load_wasm_node_1.loadWasm)(wasmUrl);
9
- const arr = new Uint8Array(arrayBuffer);
10
- const arrowIPCUint8Arr = wasm.readParquet(arr);
11
- const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(arrowIPCUint8Arr.byteOffset, arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset);
12
- const arrowTable = tableFromIPC(arrowIPCBuffer);
13
- return arrowTable;
14
- }
15
- exports.parseParquetWasm = parseParquetWasm;
16
- /**
17
- * Deserialize the IPC format into a {@link Table}. This function is a
18
- * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
19
- */
20
- function tableFromIPC(input) {
21
- const reader = apache_arrow_1.RecordBatchStreamReader.from(input);
22
- const recordBatches = [];
23
- for (const recordBatch of reader) {
24
- recordBatches.push(recordBatch);
25
- }
26
- return new apache_arrow_1.Table(recordBatches);
27
- }
@@ -1,41 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ParqueColumnnartLoader = exports.ParquetLoader = void 0;
4
- // __VERSION__ is injected by babel-plugin-version-inline
5
- // @ts-ignore TS2304: Cannot find name '__VERSION__'.
6
- const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
7
- const DEFAULT_PARQUET_LOADER_OPTIONS = {
8
- parquet: {
9
- type: 'object-row-table',
10
- url: undefined,
11
- columnList: [],
12
- geoparquet: true
13
- }
14
- };
15
- /** ParquetJS table loader */
16
- exports.ParquetLoader = {
17
- name: 'Apache Parquet',
18
- id: 'parquet',
19
- module: 'parquet',
20
- version: VERSION,
21
- worker: true,
22
- category: 'table',
23
- extensions: ['parquet'],
24
- mimeTypes: ['application/octet-stream'],
25
- binary: true,
26
- tests: ['PAR1', 'PARE'],
27
- options: DEFAULT_PARQUET_LOADER_OPTIONS
28
- };
29
- exports.ParqueColumnnartLoader = {
30
- name: 'Apache Parquet',
31
- id: 'parquet',
32
- module: 'parquet',
33
- version: VERSION,
34
- worker: true,
35
- category: 'table',
36
- extensions: ['parquet'],
37
- mimeTypes: ['application/octet-stream'],
38
- binary: true,
39
- tests: ['PAR1', 'PARE'],
40
- options: DEFAULT_PARQUET_LOADER_OPTIONS
41
- };
@@ -1,26 +0,0 @@
1
- "use strict";
2
- // loaders.gl, MIT license
3
- Object.defineProperty(exports, "__esModule", { value: true });
4
- exports.ParquetWasmLoader = void 0;
5
- // __VERSION__ is injected by babel-plugin-version-inline
6
- // @ts-ignore TS2304: Cannot find name '__VERSION__'.
7
- const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
8
- /** Parquet WASM table loader */
9
- exports.ParquetWasmLoader = {
10
- name: 'Apache Parquet',
11
- id: 'parquet-wasm',
12
- module: 'parquet',
13
- version: VERSION,
14
- worker: false,
15
- category: 'table',
16
- extensions: ['parquet'],
17
- mimeTypes: ['application/octet-stream'],
18
- binary: true,
19
- tests: ['PAR1', 'PARE'],
20
- options: {
21
- parquet: {
22
- type: 'arrow-table',
23
- wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
24
- }
25
- }
26
- };
@@ -1,24 +0,0 @@
1
- "use strict";
2
- // loaders.gl, MIT license
3
- Object.defineProperty(exports, "__esModule", { value: true });
4
- exports.ParquetWasmWriter = void 0;
5
- const encode_parquet_wasm_1 = require("./lib/wasm/encode-parquet-wasm");
6
- // __VERSION__ is injected by babel-plugin-version-inline
7
- // @ts-ignore TS2304: Cannot find name '__VERSION__'.
8
- const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
9
- /** Parquet WASM writer */
10
- exports.ParquetWasmWriter = {
11
- name: 'Apache Parquet',
12
- id: 'parquet-wasm',
13
- module: 'parquet',
14
- version: VERSION,
15
- extensions: ['parquet'],
16
- mimeTypes: ['application/octet-stream'],
17
- encode: encode_parquet_wasm_1.encode,
18
- binary: true,
19
- options: {
20
- parquet: {
21
- wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
22
- }
23
- }
24
- };
@@ -1,22 +0,0 @@
1
- "use strict";
2
- // loaders.gl, MIT license
3
- Object.defineProperty(exports, "__esModule", { value: true });
4
- exports.ParquetWriter = void 0;
5
- // __VERSION__ is injected by babel-plugin-version-inline
6
- // @ts-ignore TS2304: Cannot find name '__VERSION__'.
7
- const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
8
- const DEFAULT_PARQUET_LOADER_OPTIONS = {};
9
- exports.ParquetWriter = {
10
- name: 'Apache Parquet',
11
- id: 'parquet',
12
- module: 'parquet',
13
- version: VERSION,
14
- extensions: ['parquet'],
15
- mimeTypes: ['application/octet-stream'],
16
- encodeSync,
17
- binary: true,
18
- options: DEFAULT_PARQUET_LOADER_OPTIONS
19
- };
20
- function encodeSync(data, options) {
21
- return new ArrayBuffer(0);
22
- }
@@ -1,2 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
@@ -1,14 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.encodeValues = exports.decodeValues = void 0;
4
- const rle_1 = require("./rle");
5
- function decodeValues(type, cursor, count, opts) {
6
- opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
7
- cursor.offset += 1;
8
- return (0, rle_1.decodeValues)(type, cursor, count, { ...opts, disableEnvelope: true });
9
- }
10
- exports.decodeValues = decodeValues;
11
- function encodeValues(type, cursor, count, opts) {
12
- throw new Error('Encode dictionary functionality is not supported');
13
- }
14
- exports.encodeValues = encodeValues;
@@ -1,55 +0,0 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
- var __exportStar = (this && this.__exportStar) || function(m, exports) {
26
- for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
27
- };
28
- Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.PARQUET_CODECS = void 0;
30
- const PLAIN = __importStar(require("./plain"));
31
- const RLE = __importStar(require("./rle"));
32
- const DICTIONARY = __importStar(require("./dictionary"));
33
- __exportStar(require("./declare"), exports);
34
- exports.PARQUET_CODECS = {
35
- PLAIN: {
36
- encodeValues: PLAIN.encodeValues,
37
- decodeValues: PLAIN.decodeValues
38
- },
39
- RLE: {
40
- encodeValues: RLE.encodeValues,
41
- decodeValues: RLE.decodeValues
42
- },
43
- // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.
44
- PLAIN_DICTIONARY: {
45
- // @ts-ignore
46
- encodeValues: DICTIONARY.encodeValues,
47
- decodeValues: DICTIONARY.decodeValues
48
- },
49
- // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.
50
- RLE_DICTIONARY: {
51
- // @ts-ignore
52
- encodeValues: DICTIONARY.encodeValues,
53
- decodeValues: DICTIONARY.decodeValues
54
- }
55
- };
@@ -1,211 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.decodeValues = exports.encodeValues = void 0;
7
- const int53_1 = __importDefault(require("int53"));
8
- function encodeValues(type, values, opts) {
9
- switch (type) {
10
- case 'BOOLEAN':
11
- return encodeValues_BOOLEAN(values);
12
- case 'INT32':
13
- return encodeValues_INT32(values);
14
- case 'INT64':
15
- return encodeValues_INT64(values);
16
- case 'INT96':
17
- return encodeValues_INT96(values);
18
- case 'FLOAT':
19
- return encodeValues_FLOAT(values);
20
- case 'DOUBLE':
21
- return encodeValues_DOUBLE(values);
22
- case 'BYTE_ARRAY':
23
- return encodeValues_BYTE_ARRAY(values);
24
- case 'FIXED_LEN_BYTE_ARRAY':
25
- return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
26
- default:
27
- throw new Error(`unsupported type: ${type}`);
28
- }
29
- }
30
- exports.encodeValues = encodeValues;
31
- function decodeValues(type, cursor, count, opts) {
32
- switch (type) {
33
- case 'BOOLEAN':
34
- return decodeValues_BOOLEAN(cursor, count);
35
- case 'INT32':
36
- return decodeValues_INT32(cursor, count);
37
- case 'INT64':
38
- return decodeValues_INT64(cursor, count);
39
- case 'INT96':
40
- return decodeValues_INT96(cursor, count);
41
- case 'FLOAT':
42
- return decodeValues_FLOAT(cursor, count);
43
- case 'DOUBLE':
44
- return decodeValues_DOUBLE(cursor, count);
45
- case 'BYTE_ARRAY':
46
- return decodeValues_BYTE_ARRAY(cursor, count);
47
- case 'FIXED_LEN_BYTE_ARRAY':
48
- return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
49
- default:
50
- throw new Error(`unsupported type: ${type}`);
51
- }
52
- }
53
- exports.decodeValues = decodeValues;
54
- function encodeValues_BOOLEAN(values) {
55
- const buf = Buffer.alloc(Math.ceil(values.length / 8));
56
- buf.fill(0);
57
- for (let i = 0; i < values.length; i++) {
58
- if (values[i]) {
59
- buf[Math.floor(i / 8)] |= 1 << i % 8;
60
- }
61
- }
62
- return buf;
63
- }
64
- function decodeValues_BOOLEAN(cursor, count) {
65
- const values = [];
66
- for (let i = 0; i < count; i++) {
67
- const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
68
- values.push((b & (1 << i % 8)) > 0);
69
- }
70
- cursor.offset += Math.ceil(count / 8);
71
- return values;
72
- }
73
- function encodeValues_INT32(values) {
74
- const buf = Buffer.alloc(4 * values.length);
75
- for (let i = 0; i < values.length; i++) {
76
- buf.writeInt32LE(values[i], i * 4);
77
- }
78
- return buf;
79
- }
80
- function decodeValues_INT32(cursor, count) {
81
- const values = [];
82
- for (let i = 0; i < count; i++) {
83
- values.push(cursor.buffer.readInt32LE(cursor.offset));
84
- cursor.offset += 4;
85
- }
86
- return values;
87
- }
88
- function encodeValues_INT64(values) {
89
- const buf = Buffer.alloc(8 * values.length);
90
- for (let i = 0; i < values.length; i++) {
91
- int53_1.default.writeInt64LE(values[i], buf, i * 8);
92
- }
93
- return buf;
94
- }
95
- function decodeValues_INT64(cursor, count) {
96
- const values = [];
97
- for (let i = 0; i < count; i++) {
98
- values.push(int53_1.default.readInt64LE(cursor.buffer, cursor.offset));
99
- cursor.offset += 8;
100
- }
101
- return values;
102
- }
103
- function encodeValues_INT96(values) {
104
- const buf = Buffer.alloc(12 * values.length);
105
- for (let i = 0; i < values.length; i++) {
106
- if (values[i] >= 0) {
107
- int53_1.default.writeInt64LE(values[i], buf, i * 12);
108
- buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision
109
- }
110
- else {
111
- int53_1.default.writeInt64LE(~-values[i] + 1, buf, i * 12);
112
- buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision
113
- }
114
- }
115
- return buf;
116
- }
117
- function decodeValues_INT96(cursor, count) {
118
- const values = [];
119
- for (let i = 0; i < count; i++) {
120
- const low = int53_1.default.readInt64LE(cursor.buffer, cursor.offset);
121
- const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
122
- if (high === 0xffffffff) {
123
- values.push(~-low + 1); // truncate to 64 actual precision
124
- }
125
- else {
126
- values.push(low); // truncate to 64 actual precision
127
- }
128
- cursor.offset += 12;
129
- }
130
- return values;
131
- }
132
- function encodeValues_FLOAT(values) {
133
- const buf = Buffer.alloc(4 * values.length);
134
- for (let i = 0; i < values.length; i++) {
135
- buf.writeFloatLE(values[i], i * 4);
136
- }
137
- return buf;
138
- }
139
- function decodeValues_FLOAT(cursor, count) {
140
- const values = [];
141
- for (let i = 0; i < count; i++) {
142
- values.push(cursor.buffer.readFloatLE(cursor.offset));
143
- cursor.offset += 4;
144
- }
145
- return values;
146
- }
147
- function encodeValues_DOUBLE(values) {
148
- const buf = Buffer.alloc(8 * values.length);
149
- for (let i = 0; i < values.length; i++) {
150
- buf.writeDoubleLE(values[i], i * 8);
151
- }
152
- return buf;
153
- }
154
- function decodeValues_DOUBLE(cursor, count) {
155
- const values = [];
156
- for (let i = 0; i < count; i++) {
157
- values.push(cursor.buffer.readDoubleLE(cursor.offset));
158
- cursor.offset += 8;
159
- }
160
- return values;
161
- }
162
- function encodeValues_BYTE_ARRAY(values) {
163
- // tslint:disable-next-line:variable-name
164
- let buf_len = 0;
165
- for (let i = 0; i < values.length; i++) {
166
- values[i] = Buffer.from(values[i]);
167
- buf_len += 4 + values[i].length;
168
- }
169
- const buf = Buffer.alloc(buf_len);
170
- // tslint:disable-next-line:variable-name
171
- let buf_pos = 0;
172
- for (let i = 0; i < values.length; i++) {
173
- buf.writeUInt32LE(values[i].length, buf_pos);
174
- values[i].copy(buf, buf_pos + 4);
175
- buf_pos += 4 + values[i].length;
176
- }
177
- return buf;
178
- }
179
- function decodeValues_BYTE_ARRAY(cursor, count) {
180
- const values = [];
181
- for (let i = 0; i < count; i++) {
182
- const len = cursor.buffer.readUInt32LE(cursor.offset);
183
- cursor.offset += 4;
184
- values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
185
- cursor.offset += len;
186
- }
187
- return values;
188
- }
189
- function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
190
- if (!opts.typeLength) {
191
- throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
192
- }
193
- for (let i = 0; i < values.length; i++) {
194
- values[i] = Buffer.from(values[i]);
195
- if (values[i].length !== opts.typeLength) {
196
- throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
197
- }
198
- }
199
- return Buffer.concat(values);
200
- }
201
- function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
202
- const values = [];
203
- if (!opts.typeLength) {
204
- throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
205
- }
206
- for (let i = 0; i < count; i++) {
207
- values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
208
- cursor.offset += opts.typeLength;
209
- }
210
- return values;
211
- }
@@ -1,145 +0,0 @@
1
- "use strict";
2
- // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
3
- var __importDefault = (this && this.__importDefault) || function (mod) {
4
- return (mod && mod.__esModule) ? mod : { "default": mod };
5
- };
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- exports.decodeValues = exports.encodeValues = void 0;
8
- const varint_1 = __importDefault(require("varint"));
9
- // eslint-disable-next-line max-statements, complexity
10
- function encodeValues(type, values, opts) {
11
- if (!('bitWidth' in opts)) {
12
- throw new Error('bitWidth is required');
13
- }
14
- switch (type) {
15
- case 'BOOLEAN':
16
- case 'INT32':
17
- case 'INT64':
18
- // tslint:disable-next-line:no-parameter-reassignment
19
- values = values.map((x) => parseInt(x, 10));
20
- break;
21
- default:
22
- throw new Error(`unsupported type: ${type}`);
23
- }
24
- let buf = Buffer.alloc(0);
25
- let run = [];
26
- let repeats = 0;
27
- for (let i = 0; i < values.length; i++) {
28
- // If we are at the beginning of a run and the next value is same we start
29
- // collecting repeated values
30
- if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
31
- // If we have any data in runs we need to encode them
32
- if (run.length) {
33
- buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
34
- run = [];
35
- }
36
- repeats = 1;
37
- }
38
- else if (repeats > 0 && values[i] === values[i - 1]) {
39
- repeats += 1;
40
- }
41
- else {
42
- // If values changes we need to post any previous repeated values
43
- if (repeats) {
44
- buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
45
- repeats = 0;
46
- }
47
- run.push(values[i]);
48
- }
49
- }
50
- if (repeats) {
51
- buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
52
- }
53
- else if (run.length) {
54
- buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
55
- }
56
- if (opts.disableEnvelope) {
57
- return buf;
58
- }
59
- const envelope = Buffer.alloc(buf.length + 4);
60
- envelope.writeUInt32LE(buf.length, undefined);
61
- buf.copy(envelope, 4);
62
- return envelope;
63
- }
64
- exports.encodeValues = encodeValues;
65
- function decodeValues(type, cursor, count, opts) {
66
- if (!('bitWidth' in opts)) {
67
- throw new Error('bitWidth is required');
68
- }
69
- if (!opts.disableEnvelope) {
70
- cursor.offset += 4;
71
- }
72
- let values = [];
73
- while (values.length < count) {
74
- const header = varint_1.default.decode(cursor.buffer, cursor.offset);
75
- cursor.offset += varint_1.default.encodingLength(header);
76
- if (header & 1) {
77
- const count = (header >> 1) * 8;
78
- values.push(...decodeRunBitpacked(cursor, count, opts));
79
- }
80
- else {
81
- const count = header >> 1;
82
- values.push(...decodeRunRepeated(cursor, count, opts));
83
- }
84
- }
85
- values = values.slice(0, count);
86
- if (values.length !== count) {
87
- throw new Error('invalid RLE encoding');
88
- }
89
- return values;
90
- }
91
- exports.decodeValues = decodeValues;
92
- function decodeRunBitpacked(cursor, count, opts) {
93
- // @ts-ignore
94
- const bitWidth = opts.bitWidth;
95
- if (count % 8 !== 0) {
96
- throw new Error('must be a multiple of 8');
97
- }
98
- // tslint:disable-next-line:prefer-array-literal
99
- const values = new Array(count).fill(0);
100
- for (let b = 0; b < bitWidth * count; b++) {
101
- if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {
102
- values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
103
- }
104
- }
105
- cursor.offset += bitWidth * (count / 8);
106
- return values;
107
- }
108
- function decodeRunRepeated(cursor, count, opts) {
109
- // @ts-ignore
110
- const bitWidth = opts.bitWidth;
111
- let value = 0;
112
- for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
113
- // eslint-disable-next-line
114
- value << 8; // TODO - this looks wrong
115
- value += cursor.buffer[cursor.offset];
116
- cursor.offset += 1;
117
- }
118
- // tslint:disable-next-line:prefer-array-literal
119
- return new Array(count).fill(value);
120
- }
121
- function encodeRunBitpacked(values, opts) {
122
- // @ts-ignore
123
- const bitWidth = opts.bitWidth;
124
- for (let i = 0; i < values.length % 8; i++) {
125
- values.push(0);
126
- }
127
- const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
128
- for (let b = 0; b < bitWidth * values.length; b++) {
129
- if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {
130
- buf[Math.floor(b / 8)] |= 1 << b % 8;
131
- }
132
- }
133
- return Buffer.concat([Buffer.from(varint_1.default.encode(((values.length / 8) << 1) | 1)), buf]);
134
- }
135
- function encodeRunRepeated(value, count, opts) {
136
- // @ts-ignore
137
- const bitWidth = opts.bitWidth;
138
- const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
139
- for (let i = 0; i < buf.length; i++) {
140
- buf.writeUInt8(value & 0xff, i);
141
- // eslint-disable-next-line
142
- value >> 8; // TODO - this looks wrong
143
- }
144
- return Buffer.concat([Buffer.from(varint_1.default.encode(count << 1)), buf]);
145
- }