@loaders.gl/parquet 4.0.0-alpha.22 → 4.0.0-alpha.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +17 -17
- package/dist/dist.min.js.map +3 -3
- package/dist/es5/index.js +0 -3
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/parquet-loader.js +1 -1
- package/dist/es5/parquet-wasm-loader.js +1 -1
- package/dist/es5/parquet-wasm-writer.js +1 -1
- package/dist/es5/parquet-writer.js +2 -3
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/esm/index.js +0 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/parquet-loader.js +1 -1
- package/dist/esm/parquet-wasm-loader.js +1 -1
- package/dist/esm/parquet-wasm-writer.js +1 -1
- package/dist/esm/parquet-writer.js +2 -3
- package/dist/esm/parquet-writer.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/parquet-worker.js +17 -17
- package/dist/parquet-worker.js.map +3 -3
- package/dist/parquet-writer.d.ts.map +1 -1
- package/package.json +11 -9
- package/src/index.ts +0 -3
- package/src/parquet-writer.ts +1 -3
- package/dist/bundle.js +0 -5
- package/dist/constants.js +0 -18
- package/dist/index.js +0 -58
- package/dist/lib/arrow/convert-columns-to-row-group.js +0 -1
- package/dist/lib/arrow/convert-row-group-to-columns.js +0 -12
- package/dist/lib/arrow/convert-schema-from-parquet.js +0 -86
- package/dist/lib/arrow/convert-schema-to-parquet.js +0 -71
- package/dist/lib/geo/decode-geo-metadata.js +0 -77
- package/dist/lib/geo/geoparquet-schema.js +0 -69
- package/dist/lib/parsers/parse-parquet-to-columns.js +0 -46
- package/dist/lib/parsers/parse-parquet-to-rows.js +0 -37
- package/dist/lib/wasm/encode-parquet-wasm.js +0 -30
- package/dist/lib/wasm/load-wasm/index.js +0 -5
- package/dist/lib/wasm/load-wasm/load-wasm-browser.js +0 -38
- package/dist/lib/wasm/load-wasm/load-wasm-node.js +0 -31
- package/dist/lib/wasm/parse-parquet-wasm.js +0 -27
- package/dist/parquet-loader.js +0 -41
- package/dist/parquet-wasm-loader.js +0 -26
- package/dist/parquet-wasm-writer.js +0 -24
- package/dist/parquet-writer.js +0 -22
- package/dist/parquetjs/codecs/declare.js +0 -2
- package/dist/parquetjs/codecs/dictionary.js +0 -14
- package/dist/parquetjs/codecs/index.js +0 -55
- package/dist/parquetjs/codecs/plain.js +0 -211
- package/dist/parquetjs/codecs/rle.js +0 -145
- package/dist/parquetjs/compression.js +0 -183
- package/dist/parquetjs/encoder/parquet-encoder.js +0 -484
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +0 -15
- package/dist/parquetjs/parquet-thrift/BsonType.js +0 -62
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +0 -211
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +0 -217
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +0 -402
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +0 -108
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +0 -20
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +0 -34
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +0 -170
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +0 -230
- package/dist/parquetjs/parquet-thrift/DateType.js +0 -62
- package/dist/parquetjs/parquet-thrift/DecimalType.js +0 -109
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +0 -126
- package/dist/parquetjs/parquet-thrift/Encoding.js +0 -20
- package/dist/parquetjs/parquet-thrift/EnumType.js +0 -62
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +0 -15
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +0 -260
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +0 -62
- package/dist/parquetjs/parquet-thrift/IntType.js +0 -109
- package/dist/parquetjs/parquet-thrift/JsonType.js +0 -62
- package/dist/parquetjs/parquet-thrift/KeyValue.js +0 -106
- package/dist/parquetjs/parquet-thrift/ListType.js +0 -62
- package/dist/parquetjs/parquet-thrift/LogicalType.js +0 -384
- package/dist/parquetjs/parquet-thrift/MapType.js +0 -62
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +0 -62
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +0 -62
- package/dist/parquetjs/parquet-thrift/NullType.js +0 -62
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +0 -101
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +0 -131
- package/dist/parquetjs/parquet-thrift/PageHeader.js +0 -220
- package/dist/parquetjs/parquet-thrift/PageLocation.js +0 -145
- package/dist/parquetjs/parquet-thrift/PageType.js +0 -16
- package/dist/parquetjs/parquet-thrift/RowGroup.js +0 -186
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +0 -243
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +0 -131
- package/dist/parquetjs/parquet-thrift/Statistics.js +0 -180
- package/dist/parquetjs/parquet-thrift/StringType.js +0 -62
- package/dist/parquetjs/parquet-thrift/TimeType.js +0 -110
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +0 -131
- package/dist/parquetjs/parquet-thrift/TimestampType.js +0 -110
- package/dist/parquetjs/parquet-thrift/Type.js +0 -20
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +0 -62
- package/dist/parquetjs/parquet-thrift/UUIDType.js +0 -62
- package/dist/parquetjs/parquet-thrift/index.js +0 -65
- package/dist/parquetjs/parser/decoders.js +0 -318
- package/dist/parquetjs/parser/parquet-reader.js +0 -200
- package/dist/parquetjs/schema/declare.js +0 -12
- package/dist/parquetjs/schema/schema.js +0 -162
- package/dist/parquetjs/schema/shred.js +0 -355
- package/dist/parquetjs/schema/types.js +0 -416
- package/dist/parquetjs/utils/file-utils.js +0 -43
- package/dist/parquetjs/utils/read-utils.js +0 -109
- package/dist/workers/parquet-worker.js +0 -5
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.loadWasm = void 0;
|
|
27
|
-
const wasmNode = __importStar(require("parquet-wasm/node/arrow1"));
|
|
28
|
-
async function loadWasm(wasmUrl) {
|
|
29
|
-
return wasmNode;
|
|
30
|
-
}
|
|
31
|
-
exports.loadWasm = loadWasm;
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseParquetWasm = void 0;
|
|
4
|
-
const apache_arrow_1 = require("apache-arrow");
|
|
5
|
-
const load_wasm_node_1 = require("./load-wasm/load-wasm-node");
|
|
6
|
-
async function parseParquetWasm(arrayBuffer, options) {
|
|
7
|
-
const wasmUrl = options?.parquet?.wasmUrl;
|
|
8
|
-
const wasm = await (0, load_wasm_node_1.loadWasm)(wasmUrl);
|
|
9
|
-
const arr = new Uint8Array(arrayBuffer);
|
|
10
|
-
const arrowIPCUint8Arr = wasm.readParquet(arr);
|
|
11
|
-
const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(arrowIPCUint8Arr.byteOffset, arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset);
|
|
12
|
-
const arrowTable = tableFromIPC(arrowIPCBuffer);
|
|
13
|
-
return arrowTable;
|
|
14
|
-
}
|
|
15
|
-
exports.parseParquetWasm = parseParquetWasm;
|
|
16
|
-
/**
|
|
17
|
-
* Deserialize the IPC format into a {@link Table}. This function is a
|
|
18
|
-
* convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
|
|
19
|
-
*/
|
|
20
|
-
function tableFromIPC(input) {
|
|
21
|
-
const reader = apache_arrow_1.RecordBatchStreamReader.from(input);
|
|
22
|
-
const recordBatches = [];
|
|
23
|
-
for (const recordBatch of reader) {
|
|
24
|
-
recordBatches.push(recordBatch);
|
|
25
|
-
}
|
|
26
|
-
return new apache_arrow_1.Table(recordBatches);
|
|
27
|
-
}
|
package/dist/parquet-loader.js
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ParqueColumnnartLoader = exports.ParquetLoader = void 0;
|
|
4
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
7
|
-
const DEFAULT_PARQUET_LOADER_OPTIONS = {
|
|
8
|
-
parquet: {
|
|
9
|
-
type: 'object-row-table',
|
|
10
|
-
url: undefined,
|
|
11
|
-
columnList: [],
|
|
12
|
-
geoparquet: true
|
|
13
|
-
}
|
|
14
|
-
};
|
|
15
|
-
/** ParquetJS table loader */
|
|
16
|
-
exports.ParquetLoader = {
|
|
17
|
-
name: 'Apache Parquet',
|
|
18
|
-
id: 'parquet',
|
|
19
|
-
module: 'parquet',
|
|
20
|
-
version: VERSION,
|
|
21
|
-
worker: true,
|
|
22
|
-
category: 'table',
|
|
23
|
-
extensions: ['parquet'],
|
|
24
|
-
mimeTypes: ['application/octet-stream'],
|
|
25
|
-
binary: true,
|
|
26
|
-
tests: ['PAR1', 'PARE'],
|
|
27
|
-
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
28
|
-
};
|
|
29
|
-
exports.ParqueColumnnartLoader = {
|
|
30
|
-
name: 'Apache Parquet',
|
|
31
|
-
id: 'parquet',
|
|
32
|
-
module: 'parquet',
|
|
33
|
-
version: VERSION,
|
|
34
|
-
worker: true,
|
|
35
|
-
category: 'table',
|
|
36
|
-
extensions: ['parquet'],
|
|
37
|
-
mimeTypes: ['application/octet-stream'],
|
|
38
|
-
binary: true,
|
|
39
|
-
tests: ['PAR1', 'PARE'],
|
|
40
|
-
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
41
|
-
};
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.ParquetWasmLoader = void 0;
|
|
5
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
8
|
-
/** Parquet WASM table loader */
|
|
9
|
-
exports.ParquetWasmLoader = {
|
|
10
|
-
name: 'Apache Parquet',
|
|
11
|
-
id: 'parquet-wasm',
|
|
12
|
-
module: 'parquet',
|
|
13
|
-
version: VERSION,
|
|
14
|
-
worker: false,
|
|
15
|
-
category: 'table',
|
|
16
|
-
extensions: ['parquet'],
|
|
17
|
-
mimeTypes: ['application/octet-stream'],
|
|
18
|
-
binary: true,
|
|
19
|
-
tests: ['PAR1', 'PARE'],
|
|
20
|
-
options: {
|
|
21
|
-
parquet: {
|
|
22
|
-
type: 'arrow-table',
|
|
23
|
-
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
};
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.ParquetWasmWriter = void 0;
|
|
5
|
-
const encode_parquet_wasm_1 = require("./lib/wasm/encode-parquet-wasm");
|
|
6
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
7
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
8
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
9
|
-
/** Parquet WASM writer */
|
|
10
|
-
exports.ParquetWasmWriter = {
|
|
11
|
-
name: 'Apache Parquet',
|
|
12
|
-
id: 'parquet-wasm',
|
|
13
|
-
module: 'parquet',
|
|
14
|
-
version: VERSION,
|
|
15
|
-
extensions: ['parquet'],
|
|
16
|
-
mimeTypes: ['application/octet-stream'],
|
|
17
|
-
encode: encode_parquet_wasm_1.encode,
|
|
18
|
-
binary: true,
|
|
19
|
-
options: {
|
|
20
|
-
parquet: {
|
|
21
|
-
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
};
|
package/dist/parquet-writer.js
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.ParquetWriter = void 0;
|
|
5
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
8
|
-
const DEFAULT_PARQUET_LOADER_OPTIONS = {};
|
|
9
|
-
exports.ParquetWriter = {
|
|
10
|
-
name: 'Apache Parquet',
|
|
11
|
-
id: 'parquet',
|
|
12
|
-
module: 'parquet',
|
|
13
|
-
version: VERSION,
|
|
14
|
-
extensions: ['parquet'],
|
|
15
|
-
mimeTypes: ['application/octet-stream'],
|
|
16
|
-
encodeSync,
|
|
17
|
-
binary: true,
|
|
18
|
-
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
19
|
-
};
|
|
20
|
-
function encodeSync(data, options) {
|
|
21
|
-
return new ArrayBuffer(0);
|
|
22
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.encodeValues = exports.decodeValues = void 0;
|
|
4
|
-
const rle_1 = require("./rle");
|
|
5
|
-
function decodeValues(type, cursor, count, opts) {
|
|
6
|
-
opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
|
|
7
|
-
cursor.offset += 1;
|
|
8
|
-
return (0, rle_1.decodeValues)(type, cursor, count, { ...opts, disableEnvelope: true });
|
|
9
|
-
}
|
|
10
|
-
exports.decodeValues = decodeValues;
|
|
11
|
-
function encodeValues(type, cursor, count, opts) {
|
|
12
|
-
throw new Error('Encode dictionary functionality is not supported');
|
|
13
|
-
}
|
|
14
|
-
exports.encodeValues = encodeValues;
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
26
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.PARQUET_CODECS = void 0;
|
|
30
|
-
const PLAIN = __importStar(require("./plain"));
|
|
31
|
-
const RLE = __importStar(require("./rle"));
|
|
32
|
-
const DICTIONARY = __importStar(require("./dictionary"));
|
|
33
|
-
__exportStar(require("./declare"), exports);
|
|
34
|
-
exports.PARQUET_CODECS = {
|
|
35
|
-
PLAIN: {
|
|
36
|
-
encodeValues: PLAIN.encodeValues,
|
|
37
|
-
decodeValues: PLAIN.decodeValues
|
|
38
|
-
},
|
|
39
|
-
RLE: {
|
|
40
|
-
encodeValues: RLE.encodeValues,
|
|
41
|
-
decodeValues: RLE.decodeValues
|
|
42
|
-
},
|
|
43
|
-
// Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.
|
|
44
|
-
PLAIN_DICTIONARY: {
|
|
45
|
-
// @ts-ignore
|
|
46
|
-
encodeValues: DICTIONARY.encodeValues,
|
|
47
|
-
decodeValues: DICTIONARY.decodeValues
|
|
48
|
-
},
|
|
49
|
-
// Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.
|
|
50
|
-
RLE_DICTIONARY: {
|
|
51
|
-
// @ts-ignore
|
|
52
|
-
encodeValues: DICTIONARY.encodeValues,
|
|
53
|
-
decodeValues: DICTIONARY.decodeValues
|
|
54
|
-
}
|
|
55
|
-
};
|
|
@@ -1,211 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.decodeValues = exports.encodeValues = void 0;
|
|
7
|
-
const int53_1 = __importDefault(require("int53"));
|
|
8
|
-
function encodeValues(type, values, opts) {
|
|
9
|
-
switch (type) {
|
|
10
|
-
case 'BOOLEAN':
|
|
11
|
-
return encodeValues_BOOLEAN(values);
|
|
12
|
-
case 'INT32':
|
|
13
|
-
return encodeValues_INT32(values);
|
|
14
|
-
case 'INT64':
|
|
15
|
-
return encodeValues_INT64(values);
|
|
16
|
-
case 'INT96':
|
|
17
|
-
return encodeValues_INT96(values);
|
|
18
|
-
case 'FLOAT':
|
|
19
|
-
return encodeValues_FLOAT(values);
|
|
20
|
-
case 'DOUBLE':
|
|
21
|
-
return encodeValues_DOUBLE(values);
|
|
22
|
-
case 'BYTE_ARRAY':
|
|
23
|
-
return encodeValues_BYTE_ARRAY(values);
|
|
24
|
-
case 'FIXED_LEN_BYTE_ARRAY':
|
|
25
|
-
return encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts);
|
|
26
|
-
default:
|
|
27
|
-
throw new Error(`unsupported type: ${type}`);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
exports.encodeValues = encodeValues;
|
|
31
|
-
function decodeValues(type, cursor, count, opts) {
|
|
32
|
-
switch (type) {
|
|
33
|
-
case 'BOOLEAN':
|
|
34
|
-
return decodeValues_BOOLEAN(cursor, count);
|
|
35
|
-
case 'INT32':
|
|
36
|
-
return decodeValues_INT32(cursor, count);
|
|
37
|
-
case 'INT64':
|
|
38
|
-
return decodeValues_INT64(cursor, count);
|
|
39
|
-
case 'INT96':
|
|
40
|
-
return decodeValues_INT96(cursor, count);
|
|
41
|
-
case 'FLOAT':
|
|
42
|
-
return decodeValues_FLOAT(cursor, count);
|
|
43
|
-
case 'DOUBLE':
|
|
44
|
-
return decodeValues_DOUBLE(cursor, count);
|
|
45
|
-
case 'BYTE_ARRAY':
|
|
46
|
-
return decodeValues_BYTE_ARRAY(cursor, count);
|
|
47
|
-
case 'FIXED_LEN_BYTE_ARRAY':
|
|
48
|
-
return decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts);
|
|
49
|
-
default:
|
|
50
|
-
throw new Error(`unsupported type: ${type}`);
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
exports.decodeValues = decodeValues;
|
|
54
|
-
function encodeValues_BOOLEAN(values) {
|
|
55
|
-
const buf = Buffer.alloc(Math.ceil(values.length / 8));
|
|
56
|
-
buf.fill(0);
|
|
57
|
-
for (let i = 0; i < values.length; i++) {
|
|
58
|
-
if (values[i]) {
|
|
59
|
-
buf[Math.floor(i / 8)] |= 1 << i % 8;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
return buf;
|
|
63
|
-
}
|
|
64
|
-
function decodeValues_BOOLEAN(cursor, count) {
|
|
65
|
-
const values = [];
|
|
66
|
-
for (let i = 0; i < count; i++) {
|
|
67
|
-
const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
|
|
68
|
-
values.push((b & (1 << i % 8)) > 0);
|
|
69
|
-
}
|
|
70
|
-
cursor.offset += Math.ceil(count / 8);
|
|
71
|
-
return values;
|
|
72
|
-
}
|
|
73
|
-
function encodeValues_INT32(values) {
|
|
74
|
-
const buf = Buffer.alloc(4 * values.length);
|
|
75
|
-
for (let i = 0; i < values.length; i++) {
|
|
76
|
-
buf.writeInt32LE(values[i], i * 4);
|
|
77
|
-
}
|
|
78
|
-
return buf;
|
|
79
|
-
}
|
|
80
|
-
function decodeValues_INT32(cursor, count) {
|
|
81
|
-
const values = [];
|
|
82
|
-
for (let i = 0; i < count; i++) {
|
|
83
|
-
values.push(cursor.buffer.readInt32LE(cursor.offset));
|
|
84
|
-
cursor.offset += 4;
|
|
85
|
-
}
|
|
86
|
-
return values;
|
|
87
|
-
}
|
|
88
|
-
function encodeValues_INT64(values) {
|
|
89
|
-
const buf = Buffer.alloc(8 * values.length);
|
|
90
|
-
for (let i = 0; i < values.length; i++) {
|
|
91
|
-
int53_1.default.writeInt64LE(values[i], buf, i * 8);
|
|
92
|
-
}
|
|
93
|
-
return buf;
|
|
94
|
-
}
|
|
95
|
-
function decodeValues_INT64(cursor, count) {
|
|
96
|
-
const values = [];
|
|
97
|
-
for (let i = 0; i < count; i++) {
|
|
98
|
-
values.push(int53_1.default.readInt64LE(cursor.buffer, cursor.offset));
|
|
99
|
-
cursor.offset += 8;
|
|
100
|
-
}
|
|
101
|
-
return values;
|
|
102
|
-
}
|
|
103
|
-
function encodeValues_INT96(values) {
|
|
104
|
-
const buf = Buffer.alloc(12 * values.length);
|
|
105
|
-
for (let i = 0; i < values.length; i++) {
|
|
106
|
-
if (values[i] >= 0) {
|
|
107
|
-
int53_1.default.writeInt64LE(values[i], buf, i * 12);
|
|
108
|
-
buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
int53_1.default.writeInt64LE(~-values[i] + 1, buf, i * 12);
|
|
112
|
-
buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
return buf;
|
|
116
|
-
}
|
|
117
|
-
function decodeValues_INT96(cursor, count) {
|
|
118
|
-
const values = [];
|
|
119
|
-
for (let i = 0; i < count; i++) {
|
|
120
|
-
const low = int53_1.default.readInt64LE(cursor.buffer, cursor.offset);
|
|
121
|
-
const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
|
|
122
|
-
if (high === 0xffffffff) {
|
|
123
|
-
values.push(~-low + 1); // truncate to 64 actual precision
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
values.push(low); // truncate to 64 actual precision
|
|
127
|
-
}
|
|
128
|
-
cursor.offset += 12;
|
|
129
|
-
}
|
|
130
|
-
return values;
|
|
131
|
-
}
|
|
132
|
-
function encodeValues_FLOAT(values) {
|
|
133
|
-
const buf = Buffer.alloc(4 * values.length);
|
|
134
|
-
for (let i = 0; i < values.length; i++) {
|
|
135
|
-
buf.writeFloatLE(values[i], i * 4);
|
|
136
|
-
}
|
|
137
|
-
return buf;
|
|
138
|
-
}
|
|
139
|
-
function decodeValues_FLOAT(cursor, count) {
|
|
140
|
-
const values = [];
|
|
141
|
-
for (let i = 0; i < count; i++) {
|
|
142
|
-
values.push(cursor.buffer.readFloatLE(cursor.offset));
|
|
143
|
-
cursor.offset += 4;
|
|
144
|
-
}
|
|
145
|
-
return values;
|
|
146
|
-
}
|
|
147
|
-
function encodeValues_DOUBLE(values) {
|
|
148
|
-
const buf = Buffer.alloc(8 * values.length);
|
|
149
|
-
for (let i = 0; i < values.length; i++) {
|
|
150
|
-
buf.writeDoubleLE(values[i], i * 8);
|
|
151
|
-
}
|
|
152
|
-
return buf;
|
|
153
|
-
}
|
|
154
|
-
function decodeValues_DOUBLE(cursor, count) {
|
|
155
|
-
const values = [];
|
|
156
|
-
for (let i = 0; i < count; i++) {
|
|
157
|
-
values.push(cursor.buffer.readDoubleLE(cursor.offset));
|
|
158
|
-
cursor.offset += 8;
|
|
159
|
-
}
|
|
160
|
-
return values;
|
|
161
|
-
}
|
|
162
|
-
function encodeValues_BYTE_ARRAY(values) {
|
|
163
|
-
// tslint:disable-next-line:variable-name
|
|
164
|
-
let buf_len = 0;
|
|
165
|
-
for (let i = 0; i < values.length; i++) {
|
|
166
|
-
values[i] = Buffer.from(values[i]);
|
|
167
|
-
buf_len += 4 + values[i].length;
|
|
168
|
-
}
|
|
169
|
-
const buf = Buffer.alloc(buf_len);
|
|
170
|
-
// tslint:disable-next-line:variable-name
|
|
171
|
-
let buf_pos = 0;
|
|
172
|
-
for (let i = 0; i < values.length; i++) {
|
|
173
|
-
buf.writeUInt32LE(values[i].length, buf_pos);
|
|
174
|
-
values[i].copy(buf, buf_pos + 4);
|
|
175
|
-
buf_pos += 4 + values[i].length;
|
|
176
|
-
}
|
|
177
|
-
return buf;
|
|
178
|
-
}
|
|
179
|
-
function decodeValues_BYTE_ARRAY(cursor, count) {
|
|
180
|
-
const values = [];
|
|
181
|
-
for (let i = 0; i < count; i++) {
|
|
182
|
-
const len = cursor.buffer.readUInt32LE(cursor.offset);
|
|
183
|
-
cursor.offset += 4;
|
|
184
|
-
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
|
|
185
|
-
cursor.offset += len;
|
|
186
|
-
}
|
|
187
|
-
return values;
|
|
188
|
-
}
|
|
189
|
-
function encodeValues_FIXED_LEN_BYTE_ARRAY(values, opts) {
|
|
190
|
-
if (!opts.typeLength) {
|
|
191
|
-
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
|
|
192
|
-
}
|
|
193
|
-
for (let i = 0; i < values.length; i++) {
|
|
194
|
-
values[i] = Buffer.from(values[i]);
|
|
195
|
-
if (values[i].length !== opts.typeLength) {
|
|
196
|
-
throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
return Buffer.concat(values);
|
|
200
|
-
}
|
|
201
|
-
function decodeValues_FIXED_LEN_BYTE_ARRAY(cursor, count, opts) {
|
|
202
|
-
const values = [];
|
|
203
|
-
if (!opts.typeLength) {
|
|
204
|
-
throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
|
|
205
|
-
}
|
|
206
|
-
for (let i = 0; i < count; i++) {
|
|
207
|
-
values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
|
|
208
|
-
cursor.offset += opts.typeLength;
|
|
209
|
-
}
|
|
210
|
-
return values;
|
|
211
|
-
}
|
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
|
|
3
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
-
};
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.decodeValues = exports.encodeValues = void 0;
|
|
8
|
-
const varint_1 = __importDefault(require("varint"));
|
|
9
|
-
// eslint-disable-next-line max-statements, complexity
|
|
10
|
-
function encodeValues(type, values, opts) {
|
|
11
|
-
if (!('bitWidth' in opts)) {
|
|
12
|
-
throw new Error('bitWidth is required');
|
|
13
|
-
}
|
|
14
|
-
switch (type) {
|
|
15
|
-
case 'BOOLEAN':
|
|
16
|
-
case 'INT32':
|
|
17
|
-
case 'INT64':
|
|
18
|
-
// tslint:disable-next-line:no-parameter-reassignment
|
|
19
|
-
values = values.map((x) => parseInt(x, 10));
|
|
20
|
-
break;
|
|
21
|
-
default:
|
|
22
|
-
throw new Error(`unsupported type: ${type}`);
|
|
23
|
-
}
|
|
24
|
-
let buf = Buffer.alloc(0);
|
|
25
|
-
let run = [];
|
|
26
|
-
let repeats = 0;
|
|
27
|
-
for (let i = 0; i < values.length; i++) {
|
|
28
|
-
// If we are at the beginning of a run and the next value is same we start
|
|
29
|
-
// collecting repeated values
|
|
30
|
-
if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
|
|
31
|
-
// If we have any data in runs we need to encode them
|
|
32
|
-
if (run.length) {
|
|
33
|
-
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
34
|
-
run = [];
|
|
35
|
-
}
|
|
36
|
-
repeats = 1;
|
|
37
|
-
}
|
|
38
|
-
else if (repeats > 0 && values[i] === values[i - 1]) {
|
|
39
|
-
repeats += 1;
|
|
40
|
-
}
|
|
41
|
-
else {
|
|
42
|
-
// If values changes we need to post any previous repeated values
|
|
43
|
-
if (repeats) {
|
|
44
|
-
buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
|
|
45
|
-
repeats = 0;
|
|
46
|
-
}
|
|
47
|
-
run.push(values[i]);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
if (repeats) {
|
|
51
|
-
buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
|
|
52
|
-
}
|
|
53
|
-
else if (run.length) {
|
|
54
|
-
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
55
|
-
}
|
|
56
|
-
if (opts.disableEnvelope) {
|
|
57
|
-
return buf;
|
|
58
|
-
}
|
|
59
|
-
const envelope = Buffer.alloc(buf.length + 4);
|
|
60
|
-
envelope.writeUInt32LE(buf.length, undefined);
|
|
61
|
-
buf.copy(envelope, 4);
|
|
62
|
-
return envelope;
|
|
63
|
-
}
|
|
64
|
-
exports.encodeValues = encodeValues;
|
|
65
|
-
function decodeValues(type, cursor, count, opts) {
|
|
66
|
-
if (!('bitWidth' in opts)) {
|
|
67
|
-
throw new Error('bitWidth is required');
|
|
68
|
-
}
|
|
69
|
-
if (!opts.disableEnvelope) {
|
|
70
|
-
cursor.offset += 4;
|
|
71
|
-
}
|
|
72
|
-
let values = [];
|
|
73
|
-
while (values.length < count) {
|
|
74
|
-
const header = varint_1.default.decode(cursor.buffer, cursor.offset);
|
|
75
|
-
cursor.offset += varint_1.default.encodingLength(header);
|
|
76
|
-
if (header & 1) {
|
|
77
|
-
const count = (header >> 1) * 8;
|
|
78
|
-
values.push(...decodeRunBitpacked(cursor, count, opts));
|
|
79
|
-
}
|
|
80
|
-
else {
|
|
81
|
-
const count = header >> 1;
|
|
82
|
-
values.push(...decodeRunRepeated(cursor, count, opts));
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
values = values.slice(0, count);
|
|
86
|
-
if (values.length !== count) {
|
|
87
|
-
throw new Error('invalid RLE encoding');
|
|
88
|
-
}
|
|
89
|
-
return values;
|
|
90
|
-
}
|
|
91
|
-
exports.decodeValues = decodeValues;
|
|
92
|
-
function decodeRunBitpacked(cursor, count, opts) {
|
|
93
|
-
// @ts-ignore
|
|
94
|
-
const bitWidth = opts.bitWidth;
|
|
95
|
-
if (count % 8 !== 0) {
|
|
96
|
-
throw new Error('must be a multiple of 8');
|
|
97
|
-
}
|
|
98
|
-
// tslint:disable-next-line:prefer-array-literal
|
|
99
|
-
const values = new Array(count).fill(0);
|
|
100
|
-
for (let b = 0; b < bitWidth * count; b++) {
|
|
101
|
-
if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {
|
|
102
|
-
values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
cursor.offset += bitWidth * (count / 8);
|
|
106
|
-
return values;
|
|
107
|
-
}
|
|
108
|
-
function decodeRunRepeated(cursor, count, opts) {
|
|
109
|
-
// @ts-ignore
|
|
110
|
-
const bitWidth = opts.bitWidth;
|
|
111
|
-
let value = 0;
|
|
112
|
-
for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
|
|
113
|
-
// eslint-disable-next-line
|
|
114
|
-
value << 8; // TODO - this looks wrong
|
|
115
|
-
value += cursor.buffer[cursor.offset];
|
|
116
|
-
cursor.offset += 1;
|
|
117
|
-
}
|
|
118
|
-
// tslint:disable-next-line:prefer-array-literal
|
|
119
|
-
return new Array(count).fill(value);
|
|
120
|
-
}
|
|
121
|
-
function encodeRunBitpacked(values, opts) {
|
|
122
|
-
// @ts-ignore
|
|
123
|
-
const bitWidth = opts.bitWidth;
|
|
124
|
-
for (let i = 0; i < values.length % 8; i++) {
|
|
125
|
-
values.push(0);
|
|
126
|
-
}
|
|
127
|
-
const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
|
|
128
|
-
for (let b = 0; b < bitWidth * values.length; b++) {
|
|
129
|
-
if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {
|
|
130
|
-
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
return Buffer.concat([Buffer.from(varint_1.default.encode(((values.length / 8) << 1) | 1)), buf]);
|
|
134
|
-
}
|
|
135
|
-
function encodeRunRepeated(value, count, opts) {
|
|
136
|
-
// @ts-ignore
|
|
137
|
-
const bitWidth = opts.bitWidth;
|
|
138
|
-
const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
|
|
139
|
-
for (let i = 0; i < buf.length; i++) {
|
|
140
|
-
buf.writeUInt8(value & 0xff, i);
|
|
141
|
-
// eslint-disable-next-line
|
|
142
|
-
value >> 8; // TODO - this looks wrong
|
|
143
|
-
}
|
|
144
|
-
return Buffer.concat([Buffer.from(varint_1.default.encode(count << 1)), buf]);
|
|
145
|
-
}
|