@loaders.gl/parquet 4.0.2 → 4.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow1_bg.wasm +0 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js.map +1 -1
- package/dist/index.cjs +50 -33
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -7
- package/dist/index.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +4 -4
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-columns.js +5 -8
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-rows.d.ts +4 -4
- package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-rows.js +30 -19
- package/dist/lib/parsers/parse-parquet-to-rows.js.map +1 -1
- package/dist/lib/wasm/encode-parquet-wasm.d.ts +2 -10
- package/dist/lib/wasm/encode-parquet-wasm.d.ts.map +1 -1
- package/dist/lib/wasm/encode-parquet-wasm.js +4 -5
- package/dist/lib/wasm/encode-parquet-wasm.js.map +1 -1
- package/dist/lib/wasm/load-wasm-browser.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm-browser.js.map +1 -0
- package/dist/lib/wasm/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm-node.js.map +1 -0
- package/dist/lib/wasm/{load-wasm/index.d.ts → load-wasm.d.ts} +1 -1
- package/dist/lib/wasm/load-wasm.d.ts.map +1 -0
- package/dist/lib/wasm/{load-wasm/index.js → load-wasm.js} +1 -1
- package/dist/lib/wasm/load-wasm.js.map +1 -0
- package/dist/lib/wasm/parse-parquet-wasm.d.ts +2 -2
- package/dist/lib/wasm/parse-parquet-wasm.d.ts.map +1 -1
- package/dist/lib/wasm/parse-parquet-wasm.js +9 -7
- package/dist/lib/wasm/parse-parquet-wasm.js.map +1 -1
- package/dist/parquet-loader.d.ts +3 -3
- package/dist/parquet-loader.d.ts.map +1 -1
- package/dist/parquet-loader.js +1 -1
- package/dist/parquet-loader.js.map +1 -1
- package/dist/parquet-wasm-loader.d.ts +2 -2
- package/dist/parquet-wasm-loader.d.ts.map +1 -1
- package/dist/parquet-wasm-loader.js.map +1 -1
- package/dist/parquet-wasm-writer.d.ts +3 -3
- package/dist/parquet-wasm-writer.d.ts.map +1 -1
- package/dist/parquet-wasm-writer.js +2 -2
- package/dist/parquet-wasm-writer.js.map +1 -1
- package/dist/parquet-writer.d.ts +2 -2
- package/dist/parquet-writer.d.ts.map +1 -1
- package/dist/parquet-writer.js.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.d.ts +1 -1
- package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
- package/dist/parquetjs/parquet-thrift/index.js +1 -1
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
- package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +1 -0
- package/dist/polyfills/buffer/buffer.d.ts.map +1 -0
- package/dist/polyfills/buffer/buffer.js.map +1 -0
- package/dist/polyfills/buffer/index.d.ts.map +1 -0
- package/dist/polyfills/buffer/index.js.map +1 -0
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +1 -0
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +1 -0
- package/dist/polyfills/util.d.ts +9 -0
- package/dist/polyfills/util.d.ts.map +1 -0
- package/dist/polyfills/util.js +3 -0
- package/dist/polyfills/util.js.map +1 -0
- package/dist/workers/parquet-worker.js.map +1 -1
- package/package.json +16 -14
- package/src/constants.ts +3 -0
- package/src/index.ts +18 -17
- package/src/lib/parsers/parse-parquet-to-columns.ts +9 -11
- package/src/lib/parsers/parse-parquet-to-rows.ts +49 -25
- package/src/lib/wasm/encode-parquet-wasm.ts +8 -13
- package/src/lib/wasm/parse-parquet-wasm.ts +12 -12
- package/src/parquet-loader.ts +4 -9
- package/src/parquet-wasm-loader.ts +2 -2
- package/src/parquet-wasm-writer.ts +5 -5
- package/src/parquet-writer.ts +2 -2
- package/src/parquetjs/parquet-thrift/index.ts +1 -1
- package/src/polyfills/util.js +7 -0
- package/src/workers/parquet-worker.ts +3 -0
- package/dist/buffer-polyfill/buffer-polyfill.browser.d.ts.map +0 -1
- package/dist/buffer-polyfill/buffer-polyfill.browser.js.map +0 -1
- package/dist/buffer-polyfill/buffer-polyfill.node.d.ts.map +0 -1
- package/dist/buffer-polyfill/buffer-polyfill.node.js.map +0 -1
- package/dist/buffer-polyfill/buffer.d.ts.map +0 -1
- package/dist/buffer-polyfill/buffer.js.map +0 -1
- package/dist/buffer-polyfill/index.d.ts.map +0 -1
- package/dist/buffer-polyfill/index.js.map +0 -1
- package/dist/buffer-polyfill/install-buffer-polyfill.d.ts.map +0 -1
- package/dist/buffer-polyfill/install-buffer-polyfill.js.map +0 -1
- package/dist/lib/wasm/load-wasm/index.d.ts.map +0 -1
- package/dist/lib/wasm/load-wasm/index.js.map +0 -1
- package/dist/lib/wasm/load-wasm/load-wasm-browser.d.ts.map +0 -1
- package/dist/lib/wasm/load-wasm/load-wasm-browser.js.map +0 -1
- package/dist/lib/wasm/load-wasm/load-wasm-node.d.ts.map +0 -1
- package/dist/lib/wasm/load-wasm/load-wasm-node.js.map +0 -1
- /package/dist/lib/wasm/{load-wasm/load-wasm-browser.d.ts → load-wasm-browser.d.ts} +0 -0
- /package/dist/lib/wasm/{load-wasm/load-wasm-browser.js → load-wasm-browser.js} +0 -0
- /package/dist/lib/wasm/{load-wasm/load-wasm-node.d.ts → load-wasm-node.d.ts} +0 -0
- /package/dist/lib/wasm/{load-wasm/load-wasm-node.js → load-wasm-node.js} +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/buffer-polyfill.browser.d.ts +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/buffer-polyfill.browser.js +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/buffer-polyfill.node.d.ts +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/buffer-polyfill.node.js +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/buffer.d.ts +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/buffer.js +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/index.d.ts +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/index.js +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/install-buffer-polyfill.d.ts +0 -0
- /package/dist/{buffer-polyfill → polyfills/buffer}/install-buffer-polyfill.js +0 -0
- /package/src/lib/wasm/{load-wasm/load-wasm-browser.ts → load-wasm-browser.ts} +0 -0
- /package/src/lib/wasm/{load-wasm/load-wasm-node.ts → load-wasm-node.ts} +0 -0
- /package/src/lib/wasm/{load-wasm/index.ts → load-wasm.ts} +0 -0
- /package/src/{buffer-polyfill → polyfills/buffer}/buffer-polyfill.browser.ts +0 -0
- /package/src/{buffer-polyfill → polyfills/buffer}/buffer-polyfill.node.ts +0 -0
- /package/src/{buffer-polyfill → polyfills/buffer}/buffer.ts +0 -0
- /package/src/{buffer-polyfill → polyfills/buffer}/index.ts +0 -0
- /package/src/{buffer-polyfill → polyfills/buffer}/install-buffer-polyfill.ts +0 -0
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
2
2
|
// import {ColumnarTableBatch} from '@loaders.gl/schema';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import type {ReadableFile} from '@loaders.gl/loader-utils';
|
|
4
|
+
import type {
|
|
5
|
+
GeoJSONTable,
|
|
6
|
+
GeoJSONTableBatch,
|
|
7
|
+
ObjectRowTable,
|
|
8
|
+
ObjectRowTableBatch
|
|
9
|
+
} from '@loaders.gl/schema';
|
|
5
10
|
import {convertWKBTableToGeoJSON} from '@loaders.gl/gis';
|
|
6
11
|
import {WKTLoader, WKBLoader} from '@loaders.gl/wkt';
|
|
7
12
|
|
|
@@ -9,16 +14,14 @@ import type {ParquetLoaderOptions} from '../../parquet-loader';
|
|
|
9
14
|
import type {ParquetRow} from '../../parquetjs/schema/declare';
|
|
10
15
|
import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
|
|
11
16
|
import {getSchemaFromParquetReader} from './get-parquet-schema';
|
|
12
|
-
import {installBufferPolyfill} from '../../buffer
|
|
17
|
+
import {installBufferPolyfill} from '../../polyfills/buffer';
|
|
13
18
|
|
|
14
|
-
export async function
|
|
15
|
-
|
|
19
|
+
export async function parseParquetFile(
|
|
20
|
+
file: ReadableFile,
|
|
16
21
|
options?: ParquetLoaderOptions
|
|
17
22
|
): Promise<ObjectRowTable | GeoJSONTable> {
|
|
18
23
|
installBufferPolyfill();
|
|
19
24
|
|
|
20
|
-
const blob = new Blob([arrayBuffer]);
|
|
21
|
-
const file = new BlobFile(blob);
|
|
22
25
|
const reader = new ParquetReader(file, {
|
|
23
26
|
preserveBinary: options?.parquet?.preserveBinary
|
|
24
27
|
});
|
|
@@ -41,35 +44,56 @@ export async function parseParquet(
|
|
|
41
44
|
};
|
|
42
45
|
|
|
43
46
|
const shape = options?.parquet?.shape;
|
|
44
|
-
|
|
45
|
-
case 'object-row-table':
|
|
46
|
-
return objectRowTable;
|
|
47
|
-
|
|
48
|
-
case 'geojson-table':
|
|
49
|
-
try {
|
|
50
|
-
return convertWKBTableToGeoJSON(objectRowTable, schema, [WKTLoader, WKBLoader]);
|
|
51
|
-
} catch (error) {
|
|
52
|
-
return objectRowTable;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
default:
|
|
56
|
-
throw new Error(shape);
|
|
57
|
-
}
|
|
47
|
+
return convertTable(objectRowTable, shape);
|
|
58
48
|
}
|
|
59
49
|
|
|
60
50
|
export async function* parseParquetFileInBatches(
|
|
61
|
-
|
|
51
|
+
file: ReadableFile,
|
|
62
52
|
options?: ParquetLoaderOptions
|
|
63
|
-
): AsyncIterable<ObjectRowTableBatch> {
|
|
53
|
+
): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {
|
|
54
|
+
const reader = new ParquetReader(file, {
|
|
55
|
+
preserveBinary: options?.parquet?.preserveBinary
|
|
56
|
+
});
|
|
57
|
+
|
|
64
58
|
const schema = await getSchemaFromParquetReader(reader);
|
|
65
59
|
const rowBatches = reader.rowBatchIterator(options?.parquet);
|
|
66
60
|
for await (const rows of rowBatches) {
|
|
61
|
+
const objectRowTable: ObjectRowTable = {
|
|
62
|
+
shape: 'object-row-table',
|
|
63
|
+
schema,
|
|
64
|
+
data: rows
|
|
65
|
+
};
|
|
66
|
+
const shape = options?.parquet?.shape;
|
|
67
|
+
const table = convertTable(objectRowTable, shape);
|
|
68
|
+
|
|
67
69
|
yield {
|
|
68
70
|
batchType: 'data',
|
|
69
|
-
shape: 'object-row-table',
|
|
70
71
|
schema,
|
|
71
|
-
|
|
72
|
+
...table,
|
|
72
73
|
length: rows.length
|
|
73
74
|
};
|
|
74
75
|
}
|
|
75
76
|
}
|
|
77
|
+
|
|
78
|
+
function convertTable(
|
|
79
|
+
objectRowTable: ObjectRowTable,
|
|
80
|
+
shape?: 'object-row-table' | 'geojson-table'
|
|
81
|
+
): ObjectRowTable | GeoJSONTable {
|
|
82
|
+
switch (shape) {
|
|
83
|
+
case 'object-row-table':
|
|
84
|
+
return objectRowTable;
|
|
85
|
+
|
|
86
|
+
case 'geojson-table':
|
|
87
|
+
try {
|
|
88
|
+
return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!, [
|
|
89
|
+
WKTLoader,
|
|
90
|
+
WKBLoader
|
|
91
|
+
]);
|
|
92
|
+
} catch (error) {
|
|
93
|
+
return objectRowTable;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
default:
|
|
97
|
+
throw new Error(shape);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type {WriterOptions} from '@loaders.gl/loader-utils';
|
|
2
|
+
import type {ArrowTable} from '@loaders.gl/arrow';
|
|
2
3
|
|
|
3
4
|
import * as arrow from 'apache-arrow';
|
|
4
5
|
import {loadWasm} from './load-wasm';
|
|
@@ -13,13 +14,18 @@ export type ParquetWriterOptions = WriterOptions & {
|
|
|
13
14
|
* Encode Arrow arrow.Table to Parquet buffer
|
|
14
15
|
*/
|
|
15
16
|
export async function encode(
|
|
16
|
-
table:
|
|
17
|
+
table: ArrowTable,
|
|
17
18
|
options?: ParquetWriterOptions
|
|
18
19
|
): Promise<ArrayBuffer> {
|
|
19
20
|
const wasmUrl = options?.parquet?.wasmUrl;
|
|
20
21
|
const wasm = await loadWasm(wasmUrl);
|
|
21
22
|
|
|
22
|
-
const
|
|
23
|
+
const arrowTable: arrow.Table = table.data;
|
|
24
|
+
|
|
25
|
+
// Serialize a table to the IPC format.
|
|
26
|
+
const writer = arrow.RecordBatchStreamWriter.writeAll(arrowTable);
|
|
27
|
+
const arrowIPCBytes = writer.toUint8Array(true);
|
|
28
|
+
|
|
23
29
|
// TODO: provide options for how to write table.
|
|
24
30
|
const writerProperties = new wasm.WriterPropertiesBuilder().build();
|
|
25
31
|
const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
|
|
@@ -28,14 +34,3 @@ export async function encode(
|
|
|
28
34
|
parquetBytes.byteLength + parquetBytes.byteOffset
|
|
29
35
|
);
|
|
30
36
|
}
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Serialize a table to the IPC format. This function is a convenience
|
|
34
|
-
* Opposite of {@link tableFromIPC}.
|
|
35
|
-
*
|
|
36
|
-
* @param table The arrow.Table to serialize.
|
|
37
|
-
* @param type Whether to serialize the arrow.Table as a file or a stream.
|
|
38
|
-
*/
|
|
39
|
-
export function tableToIPC(table: arrow.Table): Uint8Array {
|
|
40
|
-
return arrow.RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
|
|
41
|
-
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
// eslint-disable
|
|
2
2
|
import type {LoaderOptions} from '@loaders.gl/loader-utils';
|
|
3
|
+
import type {ArrowTable} from '@loaders.gl/arrow';
|
|
4
|
+
import {serializeArrowSchema} from '@loaders.gl/arrow';
|
|
3
5
|
import * as arrow from 'apache-arrow';
|
|
4
|
-
import {loadWasm} from './load-wasm
|
|
6
|
+
import {loadWasm} from './load-wasm';
|
|
5
7
|
|
|
6
8
|
export type ParquetWasmLoaderOptions = LoaderOptions & {
|
|
7
9
|
parquet?: {
|
|
@@ -13,7 +15,7 @@ export type ParquetWasmLoaderOptions = LoaderOptions & {
|
|
|
13
15
|
export async function parseParquetWasm(
|
|
14
16
|
arrayBuffer: ArrayBuffer,
|
|
15
17
|
options?: ParquetWasmLoaderOptions
|
|
16
|
-
): Promise<
|
|
18
|
+
): Promise<ArrowTable> {
|
|
17
19
|
const wasmUrl = options?.parquet?.wasmUrl;
|
|
18
20
|
const wasm = await loadWasm(wasmUrl);
|
|
19
21
|
|
|
@@ -23,19 +25,17 @@ export async function parseParquetWasm(
|
|
|
23
25
|
arrowIPCUint8Arr.byteOffset,
|
|
24
26
|
arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset
|
|
25
27
|
);
|
|
26
|
-
const arrowTable = tableFromIPC(arrowIPCBuffer);
|
|
27
|
-
return arrowTable;
|
|
28
|
-
}
|
|
29
28
|
|
|
30
|
-
|
|
31
|
-
* Deserialize the IPC format into a {@link Table}. This function is a
|
|
32
|
-
* convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
|
|
33
|
-
*/
|
|
34
|
-
function tableFromIPC(input: ArrayBuffer): arrow.Table {
|
|
35
|
-
const reader = arrow.RecordBatchStreamReader.from(input);
|
|
29
|
+
const reader = arrow.RecordBatchStreamReader.from(arrowIPCBuffer);
|
|
36
30
|
const recordBatches: arrow.RecordBatch[] = [];
|
|
37
31
|
for (const recordBatch of reader) {
|
|
38
32
|
recordBatches.push(recordBatch);
|
|
39
33
|
}
|
|
40
|
-
|
|
34
|
+
const arrowTable = new arrow.Table(recordBatches);
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
shape: 'arrow-table',
|
|
38
|
+
schema: serializeArrowSchema(arrowTable.schema),
|
|
39
|
+
data: arrowTable
|
|
40
|
+
};
|
|
41
41
|
}
|
package/src/parquet-loader.ts
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
3
4
|
import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
4
5
|
import type {
|
|
5
6
|
ObjectRowTable,
|
|
6
7
|
ObjectRowTableBatch,
|
|
7
8
|
ColumnarTable,
|
|
8
|
-
ColumnarTableBatch
|
|
9
|
-
GeoJSONTable,
|
|
10
|
-
GeoJSONTableBatch
|
|
9
|
+
ColumnarTableBatch
|
|
11
10
|
} from '@loaders.gl/schema';
|
|
12
11
|
|
|
13
|
-
export {Buffer} from './buffer
|
|
12
|
+
export {Buffer} from './polyfills/buffer/install-buffer-polyfill';
|
|
14
13
|
|
|
15
14
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
16
15
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
@@ -34,11 +33,7 @@ export type ParquetLoaderOptions = LoaderOptions & {
|
|
|
34
33
|
};
|
|
35
34
|
|
|
36
35
|
/** ParquetJS table loader */
|
|
37
|
-
export const ParquetLoader: Loader<
|
|
38
|
-
ObjectRowTable | GeoJSONTable,
|
|
39
|
-
ObjectRowTableBatch | GeoJSONTableBatch,
|
|
40
|
-
ParquetLoaderOptions
|
|
41
|
-
> = {
|
|
36
|
+
export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions> = {
|
|
42
37
|
name: 'Apache Parquet',
|
|
43
38
|
id: 'parquet',
|
|
44
39
|
module: 'parquet',
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
3
|
|
|
4
4
|
import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
5
|
-
import type
|
|
5
|
+
import type {ArrowTable} from '@loaders.gl/arrow';
|
|
6
6
|
|
|
7
7
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
8
8
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
@@ -17,7 +17,7 @@ export type ParquetWasmLoaderOptions = LoaderOptions & {
|
|
|
17
17
|
};
|
|
18
18
|
|
|
19
19
|
/** Parquet WASM table loader */
|
|
20
|
-
export const ParquetWasmLoader: Loader<
|
|
20
|
+
export const ParquetWasmLoader: Loader<ArrowTable, never, ParquetWasmLoaderOptions> = {
|
|
21
21
|
name: 'Apache Parquet',
|
|
22
22
|
id: 'parquet-wasm',
|
|
23
23
|
module: 'parquet',
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
3
|
|
|
4
|
-
import type {
|
|
4
|
+
import type {WriterWithEncoder} from '@loaders.gl/loader-utils';
|
|
5
|
+
import type {ArrowTable} from '@loaders.gl/arrow';
|
|
5
6
|
import {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';
|
|
6
|
-
import type * as arrow from 'apache-arrow';
|
|
7
7
|
|
|
8
8
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
9
9
|
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
10
10
|
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
11
11
|
|
|
12
12
|
/** Parquet WASM writer */
|
|
13
|
-
export const ParquetWasmWriter:
|
|
13
|
+
export const ParquetWasmWriter: WriterWithEncoder<ArrowTable, never, ParquetWriterOptions> = {
|
|
14
14
|
name: 'Apache Parquet',
|
|
15
15
|
id: 'parquet-wasm',
|
|
16
16
|
module: 'parquet',
|
|
17
17
|
version: VERSION,
|
|
18
18
|
extensions: ['parquet'],
|
|
19
19
|
mimeTypes: ['application/octet-stream'],
|
|
20
|
-
encode,
|
|
21
20
|
binary: true,
|
|
22
21
|
options: {
|
|
23
22
|
parquet: {
|
|
24
23
|
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
25
24
|
}
|
|
26
|
-
}
|
|
25
|
+
},
|
|
26
|
+
encode
|
|
27
27
|
};
|
package/src/parquet-writer.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
3
|
|
|
4
|
-
import type {
|
|
4
|
+
import type {WriterWithEncoder} from '@loaders.gl/loader-utils';
|
|
5
5
|
import {Table, TableBatch} from '@loaders.gl/schema';
|
|
6
6
|
|
|
7
7
|
// __VERSION__ is injected by babel-plugin-version-inline
|
|
@@ -10,7 +10,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
|
10
10
|
|
|
11
11
|
export type ParquetWriterOptions = {};
|
|
12
12
|
|
|
13
|
-
export const ParquetWriter:
|
|
13
|
+
export const ParquetWriter: WriterWithEncoder<Table, TableBatch, ParquetWriterOptions> = {
|
|
14
14
|
name: 'Apache Parquet',
|
|
15
15
|
id: 'parquet',
|
|
16
16
|
module: 'parquet',
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Autogenerated by @creditkarma/thrift-typescript v3.7.2
|
|
5
5
|
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
6
6
|
*/
|
|
7
|
-
export {Buffer} from '../../buffer
|
|
7
|
+
export {Buffer} from '../../polyfills/buffer/install-buffer-polyfill';
|
|
8
8
|
|
|
9
9
|
export {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';
|
|
10
10
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"buffer-polyfill.browser.d.ts","sourceRoot":"","sources":["../../src/buffer-polyfill/buffer-polyfill.browser.ts"],"names":[],"mappings":";AAIA,yDAAyD;AACzD,wBAAgB,qBAAqB,IAAI,OAAO,MAAM,CAMrD"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"buffer-polyfill.browser.js","names":["Buffer","BufferPolyfill","installBufferPolyfill","globalThis"],"sources":["../../src/buffer-polyfill/buffer-polyfill.browser.ts"],"sourcesContent":["// luma.gl, MIT license\n\nimport {Buffer as BufferPolyfill} from './buffer';\n\n/** Install Node.js Buffer polyfill (NO-OP in Node.js) */\nexport function installBufferPolyfill(): typeof Buffer {\n // @ts-ignore\n globalThis.Buffer = globalThis.Buffer || BufferPolyfill;\n\n // @ts-ignore\n return globalThis.Buffer;\n}\n"],"mappings":"SAEQA,MAAM,IAAIC,cAAc;AAGhC,OAAO,SAASC,qBAAqBA,CAAA,EAAkB;EAErDC,UAAU,CAACH,MAAM,GAAGG,UAAU,CAACH,MAAM,IAAIC,cAAc;EAGvD,OAAOE,UAAU,CAACH,MAAM;AAC1B"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"buffer-polyfill.node.d.ts","sourceRoot":"","sources":["../../src/buffer-polyfill/buffer-polyfill.node.ts"],"names":[],"mappings":";AAIA,6DAA6D;AAC7D,wBAAgB,qBAAqB,IAAI,OAAO,MAAM,CAWrD"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"buffer-polyfill.node.js","names":["Buffer","BufferPolyfill","installBufferPolyfill","Buffer_","globalThis","process"],"sources":["../../src/buffer-polyfill/buffer-polyfill.node.ts"],"sourcesContent":["// luma.gl, MIT license\n\nimport {Buffer as BufferPolyfill} from './buffer';\n\n/** Install the Node.js Buffer polyfill (NO-OP in Node.js) */\nexport function installBufferPolyfill(): typeof Buffer {\n const Buffer_ = typeof Buffer !== 'undefined' ? Buffer : null;\n if (!Buffer_) {\n // @ts-expect-error\n globalThis.Buffer = BufferPolyfill;\n return BufferPolyfill as unknown as typeof Buffer;\n }\n\n globalThis.process = globalThis.process || {};\n // Buffer is a global variable in Node.js\n return Buffer_;\n}\n"],"mappings":"SAEQA,MAAM,IAAIC,cAAc;AAGhC,OAAO,SAASC,qBAAqBA,CAAA,EAAkB;EACrD,MAAMC,OAAO,GAAG,OAAOH,MAAM,KAAK,WAAW,GAAGA,MAAM,GAAG,IAAI;EAC7D,IAAI,CAACG,OAAO,EAAE;IAEZC,UAAU,CAACJ,MAAM,GAAGC,cAAc;IAClC,OAAOA,cAAc;EACvB;EAEAG,UAAU,CAACC,OAAO,GAAGD,UAAU,CAACC,OAAO,IAAI,CAAC,CAAC;EAE7C,OAAOF,OAAO;AAChB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"buffer.d.ts","sourceRoot":"","sources":["../../src/buffer-polyfill/buffer.ts"],"names":[],"mappings":"AAgCA,eAAO,MAAM,UAAU,aAAa,CAAC;AACrC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,KAAK,cAAc,GAAG,MAAM,CAAC;AAO7B;;;;;;;;GAQG;AACH,qBAAa,MAAO,SAAQ,UAAU;IACpC,MAAM,CAAC,QAAQ,SAAQ;IAIvB,IAAI,MAAM,4BAGT;IAED,IAAI,MAAM,uBAGT;IAED;;;;;;OAMG;IACH,SAAS,UAAQ;IAEjB;;;;;OAKG;gBACS,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM;IAC1C;;;;OAIG;gBACS,IAAI,EAAE,MAAM;IACxB;;;;OAIG;gBACS,KAAK,EAAE,UAAU;IAC7B;;;;;;OAMG;gBACS,WAAW,EAAE,WAAW;IACpC;;;;OAIG;gBACS,KAAK,EAAE,GAAG,EAAE;IACxB;;;;OAIG;gBACS,MAAM,EAAE,MAAM;IAmB1B;;;;;;;QAOI;IAEJ;;;;OAIG;IACH,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM;IACjC;;;;;;;;;OASG;IACH,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM;IACnF;;;;OAIG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM;IAChD;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IA4DnD;;;;OAIG;IACH,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM;IAItC;;OAEG;IACH,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,EAAE,CAAC,EAAE,UAAU,GAAG,MAAM,GAAG,MAAM;IAyBtE;;;;;OAKG;IACH,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAmB5C;;;;;;;;;;OAUG;IACH,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM;IAwCrE;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAItF;;;;;OAKG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAKxC;;;;;OAKG;IACH,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAI5C,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO;IAIxF,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAItF,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAI1F,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAOpD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAOvD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAOvD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASvD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASvD,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAkBzE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAkBzE,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAkBtC,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAqBtC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMrD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMxD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMxD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAUxD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAUxD,UAAU,CAAC,MAAM,KAAA,EAAE,UAAU,KAAA,EAAE,QAAQ,KAAA;IAevC,UAAU,CAAC,MAAM,KAAA,EAAE,UAAU,KAAA,EAAE,QAAQ,KAAA;IAgBvC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAiBvC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAiBvC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMvD,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMvD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMxD,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAMxD,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAQrE,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASxE,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASxE,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAWxE,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAWxE,WAAW,CAAC,KAAK,KAAA,EAAE,MAAM,KAAA,EAAE,UAAU,KAAA,EAAE,QAAQ,KAAA;IAmB/C,WAAW,CAAC,KAAK,KAAA,EAAE,MAAM,KAAA,EAAE,UAAU,KAAA,EAAE,QAAQ,KAAA;IAmB/C,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAU,GAAG,MAAM;IAI3D,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAU,GAAG,MAAM;IAI3D,UAAU,CAAC,KAAK,KAAA,EAAE,MAAM,KAAA,EAAE,UAAU,KAAA,EAAE,QAAQ,KAAA;IAuB9C,UAAU,CAAC,KAAK,KAAA,EAAE,MAAM,KAAA,EAAE,UAAU,KAAA,EAAE,QAAQ,KAAA;IAuB9C,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASpE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASvE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IASvE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAWvE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAYvE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAU,GAAG,MAAM;IAU1D,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAU,GAAG,MAAM;IAU1D,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAIvE,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAIvE,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAIxE,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM;IAIxE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,MAAM;IACxD,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,MAAM;IACxE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,MAAM;IAqExF,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM;IAyChF,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAA,GAAG,IAAI;IA+D7D,MAAM;IAWN,MAAM;IAYN,MAAM;IAcN,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM;IAajE,OAAO;IAcP,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO;IAM1B,OAAO,CACL,MAAM,EAAE,MAAM,GAAG,UAAU,EAC3B,KAAK,CAAC,EAAE,MAAM,EACd,GAAG,CAAC,EAAE,MAAM,EACZ,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM;IAkET,MAAM,IAAI;QAAC,IAAI,EAAE,QAAQ,CAAC;QAAC,IAAI,EAAE,GAAG,EAAE,CAAA;KAAC;IAOvC,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI;IA4DzC,SAAS,CAAC,aAAa,CAAC,QAAQ,KAAA,EAAE,KAAK,KAAA,EAAE,GAAG,KAAA;CAqE7C"}
|