@loaders.gl/parquet 3.1.8 → 3.2.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +31 -15
- package/dist/dist.min.js.map +3 -3
- package/dist/es5/index.js +25 -1
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/wasm/encode-parquet-wasm.js +56 -0
- package/dist/es5/lib/wasm/encode-parquet-wasm.js.map +1 -0
- package/dist/es5/lib/wasm/load-wasm/index.js +14 -0
- package/dist/es5/lib/wasm/load-wasm/index.js.map +1 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js +58 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js +43 -0
- package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map +1 -0
- package/dist/es5/lib/wasm/parse-parquet-wasm.js +80 -0
- package/dist/es5/lib/wasm/parse-parquet-wasm.js.map +1 -0
- package/dist/es5/parquet-loader.js +1 -1
- package/dist/es5/parquet-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-loader.js +30 -0
- package/dist/es5/parquet-wasm-loader.js.map +1 -0
- package/dist/es5/parquet-wasm-writer.js +28 -0
- package/dist/es5/parquet-wasm-writer.js.map +1 -0
- package/dist/es5/parquet-writer.js +1 -1
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/esm/index.js +7 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/wasm/encode-parquet-wasm.js +16 -0
- package/dist/esm/lib/wasm/encode-parquet-wasm.js.map +1 -0
- package/dist/esm/lib/wasm/load-wasm/index.js +2 -0
- package/dist/esm/lib/wasm/load-wasm/index.js.map +1 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js +12 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map +1 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js +5 -0
- package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map +1 -0
- package/dist/esm/lib/wasm/parse-parquet-wasm.js +25 -0
- package/dist/esm/lib/wasm/parse-parquet-wasm.js.map +1 -0
- package/dist/esm/parquet-loader.js +1 -1
- package/dist/esm/parquet-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-loader.js +22 -0
- package/dist/esm/parquet-wasm-loader.js.map +1 -0
- package/dist/esm/parquet-wasm-writer.js +19 -0
- package/dist/esm/parquet-wasm-writer.js.map +1 -0
- package/dist/esm/parquet-writer.js +1 -1
- package/dist/esm/parquet-writer.js.map +1 -1
- package/dist/index.d.ts +18 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -1
- package/dist/lib/wasm/encode-parquet-wasm.d.ts +21 -0
- package/dist/lib/wasm/encode-parquet-wasm.d.ts.map +1 -0
- package/dist/lib/wasm/encode-parquet-wasm.js +30 -0
- package/dist/lib/wasm/load-wasm/index.d.ts +2 -0
- package/dist/lib/wasm/load-wasm/index.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm/index.js +5 -0
- package/dist/lib/wasm/load-wasm/load-wasm-browser.d.ts +3 -0
- package/dist/lib/wasm/load-wasm/load-wasm-browser.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm/load-wasm-browser.js +34 -0
- package/dist/lib/wasm/load-wasm/load-wasm-node.d.ts +3 -0
- package/dist/lib/wasm/load-wasm/load-wasm-node.d.ts.map +1 -0
- package/dist/lib/wasm/load-wasm/load-wasm-node.js +27 -0
- package/dist/lib/wasm/parse-parquet-wasm.d.ts +10 -0
- package/dist/lib/wasm/parse-parquet-wasm.d.ts.map +1 -0
- package/dist/lib/wasm/parse-parquet-wasm.js +27 -0
- package/dist/parquet-wasm-loader.d.ts +23 -0
- package/dist/parquet-wasm-loader.d.ts.map +1 -0
- package/dist/parquet-wasm-loader.js +27 -0
- package/dist/parquet-wasm-writer.d.ts +3 -0
- package/dist/parquet-wasm-writer.d.ts.map +1 -0
- package/dist/parquet-wasm-writer.js +23 -0
- package/dist/parquet-worker.js +32 -16
- package/dist/parquet-worker.js.map +3 -3
- package/package.json +13 -7
- package/src/index.ts +9 -1
- package/src/lib/wasm/encode-parquet-wasm.ts +40 -0
- package/src/lib/wasm/load-wasm/index.ts +1 -0
- package/src/lib/wasm/load-wasm/load-wasm-browser.ts +15 -0
- package/src/lib/wasm/load-wasm/load-wasm-node.ts +5 -0
- package/src/lib/wasm/parse-parquet-wasm.ts +42 -0
- package/src/parquet-wasm-loader.ts +36 -0
- package/src/parquet-wasm-writer.ts +24 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,
|
|
1
|
+
{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQA,OAAO,MAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;AAcP,OAAO,MAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'object-row-table';\n url?: string;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'object-row-table',\n url: undefined\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
const VERSION = typeof "3.2.0-alpha.3" !== 'undefined' ? "3.2.0-alpha.3" : 'latest';
|
|
2
|
+
const DEFAULT_PARQUET_LOADER_OPTIONS = {
|
|
3
|
+
parquet: {
|
|
4
|
+
type: 'arrow-table',
|
|
5
|
+
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
6
|
+
}
|
|
7
|
+
};
|
|
8
|
+
export const ParquetWasmLoader = {
|
|
9
|
+
name: 'Apache Parquet',
|
|
10
|
+
id: 'parquet-wasm',
|
|
11
|
+
module: 'parquet',
|
|
12
|
+
version: VERSION,
|
|
13
|
+
worker: false,
|
|
14
|
+
category: 'table',
|
|
15
|
+
extensions: ['parquet'],
|
|
16
|
+
mimeTypes: ['application/octet-stream'],
|
|
17
|
+
binary: true,
|
|
18
|
+
tests: ['PAR1', 'PARE'],
|
|
19
|
+
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
20
|
+
};
|
|
21
|
+
export const _typecheckParquetLoader = ParquetWasmLoader;
|
|
22
|
+
//# sourceMappingURL=parquet-wasm-loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/parquet-wasm-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","wasmUrl","ParquetWasmLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,aADC;AAEPC,IAAAA,OAAO,EAAE;AAFF;AADkD,CAA7D;AAQA,OAAO,MAAMC,iBAAiB,GAAG;AAC/BC,EAAAA,IAAI,EAAE,gBADyB;AAE/BC,EAAAA,EAAE,EAAE,cAF2B;AAG/BC,EAAAA,MAAM,EAAE,SAHuB;AAI/BC,EAAAA,OAAO,EAAET,OAJsB;AAK/BU,EAAAA,MAAM,EAAE,KALuB;AAM/BC,EAAAA,QAAQ,EAAE,OANqB;AAO/BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPmB;AAQ/BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARoB;AAS/BC,EAAAA,MAAM,EAAE,IATuB;AAU/BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVwB;AAW/BC,EAAAA,OAAO,EAAEf;AAXsB,CAA1B;AAcP,OAAO,MAAMgB,uBAA+B,GAAGZ,iBAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'arrow-table';\n wasmUrl?: string;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'arrow-table',\n wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWasmLoader = {\n name: 'Apache Parquet',\n id: 'parquet-wasm',\n module: 'parquet',\n version: VERSION,\n worker: false,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetWasmLoader;\n"],"file":"parquet-wasm-loader.js"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { encode } from './lib/wasm/encode-parquet-wasm';
|
|
2
|
+
const VERSION = typeof "3.2.0-alpha.3" !== 'undefined' ? "3.2.0-alpha.3" : 'latest';
|
|
3
|
+
const DEFAULT_PARQUET_WRITER_OPTIONS = {
|
|
4
|
+
parquet: {
|
|
5
|
+
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
6
|
+
}
|
|
7
|
+
};
|
|
8
|
+
export const ParquetWasmWriter = {
|
|
9
|
+
name: 'Apache Parquet',
|
|
10
|
+
id: 'parquet-wasm',
|
|
11
|
+
module: 'parquet',
|
|
12
|
+
version: VERSION,
|
|
13
|
+
extensions: ['parquet'],
|
|
14
|
+
mimeTypes: ['application/octet-stream'],
|
|
15
|
+
encode,
|
|
16
|
+
binary: true,
|
|
17
|
+
options: DEFAULT_PARQUET_WRITER_OPTIONS
|
|
18
|
+
};
|
|
19
|
+
//# sourceMappingURL=parquet-wasm-writer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/parquet-wasm-writer.ts"],"names":["encode","VERSION","DEFAULT_PARQUET_WRITER_OPTIONS","parquet","wasmUrl","ParquetWasmWriter","name","id","module","version","extensions","mimeTypes","binary","options"],"mappings":"AACA,SAAQA,MAAR,QAA2C,gCAA3C;AAIA,MAAMC,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAEA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,OAAO,EAAE;AADF;AADkD,CAA7D;AAMA,OAAO,MAAMC,iBAAyB,GAAG;AACvCC,EAAAA,IAAI,EAAE,gBADiC;AAEvCC,EAAAA,EAAE,EAAE,cAFmC;AAGvCC,EAAAA,MAAM,EAAE,SAH+B;AAIvCC,EAAAA,OAAO,EAAER,OAJ8B;AAKvCS,EAAAA,UAAU,EAAE,CAAC,SAAD,CAL2B;AAMvCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CAN4B;AAOvCX,EAAAA,MAPuC;AAQvCY,EAAAA,MAAM,EAAE,IAR+B;AASvCC,EAAAA,OAAO,EAAEX;AAT8B,CAAlC","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\nimport {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {\n parquet: {\n wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n }\n};\n\nexport const ParquetWasmWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet-wasm',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encode,\n binary: true,\n options: DEFAULT_PARQUET_WRITER_OPTIONS\n};\n"],"file":"parquet-wasm-writer.js"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":"AAIA,MAAMA,OAAO,GAAG,
|
|
1
|
+
{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":"AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEA,OAAO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { LoaderWithParser } from '@loaders.gl/loader-utils';
|
|
2
|
+
import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
|
|
2
3
|
import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
|
|
3
4
|
import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
|
|
4
|
-
|
|
5
|
+
import { parseParquet as parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
|
|
6
|
+
export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
|
|
5
7
|
/** ParquetJS table loader */
|
|
6
8
|
export declare const ParquetLoader: {
|
|
7
9
|
parse: typeof parseParquet;
|
|
@@ -18,7 +20,22 @@ export declare const ParquetLoader: {
|
|
|
18
20
|
tests: string[];
|
|
19
21
|
options: import("./parquet-loader").ParquetLoaderOptions;
|
|
20
22
|
};
|
|
23
|
+
export declare const ParquetWasmLoader: {
|
|
24
|
+
parse: typeof parseParquetWasm;
|
|
25
|
+
name: string;
|
|
26
|
+
id: string;
|
|
27
|
+
module: string;
|
|
28
|
+
version: any;
|
|
29
|
+
worker: boolean;
|
|
30
|
+
category: string;
|
|
31
|
+
extensions: string[];
|
|
32
|
+
mimeTypes: string[];
|
|
33
|
+
binary: boolean;
|
|
34
|
+
tests: string[];
|
|
35
|
+
options: import("./parquet-wasm-loader").ParquetLoaderOptions;
|
|
36
|
+
};
|
|
21
37
|
export { ParquetWriter as _ParquetWriter } from './parquet-writer';
|
|
38
|
+
export { ParquetWasmWriter } from './parquet-wasm-writer';
|
|
22
39
|
export { preloadCompressions } from './parquetjs/compression';
|
|
23
40
|
export { ParquetSchema } from './parquetjs/schema/schema';
|
|
24
41
|
export { ParquetReader } from './parquetjs/parser/parquet-reader';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAI/D,OAAO,EAAC,aAAa,IAAI,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAC,YAAY,EAAE,yBAAyB,EAAC,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAI/D,OAAO,EAAC,iBAAiB,IAAI,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAC,aAAa,IAAI,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAC,YAAY,EAAE,yBAAyB,EAAC,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAC,YAAY,IAAI,gBAAgB,EAAC,MAAM,+BAA+B,CAAC;AAE/E,OAAO,EAAC,mBAAmB,EAAE,uBAAuB,EAAC,CAAC;AAEtD,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;;;CAIzB,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;CAG7B,CAAC;AAIF,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAC,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAC,qBAAqB,EAAC,MAAM,4CAA4C,CAAC;AAEjF,OAAO,EAAC,2BAA2B,EAAC,MAAM,sBAAsB,CAAC;AAGjE,eAAO,MAAM,uBAAuB,EAAE,gBAAgC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,19 +1,28 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.ParquetEnvelopeReader = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports._ParquetWriter = exports.ParquetLoader = exports.ParquetWorkerLoader = void 0;
|
|
3
|
+
exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.ParquetEnvelopeReader = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
|
|
4
4
|
// ParquetLoader
|
|
5
|
+
const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
|
|
6
|
+
Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
|
|
5
7
|
const parquet_loader_1 = require("./parquet-loader");
|
|
6
8
|
Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
|
|
7
9
|
const parse_parquet_1 = require("./lib/parse-parquet");
|
|
10
|
+
const parse_parquet_wasm_1 = require("./lib/wasm/parse-parquet-wasm");
|
|
8
11
|
/** ParquetJS table loader */
|
|
9
12
|
exports.ParquetLoader = {
|
|
10
13
|
...parquet_loader_1.ParquetLoader,
|
|
11
14
|
parse: parse_parquet_1.parseParquet,
|
|
12
15
|
parseFileInBatches: parse_parquet_1.parseParquetFileInBatches
|
|
13
16
|
};
|
|
17
|
+
exports.ParquetWasmLoader = {
|
|
18
|
+
...parquet_wasm_loader_1.ParquetWasmLoader,
|
|
19
|
+
parse: parse_parquet_wasm_1.parseParquet
|
|
20
|
+
};
|
|
14
21
|
// ParquetWriter
|
|
15
22
|
var parquet_writer_1 = require("./parquet-writer");
|
|
16
23
|
Object.defineProperty(exports, "_ParquetWriter", { enumerable: true, get: function () { return parquet_writer_1.ParquetWriter; } });
|
|
24
|
+
var parquet_wasm_writer_1 = require("./parquet-wasm-writer");
|
|
25
|
+
Object.defineProperty(exports, "ParquetWasmWriter", { enumerable: true, get: function () { return parquet_wasm_writer_1.ParquetWasmWriter; } });
|
|
17
26
|
// EXPERIMENTAL - expose the internal parquetjs API
|
|
18
27
|
var compression_1 = require("./parquetjs/compression");
|
|
19
28
|
Object.defineProperty(exports, "preloadCompressions", { enumerable: true, get: function () { return compression_1.preloadCompressions; } });
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Table } from 'apache-arrow';
|
|
2
|
+
import type { WriterOptions } from '@loaders.gl/loader-utils';
|
|
3
|
+
export declare type ParquetWriterOptions = WriterOptions & {
|
|
4
|
+
parquet?: {
|
|
5
|
+
wasmUrl?: string;
|
|
6
|
+
};
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Encode Arrow Table to Parquet buffer
|
|
10
|
+
*/
|
|
11
|
+
export declare function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer>;
|
|
12
|
+
/**
|
|
13
|
+
* Serialize a {@link Table} to the IPC format. This function is a convenience
|
|
14
|
+
* wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.
|
|
15
|
+
* Opposite of {@link tableFromIPC}.
|
|
16
|
+
*
|
|
17
|
+
* @param table The Table to serialize.
|
|
18
|
+
* @param type Whether to serialize the Table as a file or a stream.
|
|
19
|
+
*/
|
|
20
|
+
export declare function tableToIPC(table: Table): Uint8Array;
|
|
21
|
+
//# sourceMappingURL=encode-parquet-wasm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"encode-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/wasm/encode-parquet-wasm.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,cAAc,CAAC;AACxC,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAK5D,oBAAY,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF;;GAEG;AACH,wBAAsB,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,WAAW,CAAC,CAY/F;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAEnD"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.tableToIPC = exports.encode = void 0;
|
|
4
|
+
const apache_arrow_1 = require("apache-arrow");
|
|
5
|
+
const load_wasm_1 = require("./load-wasm");
|
|
6
|
+
/**
|
|
7
|
+
* Encode Arrow Table to Parquet buffer
|
|
8
|
+
*/
|
|
9
|
+
async function encode(table, options) {
|
|
10
|
+
const wasmUrl = options?.parquet?.wasmUrl;
|
|
11
|
+
const wasm = await (0, load_wasm_1.loadWasm)(wasmUrl);
|
|
12
|
+
const arrowIPCBytes = tableToIPC(table);
|
|
13
|
+
// TODO: provide options for how to write table.
|
|
14
|
+
const writerProperties = new wasm.WriterPropertiesBuilder().build();
|
|
15
|
+
const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
|
|
16
|
+
return parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset);
|
|
17
|
+
}
|
|
18
|
+
exports.encode = encode;
|
|
19
|
+
/**
|
|
20
|
+
* Serialize a {@link Table} to the IPC format. This function is a convenience
|
|
21
|
+
* wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.
|
|
22
|
+
* Opposite of {@link tableFromIPC}.
|
|
23
|
+
*
|
|
24
|
+
* @param table The Table to serialize.
|
|
25
|
+
* @param type Whether to serialize the Table as a file or a stream.
|
|
26
|
+
*/
|
|
27
|
+
function tableToIPC(table) {
|
|
28
|
+
return apache_arrow_1.RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
|
|
29
|
+
}
|
|
30
|
+
exports.tableToIPC = tableToIPC;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/wasm/load-wasm/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,QAAQ,EAAC,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.loadWasm = void 0;
|
|
4
|
+
var load_wasm_node_1 = require("./load-wasm-node");
|
|
5
|
+
Object.defineProperty(exports, "loadWasm", { enumerable: true, get: function () { return load_wasm_node_1.loadWasm; } });
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-wasm-browser.d.ts","sourceRoot":"","sources":["../../../../src/lib/wasm/load-wasm/load-wasm-browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,0BAA0B,CAAC;AAIpD,wBAAsB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,2BAU9C"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
+
}) : (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
o[k2] = m[k];
|
|
8
|
+
}));
|
|
9
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
+
}) : function(o, v) {
|
|
12
|
+
o["default"] = v;
|
|
13
|
+
});
|
|
14
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
+
if (mod && mod.__esModule) return mod;
|
|
16
|
+
var result = {};
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
+
__setModuleDefault(result, mod);
|
|
19
|
+
return result;
|
|
20
|
+
};
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.loadWasm = void 0;
|
|
23
|
+
const wasmEsm = __importStar(require("parquet-wasm/esm2/arrow1"));
|
|
24
|
+
let cached = null;
|
|
25
|
+
async function loadWasm(wasmUrl) {
|
|
26
|
+
if (cached !== null) {
|
|
27
|
+
return cached;
|
|
28
|
+
}
|
|
29
|
+
// For ESM bundles, need to await the default export, which loads the WASM
|
|
30
|
+
await wasmEsm.default(wasmUrl);
|
|
31
|
+
cached = wasmEsm;
|
|
32
|
+
return wasmEsm;
|
|
33
|
+
}
|
|
34
|
+
exports.loadWasm = loadWasm;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-wasm-node.d.ts","sourceRoot":"","sources":["../../../../src/lib/wasm/load-wasm/load-wasm-node.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,0BAA0B,CAAC;AAErD,wBAAsB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,4BAE9C"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
+
}) : (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
o[k2] = m[k];
|
|
8
|
+
}));
|
|
9
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
+
}) : function(o, v) {
|
|
12
|
+
o["default"] = v;
|
|
13
|
+
});
|
|
14
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
+
if (mod && mod.__esModule) return mod;
|
|
16
|
+
var result = {};
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
+
__setModuleDefault(result, mod);
|
|
19
|
+
return result;
|
|
20
|
+
};
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.loadWasm = void 0;
|
|
23
|
+
const wasmNode = __importStar(require("parquet-wasm/node/arrow1"));
|
|
24
|
+
async function loadWasm(wasmUrl) {
|
|
25
|
+
return wasmNode;
|
|
26
|
+
}
|
|
27
|
+
exports.loadWasm = loadWasm;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { LoaderOptions } from '@loaders.gl/loader-utils';
|
|
2
|
+
import { Table } from 'apache-arrow';
|
|
3
|
+
export declare type ParquetLoaderOptions = LoaderOptions & {
|
|
4
|
+
parquet?: {
|
|
5
|
+
type?: 'arrow-table';
|
|
6
|
+
wasmUrl?: string;
|
|
7
|
+
};
|
|
8
|
+
};
|
|
9
|
+
export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<Table>;
|
|
10
|
+
//# sourceMappingURL=parse-parquet-wasm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/wasm/parse-parquet-wasm.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAC,KAAK,EAA0B,MAAM,cAAc,CAAC;AAG5D,oBAAY,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,wBAAsB,YAAY,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,KAAK,CAAC,CAYhB"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseParquet = void 0;
|
|
4
|
+
const apache_arrow_1 = require("apache-arrow");
|
|
5
|
+
const load_wasm_node_1 = require("./load-wasm/load-wasm-node");
|
|
6
|
+
async function parseParquet(arrayBuffer, options) {
|
|
7
|
+
const wasmUrl = options?.parquet?.wasmUrl;
|
|
8
|
+
const wasm = await (0, load_wasm_node_1.loadWasm)(wasmUrl);
|
|
9
|
+
const arr = new Uint8Array(arrayBuffer);
|
|
10
|
+
const arrowIPCUint8Arr = wasm.readParquet(arr);
|
|
11
|
+
const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(arrowIPCUint8Arr.byteOffset, arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset);
|
|
12
|
+
const arrowTable = tableFromIPC(arrowIPCBuffer);
|
|
13
|
+
return arrowTable;
|
|
14
|
+
}
|
|
15
|
+
exports.parseParquet = parseParquet;
|
|
16
|
+
/**
|
|
17
|
+
* Deserialize the IPC format into a {@link Table}. This function is a
|
|
18
|
+
* convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
|
|
19
|
+
*/
|
|
20
|
+
function tableFromIPC(input) {
|
|
21
|
+
const reader = apache_arrow_1.RecordBatchStreamReader.from(input);
|
|
22
|
+
const recordBatches = [];
|
|
23
|
+
for (const recordBatch of reader) {
|
|
24
|
+
recordBatches.push(recordBatch);
|
|
25
|
+
}
|
|
26
|
+
return new apache_arrow_1.Table(recordBatches);
|
|
27
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Loader, LoaderOptions } from '@loaders.gl/loader-utils';
|
|
2
|
+
export declare type ParquetLoaderOptions = LoaderOptions & {
|
|
3
|
+
parquet?: {
|
|
4
|
+
type?: 'arrow-table';
|
|
5
|
+
wasmUrl?: string;
|
|
6
|
+
};
|
|
7
|
+
};
|
|
8
|
+
/** ParquetJS table loader */
|
|
9
|
+
export declare const ParquetWasmLoader: {
|
|
10
|
+
name: string;
|
|
11
|
+
id: string;
|
|
12
|
+
module: string;
|
|
13
|
+
version: any;
|
|
14
|
+
worker: boolean;
|
|
15
|
+
category: string;
|
|
16
|
+
extensions: string[];
|
|
17
|
+
mimeTypes: string[];
|
|
18
|
+
binary: boolean;
|
|
19
|
+
tests: string[];
|
|
20
|
+
options: ParquetLoaderOptions;
|
|
21
|
+
};
|
|
22
|
+
export declare const _typecheckParquetLoader: Loader;
|
|
23
|
+
//# sourceMappingURL=parquet-wasm-loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet-wasm-loader.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,oBAAY,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AASF,6BAA6B;AAC7B,eAAO,MAAM,iBAAiB;;;;;;;;;;;;CAY7B,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAA0B,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports._typecheckParquetLoader = exports.ParquetWasmLoader = void 0;
|
|
4
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
5
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
6
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
7
|
+
const DEFAULT_PARQUET_LOADER_OPTIONS = {
|
|
8
|
+
parquet: {
|
|
9
|
+
type: 'arrow-table',
|
|
10
|
+
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
/** ParquetJS table loader */
|
|
14
|
+
exports.ParquetWasmLoader = {
|
|
15
|
+
name: 'Apache Parquet',
|
|
16
|
+
id: 'parquet-wasm',
|
|
17
|
+
module: 'parquet',
|
|
18
|
+
version: VERSION,
|
|
19
|
+
worker: false,
|
|
20
|
+
category: 'table',
|
|
21
|
+
extensions: ['parquet'],
|
|
22
|
+
mimeTypes: ['application/octet-stream'],
|
|
23
|
+
binary: true,
|
|
24
|
+
tests: ['PAR1', 'PARE'],
|
|
25
|
+
options: DEFAULT_PARQUET_LOADER_OPTIONS
|
|
26
|
+
};
|
|
27
|
+
exports._typecheckParquetLoader = exports.ParquetWasmLoader;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parquet-wasm-writer.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAarD,eAAO,MAAM,iBAAiB,EAAE,MAU/B,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ParquetWasmWriter = void 0;
|
|
4
|
+
const encode_parquet_wasm_1 = require("./lib/wasm/encode-parquet-wasm");
|
|
5
|
+
// __VERSION__ is injected by babel-plugin-version-inline
|
|
6
|
+
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
7
|
+
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
8
|
+
const DEFAULT_PARQUET_WRITER_OPTIONS = {
|
|
9
|
+
parquet: {
|
|
10
|
+
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
exports.ParquetWasmWriter = {
|
|
14
|
+
name: 'Apache Parquet',
|
|
15
|
+
id: 'parquet-wasm',
|
|
16
|
+
module: 'parquet',
|
|
17
|
+
version: VERSION,
|
|
18
|
+
extensions: ['parquet'],
|
|
19
|
+
mimeTypes: ['application/octet-stream'],
|
|
20
|
+
encode: encode_parquet_wasm_1.encode,
|
|
21
|
+
binary: true,
|
|
22
|
+
options: DEFAULT_PARQUET_WRITER_OPTIONS
|
|
23
|
+
};
|