@loaders.gl/parquet 3.3.2 → 3.4.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +16 -16
- package/dist/dist.min.js.map +3 -3
- package/dist/es5/index.js +46 -5
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/arrow/convert-columns-to-row-group.js +2 -0
- package/dist/es5/lib/arrow/convert-columns-to-row-group.js.map +1 -0
- package/dist/es5/lib/arrow/convert-row-group-to-columns.js +20 -0
- package/dist/es5/lib/arrow/convert-row-group-to-columns.js.map +1 -0
- package/dist/es5/lib/arrow/convert-schema-from-parquet.js +98 -0
- package/dist/es5/lib/arrow/convert-schema-from-parquet.js.map +1 -0
- package/dist/es5/lib/{convert-schema.js → arrow/convert-schema-to-parquet.js} +5 -31
- package/dist/es5/lib/arrow/convert-schema-to-parquet.js.map +1 -0
- package/dist/es5/lib/geo/decode-geo-metadata.js +82 -0
- package/dist/es5/lib/geo/decode-geo-metadata.js.map +1 -0
- package/dist/es5/lib/geo/geoparquet-schema.js +83 -0
- package/dist/es5/lib/geo/geoparquet-schema.js.map +1 -0
- package/dist/es5/lib/parsers/parse-parquet-to-columns.js +177 -0
- package/dist/es5/lib/parsers/parse-parquet-to-columns.js.map +1 -0
- package/dist/es5/lib/{parse-parquet.js → parsers/parse-parquet-to-rows.js} +2 -2
- package/dist/es5/lib/parsers/parse-parquet-to-rows.js.map +1 -0
- package/dist/es5/lib/wip/convert-schema-deep.rs.disabled +976 -0
- package/dist/es5/parquet-loader.js +3 -2
- package/dist/es5/parquet-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-loader.js +1 -1
- package/dist/es5/parquet-wasm-loader.js.map +1 -1
- package/dist/es5/parquet-wasm-writer.js +1 -1
- package/dist/es5/parquet-wasm-writer.js.map +1 -1
- package/dist/es5/parquet-writer.js +1 -1
- package/dist/es5/parquet-writer.js.map +1 -1
- package/dist/esm/index.js +12 -2
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/arrow/convert-columns-to-row-group.js +2 -0
- package/dist/esm/lib/arrow/convert-columns-to-row-group.js.map +1 -0
- package/dist/esm/lib/arrow/convert-row-group-to-columns.js +10 -0
- package/dist/esm/lib/arrow/convert-row-group-to-columns.js.map +1 -0
- package/dist/esm/lib/{convert-schema.js → arrow/convert-schema-from-parquet.js} +32 -16
- package/dist/esm/lib/arrow/convert-schema-from-parquet.js.map +1 -0
- package/dist/esm/lib/arrow/convert-schema-to-parquet.js +40 -0
- package/dist/esm/lib/arrow/convert-schema-to-parquet.js.map +1 -0
- package/dist/esm/lib/geo/decode-geo-metadata.js +64 -0
- package/dist/esm/lib/geo/decode-geo-metadata.js.map +1 -0
- package/dist/esm/lib/geo/geoparquet-schema.js +78 -0
- package/dist/esm/lib/geo/geoparquet-schema.js.map +1 -0
- package/dist/esm/lib/parsers/parse-parquet-to-columns.js +37 -0
- package/dist/esm/lib/parsers/parse-parquet-to-columns.js.map +1 -0
- package/dist/esm/lib/{parse-parquet.js → parsers/parse-parquet-to-rows.js} +2 -2
- package/dist/esm/lib/parsers/parse-parquet-to-rows.js.map +1 -0
- package/dist/esm/lib/wip/convert-schema-deep.rs.disabled +976 -0
- package/dist/esm/parquet-loader.js +3 -2
- package/dist/esm/parquet-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-loader.js +1 -1
- package/dist/esm/parquet-wasm-loader.js.map +1 -1
- package/dist/esm/parquet-wasm-writer.js +1 -1
- package/dist/esm/parquet-wasm-writer.js.map +1 -1
- package/dist/esm/parquet-writer.js +1 -1
- package/dist/esm/parquet-writer.js.map +1 -1
- package/dist/index.d.ts +23 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +24 -6
- package/dist/lib/arrow/convert-columns-to-row-group.d.ts +1 -0
- package/dist/lib/arrow/convert-columns-to-row-group.d.ts.map +1 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js +1 -0
- package/dist/lib/arrow/convert-row-group-to-columns.d.ts +4 -0
- package/dist/lib/arrow/convert-row-group-to-columns.d.ts.map +1 -0
- package/dist/lib/arrow/convert-row-group-to-columns.js +12 -0
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts +9 -0
- package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map +1 -0
- package/dist/lib/{convert-schema.js → arrow/convert-schema-from-parquet.js} +30 -18
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts +7 -0
- package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map +1 -0
- package/dist/lib/arrow/convert-schema-to-parquet.js +72 -0
- package/dist/lib/geo/decode-geo-metadata.d.ts +31 -0
- package/dist/lib/geo/decode-geo-metadata.d.ts.map +1 -0
- package/dist/lib/geo/decode-geo-metadata.js +73 -0
- package/dist/lib/geo/geoparquet-schema.d.ts +80 -0
- package/dist/lib/geo/geoparquet-schema.d.ts.map +1 -0
- package/dist/lib/geo/geoparquet-schema.js +69 -0
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts +5 -0
- package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-columns.js +40 -0
- package/dist/lib/parsers/parse-parquet-to-rows.d.ts +4 -0
- package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map +1 -0
- package/dist/lib/{parse-parquet.js → parsers/parse-parquet-to-rows.js} +1 -1
- package/dist/parquet-loader.d.ts +1 -0
- package/dist/parquet-loader.d.ts.map +1 -1
- package/dist/parquet-loader.js +2 -1
- package/dist/parquet-worker.js +19 -19
- package/dist/parquet-worker.js.map +3 -3
- package/package.json +5 -5
- package/src/index.ts +22 -2
- package/src/lib/arrow/convert-columns-to-row-group.ts +0 -0
- package/src/lib/arrow/convert-row-group-to-columns.ts +15 -0
- package/src/lib/{convert-schema.ts → arrow/convert-schema-from-parquet.ts} +41 -22
- package/src/lib/arrow/convert-schema-to-parquet.ts +102 -0
- package/src/lib/geo/decode-geo-metadata.ts +99 -0
- package/src/lib/geo/geoparquet-schema.ts +69 -0
- package/src/lib/parsers/parse-parquet-to-columns.ts +49 -0
- package/src/lib/{parse-parquet.ts → parsers/parse-parquet-to-rows.ts} +2 -2
- package/src/lib/wip/convert-schema-deep.rs.disabled +976 -0
- package/src/parquet-loader.ts +3 -1
- package/dist/es5/lib/convert-schema.js.map +0 -1
- package/dist/es5/lib/parse-parquet.js.map +0 -1
- package/dist/es5/lib/read-array-buffer.js +0 -43
- package/dist/es5/lib/read-array-buffer.js.map +0 -1
- package/dist/esm/lib/convert-schema.js.map +0 -1
- package/dist/esm/lib/parse-parquet.js.map +0 -1
- package/dist/esm/lib/read-array-buffer.js +0 -10
- package/dist/esm/lib/read-array-buffer.js.map +0 -1
- package/dist/lib/convert-schema.d.ts +0 -8
- package/dist/lib/convert-schema.d.ts.map +0 -1
- package/dist/lib/parse-parquet.d.ts +0 -4
- package/dist/lib/parse-parquet.d.ts.map +0 -1
- package/dist/lib/read-array-buffer.d.ts +0 -19
- package/dist/lib/read-array-buffer.d.ts.map +0 -1
- package/dist/lib/read-array-buffer.js +0 -29
- package/src/lib/read-array-buffer.ts +0 -31
- /package/dist/es5/lib/{convert-schema-deep.ts.disabled → wip/convert-schema-deep.java.disabled} +0 -0
- /package/dist/esm/lib/{convert-schema-deep.ts.disabled → wip/convert-schema-deep.java.disabled} +0 -0
- /package/src/lib/{convert-schema-deep.ts.disabled → wip/convert-schema-deep.java.disabled} +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
|
|
2
|
-
const VERSION = typeof "3.
|
|
2
|
+
const VERSION = typeof "3.4.0-alpha.2" !== 'undefined' ? "3.4.0-alpha.2" : 'latest';
|
|
3
3
|
const DEFAULT_PARQUET_LOADER_OPTIONS = {
|
|
4
4
|
parquet: {
|
|
5
5
|
type: 'object-row-table',
|
|
6
6
|
url: undefined,
|
|
7
|
-
columnList: []
|
|
7
|
+
columnList: [],
|
|
8
|
+
geoparquet: true
|
|
8
9
|
}
|
|
9
10
|
};
|
|
10
11
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","columnList","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'object-row-table';\n url?: string;\n columnList?: string[] | string[][];\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'object-row-table',\n url: undefined,\n columnList: []\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,
|
|
1
|
+
{"version":3,"file":"parquet-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","columnList","geoparquet","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'object-row-table';\n url?: string;\n columnList?: string[] | string[][];\n geoparquet?: boolean;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'object-row-table',\n url: undefined,\n columnList: [],\n geoparquet: true\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAW3E,MAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,kBAAkB;IACxBC,GAAG,EAAEC,SAAS;IACdC,UAAU,EAAE,EAAE;IACdC,UAAU,EAAE;EACd;AACF,CAAC;;AAGD,OAAO,MAAMC,aAAa,GAAG;EAC3BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEZ,OAAO;EAChBa,MAAM,EAAE,IAAI;EACZC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAElB;AACX,CAAC;AAED,OAAO,MAAMmB,uBAA+B,GAAGZ,aAAa"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-wasm-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","wasmUrl","ParquetWasmLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-wasm-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'arrow-table';\n wasmUrl?: string;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'arrow-table',\n wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWasmLoader = {\n name: 'Apache Parquet',\n id: 'parquet-wasm',\n module: 'parquet',\n version: VERSION,\n worker: false,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetWasmLoader;\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,
|
|
1
|
+
{"version":3,"file":"parquet-wasm-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","wasmUrl","ParquetWasmLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-wasm-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n parquet?: {\n type?: 'arrow-table';\n wasmUrl?: string;\n };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n parquet: {\n type: 'arrow-table',\n wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWasmLoader = {\n name: 'Apache Parquet',\n id: 'parquet-wasm',\n module: 'parquet',\n version: VERSION,\n worker: false,\n category: 'table',\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['PAR1', 'PARE'],\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetWasmLoader;\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAS3E,MAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,aAAa;IACnBC,OAAO,EAAE;EACX;AACF,CAAC;;AAGD,OAAO,MAAMC,iBAAiB,GAAG;EAC/BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,cAAc;EAClBC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAET,OAAO;EAChBU,MAAM,EAAE,KAAK;EACbC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAEf;AACX,CAAC;AAED,OAAO,MAAMgB,uBAA+B,GAAGZ,iBAAiB"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { encode } from './lib/wasm/encode-parquet-wasm';
|
|
2
2
|
|
|
3
|
-
const VERSION = typeof "3.
|
|
3
|
+
const VERSION = typeof "3.4.0-alpha.2" !== 'undefined' ? "3.4.0-alpha.2" : 'latest';
|
|
4
4
|
const DEFAULT_PARQUET_WRITER_OPTIONS = {
|
|
5
5
|
parquet: {
|
|
6
6
|
wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-wasm-writer.js","names":["encode","VERSION","DEFAULT_PARQUET_WRITER_OPTIONS","parquet","wasmUrl","ParquetWasmWriter","name","id","module","version","extensions","mimeTypes","binary","options"],"sources":["../../src/parquet-wasm-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\nimport {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {\n parquet: {\n wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n }\n};\n\nexport const ParquetWasmWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet-wasm',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encode,\n binary: true,\n options: DEFAULT_PARQUET_WRITER_OPTIONS\n};\n"],"mappings":"AACA,SAAQA,MAAM,QAA6B,gCAAgC;;AAI3E,MAAMC,OAAO,GAAG,
|
|
1
|
+
{"version":3,"file":"parquet-wasm-writer.js","names":["encode","VERSION","DEFAULT_PARQUET_WRITER_OPTIONS","parquet","wasmUrl","ParquetWasmWriter","name","id","module","version","extensions","mimeTypes","binary","options"],"sources":["../../src/parquet-wasm-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\nimport {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {\n parquet: {\n wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n }\n};\n\nexport const ParquetWasmWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet-wasm',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encode,\n binary: true,\n options: DEFAULT_PARQUET_WRITER_OPTIONS\n};\n"],"mappings":"AACA,SAAQA,MAAM,QAA6B,gCAAgC;;AAI3E,MAAMC,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAE3E,MAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,OAAO,EAAE;EACX;AACF,CAAC;AAED,OAAO,MAAMC,iBAAyB,GAAG;EACvCC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,cAAc;EAClBC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAER,OAAO;EAChBS,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCX,MAAM;EACNY,MAAM,EAAE,IAAI;EACZC,OAAO,EAAEX;AACX,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet-writer.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"sources":["../../src/parquet-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n return new ArrayBuffer(0);\n}\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,
|
|
1
|
+
{"version":3,"file":"parquet-writer.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"sources":["../../src/parquet-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n name: 'Apache Parquet',\n id: 'parquet',\n module: 'parquet',\n version: VERSION,\n extensions: ['parquet'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n return new ArrayBuffer(0);\n}\n"],"mappings":";AAIA,MAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAI3E,MAAMC,8BAA8B,GAAG,CAAC,CAAC;AAEzC,OAAO,MAAMC,aAAqB,GAAG;EACnCC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEN,OAAO;EAChBO,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,UAAU;EACVC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAEV;AACX,CAAC;AAED,SAASQ,UAAU,CAACG,IAAI,EAAED,OAA8B,EAAE;EACxD,OAAO,IAAIE,WAAW,CAAC,CAAC,CAAC;AAC3B"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { LoaderWithParser } from '@loaders.gl/loader-utils';
|
|
2
2
|
import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
|
|
3
3
|
import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
|
|
4
|
-
import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
|
|
4
|
+
import { parseParquet, parseParquetFileInBatches } from './lib/parsers/parse-parquet-to-rows';
|
|
5
|
+
import { parseParquetInColumns, parseParquetFileInColumnarBatches } from './lib/parsers/parse-parquet-to-columns';
|
|
5
6
|
import { parseParquet as parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
|
|
6
7
|
export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
|
|
7
8
|
/** ParquetJS table loader */
|
|
@@ -10,7 +11,23 @@ export declare const ParquetLoader: {
|
|
|
10
11
|
parseFileInBatches: typeof parseParquetFileInBatches;
|
|
11
12
|
name: string;
|
|
12
13
|
id: string;
|
|
13
|
-
module: string;
|
|
14
|
+
module: string; /** ParquetJS table loader */
|
|
15
|
+
version: any;
|
|
16
|
+
worker: boolean;
|
|
17
|
+
category: string;
|
|
18
|
+
extensions: string[];
|
|
19
|
+
mimeTypes: string[];
|
|
20
|
+
binary: boolean;
|
|
21
|
+
tests: string[];
|
|
22
|
+
options: import("./parquet-loader").ParquetLoaderOptions;
|
|
23
|
+
};
|
|
24
|
+
/** ParquetJS table loader */
|
|
25
|
+
export declare const ParquetColumnarLoader: {
|
|
26
|
+
parse: typeof parseParquetInColumns;
|
|
27
|
+
parseFileInBatches: typeof parseParquetFileInColumnarBatches;
|
|
28
|
+
name: string;
|
|
29
|
+
id: string;
|
|
30
|
+
module: string; /** ParquetJS table loader */
|
|
14
31
|
version: any;
|
|
15
32
|
worker: boolean;
|
|
16
33
|
category: string;
|
|
@@ -40,6 +57,9 @@ export { preloadCompressions } from './parquetjs/compression';
|
|
|
40
57
|
export { ParquetSchema } from './parquetjs/schema/schema';
|
|
41
58
|
export { ParquetReader } from './parquetjs/parser/parquet-reader';
|
|
42
59
|
export { ParquetEncoder } from './parquetjs/encoder/parquet-encoder';
|
|
43
|
-
export { convertParquetToArrowSchema } from './lib/convert-schema';
|
|
60
|
+
export { convertSchemaFromParquet, convertSchemaFromParquet as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
|
|
44
61
|
export declare const _typecheckParquetLoader: LoaderWithParser;
|
|
62
|
+
export { default as geoJSONSchema } from './lib/geo/geoparquet-schema';
|
|
63
|
+
export type { GeoMetadata } from './lib/geo/decode-geo-metadata';
|
|
64
|
+
export { getGeoMetadata, setGeoMetadata, unpackGeoMetadata } from './lib/geo/decode-geo-metadata';
|
|
45
65
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAI/D,OAAO,EAAC,iBAAiB,IAAI,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAC,aAAa,IAAI,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAC,YAAY,EAAE,yBAAyB,EAAC,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAI/D,OAAO,EAAC,iBAAiB,IAAI,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAC,aAAa,IAAI,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAC,YAAY,EAAE,yBAAyB,EAAC,MAAM,qCAAqC,CAAC;AAC5F,OAAO,EACL,qBAAqB,EACrB,iCAAiC,EAClC,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAC,YAAY,IAAI,gBAAgB,EAAC,MAAM,+BAA+B,CAAC;AAE/E,OAAO,EAAC,mBAAmB,EAAE,uBAAuB,EAAC,CAAC;AAEtD,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;oBAM1B,6BAA6B;;;;;;;;;CAF5B,CAAC;AAEF,6BAA6B;AAC7B,eAAO,MAAM,qBAAqB;;;;;oBADlC,6BAA6B;;;;;;;;;CAK5B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;CAG7B,CAAC;AAIF,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAC,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EACL,wBAAwB,EACxB,wBAAwB,IAAI,2BAA2B,EACxD,MAAM,yCAAyC,CAAC;AAGjD,eAAO,MAAM,uBAAuB,EAAE,gBAAgC,CAAC;AAGvE,OAAO,EAAC,OAAO,IAAI,aAAa,EAAC,MAAM,6BAA6B,CAAC;AAErE,YAAY,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAC,cAAc,EAAE,cAAc,EAAE,iBAAiB,EAAC,MAAM,+BAA+B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,18 +1,28 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
|
|
6
|
+
exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = exports.geoJSONSchema = exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.convertSchemaFromParquet = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetColumnarLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
|
|
4
7
|
// ParquetLoader
|
|
5
8
|
const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
|
|
6
9
|
Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
|
|
7
10
|
const parquet_loader_1 = require("./parquet-loader");
|
|
8
11
|
Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
|
|
9
|
-
const
|
|
12
|
+
const parse_parquet_to_rows_1 = require("./lib/parsers/parse-parquet-to-rows");
|
|
13
|
+
const parse_parquet_to_columns_1 = require("./lib/parsers/parse-parquet-to-columns");
|
|
10
14
|
const parse_parquet_wasm_1 = require("./lib/wasm/parse-parquet-wasm");
|
|
11
15
|
/** ParquetJS table loader */
|
|
12
16
|
exports.ParquetLoader = {
|
|
13
17
|
...parquet_loader_1.ParquetLoader,
|
|
14
|
-
parse:
|
|
15
|
-
parseFileInBatches:
|
|
18
|
+
parse: parse_parquet_to_rows_1.parseParquet,
|
|
19
|
+
parseFileInBatches: parse_parquet_to_rows_1.parseParquetFileInBatches
|
|
20
|
+
};
|
|
21
|
+
/** ParquetJS table loader */
|
|
22
|
+
exports.ParquetColumnarLoader = {
|
|
23
|
+
...parquet_loader_1.ParquetLoader,
|
|
24
|
+
parse: parse_parquet_to_columns_1.parseParquetInColumns,
|
|
25
|
+
parseFileInBatches: parse_parquet_to_columns_1.parseParquetFileInColumnarBatches
|
|
16
26
|
};
|
|
17
27
|
exports.ParquetWasmLoader = {
|
|
18
28
|
...parquet_wasm_loader_1.ParquetWasmLoader,
|
|
@@ -32,7 +42,15 @@ var parquet_reader_1 = require("./parquetjs/parser/parquet-reader");
|
|
|
32
42
|
Object.defineProperty(exports, "ParquetReader", { enumerable: true, get: function () { return parquet_reader_1.ParquetReader; } });
|
|
33
43
|
var parquet_encoder_1 = require("./parquetjs/encoder/parquet-encoder");
|
|
34
44
|
Object.defineProperty(exports, "ParquetEncoder", { enumerable: true, get: function () { return parquet_encoder_1.ParquetEncoder; } });
|
|
35
|
-
var
|
|
36
|
-
Object.defineProperty(exports, "
|
|
45
|
+
var convert_schema_from_parquet_1 = require("./lib/arrow/convert-schema-from-parquet");
|
|
46
|
+
Object.defineProperty(exports, "convertSchemaFromParquet", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertSchemaFromParquet; } });
|
|
47
|
+
Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertSchemaFromParquet; } });
|
|
37
48
|
// TESTS
|
|
38
49
|
exports._typecheckParquetLoader = exports.ParquetLoader;
|
|
50
|
+
// Geo Metadata
|
|
51
|
+
var geoparquet_schema_1 = require("./lib/geo/geoparquet-schema");
|
|
52
|
+
Object.defineProperty(exports, "geoJSONSchema", { enumerable: true, get: function () { return __importDefault(geoparquet_schema_1).default; } });
|
|
53
|
+
var decode_geo_metadata_1 = require("./lib/geo/decode-geo-metadata");
|
|
54
|
+
Object.defineProperty(exports, "getGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.getGeoMetadata; } });
|
|
55
|
+
Object.defineProperty(exports, "setGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.setGeoMetadata; } });
|
|
56
|
+
Object.defineProperty(exports, "unpackGeoMetadata", { enumerable: true, get: function () { return decode_geo_metadata_1.unpackGeoMetadata; } });
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
//# sourceMappingURL=convert-columns-to-row-group.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"convert-columns-to-row-group.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-columns-to-row-group.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { Schema } from '@loaders.gl/schema';
|
|
2
|
+
import { ParquetBuffer } from '@loaders.gl/parquet/parquetjs/schema/declare';
|
|
3
|
+
export declare function convertParquetRowGroupToColumns(schema: Schema, rowGroup: ParquetBuffer): Record<string, any[]>;
|
|
4
|
+
//# sourceMappingURL=convert-row-group-to-columns.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"convert-row-group-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-row-group-to-columns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,aAAa,EAAC,MAAM,8CAA8C,CAAC;AAE3E,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,aAAa,GACtB,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAMvB"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// loaders.gl, MIT license
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.convertParquetRowGroupToColumns = void 0;
|
|
5
|
+
function convertParquetRowGroupToColumns(schema, rowGroup) {
|
|
6
|
+
const columns = {};
|
|
7
|
+
for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
|
|
8
|
+
columns[columnName] = columns[columnName] || data.values;
|
|
9
|
+
}
|
|
10
|
+
return columns;
|
|
11
|
+
}
|
|
12
|
+
exports.convertParquetRowGroupToColumns = convertParquetRowGroupToColumns;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ParquetSchema } from '../../parquetjs/schema/schema';
|
|
2
|
+
import type { ParquetType } from '../../parquetjs/schema/declare';
|
|
3
|
+
import { FileMetaData } from '@loaders.gl/parquet/parquetjs/parquet-thrift';
|
|
4
|
+
import { Schema, DataType } from '@loaders.gl/schema';
|
|
5
|
+
export declare const PARQUET_TYPE_MAPPING: {
|
|
6
|
+
[type in ParquetType]: typeof DataType;
|
|
7
|
+
};
|
|
8
|
+
export declare function convertSchemaFromParquet(parquetSchema: ParquetSchema, parquetMetadata?: FileMetaData): Schema;
|
|
9
|
+
//# sourceMappingURL=convert-schema-from-parquet.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAC,YAAY,EAAC,MAAM,8CAA8C,CAAC;AAE1E,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,aAAa,EAC5B,eAAe,CAAC,EAAE,YAAY,GAC7B,MAAM,CAIR"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
// loaders.gl, MIT license
|
|
2
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
+
exports.convertSchemaFromParquet = exports.PARQUET_TYPE_MAPPING = void 0;
|
|
4
5
|
const schema_1 = require("@loaders.gl/schema");
|
|
5
6
|
exports.PARQUET_TYPE_MAPPING = {
|
|
6
7
|
BOOLEAN: schema_1.Bool,
|
|
@@ -27,36 +28,26 @@ exports.PARQUET_TYPE_MAPPING = {
|
|
|
27
28
|
INT_64: schema_1.Int64,
|
|
28
29
|
JSON: schema_1.Binary,
|
|
29
30
|
BSON: schema_1.Binary,
|
|
30
|
-
// TODO check
|
|
31
|
+
// TODO check interval type
|
|
31
32
|
INTERVAL: schema_1.Binary,
|
|
32
33
|
DECIMAL_INT32: schema_1.Float32,
|
|
33
34
|
DECIMAL_INT64: schema_1.Float64,
|
|
34
35
|
DECIMAL_BYTE_ARRAY: schema_1.Float64,
|
|
35
36
|
DECIMAL_FIXED_LEN_BYTE_ARRAY: schema_1.Float64
|
|
36
37
|
};
|
|
37
|
-
function
|
|
38
|
+
function convertSchemaFromParquet(parquetSchema, parquetMetadata) {
|
|
38
39
|
const fields = getFields(parquetSchema.schema);
|
|
39
|
-
|
|
40
|
-
return new schema_1.Schema(fields);
|
|
41
|
-
}
|
|
42
|
-
exports.convertParquetToArrowSchema = convertParquetToArrowSchema;
|
|
43
|
-
function getFieldMetadata(field) {
|
|
44
|
-
const metadata = new Map();
|
|
45
|
-
for (const key in field) {
|
|
46
|
-
if (key !== 'name') {
|
|
47
|
-
const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
48
|
-
metadata.set(key, value);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
return metadata;
|
|
40
|
+
const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
|
|
41
|
+
return new schema_1.Schema(fields, metadata);
|
|
52
42
|
}
|
|
43
|
+
exports.convertSchemaFromParquet = convertSchemaFromParquet;
|
|
53
44
|
function getFields(schema) {
|
|
54
45
|
const fields = [];
|
|
55
46
|
for (const name in schema) {
|
|
56
47
|
const field = schema[name];
|
|
57
48
|
if (field.fields) {
|
|
58
|
-
const
|
|
59
|
-
const nestedField = new schema_1.Field(name, new schema_1.Struct(
|
|
49
|
+
const childFields = getFields(field.fields);
|
|
50
|
+
const nestedField = new schema_1.Field(name, new schema_1.Struct(childFields), field.optional);
|
|
60
51
|
fields.push(nestedField);
|
|
61
52
|
}
|
|
62
53
|
else {
|
|
@@ -68,3 +59,24 @@ function getFields(schema) {
|
|
|
68
59
|
}
|
|
69
60
|
return fields;
|
|
70
61
|
}
|
|
62
|
+
function getFieldMetadata(field) {
|
|
63
|
+
const metadata = new Map();
|
|
64
|
+
for (const key in field) {
|
|
65
|
+
if (key !== 'name') {
|
|
66
|
+
let value = field[key] || '';
|
|
67
|
+
value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
68
|
+
metadata.set(key, value);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return metadata;
|
|
72
|
+
}
|
|
73
|
+
function getSchemaMetadata(parquetMetadata) {
|
|
74
|
+
const metadata = new Map();
|
|
75
|
+
const keyValueList = parquetMetadata.key_value_metadata || [];
|
|
76
|
+
for (const { key, value } of keyValueList) {
|
|
77
|
+
if (typeof value === 'string') {
|
|
78
|
+
metadata.set(key, value);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return metadata;
|
|
82
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ParquetType } from '../../parquetjs/schema/declare';
|
|
2
|
+
import { Schema, DataType } from '@loaders.gl/schema';
|
|
3
|
+
export declare const PARQUET_TYPE_MAPPING: {
|
|
4
|
+
[type in ParquetType]: typeof DataType;
|
|
5
|
+
};
|
|
6
|
+
export declare function convertToParquetSchema(schema: Schema): Schema;
|
|
7
|
+
//# sourceMappingURL=convert-schema-to-parquet.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// loaders.gl, MIT license
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.convertToParquetSchema = exports.PARQUET_TYPE_MAPPING = void 0;
|
|
5
|
+
const schema_1 = require("@loaders.gl/schema");
|
|
6
|
+
exports.PARQUET_TYPE_MAPPING = {
|
|
7
|
+
BOOLEAN: schema_1.Bool,
|
|
8
|
+
INT32: schema_1.Int32,
|
|
9
|
+
INT64: schema_1.Float64,
|
|
10
|
+
INT96: schema_1.Float64,
|
|
11
|
+
FLOAT: schema_1.Float32,
|
|
12
|
+
DOUBLE: schema_1.Float64,
|
|
13
|
+
BYTE_ARRAY: schema_1.Binary,
|
|
14
|
+
FIXED_LEN_BYTE_ARRAY: schema_1.Binary,
|
|
15
|
+
UTF8: schema_1.Utf8,
|
|
16
|
+
DATE: schema_1.Int32,
|
|
17
|
+
TIME_MILLIS: schema_1.Int64,
|
|
18
|
+
TIME_MICROS: schema_1.Int64,
|
|
19
|
+
TIMESTAMP_MILLIS: schema_1.Int64,
|
|
20
|
+
TIMESTAMP_MICROS: schema_1.Int64,
|
|
21
|
+
UINT_8: schema_1.Int32,
|
|
22
|
+
UINT_16: schema_1.Uint16,
|
|
23
|
+
UINT_32: schema_1.Uint32,
|
|
24
|
+
UINT_64: schema_1.Uint64,
|
|
25
|
+
INT_8: schema_1.Int8,
|
|
26
|
+
INT_16: schema_1.Int16,
|
|
27
|
+
INT_32: schema_1.Int32,
|
|
28
|
+
INT_64: schema_1.Int64,
|
|
29
|
+
JSON: schema_1.Binary,
|
|
30
|
+
BSON: schema_1.Binary,
|
|
31
|
+
// TODO check interval type
|
|
32
|
+
INTERVAL: schema_1.Binary,
|
|
33
|
+
DECIMAL_INT32: schema_1.Float32,
|
|
34
|
+
DECIMAL_INT64: schema_1.Float64,
|
|
35
|
+
DECIMAL_BYTE_ARRAY: schema_1.Float64,
|
|
36
|
+
DECIMAL_FIXED_LEN_BYTE_ARRAY: schema_1.Float64
|
|
37
|
+
};
|
|
38
|
+
function convertToParquetSchema(schema) {
|
|
39
|
+
const fields = []; // getFields(schema.fields);
|
|
40
|
+
// TODO add metadata if needed.
|
|
41
|
+
return new schema_1.Schema(fields);
|
|
42
|
+
}
|
|
43
|
+
exports.convertToParquetSchema = convertToParquetSchema;
|
|
44
|
+
// function getFields(schema: Field[]): Definition[] {
|
|
45
|
+
// const fields: Field[] = [];
|
|
46
|
+
// for (const name in schema) {
|
|
47
|
+
// const field = schema[name];
|
|
48
|
+
// // @ts-ignore
|
|
49
|
+
// const children = field.children as DataType[];
|
|
50
|
+
// if (children) {
|
|
51
|
+
// const childField = getFields(field.fields);
|
|
52
|
+
// const nestedField = new Field(name, new Struct(childField), field.optional);
|
|
53
|
+
// fields.push(nestedField);
|
|
54
|
+
// } else {
|
|
55
|
+
// const FieldType = PARQUET_TYPE_MAPPING[field.type];
|
|
56
|
+
// const metadata = getFieldMetadata(field);
|
|
57
|
+
// const arrowField = new Field(name, new FieldType(), field.optional, metadata);
|
|
58
|
+
// fields.push(arrowField);
|
|
59
|
+
// }
|
|
60
|
+
// }
|
|
61
|
+
// return fields;
|
|
62
|
+
// }
|
|
63
|
+
// function getFieldMetadata(field: ParquetField): Map<string, string> {
|
|
64
|
+
// const metadata = new Map();
|
|
65
|
+
// for (const key in field) {
|
|
66
|
+
// if (key !== 'name') {
|
|
67
|
+
// const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
|
|
68
|
+
// metadata.set(key, value);
|
|
69
|
+
// }
|
|
70
|
+
// }
|
|
71
|
+
// return metadata;
|
|
72
|
+
// }
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Schema } from '@loaders.gl/schema';
|
|
2
|
+
/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */
|
|
3
|
+
export type GeoMetadata = {
|
|
4
|
+
version?: string;
|
|
5
|
+
primary_column?: string;
|
|
6
|
+
columns: Record<string, GeoColumnMetadata>;
|
|
7
|
+
[key: string]: unknown;
|
|
8
|
+
};
|
|
9
|
+
/** A geoarrow / geoparquet geo metadata for one geometry column */
|
|
10
|
+
export type GeoColumnMetadata = {
|
|
11
|
+
bounding_box?: [number, number, number, number] | [number, number, number, number, number, number];
|
|
12
|
+
crs?: string;
|
|
13
|
+
geometry_type?: string[];
|
|
14
|
+
edges?: string;
|
|
15
|
+
[key: string]: unknown;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Reads the GeoMetadata object from the metadata
|
|
19
|
+
* @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
|
|
20
|
+
export declare function getGeoMetadata(schema: Schema): GeoMetadata | null;
|
|
21
|
+
/**
|
|
22
|
+
* Stores a geoarrow / geoparquet geo metadata object in the schema
|
|
23
|
+
* @note geoarrow / geoparquet geo metadata is a single stringified JSON field
|
|
24
|
+
*/
|
|
25
|
+
export declare function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void;
|
|
26
|
+
/**
|
|
27
|
+
* Unpacks geo metadata into separate metadata fields (parses the long JSON string)
|
|
28
|
+
* @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
|
|
29
|
+
*/
|
|
30
|
+
export declare function unpackGeoMetadata(schema: Schema): void;
|
|
31
|
+
//# sourceMappingURL=decode-geo-metadata.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"decode-geo-metadata.d.ts","sourceRoot":"","sources":["../../../src/lib/geo/decode-geo-metadata.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAQ,MAAM,oBAAoB,CAAC;AAIjD,mHAAmH;AACnH,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAC3C,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,YAAY,CAAC,EACT,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAChC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AAEF;;yGAEyG;AACzG,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAYjE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,GAAG,IAAI,CAG7E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CA6BtD"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Reads the GeoMetadata object from the metadata
|
|
6
|
+
* @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */
|
|
7
|
+
function getGeoMetadata(schema) {
|
|
8
|
+
const stringifiedGeoMetadata = schema.metadata.get('geo');
|
|
9
|
+
if (!stringifiedGeoMetadata) {
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
try {
|
|
13
|
+
const geoMetadata = JSON.parse(stringifiedGeoMetadata);
|
|
14
|
+
return geoMetadata;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
exports.getGeoMetadata = getGeoMetadata;
|
|
21
|
+
/**
|
|
22
|
+
* Stores a geoarrow / geoparquet geo metadata object in the schema
|
|
23
|
+
* @note geoarrow / geoparquet geo metadata is a single stringified JSON field
|
|
24
|
+
*/
|
|
25
|
+
function setGeoMetadata(schema, geoMetadata) {
|
|
26
|
+
const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
|
|
27
|
+
schema.metadata.set('geo', stringifiedGeoMetadata);
|
|
28
|
+
}
|
|
29
|
+
exports.setGeoMetadata = setGeoMetadata;
|
|
30
|
+
/**
|
|
31
|
+
* Unpacks geo metadata into separate metadata fields (parses the long JSON string)
|
|
32
|
+
* @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata
|
|
33
|
+
*/
|
|
34
|
+
function unpackGeoMetadata(schema) {
|
|
35
|
+
const geoMetadata = getGeoMetadata(schema);
|
|
36
|
+
if (!geoMetadata) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
// Store Parquet Schema Level Metadata
|
|
40
|
+
const { version, primary_column, columns } = geoMetadata;
|
|
41
|
+
if (version) {
|
|
42
|
+
schema.metadata.set('geo.version', version);
|
|
43
|
+
}
|
|
44
|
+
if (primary_column) {
|
|
45
|
+
schema.metadata.set('geo.primary_column', primary_column);
|
|
46
|
+
}
|
|
47
|
+
// store column names as comma separated list
|
|
48
|
+
schema.metadata.set('geo.columns', Object.keys(columns || {}).join(''));
|
|
49
|
+
for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
|
|
50
|
+
const field = schema.fields.find((field) => field.name === columnName);
|
|
51
|
+
if (field) {
|
|
52
|
+
if (field.name === primary_column) {
|
|
53
|
+
field.metadata.set('geo.primary_field', 'true');
|
|
54
|
+
}
|
|
55
|
+
unpackGeoFieldMetadata(field, columnMetadata);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
exports.unpackGeoMetadata = unpackGeoMetadata;
|
|
60
|
+
function unpackGeoFieldMetadata(field, columnMetadata) {
|
|
61
|
+
for (const [key, value] of Object.entries(columnMetadata || {})) {
|
|
62
|
+
switch (key) {
|
|
63
|
+
case 'geometry_type':
|
|
64
|
+
field.metadata.set(`geo.${key}`, value.join(','));
|
|
65
|
+
break;
|
|
66
|
+
case 'bbox':
|
|
67
|
+
case 'crs':
|
|
68
|
+
case 'edges':
|
|
69
|
+
default:
|
|
70
|
+
field.metadata.set(`geo.${key}`, typeof value === 'string' ? value : JSON.stringify(value));
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Geoparquet JSON schema for geo metadata
|
|
3
|
+
* @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md
|
|
4
|
+
* @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
|
|
5
|
+
*/
|
|
6
|
+
declare const _default: {
|
|
7
|
+
$schema: string;
|
|
8
|
+
title: string;
|
|
9
|
+
description: string;
|
|
10
|
+
type: string;
|
|
11
|
+
required: string[];
|
|
12
|
+
properties: {
|
|
13
|
+
version: {
|
|
14
|
+
type: string;
|
|
15
|
+
const: string;
|
|
16
|
+
};
|
|
17
|
+
primary_column: {
|
|
18
|
+
type: string;
|
|
19
|
+
minLength: number;
|
|
20
|
+
};
|
|
21
|
+
columns: {
|
|
22
|
+
type: string;
|
|
23
|
+
minProperties: number;
|
|
24
|
+
patternProperties: {
|
|
25
|
+
'.+': {
|
|
26
|
+
type: string;
|
|
27
|
+
required: string[];
|
|
28
|
+
properties: {
|
|
29
|
+
encoding: {
|
|
30
|
+
type: string;
|
|
31
|
+
const: string;
|
|
32
|
+
};
|
|
33
|
+
geometry_types: {
|
|
34
|
+
type: string;
|
|
35
|
+
uniqueItems: boolean;
|
|
36
|
+
items: {
|
|
37
|
+
type: string;
|
|
38
|
+
pattern: string;
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
crs: {
|
|
42
|
+
oneOf: ({
|
|
43
|
+
$ref: string;
|
|
44
|
+
type?: undefined;
|
|
45
|
+
} | {
|
|
46
|
+
type: string;
|
|
47
|
+
$ref?: undefined;
|
|
48
|
+
})[];
|
|
49
|
+
};
|
|
50
|
+
edges: {
|
|
51
|
+
type: string;
|
|
52
|
+
enum: string[];
|
|
53
|
+
};
|
|
54
|
+
orientation: {
|
|
55
|
+
type: string;
|
|
56
|
+
const: string;
|
|
57
|
+
};
|
|
58
|
+
bbox: {
|
|
59
|
+
type: string;
|
|
60
|
+
items: {
|
|
61
|
+
type: string;
|
|
62
|
+
};
|
|
63
|
+
oneOf: {
|
|
64
|
+
description: string;
|
|
65
|
+
minItems: number;
|
|
66
|
+
maxItems: number;
|
|
67
|
+
}[];
|
|
68
|
+
};
|
|
69
|
+
epoch: {
|
|
70
|
+
type: string;
|
|
71
|
+
};
|
|
72
|
+
};
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
additionalProperties: boolean;
|
|
76
|
+
};
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
export default _default;
|
|
80
|
+
//# sourceMappingURL=geoparquet-schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"geoparquet-schema.d.ts","sourceRoot":"","sources":["../../../src/lib/geo/geoparquet-schema.ts"],"names":[],"mappings":"AAIA;;;;GAIG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACH,wBA2DE"}
|