@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (267) hide show
  1. package/dist/bundled.d.ts +6 -0
  2. package/dist/bundled.d.ts.map +1 -0
  3. package/dist/bundled.js +7 -0
  4. package/dist/bundled.js.map +1 -0
  5. package/dist/geoparquet-loader-with-parser.d.ts +34 -0
  6. package/dist/geoparquet-loader-with-parser.d.ts.map +1 -0
  7. package/dist/geoparquet-loader-with-parser.js +52 -0
  8. package/dist/geoparquet-loader-with-parser.js.map +1 -0
  9. package/dist/geoparquet-loader.d.ts +61 -0
  10. package/dist/geoparquet-loader.d.ts.map +1 -0
  11. package/dist/geoparquet-loader.js +30 -0
  12. package/dist/geoparquet-loader.js.map +1 -0
  13. package/dist/index.cjs +5879 -5643
  14. package/dist/index.cjs.map +4 -4
  15. package/dist/index.d.ts +7 -7
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +5 -11
  18. package/dist/index.js.map +1 -1
  19. package/dist/lib/constants.d.ts +1 -1
  20. package/dist/lib/constants.js +2 -2
  21. package/dist/lib/constants.js.map +1 -1
  22. package/dist/lib/encoders/encode-arrow-to-parquet.d.ts +2 -2
  23. package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
  24. package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -10
  25. package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -1
  26. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts +5 -0
  27. package/dist/lib/encoders/encode-table-to-parquet-js.d.ts.map +1 -0
  28. package/dist/lib/encoders/encode-table-to-parquet-js.js +149 -0
  29. package/dist/lib/encoders/encode-table-to-parquet-js.js.map +1 -0
  30. package/dist/lib/geo/geospatial-metadata.d.ts +67 -0
  31. package/dist/lib/geo/geospatial-metadata.d.ts.map +1 -0
  32. package/dist/lib/geo/geospatial-metadata.js +357 -0
  33. package/dist/lib/geo/geospatial-metadata.js.map +1 -0
  34. package/dist/lib/parsers/get-parquet-schema.d.ts.map +1 -1
  35. package/dist/lib/parsers/get-parquet-schema.js +3 -1
  36. package/dist/lib/parsers/get-parquet-schema.js.map +1 -1
  37. package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts +4 -4
  38. package/dist/lib/parsers/parse-geoparquet-to-geojson.d.ts.map +1 -1
  39. package/dist/lib/parsers/parse-geoparquet-to-geojson.js +19 -6
  40. package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -1
  41. package/dist/lib/parsers/parse-parquet-tables.d.ts +50 -0
  42. package/dist/lib/parsers/parse-parquet-tables.d.ts.map +1 -0
  43. package/dist/lib/parsers/parse-parquet-tables.js +101 -0
  44. package/dist/lib/parsers/parse-parquet-tables.js.map +1 -0
  45. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts +6 -0
  46. package/dist/lib/parsers/parse-parquet-to-arrow-js.d.ts.map +1 -0
  47. package/dist/lib/parsers/parse-parquet-to-arrow-js.js +23 -0
  48. package/dist/lib/parsers/parse-parquet-to-arrow-js.js.map +1 -0
  49. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts +7 -7
  50. package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map +1 -1
  51. package/dist/lib/parsers/parse-parquet-to-arrow.js +45 -32
  52. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -1
  53. package/dist/lib/parsers/parse-parquet-to-columns.d.ts +3 -3
  54. package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map +1 -1
  55. package/dist/lib/parsers/parse-parquet-to-columns.js +1 -4
  56. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -1
  57. package/dist/lib/parsers/parse-parquet-to-json.d.ts +3 -3
  58. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  59. package/dist/lib/parsers/parse-parquet-to-json.js +77 -26
  60. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -1
  61. package/dist/lib/utils/load-wasm.d.ts +2 -2
  62. package/dist/lib/utils/load-wasm.d.ts.map +1 -1
  63. package/dist/lib/utils/load-wasm.js +7 -7
  64. package/dist/lib/utils/load-wasm.js.map +1 -1
  65. package/dist/lib/utils/make-stream-iterator.js +1 -1
  66. package/dist/lib/utils/make-stream-iterator.js.map +1 -1
  67. package/dist/lib/utils/normalize-parquet-options.d.ts +5 -0
  68. package/dist/lib/utils/normalize-parquet-options.d.ts.map +1 -0
  69. package/dist/lib/utils/normalize-parquet-options.js +11 -0
  70. package/dist/lib/utils/normalize-parquet-options.js.map +1 -0
  71. package/dist/parquet-format.d.ts +2 -0
  72. package/dist/parquet-format.d.ts.map +1 -1
  73. package/dist/parquet-format.js +2 -0
  74. package/dist/parquet-format.js.map +1 -1
  75. package/dist/parquet-js-loader.d.ts +57 -0
  76. package/dist/parquet-js-loader.d.ts.map +1 -0
  77. package/dist/parquet-js-loader.js +33 -0
  78. package/dist/parquet-js-loader.js.map +1 -0
  79. package/dist/parquet-js-writer.d.ts +34 -0
  80. package/dist/parquet-js-writer.d.ts.map +1 -0
  81. package/dist/parquet-js-writer.js +28 -0
  82. package/dist/parquet-js-writer.js.map +1 -0
  83. package/dist/parquet-loader-options.d.ts +56 -0
  84. package/dist/parquet-loader-options.d.ts.map +1 -0
  85. package/dist/parquet-loader-options.js +11 -0
  86. package/dist/parquet-loader-options.js.map +1 -0
  87. package/dist/parquet-loader-with-parser.d.ts +33 -0
  88. package/dist/parquet-loader-with-parser.d.ts.map +1 -0
  89. package/dist/parquet-loader-with-parser.js +40 -0
  90. package/dist/parquet-loader-with-parser.js.map +1 -0
  91. package/dist/parquet-loader.d.ts +62 -0
  92. package/dist/parquet-loader.d.ts.map +1 -0
  93. package/dist/parquet-loader.js +53 -0
  94. package/dist/parquet-loader.js.map +1 -0
  95. package/dist/parquet-wasm-loader-with-parser.d.ts +40 -0
  96. package/dist/parquet-wasm-loader-with-parser.d.ts.map +1 -0
  97. package/dist/parquet-wasm-loader-with-parser.js +68 -0
  98. package/dist/parquet-wasm-loader-with-parser.js.map +1 -0
  99. package/dist/parquet-writer.d.ts +10 -0
  100. package/dist/parquet-writer.d.ts.map +1 -0
  101. package/dist/parquet-writer.js +29 -0
  102. package/dist/parquet-writer.js.map +1 -0
  103. package/dist/parquet_wasm_bg.wasm +0 -0
  104. package/dist/parquetjs/codecs/declare.d.ts +2 -2
  105. package/dist/parquetjs/codecs/declare.d.ts.map +1 -1
  106. package/dist/parquetjs/codecs/dictionary.js +1 -1
  107. package/dist/parquetjs/codecs/dictionary.js.map +1 -1
  108. package/dist/parquetjs/codecs/plain.d.ts +1 -1
  109. package/dist/parquetjs/codecs/plain.d.ts.map +1 -1
  110. package/dist/parquetjs/codecs/plain.js +36 -41
  111. package/dist/parquetjs/codecs/plain.js.map +1 -1
  112. package/dist/parquetjs/codecs/rle.d.ts +1 -1
  113. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  114. package/dist/parquetjs/codecs/rle.js +16 -21
  115. package/dist/parquetjs/codecs/rle.js.map +1 -1
  116. package/dist/parquetjs/compression.d.ts +2 -3
  117. package/dist/parquetjs/compression.d.ts.map +1 -1
  118. package/dist/parquetjs/compression.js +4 -105
  119. package/dist/parquetjs/compression.js.map +1 -1
  120. package/dist/parquetjs/encoder/parquet-encoder.d.ts +4 -4
  121. package/dist/parquetjs/encoder/parquet-encoder.d.ts.map +1 -1
  122. package/dist/parquetjs/encoder/parquet-encoder.js +38 -33
  123. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -1
  124. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +1 -1
  125. package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -1
  126. package/dist/parquetjs/parquet-thrift/index.d.ts +0 -1
  127. package/dist/parquetjs/parquet-thrift/index.d.ts.map +1 -1
  128. package/dist/parquetjs/parquet-thrift/index.js +0 -1
  129. package/dist/parquetjs/parquet-thrift/index.js.map +1 -1
  130. package/dist/parquetjs/parser/decoders.d.ts +1 -1
  131. package/dist/parquetjs/parser/decoders.d.ts.map +1 -1
  132. package/dist/parquetjs/parser/decoders.js +4 -22
  133. package/dist/parquetjs/parser/decoders.js.map +1 -1
  134. package/dist/parquetjs/parser/parquet-reader.d.ts +1 -1
  135. package/dist/parquetjs/parser/parquet-reader.d.ts.map +1 -1
  136. package/dist/parquetjs/parser/parquet-reader.js +9 -9
  137. package/dist/parquetjs/parser/parquet-reader.js.map +1 -1
  138. package/dist/parquetjs/schema/declare.d.ts +1 -1
  139. package/dist/parquetjs/schema/declare.d.ts.map +1 -1
  140. package/dist/parquetjs/schema/types.d.ts.map +1 -1
  141. package/dist/parquetjs/schema/types.js +36 -35
  142. package/dist/parquetjs/schema/types.js.map +1 -1
  143. package/dist/parquetjs/utils/binary-utils.d.ts +39 -0
  144. package/dist/parquetjs/utils/binary-utils.d.ts.map +1 -0
  145. package/dist/parquetjs/utils/binary-utils.js +102 -0
  146. package/dist/parquetjs/utils/binary-utils.js.map +1 -0
  147. package/dist/parquetjs/utils/file-utils.d.ts +1 -1
  148. package/dist/parquetjs/utils/file-utils.d.ts.map +1 -1
  149. package/dist/parquetjs/utils/file-utils.js +3 -3
  150. package/dist/parquetjs/utils/file-utils.js.map +1 -1
  151. package/dist/parquetjs/utils/read-utils.d.ts +4 -4
  152. package/dist/parquetjs/utils/read-utils.d.ts.map +1 -1
  153. package/dist/parquetjs/utils/read-utils.js +12 -17
  154. package/dist/parquetjs/utils/read-utils.js.map +1 -1
  155. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts +78 -0
  156. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.d.ts.map +1 -0
  157. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js +213 -0
  158. package/dist/parquetjs/utils/uint8-array-compact-protocol-writer.js.map +1 -0
  159. package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts +78 -0
  160. package/dist/parquetjs/utils/uint8-array-compact-protocol.d.ts.map +1 -0
  161. package/dist/parquetjs/utils/uint8-array-compact-protocol.js +276 -0
  162. package/dist/parquetjs/utils/uint8-array-compact-protocol.js.map +1 -0
  163. package/dist/parquetjs/utils/uint8-array-transport.d.ts +25 -0
  164. package/dist/parquetjs/utils/uint8-array-transport.d.ts.map +1 -0
  165. package/dist/parquetjs/utils/uint8-array-transport.js +62 -0
  166. package/dist/parquetjs/utils/uint8-array-transport.js.map +1 -0
  167. package/dist/unbundled.d.ts +6 -0
  168. package/dist/unbundled.d.ts.map +1 -0
  169. package/dist/unbundled.js +7 -0
  170. package/dist/unbundled.js.map +1 -0
  171. package/package.json +21 -25
  172. package/src/bundled.ts +9 -0
  173. package/src/geoparquet-loader-with-parser.ts +95 -0
  174. package/src/geoparquet-loader.ts +54 -0
  175. package/src/index.ts +7 -25
  176. package/src/lib/constants.ts +1 -1
  177. package/src/lib/encoders/encode-arrow-to-parquet.ts +9 -15
  178. package/src/lib/encoders/encode-table-to-parquet-js.ts +199 -0
  179. package/src/lib/geo/geospatial-metadata.ts +507 -0
  180. package/src/lib/parsers/get-parquet-schema.ts +3 -1
  181. package/src/lib/parsers/parse-geoparquet-to-geojson.ts +27 -14
  182. package/src/lib/parsers/parse-parquet-tables.ts +137 -0
  183. package/src/lib/parsers/parse-parquet-to-arrow-js.ts +36 -0
  184. package/src/lib/parsers/parse-parquet-to-arrow.ts +63 -36
  185. package/src/lib/parsers/parse-parquet-to-columns.ts +6 -7
  186. package/src/lib/parsers/parse-parquet-to-json.ts +101 -33
  187. package/src/lib/utils/load-wasm.ts +16 -9
  188. package/src/lib/utils/make-stream-iterator.ts +1 -1
  189. package/src/lib/utils/normalize-parquet-options.ts +14 -0
  190. package/src/parquet-format.ts +2 -0
  191. package/src/parquet-js-loader.ts +43 -0
  192. package/src/parquet-js-writer.ts +52 -0
  193. package/src/parquet-loader-options.ts +63 -0
  194. package/src/parquet-loader-with-parser.ts +68 -0
  195. package/src/parquet-loader.ts +79 -0
  196. package/src/parquet-wasm-loader-with-parser.ts +115 -0
  197. package/src/parquet-writer.ts +44 -0
  198. package/src/parquetjs/codecs/declare.ts +2 -2
  199. package/src/parquetjs/codecs/dictionary.ts +1 -1
  200. package/src/parquetjs/codecs/plain.ts +67 -53
  201. package/src/parquetjs/codecs/rle.ts +20 -26
  202. package/src/parquetjs/compression.ts +7 -114
  203. package/src/parquetjs/encoder/parquet-encoder.ts +49 -43
  204. package/src/parquetjs/parquet-thrift/ColumnIndex.ts +1 -1
  205. package/src/parquetjs/parquet-thrift/index.ts +0 -2
  206. package/src/parquetjs/parser/decoders.ts +5 -23
  207. package/src/parquetjs/parser/parquet-reader.ts +10 -12
  208. package/src/parquetjs/schema/declare.ts +1 -1
  209. package/src/parquetjs/schema/types.ts +48 -40
  210. package/src/parquetjs/utils/binary-utils.ts +127 -0
  211. package/src/parquetjs/utils/file-utils.ts +4 -4
  212. package/src/parquetjs/utils/read-utils.ts +19 -34
  213. package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts +254 -0
  214. package/src/parquetjs/utils/uint8-array-compact-protocol.ts +322 -0
  215. package/src/parquetjs/utils/uint8-array-transport.ts +79 -0
  216. package/src/unbundled.ts +9 -0
  217. package/dist/parquet-arrow-loader.d.ts +0 -43
  218. package/dist/parquet-arrow-loader.d.ts.map +0 -1
  219. package/dist/parquet-arrow-loader.js +0 -49
  220. package/dist/parquet-arrow-loader.js.map +0 -1
  221. package/dist/parquet-arrow-writer.d.ts +0 -9
  222. package/dist/parquet-arrow-writer.d.ts.map +0 -1
  223. package/dist/parquet-arrow-writer.js +0 -23
  224. package/dist/parquet-arrow-writer.js.map +0 -1
  225. package/dist/parquet-json-loader.d.ts +0 -148
  226. package/dist/parquet-json-loader.d.ts.map +0 -1
  227. package/dist/parquet-json-loader.js +0 -106
  228. package/dist/parquet-json-loader.js.map +0 -1
  229. package/dist/parquet-json-writer.d.ts +0 -19
  230. package/dist/parquet-json-writer.d.ts.map +0 -1
  231. package/dist/parquet-json-writer.js +0 -20
  232. package/dist/parquet-json-writer.js.map +0 -1
  233. package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts +0 -3
  234. package/dist/polyfills/buffer/buffer-polyfill.browser.d.ts.map +0 -1
  235. package/dist/polyfills/buffer/buffer-polyfill.browser.js +0 -12
  236. package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +0 -1
  237. package/dist/polyfills/buffer/buffer-polyfill.node.d.ts +0 -3
  238. package/dist/polyfills/buffer/buffer-polyfill.node.d.ts.map +0 -1
  239. package/dist/polyfills/buffer/buffer-polyfill.node.js +0 -17
  240. package/dist/polyfills/buffer/buffer-polyfill.node.js.map +0 -1
  241. package/dist/polyfills/buffer/buffer.d.ts +0 -222
  242. package/dist/polyfills/buffer/buffer.d.ts.map +0 -1
  243. package/dist/polyfills/buffer/buffer.js +0 -1868
  244. package/dist/polyfills/buffer/buffer.js.map +0 -1
  245. package/dist/polyfills/buffer/index.d.ts +0 -4
  246. package/dist/polyfills/buffer/index.d.ts.map +0 -1
  247. package/dist/polyfills/buffer/index.js +0 -9
  248. package/dist/polyfills/buffer/index.js.map +0 -1
  249. package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +0 -2
  250. package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +0 -1
  251. package/dist/polyfills/buffer/install-buffer-polyfill.js +0 -10
  252. package/dist/polyfills/buffer/install-buffer-polyfill.js.map +0 -1
  253. package/dist/workers/parquet-json-worker.d.ts +0 -2
  254. package/dist/workers/parquet-json-worker.d.ts.map +0 -1
  255. package/dist/workers/parquet-json-worker.js +0 -7
  256. package/dist/workers/parquet-json-worker.js.map +0 -1
  257. package/src/parquet-arrow-loader.ts +0 -74
  258. package/src/parquet-arrow-writer.ts +0 -33
  259. package/src/parquet-json-loader.ts +0 -160
  260. package/src/parquet-json-writer.ts +0 -27
  261. package/src/parquetjs/modules.d.ts +0 -25
  262. package/src/polyfills/buffer/buffer-polyfill.browser.ts +0 -14
  263. package/src/polyfills/buffer/buffer-polyfill.node.ts +0 -19
  264. package/src/polyfills/buffer/buffer.ts +0 -2205
  265. package/src/polyfills/buffer/index.ts +0 -11
  266. package/src/polyfills/buffer/install-buffer-polyfill.ts +0 -12
  267. package/src/workers/parquet-json-worker.ts +0 -8
@@ -0,0 +1,199 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {DataType, Field, ObjectRowTable, Schema, Table} from '@loaders.gl/schema';
6
+ import {concatUint8Arrays} from '../../parquetjs/utils/binary-utils';
7
+ import {ParquetEncoder} from '../../parquetjs/encoder/parquet-encoder';
8
+ import {ParquetSchema} from '../../parquetjs/schema/schema';
9
+ import type {FieldDefinition, SchemaDefinition} from '../../parquetjs/schema/declare';
10
+ import type {ParquetJSWriterOptions} from '../../parquet-js-writer';
11
+
12
+ /** Encode a plain loaders.gl table with the parquetjs implementation. */
13
+ export async function encodeTableToParquetJs(
14
+ table: Table,
15
+ objectRowTable: ObjectRowTable,
16
+ options: ParquetJSWriterOptions
17
+ ): Promise<ArrayBuffer> {
18
+ const parquetSchema = new ParquetSchema(
19
+ convertSchemaToParquetSchema(table.schema!, objectRowTable)
20
+ );
21
+ const chunks: Uint8Array[] = [];
22
+ const outputStream = {
23
+ write(chunk: Uint8Array, callback: (error?: Error) => void) {
24
+ chunks.push(chunk.slice());
25
+ callback();
26
+ },
27
+ close(callback: (error?: Error) => void) {
28
+ callback();
29
+ }
30
+ };
31
+
32
+ const encoder = await ParquetEncoder.openStream(
33
+ parquetSchema,
34
+ outputStream as any,
35
+ options.parquet
36
+ );
37
+
38
+ for (const [key, value] of Object.entries(table.schema?.metadata || {})) {
39
+ encoder.setMetadata(key, value);
40
+ }
41
+
42
+ for (const row of objectRowTable.data) {
43
+ await encoder.appendRow(row);
44
+ }
45
+
46
+ await encoder.close();
47
+
48
+ return concatUint8Arrays(chunks).slice().buffer;
49
+ }
50
+
51
+ /**
52
+ * Converts a loaders.gl schema into a parquetjs schema definition.
53
+ * @param schema table schema
54
+ * @param objectRowTable table data in object-row form
55
+ * @returns parquetjs schema definition
56
+ */
57
+ function convertSchemaToParquetSchema(
58
+ schema: Schema,
59
+ objectRowTable: ObjectRowTable
60
+ ): SchemaDefinition {
61
+ const parquetFields: SchemaDefinition = {};
62
+
63
+ for (const field of schema.fields) {
64
+ parquetFields[field.name] = convertFieldToParquetFieldDefinition(field, objectRowTable.data);
65
+ }
66
+
67
+ return parquetFields;
68
+ }
69
+
70
+ /**
71
+ * Converts one loaders.gl field to the closest parquetjs field definition.
72
+ * @param field table field metadata
73
+ * @param rows object rows used for nullability and fallback inference
74
+ * @returns parquetjs field definition
75
+ */
76
+ function convertFieldToParquetFieldDefinition(
77
+ field: Field,
78
+ rows: Array<Record<string, unknown>>
79
+ ): FieldDefinition {
80
+ const sampleValue = getFirstDefinedValue(rows, field.name);
81
+ const nullable = field.nullable ?? sampleValue === undefined;
82
+ const dataType = field.type === 'null' ? getDataTypeFromValue(sampleValue) : field.type;
83
+
84
+ switch (dataType) {
85
+ case 'bool':
86
+ return {type: 'BOOLEAN', optional: nullable};
87
+
88
+ case 'int':
89
+ case 'int8':
90
+ case 'int16':
91
+ case 'int32':
92
+ case 'uint8':
93
+ case 'uint16':
94
+ case 'uint32':
95
+ return {type: 'INT32', optional: nullable};
96
+
97
+ case 'int64':
98
+ case 'uint64':
99
+ return {type: 'INT64', optional: nullable};
100
+
101
+ case 'float':
102
+ case 'float16':
103
+ case 'float32':
104
+ return {type: 'FLOAT', optional: nullable};
105
+
106
+ case 'float64':
107
+ return {type: 'DOUBLE', optional: nullable};
108
+
109
+ case 'utf8':
110
+ return {type: 'UTF8', optional: nullable};
111
+
112
+ case 'binary':
113
+ return {type: 'BYTE_ARRAY', optional: nullable};
114
+
115
+ case 'date-day':
116
+ return {type: 'DATE', optional: nullable};
117
+
118
+ case 'date-millisecond':
119
+ case 'timestamp-second':
120
+ case 'timestamp-millisecond':
121
+ return {type: 'TIMESTAMP_MILLIS', optional: nullable};
122
+
123
+ case 'timestamp-microsecond':
124
+ case 'timestamp-nanosecond':
125
+ return {type: 'TIMESTAMP_MICROS', optional: nullable};
126
+
127
+ case 'time-second':
128
+ case 'time-millisecond':
129
+ return {type: 'TIME_MILLIS', optional: nullable};
130
+
131
+ case 'time-microsecond':
132
+ case 'time-nanosecond':
133
+ return {type: 'TIME_MICROS', optional: nullable};
134
+
135
+ default:
136
+ if (typeof dataType === 'object' && dataType.type === 'fixed-size-binary') {
137
+ return {
138
+ type: 'FIXED_LEN_BYTE_ARRAY',
139
+ typeLength: dataType.byteWidth,
140
+ optional: nullable
141
+ };
142
+ }
143
+
144
+ throw new Error(
145
+ `ParquetJSWriter: Unsupported field "${field.name}" with type ${formatDataType(dataType)}`
146
+ );
147
+ }
148
+ }
149
+
150
+ /**
151
+ * Finds the first non-nullish value in a column.
152
+ * @param rows table rows
153
+ * @param columnName column name
154
+ * @returns first defined value, if any
155
+ */
156
+ function getFirstDefinedValue(rows: Array<Record<string, unknown>>, columnName: string): unknown {
157
+ for (const row of rows) {
158
+ const value = row[columnName];
159
+ if (value !== null && value !== undefined) {
160
+ return value;
161
+ }
162
+ }
163
+
164
+ return undefined;
165
+ }
166
+
167
+ /**
168
+ * Infers a loaders.gl data type from a JavaScript value.
169
+ * @param value sample value
170
+ * @returns inferred data type
171
+ */
172
+ function getDataTypeFromValue(value: unknown): DataType {
173
+ if (typeof value === 'boolean') {
174
+ return 'bool';
175
+ }
176
+ if (typeof value === 'number') {
177
+ return Number.isInteger(value) ? 'int32' : 'float64';
178
+ }
179
+ if (typeof value === 'string') {
180
+ return 'utf8';
181
+ }
182
+ if (value instanceof Date) {
183
+ return 'timestamp-millisecond';
184
+ }
185
+ if (value instanceof Uint8Array) {
186
+ return 'binary';
187
+ }
188
+
189
+ throw new Error('ParquetJSWriter: Unable to infer a Parquet type from the provided row data');
190
+ }
191
+
192
+ /**
193
+ * Formats a loaders.gl data type for error messages.
194
+ * @param dataType input data type
195
+ * @returns human-readable type string
196
+ */
197
+ function formatDataType(dataType: DataType): string {
198
+ return typeof dataType === 'string' ? dataType : dataType.type;
199
+ }
@@ -0,0 +1,507 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import * as arrow from 'apache-arrow';
6
+
7
+ import type {Field, Schema} from '@loaders.gl/schema';
8
+ import {
9
+ getGeoMetadata,
10
+ getMetadataValue,
11
+ setGeoMetadata,
12
+ setMetadataValue,
13
+ type GeoColumnMetadata,
14
+ type GeoMetadata,
15
+ type GeoParquetGeometryType
16
+ } from '@loaders.gl/gis';
17
+ import {convertArrowToSchema} from '@loaders.gl/schema-utils';
18
+
19
+ const GEOARROW_EXTENSION_NAME_KEY = 'ARROW:extension:name';
20
+ const GEOARROW_EXTENSION_METADATA_KEY = 'ARROW:extension:metadata';
21
+ const GEOPARQUET_VERSION = '1.1.0';
22
+
23
+ type GeoArrowEncoding =
24
+ | 'geoarrow.geometry'
25
+ | 'geoarrow.geometrycollection'
26
+ | 'geoarrow.multipolygon'
27
+ | 'geoarrow.polygon'
28
+ | 'geoarrow.multilinestring'
29
+ | 'geoarrow.linestring'
30
+ | 'geoarrow.multipoint'
31
+ | 'geoarrow.point'
32
+ | 'geoarrow.wkb'
33
+ | 'geoarrow.wkt';
34
+
35
+ type GeoArrowMetadata = {
36
+ encoding?: GeoArrowEncoding;
37
+ crs?: Record<string, unknown>;
38
+ edges?: 'spherical';
39
+ [key: string]: unknown;
40
+ };
41
+
42
+ const GEOARROW_ENCODINGS = [
43
+ 'geoarrow.geometry',
44
+ 'geoarrow.geometrycollection',
45
+ 'geoarrow.multipolygon',
46
+ 'geoarrow.polygon',
47
+ 'geoarrow.multilinestring',
48
+ 'geoarrow.linestring',
49
+ 'geoarrow.multipoint',
50
+ 'geoarrow.point',
51
+ 'geoarrow.wkb',
52
+ 'geoarrow.wkt'
53
+ ] as const satisfies GeoArrowEncoding[];
54
+
55
+ const GEOPARQUET_TO_GEOARROW_ENCODINGS = {
56
+ wkb: 'geoarrow.wkb',
57
+ point: 'geoarrow.point',
58
+ linestring: 'geoarrow.linestring',
59
+ polygon: 'geoarrow.polygon',
60
+ multipoint: 'geoarrow.multipoint',
61
+ multilinestring: 'geoarrow.multilinestring',
62
+ multipolygon: 'geoarrow.multipolygon'
63
+ } as const satisfies Record<string, GeoArrowEncoding>;
64
+
65
+ const GEOARROW_TO_GEOPARQUET_ENCODINGS = {
66
+ 'geoarrow.wkb': 'wkb',
67
+ 'geoarrow.point': 'point',
68
+ 'geoarrow.linestring': 'linestring',
69
+ 'geoarrow.polygon': 'polygon',
70
+ 'geoarrow.multipoint': 'multipoint',
71
+ 'geoarrow.multilinestring': 'multilinestring',
72
+ 'geoarrow.multipolygon': 'multipolygon'
73
+ } as const;
74
+
75
+ const GEOARROW_TO_GEOMETRY_TYPE = {
76
+ 'geoarrow.point': 'Point',
77
+ 'geoarrow.linestring': 'LineString',
78
+ 'geoarrow.polygon': 'Polygon',
79
+ 'geoarrow.multipoint': 'MultiPoint',
80
+ 'geoarrow.multilinestring': 'MultiLineString',
81
+ 'geoarrow.multipolygon': 'MultiPolygon'
82
+ } as const satisfies Record<string, GeoParquetGeometryType>;
83
+
84
+ /** Returns GeoArrow geometry field metadata keyed by column name. */
85
+ function getGeometryColumnsFromSchema(schema: Schema): Record<string, GeoArrowMetadata> {
86
+ const geometryColumns: Record<string, GeoArrowMetadata> = {};
87
+ for (const field of schema.fields || []) {
88
+ const metadata = getGeometryMetadataForField(field.metadata || {});
89
+ if (metadata) {
90
+ geometryColumns[field.name] = metadata;
91
+ }
92
+ }
93
+ return geometryColumns;
94
+ }
95
+
96
+ /** Extracts GeoArrow metadata from one schema field metadata object. */
97
+ function getGeometryMetadataForField(
98
+ fieldMetadata: Record<string, string>
99
+ ): GeoArrowMetadata | null {
100
+ let metadata: GeoArrowMetadata | null = null;
101
+ let geoEncoding = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_NAME_KEY);
102
+ if (geoEncoding) {
103
+ geoEncoding = geoEncoding.toLowerCase();
104
+ if (geoEncoding === 'wkb') {
105
+ geoEncoding = 'geoarrow.wkb';
106
+ }
107
+ if (geoEncoding === 'wkt') {
108
+ geoEncoding = 'geoarrow.wkt';
109
+ }
110
+ if (GEOARROW_ENCODINGS.includes(geoEncoding as GeoArrowEncoding)) {
111
+ metadata ||= {};
112
+ metadata.encoding = geoEncoding as GeoArrowEncoding;
113
+ }
114
+ }
115
+
116
+ const columnMetadata = getMetadataValue(fieldMetadata, GEOARROW_EXTENSION_METADATA_KEY);
117
+ if (columnMetadata) {
118
+ try {
119
+ metadata = {
120
+ ...(metadata || {}),
121
+ ...(JSON.parse(columnMetadata) as GeoArrowMetadata)
122
+ };
123
+ } catch {
124
+ return metadata;
125
+ }
126
+ }
127
+
128
+ return metadata;
129
+ }
130
+
131
+ /**
132
+ * Applies GeoParquet schema metadata to matching geometry fields as GeoArrow field metadata.
133
+ *
134
+ * @param schema - Schema to annotate.
135
+ * @returns The same schema reference.
136
+ */
137
+ export function applyGeoParquetToFieldMetadata(schema: Schema): Schema {
138
+ const geoMetadata = getGeoMetadata(schema.metadata);
139
+ if (!geoMetadata) {
140
+ return schema;
141
+ }
142
+
143
+ for (const field of schema.fields) {
144
+ const columnMetadata = geoMetadata.columns?.[field.name];
145
+ if (!columnMetadata) {
146
+ continue;
147
+ }
148
+
149
+ const fieldMetadata = getGeoArrowMetadataFromGeoParquetField(field, columnMetadata);
150
+ if (!fieldMetadata) {
151
+ continue;
152
+ }
153
+
154
+ field.metadata ||= {};
155
+ setMetadataValue(field.metadata, GEOARROW_EXTENSION_NAME_KEY, fieldMetadata.extensionName);
156
+
157
+ if (fieldMetadata.extensionMetadata) {
158
+ setMetadataValue(
159
+ field.metadata,
160
+ GEOARROW_EXTENSION_METADATA_KEY,
161
+ JSON.stringify(fieldMetadata.extensionMetadata)
162
+ );
163
+ }
164
+ }
165
+
166
+ return schema;
167
+ }
168
+
169
+ /**
170
+ * Creates an Arrow JS table whose schema metadata matches a loaders.gl schema.
171
+ *
172
+ * @param table - Arrow JS table to rewrap.
173
+ * @param schema - loaders.gl schema containing desired field/schema metadata.
174
+ * @returns Table using the updated Arrow JS schema.
175
+ */
176
+ export function applyLoadersSchemaToArrowTable(table: arrow.Table, schema: Schema): arrow.Table {
177
+ const arrowSchema = createArrowJsSchema(table.schema, schema);
178
+ return new arrow.Table(arrowSchema, table.batches);
179
+ }
180
+
181
+ /**
182
+ * Normalizes an Arrow wrapper so GeoParquet metadata is preserved and GeoArrow field metadata is attached.
183
+ *
184
+ * @param arrowTable - loaders.gl Arrow table wrapper.
185
+ * @param schemaMetadata - Optional schema metadata override from the Parquet footer.
186
+ * @returns Arrow table wrapper with synchronized schema metadata.
187
+ */
188
+ export function normalizeArrowTableGeoMetadata(
189
+ arrowTable: {shape: 'arrow-table'; data: arrow.Table; schema?: Schema},
190
+ schemaMetadata?: Map<string, string> | Record<string, string>
191
+ ): {shape: 'arrow-table'; data: arrow.Table; schema: Schema} {
192
+ const schema = convertArrowToSchema(arrowTable.data.schema);
193
+ mergeSchemaMetadata(schema, schemaMetadata);
194
+ applyGeoParquetToFieldMetadata(schema);
195
+
196
+ return {
197
+ shape: 'arrow-table',
198
+ data: applyLoadersSchemaToArrowTable(arrowTable.data, schema),
199
+ schema
200
+ };
201
+ }
202
+
203
+ /**
204
+ * Synthesizes GeoParquet schema metadata from GeoArrow field metadata when needed.
205
+ *
206
+ * @param schema - Schema to inspect and update.
207
+ * @returns The same schema reference.
208
+ */
209
+ export function ensureGeoParquetMetadata(schema: Schema): Schema {
210
+ schema.metadata ||= {};
211
+ const rawExistingGeoMetadata = getGeoMetadata(schema.metadata);
212
+ if (isValidGeoParquetMetadata(rawExistingGeoMetadata)) {
213
+ return schema;
214
+ }
215
+ const existingGeoMetadata: GeoMetadata | undefined = rawExistingGeoMetadata || undefined;
216
+
217
+ const geometryColumns = getGeometryColumnsFromSchema(schema);
218
+ const geometryColumnNames = Object.keys(geometryColumns);
219
+ if (geometryColumnNames.length === 0) {
220
+ return schema;
221
+ }
222
+
223
+ const geoMetadata: GeoMetadata = {
224
+ version: GEOPARQUET_VERSION,
225
+ primary_column: geometryColumnNames[0],
226
+ columns: {}
227
+ };
228
+
229
+ for (const geometryColumnName of geometryColumnNames) {
230
+ const geometryMetadata = geometryColumns[geometryColumnName];
231
+ const synthesizedColumnMetadata = synthesizeGeoParquetColumnMetadata(geometryMetadata);
232
+ if (!synthesizedColumnMetadata) {
233
+ continue;
234
+ }
235
+
236
+ const existingColumnMetadata = existingGeoMetadata?.columns?.[geometryColumnName];
237
+ geoMetadata.columns[geometryColumnName] = {
238
+ ...pickValidOptionalGeoParquetColumnMetadata(existingColumnMetadata),
239
+ ...synthesizedColumnMetadata
240
+ };
241
+ }
242
+
243
+ if (Object.keys(geoMetadata.columns).length === 0) {
244
+ return schema;
245
+ }
246
+
247
+ setGeoMetadata(schema.metadata, geoMetadata);
248
+ return schema;
249
+ }
250
+
251
+ /**
252
+ * Ensures an Arrow table wrapper carries valid GeoParquet schema metadata when GeoArrow fields are present.
253
+ *
254
+ * @param arrowTable - Arrow table wrapper to inspect and update.
255
+ * @returns Updated Arrow table wrapper.
256
+ */
257
+ export function ensureGeoParquetMetadataOnArrowTable(arrowTable: {
258
+ shape: 'arrow-table';
259
+ data: arrow.Table;
260
+ schema?: Schema;
261
+ }): {shape: 'arrow-table'; data: arrow.Table; schema: Schema} {
262
+ const schema = arrowTable.schema || convertArrowToSchema(arrowTable.data.schema);
263
+ ensureGeoParquetMetadata(schema);
264
+
265
+ return {
266
+ shape: 'arrow-table',
267
+ data: applyLoadersSchemaToArrowTable(arrowTable.data, schema),
268
+ schema
269
+ };
270
+ }
271
+
272
+ /**
273
+ * Returns Parquet file key/value metadata as a plain map.
274
+ *
275
+ * @param parquetMetadata - parquet-wasm metadata object.
276
+ * @returns Metadata map.
277
+ */
278
+ export function getParquetFileMetadataMap(parquetMetadata: {
279
+ fileMetadata(): {keyValueMetadata(): Map<string, string>};
280
+ }): Map<string, string> {
281
+ return parquetMetadata.fileMetadata().keyValueMetadata();
282
+ }
283
+
284
+ function mergeSchemaMetadata(
285
+ schema: Schema,
286
+ schemaMetadata?: Map<string, string> | Record<string, string>
287
+ ): void {
288
+ if (!schemaMetadata) {
289
+ return;
290
+ }
291
+
292
+ schema.metadata ||= {};
293
+ for (const [metadataKey, metadataValue] of getMetadataEntries(schemaMetadata)) {
294
+ setMetadataValue(schema.metadata, metadataKey, metadataValue);
295
+ }
296
+ }
297
+
298
+ function createArrowJsSchema(baseSchema: arrow.Schema, schema: Schema): arrow.Schema {
299
+ const fields = baseSchema.fields.map(baseField => {
300
+ const nextField = schema.fields.find(field => field.name === baseField.name);
301
+ if (!nextField?.metadata) {
302
+ return baseField;
303
+ }
304
+
305
+ return baseField.clone({
306
+ metadata: createMetadataMap(nextField.metadata)
307
+ });
308
+ });
309
+
310
+ return new arrow.Schema(
311
+ fields,
312
+ createMetadataMap(schema.metadata),
313
+ baseSchema.dictionaries,
314
+ baseSchema.metadataVersion
315
+ );
316
+ }
317
+
318
+ function getGeoArrowMetadataFromGeoParquetField(
319
+ field: Field,
320
+ columnMetadata: GeoColumnMetadata
321
+ ): {extensionName: GeoArrowEncoding; extensionMetadata?: Record<string, unknown>} | null {
322
+ const normalizedEncoding = String(columnMetadata.encoding || '').toLowerCase();
323
+ const extensionName =
324
+ GEOPARQUET_TO_GEOARROW_ENCODINGS[
325
+ normalizedEncoding as keyof typeof GEOPARQUET_TO_GEOARROW_ENCODINGS
326
+ ];
327
+
328
+ if (!extensionName) {
329
+ return null;
330
+ }
331
+
332
+ if (normalizedEncoding === 'wkb' && field.type !== 'binary') {
333
+ return null;
334
+ }
335
+
336
+ if (normalizedEncoding !== 'wkb' && (field.type === 'binary' || field.type === 'utf8')) {
337
+ return null;
338
+ }
339
+
340
+ const extensionMetadata: Record<string, unknown> = {};
341
+ if (columnMetadata.crs !== undefined) {
342
+ extensionMetadata.crs = columnMetadata.crs;
343
+ }
344
+ if (columnMetadata.crs_type !== undefined) {
345
+ extensionMetadata.crs_type = columnMetadata.crs_type;
346
+ }
347
+ if (columnMetadata.edges === 'spherical') {
348
+ extensionMetadata.edges = 'spherical';
349
+ }
350
+
351
+ return Object.keys(extensionMetadata).length > 0
352
+ ? {extensionName, extensionMetadata}
353
+ : {extensionName};
354
+ }
355
+
356
+ function isValidGeoParquetMetadata(geoMetadata: GeoMetadata | null): boolean {
357
+ if (!geoMetadata || typeof geoMetadata !== 'object') {
358
+ return false;
359
+ }
360
+
361
+ if (
362
+ typeof geoMetadata.version !== 'string' ||
363
+ typeof geoMetadata.primary_column !== 'string' ||
364
+ !geoMetadata.columns ||
365
+ typeof geoMetadata.columns !== 'object'
366
+ ) {
367
+ return false;
368
+ }
369
+
370
+ const primaryColumnMetadata = geoMetadata.columns[geoMetadata.primary_column];
371
+ if (!primaryColumnMetadata) {
372
+ return false;
373
+ }
374
+
375
+ return Object.values(geoMetadata.columns).every(isValidGeoParquetColumnMetadata);
376
+ }
377
+
378
+ function isValidGeoParquetColumnMetadata(columnMetadata: unknown): boolean {
379
+ if (!columnMetadata || typeof columnMetadata !== 'object') {
380
+ return false;
381
+ }
382
+
383
+ const {encoding, geometry_types} = columnMetadata as GeoColumnMetadata;
384
+ return (
385
+ typeof encoding === 'string' &&
386
+ Boolean(
387
+ GEOPARQUET_TO_GEOARROW_ENCODINGS[
388
+ encoding.toLowerCase() as keyof typeof GEOPARQUET_TO_GEOARROW_ENCODINGS
389
+ ]
390
+ ) &&
391
+ Array.isArray(geometry_types)
392
+ );
393
+ }
394
+
395
+ function synthesizeGeoParquetColumnMetadata(geometryMetadata: {
396
+ encoding?: GeoArrowEncoding;
397
+ crs?: Record<string, unknown>;
398
+ crs_type?: 'projjson' | 'wkt2:2019';
399
+ edges?: 'spherical';
400
+ geometry_types?: GeoParquetGeometryType[];
401
+ }): GeoColumnMetadata | null {
402
+ const encoding = geometryMetadata.encoding
403
+ ? GEOARROW_TO_GEOPARQUET_ENCODINGS[
404
+ geometryMetadata.encoding as keyof typeof GEOARROW_TO_GEOPARQUET_ENCODINGS
405
+ ]
406
+ : null;
407
+
408
+ if (!encoding) {
409
+ return null;
410
+ }
411
+
412
+ const geometryTypes = Array.isArray(geometryMetadata.geometry_types)
413
+ ? geometryMetadata.geometry_types
414
+ : inferGeometryTypesFromGeoArrowEncoding(geometryMetadata.encoding);
415
+
416
+ const columnMetadata: GeoColumnMetadata = {
417
+ encoding,
418
+ geometry_types: geometryTypes
419
+ };
420
+
421
+ if (geometryMetadata.crs !== undefined) {
422
+ columnMetadata.crs = geometryMetadata.crs;
423
+ }
424
+ if (geometryMetadata.crs_type !== undefined) {
425
+ columnMetadata.crs_type = geometryMetadata.crs_type;
426
+ }
427
+ if (geometryMetadata.edges === 'spherical') {
428
+ columnMetadata.edges = 'spherical';
429
+ }
430
+
431
+ return columnMetadata;
432
+ }
433
+
434
+ function inferGeometryTypesFromGeoArrowEncoding(
435
+ encoding?: GeoArrowEncoding
436
+ ): GeoParquetGeometryType[] {
437
+ const geometryType = encoding
438
+ ? GEOARROW_TO_GEOMETRY_TYPE[encoding as keyof typeof GEOARROW_TO_GEOMETRY_TYPE]
439
+ : null;
440
+ return geometryType ? [geometryType] : [];
441
+ }
442
+
443
+ function pickValidOptionalGeoParquetColumnMetadata(
444
+ columnMetadata: GeoColumnMetadata | undefined
445
+ ): Partial<GeoColumnMetadata> {
446
+ if (!columnMetadata) {
447
+ return {};
448
+ }
449
+
450
+ const nextColumnMetadata: Partial<GeoColumnMetadata> = {};
451
+
452
+ if (columnMetadata.orientation === 'counterclockwise') {
453
+ nextColumnMetadata.orientation = columnMetadata.orientation;
454
+ }
455
+
456
+ if (isValidBBox(columnMetadata.bbox)) {
457
+ nextColumnMetadata.bbox = columnMetadata.bbox;
458
+ }
459
+
460
+ if (typeof columnMetadata.epoch === 'number' && Number.isFinite(columnMetadata.epoch)) {
461
+ nextColumnMetadata.epoch = columnMetadata.epoch;
462
+ }
463
+
464
+ if (columnMetadata.covering && typeof columnMetadata.covering === 'object') {
465
+ nextColumnMetadata.covering = columnMetadata.covering;
466
+ }
467
+
468
+ return nextColumnMetadata;
469
+ }
470
+
471
+ function isValidBBox(
472
+ bbox: GeoColumnMetadata['bbox']
473
+ ): bbox is [number, number, number, number] | [number, number, number, number, number, number] {
474
+ if (!Array.isArray(bbox) || (bbox.length !== 4 && bbox.length !== 6)) {
475
+ return false;
476
+ }
477
+
478
+ return bbox.every(value => typeof value === 'number' && Number.isFinite(value));
479
+ }
480
+
481
+ function getMetadataEntries(
482
+ metadata: Map<string, string> | Record<string, string>
483
+ ): [string, string][] {
484
+ if (metadata instanceof Map) {
485
+ return [...metadata.entries()];
486
+ }
487
+ return Object.entries(metadata);
488
+ }
489
+
490
+ function createMetadataMap(
491
+ metadata: Map<string, string> | Record<string, string> | undefined
492
+ ): Map<string, string> {
493
+ if (!metadata) {
494
+ return new Map();
495
+ }
496
+
497
+ if (metadata instanceof Map) {
498
+ return new Map(metadata);
499
+ }
500
+
501
+ return new Map(
502
+ Object.entries(metadata).map(([metadataKey, metadataValue]) => [
503
+ metadataKey,
504
+ String(metadataValue)
505
+ ])
506
+ );
507
+ }