@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/cjs/data-processor.js +674 -0
  2. package/dist/cjs/data-processor.js.map +1 -0
  3. package/dist/cjs/export/dataset-to-geojson.js +138 -0
  4. package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
  5. package/dist/cjs/export/geopackage-writer.js +385 -0
  6. package/dist/cjs/export/geopackage-writer.js.map +1 -0
  7. package/dist/cjs/export/geoparquet-writer.js +351 -0
  8. package/dist/cjs/export/geoparquet-writer.js.map +1 -0
  9. package/dist/cjs/export/kml-writer.js +347 -0
  10. package/dist/cjs/export/kml-writer.js.map +1 -0
  11. package/dist/cjs/export/shapefile-writer.js +708 -0
  12. package/dist/cjs/export/shapefile-writer.js.map +1 -0
  13. package/dist/cjs/export/sql-js.js +95 -0
  14. package/dist/cjs/export/sql-js.js.map +1 -0
  15. package/dist/cjs/export/vector-export.js +187 -0
  16. package/dist/cjs/export/vector-export.js.map +1 -0
  17. package/dist/cjs/file-handler.js +997 -0
  18. package/dist/cjs/file-handler.js.map +1 -0
  19. package/dist/cjs/index.js +114 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/kepler-csv-loader.js +163 -0
  22. package/dist/cjs/kepler-csv-loader.js.map +1 -0
  23. package/dist/cjs/loader-registry.js +542 -0
  24. package/dist/cjs/loader-registry.js.map +1 -0
  25. package/dist/cjs/remote-file.js +675 -0
  26. package/dist/cjs/remote-file.js.map +1 -0
  27. package/dist/cjs/shapefile-files.js +281 -0
  28. package/dist/cjs/shapefile-files.js.map +1 -0
  29. package/dist/cjs/types.js +6 -0
  30. package/dist/cjs/types.js.map +1 -0
  31. package/dist/data-processor.d.ts +30 -16
  32. package/dist/esm/data-processor.js +655 -0
  33. package/dist/esm/data-processor.js.map +1 -0
  34. package/dist/esm/export/dataset-to-geojson.js +132 -0
  35. package/dist/esm/export/dataset-to-geojson.js.map +1 -0
  36. package/dist/esm/export/geopackage-writer.js +381 -0
  37. package/dist/esm/export/geopackage-writer.js.map +1 -0
  38. package/dist/esm/export/geoparquet-writer.js +336 -0
  39. package/dist/esm/export/geoparquet-writer.js.map +1 -0
  40. package/dist/esm/export/kml-writer.js +341 -0
  41. package/dist/esm/export/kml-writer.js.map +1 -0
  42. package/dist/esm/export/shapefile-writer.js +701 -0
  43. package/dist/esm/export/shapefile-writer.js.map +1 -0
  44. package/dist/esm/export/sql-js.js +80 -0
  45. package/dist/esm/export/sql-js.js.map +1 -0
  46. package/dist/esm/export/vector-export.js +176 -0
  47. package/dist/esm/export/vector-export.js.map +1 -0
  48. package/dist/esm/file-handler.js +972 -0
  49. package/dist/esm/file-handler.js.map +1 -0
  50. package/dist/esm/index.js +12 -0
  51. package/dist/esm/index.js.map +1 -0
  52. package/dist/esm/kepler-csv-loader.js +152 -0
  53. package/dist/esm/kepler-csv-loader.js.map +1 -0
  54. package/dist/esm/loader-registry.js +510 -0
  55. package/dist/esm/loader-registry.js.map +1 -0
  56. package/dist/esm/remote-file.js +664 -0
  57. package/dist/esm/remote-file.js.map +1 -0
  58. package/dist/esm/shapefile-files.js +264 -0
  59. package/dist/esm/shapefile-files.js.map +1 -0
  60. package/dist/esm/types.js +3 -0
  61. package/dist/esm/types.js.map +1 -0
  62. package/dist/export/dataset-to-geojson.d.ts +12 -0
  63. package/dist/export/geopackage-writer.d.ts +28 -0
  64. package/dist/export/geoparquet-writer.d.ts +7 -0
  65. package/dist/export/kml-writer.d.ts +13 -0
  66. package/dist/export/shapefile-writer.d.ts +28 -0
  67. package/dist/export/sql-js.d.ts +8 -0
  68. package/dist/export/vector-export.d.ts +22 -0
  69. package/dist/file-handler.d.ts +55 -8
  70. package/dist/index.d.ts +6 -0
  71. package/dist/kepler-csv-loader.d.ts +12 -0
  72. package/dist/loader-registry.d.ts +29 -0
  73. package/dist/remote-file.d.ts +79 -0
  74. package/dist/shapefile-files.d.ts +39 -0
  75. package/dist/types.d.ts +13 -0
  76. package/package.json +44 -26
  77. package/dist/data-processor.js +0 -637
  78. package/dist/file-handler.js +0 -409
  79. package/dist/index.js +0 -39
  80. package/dist/types.js +0 -6
@@ -0,0 +1,655 @@
1
+ import _toConsumableArray from "@babel/runtime/helpers/esm/toConsumableArray";
2
+ import _typeof from "@babel/runtime/helpers/esm/typeof";
3
+ import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
4
+ function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
5
+ function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
6
+ function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
7
+ function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
8
+ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
9
+ // SPDX-License-Identifier: MIT
10
+ // Copyright contributors to the kepler.gl project
11
+
12
+ import * as arrow from 'apache-arrow';
13
+ import { csvParseRows, tsvParseRows, dsvFormat } from 'd3-dsv';
14
+ import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';
15
+ import normalize from '@mapbox/geojson-normalize';
16
+ import { parseSync } from '@loaders.gl/core';
17
+ import { WKBLoader } from '@loaders.gl/wkt';
18
+ import { ALL_FIELD_TYPES, DATASET_FORMATS, GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, GUIDES_FILE_FORMAT_DOC } from '@kepler.gl/constants';
19
+ import { arrowDataTypeToAnalyzerDataType, arrowDataTypeToFieldType, compactArrowTable, hasOwnProperty, isPlainObject, optionalMapInfo } from '@kepler.gl/utils';
20
+ import { analyzerTypeToFieldType, getSampleForTypeAnalyze, getSampleForTypeAnalyzeArrow, getFieldsFromData, h3IsValid, notNullorUndefined, toArray } from '@kepler.gl/common-utils';
21
+ import { KeplerGlSchema } from '@kepler.gl/schemas';
22
+ // if any of these value occurs in csv, parse it to null;
23
+ // const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN', '/N'];
24
+ // matches empty string
25
+ export var CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;
26
+ var SUPPORTED_DELIMITERS = [',', '\t', ';', '|'];
27
+ function getRowParser(delimiter) {
28
+ if (delimiter === ',') return csvParseRows;
29
+ if (delimiter === '\t') return tsvParseRows;
30
+ return dsvFormat(delimiter).parseRows;
31
+ }
32
+
33
+ /**
34
+ * Detect the delimiter used in a DSV string by checking the first line.
35
+ * Returns the delimiter that produces the most columns (minimum 2).
36
+ * Falls back to comma if no delimiter produces multiple columns.
37
+ */
38
+ export function detectDelimiter(rawData) {
39
+ var newlineIdx = rawData.indexOf('\n');
40
+ var firstLine = newlineIdx === -1 ? rawData : rawData.slice(0, newlineIdx);
41
+ if (!firstLine) return ',';
42
+ var bestDelimiter = ',';
43
+ var bestCount = 1;
44
+ var _iterator = _createForOfIteratorHelper(SUPPORTED_DELIMITERS),
45
+ _step;
46
+ try {
47
+ for (_iterator.s(); !(_step = _iterator.n()).done;) {
48
+ var _parsed$;
49
+ var delimiter = _step.value;
50
+ var parsed = getRowParser(delimiter)(firstLine);
51
+ var count = ((_parsed$ = parsed[0]) === null || _parsed$ === void 0 ? void 0 : _parsed$.length) || 0;
52
+ if (count > bestCount) {
53
+ bestCount = count;
54
+ bestDelimiter = delimiter;
55
+ }
56
+ }
57
+ } catch (err) {
58
+ _iterator.e(err);
59
+ } finally {
60
+ _iterator.f();
61
+ }
62
+ return bestDelimiter;
63
+ }
64
+ function tryParseJsonString(str) {
65
+ try {
66
+ return JSON.parse(str);
67
+ } catch (e) {
68
+ return null;
69
+ }
70
+ }
71
+ export var PARSE_FIELD_VALUE_FROM_STRING = _defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty({}, ALL_FIELD_TYPES["boolean"], {
72
+ valid: function valid(d) {
73
+ return typeof d === 'boolean';
74
+ },
75
+ parse: function parse(d) {
76
+ var s = String(d).toLowerCase();
77
+ return s === 'true' || s === 'yes' || s === '1';
78
+ }
79
+ }), ALL_FIELD_TYPES.integer, {
80
+ // @ts-ignore
81
+ valid: function valid(d) {
82
+ return parseInt(d, 10) === d;
83
+ },
84
+ // @ts-ignore
85
+ parse: function parse(d) {
86
+ return parseInt(d, 10);
87
+ }
88
+ }), ALL_FIELD_TYPES.timestamp, {
89
+ valid: function valid(d, field) {
90
+ return ['x', 'X'].includes(field.format) ? typeof d === 'number' : typeof d === 'string';
91
+ },
92
+ parse: function parse(d, field) {
93
+ return ['x', 'X'].includes(field.format) ? Number(d) : d;
94
+ }
95
+ }), ALL_FIELD_TYPES.real, {
96
+ // @ts-ignore
97
+ valid: function valid(d) {
98
+ return parseFloat(d) === d;
99
+ },
100
+ // Note this will result in NaN for some string
101
+ parse: parseFloat
102
+ }), ALL_FIELD_TYPES.object, {
103
+ valid: isPlainObject,
104
+ parse: tryParseJsonString
105
+ }), ALL_FIELD_TYPES.array, {
106
+ valid: Array.isArray,
107
+ parse: tryParseJsonString
108
+ }), ALL_FIELD_TYPES.h3, {
109
+ valid: function valid(d) {
110
+ return h3IsValid(d);
111
+ },
112
+ parse: function parse(d) {
113
+ return d;
114
+ }
115
+ });
116
+
117
+ /**
118
+ * Process csv data, output a data object with `{fields: [], rows: []}`.
119
+ * The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)
120
+ * @param rawData raw csv string
121
+ * @returns data object `{fields: [], rows: []}` can be passed to addDataToMaps
122
+ * @public
123
+ * @example
124
+ * import {processCsvData} from '@kepler.gl/processors';
125
+ *
126
+ * const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date
127
+ * 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23
128
+ * 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23
129
+ * 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23
130
+ * 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`
131
+ *
132
+ * const dataset = {
133
+ * info: {id: 'test_data', label: 'My Csv'},
134
+ * data: processCsvData(testData)
135
+ * };
136
+ *
137
+ * dispatch(addDataToMap({
138
+ * datasets: [dataset],
139
+ * options: {centerMap: true, readOnly: true}
140
+ * }));
141
+ */
142
+ export function processCsvData(rawData, header) {
143
+ var rows;
144
+ var headerRow;
145
+ if (typeof rawData === 'string') {
146
+ var delimiter = detectDelimiter(rawData);
147
+ var _parsedRows = getRowParser(delimiter)(rawData);
148
+ if (!Array.isArray(_parsedRows) || _parsedRows.length < 2) {
149
+ throw new Error('processCsvData Failed: delimited text is empty or has no data rows');
150
+ }
151
+ headerRow = _parsedRows[0];
152
+ rows = _parsedRows.slice(1);
153
+ } else if (Array.isArray(rawData) && rawData.length) {
154
+ rows = rawData;
155
+ headerRow = header;
156
+ if (!Array.isArray(headerRow)) {
157
+ // if data is passed in as array of rows and missing header
158
+ // assume first row is header
159
+ // @ts-ignore
160
+ headerRow = rawData[0];
161
+ rows = rawData.slice(1);
162
+ }
163
+ }
164
+ if (!rows || !headerRow) {
165
+ throw new Error('invalid input passed to processCsvData');
166
+ }
167
+
168
+ // here we assume the csv file that people uploaded will have first row
169
+ // as name of the column
170
+
171
+ cleanUpFalsyCsvValue(rows);
172
+ // No need to run type detection on every data point
173
+ // here we get a list of none null values to run analyze on
174
+ var sample = getSampleForTypeAnalyze({
175
+ fields: headerRow,
176
+ rows: rows
177
+ });
178
+ var fields = getFieldsFromData(sample, headerRow);
179
+ var parsedRows = parseRowsByFields(rows, fields);
180
+ return {
181
+ fields: fields,
182
+ rows: parsedRows
183
+ };
184
+ }
185
+
186
+ /**
187
+ * Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`
188
+ * @param rows
189
+ * @param fields
190
+ */
191
+ export function parseRowsByFields(rows, fields) {
192
+ // Edit rows in place
193
+ var geojsonFieldIdx = fields.findIndex(function (f) {
194
+ return f.name === '_geojson';
195
+ });
196
+ fields.forEach(parseCsvRowsByFieldType.bind(null, rows, geojsonFieldIdx));
197
+ return rows;
198
+ }
199
+
200
+ /**
201
+ * Convert falsy value in csv including `'', 'null', 'NULL', 'Null', 'NaN'` to `null`,
202
+ * so that type-analyzer won't detect it as string
203
+ *
204
+ * @param rows
205
+ */
206
+ function cleanUpFalsyCsvValue(rows) {
207
+ var re = new RegExp(CSV_NULLS, 'g');
208
+ for (var i = 0; i < rows.length; i++) {
209
+ for (var j = 0; j < rows[i].length; j++) {
210
+ // analyzer will set any fields to 'string' if there are empty values
211
+ // which will be parsed as '' by d3.csv
212
+ // here we parse empty data as null
213
+ // TODO: create warning when deltect `CSV_NULLS` in the data
214
+ if (typeof rows[i][j] === 'string' && rows[i][j].match(re)) {
215
+ rows[i][j] = null;
216
+ }
217
+ }
218
+ }
219
+ }
220
+
221
+ /**
222
+ * Process uploaded csv file to parse value by field type
223
+ *
224
+ * @param rows
225
+ * @param geoFieldIdx field index
226
+ * @param field
227
+ * @param i
228
+ */
229
+ export function parseCsvRowsByFieldType(rows, geoFieldIdx, field, i) {
230
+ var parser = PARSE_FIELD_VALUE_FROM_STRING[field.type];
231
+ if (parser) {
232
+ // check first not null value of it's already parsed
233
+ var first = rows.find(function (r) {
234
+ return notNullorUndefined(r[i]);
235
+ });
236
+ if (!first || parser.valid(first[i], field)) {
237
+ return;
238
+ }
239
+ rows.forEach(function (row) {
240
+ // parse string value based on field type
241
+ if (row[i] !== null) {
242
+ row[i] = parser.parse(row[i], field);
243
+ if (geoFieldIdx > -1 && isPlainObject(row[geoFieldIdx]) &&
244
+ // @ts-ignore
245
+ hasOwnProperty(row[geoFieldIdx], 'properties')) {
246
+ // @ts-ignore
247
+ row[geoFieldIdx].properties[field.name] = row[i];
248
+ }
249
+ }
250
+ });
251
+ }
252
+ }
253
+
254
+ /* eslint-enable complexity */
255
+
256
+ /**
257
+ * Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
258
+ * NOTE: This function may mutate input.
259
+ * @param rawData an array of row object, each object should have the same number of keys
260
+ * @returns dataset containing `fields` and `rows`
261
+ * @public
262
+ * @example
263
+ * import {addDataToMap} from '@kepler.gl/actions';
264
+ * import {processRowObject} from '@kepler.gl/processors';
265
+ *
266
+ * const data = [
267
+ * {lat: 31.27, lng: 127.56, value: 3},
268
+ * {lat: 31.22, lng: 126.26, value: 1}
269
+ * ];
270
+ *
271
+ * dispatch(addDataToMap({
272
+ * datasets: {
273
+ * info: {label: 'My Data', id: 'my_data'},
274
+ * data: processRowObject(data)
275
+ * }
276
+ * }));
277
+ */
278
+ export function processRowObject(rawData) {
279
+ if (!Array.isArray(rawData)) {
280
+ return null;
281
+ } else if (!rawData.length) {
282
+ // data is empty
283
+ return {
284
+ fields: [],
285
+ rows: []
286
+ };
287
+ }
288
+ var firstRow = rawData[0];
289
+ var keys = Object.keys(firstRow); // [lat, lng, value]
290
+ var rows = rawData.map(function (d) {
291
+ return keys.map(function (key) {
292
+ return d[key];
293
+ });
294
+ }); // [[31.27, 127.56, 3]]
295
+
296
+ // row object can still contain values like `Null` or `N/A`
297
+ cleanUpFalsyCsvValue(rows);
298
+ return processCsvData(rows, keys);
299
+ }
300
+
301
+ /**
302
+ * Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),
303
+ * output a data object with `{fields: [], rows: []}`.
304
+ * The data object can be wrapped in a `dataset` and passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
305
+ * NOTE: This function may mutate input.
306
+ *
307
+ * @param rawData raw geojson feature collection
308
+ * @returns dataset containing `fields` and `rows`
309
+ * @public
310
+ * @example
311
+ * import {addDataToMap} from '@kepler.gl/actions';
312
+ * import {processGeojson} from '@kepler.gl/processors';
313
+ *
314
+ * const geojson = {
315
+ * "type" : "FeatureCollection",
316
+ * "features" : [{
317
+ * "type" : "Feature",
318
+ * "properties" : {
319
+ * "capacity" : "10",
320
+ * "type" : "U-Rack"
321
+ * },
322
+ * "geometry" : {
323
+ * "type" : "Point",
324
+ * "coordinates" : [ -71.073283, 42.417500 ]
325
+ * }
326
+ * }]
327
+ * };
328
+ *
329
+ * dispatch(addDataToMap({
330
+ * datasets: {
331
+ * info: {
332
+ * label: 'Sample Taxi Trips in New York City',
333
+ * id: 'test_trip_data'
334
+ * },
335
+ * data: processGeojson(geojson)
336
+ * }
337
+ * }));
338
+ */
339
+ export function processGeojson(rawData) {
340
+ var normalizedGeojson = normalize(rawData);
341
+ if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {
342
+ throw new Error("Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](".concat(GUIDES_FILE_FORMAT_DOC, ")"));
343
+ }
344
+
345
+ // getting all feature fields
346
+ var allDataRows = [];
347
+ for (var i = 0; i < normalizedGeojson.features.length; i++) {
348
+ var f = normalizedGeojson.features[i];
349
+ if (f.geometry) {
350
+ var row = _objectSpread({
351
+ // Reserve the first field for geometry to preserve existing field indices.
352
+ _geojson: f
353
+ }, f.properties || {});
354
+ // A property with the reserved name must not replace the feature.
355
+ row._geojson = f;
356
+ allDataRows.push(row);
357
+ }
358
+ }
359
+ // get all the field
360
+ var fields = allDataRows.reduce(function (accu, curr) {
361
+ Object.keys(curr).forEach(function (key) {
362
+ if (!accu.includes(key)) {
363
+ accu.push(key);
364
+ }
365
+ });
366
+ return accu;
367
+ }, []);
368
+
369
+ // make sure each feature has exact same fields
370
+ allDataRows.forEach(function (d) {
371
+ fields.forEach(function (f) {
372
+ if (!(f in d)) {
373
+ d[f] = null;
374
+ if (d._geojson.properties) {
375
+ d._geojson.properties[f] = null;
376
+ }
377
+ }
378
+ });
379
+ });
380
+ return processRowObject(allDataRows);
381
+ }
382
+
383
+ /**
384
+ * Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
385
+ * The json object should contain `datasets` and `config`.
386
+ * @param rawData
387
+ * @param schema
388
+ * @returns datasets and config `{datasets: {}, config: {}}`
389
+ * @public
390
+ * @example
391
+ * import {addDataToMap} from '@kepler.gl/actions';
392
+ * import {processKeplerglJSON} from '@kepler.gl/processors';
393
+ *
394
+ * dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
395
+ */
396
+ function savedMapDisplayInfo(info) {
397
+ var saved = info && _typeof(info) === 'object' ? optionalMapInfo(info) : undefined;
398
+ // Both keys are always present: setMapInfo merges, so omitting one would keep
399
+ // the previously loaded map's value instead of clearing it.
400
+ return {
401
+ title: saved === null || saved === void 0 ? void 0 : saved.title,
402
+ description: saved === null || saved === void 0 ? void 0 : saved.description
403
+ };
404
+ }
405
+ export function processKeplerglJSON(rawData) {
406
+ var schema = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : KeplerGlSchema;
407
+ if (!rawData) {
408
+ return null;
409
+ }
410
+ return _objectSpread(_objectSpread({}, schema.load(rawData.datasets, rawData.config)), {}, {
411
+ info: savedMapDisplayInfo(rawData.info)
412
+ });
413
+ }
414
+
415
+ /**
416
+ * Parse a single or an array of datasets saved using kepler.gl schema
417
+ * @param rawData
418
+ * @param schema
419
+ */
420
+ export function processKeplerglDataset(rawData) {
421
+ var schema = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : KeplerGlSchema;
422
+ if (!rawData) {
423
+ return null;
424
+ }
425
+ var results = schema.parseSavedData(toArray(rawData));
426
+ if (!results) {
427
+ return null;
428
+ }
429
+ return Array.isArray(rawData) ? results : results[0];
430
+ }
431
+
432
+ /**
433
+ * Process a loaders.gl [`ArrowTable`](https://loaders.gl/docs/specifications/category-table) (`{shape: 'arrow-table', data}`)
434
+ * and return a data object that can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap).
435
+ * Load arrow files with `GeoArrowLoader` and `{arrow: {shape: 'arrow-table'}}`.
436
+ *
437
+ * @param arrowTable ArrowTable to parse, see loaders.gl/schema
438
+ * @returns dataset containing `fields` and `rows` or null
439
+ * @public
440
+ * @example
441
+ * import {addDataToMap} from '@kepler.gl/actions';
442
+ * import {processArrowTable} from '@kepler.gl/processors';
443
+ * import {load} from '@loaders.gl/core';
444
+ * import {GeoArrowLoader} from '@loaders.gl/arrow';
445
+ *
446
+ * const arrowTable = await load(url, GeoArrowLoader, {arrow: {shape: 'arrow-table'}});
447
+ *
448
+ * dispatch(addDataToMap({
449
+ * datasets: {
450
+ * info: {id: 'arrow_dataset', label: 'My Arrow'},
451
+ * data: processArrowTable(arrowTable)
452
+ * }
453
+ * }));
454
+ */
455
+ export function processArrowTable(arrowTable) {
456
+ // @ts-ignore - Unknown data type causing build failures
457
+ return processArrowBatches(arrowTable.data.batches);
458
+ }
459
+
460
+ /**
461
+ * Extracts GeoArrow metadata from an Apache Arrow table schema.
462
+ * For geoparquet files geoarrow metadata isn't present in fields, so extract extra info from schema.
463
+ * @param table The Apache Arrow table to extract metadata from.
464
+ * @returns An object mapping column names to their GeoArrow encoding type.
465
+ * @throws Logs an error message if parsing of metadata fails.
466
+ */
467
+ export function getGeoArrowMetadataFromSchema(table) {
468
+ var geoArrowMetadata = {};
469
+ try {
470
+ var _table$schema$metadat;
471
+ var geoString = (_table$schema$metadat = table.schema.metadata) === null || _table$schema$metadat === void 0 ? void 0 : _table$schema$metadat.get('geo');
472
+ if (geoString) {
473
+ var parsedGeoString = JSON.parse(geoString);
474
+ if (parsedGeoString.columns) {
475
+ Object.keys(parsedGeoString.columns).forEach(function (columnName) {
476
+ var columnData = parsedGeoString.columns[columnName];
477
+ if ((columnData === null || columnData === void 0 ? void 0 : columnData.encoding) === 'WKB') {
478
+ geoArrowMetadata[columnName] = GEOARROW_EXTENSIONS.WKB;
479
+ }
480
+ // TODO potentially there are other types but no datasets to test
481
+ });
482
+ }
483
+ }
484
+ } catch (error) {
485
+ console.error('An error during arrow table schema metadata parsing');
486
+ }
487
+ return geoArrowMetadata;
488
+ }
489
+
490
+ /**
491
+ * Converts an Apache Arrow table schema into an array of Kepler.gl field objects.
492
+ * @param table The Apache Arrow table whose schema needs to be converted.
493
+ * @param fieldTypeSuggestions Optional mapping of field names to suggested field types.
494
+ * @returns An array of field objects suitable for Kepler.gl.
495
+ */
496
+ export function arrowSchemaToFields(table) {
497
+ var fieldTypeSuggestions = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
498
+ var headerRow = table.schema.fields.map(function (f) {
499
+ return f.name;
500
+ });
501
+ var sample = getSampleForTypeAnalyzeArrow(table, headerRow);
502
+ var keplerFields = getFieldsFromData(sample, headerRow);
503
+ var geoArrowMetadata = getGeoArrowMetadataFromSchema(table);
504
+ return table.schema.fields.map(function (field, fieldIndex) {
505
+ var _field$metadata$get;
506
+ var type = arrowDataTypeToFieldType(field.type);
507
+ var analyzerType = arrowDataTypeToAnalyzerDataType(field.type);
508
+ var format = '';
509
+ var fieldTypeSuggestion = fieldTypeSuggestions[field.name];
510
+ var keplerField = keplerFields[fieldIndex];
511
+
512
+ // geometry fields produced by DuckDB's st_asgeojson()
513
+ if (fieldTypeSuggestion === 'JSON') {
514
+ type = ALL_FIELD_TYPES.geojson;
515
+ analyzerType = AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING;
516
+ } else if (fieldTypeSuggestion === 'GEOMETRY' || (_field$metadata$get = field.metadata.get(GEOARROW_METADATA_KEY)) !== null && _field$metadata$get !== void 0 && _field$metadata$get.startsWith('geoarrow')) {
517
+ type = ALL_FIELD_TYPES.geoarrow;
518
+ analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
519
+ } else if (geoArrowMetadata[field.name]) {
520
+ var _field$metadata;
521
+ type = ALL_FIELD_TYPES.geoarrow;
522
+ analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
523
+ (_field$metadata = field.metadata) === null || _field$metadata === void 0 || _field$metadata.set(GEOARROW_METADATA_KEY, geoArrowMetadata[field.name]);
524
+ } else if (fieldTypeSuggestion === 'BLOB') {
525
+ // When arrow wkb column saved to DuckDB as BLOB without any metadata, then queried back
526
+ try {
527
+ var _table$getChildAt;
528
+ var data = (_table$getChildAt = table.getChildAt(fieldIndex)) === null || _table$getChildAt === void 0 ? void 0 : _table$getChildAt.get(0);
529
+ if (data) {
530
+ var binaryGeo = parseSync(data, WKBLoader);
531
+ if (binaryGeo) {
532
+ var _field$metadata2;
533
+ type = ALL_FIELD_TYPES.geoarrow;
534
+ analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
535
+ (_field$metadata2 = field.metadata) === null || _field$metadata2 === void 0 || _field$metadata2.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB);
536
+ }
537
+ }
538
+ } catch (error) {
539
+ // ignore, not WKB
540
+ }
541
+ } else if (fieldTypeSuggestion === 'VARCHAR' && (keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY || keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING)) {
542
+ // When wkb/wkt was saved as varchar in DuckDB
543
+ type = keplerField.type;
544
+ analyzerType = keplerField.analyzerType;
545
+ format = keplerField.format;
546
+ } else if (fieldTypeSuggestion === 'VARCHAR' && keplerField.type === ALL_FIELD_TYPES.h3) {
547
+ // when kepler detected h3 column using getFieldsFromData(), set type to h3 and analyzerType to H3
548
+ type = ALL_FIELD_TYPES.h3;
549
+ analyzerType = keplerField.analyzerType;
550
+ } else {
551
+ // TODO should we use Kepler getFieldsFromData instead
552
+ // of arrowDataTypeToFieldType for all fields?
553
+ if (keplerField.type === ALL_FIELD_TYPES.timestamp) {
554
+ type = keplerField.type;
555
+ analyzerType = keplerField.analyzerType;
556
+ format = keplerField.format;
557
+ }
558
+ }
559
+ return _objectSpread(_objectSpread({}, field), {}, {
560
+ name: field.name,
561
+ id: field.name,
562
+ displayName: field.name,
563
+ format: format,
564
+ fieldIdx: fieldIndex,
565
+ type: type,
566
+ analyzerType: analyzerType,
567
+ valueAccessor: function valueAccessor(dc) {
568
+ return function (d) {
569
+ return dc.valueAt(d.index, fieldIndex);
570
+ };
571
+ },
572
+ metadata: field.metadata
573
+ });
574
+ });
575
+ }
576
+ var CAST_BIGINTS = false;
577
+
578
+ /**
579
+ * Cast 64-bit integer Arrow columns (Int64, Uint64) to Float64 to avoid BigInt values
580
+ * that are incompatible with d3 scales, sorting, and other numeric operations.
581
+ * Mirrors the DuckDB approach of casting BIGINT/UBIGINT to DOUBLE.
582
+ *
583
+ * Left off so Uint64 values are not rounded through Float64. Samples are
584
+ * stringified for type-analyzer instead.
585
+ */
586
+ function castBigIntColumnsToFloat64(arrowTable) {
587
+ if (!CAST_BIGINTS) {
588
+ return arrowTable;
589
+ }
590
+ var needsCast = arrowTable.schema.fields.some(function (f) {
591
+ return arrow.DataType.isInt(f.type) && f.type.bitWidth === 64;
592
+ });
593
+ if (!needsCast) {
594
+ return arrowTable;
595
+ }
596
+ var newColumns = {};
597
+ for (var i = 0; i < arrowTable.numCols; i++) {
598
+ var field = arrowTable.schema.fields[i];
599
+ var col = arrowTable.getChildAt(i);
600
+ if (!col) continue;
601
+ if (arrow.DataType.isInt(field.type) && field.type.bitWidth === 64) {
602
+ var float64Array = new Float64Array(col.length);
603
+ for (var j = 0; j < col.length; j++) {
604
+ var val = col.get(j);
605
+ float64Array[j] = val === null ? NaN : Number(val);
606
+ }
607
+ newColumns[field.name] = arrow.makeVector(float64Array);
608
+ } else {
609
+ newColumns[field.name] = col;
610
+ }
611
+ }
612
+ return new arrow.Table(newColumns);
613
+ }
614
+ /**
615
+ * Parse arrow batches returned from parseInBatches()
616
+ *
617
+ * @param arrowBatches the arrow record batches to parse
618
+ * @param options optional processing flags
619
+ * @returns dataset containing `fields` and `rows` or null
620
+ */
621
+ export function processArrowBatches(arrowBatches, options) {
622
+ if (arrowBatches.length === 0) {
623
+ return null;
624
+ }
625
+ var arrowTable = (options === null || options === void 0 ? void 0 : options.compact) === false ? castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)) : compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)));
626
+ var fields = arrowSchemaToFields(arrowTable);
627
+ var cols = _toConsumableArray(Array(arrowTable.numCols).keys()).map(function (i) {
628
+ return arrowTable.getChildAt(i);
629
+ });
630
+
631
+ // return empty rows and use raw arrow table to construct column-wise data container
632
+ return {
633
+ fields: fields,
634
+ rows: [],
635
+ cols: cols,
636
+ metadata: arrowTable.schema.metadata,
637
+ // Save original arrow schema, for better ingestion into DuckDB.
638
+ // TODO consider returning arrowTable in cols, not an array of Vectors from arrowTable.
639
+ arrowSchema: arrowTable.schema
640
+ };
641
+ }
642
+ export var DATASET_HANDLERS = _defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty({}, DATASET_FORMATS.row, processRowObject), DATASET_FORMATS.geojson, processGeojson), DATASET_FORMATS.csv, processCsvData), DATASET_FORMATS.arrow, processArrowTable), DATASET_FORMATS.keplergl, processKeplerglDataset);
643
+ export var Processors = {
644
+ processGeojson: processGeojson,
645
+ processCsvData: processCsvData,
646
+ processArrowTable: processArrowTable,
647
+ processArrowBatches: processArrowBatches,
648
+ processRowObject: processRowObject,
649
+ processKeplerglJSON: processKeplerglJSON,
650
+ processKeplerglDataset: processKeplerglDataset,
651
+ analyzerTypeToFieldType: analyzerTypeToFieldType,
652
+ getFieldsFromData: getFieldsFromData,
653
+ parseCsvRowsByFieldType: parseCsvRowsByFieldType
654
+ };
655
+ //# sourceMappingURL=data-processor.js.map