@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/cjs/data-processor.js +674 -0
  2. package/dist/cjs/data-processor.js.map +1 -0
  3. package/dist/cjs/export/dataset-to-geojson.js +138 -0
  4. package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
  5. package/dist/cjs/export/geopackage-writer.js +385 -0
  6. package/dist/cjs/export/geopackage-writer.js.map +1 -0
  7. package/dist/cjs/export/geoparquet-writer.js +351 -0
  8. package/dist/cjs/export/geoparquet-writer.js.map +1 -0
  9. package/dist/cjs/export/kml-writer.js +347 -0
  10. package/dist/cjs/export/kml-writer.js.map +1 -0
  11. package/dist/cjs/export/shapefile-writer.js +708 -0
  12. package/dist/cjs/export/shapefile-writer.js.map +1 -0
  13. package/dist/cjs/export/sql-js.js +95 -0
  14. package/dist/cjs/export/sql-js.js.map +1 -0
  15. package/dist/cjs/export/vector-export.js +187 -0
  16. package/dist/cjs/export/vector-export.js.map +1 -0
  17. package/dist/cjs/file-handler.js +997 -0
  18. package/dist/cjs/file-handler.js.map +1 -0
  19. package/dist/cjs/index.js +114 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/kepler-csv-loader.js +163 -0
  22. package/dist/cjs/kepler-csv-loader.js.map +1 -0
  23. package/dist/cjs/loader-registry.js +542 -0
  24. package/dist/cjs/loader-registry.js.map +1 -0
  25. package/dist/cjs/remote-file.js +675 -0
  26. package/dist/cjs/remote-file.js.map +1 -0
  27. package/dist/cjs/shapefile-files.js +281 -0
  28. package/dist/cjs/shapefile-files.js.map +1 -0
  29. package/dist/cjs/types.js +6 -0
  30. package/dist/cjs/types.js.map +1 -0
  31. package/dist/data-processor.d.ts +30 -16
  32. package/dist/esm/data-processor.js +655 -0
  33. package/dist/esm/data-processor.js.map +1 -0
  34. package/dist/esm/export/dataset-to-geojson.js +132 -0
  35. package/dist/esm/export/dataset-to-geojson.js.map +1 -0
  36. package/dist/esm/export/geopackage-writer.js +381 -0
  37. package/dist/esm/export/geopackage-writer.js.map +1 -0
  38. package/dist/esm/export/geoparquet-writer.js +336 -0
  39. package/dist/esm/export/geoparquet-writer.js.map +1 -0
  40. package/dist/esm/export/kml-writer.js +341 -0
  41. package/dist/esm/export/kml-writer.js.map +1 -0
  42. package/dist/esm/export/shapefile-writer.js +701 -0
  43. package/dist/esm/export/shapefile-writer.js.map +1 -0
  44. package/dist/esm/export/sql-js.js +80 -0
  45. package/dist/esm/export/sql-js.js.map +1 -0
  46. package/dist/esm/export/vector-export.js +176 -0
  47. package/dist/esm/export/vector-export.js.map +1 -0
  48. package/dist/esm/file-handler.js +972 -0
  49. package/dist/esm/file-handler.js.map +1 -0
  50. package/dist/esm/index.js +12 -0
  51. package/dist/esm/index.js.map +1 -0
  52. package/dist/esm/kepler-csv-loader.js +152 -0
  53. package/dist/esm/kepler-csv-loader.js.map +1 -0
  54. package/dist/esm/loader-registry.js +510 -0
  55. package/dist/esm/loader-registry.js.map +1 -0
  56. package/dist/esm/remote-file.js +664 -0
  57. package/dist/esm/remote-file.js.map +1 -0
  58. package/dist/esm/shapefile-files.js +264 -0
  59. package/dist/esm/shapefile-files.js.map +1 -0
  60. package/dist/esm/types.js +3 -0
  61. package/dist/esm/types.js.map +1 -0
  62. package/dist/export/dataset-to-geojson.d.ts +12 -0
  63. package/dist/export/geopackage-writer.d.ts +28 -0
  64. package/dist/export/geoparquet-writer.d.ts +7 -0
  65. package/dist/export/kml-writer.d.ts +13 -0
  66. package/dist/export/shapefile-writer.d.ts +28 -0
  67. package/dist/export/sql-js.d.ts +8 -0
  68. package/dist/export/vector-export.d.ts +22 -0
  69. package/dist/file-handler.d.ts +55 -8
  70. package/dist/index.d.ts +6 -0
  71. package/dist/kepler-csv-loader.d.ts +12 -0
  72. package/dist/loader-registry.d.ts +29 -0
  73. package/dist/remote-file.d.ts +79 -0
  74. package/dist/shapefile-files.d.ts +39 -0
  75. package/dist/types.d.ts +13 -0
  76. package/package.json +44 -26
  77. package/dist/data-processor.js +0 -637
  78. package/dist/file-handler.js +0 -409
  79. package/dist/index.js +0 -39
  80. package/dist/types.js +0 -6
@@ -0,0 +1,674 @@
1
+ "use strict";
2
+
3
+ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
4
+ var _typeof3 = require("@babel/runtime/helpers/typeof");
5
+ Object.defineProperty(exports, "__esModule", {
6
+ value: true
7
+ });
8
+ exports.Processors = exports.PARSE_FIELD_VALUE_FROM_STRING = exports.DATASET_HANDLERS = exports.CSV_NULLS = void 0;
9
+ exports.arrowSchemaToFields = arrowSchemaToFields;
10
+ exports.detectDelimiter = detectDelimiter;
11
+ exports.getGeoArrowMetadataFromSchema = getGeoArrowMetadataFromSchema;
12
+ exports.parseCsvRowsByFieldType = parseCsvRowsByFieldType;
13
+ exports.parseRowsByFields = parseRowsByFields;
14
+ exports.processArrowBatches = processArrowBatches;
15
+ exports.processArrowTable = processArrowTable;
16
+ exports.processCsvData = processCsvData;
17
+ exports.processGeojson = processGeojson;
18
+ exports.processKeplerglDataset = processKeplerglDataset;
19
+ exports.processKeplerglJSON = processKeplerglJSON;
20
+ exports.processRowObject = processRowObject;
21
+ var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
22
+ var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
23
+ var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
24
+ var arrow = _interopRequireWildcard(require("apache-arrow"));
25
+ var _d3Dsv = require("d3-dsv");
26
+ var _typeAnalyzer = require("type-analyzer");
27
+ var _geojsonNormalize = _interopRequireDefault(require("@mapbox/geojson-normalize"));
28
+ var _core = require("@loaders.gl/core");
29
+ var _wkt = require("@loaders.gl/wkt");
30
+ var _constants = require("@kepler.gl/constants");
31
+ var _utils = require("@kepler.gl/utils");
32
+ var _commonUtils = require("@kepler.gl/common-utils");
33
+ var _schemas = require("@kepler.gl/schemas");
34
+ function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function _interopRequireWildcard(e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, "default": e }; if (null === e || "object" != _typeof3(e) && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (var _t in e) "default" !== _t && {}.hasOwnProperty.call(e, _t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, _t)) && (i.get || i.set) ? o(f, _t, i) : f[_t] = e[_t]); return f; })(e, t); }
35
+ function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
36
+ function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { (0, _defineProperty2["default"])(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
37
+ function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
38
+ function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
39
+ function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; } // SPDX-License-Identifier: MIT
40
+ // Copyright contributors to the kepler.gl project
41
+ // if any of these value occurs in csv, parse it to null;
42
+ // const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN', '/N'];
43
+ // matches empty string
44
+ var CSV_NULLS = exports.CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;
45
+ var SUPPORTED_DELIMITERS = [',', '\t', ';', '|'];
46
+ function getRowParser(delimiter) {
47
+ if (delimiter === ',') return _d3Dsv.csvParseRows;
48
+ if (delimiter === '\t') return _d3Dsv.tsvParseRows;
49
+ return (0, _d3Dsv.dsvFormat)(delimiter).parseRows;
50
+ }
51
+
52
+ /**
53
+ * Detect the delimiter used in a DSV string by checking the first line.
54
+ * Returns the delimiter that produces the most columns (minimum 2).
55
+ * Falls back to comma if no delimiter produces multiple columns.
56
+ */
57
+ function detectDelimiter(rawData) {
58
+ var newlineIdx = rawData.indexOf('\n');
59
+ var firstLine = newlineIdx === -1 ? rawData : rawData.slice(0, newlineIdx);
60
+ if (!firstLine) return ',';
61
+ var bestDelimiter = ',';
62
+ var bestCount = 1;
63
+ var _iterator = _createForOfIteratorHelper(SUPPORTED_DELIMITERS),
64
+ _step;
65
+ try {
66
+ for (_iterator.s(); !(_step = _iterator.n()).done;) {
67
+ var _parsed$;
68
+ var delimiter = _step.value;
69
+ var parsed = getRowParser(delimiter)(firstLine);
70
+ var count = ((_parsed$ = parsed[0]) === null || _parsed$ === void 0 ? void 0 : _parsed$.length) || 0;
71
+ if (count > bestCount) {
72
+ bestCount = count;
73
+ bestDelimiter = delimiter;
74
+ }
75
+ }
76
+ } catch (err) {
77
+ _iterator.e(err);
78
+ } finally {
79
+ _iterator.f();
80
+ }
81
+ return bestDelimiter;
82
+ }
83
+ function tryParseJsonString(str) {
84
+ try {
85
+ return JSON.parse(str);
86
+ } catch (e) {
87
+ return null;
88
+ }
89
+ }
90
+ var PARSE_FIELD_VALUE_FROM_STRING = exports.PARSE_FIELD_VALUE_FROM_STRING = (0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])({}, _constants.ALL_FIELD_TYPES["boolean"], {
91
+ valid: function valid(d) {
92
+ return typeof d === 'boolean';
93
+ },
94
+ parse: function parse(d) {
95
+ var s = String(d).toLowerCase();
96
+ return s === 'true' || s === 'yes' || s === '1';
97
+ }
98
+ }), _constants.ALL_FIELD_TYPES.integer, {
99
+ // @ts-ignore
100
+ valid: function valid(d) {
101
+ return parseInt(d, 10) === d;
102
+ },
103
+ // @ts-ignore
104
+ parse: function parse(d) {
105
+ return parseInt(d, 10);
106
+ }
107
+ }), _constants.ALL_FIELD_TYPES.timestamp, {
108
+ valid: function valid(d, field) {
109
+ return ['x', 'X'].includes(field.format) ? typeof d === 'number' : typeof d === 'string';
110
+ },
111
+ parse: function parse(d, field) {
112
+ return ['x', 'X'].includes(field.format) ? Number(d) : d;
113
+ }
114
+ }), _constants.ALL_FIELD_TYPES.real, {
115
+ // @ts-ignore
116
+ valid: function valid(d) {
117
+ return parseFloat(d) === d;
118
+ },
119
+ // Note this will result in NaN for some string
120
+ parse: parseFloat
121
+ }), _constants.ALL_FIELD_TYPES.object, {
122
+ valid: _utils.isPlainObject,
123
+ parse: tryParseJsonString
124
+ }), _constants.ALL_FIELD_TYPES.array, {
125
+ valid: Array.isArray,
126
+ parse: tryParseJsonString
127
+ }), _constants.ALL_FIELD_TYPES.h3, {
128
+ valid: function valid(d) {
129
+ return (0, _commonUtils.h3IsValid)(d);
130
+ },
131
+ parse: function parse(d) {
132
+ return d;
133
+ }
134
+ });
135
+
136
+ /**
137
+ * Process csv data, output a data object with `{fields: [], rows: []}`.
138
+ * The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)
139
+ * @param rawData raw csv string
140
+ * @returns data object `{fields: [], rows: []}` can be passed to addDataToMaps
141
+ * @public
142
+ * @example
143
+ * import {processCsvData} from '@kepler.gl/processors';
144
+ *
145
+ * const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date
146
+ * 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23
147
+ * 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23
148
+ * 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23
149
+ * 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`
150
+ *
151
+ * const dataset = {
152
+ * info: {id: 'test_data', label: 'My Csv'},
153
+ * data: processCsvData(testData)
154
+ * };
155
+ *
156
+ * dispatch(addDataToMap({
157
+ * datasets: [dataset],
158
+ * options: {centerMap: true, readOnly: true}
159
+ * }));
160
+ */
161
+ function processCsvData(rawData, header) {
162
+ var rows;
163
+ var headerRow;
164
+ if (typeof rawData === 'string') {
165
+ var delimiter = detectDelimiter(rawData);
166
+ var _parsedRows = getRowParser(delimiter)(rawData);
167
+ if (!Array.isArray(_parsedRows) || _parsedRows.length < 2) {
168
+ throw new Error('processCsvData Failed: delimited text is empty or has no data rows');
169
+ }
170
+ headerRow = _parsedRows[0];
171
+ rows = _parsedRows.slice(1);
172
+ } else if (Array.isArray(rawData) && rawData.length) {
173
+ rows = rawData;
174
+ headerRow = header;
175
+ if (!Array.isArray(headerRow)) {
176
+ // if data is passed in as array of rows and missing header
177
+ // assume first row is header
178
+ // @ts-ignore
179
+ headerRow = rawData[0];
180
+ rows = rawData.slice(1);
181
+ }
182
+ }
183
+ if (!rows || !headerRow) {
184
+ throw new Error('invalid input passed to processCsvData');
185
+ }
186
+
187
+ // here we assume the csv file that people uploaded will have first row
188
+ // as name of the column
189
+
190
+ cleanUpFalsyCsvValue(rows);
191
+ // No need to run type detection on every data point
192
+ // here we get a list of none null values to run analyze on
193
+ var sample = (0, _commonUtils.getSampleForTypeAnalyze)({
194
+ fields: headerRow,
195
+ rows: rows
196
+ });
197
+ var fields = (0, _commonUtils.getFieldsFromData)(sample, headerRow);
198
+ var parsedRows = parseRowsByFields(rows, fields);
199
+ return {
200
+ fields: fields,
201
+ rows: parsedRows
202
+ };
203
+ }
204
+
205
+ /**
206
+ * Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`
207
+ * @param rows
208
+ * @param fields
209
+ */
210
+ function parseRowsByFields(rows, fields) {
211
+ // Edit rows in place
212
+ var geojsonFieldIdx = fields.findIndex(function (f) {
213
+ return f.name === '_geojson';
214
+ });
215
+ fields.forEach(parseCsvRowsByFieldType.bind(null, rows, geojsonFieldIdx));
216
+ return rows;
217
+ }
218
+
219
+ /**
220
+ * Convert falsy value in csv including `'', 'null', 'NULL', 'Null', 'NaN'` to `null`,
221
+ * so that type-analyzer won't detect it as string
222
+ *
223
+ * @param rows
224
+ */
225
+ function cleanUpFalsyCsvValue(rows) {
226
+ var re = new RegExp(CSV_NULLS, 'g');
227
+ for (var i = 0; i < rows.length; i++) {
228
+ for (var j = 0; j < rows[i].length; j++) {
229
+ // analyzer will set any fields to 'string' if there are empty values
230
+ // which will be parsed as '' by d3.csv
231
+ // here we parse empty data as null
232
+ // TODO: create warning when deltect `CSV_NULLS` in the data
233
+ if (typeof rows[i][j] === 'string' && rows[i][j].match(re)) {
234
+ rows[i][j] = null;
235
+ }
236
+ }
237
+ }
238
+ }
239
+
240
+ /**
241
+ * Process uploaded csv file to parse value by field type
242
+ *
243
+ * @param rows
244
+ * @param geoFieldIdx field index
245
+ * @param field
246
+ * @param i
247
+ */
248
+ function parseCsvRowsByFieldType(rows, geoFieldIdx, field, i) {
249
+ var parser = PARSE_FIELD_VALUE_FROM_STRING[field.type];
250
+ if (parser) {
251
+ // check first not null value of it's already parsed
252
+ var first = rows.find(function (r) {
253
+ return (0, _commonUtils.notNullorUndefined)(r[i]);
254
+ });
255
+ if (!first || parser.valid(first[i], field)) {
256
+ return;
257
+ }
258
+ rows.forEach(function (row) {
259
+ // parse string value based on field type
260
+ if (row[i] !== null) {
261
+ row[i] = parser.parse(row[i], field);
262
+ if (geoFieldIdx > -1 && (0, _utils.isPlainObject)(row[geoFieldIdx]) &&
263
+ // @ts-ignore
264
+ (0, _utils.hasOwnProperty)(row[geoFieldIdx], 'properties')) {
265
+ // @ts-ignore
266
+ row[geoFieldIdx].properties[field.name] = row[i];
267
+ }
268
+ }
269
+ });
270
+ }
271
+ }
272
+
273
+ /* eslint-enable complexity */
274
+
275
+ /**
276
+ * Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
277
+ * NOTE: This function may mutate input.
278
+ * @param rawData an array of row object, each object should have the same number of keys
279
+ * @returns dataset containing `fields` and `rows`
280
+ * @public
281
+ * @example
282
+ * import {addDataToMap} from '@kepler.gl/actions';
283
+ * import {processRowObject} from '@kepler.gl/processors';
284
+ *
285
+ * const data = [
286
+ * {lat: 31.27, lng: 127.56, value: 3},
287
+ * {lat: 31.22, lng: 126.26, value: 1}
288
+ * ];
289
+ *
290
+ * dispatch(addDataToMap({
291
+ * datasets: {
292
+ * info: {label: 'My Data', id: 'my_data'},
293
+ * data: processRowObject(data)
294
+ * }
295
+ * }));
296
+ */
297
+ function processRowObject(rawData) {
298
+ if (!Array.isArray(rawData)) {
299
+ return null;
300
+ } else if (!rawData.length) {
301
+ // data is empty
302
+ return {
303
+ fields: [],
304
+ rows: []
305
+ };
306
+ }
307
+ var firstRow = rawData[0];
308
+ var keys = Object.keys(firstRow); // [lat, lng, value]
309
+ var rows = rawData.map(function (d) {
310
+ return keys.map(function (key) {
311
+ return d[key];
312
+ });
313
+ }); // [[31.27, 127.56, 3]]
314
+
315
+ // row object can still contain values like `Null` or `N/A`
316
+ cleanUpFalsyCsvValue(rows);
317
+ return processCsvData(rows, keys);
318
+ }
319
+
320
+ /**
321
+ * Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),
322
+ * output a data object with `{fields: [], rows: []}`.
323
+ * The data object can be wrapped in a `dataset` and passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
324
+ * NOTE: This function may mutate input.
325
+ *
326
+ * @param rawData raw geojson feature collection
327
+ * @returns dataset containing `fields` and `rows`
328
+ * @public
329
+ * @example
330
+ * import {addDataToMap} from '@kepler.gl/actions';
331
+ * import {processGeojson} from '@kepler.gl/processors';
332
+ *
333
+ * const geojson = {
334
+ * "type" : "FeatureCollection",
335
+ * "features" : [{
336
+ * "type" : "Feature",
337
+ * "properties" : {
338
+ * "capacity" : "10",
339
+ * "type" : "U-Rack"
340
+ * },
341
+ * "geometry" : {
342
+ * "type" : "Point",
343
+ * "coordinates" : [ -71.073283, 42.417500 ]
344
+ * }
345
+ * }]
346
+ * };
347
+ *
348
+ * dispatch(addDataToMap({
349
+ * datasets: {
350
+ * info: {
351
+ * label: 'Sample Taxi Trips in New York City',
352
+ * id: 'test_trip_data'
353
+ * },
354
+ * data: processGeojson(geojson)
355
+ * }
356
+ * }));
357
+ */
358
+ function processGeojson(rawData) {
359
+ var normalizedGeojson = (0, _geojsonNormalize["default"])(rawData);
360
+ if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {
361
+ throw new Error("Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](".concat(_constants.GUIDES_FILE_FORMAT_DOC, ")"));
362
+ }
363
+
364
+ // getting all feature fields
365
+ var allDataRows = [];
366
+ for (var i = 0; i < normalizedGeojson.features.length; i++) {
367
+ var f = normalizedGeojson.features[i];
368
+ if (f.geometry) {
369
+ var row = _objectSpread({
370
+ // Reserve the first field for geometry to preserve existing field indices.
371
+ _geojson: f
372
+ }, f.properties || {});
373
+ // A property with the reserved name must not replace the feature.
374
+ row._geojson = f;
375
+ allDataRows.push(row);
376
+ }
377
+ }
378
+ // get all the field
379
+ var fields = allDataRows.reduce(function (accu, curr) {
380
+ Object.keys(curr).forEach(function (key) {
381
+ if (!accu.includes(key)) {
382
+ accu.push(key);
383
+ }
384
+ });
385
+ return accu;
386
+ }, []);
387
+
388
+ // make sure each feature has exact same fields
389
+ allDataRows.forEach(function (d) {
390
+ fields.forEach(function (f) {
391
+ if (!(f in d)) {
392
+ d[f] = null;
393
+ if (d._geojson.properties) {
394
+ d._geojson.properties[f] = null;
395
+ }
396
+ }
397
+ });
398
+ });
399
+ return processRowObject(allDataRows);
400
+ }
401
+
402
+ /**
403
+ * Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
404
+ * The json object should contain `datasets` and `config`.
405
+ * @param rawData
406
+ * @param schema
407
+ * @returns datasets and config `{datasets: {}, config: {}}`
408
+ * @public
409
+ * @example
410
+ * import {addDataToMap} from '@kepler.gl/actions';
411
+ * import {processKeplerglJSON} from '@kepler.gl/processors';
412
+ *
413
+ * dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
414
+ */
415
+ function savedMapDisplayInfo(info) {
416
+ var saved = info && (0, _typeof2["default"])(info) === 'object' ? (0, _utils.optionalMapInfo)(info) : undefined;
417
+ // Both keys are always present: setMapInfo merges, so omitting one would keep
418
+ // the previously loaded map's value instead of clearing it.
419
+ return {
420
+ title: saved === null || saved === void 0 ? void 0 : saved.title,
421
+ description: saved === null || saved === void 0 ? void 0 : saved.description
422
+ };
423
+ }
424
+ function processKeplerglJSON(rawData) {
425
+ var schema = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : _schemas.KeplerGlSchema;
426
+ if (!rawData) {
427
+ return null;
428
+ }
429
+ return _objectSpread(_objectSpread({}, schema.load(rawData.datasets, rawData.config)), {}, {
430
+ info: savedMapDisplayInfo(rawData.info)
431
+ });
432
+ }
433
+
434
+ /**
435
+ * Parse a single or an array of datasets saved using kepler.gl schema
436
+ * @param rawData
437
+ * @param schema
438
+ */
439
+ function processKeplerglDataset(rawData) {
440
+ var schema = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : _schemas.KeplerGlSchema;
441
+ if (!rawData) {
442
+ return null;
443
+ }
444
+ var results = schema.parseSavedData((0, _commonUtils.toArray)(rawData));
445
+ if (!results) {
446
+ return null;
447
+ }
448
+ return Array.isArray(rawData) ? results : results[0];
449
+ }
450
+
451
+ /**
452
+ * Process a loaders.gl [`ArrowTable`](https://loaders.gl/docs/specifications/category-table) (`{shape: 'arrow-table', data}`)
453
+ * and return a data object that can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap).
454
+ * Load arrow files with `GeoArrowLoader` and `{arrow: {shape: 'arrow-table'}}`.
455
+ *
456
+ * @param arrowTable ArrowTable to parse, see loaders.gl/schema
457
+ * @returns dataset containing `fields` and `rows` or null
458
+ * @public
459
+ * @example
460
+ * import {addDataToMap} from '@kepler.gl/actions';
461
+ * import {processArrowTable} from '@kepler.gl/processors';
462
+ * import {load} from '@loaders.gl/core';
463
+ * import {GeoArrowLoader} from '@loaders.gl/arrow';
464
+ *
465
+ * const arrowTable = await load(url, GeoArrowLoader, {arrow: {shape: 'arrow-table'}});
466
+ *
467
+ * dispatch(addDataToMap({
468
+ * datasets: {
469
+ * info: {id: 'arrow_dataset', label: 'My Arrow'},
470
+ * data: processArrowTable(arrowTable)
471
+ * }
472
+ * }));
473
+ */
474
+ function processArrowTable(arrowTable) {
475
+ // @ts-ignore - Unknown data type causing build failures
476
+ return processArrowBatches(arrowTable.data.batches);
477
+ }
478
+
479
+ /**
480
+ * Extracts GeoArrow metadata from an Apache Arrow table schema.
481
+ * For geoparquet files geoarrow metadata isn't present in fields, so extract extra info from schema.
482
+ * @param table The Apache Arrow table to extract metadata from.
483
+ * @returns An object mapping column names to their GeoArrow encoding type.
484
+ * @throws Logs an error message if parsing of metadata fails.
485
+ */
486
+ function getGeoArrowMetadataFromSchema(table) {
487
+ var geoArrowMetadata = {};
488
+ try {
489
+ var _table$schema$metadat;
490
+ var geoString = (_table$schema$metadat = table.schema.metadata) === null || _table$schema$metadat === void 0 ? void 0 : _table$schema$metadat.get('geo');
491
+ if (geoString) {
492
+ var parsedGeoString = JSON.parse(geoString);
493
+ if (parsedGeoString.columns) {
494
+ Object.keys(parsedGeoString.columns).forEach(function (columnName) {
495
+ var columnData = parsedGeoString.columns[columnName];
496
+ if ((columnData === null || columnData === void 0 ? void 0 : columnData.encoding) === 'WKB') {
497
+ geoArrowMetadata[columnName] = _constants.GEOARROW_EXTENSIONS.WKB;
498
+ }
499
+ // TODO potentially there are other types but no datasets to test
500
+ });
501
+ }
502
+ }
503
+ } catch (error) {
504
+ console.error('An error during arrow table schema metadata parsing');
505
+ }
506
+ return geoArrowMetadata;
507
+ }
508
+
509
+ /**
510
+ * Converts an Apache Arrow table schema into an array of Kepler.gl field objects.
511
+ * @param table The Apache Arrow table whose schema needs to be converted.
512
+ * @param fieldTypeSuggestions Optional mapping of field names to suggested field types.
513
+ * @returns An array of field objects suitable for Kepler.gl.
514
+ */
515
+ function arrowSchemaToFields(table) {
516
+ var fieldTypeSuggestions = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
517
+ var headerRow = table.schema.fields.map(function (f) {
518
+ return f.name;
519
+ });
520
+ var sample = (0, _commonUtils.getSampleForTypeAnalyzeArrow)(table, headerRow);
521
+ var keplerFields = (0, _commonUtils.getFieldsFromData)(sample, headerRow);
522
+ var geoArrowMetadata = getGeoArrowMetadataFromSchema(table);
523
+ return table.schema.fields.map(function (field, fieldIndex) {
524
+ var _field$metadata$get;
525
+ var type = (0, _utils.arrowDataTypeToFieldType)(field.type);
526
+ var analyzerType = (0, _utils.arrowDataTypeToAnalyzerDataType)(field.type);
527
+ var format = '';
528
+ var fieldTypeSuggestion = fieldTypeSuggestions[field.name];
529
+ var keplerField = keplerFields[fieldIndex];
530
+
531
+ // geometry fields produced by DuckDB's st_asgeojson()
532
+ if (fieldTypeSuggestion === 'JSON') {
533
+ type = _constants.ALL_FIELD_TYPES.geojson;
534
+ analyzerType = _typeAnalyzer.DATA_TYPES.GEOMETRY_FROM_STRING;
535
+ } else if (fieldTypeSuggestion === 'GEOMETRY' || (_field$metadata$get = field.metadata.get(_constants.GEOARROW_METADATA_KEY)) !== null && _field$metadata$get !== void 0 && _field$metadata$get.startsWith('geoarrow')) {
536
+ type = _constants.ALL_FIELD_TYPES.geoarrow;
537
+ analyzerType = _typeAnalyzer.DATA_TYPES.GEOMETRY;
538
+ } else if (geoArrowMetadata[field.name]) {
539
+ var _field$metadata;
540
+ type = _constants.ALL_FIELD_TYPES.geoarrow;
541
+ analyzerType = _typeAnalyzer.DATA_TYPES.GEOMETRY;
542
+ (_field$metadata = field.metadata) === null || _field$metadata === void 0 || _field$metadata.set(_constants.GEOARROW_METADATA_KEY, geoArrowMetadata[field.name]);
543
+ } else if (fieldTypeSuggestion === 'BLOB') {
544
+ // When arrow wkb column saved to DuckDB as BLOB without any metadata, then queried back
545
+ try {
546
+ var _table$getChildAt;
547
+ var data = (_table$getChildAt = table.getChildAt(fieldIndex)) === null || _table$getChildAt === void 0 ? void 0 : _table$getChildAt.get(0);
548
+ if (data) {
549
+ var binaryGeo = (0, _core.parseSync)(data, _wkt.WKBLoader);
550
+ if (binaryGeo) {
551
+ var _field$metadata2;
552
+ type = _constants.ALL_FIELD_TYPES.geoarrow;
553
+ analyzerType = _typeAnalyzer.DATA_TYPES.GEOMETRY;
554
+ (_field$metadata2 = field.metadata) === null || _field$metadata2 === void 0 || _field$metadata2.set(_constants.GEOARROW_METADATA_KEY, _constants.GEOARROW_EXTENSIONS.WKB);
555
+ }
556
+ }
557
+ } catch (error) {
558
+ // ignore, not WKB
559
+ }
560
+ } else if (fieldTypeSuggestion === 'VARCHAR' && (keplerField.analyzerType === _typeAnalyzer.DATA_TYPES.GEOMETRY || keplerField.analyzerType === _typeAnalyzer.DATA_TYPES.GEOMETRY_FROM_STRING)) {
561
+ // When wkb/wkt was saved as varchar in DuckDB
562
+ type = keplerField.type;
563
+ analyzerType = keplerField.analyzerType;
564
+ format = keplerField.format;
565
+ } else if (fieldTypeSuggestion === 'VARCHAR' && keplerField.type === _constants.ALL_FIELD_TYPES.h3) {
566
+ // when kepler detected h3 column using getFieldsFromData(), set type to h3 and analyzerType to H3
567
+ type = _constants.ALL_FIELD_TYPES.h3;
568
+ analyzerType = keplerField.analyzerType;
569
+ } else {
570
+ // TODO should we use Kepler getFieldsFromData instead
571
+ // of arrowDataTypeToFieldType for all fields?
572
+ if (keplerField.type === _constants.ALL_FIELD_TYPES.timestamp) {
573
+ type = keplerField.type;
574
+ analyzerType = keplerField.analyzerType;
575
+ format = keplerField.format;
576
+ }
577
+ }
578
+ return _objectSpread(_objectSpread({}, field), {}, {
579
+ name: field.name,
580
+ id: field.name,
581
+ displayName: field.name,
582
+ format: format,
583
+ fieldIdx: fieldIndex,
584
+ type: type,
585
+ analyzerType: analyzerType,
586
+ valueAccessor: function valueAccessor(dc) {
587
+ return function (d) {
588
+ return dc.valueAt(d.index, fieldIndex);
589
+ };
590
+ },
591
+ metadata: field.metadata
592
+ });
593
+ });
594
+ }
595
+ var CAST_BIGINTS = false;
596
+
597
+ /**
598
+ * Cast 64-bit integer Arrow columns (Int64, Uint64) to Float64 to avoid BigInt values
599
+ * that are incompatible with d3 scales, sorting, and other numeric operations.
600
+ * Mirrors the DuckDB approach of casting BIGINT/UBIGINT to DOUBLE.
601
+ *
602
+ * Left off so Uint64 values are not rounded through Float64. Samples are
603
+ * stringified for type-analyzer instead.
604
+ */
605
+ function castBigIntColumnsToFloat64(arrowTable) {
606
+ if (!CAST_BIGINTS) {
607
+ return arrowTable;
608
+ }
609
+ var needsCast = arrowTable.schema.fields.some(function (f) {
610
+ return arrow.DataType.isInt(f.type) && f.type.bitWidth === 64;
611
+ });
612
+ if (!needsCast) {
613
+ return arrowTable;
614
+ }
615
+ var newColumns = {};
616
+ for (var i = 0; i < arrowTable.numCols; i++) {
617
+ var field = arrowTable.schema.fields[i];
618
+ var col = arrowTable.getChildAt(i);
619
+ if (!col) continue;
620
+ if (arrow.DataType.isInt(field.type) && field.type.bitWidth === 64) {
621
+ var float64Array = new Float64Array(col.length);
622
+ for (var j = 0; j < col.length; j++) {
623
+ var val = col.get(j);
624
+ float64Array[j] = val === null ? NaN : Number(val);
625
+ }
626
+ newColumns[field.name] = arrow.makeVector(float64Array);
627
+ } else {
628
+ newColumns[field.name] = col;
629
+ }
630
+ }
631
+ return new arrow.Table(newColumns);
632
+ }
633
+ /**
634
+ * Parse arrow batches returned from parseInBatches()
635
+ *
636
+ * @param arrowBatches the arrow record batches to parse
637
+ * @param options optional processing flags
638
+ * @returns dataset containing `fields` and `rows` or null
639
+ */
640
+ function processArrowBatches(arrowBatches, options) {
641
+ if (arrowBatches.length === 0) {
642
+ return null;
643
+ }
644
+ var arrowTable = (options === null || options === void 0 ? void 0 : options.compact) === false ? castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)) : (0, _utils.compactArrowTable)(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)));
645
+ var fields = arrowSchemaToFields(arrowTable);
646
+ var cols = (0, _toConsumableArray2["default"])(Array(arrowTable.numCols).keys()).map(function (i) {
647
+ return arrowTable.getChildAt(i);
648
+ });
649
+
650
+ // return empty rows and use raw arrow table to construct column-wise data container
651
+ return {
652
+ fields: fields,
653
+ rows: [],
654
+ cols: cols,
655
+ metadata: arrowTable.schema.metadata,
656
+ // Save original arrow schema, for better ingestion into DuckDB.
657
+ // TODO consider returning arrowTable in cols, not an array of Vectors from arrowTable.
658
+ arrowSchema: arrowTable.schema
659
+ };
660
+ }
661
+ var DATASET_HANDLERS = exports.DATASET_HANDLERS = (0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])((0, _defineProperty2["default"])({}, _constants.DATASET_FORMATS.row, processRowObject), _constants.DATASET_FORMATS.geojson, processGeojson), _constants.DATASET_FORMATS.csv, processCsvData), _constants.DATASET_FORMATS.arrow, processArrowTable), _constants.DATASET_FORMATS.keplergl, processKeplerglDataset);
662
+ var Processors = exports.Processors = {
663
+ processGeojson: processGeojson,
664
+ processCsvData: processCsvData,
665
+ processArrowTable: processArrowTable,
666
+ processArrowBatches: processArrowBatches,
667
+ processRowObject: processRowObject,
668
+ processKeplerglJSON: processKeplerglJSON,
669
+ processKeplerglDataset: processKeplerglDataset,
670
+ analyzerTypeToFieldType: _commonUtils.analyzerTypeToFieldType,
671
+ getFieldsFromData: _commonUtils.getFieldsFromData,
672
+ parseCsvRowsByFieldType: parseCsvRowsByFieldType
673
+ };
674
+ //# sourceMappingURL=data-processor.js.map