@kepler.gl/processors 3.3.0-alpha.2 → 3.3.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/data-processor.js +674 -0
- package/dist/cjs/data-processor.js.map +1 -0
- package/dist/cjs/export/dataset-to-geojson.js +138 -0
- package/dist/cjs/export/dataset-to-geojson.js.map +1 -0
- package/dist/cjs/export/geopackage-writer.js +385 -0
- package/dist/cjs/export/geopackage-writer.js.map +1 -0
- package/dist/cjs/export/geoparquet-writer.js +351 -0
- package/dist/cjs/export/geoparquet-writer.js.map +1 -0
- package/dist/cjs/export/kml-writer.js +347 -0
- package/dist/cjs/export/kml-writer.js.map +1 -0
- package/dist/cjs/export/shapefile-writer.js +708 -0
- package/dist/cjs/export/shapefile-writer.js.map +1 -0
- package/dist/cjs/export/sql-js.js +95 -0
- package/dist/cjs/export/sql-js.js.map +1 -0
- package/dist/cjs/export/vector-export.js +187 -0
- package/dist/cjs/export/vector-export.js.map +1 -0
- package/dist/cjs/file-handler.js +997 -0
- package/dist/cjs/file-handler.js.map +1 -0
- package/dist/cjs/index.js +114 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/kepler-csv-loader.js +163 -0
- package/dist/cjs/kepler-csv-loader.js.map +1 -0
- package/dist/cjs/loader-registry.js +542 -0
- package/dist/cjs/loader-registry.js.map +1 -0
- package/dist/cjs/remote-file.js +675 -0
- package/dist/cjs/remote-file.js.map +1 -0
- package/dist/cjs/shapefile-files.js +281 -0
- package/dist/cjs/shapefile-files.js.map +1 -0
- package/dist/cjs/types.js +6 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/data-processor.d.ts +30 -16
- package/dist/esm/data-processor.js +655 -0
- package/dist/esm/data-processor.js.map +1 -0
- package/dist/esm/export/dataset-to-geojson.js +132 -0
- package/dist/esm/export/dataset-to-geojson.js.map +1 -0
- package/dist/esm/export/geopackage-writer.js +381 -0
- package/dist/esm/export/geopackage-writer.js.map +1 -0
- package/dist/esm/export/geoparquet-writer.js +336 -0
- package/dist/esm/export/geoparquet-writer.js.map +1 -0
- package/dist/esm/export/kml-writer.js +341 -0
- package/dist/esm/export/kml-writer.js.map +1 -0
- package/dist/esm/export/shapefile-writer.js +701 -0
- package/dist/esm/export/shapefile-writer.js.map +1 -0
- package/dist/esm/export/sql-js.js +80 -0
- package/dist/esm/export/sql-js.js.map +1 -0
- package/dist/esm/export/vector-export.js +176 -0
- package/dist/esm/export/vector-export.js.map +1 -0
- package/dist/esm/file-handler.js +972 -0
- package/dist/esm/file-handler.js.map +1 -0
- package/dist/esm/index.js +12 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/kepler-csv-loader.js +152 -0
- package/dist/esm/kepler-csv-loader.js.map +1 -0
- package/dist/esm/loader-registry.js +510 -0
- package/dist/esm/loader-registry.js.map +1 -0
- package/dist/esm/remote-file.js +664 -0
- package/dist/esm/remote-file.js.map +1 -0
- package/dist/esm/shapefile-files.js +264 -0
- package/dist/esm/shapefile-files.js.map +1 -0
- package/dist/esm/types.js +3 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/export/dataset-to-geojson.d.ts +12 -0
- package/dist/export/geopackage-writer.d.ts +28 -0
- package/dist/export/geoparquet-writer.d.ts +7 -0
- package/dist/export/kml-writer.d.ts +13 -0
- package/dist/export/shapefile-writer.d.ts +28 -0
- package/dist/export/sql-js.d.ts +8 -0
- package/dist/export/vector-export.d.ts +22 -0
- package/dist/file-handler.d.ts +55 -8
- package/dist/index.d.ts +6 -0
- package/dist/kepler-csv-loader.d.ts +12 -0
- package/dist/loader-registry.d.ts +29 -0
- package/dist/remote-file.d.ts +79 -0
- package/dist/shapefile-files.d.ts +39 -0
- package/dist/types.d.ts +13 -0
- package/package.json +44 -26
- package/dist/data-processor.js +0 -637
- package/dist/file-handler.js +0 -409
- package/dist/index.js +0 -39
- package/dist/types.js +0 -6
|
@@ -0,0 +1,655 @@
|
|
|
1
|
+
import _toConsumableArray from "@babel/runtime/helpers/esm/toConsumableArray";
|
|
2
|
+
import _typeof from "@babel/runtime/helpers/esm/typeof";
|
|
3
|
+
import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
|
|
4
|
+
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
|
|
5
|
+
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
|
|
6
|
+
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
|
|
7
|
+
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
|
|
8
|
+
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
|
|
9
|
+
// SPDX-License-Identifier: MIT
|
|
10
|
+
// Copyright contributors to the kepler.gl project
|
|
11
|
+
|
|
12
|
+
import * as arrow from 'apache-arrow';
|
|
13
|
+
import { csvParseRows, tsvParseRows, dsvFormat } from 'd3-dsv';
|
|
14
|
+
import { DATA_TYPES as AnalyzerDATA_TYPES } from 'type-analyzer';
|
|
15
|
+
import normalize from '@mapbox/geojson-normalize';
|
|
16
|
+
import { parseSync } from '@loaders.gl/core';
|
|
17
|
+
import { WKBLoader } from '@loaders.gl/wkt';
|
|
18
|
+
import { ALL_FIELD_TYPES, DATASET_FORMATS, GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY, GUIDES_FILE_FORMAT_DOC } from '@kepler.gl/constants';
|
|
19
|
+
import { arrowDataTypeToAnalyzerDataType, arrowDataTypeToFieldType, compactArrowTable, hasOwnProperty, isPlainObject, optionalMapInfo } from '@kepler.gl/utils';
|
|
20
|
+
import { analyzerTypeToFieldType, getSampleForTypeAnalyze, getSampleForTypeAnalyzeArrow, getFieldsFromData, h3IsValid, notNullorUndefined, toArray } from '@kepler.gl/common-utils';
|
|
21
|
+
import { KeplerGlSchema } from '@kepler.gl/schemas';
|
|
22
|
+
// if any of these value occurs in csv, parse it to null;
|
|
23
|
+
// const CSV_NULLS = ['', 'null', 'NULL', 'Null', 'NaN', '/N'];
|
|
24
|
+
// matches empty string
|
|
25
|
+
export var CSV_NULLS = /^(null|NULL|Null|NaN|\/N||)$/;
|
|
26
|
+
var SUPPORTED_DELIMITERS = [',', '\t', ';', '|'];
|
|
27
|
+
function getRowParser(delimiter) {
|
|
28
|
+
if (delimiter === ',') return csvParseRows;
|
|
29
|
+
if (delimiter === '\t') return tsvParseRows;
|
|
30
|
+
return dsvFormat(delimiter).parseRows;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Detect the delimiter used in a DSV string by checking the first line.
|
|
35
|
+
* Returns the delimiter that produces the most columns (minimum 2).
|
|
36
|
+
* Falls back to comma if no delimiter produces multiple columns.
|
|
37
|
+
*/
|
|
38
|
+
export function detectDelimiter(rawData) {
|
|
39
|
+
var newlineIdx = rawData.indexOf('\n');
|
|
40
|
+
var firstLine = newlineIdx === -1 ? rawData : rawData.slice(0, newlineIdx);
|
|
41
|
+
if (!firstLine) return ',';
|
|
42
|
+
var bestDelimiter = ',';
|
|
43
|
+
var bestCount = 1;
|
|
44
|
+
var _iterator = _createForOfIteratorHelper(SUPPORTED_DELIMITERS),
|
|
45
|
+
_step;
|
|
46
|
+
try {
|
|
47
|
+
for (_iterator.s(); !(_step = _iterator.n()).done;) {
|
|
48
|
+
var _parsed$;
|
|
49
|
+
var delimiter = _step.value;
|
|
50
|
+
var parsed = getRowParser(delimiter)(firstLine);
|
|
51
|
+
var count = ((_parsed$ = parsed[0]) === null || _parsed$ === void 0 ? void 0 : _parsed$.length) || 0;
|
|
52
|
+
if (count > bestCount) {
|
|
53
|
+
bestCount = count;
|
|
54
|
+
bestDelimiter = delimiter;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
} catch (err) {
|
|
58
|
+
_iterator.e(err);
|
|
59
|
+
} finally {
|
|
60
|
+
_iterator.f();
|
|
61
|
+
}
|
|
62
|
+
return bestDelimiter;
|
|
63
|
+
}
|
|
64
|
+
function tryParseJsonString(str) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(str);
|
|
67
|
+
} catch (e) {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export var PARSE_FIELD_VALUE_FROM_STRING = _defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty({}, ALL_FIELD_TYPES["boolean"], {
|
|
72
|
+
valid: function valid(d) {
|
|
73
|
+
return typeof d === 'boolean';
|
|
74
|
+
},
|
|
75
|
+
parse: function parse(d) {
|
|
76
|
+
var s = String(d).toLowerCase();
|
|
77
|
+
return s === 'true' || s === 'yes' || s === '1';
|
|
78
|
+
}
|
|
79
|
+
}), ALL_FIELD_TYPES.integer, {
|
|
80
|
+
// @ts-ignore
|
|
81
|
+
valid: function valid(d) {
|
|
82
|
+
return parseInt(d, 10) === d;
|
|
83
|
+
},
|
|
84
|
+
// @ts-ignore
|
|
85
|
+
parse: function parse(d) {
|
|
86
|
+
return parseInt(d, 10);
|
|
87
|
+
}
|
|
88
|
+
}), ALL_FIELD_TYPES.timestamp, {
|
|
89
|
+
valid: function valid(d, field) {
|
|
90
|
+
return ['x', 'X'].includes(field.format) ? typeof d === 'number' : typeof d === 'string';
|
|
91
|
+
},
|
|
92
|
+
parse: function parse(d, field) {
|
|
93
|
+
return ['x', 'X'].includes(field.format) ? Number(d) : d;
|
|
94
|
+
}
|
|
95
|
+
}), ALL_FIELD_TYPES.real, {
|
|
96
|
+
// @ts-ignore
|
|
97
|
+
valid: function valid(d) {
|
|
98
|
+
return parseFloat(d) === d;
|
|
99
|
+
},
|
|
100
|
+
// Note this will result in NaN for some string
|
|
101
|
+
parse: parseFloat
|
|
102
|
+
}), ALL_FIELD_TYPES.object, {
|
|
103
|
+
valid: isPlainObject,
|
|
104
|
+
parse: tryParseJsonString
|
|
105
|
+
}), ALL_FIELD_TYPES.array, {
|
|
106
|
+
valid: Array.isArray,
|
|
107
|
+
parse: tryParseJsonString
|
|
108
|
+
}), ALL_FIELD_TYPES.h3, {
|
|
109
|
+
valid: function valid(d) {
|
|
110
|
+
return h3IsValid(d);
|
|
111
|
+
},
|
|
112
|
+
parse: function parse(d) {
|
|
113
|
+
return d;
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Process csv data, output a data object with `{fields: [], rows: []}`.
|
|
119
|
+
* The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)
|
|
120
|
+
* @param rawData raw csv string
|
|
121
|
+
* @returns data object `{fields: [], rows: []}` can be passed to addDataToMaps
|
|
122
|
+
* @public
|
|
123
|
+
* @example
|
|
124
|
+
* import {processCsvData} from '@kepler.gl/processors';
|
|
125
|
+
*
|
|
126
|
+
* const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date
|
|
127
|
+
* 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23
|
|
128
|
+
* 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23
|
|
129
|
+
* 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23
|
|
130
|
+
* 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`
|
|
131
|
+
*
|
|
132
|
+
* const dataset = {
|
|
133
|
+
* info: {id: 'test_data', label: 'My Csv'},
|
|
134
|
+
* data: processCsvData(testData)
|
|
135
|
+
* };
|
|
136
|
+
*
|
|
137
|
+
* dispatch(addDataToMap({
|
|
138
|
+
* datasets: [dataset],
|
|
139
|
+
* options: {centerMap: true, readOnly: true}
|
|
140
|
+
* }));
|
|
141
|
+
*/
|
|
142
|
+
export function processCsvData(rawData, header) {
|
|
143
|
+
var rows;
|
|
144
|
+
var headerRow;
|
|
145
|
+
if (typeof rawData === 'string') {
|
|
146
|
+
var delimiter = detectDelimiter(rawData);
|
|
147
|
+
var _parsedRows = getRowParser(delimiter)(rawData);
|
|
148
|
+
if (!Array.isArray(_parsedRows) || _parsedRows.length < 2) {
|
|
149
|
+
throw new Error('processCsvData Failed: delimited text is empty or has no data rows');
|
|
150
|
+
}
|
|
151
|
+
headerRow = _parsedRows[0];
|
|
152
|
+
rows = _parsedRows.slice(1);
|
|
153
|
+
} else if (Array.isArray(rawData) && rawData.length) {
|
|
154
|
+
rows = rawData;
|
|
155
|
+
headerRow = header;
|
|
156
|
+
if (!Array.isArray(headerRow)) {
|
|
157
|
+
// if data is passed in as array of rows and missing header
|
|
158
|
+
// assume first row is header
|
|
159
|
+
// @ts-ignore
|
|
160
|
+
headerRow = rawData[0];
|
|
161
|
+
rows = rawData.slice(1);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (!rows || !headerRow) {
|
|
165
|
+
throw new Error('invalid input passed to processCsvData');
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// here we assume the csv file that people uploaded will have first row
|
|
169
|
+
// as name of the column
|
|
170
|
+
|
|
171
|
+
cleanUpFalsyCsvValue(rows);
|
|
172
|
+
// No need to run type detection on every data point
|
|
173
|
+
// here we get a list of none null values to run analyze on
|
|
174
|
+
var sample = getSampleForTypeAnalyze({
|
|
175
|
+
fields: headerRow,
|
|
176
|
+
rows: rows
|
|
177
|
+
});
|
|
178
|
+
var fields = getFieldsFromData(sample, headerRow);
|
|
179
|
+
var parsedRows = parseRowsByFields(rows, fields);
|
|
180
|
+
return {
|
|
181
|
+
fields: fields,
|
|
182
|
+
rows: parsedRows
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`
|
|
188
|
+
* @param rows
|
|
189
|
+
* @param fields
|
|
190
|
+
*/
|
|
191
|
+
export function parseRowsByFields(rows, fields) {
|
|
192
|
+
// Edit rows in place
|
|
193
|
+
var geojsonFieldIdx = fields.findIndex(function (f) {
|
|
194
|
+
return f.name === '_geojson';
|
|
195
|
+
});
|
|
196
|
+
fields.forEach(parseCsvRowsByFieldType.bind(null, rows, geojsonFieldIdx));
|
|
197
|
+
return rows;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Convert falsy value in csv including `'', 'null', 'NULL', 'Null', 'NaN'` to `null`,
|
|
202
|
+
* so that type-analyzer won't detect it as string
|
|
203
|
+
*
|
|
204
|
+
* @param rows
|
|
205
|
+
*/
|
|
206
|
+
function cleanUpFalsyCsvValue(rows) {
|
|
207
|
+
var re = new RegExp(CSV_NULLS, 'g');
|
|
208
|
+
for (var i = 0; i < rows.length; i++) {
|
|
209
|
+
for (var j = 0; j < rows[i].length; j++) {
|
|
210
|
+
// analyzer will set any fields to 'string' if there are empty values
|
|
211
|
+
// which will be parsed as '' by d3.csv
|
|
212
|
+
// here we parse empty data as null
|
|
213
|
+
// TODO: create warning when deltect `CSV_NULLS` in the data
|
|
214
|
+
if (typeof rows[i][j] === 'string' && rows[i][j].match(re)) {
|
|
215
|
+
rows[i][j] = null;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Process uploaded csv file to parse value by field type
|
|
223
|
+
*
|
|
224
|
+
* @param rows
|
|
225
|
+
* @param geoFieldIdx field index
|
|
226
|
+
* @param field
|
|
227
|
+
* @param i
|
|
228
|
+
*/
|
|
229
|
+
export function parseCsvRowsByFieldType(rows, geoFieldIdx, field, i) {
|
|
230
|
+
var parser = PARSE_FIELD_VALUE_FROM_STRING[field.type];
|
|
231
|
+
if (parser) {
|
|
232
|
+
// check first not null value of it's already parsed
|
|
233
|
+
var first = rows.find(function (r) {
|
|
234
|
+
return notNullorUndefined(r[i]);
|
|
235
|
+
});
|
|
236
|
+
if (!first || parser.valid(first[i], field)) {
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
rows.forEach(function (row) {
|
|
240
|
+
// parse string value based on field type
|
|
241
|
+
if (row[i] !== null) {
|
|
242
|
+
row[i] = parser.parse(row[i], field);
|
|
243
|
+
if (geoFieldIdx > -1 && isPlainObject(row[geoFieldIdx]) &&
|
|
244
|
+
// @ts-ignore
|
|
245
|
+
hasOwnProperty(row[geoFieldIdx], 'properties')) {
|
|
246
|
+
// @ts-ignore
|
|
247
|
+
row[geoFieldIdx].properties[field.name] = row[i];
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/* eslint-enable complexity */
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
|
|
258
|
+
* NOTE: This function may mutate input.
|
|
259
|
+
* @param rawData an array of row object, each object should have the same number of keys
|
|
260
|
+
* @returns dataset containing `fields` and `rows`
|
|
261
|
+
* @public
|
|
262
|
+
* @example
|
|
263
|
+
* import {addDataToMap} from '@kepler.gl/actions';
|
|
264
|
+
* import {processRowObject} from '@kepler.gl/processors';
|
|
265
|
+
*
|
|
266
|
+
* const data = [
|
|
267
|
+
* {lat: 31.27, lng: 127.56, value: 3},
|
|
268
|
+
* {lat: 31.22, lng: 126.26, value: 1}
|
|
269
|
+
* ];
|
|
270
|
+
*
|
|
271
|
+
* dispatch(addDataToMap({
|
|
272
|
+
* datasets: {
|
|
273
|
+
* info: {label: 'My Data', id: 'my_data'},
|
|
274
|
+
* data: processRowObject(data)
|
|
275
|
+
* }
|
|
276
|
+
* }));
|
|
277
|
+
*/
|
|
278
|
+
export function processRowObject(rawData) {
|
|
279
|
+
if (!Array.isArray(rawData)) {
|
|
280
|
+
return null;
|
|
281
|
+
} else if (!rawData.length) {
|
|
282
|
+
// data is empty
|
|
283
|
+
return {
|
|
284
|
+
fields: [],
|
|
285
|
+
rows: []
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
var firstRow = rawData[0];
|
|
289
|
+
var keys = Object.keys(firstRow); // [lat, lng, value]
|
|
290
|
+
var rows = rawData.map(function (d) {
|
|
291
|
+
return keys.map(function (key) {
|
|
292
|
+
return d[key];
|
|
293
|
+
});
|
|
294
|
+
}); // [[31.27, 127.56, 3]]
|
|
295
|
+
|
|
296
|
+
// row object can still contain values like `Null` or `N/A`
|
|
297
|
+
cleanUpFalsyCsvValue(rows);
|
|
298
|
+
return processCsvData(rows, keys);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),
|
|
303
|
+
* output a data object with `{fields: [], rows: []}`.
|
|
304
|
+
* The data object can be wrapped in a `dataset` and passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
|
|
305
|
+
* NOTE: This function may mutate input.
|
|
306
|
+
*
|
|
307
|
+
* @param rawData raw geojson feature collection
|
|
308
|
+
* @returns dataset containing `fields` and `rows`
|
|
309
|
+
* @public
|
|
310
|
+
* @example
|
|
311
|
+
* import {addDataToMap} from '@kepler.gl/actions';
|
|
312
|
+
* import {processGeojson} from '@kepler.gl/processors';
|
|
313
|
+
*
|
|
314
|
+
* const geojson = {
|
|
315
|
+
* "type" : "FeatureCollection",
|
|
316
|
+
* "features" : [{
|
|
317
|
+
* "type" : "Feature",
|
|
318
|
+
* "properties" : {
|
|
319
|
+
* "capacity" : "10",
|
|
320
|
+
* "type" : "U-Rack"
|
|
321
|
+
* },
|
|
322
|
+
* "geometry" : {
|
|
323
|
+
* "type" : "Point",
|
|
324
|
+
* "coordinates" : [ -71.073283, 42.417500 ]
|
|
325
|
+
* }
|
|
326
|
+
* }]
|
|
327
|
+
* };
|
|
328
|
+
*
|
|
329
|
+
* dispatch(addDataToMap({
|
|
330
|
+
* datasets: {
|
|
331
|
+
* info: {
|
|
332
|
+
* label: 'Sample Taxi Trips in New York City',
|
|
333
|
+
* id: 'test_trip_data'
|
|
334
|
+
* },
|
|
335
|
+
* data: processGeojson(geojson)
|
|
336
|
+
* }
|
|
337
|
+
* }));
|
|
338
|
+
*/
|
|
339
|
+
export function processGeojson(rawData) {
|
|
340
|
+
var normalizedGeojson = normalize(rawData);
|
|
341
|
+
if (!normalizedGeojson || !Array.isArray(normalizedGeojson.features)) {
|
|
342
|
+
throw new Error("Read File Failed: File is not a valid GeoJSON. Read more about [supported file format](".concat(GUIDES_FILE_FORMAT_DOC, ")"));
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// getting all feature fields
|
|
346
|
+
var allDataRows = [];
|
|
347
|
+
for (var i = 0; i < normalizedGeojson.features.length; i++) {
|
|
348
|
+
var f = normalizedGeojson.features[i];
|
|
349
|
+
if (f.geometry) {
|
|
350
|
+
var row = _objectSpread({
|
|
351
|
+
// Reserve the first field for geometry to preserve existing field indices.
|
|
352
|
+
_geojson: f
|
|
353
|
+
}, f.properties || {});
|
|
354
|
+
// A property with the reserved name must not replace the feature.
|
|
355
|
+
row._geojson = f;
|
|
356
|
+
allDataRows.push(row);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
// get all the field
|
|
360
|
+
var fields = allDataRows.reduce(function (accu, curr) {
|
|
361
|
+
Object.keys(curr).forEach(function (key) {
|
|
362
|
+
if (!accu.includes(key)) {
|
|
363
|
+
accu.push(key);
|
|
364
|
+
}
|
|
365
|
+
});
|
|
366
|
+
return accu;
|
|
367
|
+
}, []);
|
|
368
|
+
|
|
369
|
+
// make sure each feature has exact same fields
|
|
370
|
+
allDataRows.forEach(function (d) {
|
|
371
|
+
fields.forEach(function (f) {
|
|
372
|
+
if (!(f in d)) {
|
|
373
|
+
d[f] = null;
|
|
374
|
+
if (d._geojson.properties) {
|
|
375
|
+
d._geojson.properties[f] = null;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
});
|
|
379
|
+
});
|
|
380
|
+
return processRowObject(allDataRows);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
|
|
385
|
+
* The json object should contain `datasets` and `config`.
|
|
386
|
+
* @param rawData
|
|
387
|
+
* @param schema
|
|
388
|
+
* @returns datasets and config `{datasets: {}, config: {}}`
|
|
389
|
+
* @public
|
|
390
|
+
* @example
|
|
391
|
+
* import {addDataToMap} from '@kepler.gl/actions';
|
|
392
|
+
* import {processKeplerglJSON} from '@kepler.gl/processors';
|
|
393
|
+
*
|
|
394
|
+
* dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
|
|
395
|
+
*/
|
|
396
|
+
function savedMapDisplayInfo(info) {
|
|
397
|
+
var saved = info && _typeof(info) === 'object' ? optionalMapInfo(info) : undefined;
|
|
398
|
+
// Both keys are always present: setMapInfo merges, so omitting one would keep
|
|
399
|
+
// the previously loaded map's value instead of clearing it.
|
|
400
|
+
return {
|
|
401
|
+
title: saved === null || saved === void 0 ? void 0 : saved.title,
|
|
402
|
+
description: saved === null || saved === void 0 ? void 0 : saved.description
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
export function processKeplerglJSON(rawData) {
|
|
406
|
+
var schema = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : KeplerGlSchema;
|
|
407
|
+
if (!rawData) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
return _objectSpread(_objectSpread({}, schema.load(rawData.datasets, rawData.config)), {}, {
|
|
411
|
+
info: savedMapDisplayInfo(rawData.info)
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Parse a single or an array of datasets saved using kepler.gl schema
|
|
417
|
+
* @param rawData
|
|
418
|
+
* @param schema
|
|
419
|
+
*/
|
|
420
|
+
export function processKeplerglDataset(rawData) {
|
|
421
|
+
var schema = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : KeplerGlSchema;
|
|
422
|
+
if (!rawData) {
|
|
423
|
+
return null;
|
|
424
|
+
}
|
|
425
|
+
var results = schema.parseSavedData(toArray(rawData));
|
|
426
|
+
if (!results) {
|
|
427
|
+
return null;
|
|
428
|
+
}
|
|
429
|
+
return Array.isArray(rawData) ? results : results[0];
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Process a loaders.gl [`ArrowTable`](https://loaders.gl/docs/specifications/category-table) (`{shape: 'arrow-table', data}`)
|
|
434
|
+
* and return a data object that can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap).
|
|
435
|
+
* Load arrow files with `GeoArrowLoader` and `{arrow: {shape: 'arrow-table'}}`.
|
|
436
|
+
*
|
|
437
|
+
* @param arrowTable ArrowTable to parse, see loaders.gl/schema
|
|
438
|
+
* @returns dataset containing `fields` and `rows` or null
|
|
439
|
+
* @public
|
|
440
|
+
* @example
|
|
441
|
+
* import {addDataToMap} from '@kepler.gl/actions';
|
|
442
|
+
* import {processArrowTable} from '@kepler.gl/processors';
|
|
443
|
+
* import {load} from '@loaders.gl/core';
|
|
444
|
+
* import {GeoArrowLoader} from '@loaders.gl/arrow';
|
|
445
|
+
*
|
|
446
|
+
* const arrowTable = await load(url, GeoArrowLoader, {arrow: {shape: 'arrow-table'}});
|
|
447
|
+
*
|
|
448
|
+
* dispatch(addDataToMap({
|
|
449
|
+
* datasets: {
|
|
450
|
+
* info: {id: 'arrow_dataset', label: 'My Arrow'},
|
|
451
|
+
* data: processArrowTable(arrowTable)
|
|
452
|
+
* }
|
|
453
|
+
* }));
|
|
454
|
+
*/
|
|
455
|
+
export function processArrowTable(arrowTable) {
|
|
456
|
+
// @ts-ignore - Unknown data type causing build failures
|
|
457
|
+
return processArrowBatches(arrowTable.data.batches);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* Extracts GeoArrow metadata from an Apache Arrow table schema.
|
|
462
|
+
* For geoparquet files geoarrow metadata isn't present in fields, so extract extra info from schema.
|
|
463
|
+
* @param table The Apache Arrow table to extract metadata from.
|
|
464
|
+
* @returns An object mapping column names to their GeoArrow encoding type.
|
|
465
|
+
* @throws Logs an error message if parsing of metadata fails.
|
|
466
|
+
*/
|
|
467
|
+
export function getGeoArrowMetadataFromSchema(table) {
|
|
468
|
+
var geoArrowMetadata = {};
|
|
469
|
+
try {
|
|
470
|
+
var _table$schema$metadat;
|
|
471
|
+
var geoString = (_table$schema$metadat = table.schema.metadata) === null || _table$schema$metadat === void 0 ? void 0 : _table$schema$metadat.get('geo');
|
|
472
|
+
if (geoString) {
|
|
473
|
+
var parsedGeoString = JSON.parse(geoString);
|
|
474
|
+
if (parsedGeoString.columns) {
|
|
475
|
+
Object.keys(parsedGeoString.columns).forEach(function (columnName) {
|
|
476
|
+
var columnData = parsedGeoString.columns[columnName];
|
|
477
|
+
if ((columnData === null || columnData === void 0 ? void 0 : columnData.encoding) === 'WKB') {
|
|
478
|
+
geoArrowMetadata[columnName] = GEOARROW_EXTENSIONS.WKB;
|
|
479
|
+
}
|
|
480
|
+
// TODO potentially there are other types but no datasets to test
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
} catch (error) {
|
|
485
|
+
console.error('An error during arrow table schema metadata parsing');
|
|
486
|
+
}
|
|
487
|
+
return geoArrowMetadata;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Converts an Apache Arrow table schema into an array of Kepler.gl field objects.
|
|
492
|
+
* @param table The Apache Arrow table whose schema needs to be converted.
|
|
493
|
+
* @param fieldTypeSuggestions Optional mapping of field names to suggested field types.
|
|
494
|
+
* @returns An array of field objects suitable for Kepler.gl.
|
|
495
|
+
*/
|
|
496
|
+
export function arrowSchemaToFields(table) {
|
|
497
|
+
var fieldTypeSuggestions = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
|
|
498
|
+
var headerRow = table.schema.fields.map(function (f) {
|
|
499
|
+
return f.name;
|
|
500
|
+
});
|
|
501
|
+
var sample = getSampleForTypeAnalyzeArrow(table, headerRow);
|
|
502
|
+
var keplerFields = getFieldsFromData(sample, headerRow);
|
|
503
|
+
var geoArrowMetadata = getGeoArrowMetadataFromSchema(table);
|
|
504
|
+
return table.schema.fields.map(function (field, fieldIndex) {
|
|
505
|
+
var _field$metadata$get;
|
|
506
|
+
var type = arrowDataTypeToFieldType(field.type);
|
|
507
|
+
var analyzerType = arrowDataTypeToAnalyzerDataType(field.type);
|
|
508
|
+
var format = '';
|
|
509
|
+
var fieldTypeSuggestion = fieldTypeSuggestions[field.name];
|
|
510
|
+
var keplerField = keplerFields[fieldIndex];
|
|
511
|
+
|
|
512
|
+
// geometry fields produced by DuckDB's st_asgeojson()
|
|
513
|
+
if (fieldTypeSuggestion === 'JSON') {
|
|
514
|
+
type = ALL_FIELD_TYPES.geojson;
|
|
515
|
+
analyzerType = AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING;
|
|
516
|
+
} else if (fieldTypeSuggestion === 'GEOMETRY' || (_field$metadata$get = field.metadata.get(GEOARROW_METADATA_KEY)) !== null && _field$metadata$get !== void 0 && _field$metadata$get.startsWith('geoarrow')) {
|
|
517
|
+
type = ALL_FIELD_TYPES.geoarrow;
|
|
518
|
+
analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
|
|
519
|
+
} else if (geoArrowMetadata[field.name]) {
|
|
520
|
+
var _field$metadata;
|
|
521
|
+
type = ALL_FIELD_TYPES.geoarrow;
|
|
522
|
+
analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
|
|
523
|
+
(_field$metadata = field.metadata) === null || _field$metadata === void 0 || _field$metadata.set(GEOARROW_METADATA_KEY, geoArrowMetadata[field.name]);
|
|
524
|
+
} else if (fieldTypeSuggestion === 'BLOB') {
|
|
525
|
+
// When arrow wkb column saved to DuckDB as BLOB without any metadata, then queried back
|
|
526
|
+
try {
|
|
527
|
+
var _table$getChildAt;
|
|
528
|
+
var data = (_table$getChildAt = table.getChildAt(fieldIndex)) === null || _table$getChildAt === void 0 ? void 0 : _table$getChildAt.get(0);
|
|
529
|
+
if (data) {
|
|
530
|
+
var binaryGeo = parseSync(data, WKBLoader);
|
|
531
|
+
if (binaryGeo) {
|
|
532
|
+
var _field$metadata2;
|
|
533
|
+
type = ALL_FIELD_TYPES.geoarrow;
|
|
534
|
+
analyzerType = AnalyzerDATA_TYPES.GEOMETRY;
|
|
535
|
+
(_field$metadata2 = field.metadata) === null || _field$metadata2 === void 0 || _field$metadata2.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.WKB);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
} catch (error) {
|
|
539
|
+
// ignore, not WKB
|
|
540
|
+
}
|
|
541
|
+
} else if (fieldTypeSuggestion === 'VARCHAR' && (keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY || keplerField.analyzerType === AnalyzerDATA_TYPES.GEOMETRY_FROM_STRING)) {
|
|
542
|
+
// When wkb/wkt was saved as varchar in DuckDB
|
|
543
|
+
type = keplerField.type;
|
|
544
|
+
analyzerType = keplerField.analyzerType;
|
|
545
|
+
format = keplerField.format;
|
|
546
|
+
} else if (fieldTypeSuggestion === 'VARCHAR' && keplerField.type === ALL_FIELD_TYPES.h3) {
|
|
547
|
+
// when kepler detected h3 column using getFieldsFromData(), set type to h3 and analyzerType to H3
|
|
548
|
+
type = ALL_FIELD_TYPES.h3;
|
|
549
|
+
analyzerType = keplerField.analyzerType;
|
|
550
|
+
} else {
|
|
551
|
+
// TODO should we use Kepler getFieldsFromData instead
|
|
552
|
+
// of arrowDataTypeToFieldType for all fields?
|
|
553
|
+
if (keplerField.type === ALL_FIELD_TYPES.timestamp) {
|
|
554
|
+
type = keplerField.type;
|
|
555
|
+
analyzerType = keplerField.analyzerType;
|
|
556
|
+
format = keplerField.format;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
return _objectSpread(_objectSpread({}, field), {}, {
|
|
560
|
+
name: field.name,
|
|
561
|
+
id: field.name,
|
|
562
|
+
displayName: field.name,
|
|
563
|
+
format: format,
|
|
564
|
+
fieldIdx: fieldIndex,
|
|
565
|
+
type: type,
|
|
566
|
+
analyzerType: analyzerType,
|
|
567
|
+
valueAccessor: function valueAccessor(dc) {
|
|
568
|
+
return function (d) {
|
|
569
|
+
return dc.valueAt(d.index, fieldIndex);
|
|
570
|
+
};
|
|
571
|
+
},
|
|
572
|
+
metadata: field.metadata
|
|
573
|
+
});
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
var CAST_BIGINTS = false;
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Cast 64-bit integer Arrow columns (Int64, Uint64) to Float64 to avoid BigInt values
|
|
580
|
+
* that are incompatible with d3 scales, sorting, and other numeric operations.
|
|
581
|
+
* Mirrors the DuckDB approach of casting BIGINT/UBIGINT to DOUBLE.
|
|
582
|
+
*
|
|
583
|
+
* Left off so Uint64 values are not rounded through Float64. Samples are
|
|
584
|
+
* stringified for type-analyzer instead.
|
|
585
|
+
*/
|
|
586
|
+
function castBigIntColumnsToFloat64(arrowTable) {
|
|
587
|
+
if (!CAST_BIGINTS) {
|
|
588
|
+
return arrowTable;
|
|
589
|
+
}
|
|
590
|
+
var needsCast = arrowTable.schema.fields.some(function (f) {
|
|
591
|
+
return arrow.DataType.isInt(f.type) && f.type.bitWidth === 64;
|
|
592
|
+
});
|
|
593
|
+
if (!needsCast) {
|
|
594
|
+
return arrowTable;
|
|
595
|
+
}
|
|
596
|
+
var newColumns = {};
|
|
597
|
+
for (var i = 0; i < arrowTable.numCols; i++) {
|
|
598
|
+
var field = arrowTable.schema.fields[i];
|
|
599
|
+
var col = arrowTable.getChildAt(i);
|
|
600
|
+
if (!col) continue;
|
|
601
|
+
if (arrow.DataType.isInt(field.type) && field.type.bitWidth === 64) {
|
|
602
|
+
var float64Array = new Float64Array(col.length);
|
|
603
|
+
for (var j = 0; j < col.length; j++) {
|
|
604
|
+
var val = col.get(j);
|
|
605
|
+
float64Array[j] = val === null ? NaN : Number(val);
|
|
606
|
+
}
|
|
607
|
+
newColumns[field.name] = arrow.makeVector(float64Array);
|
|
608
|
+
} else {
|
|
609
|
+
newColumns[field.name] = col;
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
return new arrow.Table(newColumns);
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* Parse arrow batches returned from parseInBatches()
|
|
616
|
+
*
|
|
617
|
+
* @param arrowBatches the arrow record batches to parse
|
|
618
|
+
* @param options optional processing flags
|
|
619
|
+
* @returns dataset containing `fields` and `rows` or null
|
|
620
|
+
*/
|
|
621
|
+
export function processArrowBatches(arrowBatches, options) {
|
|
622
|
+
if (arrowBatches.length === 0) {
|
|
623
|
+
return null;
|
|
624
|
+
}
|
|
625
|
+
var arrowTable = (options === null || options === void 0 ? void 0 : options.compact) === false ? castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)) : compactArrowTable(castBigIntColumnsToFloat64(new arrow.Table(arrowBatches)));
|
|
626
|
+
var fields = arrowSchemaToFields(arrowTable);
|
|
627
|
+
var cols = _toConsumableArray(Array(arrowTable.numCols).keys()).map(function (i) {
|
|
628
|
+
return arrowTable.getChildAt(i);
|
|
629
|
+
});
|
|
630
|
+
|
|
631
|
+
// return empty rows and use raw arrow table to construct column-wise data container
|
|
632
|
+
return {
|
|
633
|
+
fields: fields,
|
|
634
|
+
rows: [],
|
|
635
|
+
cols: cols,
|
|
636
|
+
metadata: arrowTable.schema.metadata,
|
|
637
|
+
// Save original arrow schema, for better ingestion into DuckDB.
|
|
638
|
+
// TODO consider returning arrowTable in cols, not an array of Vectors from arrowTable.
|
|
639
|
+
arrowSchema: arrowTable.schema
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
export var DATASET_HANDLERS = _defineProperty(_defineProperty(_defineProperty(_defineProperty(_defineProperty({}, DATASET_FORMATS.row, processRowObject), DATASET_FORMATS.geojson, processGeojson), DATASET_FORMATS.csv, processCsvData), DATASET_FORMATS.arrow, processArrowTable), DATASET_FORMATS.keplergl, processKeplerglDataset);
|
|
643
|
+
export var Processors = {
|
|
644
|
+
processGeojson: processGeojson,
|
|
645
|
+
processCsvData: processCsvData,
|
|
646
|
+
processArrowTable: processArrowTable,
|
|
647
|
+
processArrowBatches: processArrowBatches,
|
|
648
|
+
processRowObject: processRowObject,
|
|
649
|
+
processKeplerglJSON: processKeplerglJSON,
|
|
650
|
+
processKeplerglDataset: processKeplerglDataset,
|
|
651
|
+
analyzerTypeToFieldType: analyzerTypeToFieldType,
|
|
652
|
+
getFieldsFromData: getFieldsFromData,
|
|
653
|
+
parseCsvRowsByFieldType: parseCsvRowsByFieldType
|
|
654
|
+
};
|
|
655
|
+
//# sourceMappingURL=data-processor.js.map
|