@kepler.gl/processors 3.1.0-alpha.1 → 3.1.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/data-processor.d.ts +196 -0
- package/dist/data-processor.js +457 -0
- package/dist/file-handler.d.ts +49 -0
- package/dist/file-handler.js +416 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +39 -0
- package/dist/types.d.ts +13 -0
- package/dist/types.js +6 -0
- package/package.json +12 -9
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import * as arrow from 'apache-arrow';
|
|
2
|
+
import { ArrowTable } from '@loaders.gl/schema';
|
|
3
|
+
import { ProcessorResult, Field } from '@kepler.gl/types';
|
|
4
|
+
import { isPlainObject } from '@kepler.gl/utils';
|
|
5
|
+
import { analyzerTypeToFieldType, getFieldsFromData } from '@kepler.gl/common-utils';
|
|
6
|
+
import { ParsedDataset, SavedMap, LoadedMap } from '@kepler.gl/schemas';
|
|
7
|
+
export declare const CSV_NULLS: RegExp;
|
|
8
|
+
declare function tryParseJsonString(str: any): any;
|
|
9
|
+
export declare const PARSE_FIELD_VALUE_FROM_STRING: {
|
|
10
|
+
boolean: {
|
|
11
|
+
valid: (d: unknown) => boolean;
|
|
12
|
+
parse: (d: unknown) => boolean;
|
|
13
|
+
};
|
|
14
|
+
integer: {
|
|
15
|
+
valid: (d: unknown) => boolean;
|
|
16
|
+
parse: (d: unknown) => number;
|
|
17
|
+
};
|
|
18
|
+
timestamp: {
|
|
19
|
+
valid: (d: unknown, field: Field) => boolean;
|
|
20
|
+
parse: (d: any, field: Field) => any;
|
|
21
|
+
};
|
|
22
|
+
real: {
|
|
23
|
+
valid: (d: unknown) => boolean;
|
|
24
|
+
parse: typeof parseFloat;
|
|
25
|
+
};
|
|
26
|
+
object: {
|
|
27
|
+
valid: typeof isPlainObject;
|
|
28
|
+
parse: typeof tryParseJsonString;
|
|
29
|
+
};
|
|
30
|
+
array: {
|
|
31
|
+
valid: (arg: any) => arg is any[];
|
|
32
|
+
parse: typeof tryParseJsonString;
|
|
33
|
+
};
|
|
34
|
+
h3: {
|
|
35
|
+
valid: (d: any) => boolean;
|
|
36
|
+
parse: (d: any) => any;
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* Process csv data, output a data object with `{fields: [], rows: []}`.
|
|
41
|
+
* The data object can be wrapped in a `dataset` and pass to [`addDataToMap`](../actions/actions.md#adddatatomap)
|
|
42
|
+
* @param rawData raw csv string
|
|
43
|
+
* @returns data object `{fields: [], rows: []}` can be passed to addDataToMaps
|
|
44
|
+
* @public
|
|
45
|
+
* @example
|
|
46
|
+
* import {processCsvData} from 'kepler.gl/processors';
|
|
47
|
+
*
|
|
48
|
+
* const testData = `gps_data.utc_timestamp,gps_data.lat,gps_data.lng,gps_data.types,epoch,has_result,id,time,begintrip_ts_utc,begintrip_ts_local,date
|
|
49
|
+
* 2016-09-17 00:09:55,29.9900937,31.2590542,driver_analytics,1472688000000,False,1,2016-09-23T00:00:00.000Z,2016-10-01 09:41:39+00:00,2016-10-01 09:41:39+00:00,2016-09-23
|
|
50
|
+
* 2016-09-17 00:10:56,29.9927699,31.2461142,driver_analytics,1472688000000,False,2,2016-09-23T00:00:00.000Z,2016-10-01 09:46:37+00:00,2016-10-01 16:46:37+00:00,2016-09-23
|
|
51
|
+
* 2016-09-17 00:11:56,29.9907261,31.2312742,driver_analytics,1472688000000,False,3,2016-09-23T00:00:00.000Z,,,2016-09-23
|
|
52
|
+
* 2016-09-17 00:12:58,29.9870074,31.2175827,driver_analytics,1472688000000,False,4,2016-09-23T00:00:00.000Z,,,2016-09-23`
|
|
53
|
+
*
|
|
54
|
+
* const dataset = {
|
|
55
|
+
* info: {id: 'test_data', label: 'My Csv'},
|
|
56
|
+
* data: processCsvData(testData)
|
|
57
|
+
* };
|
|
58
|
+
*
|
|
59
|
+
* dispatch(addDataToMap({
|
|
60
|
+
* datasets: [dataset],
|
|
61
|
+
* options: {centerMap: true, readOnly: true}
|
|
62
|
+
* }));
|
|
63
|
+
*/
|
|
64
|
+
export declare function processCsvData(rawData: unknown[][] | string, header?: string[]): ProcessorResult;
|
|
65
|
+
/**
|
|
66
|
+
* Parse rows of csv by analyzed field types. So that `'1'` -> `1`, `'True'` -> `true`
|
|
67
|
+
* @param rows
|
|
68
|
+
* @param fields
|
|
69
|
+
*/
|
|
70
|
+
export declare function parseRowsByFields(rows: any[][], fields: Field[]): any[][];
|
|
71
|
+
/**
|
|
72
|
+
* Process uploaded csv file to parse value by field type
|
|
73
|
+
*
|
|
74
|
+
* @param rows
|
|
75
|
+
* @param geoFieldIdx field index
|
|
76
|
+
* @param field
|
|
77
|
+
* @param i
|
|
78
|
+
*/
|
|
79
|
+
export declare function parseCsvRowsByFieldType(rows: unknown[][], geoFieldIdx: number, field: Field, i: number): void;
|
|
80
|
+
/**
|
|
81
|
+
* Process data where each row is an object, output can be passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
|
|
82
|
+
* NOTE: This function may mutate input.
|
|
83
|
+
* @param rawData an array of row object, each object should have the same number of keys
|
|
84
|
+
* @returns dataset containing `fields` and `rows`
|
|
85
|
+
* @public
|
|
86
|
+
* @example
|
|
87
|
+
* import {addDataToMap} from 'kepler.gl/actions';
|
|
88
|
+
* import {processRowObject} from 'kepler.gl/processors';
|
|
89
|
+
*
|
|
90
|
+
* const data = [
|
|
91
|
+
* {lat: 31.27, lng: 127.56, value: 3},
|
|
92
|
+
* {lat: 31.22, lng: 126.26, value: 1}
|
|
93
|
+
* ];
|
|
94
|
+
*
|
|
95
|
+
* dispatch(addDataToMap({
|
|
96
|
+
* datasets: {
|
|
97
|
+
* info: {label: 'My Data', id: 'my_data'},
|
|
98
|
+
* data: processRowObject(data)
|
|
99
|
+
* }
|
|
100
|
+
* }));
|
|
101
|
+
*/
|
|
102
|
+
export declare function processRowObject(rawData: unknown[]): ProcessorResult;
|
|
103
|
+
/**
|
|
104
|
+
* Process GeoJSON [`FeatureCollection`](http://wiki.geojson.org/GeoJSON_draft_version_6#FeatureCollection),
|
|
105
|
+
* output a data object with `{fields: [], rows: []}`.
|
|
106
|
+
* The data object can be wrapped in a `dataset` and passed to [`addDataToMap`](../actions/actions.md#adddatatomap)
|
|
107
|
+
* NOTE: This function may mutate input.
|
|
108
|
+
*
|
|
109
|
+
* @param rawData raw geojson feature collection
|
|
110
|
+
* @returns dataset containing `fields` and `rows`
|
|
111
|
+
* @public
|
|
112
|
+
* @example
|
|
113
|
+
* import {addDataToMap} from 'kepler.gl/actions';
|
|
114
|
+
* import {processGeojson} from 'kepler.gl/processors';
|
|
115
|
+
*
|
|
116
|
+
* const geojson = {
|
|
117
|
+
* "type" : "FeatureCollection",
|
|
118
|
+
* "features" : [{
|
|
119
|
+
* "type" : "Feature",
|
|
120
|
+
* "properties" : {
|
|
121
|
+
* "capacity" : "10",
|
|
122
|
+
* "type" : "U-Rack"
|
|
123
|
+
* },
|
|
124
|
+
* "geometry" : {
|
|
125
|
+
* "type" : "Point",
|
|
126
|
+
* "coordinates" : [ -71.073283, 42.417500 ]
|
|
127
|
+
* }
|
|
128
|
+
* }]
|
|
129
|
+
* };
|
|
130
|
+
*
|
|
131
|
+
* dispatch(addDataToMap({
|
|
132
|
+
* datasets: {
|
|
133
|
+
* info: {
|
|
134
|
+
* label: 'Sample Taxi Trips in New York City',
|
|
135
|
+
* id: 'test_trip_data'
|
|
136
|
+
* },
|
|
137
|
+
* data: processGeojson(geojson)
|
|
138
|
+
* }
|
|
139
|
+
* }));
|
|
140
|
+
*/
|
|
141
|
+
export declare function processGeojson(rawData: unknown): ProcessorResult;
|
|
142
|
+
/**
|
|
143
|
+
* Process saved kepler.gl json to be pass to [`addDataToMap`](../actions/actions.md#adddatatomap).
|
|
144
|
+
* The json object should contain `datasets` and `config`.
|
|
145
|
+
* @param rawData
|
|
146
|
+
* @param schema
|
|
147
|
+
* @returns datasets and config `{datasets: {}, config: {}}`
|
|
148
|
+
* @public
|
|
149
|
+
* @example
|
|
150
|
+
* import {addDataToMap} from 'kepler.gl/actions';
|
|
151
|
+
* import {processKeplerglJSON} from 'kepler.gl/processors';
|
|
152
|
+
*
|
|
153
|
+
* dispatch(addDataToMap(processKeplerglJSON(keplerGlJson)));
|
|
154
|
+
*/
|
|
155
|
+
export declare function processKeplerglJSON(rawData: SavedMap, schema?: import("@kepler.gl/schemas").KeplerGLSchemaClass): LoadedMap | null;
|
|
156
|
+
/**
|
|
157
|
+
* Parse a single or an array of datasets saved using kepler.gl schema
|
|
158
|
+
* @param rawData
|
|
159
|
+
* @param schema
|
|
160
|
+
*/
|
|
161
|
+
export declare function processKeplerglDataset(rawData: object | object[], schema?: import("@kepler.gl/schemas").KeplerGLSchemaClass): ParsedDataset | ParsedDataset[] | null;
|
|
162
|
+
/**
|
|
163
|
+
* Parse arrow table and return a dataset
|
|
164
|
+
*
|
|
165
|
+
* @param arrowTable ArrowTable to parse, see loaders.gl/schema
|
|
166
|
+
* @returns dataset containing `fields` and `rows` or null
|
|
167
|
+
*/
|
|
168
|
+
export declare function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null;
|
|
169
|
+
export declare function arrowSchemaToFields(schema: arrow.Schema): Field[];
|
|
170
|
+
/**
|
|
171
|
+
* Parse arrow batches returned from parseInBatches()
|
|
172
|
+
*
|
|
173
|
+
* @param arrowTable the arrow table to parse
|
|
174
|
+
* @returns dataset containing `fields` and `rows` or null
|
|
175
|
+
*/
|
|
176
|
+
export declare function processArrowBatches(arrowBatches: arrow.RecordBatch[]): ProcessorResult | null;
|
|
177
|
+
export declare const DATASET_HANDLERS: {
|
|
178
|
+
row: typeof processRowObject;
|
|
179
|
+
geojson: typeof processGeojson;
|
|
180
|
+
csv: typeof processCsvData;
|
|
181
|
+
arrow: typeof processArrowTable;
|
|
182
|
+
keplergl: typeof processKeplerglDataset;
|
|
183
|
+
};
|
|
184
|
+
export declare const Processors: {
|
|
185
|
+
processGeojson: typeof processGeojson;
|
|
186
|
+
processCsvData: typeof processCsvData;
|
|
187
|
+
processArrowTable: typeof processArrowTable;
|
|
188
|
+
processArrowBatches: typeof processArrowBatches;
|
|
189
|
+
processRowObject: typeof processRowObject;
|
|
190
|
+
processKeplerglJSON: typeof processKeplerglJSON;
|
|
191
|
+
processKeplerglDataset: typeof processKeplerglDataset;
|
|
192
|
+
analyzerTypeToFieldType: typeof analyzerTypeToFieldType;
|
|
193
|
+
getFieldsFromData: typeof getFieldsFromData;
|
|
194
|
+
parseCsvRowsByFieldType: typeof parseCsvRowsByFieldType;
|
|
195
|
+
};
|
|
196
|
+
export {};
|