@loaders.gl/arrow 4.2.0-alpha.4 → 4.2.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts +1 -1
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js +30 -19
- package/dist/arrow-worker.js +4 -2
- package/dist/arrow-writer.d.ts +1 -1
- package/dist/arrow-writer.d.ts.map +1 -1
- package/dist/arrow-writer.js +23 -16
- package/dist/dist.dev.js +361 -333
- package/dist/dist.min.js +23 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +345 -288
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js +151 -124
- package/dist/geoarrow/get-arrow-bounds.js +32 -22
- package/dist/geoarrow-loader.d.ts +1 -1
- package/dist/geoarrow-loader.d.ts.map +1 -1
- package/dist/geoarrow-loader.js +22 -15
- package/dist/geoarrow-writer.js +25 -16
- package/dist/index.cjs +55 -71
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +16 -16
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -1
- package/dist/lib/arrow-table-batch.d.ts +1 -1
- package/dist/lib/arrow-table-batch.d.ts.map +1 -1
- package/dist/lib/arrow-table-batch.js +55 -51
- package/dist/lib/arrow-table.js +3 -1
- package/dist/lib/encode-arrow.d.ts +1 -1
- package/dist/lib/encode-arrow.d.ts.map +1 -1
- package/dist/lib/encode-arrow.js +31 -16
- package/dist/lib/encode-geoarrow.d.ts +1 -1
- package/dist/lib/encode-geoarrow.d.ts.map +1 -1
- package/dist/lib/encode-geoarrow.js +31 -16
- package/dist/parsers/parse-arrow-in-batches.d.ts +2 -2
- package/dist/parsers/parse-arrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-in-batches.js +49 -21
- package/dist/parsers/parse-arrow-sync.d.ts +1 -1
- package/dist/parsers/parse-arrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-arrow-sync.js +26 -22
- package/dist/parsers/parse-geoarrow-in-batches.d.ts +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-in-batches.js +7 -2
- package/dist/parsers/parse-geoarrow-sync.d.ts +1 -1
- package/dist/parsers/parse-geoarrow-sync.d.ts.map +1 -1
- package/dist/parsers/parse-geoarrow-sync.js +12 -10
- package/dist/schema/arrow-type-utils.js +36 -9
- package/dist/schema/convert-arrow-schema.js +253 -240
- package/dist/tables/convert-arrow-to-columnar-table.d.ts +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-columnar-table.js +22 -13
- package/dist/tables/convert-arrow-to-geojson-table.d.ts +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.d.ts.map +1 -1
- package/dist/tables/convert-arrow-to-geojson-table.js +47 -32
- package/dist/tables/convert-columnar-to-row-table.js +22 -16
- package/dist/tables/convert-table-to-arrow.js +58 -1
- package/dist/triangulate-on-worker.js +24 -19
- package/dist/triangulation-worker.js +6 -1
- package/dist/types.js +8 -6
- package/dist/workers/arrow-worker.js +3 -1
- package/dist/workers/hard-clone.js +93 -49
- package/dist/workers/triangulation-worker-node.d.ts +1 -1
- package/dist/workers/triangulation-worker-node.d.ts.map +1 -1
- package/dist/workers/triangulation-worker-node.js +3 -1
- package/dist/workers/triangulation-worker.js +48 -48
- package/package.json +13 -8
- package/dist/arrow-loader.js.map +0 -1
- package/dist/arrow-writer.js.map +0 -1
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +0 -1
- package/dist/geoarrow/convert-geoarrow-to-geojson-geometry.js.map +0 -1
- package/dist/geoarrow/get-arrow-bounds.js.map +0 -1
- package/dist/geoarrow-loader.js.map +0 -1
- package/dist/geoarrow-writer.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/arrow-table-batch.js.map +0 -1
- package/dist/lib/arrow-table.js.map +0 -1
- package/dist/lib/encode-arrow.js.map +0 -1
- package/dist/lib/encode-geoarrow.js.map +0 -1
- package/dist/parsers/parse-arrow-in-batches.js.map +0 -1
- package/dist/parsers/parse-arrow-sync.js.map +0 -1
- package/dist/parsers/parse-geoarrow-in-batches.js.map +0 -1
- package/dist/parsers/parse-geoarrow-sync.js.map +0 -1
- package/dist/schema/arrow-type-utils.js.map +0 -1
- package/dist/schema/convert-arrow-schema.js.map +0 -1
- package/dist/tables/convert-arrow-to-columnar-table.js.map +0 -1
- package/dist/tables/convert-arrow-to-geojson-table.js.map +0 -1
- package/dist/tables/convert-columnar-to-row-table.js.map +0 -1
- package/dist/tables/convert-table-to-arrow.js.map +0 -1
- package/dist/triangulate-on-worker.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/workers/arrow-worker.js.map +0 -1
- package/dist/workers/hard-clone.js.map +0 -1
- package/dist/workers/triangulation-worker-node.js.map +0 -1
- package/dist/workers/triangulation-worker.js.map +0 -1
|
@@ -1,259 +1,272 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import * as arrow from 'apache-arrow';
|
|
5
|
+
/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
|
|
2
6
|
export function serializeArrowSchema(arrowSchema) {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
+
return {
|
|
8
|
+
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
9
|
+
metadata: serializeArrowMetadata(arrowSchema.metadata)
|
|
10
|
+
};
|
|
7
11
|
}
|
|
12
|
+
/** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
|
|
8
13
|
export function deserializeArrowSchema(schema) {
|
|
9
|
-
|
|
14
|
+
return new arrow.Schema(schema.fields.map((field) => deserializeArrowField(field)), deserializeArrowMetadata(schema.metadata));
|
|
10
15
|
}
|
|
16
|
+
/** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
|
|
11
17
|
export function serializeArrowMetadata(arrowMetadata) {
|
|
12
|
-
|
|
18
|
+
return Object.fromEntries(arrowMetadata);
|
|
13
19
|
}
|
|
20
|
+
/** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
|
|
14
21
|
export function deserializeArrowMetadata(metadata) {
|
|
15
|
-
|
|
22
|
+
return metadata ? new Map(Object.entries(metadata)) : new Map();
|
|
16
23
|
}
|
|
24
|
+
/** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
|
|
17
25
|
export function serializeArrowField(field) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
return {
|
|
27
|
+
name: field.name,
|
|
28
|
+
type: serializeArrowType(field.type),
|
|
29
|
+
nullable: field.nullable,
|
|
30
|
+
metadata: serializeArrowMetadata(field.metadata)
|
|
31
|
+
};
|
|
24
32
|
}
|
|
33
|
+
/** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
|
|
25
34
|
export function deserializeArrowField(field) {
|
|
26
|
-
|
|
35
|
+
return new arrow.Field(field.name, deserializeArrowType(field.type), field.nullable, deserializeArrowMetadata(field.metadata));
|
|
27
36
|
}
|
|
37
|
+
/** Converts a serializable loaders.gl data type to hydrated arrow data type */
|
|
38
|
+
// eslint-disable-next-line complexity
|
|
28
39
|
export function serializeArrowType(arrowType) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
40
|
+
switch (arrowType.constructor) {
|
|
41
|
+
case arrow.Null:
|
|
42
|
+
return 'null';
|
|
43
|
+
case arrow.Binary:
|
|
44
|
+
return 'binary';
|
|
45
|
+
case arrow.Bool:
|
|
46
|
+
return 'bool';
|
|
47
|
+
case arrow.Int:
|
|
48
|
+
const intType = arrowType;
|
|
49
|
+
return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
|
|
50
|
+
case arrow.Int8:
|
|
51
|
+
return 'int8';
|
|
52
|
+
case arrow.Int16:
|
|
53
|
+
return 'int16';
|
|
54
|
+
case arrow.Int32:
|
|
55
|
+
return 'int32';
|
|
56
|
+
case arrow.Int64:
|
|
57
|
+
return 'int64';
|
|
58
|
+
case arrow.Uint8:
|
|
59
|
+
return 'uint8';
|
|
60
|
+
case arrow.Uint16:
|
|
61
|
+
return 'uint16';
|
|
62
|
+
case arrow.Uint32:
|
|
63
|
+
return 'uint32';
|
|
64
|
+
case arrow.Uint64:
|
|
65
|
+
return 'uint64';
|
|
66
|
+
case arrow.Float:
|
|
67
|
+
const precision = arrowType.precision;
|
|
68
|
+
// return `float(precision + 1) * 16`;
|
|
69
|
+
switch (precision) {
|
|
70
|
+
case arrow.Precision.HALF:
|
|
71
|
+
return 'float16';
|
|
72
|
+
case arrow.Precision.SINGLE:
|
|
73
|
+
return 'float32';
|
|
74
|
+
case arrow.Precision.DOUBLE:
|
|
75
|
+
return 'float64';
|
|
76
|
+
default:
|
|
77
|
+
return 'float16';
|
|
78
|
+
}
|
|
79
|
+
case arrow.Float16:
|
|
80
|
+
return 'float16';
|
|
81
|
+
case arrow.Float32:
|
|
82
|
+
return 'float32';
|
|
83
|
+
case arrow.Float64:
|
|
84
|
+
return 'float64';
|
|
85
|
+
case arrow.Utf8:
|
|
86
|
+
return 'utf8';
|
|
87
|
+
case arrow.Decimal:
|
|
88
|
+
const decimal = arrowType;
|
|
89
|
+
return {
|
|
90
|
+
type: 'decimal',
|
|
91
|
+
bitWidth: decimal.bitWidth,
|
|
92
|
+
precision: decimal.precision,
|
|
93
|
+
scale: decimal.scale
|
|
94
|
+
};
|
|
95
|
+
case arrow.Date_:
|
|
96
|
+
const dateUnit = arrowType.unit;
|
|
97
|
+
return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
|
|
98
|
+
case arrow.DateDay:
|
|
99
|
+
return 'date-day';
|
|
100
|
+
case arrow.DateMillisecond:
|
|
101
|
+
return 'date-millisecond';
|
|
102
|
+
case arrow.Time:
|
|
103
|
+
const timeUnit = arrowType.unit;
|
|
104
|
+
switch (timeUnit) {
|
|
105
|
+
case arrow.TimeUnit.SECOND:
|
|
106
|
+
return 'time-second';
|
|
107
|
+
case arrow.TimeUnit.MILLISECOND:
|
|
108
|
+
return 'time-millisecond';
|
|
109
|
+
case arrow.TimeUnit.MICROSECOND:
|
|
110
|
+
return 'time-microsecond';
|
|
111
|
+
case arrow.TimeUnit.NANOSECOND:
|
|
112
|
+
return 'time-nanosecond';
|
|
113
|
+
default:
|
|
114
|
+
return 'time-second';
|
|
115
|
+
}
|
|
116
|
+
case arrow.TimeMillisecond:
|
|
117
|
+
return 'time-millisecond';
|
|
118
|
+
case arrow.TimeSecond:
|
|
119
|
+
return 'time-second';
|
|
120
|
+
case arrow.TimeMicrosecond:
|
|
121
|
+
return 'time-microsecond';
|
|
122
|
+
case arrow.TimeNanosecond:
|
|
123
|
+
return 'time-nanosecond';
|
|
124
|
+
case arrow.Timestamp:
|
|
125
|
+
const timeStampUnit = arrowType.unit;
|
|
126
|
+
switch (timeStampUnit) {
|
|
127
|
+
case arrow.TimeUnit.SECOND:
|
|
128
|
+
return 'timestamp-second';
|
|
129
|
+
case arrow.TimeUnit.MILLISECOND:
|
|
130
|
+
return 'timestamp-millisecond';
|
|
131
|
+
case arrow.TimeUnit.MICROSECOND:
|
|
132
|
+
return 'timestamp-microsecond';
|
|
133
|
+
case arrow.TimeUnit.NANOSECOND:
|
|
134
|
+
return 'timestamp-nanosecond';
|
|
135
|
+
default:
|
|
136
|
+
return 'timestamp-second';
|
|
137
|
+
}
|
|
138
|
+
case arrow.TimestampSecond:
|
|
139
|
+
return 'timestamp-second';
|
|
140
|
+
case arrow.TimestampMillisecond:
|
|
141
|
+
return 'timestamp-millisecond';
|
|
142
|
+
case arrow.TimestampMicrosecond:
|
|
143
|
+
return 'timestamp-microsecond';
|
|
144
|
+
case arrow.TimestampNanosecond:
|
|
145
|
+
return 'timestamp-nanosecond';
|
|
146
|
+
case arrow.Interval:
|
|
147
|
+
const intervalUnit = arrowType.unit;
|
|
148
|
+
switch (intervalUnit) {
|
|
149
|
+
case arrow.IntervalUnit.DAY_TIME:
|
|
150
|
+
return 'interval-daytime';
|
|
151
|
+
case arrow.IntervalUnit.YEAR_MONTH:
|
|
152
|
+
return 'interval-yearmonth';
|
|
153
|
+
default:
|
|
154
|
+
return 'interval-daytime';
|
|
155
|
+
}
|
|
156
|
+
case arrow.IntervalDayTime:
|
|
157
|
+
return 'interval-daytime';
|
|
158
|
+
case arrow.IntervalYearMonth:
|
|
159
|
+
return 'interval-yearmonth';
|
|
160
|
+
case arrow.Map_:
|
|
161
|
+
const mapType = arrowType;
|
|
162
|
+
return {
|
|
163
|
+
type: 'map',
|
|
164
|
+
keysSorted: mapType.keysSorted,
|
|
165
|
+
children: mapType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
166
|
+
};
|
|
167
|
+
case arrow.List:
|
|
168
|
+
const listType = arrowType;
|
|
169
|
+
const listField = listType.valueField;
|
|
170
|
+
return {
|
|
171
|
+
type: 'list',
|
|
172
|
+
children: [serializeArrowField(listField)]
|
|
173
|
+
};
|
|
174
|
+
case arrow.FixedSizeList:
|
|
175
|
+
const fixedSizeList = arrowType;
|
|
176
|
+
return {
|
|
177
|
+
type: 'fixed-size-list',
|
|
178
|
+
listSize: fixedSizeList.listSize,
|
|
179
|
+
children: [serializeArrowField(fixedSizeList.children[0])]
|
|
180
|
+
};
|
|
181
|
+
case arrow.Struct:
|
|
182
|
+
const structType = arrowType;
|
|
183
|
+
return {
|
|
184
|
+
type: 'struct',
|
|
185
|
+
children: structType.children.map((arrowField) => serializeArrowField(arrowField))
|
|
186
|
+
};
|
|
64
187
|
default:
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
case arrow.Float16:
|
|
68
|
-
return 'float16';
|
|
69
|
-
case arrow.Float32:
|
|
70
|
-
return 'float32';
|
|
71
|
-
case arrow.Float64:
|
|
72
|
-
return 'float64';
|
|
73
|
-
case arrow.Utf8:
|
|
74
|
-
return 'utf8';
|
|
75
|
-
case arrow.Decimal:
|
|
76
|
-
const decimal = arrowType;
|
|
77
|
-
return {
|
|
78
|
-
type: 'decimal',
|
|
79
|
-
bitWidth: decimal.bitWidth,
|
|
80
|
-
precision: decimal.precision,
|
|
81
|
-
scale: decimal.scale
|
|
82
|
-
};
|
|
83
|
-
case arrow.Date_:
|
|
84
|
-
const dateUnit = arrowType.unit;
|
|
85
|
-
return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
|
|
86
|
-
case arrow.DateDay:
|
|
87
|
-
return 'date-day';
|
|
88
|
-
case arrow.DateMillisecond:
|
|
89
|
-
return 'date-millisecond';
|
|
90
|
-
case arrow.Time:
|
|
91
|
-
const timeUnit = arrowType.unit;
|
|
92
|
-
switch (timeUnit) {
|
|
93
|
-
case arrow.TimeUnit.SECOND:
|
|
94
|
-
return 'time-second';
|
|
95
|
-
case arrow.TimeUnit.MILLISECOND:
|
|
96
|
-
return 'time-millisecond';
|
|
97
|
-
case arrow.TimeUnit.MICROSECOND:
|
|
98
|
-
return 'time-microsecond';
|
|
99
|
-
case arrow.TimeUnit.NANOSECOND:
|
|
100
|
-
return 'time-nanosecond';
|
|
101
|
-
default:
|
|
102
|
-
return 'time-second';
|
|
103
|
-
}
|
|
104
|
-
case arrow.TimeMillisecond:
|
|
105
|
-
return 'time-millisecond';
|
|
106
|
-
case arrow.TimeSecond:
|
|
107
|
-
return 'time-second';
|
|
108
|
-
case arrow.TimeMicrosecond:
|
|
109
|
-
return 'time-microsecond';
|
|
110
|
-
case arrow.TimeNanosecond:
|
|
111
|
-
return 'time-nanosecond';
|
|
112
|
-
case arrow.Timestamp:
|
|
113
|
-
const timeStampUnit = arrowType.unit;
|
|
114
|
-
switch (timeStampUnit) {
|
|
115
|
-
case arrow.TimeUnit.SECOND:
|
|
116
|
-
return 'timestamp-second';
|
|
117
|
-
case arrow.TimeUnit.MILLISECOND:
|
|
118
|
-
return 'timestamp-millisecond';
|
|
119
|
-
case arrow.TimeUnit.MICROSECOND:
|
|
120
|
-
return 'timestamp-microsecond';
|
|
121
|
-
case arrow.TimeUnit.NANOSECOND:
|
|
122
|
-
return 'timestamp-nanosecond';
|
|
123
|
-
default:
|
|
124
|
-
return 'timestamp-second';
|
|
125
|
-
}
|
|
126
|
-
case arrow.TimestampSecond:
|
|
127
|
-
return 'timestamp-second';
|
|
128
|
-
case arrow.TimestampMillisecond:
|
|
129
|
-
return 'timestamp-millisecond';
|
|
130
|
-
case arrow.TimestampMicrosecond:
|
|
131
|
-
return 'timestamp-microsecond';
|
|
132
|
-
case arrow.TimestampNanosecond:
|
|
133
|
-
return 'timestamp-nanosecond';
|
|
134
|
-
case arrow.Interval:
|
|
135
|
-
const intervalUnit = arrowType.unit;
|
|
136
|
-
switch (intervalUnit) {
|
|
137
|
-
case arrow.IntervalUnit.DAY_TIME:
|
|
138
|
-
return 'interval-daytime';
|
|
139
|
-
case arrow.IntervalUnit.YEAR_MONTH:
|
|
140
|
-
return 'interval-yearmonth';
|
|
141
|
-
default:
|
|
142
|
-
return 'interval-daytime';
|
|
143
|
-
}
|
|
144
|
-
case arrow.IntervalDayTime:
|
|
145
|
-
return 'interval-daytime';
|
|
146
|
-
case arrow.IntervalYearMonth:
|
|
147
|
-
return 'interval-yearmonth';
|
|
148
|
-
case arrow.Map_:
|
|
149
|
-
const mapType = arrowType;
|
|
150
|
-
return {
|
|
151
|
-
type: 'map',
|
|
152
|
-
keysSorted: mapType.keysSorted,
|
|
153
|
-
children: mapType.children.map(arrowField => serializeArrowField(arrowField))
|
|
154
|
-
};
|
|
155
|
-
case arrow.List:
|
|
156
|
-
const listType = arrowType;
|
|
157
|
-
const listField = listType.valueField;
|
|
158
|
-
return {
|
|
159
|
-
type: 'list',
|
|
160
|
-
children: [serializeArrowField(listField)]
|
|
161
|
-
};
|
|
162
|
-
case arrow.FixedSizeList:
|
|
163
|
-
const fixedSizeList = arrowType;
|
|
164
|
-
return {
|
|
165
|
-
type: 'fixed-size-list',
|
|
166
|
-
listSize: fixedSizeList.listSize,
|
|
167
|
-
children: [serializeArrowField(fixedSizeList.children[0])]
|
|
168
|
-
};
|
|
169
|
-
case arrow.Struct:
|
|
170
|
-
const structType = arrowType;
|
|
171
|
-
return {
|
|
172
|
-
type: 'struct',
|
|
173
|
-
children: structType.children.map(arrowField => serializeArrowField(arrowField))
|
|
174
|
-
};
|
|
175
|
-
default:
|
|
176
|
-
throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
|
|
177
|
-
}
|
|
188
|
+
throw new Error(`arrow type not supported: ${arrowType.constructor.name}`);
|
|
189
|
+
}
|
|
178
190
|
}
|
|
191
|
+
/** Converts a serializable loaders.gl data type to hydrated arrow data type */
|
|
192
|
+
// eslint-disable-next-line complexity
|
|
179
193
|
export function deserializeArrowType(dataType) {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
194
|
+
if (typeof dataType === 'object') {
|
|
195
|
+
switch (dataType.type) {
|
|
196
|
+
case 'decimal':
|
|
197
|
+
return new arrow.Decimal(dataType.precision, dataType.scale, dataType.bitWidth);
|
|
198
|
+
case 'map':
|
|
199
|
+
let children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
|
|
200
|
+
return new arrow.Map_(children, dataType.keysSorted);
|
|
201
|
+
case 'list':
|
|
202
|
+
const field = deserializeArrowField(dataType.children[0]);
|
|
203
|
+
return new arrow.List(field);
|
|
204
|
+
case 'fixed-size-list':
|
|
205
|
+
const child = deserializeArrowField(dataType.children[0]);
|
|
206
|
+
return new arrow.FixedSizeList(dataType.listSize, child);
|
|
207
|
+
case 'struct':
|
|
208
|
+
children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
|
|
209
|
+
return new arrow.Struct(children);
|
|
210
|
+
default:
|
|
211
|
+
throw new Error('array type not supported');
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
switch (dataType) {
|
|
215
|
+
case 'null':
|
|
216
|
+
return new arrow.Null();
|
|
217
|
+
case 'binary':
|
|
218
|
+
return new arrow.Binary();
|
|
219
|
+
case 'bool':
|
|
220
|
+
return new arrow.Bool();
|
|
221
|
+
case 'int8':
|
|
222
|
+
return new arrow.Int8();
|
|
223
|
+
case 'int16':
|
|
224
|
+
return new arrow.Int16();
|
|
225
|
+
case 'int32':
|
|
226
|
+
return new arrow.Int32();
|
|
227
|
+
case 'int64':
|
|
228
|
+
return new arrow.Int64();
|
|
229
|
+
case 'uint8':
|
|
230
|
+
return new arrow.Uint8();
|
|
231
|
+
case 'uint16':
|
|
232
|
+
return new arrow.Uint16();
|
|
233
|
+
case 'uint32':
|
|
234
|
+
return new arrow.Uint32();
|
|
235
|
+
case 'uint64':
|
|
236
|
+
return new arrow.Uint64();
|
|
237
|
+
case 'float16':
|
|
238
|
+
return new arrow.Float16();
|
|
239
|
+
case 'float32':
|
|
240
|
+
return new arrow.Float32();
|
|
241
|
+
case 'float64':
|
|
242
|
+
return new arrow.Float64();
|
|
243
|
+
case 'utf8':
|
|
244
|
+
return new arrow.Utf8();
|
|
245
|
+
case 'date-day':
|
|
246
|
+
return new arrow.DateDay();
|
|
247
|
+
case 'date-millisecond':
|
|
248
|
+
return new arrow.DateMillisecond();
|
|
249
|
+
case 'time-second':
|
|
250
|
+
return new arrow.TimeSecond();
|
|
251
|
+
case 'time-millisecond':
|
|
252
|
+
return new arrow.TimeMillisecond();
|
|
253
|
+
case 'time-microsecond':
|
|
254
|
+
return new arrow.TimeMicrosecond();
|
|
255
|
+
case 'time-nanosecond':
|
|
256
|
+
return new arrow.TimeNanosecond();
|
|
257
|
+
case 'timestamp-second':
|
|
258
|
+
return new arrow.TimestampSecond();
|
|
259
|
+
case 'timestamp-millisecond':
|
|
260
|
+
return new arrow.TimestampMillisecond();
|
|
261
|
+
case 'timestamp-microsecond':
|
|
262
|
+
return new arrow.TimestampMicrosecond();
|
|
263
|
+
case 'timestamp-nanosecond':
|
|
264
|
+
return new arrow.TimestampNanosecond();
|
|
265
|
+
case 'interval-daytime':
|
|
266
|
+
return new arrow.IntervalDayTime();
|
|
267
|
+
case 'interval-yearmonth':
|
|
268
|
+
return new arrow.IntervalYearMonth();
|
|
269
|
+
default:
|
|
270
|
+
throw new Error('array type not supported');
|
|
198
271
|
}
|
|
199
|
-
}
|
|
200
|
-
switch (dataType) {
|
|
201
|
-
case 'null':
|
|
202
|
-
return new arrow.Null();
|
|
203
|
-
case 'binary':
|
|
204
|
-
return new arrow.Binary();
|
|
205
|
-
case 'bool':
|
|
206
|
-
return new arrow.Bool();
|
|
207
|
-
case 'int8':
|
|
208
|
-
return new arrow.Int8();
|
|
209
|
-
case 'int16':
|
|
210
|
-
return new arrow.Int16();
|
|
211
|
-
case 'int32':
|
|
212
|
-
return new arrow.Int32();
|
|
213
|
-
case 'int64':
|
|
214
|
-
return new arrow.Int64();
|
|
215
|
-
case 'uint8':
|
|
216
|
-
return new arrow.Uint8();
|
|
217
|
-
case 'uint16':
|
|
218
|
-
return new arrow.Uint16();
|
|
219
|
-
case 'uint32':
|
|
220
|
-
return new arrow.Uint32();
|
|
221
|
-
case 'uint64':
|
|
222
|
-
return new arrow.Uint64();
|
|
223
|
-
case 'float16':
|
|
224
|
-
return new arrow.Float16();
|
|
225
|
-
case 'float32':
|
|
226
|
-
return new arrow.Float32();
|
|
227
|
-
case 'float64':
|
|
228
|
-
return new arrow.Float64();
|
|
229
|
-
case 'utf8':
|
|
230
|
-
return new arrow.Utf8();
|
|
231
|
-
case 'date-day':
|
|
232
|
-
return new arrow.DateDay();
|
|
233
|
-
case 'date-millisecond':
|
|
234
|
-
return new arrow.DateMillisecond();
|
|
235
|
-
case 'time-second':
|
|
236
|
-
return new arrow.TimeSecond();
|
|
237
|
-
case 'time-millisecond':
|
|
238
|
-
return new arrow.TimeMillisecond();
|
|
239
|
-
case 'time-microsecond':
|
|
240
|
-
return new arrow.TimeMicrosecond();
|
|
241
|
-
case 'time-nanosecond':
|
|
242
|
-
return new arrow.TimeNanosecond();
|
|
243
|
-
case 'timestamp-second':
|
|
244
|
-
return new arrow.TimestampSecond();
|
|
245
|
-
case 'timestamp-millisecond':
|
|
246
|
-
return new arrow.TimestampMillisecond();
|
|
247
|
-
case 'timestamp-microsecond':
|
|
248
|
-
return new arrow.TimestampMicrosecond();
|
|
249
|
-
case 'timestamp-nanosecond':
|
|
250
|
-
return new arrow.TimestampNanosecond();
|
|
251
|
-
case 'interval-daytime':
|
|
252
|
-
return new arrow.IntervalDayTime();
|
|
253
|
-
case 'interval-yearmonth':
|
|
254
|
-
return new arrow.IntervalYearMonth();
|
|
255
|
-
default:
|
|
256
|
-
throw new Error('array type not supported');
|
|
257
|
-
}
|
|
258
272
|
}
|
|
259
|
-
//# sourceMappingURL=convert-arrow-schema.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ColumnarTable } from '@loaders.gl/schema';
|
|
2
|
-
import type { ArrowTable } from
|
|
2
|
+
import type { ArrowTable } from "../lib/arrow-table.js";
|
|
3
3
|
/**
|
|
4
4
|
* Convert an Apache Arrow table to a ColumnarTable
|
|
5
5
|
* @note Currently does not convert schema
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-arrow-to-columnar-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-columnar-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAC,UAAU,EAAC,
|
|
1
|
+
{"version":3,"file":"convert-arrow-to-columnar-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-columnar-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAEnD;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAmB5E"}
|
|
@@ -1,15 +1,24 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
/**
|
|
5
|
+
* Convert an Apache Arrow table to a ColumnarTable
|
|
6
|
+
* @note Currently does not convert schema
|
|
7
|
+
*/
|
|
1
8
|
export function convertArrowToColumnarTable(table) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
// TODO - avoid calling `getColumn` on columns we are not interested in?
|
|
10
|
+
// Add options object?
|
|
11
|
+
const arrowTable = table.data;
|
|
12
|
+
const columnarTable = {};
|
|
13
|
+
for (const field of arrowTable.schema.fields) {
|
|
14
|
+
// This (is intended to) coalesce all record batches into a single typed array
|
|
15
|
+
const arrowColumn = arrowTable.getChild(field.name);
|
|
16
|
+
const values = arrowColumn?.toArray();
|
|
17
|
+
columnarTable[field.name] = values;
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
shape: 'columnar-table',
|
|
21
|
+
schema: table.schema,
|
|
22
|
+
data: columnarTable
|
|
23
|
+
};
|
|
14
24
|
}
|
|
15
|
-
//# sourceMappingURL=convert-arrow-to-columnar-table.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { GeoJSONTable } from '@loaders.gl/schema';
|
|
2
2
|
import * as arrow from 'apache-arrow';
|
|
3
|
-
import type { ArrowTable } from
|
|
3
|
+
import type { ArrowTable } from "../lib/arrow-table.js";
|
|
4
4
|
/**
|
|
5
5
|
* Wrap an apache arrow table in a loaders.gl table wrapper.
|
|
6
6
|
* From this additional conversions are available.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-arrow-to-geojson-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-geojson-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAU,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAAC,UAAU,EAAC,
|
|
1
|
+
{"version":3,"file":"convert-arrow-to-geojson-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-geojson-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAU,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAInD;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,CAMlF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,UAAU,GAAG,YAAY,CAqC1E"}
|
|
@@ -1,39 +1,54 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { serializeArrowSchema, parseGeometryFromArrow } from '@loaders.gl/arrow';
|
|
2
5
|
import { getGeometryColumnsFromSchema } from '@loaders.gl/gis';
|
|
6
|
+
/**
|
|
7
|
+
* Wrap an apache arrow table in a loaders.gl table wrapper.
|
|
8
|
+
* From this additional conversions are available.
|
|
9
|
+
* @param arrowTable
|
|
10
|
+
* @returns
|
|
11
|
+
*/
|
|
3
12
|
export function convertApacheArrowToArrowTable(arrowTable) {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
13
|
+
return {
|
|
14
|
+
shape: 'arrow-table',
|
|
15
|
+
schema: serializeArrowSchema(arrowTable.schema),
|
|
16
|
+
data: arrowTable
|
|
17
|
+
};
|
|
9
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Convert an Apache Arrow table to a GeoJSONTable
|
|
21
|
+
* @note Currently does not convert schema
|
|
22
|
+
*/
|
|
10
23
|
export function convertArrowToGeoJSONTable(table) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
type
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
const arrowTable = table.data;
|
|
25
|
+
const schema = serializeArrowSchema(arrowTable.schema);
|
|
26
|
+
const geometryColumns = getGeometryColumnsFromSchema(schema);
|
|
27
|
+
// get encoding from geometryColumns['geometry']
|
|
28
|
+
const encoding = geometryColumns.geometry.encoding;
|
|
29
|
+
const features = [];
|
|
30
|
+
// Remove geometry columns
|
|
31
|
+
const propertyColumnNames = arrowTable.schema.fields
|
|
32
|
+
.map((field) => field.name)
|
|
33
|
+
// TODO - this deletes all geometry columns
|
|
34
|
+
.filter((name) => !(name in geometryColumns));
|
|
35
|
+
const propertiesTable = arrowTable.select(propertyColumnNames);
|
|
36
|
+
const arrowGeometryColumn = arrowTable.getChild('geometry');
|
|
37
|
+
for (let row = 0; row < arrowTable.numRows; row++) {
|
|
38
|
+
// get the geometry value from arrow geometry column
|
|
39
|
+
// Note that type can vary
|
|
40
|
+
const arrowGeometry = arrowGeometryColumn?.get(row);
|
|
41
|
+
// parse arrow geometry to geojson feature
|
|
42
|
+
const feature = parseGeometryFromArrow(arrowGeometry, encoding);
|
|
43
|
+
if (feature) {
|
|
44
|
+
const properties = propertiesTable.get(row)?.toJSON() || {};
|
|
45
|
+
features.push({ type: 'Feature', geometry: feature, properties });
|
|
46
|
+
}
|
|
30
47
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
};
|
|
48
|
+
return {
|
|
49
|
+
shape: 'geojson-table',
|
|
50
|
+
type: 'FeatureCollection',
|
|
51
|
+
schema: table.schema,
|
|
52
|
+
features
|
|
53
|
+
};
|
|
38
54
|
}
|
|
39
|
-
//# sourceMappingURL=convert-arrow-to-geojson-table.js.map
|