@loaders.gl/arrow 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js.map +1 -1
- package/dist/arrow-worker.js +2 -2
- package/dist/arrow-writer.d.ts.map +1 -1
- package/dist/arrow-writer.js.map +1 -1
- package/dist/dist.dev.js +650 -2
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +24 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +190 -0
- package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +19 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +1 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.js +138 -0
- package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +1 -0
- package/dist/geoarrow/get-arrow-bounds.d.ts +11 -0
- package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -0
- package/dist/geoarrow/get-arrow-bounds.js +24 -0
- package/dist/geoarrow/get-arrow-bounds.js.map +1 -0
- package/dist/index.cjs +589 -7
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/parse-arrow-sync.d.ts.map +1 -1
- package/dist/lib/parse-arrow-sync.js +1 -1
- package/dist/lib/parse-arrow-sync.js.map +1 -1
- package/dist/schema/arrow-type-utils.d.ts +1 -0
- package/dist/schema/arrow-type-utils.d.ts.map +1 -1
- package/dist/schema/arrow-type-utils.js.map +1 -1
- package/dist/schema/{convert-schema-arrow.d.ts → convert-arrow-schema.d.ts} +7 -1
- package/dist/schema/convert-arrow-schema.d.ts.map +1 -0
- package/dist/schema/{convert-schema-arrow.js → convert-arrow-schema.js} +78 -4
- package/dist/schema/convert-arrow-schema.js.map +1 -0
- package/dist/{lib/convert-table.d.ts → tables/convert-arrow-to-table.d.ts} +2 -2
- package/dist/tables/convert-arrow-to-table.d.ts.map +1 -0
- package/dist/{lib/convert-table.js → tables/convert-arrow-to-table.js} +1 -1
- package/dist/tables/convert-arrow-to-table.js.map +1 -0
- package/dist/{schema → tables}/convert-table-to-arrow.d.ts.map +1 -1
- package/dist/{schema → tables}/convert-table-to-arrow.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/workers/arrow-worker.js.map +1 -1
- package/package.json +5 -4
- package/src/arrow-loader.ts +1 -0
- package/src/arrow-writer.ts +1 -0
- package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +262 -0
- package/src/geoarrow/convert-geoarrow-to-geojson.ts +192 -0
- package/src/geoarrow/get-arrow-bounds.ts +41 -0
- package/src/index.ts +28 -4
- package/src/lib/parse-arrow-sync.ts +4 -1
- package/src/schema/arrow-type-utils.ts +1 -0
- package/src/schema/{convert-schema-arrow.ts → convert-arrow-schema.ts} +97 -21
- package/src/{lib/convert-table.ts → tables/convert-arrow-to-table.ts} +2 -1
- package/src/types.ts +3 -0
- package/src/workers/arrow-worker.ts +3 -0
- package/dist/lib/convert-table.d.ts.map +0 -1
- package/dist/lib/convert-table.js.map +0 -1
- package/dist/schema/convert-schema-arrow.d.ts.map +0 -1
- package/dist/schema/convert-schema-arrow.js.map +0 -1
- /package/dist/{schema → tables}/convert-table-to-arrow.d.ts +0 -0
- /package/dist/{schema → tables}/convert-table-to-arrow.js +0 -0
- /package/src/{schema → tables}/convert-table-to-arrow.ts +0 -0
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
Null,
|
|
10
10
|
Binary,
|
|
11
11
|
Bool,
|
|
12
|
-
|
|
12
|
+
Int,
|
|
13
13
|
Int8,
|
|
14
14
|
Int16,
|
|
15
15
|
Int32,
|
|
@@ -18,29 +18,37 @@ import {
|
|
|
18
18
|
Uint16,
|
|
19
19
|
Uint32,
|
|
20
20
|
Uint64,
|
|
21
|
-
|
|
21
|
+
Float,
|
|
22
22
|
Float16,
|
|
23
23
|
Float32,
|
|
24
24
|
Float64,
|
|
25
|
+
Precision,
|
|
25
26
|
Utf8,
|
|
26
|
-
|
|
27
|
+
Date_,
|
|
28
|
+
DateUnit,
|
|
27
29
|
DateDay,
|
|
28
30
|
DateMillisecond,
|
|
29
|
-
|
|
31
|
+
Time,
|
|
30
32
|
TimeMillisecond,
|
|
31
33
|
TimeSecond,
|
|
32
|
-
|
|
34
|
+
Timestamp,
|
|
33
35
|
TimestampSecond,
|
|
34
36
|
TimestampMillisecond,
|
|
35
37
|
TimestampMicrosecond,
|
|
36
38
|
TimestampNanosecond,
|
|
37
|
-
|
|
39
|
+
Interval,
|
|
40
|
+
IntervalUnit,
|
|
38
41
|
IntervalDayTime,
|
|
39
42
|
IntervalYearMonth,
|
|
40
43
|
FixedSizeList,
|
|
41
|
-
Struct
|
|
44
|
+
Struct,
|
|
45
|
+
TimeUnit,
|
|
46
|
+
TimeMicrosecond,
|
|
47
|
+
TimeNanosecond,
|
|
48
|
+
List
|
|
42
49
|
} from 'apache-arrow';
|
|
43
50
|
|
|
51
|
+
/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
|
|
44
52
|
export function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {
|
|
45
53
|
return {
|
|
46
54
|
fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
|
|
@@ -48,6 +56,7 @@ export function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {
|
|
|
48
56
|
};
|
|
49
57
|
}
|
|
50
58
|
|
|
59
|
+
/** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
|
|
51
60
|
export function deserializeArrowSchema(schema: Schema): ArrowSchema {
|
|
52
61
|
return new ArrowSchema(
|
|
53
62
|
schema.fields.map((field) => deserializeArrowField(field)),
|
|
@@ -55,14 +64,17 @@ export function deserializeArrowSchema(schema: Schema): ArrowSchema {
|
|
|
55
64
|
);
|
|
56
65
|
}
|
|
57
66
|
|
|
67
|
+
/** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
|
|
58
68
|
export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
|
|
59
69
|
return Object.fromEntries(arrowMetadata);
|
|
60
70
|
}
|
|
61
71
|
|
|
72
|
+
/** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
|
|
62
73
|
export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
|
|
63
74
|
return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
|
|
64
75
|
}
|
|
65
76
|
|
|
77
|
+
/** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
|
|
66
78
|
export function serializeArrowField(field: ArrowField): Field {
|
|
67
79
|
return {
|
|
68
80
|
name: field.name,
|
|
@@ -72,6 +84,7 @@ export function serializeArrowField(field: ArrowField): Field {
|
|
|
72
84
|
};
|
|
73
85
|
}
|
|
74
86
|
|
|
87
|
+
/** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
|
|
75
88
|
export function deserializeArrowField(field: Field): ArrowField {
|
|
76
89
|
return new ArrowField(
|
|
77
90
|
field.name,
|
|
@@ -91,7 +104,9 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
|
|
|
91
104
|
return 'binary';
|
|
92
105
|
case Bool:
|
|
93
106
|
return 'bool';
|
|
94
|
-
|
|
107
|
+
case Int:
|
|
108
|
+
const intType = arrowType as Int;
|
|
109
|
+
return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
|
|
95
110
|
case Int8:
|
|
96
111
|
return 'int8';
|
|
97
112
|
case Int16:
|
|
@@ -108,7 +123,19 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
|
|
|
108
123
|
return 'uint32';
|
|
109
124
|
case Uint64:
|
|
110
125
|
return 'uint64';
|
|
111
|
-
|
|
126
|
+
case Float:
|
|
127
|
+
const precision = (arrowType as Float).precision;
|
|
128
|
+
// return `float(precision + 1) * 16`;
|
|
129
|
+
switch (precision) {
|
|
130
|
+
case Precision.HALF:
|
|
131
|
+
return 'float16';
|
|
132
|
+
case Precision.SINGLE:
|
|
133
|
+
return 'float32';
|
|
134
|
+
case Precision.DOUBLE:
|
|
135
|
+
return 'float64';
|
|
136
|
+
default:
|
|
137
|
+
return 'float16';
|
|
138
|
+
}
|
|
112
139
|
case Float16:
|
|
113
140
|
return 'float16';
|
|
114
141
|
case Float32:
|
|
@@ -117,17 +144,49 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
|
|
|
117
144
|
return 'float64';
|
|
118
145
|
case Utf8:
|
|
119
146
|
return 'utf8';
|
|
120
|
-
|
|
147
|
+
case Date:
|
|
148
|
+
const dateUnit = (arrowType as Date_).unit;
|
|
149
|
+
return dateUnit === DateUnit.DAY ? 'date-day' : 'date-millisecond';
|
|
121
150
|
case DateDay:
|
|
122
151
|
return 'date-day';
|
|
123
152
|
case DateMillisecond:
|
|
124
153
|
return 'date-millisecond';
|
|
125
|
-
|
|
154
|
+
case Time:
|
|
155
|
+
const timeUnit = (arrowType as Time).unit;
|
|
156
|
+
switch (timeUnit) {
|
|
157
|
+
case TimeUnit.SECOND:
|
|
158
|
+
return 'time-second';
|
|
159
|
+
case TimeUnit.MILLISECOND:
|
|
160
|
+
return 'time-millisecond';
|
|
161
|
+
case TimeUnit.MICROSECOND:
|
|
162
|
+
return 'time-microsecond';
|
|
163
|
+
case TimeUnit.NANOSECOND:
|
|
164
|
+
return 'time-nanosecond';
|
|
165
|
+
default:
|
|
166
|
+
return 'time-second';
|
|
167
|
+
}
|
|
126
168
|
case TimeMillisecond:
|
|
127
169
|
return 'time-millisecond';
|
|
128
170
|
case TimeSecond:
|
|
129
171
|
return 'time-second';
|
|
130
|
-
|
|
172
|
+
case TimeMicrosecond:
|
|
173
|
+
return 'time-microsecond';
|
|
174
|
+
case TimeNanosecond:
|
|
175
|
+
return 'time-nanosecond';
|
|
176
|
+
case Timestamp:
|
|
177
|
+
const timeStampUnit = (arrowType as Timestamp).unit;
|
|
178
|
+
switch (timeStampUnit) {
|
|
179
|
+
case TimeUnit.SECOND:
|
|
180
|
+
return 'timestamp-second';
|
|
181
|
+
case TimeUnit.MILLISECOND:
|
|
182
|
+
return 'timestamp-millisecond';
|
|
183
|
+
case TimeUnit.MICROSECOND:
|
|
184
|
+
return 'timestamp-microsecond';
|
|
185
|
+
case TimeUnit.NANOSECOND:
|
|
186
|
+
return 'timestamp-nanosecond';
|
|
187
|
+
default:
|
|
188
|
+
return 'timestamp-second';
|
|
189
|
+
}
|
|
131
190
|
case TimestampSecond:
|
|
132
191
|
return 'timestamp-second';
|
|
133
192
|
case TimestampMillisecond:
|
|
@@ -136,11 +195,27 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
|
|
|
136
195
|
return 'timestamp-microsecond';
|
|
137
196
|
case TimestampNanosecond:
|
|
138
197
|
return 'timestamp-nanosecond';
|
|
139
|
-
|
|
198
|
+
case Interval:
|
|
199
|
+
const intervalUnit = (arrowType as Interval).unit;
|
|
200
|
+
switch (intervalUnit) {
|
|
201
|
+
case IntervalUnit.DAY_TIME:
|
|
202
|
+
return 'interval-daytime';
|
|
203
|
+
case IntervalUnit.YEAR_MONTH:
|
|
204
|
+
return 'interval-yearmonth';
|
|
205
|
+
default:
|
|
206
|
+
return 'interval-daytime';
|
|
207
|
+
}
|
|
140
208
|
case IntervalDayTime:
|
|
141
209
|
return 'interval-daytime';
|
|
142
210
|
case IntervalYearMonth:
|
|
143
211
|
return 'interval-yearmonth';
|
|
212
|
+
case List:
|
|
213
|
+
const listType = arrowType as List;
|
|
214
|
+
const listField = listType.valueField;
|
|
215
|
+
return {
|
|
216
|
+
type: 'list',
|
|
217
|
+
children: [serializeArrowField(listField)]
|
|
218
|
+
};
|
|
144
219
|
case FixedSizeList:
|
|
145
220
|
return {
|
|
146
221
|
type: 'fixed-size-list',
|
|
@@ -159,6 +234,9 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
|
|
|
159
234
|
export function deserializeArrowType(dataType: DataType): ArrowDataType {
|
|
160
235
|
if (typeof dataType === 'object') {
|
|
161
236
|
switch (dataType.type) {
|
|
237
|
+
case 'list':
|
|
238
|
+
const field = deserializeArrowField(dataType.children[0]);
|
|
239
|
+
return new List(field);
|
|
162
240
|
case 'fixed-size-list':
|
|
163
241
|
const child = deserializeArrowField(dataType.children[0]);
|
|
164
242
|
return new FixedSizeList(dataType.listSize, child);
|
|
@@ -177,7 +255,6 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
|
|
|
177
255
|
return new Binary();
|
|
178
256
|
case 'bool':
|
|
179
257
|
return new Bool();
|
|
180
|
-
// case 'int': return new Int();
|
|
181
258
|
case 'int8':
|
|
182
259
|
return new Int8();
|
|
183
260
|
case 'int16':
|
|
@@ -194,7 +271,6 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
|
|
|
194
271
|
return new Uint32();
|
|
195
272
|
case 'uint64':
|
|
196
273
|
return new Uint64();
|
|
197
|
-
// case 'float': return new Float();
|
|
198
274
|
case 'float16':
|
|
199
275
|
return new Float16();
|
|
200
276
|
case 'float32':
|
|
@@ -203,17 +279,18 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
|
|
|
203
279
|
return new Float64();
|
|
204
280
|
case 'utf8':
|
|
205
281
|
return new Utf8();
|
|
206
|
-
// case 'date': return new Date();
|
|
207
282
|
case 'date-day':
|
|
208
283
|
return new DateDay();
|
|
209
284
|
case 'date-millisecond':
|
|
210
285
|
return new DateMillisecond();
|
|
211
|
-
// case 'time': return new Time();
|
|
212
|
-
case 'time-millisecond':
|
|
213
|
-
return new TimeMillisecond();
|
|
214
286
|
case 'time-second':
|
|
215
287
|
return new TimeSecond();
|
|
216
|
-
|
|
288
|
+
case 'time-millisecond':
|
|
289
|
+
return new TimeMillisecond();
|
|
290
|
+
case 'time-microsecond':
|
|
291
|
+
return new TimeMicrosecond();
|
|
292
|
+
case 'time-nanosecond':
|
|
293
|
+
return new TimeNanosecond();
|
|
217
294
|
case 'timestamp-second':
|
|
218
295
|
return new TimestampSecond();
|
|
219
296
|
case 'timestamp-millisecond':
|
|
@@ -222,7 +299,6 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
|
|
|
222
299
|
return new TimestampMicrosecond();
|
|
223
300
|
case 'timestamp-nanosecond':
|
|
224
301
|
return new TimestampNanosecond();
|
|
225
|
-
// case 'interval': return new Interval();
|
|
226
302
|
case 'interval-daytime':
|
|
227
303
|
return new IntervalDayTime();
|
|
228
304
|
case 'interval-yearmonth':
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// loaders.gl, MIT license
|
|
2
2
|
// Copyright (c) vis.gl contributors
|
|
3
|
+
|
|
3
4
|
import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
|
|
4
5
|
import type {Table as ApacheArrowTable} from 'apache-arrow';
|
|
5
|
-
import type {ArrowTable} from '
|
|
6
|
+
import type {ArrowTable} from '../lib/arrow-table';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Wrap an apache arrow table in a loaders.gl table wrapper.
|
package/src/types.ts
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../src/lib/convert-table.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAC,KAAK,IAAI,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAE9C;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,gBAAgB,GAAG,UAAU,CAKvF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAkB5E;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAmB5F"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\nimport type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\nimport type {ArrowTable} from './arrow-table';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n return {\n shape: 'arrow-table',\n data: arrowTable\n };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n\n const arrowTable = table.data;\n const columnarTable = {};\n\n for (const field of arrowTable.schema.fields) {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getChild(field.name);\n const values = arrowColumn?.toArray();\n columnarTable[field.name] = values;\n }\n\n return {\n shape: 'columnar-table',\n data: columnarTable\n };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n\n return {\n shape: 'object-row-table',\n data: rowFormatTable\n };\n}\n"],"mappings":"AAYA,OAAO,SAASA,8BAA8BA,CAACC,UAA4B,EAAc;EACvF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMA,OAAO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,MAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,MAAMG,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIN,UAAU,CAACO,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,OAAO;IACLX,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMA,OAAO,SAASS,+BAA+BA,CAACT,aAA4B,EAAkB;EAC5F,MAAMU,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACZ,aAAa,CAAC;EAC5C,MAAMa,cAAc,GAAGb,aAAa,CAACU,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGnB,aAAa,CAACmB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACLrB,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAEkB;EACR,CAAC;AACH"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"convert-schema-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-schema-arrow.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAChF,OAAO,EACL,KAAK,IAAI,UAAU,EACnB,MAAM,IAAI,WAAW,EACrB,QAAQ,IAAI,aAAa,EAkC1B,MAAM,cAAc,CAAC;AAEtB,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAKrE;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAKlE;AAED,wBAAgB,sBAAsB,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,cAAc,CAEzF;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,CAAC,EAAE,cAAc,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAEvF;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CAO5D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAO9D;AAED,+EAA+E;AAE/E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,aAAa,GAAG,QAAQ,CAqErE;AAED,+EAA+E;AAE/E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,GAAG,aAAa,CA0EtE"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"convert-schema-arrow.js","names":["Field","ArrowField","Schema","ArrowSchema","Null","Binary","Bool","Int8","Int16","Int32","Int64","Uint8","Uint16","Uint32","Uint64","Float16","Float32","Float64","Utf8","DateDay","DateMillisecond","TimeMillisecond","TimeSecond","TimestampSecond","TimestampMillisecond","TimestampMicrosecond","TimestampNanosecond","IntervalDayTime","IntervalYearMonth","FixedSizeList","Struct","serializeArrowSchema","arrowSchema","fields","map","arrowField","serializeArrowField","metadata","serializeArrowMetadata","deserializeArrowSchema","schema","field","deserializeArrowField","deserializeArrowMetadata","arrowMetadata","Object","fromEntries","Map","entries","name","type","serializeArrowType","nullable","deserializeArrowType","arrowType","constructor","listSize","children","Error","dataType","child"],"sources":["../../src/schema/convert-schema-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';\nimport {\n Field as ArrowField,\n Schema as ArrowSchema,\n DataType as ArrowDataType,\n Null,\n Binary,\n Bool,\n // Int,\n Int8,\n Int16,\n Int32,\n Int64,\n Uint8,\n Uint16,\n Uint32,\n Uint64,\n // Float,\n Float16,\n Float32,\n Float64,\n Utf8,\n // Date,\n DateDay,\n DateMillisecond,\n // Time,\n TimeMillisecond,\n TimeSecond,\n // Timestamp,\n TimestampSecond,\n TimestampMillisecond,\n TimestampMicrosecond,\n TimestampNanosecond,\n // Interval,\n IntervalDayTime,\n IntervalYearMonth,\n FixedSizeList,\n Struct\n} from 'apache-arrow';\n\nexport function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {\n return {\n fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),\n metadata: serializeArrowMetadata(arrowSchema.metadata)\n };\n}\n\nexport function deserializeArrowSchema(schema: Schema): ArrowSchema {\n return new ArrowSchema(\n schema.fields.map((field) => deserializeArrowField(field)),\n deserializeArrowMetadata(schema.metadata)\n );\n}\n\nexport function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {\n return Object.fromEntries(arrowMetadata);\n}\n\nexport function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {\n return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();\n}\n\nexport function serializeArrowField(field: ArrowField): Field {\n return {\n name: field.name,\n type: serializeArrowType(field.type),\n nullable: field.nullable,\n metadata: serializeArrowMetadata(field.metadata)\n };\n}\n\nexport function deserializeArrowField(field: Field): ArrowField {\n return new ArrowField(\n field.name,\n deserializeArrowType(field.type),\n field.nullable,\n deserializeArrowMetadata(field.metadata)\n );\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function serializeArrowType(arrowType: ArrowDataType): DataType {\n switch (arrowType.constructor) {\n case Null:\n return 'null';\n case Binary:\n return 'binary';\n case Bool:\n return 'bool';\n // case Int: return 'int';\n case Int8:\n return 'int8';\n case Int16:\n return 'int16';\n case Int32:\n return 'int32';\n case Int64:\n return 'int64';\n case Uint8:\n return 'uint8';\n case Uint16:\n return 'uint16';\n case Uint32:\n return 'uint32';\n case Uint64:\n return 'uint64';\n // case Float: return 'float';\n case Float16:\n return 'float16';\n case Float32:\n return 'float32';\n case Float64:\n return 'float64';\n case Utf8:\n return 'utf8';\n // case Date: return 'date';\n case DateDay:\n return 'date-day';\n case DateMillisecond:\n return 'date-millisecond';\n // case Time: return 'time';\n case TimeMillisecond:\n return 'time-millisecond';\n case TimeSecond:\n return 'time-second';\n // case Timestamp: return 'timestamp';\n case TimestampSecond:\n return 'timestamp-second';\n case TimestampMillisecond:\n return 'timestamp-millisecond';\n case TimestampMicrosecond:\n return 'timestamp-microsecond';\n case TimestampNanosecond:\n return 'timestamp-nanosecond';\n // case Interval: return 'interval';\n case IntervalDayTime:\n return 'interval-daytime';\n case IntervalYearMonth:\n return 'interval-yearmonth';\n case FixedSizeList:\n return {\n type: 'fixed-size-list',\n listSize: (arrowType as FixedSizeList).listSize,\n children: [serializeArrowField((arrowType as FixedSizeList).children[0])]\n };\n // case Struct:\n // return {type: 'struct', children: (arrowType as Struct).children};\n default:\n throw new Error('array type not supported');\n }\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function deserializeArrowType(dataType: DataType): ArrowDataType {\n if (typeof dataType === 'object') {\n switch (dataType.type) {\n case 'fixed-size-list':\n const child = deserializeArrowField(dataType.children[0]);\n return new FixedSizeList(dataType.listSize, child);\n case 'struct':\n const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));\n return new Struct(children);\n default:\n throw new Error('array type not supported');\n }\n }\n\n switch (dataType) {\n case 'null':\n return new Null();\n case 'binary':\n return new Binary();\n case 'bool':\n return new Bool();\n // case 'int': return new Int();\n case 'int8':\n return new Int8();\n case 'int16':\n return new Int16();\n case 'int32':\n return new Int32();\n case 'int64':\n return new Int64();\n case 'uint8':\n return new Uint8();\n case 'uint16':\n return new Uint16();\n case 'uint32':\n return new Uint32();\n case 'uint64':\n return new Uint64();\n // case 'float': return new Float();\n case 'float16':\n return new Float16();\n case 'float32':\n return new Float32();\n case 'float64':\n return new Float64();\n case 'utf8':\n return new Utf8();\n // case 'date': return new Date();\n case 'date-day':\n return new DateDay();\n case 'date-millisecond':\n return new DateMillisecond();\n // case 'time': return new Time();\n case 'time-millisecond':\n return new TimeMillisecond();\n case 'time-second':\n return new TimeSecond();\n // case 'timestamp': return new Timestamp();\n case 'timestamp-second':\n return new TimestampSecond();\n case 'timestamp-millisecond':\n return new TimestampMillisecond();\n case 'timestamp-microsecond':\n return new TimestampMicrosecond();\n case 'timestamp-nanosecond':\n return new TimestampNanosecond();\n // case 'interval': return new Interval();\n case 'interval-daytime':\n return new IntervalDayTime();\n case 'interval-yearmonth':\n return new IntervalYearMonth();\n default:\n throw new Error('array type not supported');\n }\n}\n"],"mappings":"AAIA,SACEA,KAAK,IAAIC,UAAU,EACnBC,MAAM,IAAIC,WAAW,EAErBC,IAAI,EACJC,MAAM,EACNC,IAAI,EAEJC,IAAI,EACJC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,MAAM,EACNC,MAAM,EACNC,MAAM,EAENC,OAAO,EACPC,OAAO,EACPC,OAAO,EACPC,IAAI,EAEJC,OAAO,EACPC,eAAe,EAEfC,eAAe,EACfC,UAAU,EAEVC,eAAe,EACfC,oBAAoB,EACpBC,oBAAoB,EACpBC,mBAAmB,EAEnBC,eAAe,EACfC,iBAAiB,EACjBC,aAAa,EACbC,MAAM,QACD,cAAc;AAErB,OAAO,SAASC,oBAAoBA,CAACC,WAAwB,EAAU;EACrE,OAAO;IACLC,MAAM,EAAED,WAAW,CAACC,MAAM,CAACC,GAAG,CAAEC,UAAU,IAAKC,mBAAmB,CAACD,UAAU,CAAC,CAAC;IAC/EE,QAAQ,EAAEC,sBAAsB,CAACN,WAAW,CAACK,QAAQ;EACvD,CAAC;AACH;AAEA,OAAO,SAASE,sBAAsBA,CAACC,MAAc,EAAe;EAClE,OAAO,IAAIrC,WAAW,CACpBqC,MAAM,CAACP,MAAM,CAACC,GAAG,CAAEO,KAAK,IAAKC,qBAAqB,CAACD,KAAK,CAAC,CAAC,EAC1DE,wBAAwB,CAACH,MAAM,CAACH,QAAQ,CAC1C,CAAC;AACH;AAEA,OAAO,SAASC,sBAAsBA,CAACM,aAAkC,EAAkB;EACzF,OAAOC,MAAM,CAACC,WAAW,CAACF,aAAa,CAAC;AAC1C;AAEA,OAAO,SAASD,wBAAwBA,CAACN,QAAyB,EAAuB;EACvF,OAAOA,QAAQ,GAAG,IAAIU,GAAG,CAACF,MAAM,CAACG,OAAO,CAACX,QAAQ,CAAC,CAAC,GAAG,IAAIU,GAAG,CAAiB,CAAC;AACjF;AAEA,OAAO,SAASX,mBAAmBA,CAACK,KAAiB,EAAS;EAC5D,OAAO;IACLQ,IAAI,EAAER,KAAK,CAACQ,IAAI;IAChBC,IAAI,EAAEC,kBAAkB,CAACV,KAAK,CAACS,IAAI,CAAC;IACpCE,QAAQ,EAAEX,KAAK,CAACW,QAAQ;IACxBf,QAAQ,EAAEC,sBAAsB,CAACG,KAAK,CAACJ,QAAQ;EACjD,CAAC;AACH;AAEA,OAAO,SAASK,qBAAqBA,CAACD,KAAY,EAAc;EAC9D,OAAO,IAAIxC,UAAU,CACnBwC,KAAK,CAACQ,IAAI,EACVI,oBAAoB,CAACZ,KAAK,CAACS,IAAI,CAAC,EAChCT,KAAK,CAACW,QAAQ,EACdT,wBAAwB,CAACF,KAAK,CAACJ,QAAQ,CACzC,CAAC;AACH;AAIA,OAAO,SAASc,kBAAkBA,CAACG,SAAwB,EAAY;EACrE,QAAQA,SAAS,CAACC,WAAW;IAC3B,KAAKnD,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IAEjB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,OAAO;MACV,OAAO,UAAU;IACnB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAE3B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,UAAU;MACb,OAAO,aAAa;IAEtB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,mBAAmB;MACtB,OAAO,sBAAsB;IAE/B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,iBAAiB;MACpB,OAAO,oBAAoB;IAC7B,KAAKC,aAAa;MAChB,OAAO;QACLqB,IAAI,EAAE,iBAAiB;QACvBM,QAAQ,EAAGF,SAAS,CAAmBE,QAAQ;QAC/CC,QAAQ,EAAE,CAACrB,mBAAmB,CAAEkB,SAAS,CAAmBG,QAAQ,CAAC,CAAC,CAAC,CAAC;MAC1E,CAAC;IAGH;MACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF;AAIA,OAAO,SAASL,oBAAoBA,CAACM,QAAkB,EAAiB;EACtE,IAAI,OAAOA,QAAQ,KAAK,QAAQ,EAAE;IAChC,QAAQA,QAAQ,CAACT,IAAI;MACnB,KAAK,iBAAiB;QACpB,MAAMU,KAAK,GAAGlB,qBAAqB,CAACiB,QAAQ,CAACF,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI5B,aAAa,CAAC8B,QAAQ,CAACH,QAAQ,EAAEI,KAAK,CAAC;MACpD,KAAK,QAAQ;QACX,MAAMH,QAAQ,GAAGE,QAAQ,CAACF,QAAQ,CAACvB,GAAG,CAAEC,UAAU,IAAKO,qBAAqB,CAACP,UAAU,CAAC,CAAC;QACzF,OAAO,IAAIL,MAAM,CAAC2B,QAAQ,CAAC;MAC7B;QACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;IAC/C;EACF;EAEA,QAAQC,QAAQ;IACd,KAAK,MAAM;MACT,OAAO,IAAIvD,IAAI,CAAC,CAAC;IACnB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IACnB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IAErB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,UAAU;MACb,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAE9B,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,aAAa;MAChB,OAAO,IAAIC,UAAU,CAAC,CAAC;IAEzB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,sBAAsB;MACzB,OAAO,IAAIC,mBAAmB,CAAC,CAAC;IAElC,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,oBAAoB;MACvB,OAAO,IAAIC,iBAAiB,CAAC,CAAC;IAChC;MACE,MAAM,IAAI8B,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|