@loaders.gl/arrow 4.0.0-beta.8 → 4.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/arrow-loader.d.ts.map +1 -1
  2. package/dist/arrow-loader.js.map +1 -1
  3. package/dist/arrow-worker.js +2 -2
  4. package/dist/arrow-writer.d.ts.map +1 -1
  5. package/dist/arrow-writer.js.map +1 -1
  6. package/dist/dist.dev.js +650 -2
  7. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts +24 -0
  8. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map +1 -0
  9. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js +190 -0
  10. package/dist/geoarrow/convert-geoarrow-to-binary-geometry.js.map +1 -0
  11. package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts +19 -0
  12. package/dist/geoarrow/convert-geoarrow-to-geojson.d.ts.map +1 -0
  13. package/dist/geoarrow/convert-geoarrow-to-geojson.js +138 -0
  14. package/dist/geoarrow/convert-geoarrow-to-geojson.js.map +1 -0
  15. package/dist/geoarrow/get-arrow-bounds.d.ts +11 -0
  16. package/dist/geoarrow/get-arrow-bounds.d.ts.map +1 -0
  17. package/dist/geoarrow/get-arrow-bounds.js +24 -0
  18. package/dist/geoarrow/get-arrow-bounds.js.map +1 -0
  19. package/dist/index.cjs +589 -7
  20. package/dist/index.d.ts +5 -0
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +4 -1
  23. package/dist/index.js.map +1 -1
  24. package/dist/lib/parse-arrow-sync.d.ts.map +1 -1
  25. package/dist/lib/parse-arrow-sync.js +1 -1
  26. package/dist/lib/parse-arrow-sync.js.map +1 -1
  27. package/dist/schema/arrow-type-utils.d.ts +1 -0
  28. package/dist/schema/arrow-type-utils.d.ts.map +1 -1
  29. package/dist/schema/arrow-type-utils.js.map +1 -1
  30. package/dist/schema/{convert-schema-arrow.d.ts → convert-arrow-schema.d.ts} +7 -1
  31. package/dist/schema/convert-arrow-schema.d.ts.map +1 -0
  32. package/dist/schema/{convert-schema-arrow.js → convert-arrow-schema.js} +78 -4
  33. package/dist/schema/convert-arrow-schema.js.map +1 -0
  34. package/dist/{lib/convert-table.d.ts → tables/convert-arrow-to-table.d.ts} +2 -2
  35. package/dist/tables/convert-arrow-to-table.d.ts.map +1 -0
  36. package/dist/{lib/convert-table.js → tables/convert-arrow-to-table.js} +1 -1
  37. package/dist/tables/convert-arrow-to-table.js.map +1 -0
  38. package/dist/{schema → tables}/convert-table-to-arrow.d.ts.map +1 -1
  39. package/dist/{schema → tables}/convert-table-to-arrow.js.map +1 -1
  40. package/dist/types.d.ts.map +1 -1
  41. package/dist/types.js.map +1 -1
  42. package/dist/workers/arrow-worker.js.map +1 -1
  43. package/package.json +5 -4
  44. package/src/arrow-loader.ts +1 -0
  45. package/src/arrow-writer.ts +1 -0
  46. package/src/geoarrow/convert-geoarrow-to-binary-geometry.ts +262 -0
  47. package/src/geoarrow/convert-geoarrow-to-geojson.ts +192 -0
  48. package/src/geoarrow/get-arrow-bounds.ts +41 -0
  49. package/src/index.ts +28 -4
  50. package/src/lib/parse-arrow-sync.ts +4 -1
  51. package/src/schema/arrow-type-utils.ts +1 -0
  52. package/src/schema/{convert-schema-arrow.ts → convert-arrow-schema.ts} +97 -21
  53. package/src/{lib/convert-table.ts → tables/convert-arrow-to-table.ts} +2 -1
  54. package/src/types.ts +3 -0
  55. package/src/workers/arrow-worker.ts +3 -0
  56. package/dist/lib/convert-table.d.ts.map +0 -1
  57. package/dist/lib/convert-table.js.map +0 -1
  58. package/dist/schema/convert-schema-arrow.d.ts.map +0 -1
  59. package/dist/schema/convert-schema-arrow.js.map +0 -1
  60. /package/dist/{schema → tables}/convert-table-to-arrow.d.ts +0 -0
  61. /package/dist/{schema → tables}/convert-table-to-arrow.js +0 -0
  62. /package/src/{schema → tables}/convert-table-to-arrow.ts +0 -0
@@ -9,7 +9,7 @@ import {
9
9
  Null,
10
10
  Binary,
11
11
  Bool,
12
- // Int,
12
+ Int,
13
13
  Int8,
14
14
  Int16,
15
15
  Int32,
@@ -18,29 +18,37 @@ import {
18
18
  Uint16,
19
19
  Uint32,
20
20
  Uint64,
21
- // Float,
21
+ Float,
22
22
  Float16,
23
23
  Float32,
24
24
  Float64,
25
+ Precision,
25
26
  Utf8,
26
- // Date,
27
+ Date_,
28
+ DateUnit,
27
29
  DateDay,
28
30
  DateMillisecond,
29
- // Time,
31
+ Time,
30
32
  TimeMillisecond,
31
33
  TimeSecond,
32
- // Timestamp,
34
+ Timestamp,
33
35
  TimestampSecond,
34
36
  TimestampMillisecond,
35
37
  TimestampMicrosecond,
36
38
  TimestampNanosecond,
37
- // Interval,
39
+ Interval,
40
+ IntervalUnit,
38
41
  IntervalDayTime,
39
42
  IntervalYearMonth,
40
43
  FixedSizeList,
41
- Struct
44
+ Struct,
45
+ TimeUnit,
46
+ TimeMicrosecond,
47
+ TimeNanosecond,
48
+ List
42
49
  } from 'apache-arrow';
43
50
 
51
+ /** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
44
52
  export function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {
45
53
  return {
46
54
  fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
@@ -48,6 +56,7 @@ export function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {
48
56
  };
49
57
  }
50
58
 
59
+ /** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
51
60
  export function deserializeArrowSchema(schema: Schema): ArrowSchema {
52
61
  return new ArrowSchema(
53
62
  schema.fields.map((field) => deserializeArrowField(field)),
@@ -55,14 +64,17 @@ export function deserializeArrowSchema(schema: Schema): ArrowSchema {
55
64
  );
56
65
  }
57
66
 
67
+ /** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
58
68
  export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
59
69
  return Object.fromEntries(arrowMetadata);
60
70
  }
61
71
 
72
+ /** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
62
73
  export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
63
74
  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
64
75
  }
65
76
 
77
+ /** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
66
78
  export function serializeArrowField(field: ArrowField): Field {
67
79
  return {
68
80
  name: field.name,
@@ -72,6 +84,7 @@ export function serializeArrowField(field: ArrowField): Field {
72
84
  };
73
85
  }
74
86
 
87
+ /** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
75
88
  export function deserializeArrowField(field: Field): ArrowField {
76
89
  return new ArrowField(
77
90
  field.name,
@@ -91,7 +104,9 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
91
104
  return 'binary';
92
105
  case Bool:
93
106
  return 'bool';
94
- // case Int: return 'int';
107
+ case Int:
108
+ const intType = arrowType as Int;
109
+ return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
95
110
  case Int8:
96
111
  return 'int8';
97
112
  case Int16:
@@ -108,7 +123,19 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
108
123
  return 'uint32';
109
124
  case Uint64:
110
125
  return 'uint64';
111
- // case Float: return 'float';
126
+ case Float:
127
+ const precision = (arrowType as Float).precision;
128
+ // return `float(precision + 1) * 16`;
129
+ switch (precision) {
130
+ case Precision.HALF:
131
+ return 'float16';
132
+ case Precision.SINGLE:
133
+ return 'float32';
134
+ case Precision.DOUBLE:
135
+ return 'float64';
136
+ default:
137
+ return 'float16';
138
+ }
112
139
  case Float16:
113
140
  return 'float16';
114
141
  case Float32:
@@ -117,17 +144,49 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
117
144
  return 'float64';
118
145
  case Utf8:
119
146
  return 'utf8';
120
- // case Date: return 'date';
147
+ case Date:
148
+ const dateUnit = (arrowType as Date_).unit;
149
+ return dateUnit === DateUnit.DAY ? 'date-day' : 'date-millisecond';
121
150
  case DateDay:
122
151
  return 'date-day';
123
152
  case DateMillisecond:
124
153
  return 'date-millisecond';
125
- // case Time: return 'time';
154
+ case Time:
155
+ const timeUnit = (arrowType as Time).unit;
156
+ switch (timeUnit) {
157
+ case TimeUnit.SECOND:
158
+ return 'time-second';
159
+ case TimeUnit.MILLISECOND:
160
+ return 'time-millisecond';
161
+ case TimeUnit.MICROSECOND:
162
+ return 'time-microsecond';
163
+ case TimeUnit.NANOSECOND:
164
+ return 'time-nanosecond';
165
+ default:
166
+ return 'time-second';
167
+ }
126
168
  case TimeMillisecond:
127
169
  return 'time-millisecond';
128
170
  case TimeSecond:
129
171
  return 'time-second';
130
- // case Timestamp: return 'timestamp';
172
+ case TimeMicrosecond:
173
+ return 'time-microsecond';
174
+ case TimeNanosecond:
175
+ return 'time-nanosecond';
176
+ case Timestamp:
177
+ const timeStampUnit = (arrowType as Timestamp).unit;
178
+ switch (timeStampUnit) {
179
+ case TimeUnit.SECOND:
180
+ return 'timestamp-second';
181
+ case TimeUnit.MILLISECOND:
182
+ return 'timestamp-millisecond';
183
+ case TimeUnit.MICROSECOND:
184
+ return 'timestamp-microsecond';
185
+ case TimeUnit.NANOSECOND:
186
+ return 'timestamp-nanosecond';
187
+ default:
188
+ return 'timestamp-second';
189
+ }
131
190
  case TimestampSecond:
132
191
  return 'timestamp-second';
133
192
  case TimestampMillisecond:
@@ -136,11 +195,27 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
136
195
  return 'timestamp-microsecond';
137
196
  case TimestampNanosecond:
138
197
  return 'timestamp-nanosecond';
139
- // case Interval: return 'interval';
198
+ case Interval:
199
+ const intervalUnit = (arrowType as Interval).unit;
200
+ switch (intervalUnit) {
201
+ case IntervalUnit.DAY_TIME:
202
+ return 'interval-daytime';
203
+ case IntervalUnit.YEAR_MONTH:
204
+ return 'interval-yearmonth';
205
+ default:
206
+ return 'interval-daytime';
207
+ }
140
208
  case IntervalDayTime:
141
209
  return 'interval-daytime';
142
210
  case IntervalYearMonth:
143
211
  return 'interval-yearmonth';
212
+ case List:
213
+ const listType = arrowType as List;
214
+ const listField = listType.valueField;
215
+ return {
216
+ type: 'list',
217
+ children: [serializeArrowField(listField)]
218
+ };
144
219
  case FixedSizeList:
145
220
  return {
146
221
  type: 'fixed-size-list',
@@ -159,6 +234,9 @@ export function serializeArrowType(arrowType: ArrowDataType): DataType {
159
234
  export function deserializeArrowType(dataType: DataType): ArrowDataType {
160
235
  if (typeof dataType === 'object') {
161
236
  switch (dataType.type) {
237
+ case 'list':
238
+ const field = deserializeArrowField(dataType.children[0]);
239
+ return new List(field);
162
240
  case 'fixed-size-list':
163
241
  const child = deserializeArrowField(dataType.children[0]);
164
242
  return new FixedSizeList(dataType.listSize, child);
@@ -177,7 +255,6 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
177
255
  return new Binary();
178
256
  case 'bool':
179
257
  return new Bool();
180
- // case 'int': return new Int();
181
258
  case 'int8':
182
259
  return new Int8();
183
260
  case 'int16':
@@ -194,7 +271,6 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
194
271
  return new Uint32();
195
272
  case 'uint64':
196
273
  return new Uint64();
197
- // case 'float': return new Float();
198
274
  case 'float16':
199
275
  return new Float16();
200
276
  case 'float32':
@@ -203,17 +279,18 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
203
279
  return new Float64();
204
280
  case 'utf8':
205
281
  return new Utf8();
206
- // case 'date': return new Date();
207
282
  case 'date-day':
208
283
  return new DateDay();
209
284
  case 'date-millisecond':
210
285
  return new DateMillisecond();
211
- // case 'time': return new Time();
212
- case 'time-millisecond':
213
- return new TimeMillisecond();
214
286
  case 'time-second':
215
287
  return new TimeSecond();
216
- // case 'timestamp': return new Timestamp();
288
+ case 'time-millisecond':
289
+ return new TimeMillisecond();
290
+ case 'time-microsecond':
291
+ return new TimeMicrosecond();
292
+ case 'time-nanosecond':
293
+ return new TimeNanosecond();
217
294
  case 'timestamp-second':
218
295
  return new TimestampSecond();
219
296
  case 'timestamp-millisecond':
@@ -222,7 +299,6 @@ export function deserializeArrowType(dataType: DataType): ArrowDataType {
222
299
  return new TimestampMicrosecond();
223
300
  case 'timestamp-nanosecond':
224
301
  return new TimestampNanosecond();
225
- // case 'interval': return new Interval();
226
302
  case 'interval-daytime':
227
303
  return new IntervalDayTime();
228
304
  case 'interval-yearmonth':
@@ -1,8 +1,9 @@
1
1
  // loaders.gl, MIT license
2
2
  // Copyright (c) vis.gl contributors
3
+
3
4
  import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
4
5
  import type {Table as ApacheArrowTable} from 'apache-arrow';
5
- import type {ArrowTable} from './arrow-table';
6
+ import type {ArrowTable} from '../lib/arrow-table';
6
7
 
7
8
  /**
8
9
  * Wrap an apache arrow table in a loaders.gl table wrapper.
package/src/types.ts CHANGED
@@ -1,3 +1,6 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
1
4
  type TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;
2
5
 
3
6
  type TypedFloatArray = Float32Array | Float64Array;
@@ -1,3 +1,6 @@
1
+ // loaders.gl, MIT license
2
+ // Copyright (c) vis.gl contributors
3
+
1
4
  import {createLoaderWorker} from '@loaders.gl/loader-utils';
2
5
  import {ArrowLoader} from '../index';
3
6
 
@@ -1 +0,0 @@
1
- {"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../src/lib/convert-table.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAC,KAAK,IAAI,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAE9C;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,gBAAgB,GAAG,UAAU,CAKvF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAkB5E;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAmB5F"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\nimport type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\nimport type {ArrowTable} from './arrow-table';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n return {\n shape: 'arrow-table',\n data: arrowTable\n };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n\n const arrowTable = table.data;\n const columnarTable = {};\n\n for (const field of arrowTable.schema.fields) {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getChild(field.name);\n const values = arrowColumn?.toArray();\n columnarTable[field.name] = values;\n }\n\n return {\n shape: 'columnar-table',\n data: columnarTable\n };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n\n return {\n shape: 'object-row-table',\n data: rowFormatTable\n };\n}\n"],"mappings":"AAYA,OAAO,SAASA,8BAA8BA,CAACC,UAA4B,EAAc;EACvF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMA,OAAO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,MAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,MAAMG,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIN,UAAU,CAACO,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,OAAO;IACLX,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMA,OAAO,SAASS,+BAA+BA,CAACT,aAA4B,EAAkB;EAC5F,MAAMU,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACZ,aAAa,CAAC;EAC5C,MAAMa,cAAc,GAAGb,aAAa,CAACU,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGnB,aAAa,CAACmB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACLrB,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAEkB;EACR,CAAC;AACH"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"convert-schema-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-schema-arrow.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAChF,OAAO,EACL,KAAK,IAAI,UAAU,EACnB,MAAM,IAAI,WAAW,EACrB,QAAQ,IAAI,aAAa,EAkC1B,MAAM,cAAc,CAAC;AAEtB,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAKrE;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAKlE;AAED,wBAAgB,sBAAsB,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,cAAc,CAEzF;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,CAAC,EAAE,cAAc,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAEvF;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CAO5D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAO9D;AAED,+EAA+E;AAE/E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,aAAa,GAAG,QAAQ,CAqErE;AAED,+EAA+E;AAE/E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,GAAG,aAAa,CA0EtE"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"convert-schema-arrow.js","names":["Field","ArrowField","Schema","ArrowSchema","Null","Binary","Bool","Int8","Int16","Int32","Int64","Uint8","Uint16","Uint32","Uint64","Float16","Float32","Float64","Utf8","DateDay","DateMillisecond","TimeMillisecond","TimeSecond","TimestampSecond","TimestampMillisecond","TimestampMicrosecond","TimestampNanosecond","IntervalDayTime","IntervalYearMonth","FixedSizeList","Struct","serializeArrowSchema","arrowSchema","fields","map","arrowField","serializeArrowField","metadata","serializeArrowMetadata","deserializeArrowSchema","schema","field","deserializeArrowField","deserializeArrowMetadata","arrowMetadata","Object","fromEntries","Map","entries","name","type","serializeArrowType","nullable","deserializeArrowType","arrowType","constructor","listSize","children","Error","dataType","child"],"sources":["../../src/schema/convert-schema-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';\nimport {\n Field as ArrowField,\n Schema as ArrowSchema,\n DataType as ArrowDataType,\n Null,\n Binary,\n Bool,\n // Int,\n Int8,\n Int16,\n Int32,\n Int64,\n Uint8,\n Uint16,\n Uint32,\n Uint64,\n // Float,\n Float16,\n Float32,\n Float64,\n Utf8,\n // Date,\n DateDay,\n DateMillisecond,\n // Time,\n TimeMillisecond,\n TimeSecond,\n // Timestamp,\n TimestampSecond,\n TimestampMillisecond,\n TimestampMicrosecond,\n TimestampNanosecond,\n // Interval,\n IntervalDayTime,\n IntervalYearMonth,\n FixedSizeList,\n Struct\n} from 'apache-arrow';\n\nexport function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {\n return {\n fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),\n metadata: serializeArrowMetadata(arrowSchema.metadata)\n };\n}\n\nexport function deserializeArrowSchema(schema: Schema): ArrowSchema {\n return new ArrowSchema(\n schema.fields.map((field) => deserializeArrowField(field)),\n deserializeArrowMetadata(schema.metadata)\n );\n}\n\nexport function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {\n return Object.fromEntries(arrowMetadata);\n}\n\nexport function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {\n return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();\n}\n\nexport function serializeArrowField(field: ArrowField): Field {\n return {\n name: field.name,\n type: serializeArrowType(field.type),\n nullable: field.nullable,\n metadata: serializeArrowMetadata(field.metadata)\n };\n}\n\nexport function deserializeArrowField(field: Field): ArrowField {\n return new ArrowField(\n field.name,\n deserializeArrowType(field.type),\n field.nullable,\n deserializeArrowMetadata(field.metadata)\n );\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function serializeArrowType(arrowType: ArrowDataType): DataType {\n switch (arrowType.constructor) {\n case Null:\n return 'null';\n case Binary:\n return 'binary';\n case Bool:\n return 'bool';\n // case Int: return 'int';\n case Int8:\n return 'int8';\n case Int16:\n return 'int16';\n case Int32:\n return 'int32';\n case Int64:\n return 'int64';\n case Uint8:\n return 'uint8';\n case Uint16:\n return 'uint16';\n case Uint32:\n return 'uint32';\n case Uint64:\n return 'uint64';\n // case Float: return 'float';\n case Float16:\n return 'float16';\n case Float32:\n return 'float32';\n case Float64:\n return 'float64';\n case Utf8:\n return 'utf8';\n // case Date: return 'date';\n case DateDay:\n return 'date-day';\n case DateMillisecond:\n return 'date-millisecond';\n // case Time: return 'time';\n case TimeMillisecond:\n return 'time-millisecond';\n case TimeSecond:\n return 'time-second';\n // case Timestamp: return 'timestamp';\n case TimestampSecond:\n return 'timestamp-second';\n case TimestampMillisecond:\n return 'timestamp-millisecond';\n case TimestampMicrosecond:\n return 'timestamp-microsecond';\n case TimestampNanosecond:\n return 'timestamp-nanosecond';\n // case Interval: return 'interval';\n case IntervalDayTime:\n return 'interval-daytime';\n case IntervalYearMonth:\n return 'interval-yearmonth';\n case FixedSizeList:\n return {\n type: 'fixed-size-list',\n listSize: (arrowType as FixedSizeList).listSize,\n children: [serializeArrowField((arrowType as FixedSizeList).children[0])]\n };\n // case Struct:\n // return {type: 'struct', children: (arrowType as Struct).children};\n default:\n throw new Error('array type not supported');\n }\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function deserializeArrowType(dataType: DataType): ArrowDataType {\n if (typeof dataType === 'object') {\n switch (dataType.type) {\n case 'fixed-size-list':\n const child = deserializeArrowField(dataType.children[0]);\n return new FixedSizeList(dataType.listSize, child);\n case 'struct':\n const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));\n return new Struct(children);\n default:\n throw new Error('array type not supported');\n }\n }\n\n switch (dataType) {\n case 'null':\n return new Null();\n case 'binary':\n return new Binary();\n case 'bool':\n return new Bool();\n // case 'int': return new Int();\n case 'int8':\n return new Int8();\n case 'int16':\n return new Int16();\n case 'int32':\n return new Int32();\n case 'int64':\n return new Int64();\n case 'uint8':\n return new Uint8();\n case 'uint16':\n return new Uint16();\n case 'uint32':\n return new Uint32();\n case 'uint64':\n return new Uint64();\n // case 'float': return new Float();\n case 'float16':\n return new Float16();\n case 'float32':\n return new Float32();\n case 'float64':\n return new Float64();\n case 'utf8':\n return new Utf8();\n // case 'date': return new Date();\n case 'date-day':\n return new DateDay();\n case 'date-millisecond':\n return new DateMillisecond();\n // case 'time': return new Time();\n case 'time-millisecond':\n return new TimeMillisecond();\n case 'time-second':\n return new TimeSecond();\n // case 'timestamp': return new Timestamp();\n case 'timestamp-second':\n return new TimestampSecond();\n case 'timestamp-millisecond':\n return new TimestampMillisecond();\n case 'timestamp-microsecond':\n return new TimestampMicrosecond();\n case 'timestamp-nanosecond':\n return new TimestampNanosecond();\n // case 'interval': return new Interval();\n case 'interval-daytime':\n return new IntervalDayTime();\n case 'interval-yearmonth':\n return new IntervalYearMonth();\n default:\n throw new Error('array type not supported');\n }\n}\n"],"mappings":"AAIA,SACEA,KAAK,IAAIC,UAAU,EACnBC,MAAM,IAAIC,WAAW,EAErBC,IAAI,EACJC,MAAM,EACNC,IAAI,EAEJC,IAAI,EACJC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,MAAM,EACNC,MAAM,EACNC,MAAM,EAENC,OAAO,EACPC,OAAO,EACPC,OAAO,EACPC,IAAI,EAEJC,OAAO,EACPC,eAAe,EAEfC,eAAe,EACfC,UAAU,EAEVC,eAAe,EACfC,oBAAoB,EACpBC,oBAAoB,EACpBC,mBAAmB,EAEnBC,eAAe,EACfC,iBAAiB,EACjBC,aAAa,EACbC,MAAM,QACD,cAAc;AAErB,OAAO,SAASC,oBAAoBA,CAACC,WAAwB,EAAU;EACrE,OAAO;IACLC,MAAM,EAAED,WAAW,CAACC,MAAM,CAACC,GAAG,CAAEC,UAAU,IAAKC,mBAAmB,CAACD,UAAU,CAAC,CAAC;IAC/EE,QAAQ,EAAEC,sBAAsB,CAACN,WAAW,CAACK,QAAQ;EACvD,CAAC;AACH;AAEA,OAAO,SAASE,sBAAsBA,CAACC,MAAc,EAAe;EAClE,OAAO,IAAIrC,WAAW,CACpBqC,MAAM,CAACP,MAAM,CAACC,GAAG,CAAEO,KAAK,IAAKC,qBAAqB,CAACD,KAAK,CAAC,CAAC,EAC1DE,wBAAwB,CAACH,MAAM,CAACH,QAAQ,CAC1C,CAAC;AACH;AAEA,OAAO,SAASC,sBAAsBA,CAACM,aAAkC,EAAkB;EACzF,OAAOC,MAAM,CAACC,WAAW,CAACF,aAAa,CAAC;AAC1C;AAEA,OAAO,SAASD,wBAAwBA,CAACN,QAAyB,EAAuB;EACvF,OAAOA,QAAQ,GAAG,IAAIU,GAAG,CAACF,MAAM,CAACG,OAAO,CAACX,QAAQ,CAAC,CAAC,GAAG,IAAIU,GAAG,CAAiB,CAAC;AACjF;AAEA,OAAO,SAASX,mBAAmBA,CAACK,KAAiB,EAAS;EAC5D,OAAO;IACLQ,IAAI,EAAER,KAAK,CAACQ,IAAI;IAChBC,IAAI,EAAEC,kBAAkB,CAACV,KAAK,CAACS,IAAI,CAAC;IACpCE,QAAQ,EAAEX,KAAK,CAACW,QAAQ;IACxBf,QAAQ,EAAEC,sBAAsB,CAACG,KAAK,CAACJ,QAAQ;EACjD,CAAC;AACH;AAEA,OAAO,SAASK,qBAAqBA,CAACD,KAAY,EAAc;EAC9D,OAAO,IAAIxC,UAAU,CACnBwC,KAAK,CAACQ,IAAI,EACVI,oBAAoB,CAACZ,KAAK,CAACS,IAAI,CAAC,EAChCT,KAAK,CAACW,QAAQ,EACdT,wBAAwB,CAACF,KAAK,CAACJ,QAAQ,CACzC,CAAC;AACH;AAIA,OAAO,SAASc,kBAAkBA,CAACG,SAAwB,EAAY;EACrE,QAAQA,SAAS,CAACC,WAAW;IAC3B,KAAKnD,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IAEjB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,OAAO;MACV,OAAO,UAAU;IACnB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAE3B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,UAAU;MACb,OAAO,aAAa;IAEtB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,mBAAmB;MACtB,OAAO,sBAAsB;IAE/B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,iBAAiB;MACpB,OAAO,oBAAoB;IAC7B,KAAKC,aAAa;MAChB,OAAO;QACLqB,IAAI,EAAE,iBAAiB;QACvBM,QAAQ,EAAGF,SAAS,CAAmBE,QAAQ;QAC/CC,QAAQ,EAAE,CAACrB,mBAAmB,CAAEkB,SAAS,CAAmBG,QAAQ,CAAC,CAAC,CAAC,CAAC;MAC1E,CAAC;IAGH;MACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF;AAIA,OAAO,SAASL,oBAAoBA,CAACM,QAAkB,EAAiB;EACtE,IAAI,OAAOA,QAAQ,KAAK,QAAQ,EAAE;IAChC,QAAQA,QAAQ,CAACT,IAAI;MACnB,KAAK,iBAAiB;QACpB,MAAMU,KAAK,GAAGlB,qBAAqB,CAACiB,QAAQ,CAACF,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI5B,aAAa,CAAC8B,QAAQ,CAACH,QAAQ,EAAEI,KAAK,CAAC;MACpD,KAAK,QAAQ;QACX,MAAMH,QAAQ,GAAGE,QAAQ,CAACF,QAAQ,CAACvB,GAAG,CAAEC,UAAU,IAAKO,qBAAqB,CAACP,UAAU,CAAC,CAAC;QACzF,OAAO,IAAIL,MAAM,CAAC2B,QAAQ,CAAC;MAC7B;QACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;IAC/C;EACF;EAEA,QAAQC,QAAQ;IACd,KAAK,MAAM;MACT,OAAO,IAAIvD,IAAI,CAAC,CAAC;IACnB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IACnB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IAErB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,UAAU;MACb,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAE9B,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,aAAa;MAChB,OAAO,IAAIC,UAAU,CAAC,CAAC;IAEzB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,sBAAsB;MACzB,OAAO,IAAIC,mBAAmB,CAAC,CAAC;IAElC,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,oBAAoB;MACvB,OAAO,IAAIC,iBAAiB,CAAC,CAAC;IAChC;MACE,MAAM,IAAI8B,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF"}