@loaders.gl/geospatial 5.0.0-beta.10 → 5.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dbf-loader-types.cjs +1 -1
- package/dist/dbf-loader.cjs +2 -4
- package/dist/dbf-loader.cjs.map +2 -2
- package/dist/dbf-worker.js +22 -79
- package/dist/flatgeobuf/flatgeobuf-loader-types.cjs +1 -1
- package/dist/flatgeobuf/flatgeobuf-loader-types.js +1 -1
- package/dist/flatgeobuf/lib/parse-flatgeobuf.cjs +6 -20
- package/dist/flatgeobuf/lib/parse-flatgeobuf.cjs.map +2 -2
- package/dist/flatgeobuf/lib/parse-flatgeobuf.d.ts +4 -4
- package/dist/flatgeobuf/lib/parse-flatgeobuf.d.ts.map +1 -1
- package/dist/flatgeobuf/lib/parse-flatgeobuf.js +9 -22
- package/dist/flatgeobuf/lib/parse-flatgeobuf.js.map +1 -1
- package/dist/flatgeobuf-loader-types.cjs +1 -1
- package/dist/flatgeobuf-worker.js +655 -77
- package/dist/geopackage/geopackage-source-loader.cjs +1 -1
- package/dist/geopackage/geopackage-source-loader.cjs.map +2 -2
- package/dist/geopackage/geopackage-source-loader.js +1 -1
- package/dist/geopackage/geopackage-source-loader.js.map +1 -1
- package/dist/geopackage/lib/parse-geopackage.cjs +2 -6
- package/dist/geopackage/lib/parse-geopackage.cjs.map +2 -2
- package/dist/geopackage/lib/parse-geopackage.d.ts +3 -3
- package/dist/geopackage/lib/parse-geopackage.d.ts.map +1 -1
- package/dist/geopackage/lib/parse-geopackage.js +2 -6
- package/dist/geopackage/lib/parse-geopackage.js.map +1 -1
- package/dist/geopackage-source-loader.cjs +1 -1
- package/dist/geopackage-source-loader.cjs.map +2 -2
- package/dist/gpx-loader-types.cjs +1 -1
- package/dist/kml/gpx-loader-types.cjs +1 -1
- package/dist/kml/gpx-loader-types.js +1 -1
- package/dist/kml/kml-loader-types.cjs +1 -1
- package/dist/kml/kml-loader-types.js +1 -1
- package/dist/kml/kml-writer.cjs +1 -1
- package/dist/kml/kml-writer.js +1 -1
- package/dist/kml/kmz-loader-types.cjs +1 -1
- package/dist/kml/kmz-loader-types.js +1 -1
- package/dist/kml/lib/feature-collection-to-arrow.cjs +1 -1
- package/dist/kml/lib/feature-collection-to-arrow.cjs.map +2 -2
- package/dist/kml/lib/feature-collection-to-arrow.js +1 -1
- package/dist/kml/lib/feature-collection-to-arrow.js.map +1 -1
- package/dist/kml/tcx-loader-types.cjs +1 -1
- package/dist/kml/tcx-loader-types.js +1 -1
- package/dist/kml-loader-types.cjs +1 -1
- package/dist/kml-writer.cjs +1 -1
- package/dist/kmz-loader-types.cjs +1 -1
- package/dist/shapefile/dbf-loader-types.cjs +1 -1
- package/dist/shapefile/dbf-loader-types.js +1 -1
- package/dist/shapefile/dbf-loader.cjs +2 -4
- package/dist/shapefile/dbf-loader.cjs.map +2 -2
- package/dist/shapefile/dbf-loader.d.ts.map +1 -1
- package/dist/shapefile/dbf-loader.js +3 -5
- package/dist/shapefile/dbf-loader.js.map +1 -1
- package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.cjs +1 -5
- package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.cjs.map +2 -2
- package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.d.ts +5 -4
- package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.d.ts.map +1 -1
- package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.js +2 -6
- package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.js.map +1 -1
- package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.cjs.map +2 -2
- package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.d.ts +4 -3
- package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.d.ts.map +1 -1
- package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.js.map +1 -1
- package/dist/shapefile/lib/parsers/parse-shp-to-arrow.cjs +7 -10
- package/dist/shapefile/lib/parsers/parse-shp-to-arrow.cjs.map +2 -2
- package/dist/shapefile/lib/parsers/parse-shp-to-arrow.d.ts +3 -3
- package/dist/shapefile/lib/parsers/parse-shp-to-arrow.d.ts.map +1 -1
- package/dist/shapefile/lib/parsers/parse-shp-to-arrow.js +11 -9
- package/dist/shapefile/lib/parsers/parse-shp-to-arrow.js.map +1 -1
- package/dist/shapefile/lib/shapefile-worker-transport.cjs +0 -3
- package/dist/shapefile/lib/shapefile-worker-transport.cjs.map +2 -2
- package/dist/shapefile/lib/shapefile-worker-transport.d.ts.map +1 -1
- package/dist/shapefile/lib/shapefile-worker-transport.js +1 -5
- package/dist/shapefile/lib/shapefile-worker-transport.js.map +1 -1
- package/dist/shapefile/shapefile-arrow-loader.cjs +53 -48
- package/dist/shapefile/shapefile-arrow-loader.cjs.map +2 -2
- package/dist/shapefile/shapefile-arrow-loader.d.ts +5 -3
- package/dist/shapefile/shapefile-arrow-loader.d.ts.map +1 -1
- package/dist/shapefile/shapefile-arrow-loader.js +54 -52
- package/dist/shapefile/shapefile-arrow-loader.js.map +1 -1
- package/dist/shapefile/shapefile-loader-types.cjs +1 -1
- package/dist/shapefile/shapefile-loader-types.d.ts +4 -4
- package/dist/shapefile/shapefile-loader-types.js +1 -1
- package/dist/shapefile/shapefile-loader.cjs +19 -55
- package/dist/shapefile/shapefile-loader.cjs.map +2 -2
- package/dist/shapefile/shapefile-loader.d.ts +5 -42
- package/dist/shapefile/shapefile-loader.d.ts.map +1 -1
- package/dist/shapefile/shapefile-loader.js +22 -64
- package/dist/shapefile/shapefile-loader.js.map +1 -1
- package/dist/shapefile/shp-loader-types.cjs +1 -1
- package/dist/shapefile/shp-loader-types.js +1 -1
- package/dist/shapefile/shp-loader.cjs +1 -44
- package/dist/shapefile/shp-loader.cjs.map +2 -2
- package/dist/shapefile/shp-loader.d.ts +1 -115
- package/dist/shapefile/shp-loader.d.ts.map +1 -1
- package/dist/shapefile/shp-loader.js +1 -48
- package/dist/shapefile/shp-loader.js.map +1 -1
- package/dist/shapefile-arrow-loader.cjs +53 -48
- package/dist/shapefile-arrow-loader.cjs.map +2 -2
- package/dist/shapefile-loader-types.cjs +1 -1
- package/dist/shapefile-loader.cjs +19 -55
- package/dist/shapefile-loader.cjs.map +2 -2
- package/dist/shapefile-worker.js +117 -272
- package/dist/shp-loader-types.cjs +1 -1
- package/dist/shp-loader.cjs +1 -44
- package/dist/shp-loader.cjs.map +2 -2
- package/dist/shp-worker.js +16964 -17085
- package/dist/tcx-loader-types.cjs +1 -1
- package/dist/wkt/lib/version.cjs +1 -1
- package/dist/wkt/lib/version.js +1 -1
- package/dist/wkt-worker.js +1 -1
- package/dist/workers/shapefile-worker.cjs +3 -3
- package/dist/workers/shapefile-worker.cjs.map +2 -2
- package/dist/workers/shapefile-worker.js +4 -4
- package/dist/workers/shapefile-worker.js.map +1 -1
- package/dist/workers/shp-worker.cjs +1 -1
- package/dist/workers/shp-worker.cjs.map +2 -2
- package/dist/workers/shp-worker.js +2 -2
- package/dist/workers/shp-worker.js.map +1 -1
- package/package.json +6 -6
- package/src/flatgeobuf/lib/parse-flatgeobuf.ts +15 -33
- package/src/geopackage/geopackage-source-loader.ts +1 -1
- package/src/geopackage/lib/parse-geopackage.ts +4 -9
- package/src/kml/lib/feature-collection-to-arrow.ts +1 -1
- package/src/shapefile/dbf-loader.ts +4 -7
- package/src/shapefile/lib/parsers/build-geoarrow-geometry-arrow.ts +5 -9
- package/src/shapefile/lib/parsers/build-wkb-geometry-arrow.ts +4 -3
- package/src/shapefile/lib/parsers/parse-shp-to-arrow.ts +14 -20
- package/src/shapefile/lib/shapefile-worker-transport.ts +1 -6
- package/src/shapefile/shapefile-arrow-loader.ts +76 -70
- package/src/shapefile/shapefile-loader.ts +40 -90
- package/src/shapefile/shp-loader.ts +7 -61
- package/src/workers/shapefile-worker.ts +4 -4
- package/src/workers/shp-worker.ts +2 -2
|
@@ -4,20 +4,14 @@
|
|
|
4
4
|
|
|
5
5
|
import {CRSReprojectionError, type LoaderContext} from '@loaders.gl/core/loader-utils';
|
|
6
6
|
import {
|
|
7
|
-
makeTableScanBatch,
|
|
8
7
|
parseFromContext,
|
|
9
8
|
parseInBatchesFromContext,
|
|
10
9
|
toArrayBufferIterator
|
|
11
10
|
} from '@loaders.gl/core/loader-utils';
|
|
12
11
|
import * as arrow from 'apache-arrow';
|
|
13
|
-
import {convertArrowToSchema,
|
|
14
|
-
import type {
|
|
15
|
-
|
|
16
|
-
ArrowTableBatch,
|
|
17
|
-
Field,
|
|
18
|
-
Schema as TableSchema
|
|
19
|
-
} from '@loaders.gl/core/schema';
|
|
20
|
-
import {ArrowTableBuilder, convertSchemaToArrow} from '@loaders.gl/core/schema-utils';
|
|
12
|
+
import {convertArrowToSchema, isArrowRecordBatch} from '@loaders.gl/core/schema-utils';
|
|
13
|
+
import type {Field, Schema as TableSchema} from '@loaders.gl/core/schema';
|
|
14
|
+
import {makeArrowMetadataBatch, convertSchemaToArrow} from '@loaders.gl/core/schema-utils';
|
|
21
15
|
import {convertFeaturesToGeoArrowTable} from '@loaders.gl/core/arrow-geometry/feature-table';
|
|
22
16
|
import {convertWKBToGeometry} from '@loaders.gl/core/arrow-geometry/geometry-codecs';
|
|
23
17
|
import {type GeoParquetGeometryType} from '@loaders.gl/core/schema';
|
|
@@ -45,12 +39,12 @@ export async function parseShapefileToArrow(
|
|
|
45
39
|
arrayBuffer: ArrayBuffer,
|
|
46
40
|
options?: ShapefileLoaderOptions,
|
|
47
41
|
context?: LoaderContext
|
|
48
|
-
): Promise<
|
|
42
|
+
): Promise<arrow.Table> {
|
|
49
43
|
const {cpg, prj, shx} = await loadShapefileSidecarFiles(options, context);
|
|
50
44
|
const transform = getReprojectionTransform(prj, options);
|
|
51
45
|
const geoArrowEncoding = getTypedGeoArrowEncoding(options);
|
|
52
46
|
let header: SHPHeader | undefined;
|
|
53
|
-
let geometryTable:
|
|
47
|
+
let geometryTable: arrow.Table;
|
|
54
48
|
|
|
55
49
|
if (geoArrowEncoding) {
|
|
56
50
|
geometryTable = makeSHPGeoArrowGeometryTable(arrayBuffer, options, {transform});
|
|
@@ -77,7 +71,7 @@ export async function parseShapefileToArrow(
|
|
|
77
71
|
options?.geoarrow?.encodingPreference === 'geoarrow.geometry' ||
|
|
78
72
|
options?.shapefile?.geoarrow?.encodingPreference === 'geoarrow.geometry'
|
|
79
73
|
) {
|
|
80
|
-
const geometryColumn = geometryTable.
|
|
74
|
+
const geometryColumn = geometryTable.getChild(GEOMETRY_COLUMN_NAME);
|
|
81
75
|
const geometries = geometryColumn?.toArray() || [];
|
|
82
76
|
geometryTable = convertFeaturesToGeoArrowTable(
|
|
83
77
|
geometries.map(geometry => ({
|
|
@@ -91,7 +85,7 @@ export async function parseShapefileToArrow(
|
|
|
91
85
|
}
|
|
92
86
|
|
|
93
87
|
let propertySchema: TableSchema | null = null;
|
|
94
|
-
let propertyTable:
|
|
88
|
+
let propertyTable: arrow.Table | null = null;
|
|
95
89
|
|
|
96
90
|
const dbfResponse = context?.url
|
|
97
91
|
? await context.fetch(replaceExtension(context.url, 'dbf')).catch(() => null)
|
|
@@ -111,14 +105,21 @@ export async function parseShapefileToArrow(
|
|
|
111
105
|
context!
|
|
112
106
|
)) as arrow.Table;
|
|
113
107
|
propertySchema = convertArrowToSchema(propertyData.schema);
|
|
114
|
-
propertyTable =
|
|
108
|
+
propertyTable = propertyData;
|
|
115
109
|
}
|
|
116
110
|
|
|
117
|
-
const schema = buildOutputSchema(
|
|
111
|
+
const schema = buildOutputSchema(
|
|
112
|
+
propertySchema,
|
|
113
|
+
header,
|
|
114
|
+
convertArrowToSchema(geometryTable.schema)
|
|
115
|
+
);
|
|
118
116
|
const table = propertyTable
|
|
119
117
|
? appendGeometryColumnToArrowTable(propertyTable, geometryTable, schema)
|
|
120
118
|
: geometryTable;
|
|
121
|
-
|
|
119
|
+
const encoded = encodeShapefileHeader({header, encoding: cpg, prj, shx});
|
|
120
|
+
table.schema.metadata.set(SHAPEFILE_HEADER_KEY, encoded);
|
|
121
|
+
for (const batch of table.batches) batch.schema.metadata.set(SHAPEFILE_HEADER_KEY, encoded);
|
|
122
|
+
return table;
|
|
122
123
|
}
|
|
123
124
|
|
|
124
125
|
/** Parses a shapefile into Arrow batches while keeping DBF-derived schema stable. */
|
|
@@ -128,7 +129,7 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
128
129
|
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
129
130
|
options?: ShapefileLoaderOptions,
|
|
130
131
|
context?: LoaderContext
|
|
131
|
-
): AsyncIterable<
|
|
132
|
+
): AsyncIterable<arrow.RecordBatch> {
|
|
132
133
|
if (getTypedGeoArrowEncoding(options)) {
|
|
133
134
|
throw new Error('Typed GeoArrow shapefile output is only supported for non-streaming parse.');
|
|
134
135
|
}
|
|
@@ -163,6 +164,7 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
163
164
|
|
|
164
165
|
const shapeHeader = await getNextNonMetadataValue(shapeIterator);
|
|
165
166
|
const header = shapeHeader as SHPHeader;
|
|
167
|
+
const encodedHeader = encodeShapefileHeader({header, encoding: cpg, prj, shx});
|
|
166
168
|
|
|
167
169
|
let propertyIterator: AsyncIterator<any> | null = null;
|
|
168
170
|
let propertySchema: TableSchema | null = null;
|
|
@@ -189,14 +191,14 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
189
191
|
propertyIterator = getAsyncIterator(propertyIterable);
|
|
190
192
|
|
|
191
193
|
const firstPropertyBatch = await getNextArrowBatch(propertyIterator);
|
|
192
|
-
propertySchema = firstPropertyBatch
|
|
194
|
+
propertySchema = firstPropertyBatch ? convertArrowToSchema(firstPropertyBatch.schema) : null;
|
|
193
195
|
const outputSchema = buildOutputSchema(propertySchema, header);
|
|
194
196
|
const propertyQueue: arrow.Table[] = [];
|
|
195
197
|
const geometryQueue: arrow.Table[] = [];
|
|
196
198
|
let yieldedDataBatch = false;
|
|
197
199
|
|
|
198
|
-
if (firstPropertyBatch && firstPropertyBatch.
|
|
199
|
-
propertyQueue.push(firstPropertyBatch.
|
|
200
|
+
if (firstPropertyBatch && firstPropertyBatch.numRows > 0) {
|
|
201
|
+
propertyQueue.push(new arrow.Table(firstPropertyBatch.schema, [firstPropertyBatch]));
|
|
200
202
|
}
|
|
201
203
|
|
|
202
204
|
let shapeDone = false;
|
|
@@ -215,7 +217,6 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
215
217
|
const transform = getReprojectionTransform(prj, options);
|
|
216
218
|
geometryQueue.push(
|
|
217
219
|
makeGeometryArrowTable(shapeBatch.value as (SHPWKBGeometry | null)[], header, transform)
|
|
218
|
-
.data
|
|
219
220
|
);
|
|
220
221
|
}
|
|
221
222
|
}
|
|
@@ -224,8 +225,8 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
224
225
|
const propertyBatch = await getNextArrowBatch(propertyIterator);
|
|
225
226
|
if (!propertyBatch) {
|
|
226
227
|
propertyDone = true;
|
|
227
|
-
} else if (propertyBatch.
|
|
228
|
-
propertyQueue.push(propertyBatch.
|
|
228
|
+
} else if (propertyBatch.numRows > 0) {
|
|
229
|
+
propertyQueue.push(new arrow.Table(propertyBatch.schema, [propertyBatch]));
|
|
229
230
|
}
|
|
230
231
|
}
|
|
231
232
|
|
|
@@ -242,16 +243,14 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
242
243
|
|
|
243
244
|
const propertyTable = takeRowsFromQueue(propertyQueue, rowCount);
|
|
244
245
|
const geometryTable = takeRowsFromQueue(geometryQueue, rowCount);
|
|
245
|
-
const batch = appendGeometryColumnToArrowTable(
|
|
246
|
-
{shape: 'arrow-table', schema: propertySchema || undefined, data: propertyTable},
|
|
247
|
-
{shape: 'arrow-table', data: geometryTable},
|
|
248
|
-
outputSchema
|
|
249
|
-
);
|
|
246
|
+
const batch = appendGeometryColumnToArrowTable(propertyTable, geometryTable, outputSchema);
|
|
250
247
|
yieldedDataBatch = true;
|
|
251
|
-
|
|
248
|
+
const recordBatch = batch.batches[0];
|
|
249
|
+
recordBatch.schema.metadata.set(SHAPEFILE_HEADER_KEY, encodedHeader);
|
|
250
|
+
yield recordBatch;
|
|
252
251
|
}
|
|
253
252
|
if (!yieldedDataBatch) {
|
|
254
|
-
yield
|
|
253
|
+
yield makeEmptyArrowBatch(outputSchema, encodedHeader);
|
|
255
254
|
}
|
|
256
255
|
return;
|
|
257
256
|
}
|
|
@@ -274,13 +273,13 @@ export async function* parseShapefileToArrowInBatches(
|
|
|
274
273
|
transform
|
|
275
274
|
);
|
|
276
275
|
yieldedDataBatch = true;
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
}
|
|
276
|
+
for (const batch of arrowTable.batches) {
|
|
277
|
+
batch.schema.metadata.set(SHAPEFILE_HEADER_KEY, encodedHeader);
|
|
278
|
+
yield batch;
|
|
279
|
+
}
|
|
281
280
|
}
|
|
282
281
|
if (!yieldedDataBatch) {
|
|
283
|
-
yield
|
|
282
|
+
yield makeEmptyArrowBatch(outputSchema, encodedHeader);
|
|
284
283
|
}
|
|
285
284
|
}
|
|
286
285
|
|
|
@@ -320,24 +319,20 @@ function makeGeometryArrowTable(
|
|
|
320
319
|
}
|
|
321
320
|
|
|
322
321
|
function appendGeometryColumnToArrowTable(
|
|
323
|
-
propertyTable:
|
|
324
|
-
geometryTable:
|
|
322
|
+
propertyTable: arrow.Table,
|
|
323
|
+
geometryTable: arrow.Table,
|
|
325
324
|
schema: TableSchema
|
|
326
|
-
):
|
|
327
|
-
const propertyBatch = propertyTable.
|
|
328
|
-
const geometryBatch = geometryTable.
|
|
325
|
+
): arrow.Table {
|
|
326
|
+
const propertyBatch = propertyTable.batches[0];
|
|
327
|
+
const geometryBatch = geometryTable.batches[0];
|
|
329
328
|
const arrowSchema = convertSchemaToArrow(schema);
|
|
330
329
|
const structField = new arrow.Struct(arrowSchema.fields);
|
|
331
330
|
const children = [...propertyBatch.data.children, geometryBatch.data.children[0]];
|
|
332
|
-
const rowCount = Math.max(propertyTable.
|
|
331
|
+
const rowCount = Math.max(propertyTable.numRows, geometryTable.numRows);
|
|
333
332
|
const structData = new arrow.Data(structField, 0, rowCount, 0, undefined, children);
|
|
334
333
|
const recordBatch = new arrow.RecordBatch(arrowSchema, structData);
|
|
335
334
|
|
|
336
|
-
return
|
|
337
|
-
shape: 'arrow-table',
|
|
338
|
-
schema,
|
|
339
|
-
data: new arrow.Table(arrowSchema, [recordBatch])
|
|
340
|
-
};
|
|
335
|
+
return new arrow.Table(arrowSchema, [recordBatch]);
|
|
341
336
|
}
|
|
342
337
|
|
|
343
338
|
function getReprojectionTransform(
|
|
@@ -429,28 +424,19 @@ async function getNextNonMetadataValue(iterator: AsyncIterator<any>): Promise<an
|
|
|
429
424
|
}
|
|
430
425
|
}
|
|
431
426
|
|
|
432
|
-
/** Reads
|
|
433
|
-
async function getNextArrowBatch(
|
|
427
|
+
/** Reads native DBF batches and imports columns originating from another Arrow build. */
|
|
428
|
+
async function getNextArrowBatch(
|
|
429
|
+
iterator: AsyncIterator<unknown>
|
|
430
|
+
): Promise<arrow.RecordBatch | null> {
|
|
434
431
|
while (true) {
|
|
435
432
|
const result = await iterator.next();
|
|
436
|
-
if (result.done)
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
const batch =
|
|
443
|
-
Object.getPrototypeOf(recordBatch) === arrow.RecordBatch.prototype
|
|
444
|
-
? recordBatch
|
|
445
|
-
: importDBFRecordBatch(recordBatch, schema);
|
|
446
|
-
const data = new arrow.Table(batch.schema, [batch]);
|
|
447
|
-
return makeTableScanBatch({
|
|
448
|
-
shape: 'arrow-table',
|
|
449
|
-
schema,
|
|
450
|
-
data
|
|
451
|
-
});
|
|
433
|
+
if (result.done) return null;
|
|
434
|
+
if (isArrowRecordBatch(result.value)) {
|
|
435
|
+
const batch = result.value;
|
|
436
|
+
return Object.getPrototypeOf(batch) === arrow.RecordBatch.prototype
|
|
437
|
+
? batch
|
|
438
|
+
: importDBFRecordBatch(batch, convertArrowToSchema(batch.schema));
|
|
452
439
|
}
|
|
453
|
-
if (result.value?.shape === 'arrow-table') return result.value;
|
|
454
440
|
}
|
|
455
441
|
}
|
|
456
442
|
|
|
@@ -492,12 +478,32 @@ function takeRowsFromQueue(queue: arrow.Table[], rowCount: number): arrow.Table
|
|
|
492
478
|
return table;
|
|
493
479
|
}
|
|
494
480
|
const result = table.slice(0, rowCount);
|
|
495
|
-
queue[0] = table.slice(rowCount
|
|
481
|
+
queue[0] = table.slice(rowCount);
|
|
496
482
|
return result;
|
|
497
483
|
}
|
|
498
484
|
|
|
499
|
-
/** Creates an explicit empty Arrow batch so zero-row shapefiles still expose schema
|
|
500
|
-
function makeEmptyArrowBatch(schema: TableSchema):
|
|
501
|
-
const
|
|
502
|
-
|
|
485
|
+
/** Creates an explicit empty Arrow batch so zero-row shapefiles still expose their schema. */
|
|
486
|
+
function makeEmptyArrowBatch(schema: TableSchema, header: string): arrow.RecordBatch {
|
|
487
|
+
const nativeSchema = convertSchemaToArrow(schema);
|
|
488
|
+
nativeSchema.metadata.set(SHAPEFILE_HEADER_KEY, header);
|
|
489
|
+
return makeArrowMetadataBatch(nativeSchema);
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** Schema metadata key for the Shapefile header and sidecar data. */
|
|
493
|
+
export const SHAPEFILE_HEADER_KEY = 'loaders.gl.shapefile.header';
|
|
494
|
+
|
|
495
|
+
/** Encodes Shapefile sidecars for schema and worker transport. */
|
|
496
|
+
function encodeShapefileHeader({
|
|
497
|
+
shx,
|
|
498
|
+
...header
|
|
499
|
+
}: {
|
|
500
|
+
header?: SHPHeader;
|
|
501
|
+
encoding?: string;
|
|
502
|
+
prj?: WKTCRSDefinition;
|
|
503
|
+
shx?: {offsets: Int32Array; lengths: Int32Array};
|
|
504
|
+
}): string {
|
|
505
|
+
return JSON.stringify({
|
|
506
|
+
...header,
|
|
507
|
+
...(shx ? {shx: {offsets: Array.from(shx.offsets), lengths: Array.from(shx.lengths)}} : {})
|
|
508
|
+
});
|
|
503
509
|
}
|
|
@@ -4,15 +4,9 @@ import {convertWKBToGeometry} from '@loaders.gl/core/arrow-geometry/geometry-cod
|
|
|
4
4
|
// Copyright (c) vis.gl contributors
|
|
5
5
|
|
|
6
6
|
import type {Table, RecordBatch} from 'apache-arrow';
|
|
7
|
-
import {
|
|
7
|
+
import {convertArrowToSchema} from '@loaders.gl/core/schema-utils';
|
|
8
8
|
import type {StrictLoaderOptions, LoaderWithParser} from '@loaders.gl/core/loader-utils';
|
|
9
|
-
import type {
|
|
10
|
-
Batch,
|
|
11
|
-
GeoArrowEncodingPreference,
|
|
12
|
-
GeoJSONTable,
|
|
13
|
-
ArrowTable,
|
|
14
|
-
ArrowTableBatch
|
|
15
|
-
} from '@loaders.gl/core/schema';
|
|
9
|
+
import type {Batch, GeoArrowEncodingPreference, GeoJSONTable} from '@loaders.gl/core/schema';
|
|
16
10
|
import {SHPLoaderOptions} from './shp-loader-types';
|
|
17
11
|
import {
|
|
18
12
|
parseShapefile,
|
|
@@ -21,7 +15,11 @@ import {
|
|
|
21
15
|
} from './lib/parsers/parse-shapefile';
|
|
22
16
|
import {DBFLoaderOptions} from './dbf-loader-types';
|
|
23
17
|
import type {SHPGeoArrowEncoding} from './lib/parsers/types';
|
|
24
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
parseShapefileToArrow,
|
|
20
|
+
parseShapefileToArrowInBatches,
|
|
21
|
+
SHAPEFILE_HEADER_KEY
|
|
22
|
+
} from './shapefile-arrow-loader';
|
|
25
23
|
import {ShapefileLoader as ShapefileLoaderMetadata} from './shapefile-loader-types';
|
|
26
24
|
import {
|
|
27
25
|
deserializeShapefileWorkerResult,
|
|
@@ -53,10 +51,10 @@ export type ShapefileLoaderOptions = StrictLoaderOptions &
|
|
|
53
51
|
* Shapefile loader
|
|
54
52
|
* @note Shapefile is multifile format and requires providing additional files
|
|
55
53
|
*/
|
|
56
|
-
export const
|
|
54
|
+
export const ShapefileLoaderWithParser = {
|
|
57
55
|
...ShapefileLoaderMetadataWithoutPreload,
|
|
58
|
-
dataType: null as unknown as ShapefileOutput | GeoJSONTable |
|
|
59
|
-
batchType: null as unknown as ShapefileOutput | Batch |
|
|
56
|
+
dataType: null as unknown as ShapefileOutput | GeoJSONTable | Table,
|
|
57
|
+
batchType: null as unknown as ShapefileOutput | Batch | RecordBatch,
|
|
60
58
|
parse: (arrayBuffer, options, context) =>
|
|
61
59
|
getShapefileShape(options) === 'arrow'
|
|
62
60
|
? parseShapefileToArrow(arrayBuffer, options, context)
|
|
@@ -66,12 +64,12 @@ export const ShapefileColumnLoader = {
|
|
|
66
64
|
? parseShapefileToArrowInBatches(asyncIterator, options, context)
|
|
67
65
|
: parseShapefileInBatches(asyncIterator, options, context),
|
|
68
66
|
serializeWorkerResult: serializeShapefileWorkerResult,
|
|
69
|
-
deserializeWorkerResult: deserializeShapefileOutput,
|
|
67
|
+
deserializeWorkerResult: (result, options) => deserializeShapefileOutput(result, options, false),
|
|
70
68
|
serializeWorkerBatch: serializeShapefileWorkerBatch,
|
|
71
|
-
deserializeWorkerBatch: deserializeShapefileOutput
|
|
69
|
+
deserializeWorkerBatch: (result, options) => deserializeShapefileOutput(result, options, true)
|
|
72
70
|
} as const satisfies LoaderWithParser<
|
|
73
|
-
ShapefileOutput | GeoJSONTable |
|
|
74
|
-
ShapefileOutput | Batch |
|
|
71
|
+
ShapefileOutput | GeoJSONTable | Table,
|
|
72
|
+
ShapefileOutput | Batch | RecordBatch,
|
|
75
73
|
ShapefileLoaderOptions
|
|
76
74
|
>;
|
|
77
75
|
|
|
@@ -81,94 +79,46 @@ function getShapefileShape(
|
|
|
81
79
|
return options?.shapefile?.shape || 'arrow';
|
|
82
80
|
}
|
|
83
81
|
|
|
84
|
-
/** Materializes requested GeoJSON features from
|
|
85
|
-
function deserializeShapefileOutput(
|
|
86
|
-
|
|
82
|
+
/** Materializes requested GeoJSON features from native worker columns on the caller. */
|
|
83
|
+
function deserializeShapefileOutput(
|
|
84
|
+
result: unknown,
|
|
85
|
+
options: ShapefileLoaderOptions | undefined,
|
|
86
|
+
isBatch: boolean
|
|
87
|
+
): any {
|
|
88
|
+
const table = deserializeShapefileWorkerResult<Table | RecordBatch>(result);
|
|
87
89
|
if (getShapefileShape(options) === 'arrow') return table;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
+
|
|
91
|
+
const header = JSON.parse(table.schema.metadata.get(SHAPEFILE_HEADER_KEY) || '{}');
|
|
92
|
+
const loaderData = header.shx
|
|
93
|
+
? {
|
|
94
|
+
...header,
|
|
95
|
+
shx: {
|
|
96
|
+
offsets: Int32Array.from(header.shx.offsets),
|
|
97
|
+
lengths: Int32Array.from(header.shx.lengths)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
: header;
|
|
101
|
+
const features = table.toArray().map(row => {
|
|
102
|
+
const {geometry, ...properties} = row!.toJSON();
|
|
90
103
|
return {
|
|
91
104
|
type: 'Feature',
|
|
92
105
|
geometry: geometry ? convertWKBToGeometry(geometry.slice().buffer) : null,
|
|
93
106
|
properties
|
|
94
107
|
};
|
|
95
108
|
});
|
|
96
|
-
if (getShapefileShape(options) === 'geojson-table' && !
|
|
109
|
+
if (getShapefileShape(options) === 'geojson-table' && !isBatch) {
|
|
97
110
|
return {
|
|
98
|
-
...
|
|
111
|
+
...loaderData,
|
|
99
112
|
shape: 'geojson-table',
|
|
100
113
|
type: 'FeatureCollection',
|
|
101
114
|
schema: {
|
|
102
|
-
fields: table.schema
|
|
115
|
+
fields: convertArrowToSchema(table.schema).fields.filter(
|
|
116
|
+
field => field.name !== 'geometry'
|
|
117
|
+
),
|
|
103
118
|
metadata: {}
|
|
104
119
|
},
|
|
105
120
|
features
|
|
106
121
|
};
|
|
107
122
|
}
|
|
108
|
-
return {...
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/** Exposes native Arrow output after parsing or worker hydration. */
|
|
112
|
-
export const ShapefileLoaderWithParser = {
|
|
113
|
-
...ShapefileColumnLoader,
|
|
114
|
-
dataType: null as unknown as ShapefileOutput | GeoJSONTable | Table,
|
|
115
|
-
batchType: null as unknown as ShapefileOutput | Batch | RecordBatch,
|
|
116
|
-
async parse(input, options, context) {
|
|
117
|
-
return unwrapShapefileTable(await ShapefileColumnLoader.parse!(input, options, context));
|
|
118
|
-
},
|
|
119
|
-
async *parseInBatches(input, options, context) {
|
|
120
|
-
for await (const batch of ShapefileColumnLoader.parseInBatches!(input, options, context)) {
|
|
121
|
-
if (isShapefileArrowTable(batch)) {
|
|
122
|
-
const table = unwrapShapefileTable(batch);
|
|
123
|
-
if (table.batches.length) yield* table.batches;
|
|
124
|
-
else yield makeArrowMetadataBatch(table.schema);
|
|
125
|
-
} else yield batch;
|
|
126
|
-
}
|
|
127
|
-
},
|
|
128
|
-
deserializeWorkerResult(result, options) {
|
|
129
|
-
return unwrapShapefileTable(deserializeShapefileOutput(result, options));
|
|
130
|
-
},
|
|
131
|
-
deserializeWorkerBatch(result, options) {
|
|
132
|
-
const batch = deserializeShapefileOutput(result, options);
|
|
133
|
-
if (!isShapefileArrowTable(batch)) return batch;
|
|
134
|
-
const table = unwrapShapefileTable(batch);
|
|
135
|
-
return table.batches[0] ?? makeArrowMetadataBatch(table.schema);
|
|
136
|
-
}
|
|
137
|
-
} as const satisfies LoaderWithParser<
|
|
138
|
-
ShapefileOutput | GeoJSONTable | Table,
|
|
139
|
-
ShapefileOutput | Batch | RecordBatch,
|
|
140
|
-
ShapefileLoaderOptions
|
|
141
|
-
>;
|
|
142
|
-
|
|
143
|
-
/** Identifies Arrow output from the private column parser. */
|
|
144
|
-
function isShapefileArrowTable(result: unknown): result is ArrowTable {
|
|
145
|
-
return (result as {shape?: string} | null)?.shape === 'arrow-table';
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/** Unwraps Arrow columns and passes through legacy results. */
|
|
149
|
-
function unwrapShapefileTable(result: ArrowTable): Table;
|
|
150
|
-
function unwrapShapefileTable<T>(result: T): Exclude<T, ArrowTable> | Table;
|
|
151
|
-
function unwrapShapefileTable(result: unknown): unknown {
|
|
152
|
-
if (!isShapefileArrowTable(result)) return result;
|
|
153
|
-
const loaderData = (
|
|
154
|
-
result as ArrowTable & {
|
|
155
|
-
loaderData?: {
|
|
156
|
-
header?: unknown;
|
|
157
|
-
encoding?: string;
|
|
158
|
-
prj?: string;
|
|
159
|
-
shx?: {offsets: Int32Array; lengths: Int32Array};
|
|
160
|
-
};
|
|
161
|
-
}
|
|
162
|
-
).loaderData;
|
|
163
|
-
if (loaderData) {
|
|
164
|
-
const {shx, ...header} = loaderData;
|
|
165
|
-
const metadata = JSON.stringify({
|
|
166
|
-
...header,
|
|
167
|
-
...(shx ? {shx: {offsets: Array.from(shx.offsets), lengths: Array.from(shx.lengths)}} : {})
|
|
168
|
-
});
|
|
169
|
-
result.data.schema.metadata.set('loaders.gl.shapefile.header', metadata);
|
|
170
|
-
for (const batch of result.data.batches)
|
|
171
|
-
batch.schema.metadata.set('loaders.gl.shapefile.header', metadata);
|
|
172
|
-
}
|
|
173
|
-
return result.data;
|
|
123
|
+
return {...loaderData, data: features};
|
|
174
124
|
}
|
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import type {Table, RecordBatch} from 'apache-arrow';
|
|
6
|
-
import {makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
|
|
7
6
|
import type {Loader, LoaderWithParser, StrictLoaderOptions} from '@loaders.gl/core/loader-utils';
|
|
8
|
-
import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/core/schema';
|
|
9
7
|
import {parseSHP, parseSHPInBatches} from './lib/parsers/parse-shp';
|
|
10
8
|
import type {SHPResult} from './lib/parsers/parse-shp';
|
|
11
9
|
import {parseSHPToArrow, parseSHPToArrowInBatches} from './lib/parsers/parse-shp-to-arrow';
|
|
@@ -46,14 +44,14 @@ export const SHPWorkerLoaderWithParser = {
|
|
|
46
44
|
} as const satisfies Loader<any, any, SHPLoaderOptions>;
|
|
47
45
|
|
|
48
46
|
/** SHP file loader */
|
|
49
|
-
export const
|
|
50
|
-
SHPResult |
|
|
51
|
-
SHPHeader | (Uint8Array | null)[] |
|
|
47
|
+
export const SHPLoaderWithParser: LoaderWithParser<
|
|
48
|
+
SHPResult | Table,
|
|
49
|
+
SHPHeader | (Uint8Array | null)[] | RecordBatch,
|
|
52
50
|
SHPLoaderOptions
|
|
53
51
|
> = {
|
|
54
52
|
...SHPLoaderMetadataWithoutPreload,
|
|
55
|
-
dataType: null as unknown as SHPResult |
|
|
56
|
-
batchType: null as unknown as SHPHeader | (Uint8Array | null)[] |
|
|
53
|
+
dataType: null as unknown as SHPResult | Table,
|
|
54
|
+
batchType: null as unknown as SHPHeader | (Uint8Array | null)[] | RecordBatch,
|
|
57
55
|
parse: async (arrayBuffer, options?) =>
|
|
58
56
|
getSHPShape(options) === 'arrow'
|
|
59
57
|
? parseSHPToArrow(arrayBuffer, options)
|
|
@@ -67,7 +65,7 @@ export const SHPColumnLoader: LoaderWithParser<
|
|
|
67
65
|
| AsyncIterable<ArrayBufferLike | ArrayBufferView>
|
|
68
66
|
| Iterable<ArrayBufferLike | ArrayBufferView>,
|
|
69
67
|
options
|
|
70
|
-
): AsyncIterable<SHPHeader | (Uint8Array | null)[] |
|
|
68
|
+
): AsyncIterable<SHPHeader | (Uint8Array | null)[] | RecordBatch> => {
|
|
71
69
|
const batchOptions =
|
|
72
70
|
typeof options?.core?.batchSize === 'number'
|
|
73
71
|
? {...options, shp: {...options.shp, batchSize: options.core.batchSize}}
|
|
@@ -76,7 +74,7 @@ export const SHPColumnLoader: LoaderWithParser<
|
|
|
76
74
|
getSHPShape(batchOptions) === 'arrow'
|
|
77
75
|
? parseSHPToArrowInBatches(arrayBufferIterator, batchOptions)
|
|
78
76
|
: parseSHPInBatches(arrayBufferIterator, batchOptions)
|
|
79
|
-
) as AsyncIterable<SHPHeader | (Uint8Array | null)[] |
|
|
77
|
+
) as AsyncIterable<SHPHeader | (Uint8Array | null)[] | RecordBatch>;
|
|
80
78
|
},
|
|
81
79
|
serializeWorkerResult: serializeShapefileWorkerResult,
|
|
82
80
|
deserializeWorkerResult: deserializeShapefileWorkerResult,
|
|
@@ -87,55 +85,3 @@ export const SHPColumnLoader: LoaderWithParser<
|
|
|
87
85
|
function getSHPShape(options?: SHPLoaderOptions): NonNullable<SHPLoaderOptions['shp']>['shape'] {
|
|
88
86
|
return options?.shp?.shape || 'arrow';
|
|
89
87
|
}
|
|
90
|
-
|
|
91
|
-
/** Exposes native Arrow output after parsing or worker hydration. */
|
|
92
|
-
export const SHPLoaderWithParser = {
|
|
93
|
-
...SHPLoaderMetadataWithoutPreload,
|
|
94
|
-
serializeWorkerResult: serializeShapefileWorkerResult,
|
|
95
|
-
serializeWorkerBatch: serializeShapefileWorkerBatch,
|
|
96
|
-
dataType: null as unknown as SHPResult | Table,
|
|
97
|
-
batchType: null as unknown as SHPHeader | (Uint8Array | null)[] | RecordBatch,
|
|
98
|
-
async parse(input, options, context) {
|
|
99
|
-
return unwrapSHPTable(await SHPColumnLoader.parse!(input, options, context));
|
|
100
|
-
},
|
|
101
|
-
parseSync(input, options, context) {
|
|
102
|
-
return unwrapSHPTable(SHPColumnLoader.parseSync!(input, options, context));
|
|
103
|
-
},
|
|
104
|
-
async *parseInBatches(input, options, context) {
|
|
105
|
-
for await (const batch of SHPColumnLoader.parseInBatches!(input, options, context)) {
|
|
106
|
-
if (isSHPArrowTable(batch)) {
|
|
107
|
-
const table = unwrapSHPTable(batch);
|
|
108
|
-
if (table.batches.length) yield* table.batches;
|
|
109
|
-
else yield makeArrowMetadataBatch(table.schema);
|
|
110
|
-
} else yield batch;
|
|
111
|
-
}
|
|
112
|
-
},
|
|
113
|
-
deserializeWorkerResult(result) {
|
|
114
|
-
return unwrapSHPTable(deserializeShapefileWorkerResult(result));
|
|
115
|
-
},
|
|
116
|
-
deserializeWorkerBatch(result) {
|
|
117
|
-
const batch = deserializeShapefileWorkerBatch<SHPHeader | (Uint8Array | null)[] | ArrowTable>(
|
|
118
|
-
result
|
|
119
|
-
);
|
|
120
|
-
if (!isSHPArrowTable(batch)) return batch;
|
|
121
|
-
const table = unwrapSHPTable(batch);
|
|
122
|
-
return table.batches[0] ?? makeArrowMetadataBatch(table.schema);
|
|
123
|
-
}
|
|
124
|
-
} as const satisfies LoaderWithParser<
|
|
125
|
-
SHPResult | Table,
|
|
126
|
-
SHPHeader | (Uint8Array | null)[] | RecordBatch,
|
|
127
|
-
SHPLoaderOptions
|
|
128
|
-
>;
|
|
129
|
-
|
|
130
|
-
/** Identifies Arrow output from the private column parser. */
|
|
131
|
-
function isSHPArrowTable(result: unknown): result is ArrowTable {
|
|
132
|
-
return (result as {shape?: string} | null)?.shape === 'arrow-table';
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/** Unwraps Arrow columns and passes through legacy results. */
|
|
136
|
-
function unwrapSHPTable(result: ArrowTable): Table;
|
|
137
|
-
function unwrapSHPTable<T>(result: T): Exclude<T, ArrowTable> | Table;
|
|
138
|
-
function unwrapSHPTable(result: unknown): unknown {
|
|
139
|
-
if (!isSHPArrowTable(result)) return result;
|
|
140
|
-
return result.data;
|
|
141
|
-
}
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
type LoaderContext,
|
|
9
9
|
type LoaderWithParser
|
|
10
10
|
} from '@loaders.gl/core/loader-utils';
|
|
11
|
-
import {
|
|
11
|
+
import {ShapefileLoaderWithParser} from '../shapefile/shapefile-loader';
|
|
12
12
|
|
|
13
13
|
/** Installs local subloader parsing so streamed sidecars and geometry stay in this worker. */
|
|
14
14
|
function createShapefileContext(context?: LoaderContext, options?: any): LoaderContext {
|
|
@@ -65,10 +65,10 @@ function getArrowOptions(options: any): any {
|
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
createLoaderWorker({
|
|
68
|
-
...
|
|
68
|
+
...ShapefileLoaderWithParser,
|
|
69
69
|
/** Parses geometries and sidecar attributes into owned transferable columns. */
|
|
70
70
|
parse(input, options, context) {
|
|
71
|
-
return
|
|
71
|
+
return ShapefileLoaderWithParser.parse(
|
|
72
72
|
input,
|
|
73
73
|
getArrowOptions(options),
|
|
74
74
|
createShapefileContext(context, options)
|
|
@@ -76,7 +76,7 @@ createLoaderWorker({
|
|
|
76
76
|
},
|
|
77
77
|
/** Joins streamed geometry and attribute batches without delegating their rows to the caller. */
|
|
78
78
|
parseInBatches(input, options, context) {
|
|
79
|
-
return
|
|
79
|
+
return ShapefileLoaderWithParser.parseInBatches(
|
|
80
80
|
input,
|
|
81
81
|
getArrowOptions(options),
|
|
82
82
|
createShapefileContext(context, options)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import {SHPLoaderWithParser} from '../shapefile/shp-loader';
|
|
6
6
|
import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
|
|
7
7
|
|
|
8
|
-
createLoaderWorker(
|
|
8
|
+
createLoaderWorker(SHPLoaderWithParser);
|