@loaders.gl/geospatial 5.0.0-beta.10 → 5.0.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/dbf-loader-types.cjs +1 -1
  2. package/dist/dbf-loader.cjs +2 -4
  3. package/dist/dbf-loader.cjs.map +2 -2
  4. package/dist/dbf-worker.js +22 -79
  5. package/dist/flatgeobuf/flatgeobuf-loader-types.cjs +1 -1
  6. package/dist/flatgeobuf/flatgeobuf-loader-types.js +1 -1
  7. package/dist/flatgeobuf/lib/parse-flatgeobuf.cjs +6 -20
  8. package/dist/flatgeobuf/lib/parse-flatgeobuf.cjs.map +2 -2
  9. package/dist/flatgeobuf/lib/parse-flatgeobuf.d.ts +4 -4
  10. package/dist/flatgeobuf/lib/parse-flatgeobuf.d.ts.map +1 -1
  11. package/dist/flatgeobuf/lib/parse-flatgeobuf.js +9 -22
  12. package/dist/flatgeobuf/lib/parse-flatgeobuf.js.map +1 -1
  13. package/dist/flatgeobuf-loader-types.cjs +1 -1
  14. package/dist/flatgeobuf-worker.js +655 -77
  15. package/dist/geopackage/geopackage-source-loader.cjs +1 -1
  16. package/dist/geopackage/geopackage-source-loader.cjs.map +2 -2
  17. package/dist/geopackage/geopackage-source-loader.js +1 -1
  18. package/dist/geopackage/geopackage-source-loader.js.map +1 -1
  19. package/dist/geopackage/lib/parse-geopackage.cjs +2 -6
  20. package/dist/geopackage/lib/parse-geopackage.cjs.map +2 -2
  21. package/dist/geopackage/lib/parse-geopackage.d.ts +3 -3
  22. package/dist/geopackage/lib/parse-geopackage.d.ts.map +1 -1
  23. package/dist/geopackage/lib/parse-geopackage.js +2 -6
  24. package/dist/geopackage/lib/parse-geopackage.js.map +1 -1
  25. package/dist/geopackage-source-loader.cjs +1 -1
  26. package/dist/geopackage-source-loader.cjs.map +2 -2
  27. package/dist/gpx-loader-types.cjs +1 -1
  28. package/dist/kml/gpx-loader-types.cjs +1 -1
  29. package/dist/kml/gpx-loader-types.js +1 -1
  30. package/dist/kml/kml-loader-types.cjs +1 -1
  31. package/dist/kml/kml-loader-types.js +1 -1
  32. package/dist/kml/kml-writer.cjs +1 -1
  33. package/dist/kml/kml-writer.js +1 -1
  34. package/dist/kml/kmz-loader-types.cjs +1 -1
  35. package/dist/kml/kmz-loader-types.js +1 -1
  36. package/dist/kml/lib/feature-collection-to-arrow.cjs +1 -1
  37. package/dist/kml/lib/feature-collection-to-arrow.cjs.map +2 -2
  38. package/dist/kml/lib/feature-collection-to-arrow.js +1 -1
  39. package/dist/kml/lib/feature-collection-to-arrow.js.map +1 -1
  40. package/dist/kml/tcx-loader-types.cjs +1 -1
  41. package/dist/kml/tcx-loader-types.js +1 -1
  42. package/dist/kml-loader-types.cjs +1 -1
  43. package/dist/kml-writer.cjs +1 -1
  44. package/dist/kmz-loader-types.cjs +1 -1
  45. package/dist/shapefile/dbf-loader-types.cjs +1 -1
  46. package/dist/shapefile/dbf-loader-types.js +1 -1
  47. package/dist/shapefile/dbf-loader.cjs +2 -4
  48. package/dist/shapefile/dbf-loader.cjs.map +2 -2
  49. package/dist/shapefile/dbf-loader.d.ts.map +1 -1
  50. package/dist/shapefile/dbf-loader.js +3 -5
  51. package/dist/shapefile/dbf-loader.js.map +1 -1
  52. package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.cjs +1 -5
  53. package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.cjs.map +2 -2
  54. package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.d.ts +5 -4
  55. package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.d.ts.map +1 -1
  56. package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.js +2 -6
  57. package/dist/shapefile/lib/parsers/build-geoarrow-geometry-arrow.js.map +1 -1
  58. package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.cjs.map +2 -2
  59. package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.d.ts +4 -3
  60. package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.d.ts.map +1 -1
  61. package/dist/shapefile/lib/parsers/build-wkb-geometry-arrow.js.map +1 -1
  62. package/dist/shapefile/lib/parsers/parse-shp-to-arrow.cjs +7 -10
  63. package/dist/shapefile/lib/parsers/parse-shp-to-arrow.cjs.map +2 -2
  64. package/dist/shapefile/lib/parsers/parse-shp-to-arrow.d.ts +3 -3
  65. package/dist/shapefile/lib/parsers/parse-shp-to-arrow.d.ts.map +1 -1
  66. package/dist/shapefile/lib/parsers/parse-shp-to-arrow.js +11 -9
  67. package/dist/shapefile/lib/parsers/parse-shp-to-arrow.js.map +1 -1
  68. package/dist/shapefile/lib/shapefile-worker-transport.cjs +0 -3
  69. package/dist/shapefile/lib/shapefile-worker-transport.cjs.map +2 -2
  70. package/dist/shapefile/lib/shapefile-worker-transport.d.ts.map +1 -1
  71. package/dist/shapefile/lib/shapefile-worker-transport.js +1 -5
  72. package/dist/shapefile/lib/shapefile-worker-transport.js.map +1 -1
  73. package/dist/shapefile/shapefile-arrow-loader.cjs +53 -48
  74. package/dist/shapefile/shapefile-arrow-loader.cjs.map +2 -2
  75. package/dist/shapefile/shapefile-arrow-loader.d.ts +5 -3
  76. package/dist/shapefile/shapefile-arrow-loader.d.ts.map +1 -1
  77. package/dist/shapefile/shapefile-arrow-loader.js +54 -52
  78. package/dist/shapefile/shapefile-arrow-loader.js.map +1 -1
  79. package/dist/shapefile/shapefile-loader-types.cjs +1 -1
  80. package/dist/shapefile/shapefile-loader-types.d.ts +4 -4
  81. package/dist/shapefile/shapefile-loader-types.js +1 -1
  82. package/dist/shapefile/shapefile-loader.cjs +19 -55
  83. package/dist/shapefile/shapefile-loader.cjs.map +2 -2
  84. package/dist/shapefile/shapefile-loader.d.ts +5 -42
  85. package/dist/shapefile/shapefile-loader.d.ts.map +1 -1
  86. package/dist/shapefile/shapefile-loader.js +22 -64
  87. package/dist/shapefile/shapefile-loader.js.map +1 -1
  88. package/dist/shapefile/shp-loader-types.cjs +1 -1
  89. package/dist/shapefile/shp-loader-types.js +1 -1
  90. package/dist/shapefile/shp-loader.cjs +1 -44
  91. package/dist/shapefile/shp-loader.cjs.map +2 -2
  92. package/dist/shapefile/shp-loader.d.ts +1 -115
  93. package/dist/shapefile/shp-loader.d.ts.map +1 -1
  94. package/dist/shapefile/shp-loader.js +1 -48
  95. package/dist/shapefile/shp-loader.js.map +1 -1
  96. package/dist/shapefile-arrow-loader.cjs +53 -48
  97. package/dist/shapefile-arrow-loader.cjs.map +2 -2
  98. package/dist/shapefile-loader-types.cjs +1 -1
  99. package/dist/shapefile-loader.cjs +19 -55
  100. package/dist/shapefile-loader.cjs.map +2 -2
  101. package/dist/shapefile-worker.js +117 -272
  102. package/dist/shp-loader-types.cjs +1 -1
  103. package/dist/shp-loader.cjs +1 -44
  104. package/dist/shp-loader.cjs.map +2 -2
  105. package/dist/shp-worker.js +16964 -17085
  106. package/dist/tcx-loader-types.cjs +1 -1
  107. package/dist/wkt/lib/version.cjs +1 -1
  108. package/dist/wkt/lib/version.js +1 -1
  109. package/dist/wkt-worker.js +1 -1
  110. package/dist/workers/shapefile-worker.cjs +3 -3
  111. package/dist/workers/shapefile-worker.cjs.map +2 -2
  112. package/dist/workers/shapefile-worker.js +4 -4
  113. package/dist/workers/shapefile-worker.js.map +1 -1
  114. package/dist/workers/shp-worker.cjs +1 -1
  115. package/dist/workers/shp-worker.cjs.map +2 -2
  116. package/dist/workers/shp-worker.js +2 -2
  117. package/dist/workers/shp-worker.js.map +1 -1
  118. package/package.json +6 -6
  119. package/src/flatgeobuf/lib/parse-flatgeobuf.ts +15 -33
  120. package/src/geopackage/geopackage-source-loader.ts +1 -1
  121. package/src/geopackage/lib/parse-geopackage.ts +4 -9
  122. package/src/kml/lib/feature-collection-to-arrow.ts +1 -1
  123. package/src/shapefile/dbf-loader.ts +4 -7
  124. package/src/shapefile/lib/parsers/build-geoarrow-geometry-arrow.ts +5 -9
  125. package/src/shapefile/lib/parsers/build-wkb-geometry-arrow.ts +4 -3
  126. package/src/shapefile/lib/parsers/parse-shp-to-arrow.ts +14 -20
  127. package/src/shapefile/lib/shapefile-worker-transport.ts +1 -6
  128. package/src/shapefile/shapefile-arrow-loader.ts +76 -70
  129. package/src/shapefile/shapefile-loader.ts +40 -90
  130. package/src/shapefile/shp-loader.ts +7 -61
  131. package/src/workers/shapefile-worker.ts +4 -4
  132. package/src/workers/shp-worker.ts +2 -2
@@ -4,20 +4,14 @@
4
4
 
5
5
  import {CRSReprojectionError, type LoaderContext} from '@loaders.gl/core/loader-utils';
6
6
  import {
7
- makeTableScanBatch,
8
7
  parseFromContext,
9
8
  parseInBatchesFromContext,
10
9
  toArrayBufferIterator
11
10
  } from '@loaders.gl/core/loader-utils';
12
11
  import * as arrow from 'apache-arrow';
13
- import {convertArrowToSchema, isNativeArrowRecordBatch} from '@loaders.gl/core/schema-utils';
14
- import type {
15
- ArrowTable,
16
- ArrowTableBatch,
17
- Field,
18
- Schema as TableSchema
19
- } from '@loaders.gl/core/schema';
20
- import {ArrowTableBuilder, convertSchemaToArrow} from '@loaders.gl/core/schema-utils';
12
+ import {convertArrowToSchema, isArrowRecordBatch} from '@loaders.gl/core/schema-utils';
13
+ import type {Field, Schema as TableSchema} from '@loaders.gl/core/schema';
14
+ import {makeArrowMetadataBatch, convertSchemaToArrow} from '@loaders.gl/core/schema-utils';
21
15
  import {convertFeaturesToGeoArrowTable} from '@loaders.gl/core/arrow-geometry/feature-table';
22
16
  import {convertWKBToGeometry} from '@loaders.gl/core/arrow-geometry/geometry-codecs';
23
17
  import {type GeoParquetGeometryType} from '@loaders.gl/core/schema';
@@ -45,12 +39,12 @@ export async function parseShapefileToArrow(
45
39
  arrayBuffer: ArrayBuffer,
46
40
  options?: ShapefileLoaderOptions,
47
41
  context?: LoaderContext
48
- ): Promise<ArrowTable> {
42
+ ): Promise<arrow.Table> {
49
43
  const {cpg, prj, shx} = await loadShapefileSidecarFiles(options, context);
50
44
  const transform = getReprojectionTransform(prj, options);
51
45
  const geoArrowEncoding = getTypedGeoArrowEncoding(options);
52
46
  let header: SHPHeader | undefined;
53
- let geometryTable: ArrowTable;
47
+ let geometryTable: arrow.Table;
54
48
 
55
49
  if (geoArrowEncoding) {
56
50
  geometryTable = makeSHPGeoArrowGeometryTable(arrayBuffer, options, {transform});
@@ -77,7 +71,7 @@ export async function parseShapefileToArrow(
77
71
  options?.geoarrow?.encodingPreference === 'geoarrow.geometry' ||
78
72
  options?.shapefile?.geoarrow?.encodingPreference === 'geoarrow.geometry'
79
73
  ) {
80
- const geometryColumn = geometryTable.data.getChild(GEOMETRY_COLUMN_NAME);
74
+ const geometryColumn = geometryTable.getChild(GEOMETRY_COLUMN_NAME);
81
75
  const geometries = geometryColumn?.toArray() || [];
82
76
  geometryTable = convertFeaturesToGeoArrowTable(
83
77
  geometries.map(geometry => ({
@@ -91,7 +85,7 @@ export async function parseShapefileToArrow(
91
85
  }
92
86
 
93
87
  let propertySchema: TableSchema | null = null;
94
- let propertyTable: ArrowTable | null = null;
88
+ let propertyTable: arrow.Table | null = null;
95
89
 
96
90
  const dbfResponse = context?.url
97
91
  ? await context.fetch(replaceExtension(context.url, 'dbf')).catch(() => null)
@@ -111,14 +105,21 @@ export async function parseShapefileToArrow(
111
105
  context!
112
106
  )) as arrow.Table;
113
107
  propertySchema = convertArrowToSchema(propertyData.schema);
114
- propertyTable = {shape: 'arrow-table', data: propertyData, schema: propertySchema};
108
+ propertyTable = propertyData;
115
109
  }
116
110
 
117
- const schema = buildOutputSchema(propertySchema, header, geometryTable.schema);
111
+ const schema = buildOutputSchema(
112
+ propertySchema,
113
+ header,
114
+ convertArrowToSchema(geometryTable.schema)
115
+ );
118
116
  const table = propertyTable
119
117
  ? appendGeometryColumnToArrowTable(propertyTable, geometryTable, schema)
120
118
  : geometryTable;
121
- return Object.assign(table, {loaderData: {header, encoding: cpg, prj, shx}});
119
+ const encoded = encodeShapefileHeader({header, encoding: cpg, prj, shx});
120
+ table.schema.metadata.set(SHAPEFILE_HEADER_KEY, encoded);
121
+ for (const batch of table.batches) batch.schema.metadata.set(SHAPEFILE_HEADER_KEY, encoded);
122
+ return table;
122
123
  }
123
124
 
124
125
  /** Parses a shapefile into Arrow batches while keeping DBF-derived schema stable. */
@@ -128,7 +129,7 @@ export async function* parseShapefileToArrowInBatches(
128
129
  | Iterable<ArrayBufferLike | ArrayBufferView>,
129
130
  options?: ShapefileLoaderOptions,
130
131
  context?: LoaderContext
131
- ): AsyncIterable<ArrowTableBatch> {
132
+ ): AsyncIterable<arrow.RecordBatch> {
132
133
  if (getTypedGeoArrowEncoding(options)) {
133
134
  throw new Error('Typed GeoArrow shapefile output is only supported for non-streaming parse.');
134
135
  }
@@ -163,6 +164,7 @@ export async function* parseShapefileToArrowInBatches(
163
164
 
164
165
  const shapeHeader = await getNextNonMetadataValue(shapeIterator);
165
166
  const header = shapeHeader as SHPHeader;
167
+ const encodedHeader = encodeShapefileHeader({header, encoding: cpg, prj, shx});
166
168
 
167
169
  let propertyIterator: AsyncIterator<any> | null = null;
168
170
  let propertySchema: TableSchema | null = null;
@@ -189,14 +191,14 @@ export async function* parseShapefileToArrowInBatches(
189
191
  propertyIterator = getAsyncIterator(propertyIterable);
190
192
 
191
193
  const firstPropertyBatch = await getNextArrowBatch(propertyIterator);
192
- propertySchema = firstPropertyBatch?.schema || null;
194
+ propertySchema = firstPropertyBatch ? convertArrowToSchema(firstPropertyBatch.schema) : null;
193
195
  const outputSchema = buildOutputSchema(propertySchema, header);
194
196
  const propertyQueue: arrow.Table[] = [];
195
197
  const geometryQueue: arrow.Table[] = [];
196
198
  let yieldedDataBatch = false;
197
199
 
198
- if (firstPropertyBatch && firstPropertyBatch.length > 0) {
199
- propertyQueue.push(firstPropertyBatch.data);
200
+ if (firstPropertyBatch && firstPropertyBatch.numRows > 0) {
201
+ propertyQueue.push(new arrow.Table(firstPropertyBatch.schema, [firstPropertyBatch]));
200
202
  }
201
203
 
202
204
  let shapeDone = false;
@@ -215,7 +217,6 @@ export async function* parseShapefileToArrowInBatches(
215
217
  const transform = getReprojectionTransform(prj, options);
216
218
  geometryQueue.push(
217
219
  makeGeometryArrowTable(shapeBatch.value as (SHPWKBGeometry | null)[], header, transform)
218
- .data
219
220
  );
220
221
  }
221
222
  }
@@ -224,8 +225,8 @@ export async function* parseShapefileToArrowInBatches(
224
225
  const propertyBatch = await getNextArrowBatch(propertyIterator);
225
226
  if (!propertyBatch) {
226
227
  propertyDone = true;
227
- } else if (propertyBatch.length > 0) {
228
- propertyQueue.push(propertyBatch.data);
228
+ } else if (propertyBatch.numRows > 0) {
229
+ propertyQueue.push(new arrow.Table(propertyBatch.schema, [propertyBatch]));
229
230
  }
230
231
  }
231
232
 
@@ -242,16 +243,14 @@ export async function* parseShapefileToArrowInBatches(
242
243
 
243
244
  const propertyTable = takeRowsFromQueue(propertyQueue, rowCount);
244
245
  const geometryTable = takeRowsFromQueue(geometryQueue, rowCount);
245
- const batch = appendGeometryColumnToArrowTable(
246
- {shape: 'arrow-table', schema: propertySchema || undefined, data: propertyTable},
247
- {shape: 'arrow-table', data: geometryTable},
248
- outputSchema
249
- );
246
+ const batch = appendGeometryColumnToArrowTable(propertyTable, geometryTable, outputSchema);
250
247
  yieldedDataBatch = true;
251
- yield {...makeTableScanBatch(batch), loaderData: {header, encoding: cpg, prj, shx}};
248
+ const recordBatch = batch.batches[0];
249
+ recordBatch.schema.metadata.set(SHAPEFILE_HEADER_KEY, encodedHeader);
250
+ yield recordBatch;
252
251
  }
253
252
  if (!yieldedDataBatch) {
254
- yield {...makeEmptyArrowBatch(outputSchema), loaderData: {header, encoding: cpg, prj, shx}};
253
+ yield makeEmptyArrowBatch(outputSchema, encodedHeader);
255
254
  }
256
255
  return;
257
256
  }
@@ -274,13 +273,13 @@ export async function* parseShapefileToArrowInBatches(
274
273
  transform
275
274
  );
276
275
  yieldedDataBatch = true;
277
- yield {
278
- ...makeTableScanBatch({...arrowTable, schema: outputSchema}),
279
- loaderData: {header, encoding: cpg, prj, shx}
280
- };
276
+ for (const batch of arrowTable.batches) {
277
+ batch.schema.metadata.set(SHAPEFILE_HEADER_KEY, encodedHeader);
278
+ yield batch;
279
+ }
281
280
  }
282
281
  if (!yieldedDataBatch) {
283
- yield {...makeEmptyArrowBatch(outputSchema), loaderData: {header, encoding: cpg, prj, shx}};
282
+ yield makeEmptyArrowBatch(outputSchema, encodedHeader);
284
283
  }
285
284
  }
286
285
 
@@ -320,24 +319,20 @@ function makeGeometryArrowTable(
320
319
  }
321
320
 
322
321
  function appendGeometryColumnToArrowTable(
323
- propertyTable: ArrowTable,
324
- geometryTable: ArrowTable,
322
+ propertyTable: arrow.Table,
323
+ geometryTable: arrow.Table,
325
324
  schema: TableSchema
326
- ): ArrowTable {
327
- const propertyBatch = propertyTable.data.batches[0];
328
- const geometryBatch = geometryTable.data.batches[0];
325
+ ): arrow.Table {
326
+ const propertyBatch = propertyTable.batches[0];
327
+ const geometryBatch = geometryTable.batches[0];
329
328
  const arrowSchema = convertSchemaToArrow(schema);
330
329
  const structField = new arrow.Struct(arrowSchema.fields);
331
330
  const children = [...propertyBatch.data.children, geometryBatch.data.children[0]];
332
- const rowCount = Math.max(propertyTable.data.numRows, geometryTable.data.numRows);
331
+ const rowCount = Math.max(propertyTable.numRows, geometryTable.numRows);
333
332
  const structData = new arrow.Data(structField, 0, rowCount, 0, undefined, children);
334
333
  const recordBatch = new arrow.RecordBatch(arrowSchema, structData);
335
334
 
336
- return {
337
- shape: 'arrow-table',
338
- schema,
339
- data: new arrow.Table(arrowSchema, [recordBatch])
340
- };
335
+ return new arrow.Table(arrowSchema, [recordBatch]);
341
336
  }
342
337
 
343
338
  function getReprojectionTransform(
@@ -429,28 +424,19 @@ async function getNextNonMetadataValue(iterator: AsyncIterator<any>): Promise<an
429
424
  }
430
425
  }
431
426
 
432
- /** Reads both native and legacy DBF column batches. */
433
- async function getNextArrowBatch(iterator: AsyncIterator<any>): Promise<ArrowTableBatch | null> {
427
+ /** Reads native DBF batches and imports columns originating from another Arrow build. */
428
+ async function getNextArrowBatch(
429
+ iterator: AsyncIterator<unknown>
430
+ ): Promise<arrow.RecordBatch | null> {
434
431
  while (true) {
435
432
  const result = await iterator.next();
436
- if (result.done) {
437
- return null;
438
- }
439
- if (isNativeArrowRecordBatch(result.value)) {
440
- const recordBatch = result.value;
441
- const schema = convertArrowToSchema(recordBatch.schema);
442
- const batch =
443
- Object.getPrototypeOf(recordBatch) === arrow.RecordBatch.prototype
444
- ? recordBatch
445
- : importDBFRecordBatch(recordBatch, schema);
446
- const data = new arrow.Table(batch.schema, [batch]);
447
- return makeTableScanBatch({
448
- shape: 'arrow-table',
449
- schema,
450
- data
451
- });
433
+ if (result.done) return null;
434
+ if (isArrowRecordBatch(result.value)) {
435
+ const batch = result.value;
436
+ return Object.getPrototypeOf(batch) === arrow.RecordBatch.prototype
437
+ ? batch
438
+ : importDBFRecordBatch(batch, convertArrowToSchema(batch.schema));
452
439
  }
453
- if (result.value?.shape === 'arrow-table') return result.value;
454
440
  }
455
441
  }
456
442
 
@@ -492,12 +478,32 @@ function takeRowsFromQueue(queue: arrow.Table[], rowCount: number): arrow.Table
492
478
  return table;
493
479
  }
494
480
  const result = table.slice(0, rowCount);
495
- queue[0] = table.slice(rowCount, table.numRows - rowCount);
481
+ queue[0] = table.slice(rowCount);
496
482
  return result;
497
483
  }
498
484
 
499
- /** Creates an explicit empty Arrow batch so zero-row shapefiles still expose schema in batch mode. */
500
- function makeEmptyArrowBatch(schema: TableSchema): ArrowTableBatch {
501
- const table = new ArrowTableBuilder(schema).finishTable();
502
- return makeTableScanBatch({shape: 'arrow-table', data: table, schema});
485
+ /** Creates an explicit empty Arrow batch so zero-row shapefiles still expose their schema. */
486
+ function makeEmptyArrowBatch(schema: TableSchema, header: string): arrow.RecordBatch {
487
+ const nativeSchema = convertSchemaToArrow(schema);
488
+ nativeSchema.metadata.set(SHAPEFILE_HEADER_KEY, header);
489
+ return makeArrowMetadataBatch(nativeSchema);
490
+ }
491
+
492
+ /** Schema metadata key for the Shapefile header and sidecar data. */
493
+ export const SHAPEFILE_HEADER_KEY = 'loaders.gl.shapefile.header';
494
+
495
+ /** Encodes Shapefile sidecars for schema and worker transport. */
496
+ function encodeShapefileHeader({
497
+ shx,
498
+ ...header
499
+ }: {
500
+ header?: SHPHeader;
501
+ encoding?: string;
502
+ prj?: WKTCRSDefinition;
503
+ shx?: {offsets: Int32Array; lengths: Int32Array};
504
+ }): string {
505
+ return JSON.stringify({
506
+ ...header,
507
+ ...(shx ? {shx: {offsets: Array.from(shx.offsets), lengths: Array.from(shx.lengths)}} : {})
508
+ });
503
509
  }
@@ -4,15 +4,9 @@ import {convertWKBToGeometry} from '@loaders.gl/core/arrow-geometry/geometry-cod
4
4
  // Copyright (c) vis.gl contributors
5
5
 
6
6
  import type {Table, RecordBatch} from 'apache-arrow';
7
- import {makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
7
+ import {convertArrowToSchema} from '@loaders.gl/core/schema-utils';
8
8
  import type {StrictLoaderOptions, LoaderWithParser} from '@loaders.gl/core/loader-utils';
9
- import type {
10
- Batch,
11
- GeoArrowEncodingPreference,
12
- GeoJSONTable,
13
- ArrowTable,
14
- ArrowTableBatch
15
- } from '@loaders.gl/core/schema';
9
+ import type {Batch, GeoArrowEncodingPreference, GeoJSONTable} from '@loaders.gl/core/schema';
16
10
  import {SHPLoaderOptions} from './shp-loader-types';
17
11
  import {
18
12
  parseShapefile,
@@ -21,7 +15,11 @@ import {
21
15
  } from './lib/parsers/parse-shapefile';
22
16
  import {DBFLoaderOptions} from './dbf-loader-types';
23
17
  import type {SHPGeoArrowEncoding} from './lib/parsers/types';
24
- import {parseShapefileToArrow, parseShapefileToArrowInBatches} from './shapefile-arrow-loader';
18
+ import {
19
+ parseShapefileToArrow,
20
+ parseShapefileToArrowInBatches,
21
+ SHAPEFILE_HEADER_KEY
22
+ } from './shapefile-arrow-loader';
25
23
  import {ShapefileLoader as ShapefileLoaderMetadata} from './shapefile-loader-types';
26
24
  import {
27
25
  deserializeShapefileWorkerResult,
@@ -53,10 +51,10 @@ export type ShapefileLoaderOptions = StrictLoaderOptions &
53
51
  * Shapefile loader
54
52
  * @note Shapefile is multifile format and requires providing additional files
55
53
  */
56
- export const ShapefileColumnLoader = {
54
+ export const ShapefileLoaderWithParser = {
57
55
  ...ShapefileLoaderMetadataWithoutPreload,
58
- dataType: null as unknown as ShapefileOutput | GeoJSONTable | ArrowTable,
59
- batchType: null as unknown as ShapefileOutput | Batch | ArrowTableBatch,
56
+ dataType: null as unknown as ShapefileOutput | GeoJSONTable | Table,
57
+ batchType: null as unknown as ShapefileOutput | Batch | RecordBatch,
60
58
  parse: (arrayBuffer, options, context) =>
61
59
  getShapefileShape(options) === 'arrow'
62
60
  ? parseShapefileToArrow(arrayBuffer, options, context)
@@ -66,12 +64,12 @@ export const ShapefileColumnLoader = {
66
64
  ? parseShapefileToArrowInBatches(asyncIterator, options, context)
67
65
  : parseShapefileInBatches(asyncIterator, options, context),
68
66
  serializeWorkerResult: serializeShapefileWorkerResult,
69
- deserializeWorkerResult: deserializeShapefileOutput,
67
+ deserializeWorkerResult: (result, options) => deserializeShapefileOutput(result, options, false),
70
68
  serializeWorkerBatch: serializeShapefileWorkerBatch,
71
- deserializeWorkerBatch: deserializeShapefileOutput
69
+ deserializeWorkerBatch: (result, options) => deserializeShapefileOutput(result, options, true)
72
70
  } as const satisfies LoaderWithParser<
73
- ShapefileOutput | GeoJSONTable | ArrowTable,
74
- ShapefileOutput | Batch | ArrowTableBatch,
71
+ ShapefileOutput | GeoJSONTable | Table,
72
+ ShapefileOutput | Batch | RecordBatch,
75
73
  ShapefileLoaderOptions
76
74
  >;
77
75
 
@@ -81,94 +79,46 @@ function getShapefileShape(
81
79
  return options?.shapefile?.shape || 'arrow';
82
80
  }
83
81
 
84
- /** Materializes requested GeoJSON features from hydrated WKB and property columns on the caller. */
85
- function deserializeShapefileOutput(result: unknown, options?: ShapefileLoaderOptions): any {
86
- const table = deserializeShapefileWorkerResult<ArrowTable & {loaderData?: object}>(result);
82
+ /** Materializes requested GeoJSON features from native worker columns on the caller. */
83
+ function deserializeShapefileOutput(
84
+ result: unknown,
85
+ options: ShapefileLoaderOptions | undefined,
86
+ isBatch: boolean
87
+ ): any {
88
+ const table = deserializeShapefileWorkerResult<Table | RecordBatch>(result);
87
89
  if (getShapefileShape(options) === 'arrow') return table;
88
- const features = table.data.toArray().map(row => {
89
- const {geometry, ...properties} = row.toJSON();
90
+
91
+ const header = JSON.parse(table.schema.metadata.get(SHAPEFILE_HEADER_KEY) || '{}');
92
+ const loaderData = header.shx
93
+ ? {
94
+ ...header,
95
+ shx: {
96
+ offsets: Int32Array.from(header.shx.offsets),
97
+ lengths: Int32Array.from(header.shx.lengths)
98
+ }
99
+ }
100
+ : header;
101
+ const features = table.toArray().map(row => {
102
+ const {geometry, ...properties} = row!.toJSON();
90
103
  return {
91
104
  type: 'Feature',
92
105
  geometry: geometry ? convertWKBToGeometry(geometry.slice().buffer) : null,
93
106
  properties
94
107
  };
95
108
  });
96
- if (getShapefileShape(options) === 'geojson-table' && !('batchType' in table)) {
109
+ if (getShapefileShape(options) === 'geojson-table' && !isBatch) {
97
110
  return {
98
- ...table.loaderData,
111
+ ...loaderData,
99
112
  shape: 'geojson-table',
100
113
  type: 'FeatureCollection',
101
114
  schema: {
102
- fields: table.schema?.fields.filter(field => field.name !== 'geometry') || [],
115
+ fields: convertArrowToSchema(table.schema).fields.filter(
116
+ field => field.name !== 'geometry'
117
+ ),
103
118
  metadata: {}
104
119
  },
105
120
  features
106
121
  };
107
122
  }
108
- return {...table.loaderData, data: features};
109
- }
110
-
111
- /** Exposes native Arrow output after parsing or worker hydration. */
112
- export const ShapefileLoaderWithParser = {
113
- ...ShapefileColumnLoader,
114
- dataType: null as unknown as ShapefileOutput | GeoJSONTable | Table,
115
- batchType: null as unknown as ShapefileOutput | Batch | RecordBatch,
116
- async parse(input, options, context) {
117
- return unwrapShapefileTable(await ShapefileColumnLoader.parse!(input, options, context));
118
- },
119
- async *parseInBatches(input, options, context) {
120
- for await (const batch of ShapefileColumnLoader.parseInBatches!(input, options, context)) {
121
- if (isShapefileArrowTable(batch)) {
122
- const table = unwrapShapefileTable(batch);
123
- if (table.batches.length) yield* table.batches;
124
- else yield makeArrowMetadataBatch(table.schema);
125
- } else yield batch;
126
- }
127
- },
128
- deserializeWorkerResult(result, options) {
129
- return unwrapShapefileTable(deserializeShapefileOutput(result, options));
130
- },
131
- deserializeWorkerBatch(result, options) {
132
- const batch = deserializeShapefileOutput(result, options);
133
- if (!isShapefileArrowTable(batch)) return batch;
134
- const table = unwrapShapefileTable(batch);
135
- return table.batches[0] ?? makeArrowMetadataBatch(table.schema);
136
- }
137
- } as const satisfies LoaderWithParser<
138
- ShapefileOutput | GeoJSONTable | Table,
139
- ShapefileOutput | Batch | RecordBatch,
140
- ShapefileLoaderOptions
141
- >;
142
-
143
- /** Identifies Arrow output from the private column parser. */
144
- function isShapefileArrowTable(result: unknown): result is ArrowTable {
145
- return (result as {shape?: string} | null)?.shape === 'arrow-table';
146
- }
147
-
148
- /** Unwraps Arrow columns and passes through legacy results. */
149
- function unwrapShapefileTable(result: ArrowTable): Table;
150
- function unwrapShapefileTable<T>(result: T): Exclude<T, ArrowTable> | Table;
151
- function unwrapShapefileTable(result: unknown): unknown {
152
- if (!isShapefileArrowTable(result)) return result;
153
- const loaderData = (
154
- result as ArrowTable & {
155
- loaderData?: {
156
- header?: unknown;
157
- encoding?: string;
158
- prj?: string;
159
- shx?: {offsets: Int32Array; lengths: Int32Array};
160
- };
161
- }
162
- ).loaderData;
163
- if (loaderData) {
164
- const {shx, ...header} = loaderData;
165
- const metadata = JSON.stringify({
166
- ...header,
167
- ...(shx ? {shx: {offsets: Array.from(shx.offsets), lengths: Array.from(shx.lengths)}} : {})
168
- });
169
- result.data.schema.metadata.set('loaders.gl.shapefile.header', metadata);
170
- for (const batch of result.data.batches)
171
- batch.schema.metadata.set('loaders.gl.shapefile.header', metadata);
172
- }
173
- return result.data;
123
+ return {...loaderData, data: features};
174
124
  }
@@ -3,9 +3,7 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {Table, RecordBatch} from 'apache-arrow';
6
- import {makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
7
6
  import type {Loader, LoaderWithParser, StrictLoaderOptions} from '@loaders.gl/core/loader-utils';
8
- import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/core/schema';
9
7
  import {parseSHP, parseSHPInBatches} from './lib/parsers/parse-shp';
10
8
  import type {SHPResult} from './lib/parsers/parse-shp';
11
9
  import {parseSHPToArrow, parseSHPToArrowInBatches} from './lib/parsers/parse-shp-to-arrow';
@@ -46,14 +44,14 @@ export const SHPWorkerLoaderWithParser = {
46
44
  } as const satisfies Loader<any, any, SHPLoaderOptions>;
47
45
 
48
46
  /** SHP file loader */
49
- export const SHPColumnLoader: LoaderWithParser<
50
- SHPResult | ArrowTable,
51
- SHPHeader | (Uint8Array | null)[] | ArrowTableBatch,
47
+ export const SHPLoaderWithParser: LoaderWithParser<
48
+ SHPResult | Table,
49
+ SHPHeader | (Uint8Array | null)[] | RecordBatch,
52
50
  SHPLoaderOptions
53
51
  > = {
54
52
  ...SHPLoaderMetadataWithoutPreload,
55
- dataType: null as unknown as SHPResult | ArrowTable,
56
- batchType: null as unknown as SHPHeader | (Uint8Array | null)[] | ArrowTableBatch,
53
+ dataType: null as unknown as SHPResult | Table,
54
+ batchType: null as unknown as SHPHeader | (Uint8Array | null)[] | RecordBatch,
57
55
  parse: async (arrayBuffer, options?) =>
58
56
  getSHPShape(options) === 'arrow'
59
57
  ? parseSHPToArrow(arrayBuffer, options)
@@ -67,7 +65,7 @@ export const SHPColumnLoader: LoaderWithParser<
67
65
  | AsyncIterable<ArrayBufferLike | ArrayBufferView>
68
66
  | Iterable<ArrayBufferLike | ArrayBufferView>,
69
67
  options
70
- ): AsyncIterable<SHPHeader | (Uint8Array | null)[] | ArrowTableBatch> => {
68
+ ): AsyncIterable<SHPHeader | (Uint8Array | null)[] | RecordBatch> => {
71
69
  const batchOptions =
72
70
  typeof options?.core?.batchSize === 'number'
73
71
  ? {...options, shp: {...options.shp, batchSize: options.core.batchSize}}
@@ -76,7 +74,7 @@ export const SHPColumnLoader: LoaderWithParser<
76
74
  getSHPShape(batchOptions) === 'arrow'
77
75
  ? parseSHPToArrowInBatches(arrayBufferIterator, batchOptions)
78
76
  : parseSHPInBatches(arrayBufferIterator, batchOptions)
79
- ) as AsyncIterable<SHPHeader | (Uint8Array | null)[] | ArrowTableBatch>;
77
+ ) as AsyncIterable<SHPHeader | (Uint8Array | null)[] | RecordBatch>;
80
78
  },
81
79
  serializeWorkerResult: serializeShapefileWorkerResult,
82
80
  deserializeWorkerResult: deserializeShapefileWorkerResult,
@@ -87,55 +85,3 @@ export const SHPColumnLoader: LoaderWithParser<
87
85
  function getSHPShape(options?: SHPLoaderOptions): NonNullable<SHPLoaderOptions['shp']>['shape'] {
88
86
  return options?.shp?.shape || 'arrow';
89
87
  }
90
-
91
- /** Exposes native Arrow output after parsing or worker hydration. */
92
- export const SHPLoaderWithParser = {
93
- ...SHPLoaderMetadataWithoutPreload,
94
- serializeWorkerResult: serializeShapefileWorkerResult,
95
- serializeWorkerBatch: serializeShapefileWorkerBatch,
96
- dataType: null as unknown as SHPResult | Table,
97
- batchType: null as unknown as SHPHeader | (Uint8Array | null)[] | RecordBatch,
98
- async parse(input, options, context) {
99
- return unwrapSHPTable(await SHPColumnLoader.parse!(input, options, context));
100
- },
101
- parseSync(input, options, context) {
102
- return unwrapSHPTable(SHPColumnLoader.parseSync!(input, options, context));
103
- },
104
- async *parseInBatches(input, options, context) {
105
- for await (const batch of SHPColumnLoader.parseInBatches!(input, options, context)) {
106
- if (isSHPArrowTable(batch)) {
107
- const table = unwrapSHPTable(batch);
108
- if (table.batches.length) yield* table.batches;
109
- else yield makeArrowMetadataBatch(table.schema);
110
- } else yield batch;
111
- }
112
- },
113
- deserializeWorkerResult(result) {
114
- return unwrapSHPTable(deserializeShapefileWorkerResult(result));
115
- },
116
- deserializeWorkerBatch(result) {
117
- const batch = deserializeShapefileWorkerBatch<SHPHeader | (Uint8Array | null)[] | ArrowTable>(
118
- result
119
- );
120
- if (!isSHPArrowTable(batch)) return batch;
121
- const table = unwrapSHPTable(batch);
122
- return table.batches[0] ?? makeArrowMetadataBatch(table.schema);
123
- }
124
- } as const satisfies LoaderWithParser<
125
- SHPResult | Table,
126
- SHPHeader | (Uint8Array | null)[] | RecordBatch,
127
- SHPLoaderOptions
128
- >;
129
-
130
- /** Identifies Arrow output from the private column parser. */
131
- function isSHPArrowTable(result: unknown): result is ArrowTable {
132
- return (result as {shape?: string} | null)?.shape === 'arrow-table';
133
- }
134
-
135
- /** Unwraps Arrow columns and passes through legacy results. */
136
- function unwrapSHPTable(result: ArrowTable): Table;
137
- function unwrapSHPTable<T>(result: T): Exclude<T, ArrowTable> | Table;
138
- function unwrapSHPTable(result: unknown): unknown {
139
- if (!isSHPArrowTable(result)) return result;
140
- return result.data;
141
- }
@@ -8,7 +8,7 @@ import {
8
8
  type LoaderContext,
9
9
  type LoaderWithParser
10
10
  } from '@loaders.gl/core/loader-utils';
11
- import {ShapefileColumnLoader} from '../shapefile/shapefile-loader';
11
+ import {ShapefileLoaderWithParser} from '../shapefile/shapefile-loader';
12
12
 
13
13
  /** Installs local subloader parsing so streamed sidecars and geometry stay in this worker. */
14
14
  function createShapefileContext(context?: LoaderContext, options?: any): LoaderContext {
@@ -65,10 +65,10 @@ function getArrowOptions(options: any): any {
65
65
  }
66
66
 
67
67
  createLoaderWorker({
68
- ...ShapefileColumnLoader,
68
+ ...ShapefileLoaderWithParser,
69
69
  /** Parses geometries and sidecar attributes into owned transferable columns. */
70
70
  parse(input, options, context) {
71
- return ShapefileColumnLoader.parse(
71
+ return ShapefileLoaderWithParser.parse(
72
72
  input,
73
73
  getArrowOptions(options),
74
74
  createShapefileContext(context, options)
@@ -76,7 +76,7 @@ createLoaderWorker({
76
76
  },
77
77
  /** Joins streamed geometry and attribute batches without delegating their rows to the caller. */
78
78
  parseInBatches(input, options, context) {
79
- return ShapefileColumnLoader.parseInBatches(
79
+ return ShapefileLoaderWithParser.parseInBatches(
80
80
  input,
81
81
  getArrowOptions(options),
82
82
  createShapefileContext(context, options)
@@ -2,7 +2,7 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import {SHPColumnLoader} from '../shapefile/shp-loader';
5
+ import {SHPLoaderWithParser} from '../shapefile/shp-loader';
6
6
  import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
7
7
 
8
- createLoaderWorker(SHPColumnLoader);
8
+ createLoaderWorker(SHPLoaderWithParser);