@loaders.gl/tile-converter 5.0.0-alpha.7 → 5.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +155 -0
- package/dist/converter.min.cjs +194 -175
- package/dist/i3s-server/bin/i3s-server.min.cjs +84 -84
- package/dist/index.cjs +35865 -0
- package/dist/index.js +35837 -0
- package/dist/v5/browser-sink.d.ts +37 -0
- package/dist/v5/browser-source.d.ts +38 -0
- package/dist/v5/browser-tileset-source.d.ts +70 -0
- package/dist/v5/browser.cjs +1962 -0
- package/dist/v5/browser.d.ts +18 -0
- package/dist/v5/browser.js +1927 -0
- package/dist/v5/conversion-api.d.ts +122 -0
- package/dist/v5/feature-arrow.d.ts +36 -0
- package/dist/v5/index.cjs +3975 -0
- package/dist/v5/index.d.ts +18 -0
- package/dist/v5/index.js +3928 -0
- package/dist/v5/point-cloud-conversion.d.ts +31 -0
- package/dist/v5/point-cloud-source-encoder.d.ts +49 -0
- package/dist/v5/point-cloud-source.d.ts +25 -0
- package/dist/v5/point-cloud.d.ts +24 -0
- package/dist/v5/resource-manifest.d.ts +58 -0
- package/dist/v5/spatial-conversion.d.ts +59 -0
- package/dist/v5/tileset-source.d.ts +21 -0
- package/package.json +41 -26
- package/src/common/README.md +5 -0
- package/src/index.ts +1 -2
- package/src/{3d-tiles-converter → v4/3d-tiles-converter}/3d-tiles-converter.ts +13 -2
- package/src/v4/3d-tiles-converter/helpers/validate-i3s-spatial-reference.ts +124 -0
- package/src/{deps-installer → v4/deps-installer}/deps-installer.ts +10 -7
- package/src/v4/i3s-converter/helpers/encode-string-attribute.ts +23 -0
- package/src/{i3s-converter → v4/i3s-converter}/helpers/geometry-converter.ts +15 -42
- package/src/{i3s-converter → v4/i3s-converter}/helpers/gltf-attributes.ts +5 -4
- package/src/{i3s-converter → v4/i3s-converter}/helpers/load-3d-tiles.ts +5 -9
- package/src/{i3s-converter → v4/i3s-converter}/i3s-converter.ts +6 -7
- package/src/{i3s-server → v4/i3s-server}/README.md +2 -2
- package/src/{i3s-server → v4/i3s-server}/routes/slpk-router.ts +1 -0
- package/src/v4/index.ts +2 -0
- package/src/v5/browser-sink.ts +129 -0
- package/src/v5/browser-source.ts +155 -0
- package/src/v5/browser-tileset-source.ts +328 -0
- package/src/v5/browser.ts +54 -0
- package/src/v5/conversion-api.ts +287 -0
- package/src/v5/feature-arrow.ts +427 -0
- package/src/v5/index.ts +47 -0
- package/src/v5/point-cloud-conversion.ts +80 -0
- package/src/v5/point-cloud-source-encoder.ts +102 -0
- package/src/v5/point-cloud-source.ts +84 -0
- package/src/v5/point-cloud.ts +271 -0
- package/src/v5/resource-manifest.ts +213 -0
- package/src/v5/spatial-conversion.ts +305 -0
- package/src/v5/tileset-source.ts +54 -0
- /package/src/{3d-tiles-converter → v4/3d-tiles-converter}/helpers/3d-tiles-content-converter.ts +0 -0
- /package/src/{3d-tiles-converter → v4/3d-tiles-converter}/helpers/i3s-obb-to-3d-tiles-obb.ts +0 -0
- /package/src/{3d-tiles-converter → v4/3d-tiles-converter}/helpers/load-i3s.ts +0 -0
- /package/src/{3d-tiles-converter → v4/3d-tiles-converter}/helpers/texture-atlas.ts +0 -0
- /package/src/{3d-tiles-converter → v4/3d-tiles-converter}/json-templates/tileset.ts +0 -0
- /package/src/{constants.ts → v4/constants.ts} +0 -0
- /package/src/{converter-cli.ts → v4/converter-cli.ts} +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/attribute-metadata-info.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/batch-ids-extensions.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/coordinate-converter.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/create-scene-server-path.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/feature-attributes.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/geometry-attributes.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/node-debug.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/node-index-document.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/node-pages.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/preprocess-3d-tiles.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/progress.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/helpers/tileset-traversal.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/geometry-definitions.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/layers.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/metadata.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/node.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/scene-server.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/shared-resources.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/json-templates/store.ts +0 -0
- /package/src/{i3s-converter → v4/i3s-converter}/types.ts +0 -0
- /package/src/{i3s-server → v4/i3s-server}/app.ts +0 -0
- /package/src/{i3s-server → v4/i3s-server}/bin/www.ts +0 -0
- /package/src/{i3s-server → v4/i3s-server}/certs/cert.pem +0 -0
- /package/src/{i3s-server → v4/i3s-server}/certs/key.pem +0 -0
- /package/src/{i3s-server → v4/i3s-server}/controllers/slpk-controller.ts +0 -0
- /package/src/{i3s-server → v4/i3s-server}/utils/create-scene-server.ts +0 -0
- /package/src/{i3s-server → v4/i3s-server}/utils/server-utils.ts +0 -0
- /package/src/{lib → v4/lib}/json-schemas/conversion-dump-json-schema.ts +0 -0
- /package/src/{lib → v4/lib}/utils/cli-utils.ts +0 -0
- /package/src/{lib → v4/lib}/utils/compress-util.ts +0 -0
- /package/src/{lib → v4/lib}/utils/conversion-dump.ts +0 -0
- /package/src/{lib → v4/lib}/utils/file-utils.ts +0 -0
- /package/src/{lib → v4/lib}/utils/geometry-utils.ts +0 -0
- /package/src/{lib → v4/lib}/utils/lod-conversion-utils.ts +0 -0
- /package/src/{lib → v4/lib}/utils/queue.ts +0 -0
- /package/src/{lib → v4/lib}/utils/statistic-utils.ts +0 -0
- /package/src/{lib → v4/lib}/utils/write-queue.ts +0 -0
- /package/src/{pgm-loader.ts → v4/pgm-loader.ts} +0 -0
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {ArrowTableBuilder} from '@loaders.gl/schema-utils';
|
|
6
|
+
import type {ArrowTableBatch, DataType, Field, Schema} from '@loaders.gl/schema';
|
|
7
|
+
import {TileConversionError} from './conversion-api.js';
|
|
8
|
+
|
|
9
|
+
const DEFAULT_FEATURE_ID_FIELD = 'feature_id';
|
|
10
|
+
const FEATURE_CLASS_METADATA_KEY = 'loaders.gl:feature-class';
|
|
11
|
+
|
|
12
|
+
/** Feature attributes and source identifier ready for Arrow normalization. */
|
|
13
|
+
export interface TileFeatureAttributes {
|
|
14
|
+
/** Stable feature identifier decoded from the source's feature-ID descriptor. */
|
|
15
|
+
readonly featureId: string | number | bigint;
|
|
16
|
+
/** Property values interpreted from the source metadata class. */
|
|
17
|
+
readonly properties: Readonly<Record<string, unknown>>;
|
|
18
|
+
/** Original class or property-table name, kept distinct between output batches. */
|
|
19
|
+
readonly metadataClass: string;
|
|
20
|
+
/** Optional original encoded metadata bytes, preserved without interpretation. */
|
|
21
|
+
readonly rawMetadata?: Uint8Array;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Explicit Arrow mapping and bounded output size for feature attribute batches. */
|
|
25
|
+
export interface FeatureArrowBatchOptions {
|
|
26
|
+
/** Declared output columns and types, including the feature-ID column. */
|
|
27
|
+
readonly schema: Schema;
|
|
28
|
+
/** Schema field receiving stable feature identifiers. Defaults to `feature_id`. */
|
|
29
|
+
readonly featureIdField?: string;
|
|
30
|
+
/** Optional binary field receiving source metadata bytes. */
|
|
31
|
+
readonly rawMetadataField?: string;
|
|
32
|
+
/** Maximum number of features in one emitted Arrow record batch. Defaults to 65,536. */
|
|
33
|
+
readonly batchSize?: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Converts decoded feature attributes to bounded Arrow batches using a caller-declared schema.
|
|
38
|
+
*
|
|
39
|
+
* Values retain their source primitive types; nested lists, structs, dictionary enums, and 64-bit
|
|
40
|
+
* identifiers use the corresponding schema field type. Missing schema fields and incompatible
|
|
41
|
+
* identifiers fail explicitly. Unknown properties are rejected so a class mapping cannot silently
|
|
42
|
+
* discard data. Separate metadata classes should be passed separately with their own schema.
|
|
43
|
+
*
|
|
44
|
+
* @param features - Features decoded from one source metadata class.
|
|
45
|
+
* @param options - Explicit Arrow schema and batch configuration.
|
|
46
|
+
* @returns Arrow table batches whose schema metadata records the source metadata class.
|
|
47
|
+
*/
|
|
48
|
+
export function convertFeatureAttributesToArrowBatches(
|
|
49
|
+
features: readonly TileFeatureAttributes[],
|
|
50
|
+
options: FeatureArrowBatchOptions
|
|
51
|
+
): ArrowTableBatch[] {
|
|
52
|
+
const featureIdField = options.featureIdField || DEFAULT_FEATURE_ID_FIELD;
|
|
53
|
+
const batchSize = options.batchSize ?? 65_536;
|
|
54
|
+
if (!Number.isInteger(batchSize) || batchSize <= 0) {
|
|
55
|
+
throw new TileConversionError(
|
|
56
|
+
'INVALID_FEATURE_BATCH_SIZE',
|
|
57
|
+
'Feature Arrow batch size must be a positive integer'
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const featureClasses = new Set(features.map(feature => feature.metadataClass));
|
|
62
|
+
if (featureClasses.size > 1) {
|
|
63
|
+
throw new TileConversionError(
|
|
64
|
+
'MULTIPLE_FEATURE_CLASSES',
|
|
65
|
+
'Build separate Arrow batches for each source metadata class'
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
const metadataClass = features[0]?.metadataClass;
|
|
69
|
+
const schema = makeFeatureSchema(
|
|
70
|
+
options.schema,
|
|
71
|
+
featureIdField,
|
|
72
|
+
options.rawMetadataField,
|
|
73
|
+
metadataClass
|
|
74
|
+
);
|
|
75
|
+
const fieldsByName = new Map(schema.fields.map(field => [field.name, field]));
|
|
76
|
+
const propertyFieldNames = new Set(
|
|
77
|
+
schema.fields
|
|
78
|
+
.filter(field => field.name !== featureIdField && field.name !== options.rawMetadataField)
|
|
79
|
+
.map(field => field.name)
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
for (const feature of features) {
|
|
83
|
+
validateFeatureIdentifier(feature.featureId, fieldsByName.get(featureIdField)?.type);
|
|
84
|
+
if (feature.rawMetadata && !options.rawMetadataField) {
|
|
85
|
+
throw new TileConversionError(
|
|
86
|
+
'RAW_METADATA_FIELD_REQUIRED',
|
|
87
|
+
'A rawMetadataField must be declared to preserve encoded source metadata'
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
for (const propertyName of Object.keys(feature.properties)) {
|
|
91
|
+
if (!propertyFieldNames.has(propertyName)) {
|
|
92
|
+
throw new TileConversionError(
|
|
93
|
+
'FEATURE_PROPERTY_NOT_MAPPED',
|
|
94
|
+
`Feature property "${propertyName}" is not declared in the Arrow schema`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const batches: ArrowTableBatch[] = [];
|
|
101
|
+
for (let batchStart = 0; batchStart < features.length; batchStart += batchSize) {
|
|
102
|
+
const tableBuilder = new ArrowTableBuilder(schema);
|
|
103
|
+
const batchFeatures = features.slice(batchStart, batchStart + batchSize);
|
|
104
|
+
for (const feature of batchFeatures) {
|
|
105
|
+
const row: Record<string, unknown> = {
|
|
106
|
+
...feature.properties,
|
|
107
|
+
[featureIdField]: feature.featureId
|
|
108
|
+
};
|
|
109
|
+
if (options.rawMetadataField) {
|
|
110
|
+
row[options.rawMetadataField] = feature.rawMetadata || null;
|
|
111
|
+
}
|
|
112
|
+
for (const field of schema.fields) {
|
|
113
|
+
if (row[field.name] === undefined) {
|
|
114
|
+
row[field.name] = null;
|
|
115
|
+
}
|
|
116
|
+
validateFeatureValue(field, row[field.name], field.name);
|
|
117
|
+
}
|
|
118
|
+
tableBuilder.addObjectRow(row);
|
|
119
|
+
}
|
|
120
|
+
const batch = tableBuilder.finishBatch();
|
|
121
|
+
if (batch) {
|
|
122
|
+
batches.push(batch);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return batches;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Adds feature class metadata and validates the required ID and raw metadata mappings. */
|
|
129
|
+
function makeFeatureSchema(
|
|
130
|
+
sourceSchema: Schema,
|
|
131
|
+
featureIdField: string,
|
|
132
|
+
rawMetadataField: string | undefined,
|
|
133
|
+
metadataClass: string | undefined
|
|
134
|
+
): Schema {
|
|
135
|
+
const fieldsByName = new Map(sourceSchema.fields.map(field => [field.name, field]));
|
|
136
|
+
if (fieldsByName.size !== sourceSchema.fields.length) {
|
|
137
|
+
throw new TileConversionError(
|
|
138
|
+
'DUPLICATE_FEATURE_FIELDS',
|
|
139
|
+
'Arrow schema fields must have unique names'
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
for (const field of sourceSchema.fields) {
|
|
143
|
+
validateFeatureSchemaType(field);
|
|
144
|
+
}
|
|
145
|
+
const featureIdType = fieldsByName.get(featureIdField)?.type;
|
|
146
|
+
if (!featureIdType) {
|
|
147
|
+
throw new TileConversionError(
|
|
148
|
+
'FEATURE_ID_FIELD_REQUIRED',
|
|
149
|
+
`Arrow schema must declare the feature identifier field "${featureIdField}"`
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
if (rawMetadataField && !fieldsByName.has(rawMetadataField)) {
|
|
153
|
+
throw new TileConversionError(
|
|
154
|
+
'RAW_METADATA_FIELD_NOT_DECLARED',
|
|
155
|
+
`Arrow schema does not declare the raw metadata field "${rawMetadataField}"`
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
if (rawMetadataField === featureIdField) {
|
|
159
|
+
throw new TileConversionError(
|
|
160
|
+
'FEATURE_FIELDS_COLLIDE',
|
|
161
|
+
'The feature identifier and raw metadata require separate Arrow fields'
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
if (rawMetadataField && !isBinaryType(fieldsByName.get(rawMetadataField)?.type)) {
|
|
165
|
+
throw new TileConversionError(
|
|
166
|
+
'RAW_METADATA_FIELD_NOT_BINARY',
|
|
167
|
+
`Arrow schema field "${rawMetadataField}" must use binary or fixed-size-binary type`
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
...sourceSchema,
|
|
172
|
+
metadata: {
|
|
173
|
+
...(sourceSchema.metadata || {}),
|
|
174
|
+
...(metadataClass ? {[FEATURE_CLASS_METADATA_KEY]: metadataClass} : {})
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Rejects identifier values that the declared Arrow type cannot preserve exactly. */
|
|
180
|
+
function validateFeatureIdentifier(featureId: string | number | bigint, dataType?: DataType): void {
|
|
181
|
+
const isStringIdentifier = typeof featureId === 'string';
|
|
182
|
+
const isIntegerIdentifier = typeof featureId === 'bigint' || Number.isSafeInteger(featureId);
|
|
183
|
+
const isStringType = dataType === 'utf8' || dataType === 'utf8-view';
|
|
184
|
+
if (
|
|
185
|
+
(isStringIdentifier && isStringType) ||
|
|
186
|
+
(isIntegerIdentifier && isIntegerDataType(dataType))
|
|
187
|
+
) {
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
throw new TileConversionError(
|
|
191
|
+
'FEATURE_ID_TYPE_MISMATCH',
|
|
192
|
+
`Feature identifier ${String(featureId)} is not compatible with the declared Arrow type`
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** Rejects nulls, implicit coercions, and out-of-range values before Arrow writes a field. */
|
|
197
|
+
function validateFeatureValue(field: Field, value: unknown, fieldPath: string): void {
|
|
198
|
+
if (value === null || value === undefined) {
|
|
199
|
+
if (field.nullable !== true) {
|
|
200
|
+
throw new TileConversionError(
|
|
201
|
+
'FEATURE_VALUE_REQUIRED',
|
|
202
|
+
`Required Arrow field "${fieldPath}" has no value`
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const dataType = field.type;
|
|
209
|
+
if (typeof dataType === 'object') {
|
|
210
|
+
if (dataType.type === 'dictionary') {
|
|
211
|
+
validateFeatureValue({...field, type: dataType.dictionary}, value, fieldPath);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (dataType.type === 'list' || dataType.type === 'large-list') {
|
|
215
|
+
if (!Array.isArray(value)) {
|
|
216
|
+
throw createFeatureValueTypeError(fieldPath, 'an array');
|
|
217
|
+
}
|
|
218
|
+
const childField = dataType.children[0];
|
|
219
|
+
for (let index = 0; index < value.length; index++) {
|
|
220
|
+
validateFeatureValue(childField, value[index], `${fieldPath}[${index}]`);
|
|
221
|
+
}
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
if (dataType.type === 'fixed-size-list') {
|
|
225
|
+
if (!Array.isArray(value) || value.length !== dataType.listSize) {
|
|
226
|
+
throw createFeatureValueTypeError(fieldPath, `an array of length ${dataType.listSize}`);
|
|
227
|
+
}
|
|
228
|
+
const childField = dataType.children[0];
|
|
229
|
+
for (let index = 0; index < value.length; index++) {
|
|
230
|
+
validateFeatureValue(childField, value[index], `${fieldPath}[${index}]`);
|
|
231
|
+
}
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
if (dataType.type === 'struct') {
|
|
235
|
+
if (!isRecord(value)) {
|
|
236
|
+
throw createFeatureValueTypeError(fieldPath, 'an object');
|
|
237
|
+
}
|
|
238
|
+
const childNames = new Set(dataType.children.map(childField => childField.name));
|
|
239
|
+
const unknownNames = Object.keys(value).filter(name => !childNames.has(name));
|
|
240
|
+
if (unknownNames.length > 0) {
|
|
241
|
+
throw new TileConversionError(
|
|
242
|
+
'FEATURE_PROPERTY_NOT_MAPPED',
|
|
243
|
+
`Nested feature properties "${unknownNames.join(', ')}" are not declared under "${fieldPath}"`
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
for (const childField of dataType.children) {
|
|
247
|
+
validateFeatureValue(childField, value[childField.name], `${fieldPath}.${childField.name}`);
|
|
248
|
+
}
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (dataType.type === 'decimal') {
|
|
252
|
+
if (typeof value !== 'bigint') {
|
|
253
|
+
throw createFeatureValueTypeError(fieldPath, 'an unscaled bigint decimal value');
|
|
254
|
+
}
|
|
255
|
+
if (
|
|
256
|
+
value <= -(10n ** BigInt(dataType.precision)) ||
|
|
257
|
+
value >= 10n ** BigInt(dataType.precision)
|
|
258
|
+
) {
|
|
259
|
+
throw new TileConversionError(
|
|
260
|
+
'FEATURE_VALUE_OUT_OF_RANGE',
|
|
261
|
+
`Value for Arrow decimal field "${fieldPath}" exceeds precision ${dataType.precision}`
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
if (dataType.type === 'fixed-size-binary') {
|
|
267
|
+
const byteLength = getByteLength(value);
|
|
268
|
+
if (byteLength !== dataType.byteWidth) {
|
|
269
|
+
throw createFeatureValueTypeError(fieldPath, `exactly ${dataType.byteWidth} bytes`);
|
|
270
|
+
}
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
// Keep future Arrow composites explicit until a lossless runtime mapping is defined.
|
|
274
|
+
throw new TileConversionError(
|
|
275
|
+
'FEATURE_VALUE_MAPPING_UNSUPPORTED',
|
|
276
|
+
`Arrow schema type "${dataType.type}" has no feature value mapping for "${fieldPath}"`
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (isIntegerDataType(dataType)) {
|
|
281
|
+
if (!isIntegerIdentifier(value)) {
|
|
282
|
+
throw createFeatureValueTypeError(fieldPath, 'an integer');
|
|
283
|
+
}
|
|
284
|
+
const [minimum, maximum] = getIntegerBounds(dataType);
|
|
285
|
+
const integerValue = typeof value === 'bigint' ? value : BigInt(value);
|
|
286
|
+
if (integerValue < minimum || integerValue > maximum) {
|
|
287
|
+
throw new TileConversionError(
|
|
288
|
+
'FEATURE_VALUE_OUT_OF_RANGE',
|
|
289
|
+
`Value for Arrow field "${fieldPath}" is outside the range of ${dataType}`
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
if (dataType === 'utf8' || dataType === 'utf8-view') {
|
|
295
|
+
if (typeof value !== 'string') throw createFeatureValueTypeError(fieldPath, 'a string');
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
if (dataType === 'bool') {
|
|
299
|
+
if (typeof value !== 'boolean') throw createFeatureValueTypeError(fieldPath, 'a boolean');
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (dataType === 'binary' || dataType === 'binary-view') {
|
|
303
|
+
if (getByteLength(value) < 0) throw createFeatureValueTypeError(fieldPath, 'binary data');
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
if (
|
|
307
|
+
dataType === 'float' ||
|
|
308
|
+
dataType === 'float16' ||
|
|
309
|
+
dataType === 'float32' ||
|
|
310
|
+
dataType === 'float64'
|
|
311
|
+
) {
|
|
312
|
+
if (typeof value !== 'number') throw createFeatureValueTypeError(fieldPath, 'a number');
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
if (dataType === 'null') {
|
|
316
|
+
throw createFeatureValueTypeError(fieldPath, 'null');
|
|
317
|
+
}
|
|
318
|
+
if (
|
|
319
|
+
dataType === 'date-day' ||
|
|
320
|
+
dataType === 'date-millisecond' ||
|
|
321
|
+
dataType.startsWith('timestamp-')
|
|
322
|
+
) {
|
|
323
|
+
if (!(value instanceof Date) || !Number.isFinite(value.getTime())) {
|
|
324
|
+
throw createFeatureValueTypeError(fieldPath, 'a valid Date');
|
|
325
|
+
}
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
throw new TileConversionError(
|
|
329
|
+
'FEATURE_VALUE_MAPPING_UNSUPPORTED',
|
|
330
|
+
`Arrow schema type "${dataType}" has no feature value mapping for "${fieldPath}"`
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Checks whether a feature identifier is exact enough for an integer Arrow field. */
|
|
335
|
+
function isIntegerIdentifier(value: unknown): value is number | bigint {
|
|
336
|
+
return typeof value === 'bigint' || (typeof value === 'number' && Number.isSafeInteger(value));
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** Identifies all Arrow integer primitive types accepted for feature identifiers. */
|
|
340
|
+
function isIntegerDataType(dataType?: DataType): boolean {
|
|
341
|
+
return (
|
|
342
|
+
dataType === 'int8' ||
|
|
343
|
+
dataType === 'int16' ||
|
|
344
|
+
dataType === 'int32' ||
|
|
345
|
+
dataType === 'int64' ||
|
|
346
|
+
dataType === 'uint8' ||
|
|
347
|
+
dataType === 'uint16' ||
|
|
348
|
+
dataType === 'uint32' ||
|
|
349
|
+
dataType === 'uint64'
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/** Rejects generic integer types and nested mappings the Arrow table builder cannot materialize. */
|
|
354
|
+
function validateFeatureSchemaType(field: Field): void {
|
|
355
|
+
const dataType = field.type;
|
|
356
|
+
if (dataType === 'int') {
|
|
357
|
+
throw new TileConversionError(
|
|
358
|
+
'FEATURE_INTEGER_TYPE_UNSUPPORTED',
|
|
359
|
+
`Arrow field "${field.name}" must declare an explicit integer width and signedness`
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
if (typeof dataType !== 'object') {
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
let childFields: readonly Field[] = [];
|
|
366
|
+
if (dataType.type === 'dictionary' && dataType.indices) {
|
|
367
|
+
childFields = [{name: `${field.name} index`, type: dataType.indices, nullable: false}];
|
|
368
|
+
} else if ('children' in dataType) {
|
|
369
|
+
childFields = dataType.children;
|
|
370
|
+
}
|
|
371
|
+
for (const childField of childFields) {
|
|
372
|
+
validateFeatureSchemaType(childField);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Returns the inclusive numeric limits for the declared integer field. */
|
|
377
|
+
function getIntegerBounds(dataType: DataType): [bigint, bigint] {
|
|
378
|
+
switch (dataType) {
|
|
379
|
+
case 'int8':
|
|
380
|
+
return [-128n, 127n];
|
|
381
|
+
case 'uint8':
|
|
382
|
+
return [0n, 255n];
|
|
383
|
+
case 'int16':
|
|
384
|
+
return [-32768n, 32767n];
|
|
385
|
+
case 'uint16':
|
|
386
|
+
return [0n, 65535n];
|
|
387
|
+
case 'int32':
|
|
388
|
+
return [-2147483648n, 2147483647n];
|
|
389
|
+
case 'uint32':
|
|
390
|
+
return [0n, 4294967295n];
|
|
391
|
+
case 'int64':
|
|
392
|
+
return [-(2n ** 63n), 2n ** 63n - 1n];
|
|
393
|
+
case 'uint64':
|
|
394
|
+
return [0n, 2n ** 64n - 1n];
|
|
395
|
+
default:
|
|
396
|
+
return [-(2n ** 31n), 2n ** 31n - 1n];
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** Creates a typed error that names the field whose schema mapping does not match its value. */
|
|
401
|
+
function createFeatureValueTypeError(fieldPath: string, expected: string): TileConversionError {
|
|
402
|
+
return new TileConversionError(
|
|
403
|
+
'FEATURE_VALUE_TYPE_MISMATCH',
|
|
404
|
+
`Arrow field "${fieldPath}" requires ${expected}`
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** Checks whether a value is a property record rather than an array or scalar. */
|
|
409
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
410
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** Returns the byte length for supported binary value containers. */
|
|
414
|
+
function getByteLength(value: unknown): number {
|
|
415
|
+
if (value instanceof Uint8Array) return value.byteLength;
|
|
416
|
+
if (value instanceof ArrayBuffer) return value.byteLength;
|
|
417
|
+
return -1;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** Checks whether the schema type stores opaque bytes without coercion. */
|
|
421
|
+
function isBinaryType(dataType?: DataType): boolean {
|
|
422
|
+
return (
|
|
423
|
+
dataType === 'binary' ||
|
|
424
|
+
dataType === 'binary-view' ||
|
|
425
|
+
(typeof dataType === 'object' && dataType.type === 'fixed-size-binary')
|
|
426
|
+
);
|
|
427
|
+
}
|
package/src/v5/index.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export {
|
|
2
|
+
convertTileset,
|
|
3
|
+
inspectTileset,
|
|
4
|
+
TileConversionError,
|
|
5
|
+
validateTileset
|
|
6
|
+
} from './conversion-api.js';
|
|
7
|
+
export type {
|
|
8
|
+
ConvertTilesetOptions,
|
|
9
|
+
TileConversionCodec,
|
|
10
|
+
TileConversionDiagnostic,
|
|
11
|
+
TileConversionProgress,
|
|
12
|
+
TileConversionReport,
|
|
13
|
+
TileConversionSink,
|
|
14
|
+
TileConversionSource,
|
|
15
|
+
TileValidationReport,
|
|
16
|
+
ValidateTilesetOptions
|
|
17
|
+
} from './conversion-api.js';
|
|
18
|
+
export {createTilesetConversionSource} from './tileset-source.js';
|
|
19
|
+
export type {TilesetConversionSourceOptions} from './tileset-source.js';
|
|
20
|
+
export {
|
|
21
|
+
createI3SConversionSpatialContext,
|
|
22
|
+
createTiles3DConversionSpatialContext
|
|
23
|
+
} from './spatial-conversion.js';
|
|
24
|
+
export type {
|
|
25
|
+
I3SConversionSpatialContext,
|
|
26
|
+
Tiles3DConversionSpatialContext
|
|
27
|
+
} from './spatial-conversion.js';
|
|
28
|
+
export {convertFeatureAttributesToArrowBatches} from './feature-arrow.js';
|
|
29
|
+
export type {FeatureArrowBatchOptions, TileFeatureAttributes} from './feature-arrow.js';
|
|
30
|
+
export {encodePointCloudTile} from './point-cloud.js';
|
|
31
|
+
export type {EncodePointCloudTileOptions} from './point-cloud.js';
|
|
32
|
+
export {traversePointCloudSource} from './point-cloud-source.js';
|
|
33
|
+
export type {PointCloudSourceTile, TraversePointCloudSourceOptions} from './point-cloud-source.js';
|
|
34
|
+
export {encodePointCloudSource, encodePointCloudSourceTile} from './point-cloud-source-encoder.js';
|
|
35
|
+
export type {
|
|
36
|
+
EncodedPointCloudSourceTile,
|
|
37
|
+
EncodePointCloudSourceOptions
|
|
38
|
+
} from './point-cloud-source-encoder.js';
|
|
39
|
+
export {convertPointCloudSource} from './point-cloud-conversion.js';
|
|
40
|
+
export type {ConvertPointCloudSourceOptions} from './point-cloud-conversion.js';
|
|
41
|
+
export {createManifestBackedTileConversionSink} from './resource-manifest.js';
|
|
42
|
+
export type {
|
|
43
|
+
ManifestBackedTileConversionSinkOptions,
|
|
44
|
+
TileResourceManifest,
|
|
45
|
+
TileResourceManifestEntry,
|
|
46
|
+
TileResourceManifestStore
|
|
47
|
+
} from './resource-manifest.js';
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {PointCloudTilesetSource} from '@loaders.gl/tiles';
|
|
6
|
+
import {convertTileset} from './conversion-api.js';
|
|
7
|
+
import type {
|
|
8
|
+
TileConversionProgress,
|
|
9
|
+
TileConversionReport,
|
|
10
|
+
TileConversionSink
|
|
11
|
+
} from './conversion-api.js';
|
|
12
|
+
import {traversePointCloudSource} from './point-cloud-source.js';
|
|
13
|
+
import type {PointCloudSourceTile} from './point-cloud-source.js';
|
|
14
|
+
import {encodePointCloudSourceTile} from './point-cloud-source-encoder.js';
|
|
15
|
+
import type {
|
|
16
|
+
EncodedPointCloudSourceTile,
|
|
17
|
+
EncodePointCloudSourceOptions
|
|
18
|
+
} from './point-cloud-source-encoder.js';
|
|
19
|
+
|
|
20
|
+
/** Point-cloud encoding options and an application-owned output destination. */
|
|
21
|
+
export interface ConvertPointCloudSourceOptions extends EncodePointCloudSourceOptions {
|
|
22
|
+
/** Destination receiving PNTS resources and their source placement metadata. */
|
|
23
|
+
readonly sink: TileConversionSink<EncodedPointCloudSourceTile>;
|
|
24
|
+
/** Input byte accounting for each visited source tile, including tiles with null content. */
|
|
25
|
+
readonly measureInputBytes: (sourceTile: PointCloudSourceTile) => number;
|
|
26
|
+
/** Maximum caller-measured bytes in one decoded source tile, checked before PNTS encoding. */
|
|
27
|
+
readonly maxInputResourceBytes?: number;
|
|
28
|
+
/** Maximum bytes in one encoded PNTS resource. Defaults to no limit. */
|
|
29
|
+
readonly maxOutputResourceBytes?: number;
|
|
30
|
+
/** Receive conversion progress, including resource counts and caller-measured input bytes. */
|
|
31
|
+
readonly onProgress?: (progress: TileConversionProgress) => void;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Converts a point-cloud source to PNTS resources through the v5 output-sink lifecycle.
|
|
36
|
+
*
|
|
37
|
+
* Each write is awaited before reading the next source tile. The destination is finalized only
|
|
38
|
+
* after traversal and writes succeed, and aborted on read, encoding, write, or cancellation errors.
|
|
39
|
+
* Input resource counts include empty tiles; output counts and byte sizes describe emitted PNTS
|
|
40
|
+
* resources. Applications supply input byte accounting because source tiles contain decoded Arrow
|
|
41
|
+
* data rather than transport bytes. The caller retains ownership of the input source.
|
|
42
|
+
*
|
|
43
|
+
* @param source - An I3S, COPC, or compatible point-cloud source to initialize and traverse.
|
|
44
|
+
* @param options - Destination, byte accounting, traversal limits, and per-tile encoding options.
|
|
45
|
+
* @returns The shared v5 conversion report after destination finalization.
|
|
46
|
+
*/
|
|
47
|
+
export function convertPointCloudSource(
|
|
48
|
+
source: PointCloudTilesetSource,
|
|
49
|
+
options: ConvertPointCloudSourceOptions
|
|
50
|
+
): Promise<TileConversionReport> {
|
|
51
|
+
return convertTileset({
|
|
52
|
+
source: {
|
|
53
|
+
async inspect() {
|
|
54
|
+
if (!source.isReady) {
|
|
55
|
+
await source.initialize();
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
read: (_inspection, signal) =>
|
|
59
|
+
traversePointCloudSource(source, {maxDepth: options.maxDepth, signal})
|
|
60
|
+
},
|
|
61
|
+
codec: {
|
|
62
|
+
async *convert(sourceTile) {
|
|
63
|
+
const encodingOptions = sourceTile.content
|
|
64
|
+
? options.getTileEncodingOptions?.(sourceTile)
|
|
65
|
+
: undefined;
|
|
66
|
+
const encodedTile = encodePointCloudSourceTile(sourceTile, encodingOptions);
|
|
67
|
+
if (encodedTile) {
|
|
68
|
+
yield encodedTile;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
sink: options.sink,
|
|
73
|
+
measureInputBytes: options.measureInputBytes,
|
|
74
|
+
measureOutputBytes: resource => resource.pnts.byteLength,
|
|
75
|
+
maxInputResourceBytes: options.maxInputResourceBytes,
|
|
76
|
+
maxOutputResourceBytes: options.maxOutputResourceBytes,
|
|
77
|
+
signal: options.signal,
|
|
78
|
+
onProgress: options.onProgress
|
|
79
|
+
});
|
|
80
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
PointCloudBoundingVolume,
|
|
7
|
+
PointCloudCoordinateSystem,
|
|
8
|
+
PointCloudTilesetSource,
|
|
9
|
+
TilesetSpatialReference
|
|
10
|
+
} from '@loaders.gl/tiles';
|
|
11
|
+
import {encodePointCloudTile} from './point-cloud.js';
|
|
12
|
+
import type {EncodePointCloudTileOptions} from './point-cloud.js';
|
|
13
|
+
import {traversePointCloudSource} from './point-cloud-source.js';
|
|
14
|
+
import type {PointCloudSourceTile, TraversePointCloudSourceOptions} from './point-cloud-source.js';
|
|
15
|
+
|
|
16
|
+
/** Traversal and per-tile encoding options for a point-cloud source. */
|
|
17
|
+
export type EncodePointCloudSourceOptions = TraversePointCloudSourceOptions & {
|
|
18
|
+
/** Select encoder options for each source tile with content, using its header and decoded data. */
|
|
19
|
+
readonly getTileEncodingOptions?: (
|
|
20
|
+
sourceTile: PointCloudSourceTile
|
|
21
|
+
) => EncodePointCloudTileOptions;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** A PNTS resource paired with the source placement metadata needed to package its tile. */
|
|
25
|
+
export type EncodedPointCloudSourceTile = {
|
|
26
|
+
/** Stable source tile identifier. */
|
|
27
|
+
readonly id: string;
|
|
28
|
+
/** Original source hierarchy and bounds. */
|
|
29
|
+
readonly header: PointCloudSourceTile['header'];
|
|
30
|
+
/** Encoded point-cloud tile content. */
|
|
31
|
+
readonly pnts: ArrayBuffer;
|
|
32
|
+
/** Number of point records encoded. */
|
|
33
|
+
readonly pointCount: number;
|
|
34
|
+
/** Coordinate system used by the returned point positions. */
|
|
35
|
+
readonly coordinateSystem: PointCloudCoordinateSystem;
|
|
36
|
+
/** Origin to add to source-relative point positions. */
|
|
37
|
+
readonly cartographicOrigin: readonly number[];
|
|
38
|
+
/** Optional placement transform supplied by the point-cloud source. */
|
|
39
|
+
readonly modelMatrix?: readonly number[];
|
|
40
|
+
/** Optional source or target CRS metadata. */
|
|
41
|
+
readonly spatialReference?: TilesetSpatialReference;
|
|
42
|
+
/** Optional output-frame tile bounds supplied by the source. */
|
|
43
|
+
readonly spatialBoundingVolume?: PointCloudBoundingVolume;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Encodes a point-cloud source as a stream of PNTS resources, one for each non-empty source tile.
|
|
48
|
+
*
|
|
49
|
+
* @param source - An I3S, COPC, or compatible point-cloud tileset source.
|
|
50
|
+
* @param options - Optional traversal depth limit, cancellation signal, and per-tile encoder options.
|
|
51
|
+
* @returns Encoded source tiles in deterministic depth-first order.
|
|
52
|
+
*/
|
|
53
|
+
export async function* encodePointCloudSource(
|
|
54
|
+
source: PointCloudTilesetSource,
|
|
55
|
+
options: EncodePointCloudSourceOptions = {}
|
|
56
|
+
): AsyncIterableIterator<EncodedPointCloudSourceTile> {
|
|
57
|
+
for await (const sourceTile of traversePointCloudSource(source, options)) {
|
|
58
|
+
const encodingOptions = sourceTile.content
|
|
59
|
+
? options.getTileEncodingOptions?.(sourceTile)
|
|
60
|
+
: undefined;
|
|
61
|
+
const encodedTile = encodePointCloudSourceTile(sourceTile, encodingOptions);
|
|
62
|
+
if (encodedTile) {
|
|
63
|
+
yield encodedTile;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Encodes one decoded source tile as PNTS and retains its hierarchy and placement metadata.
|
|
70
|
+
*
|
|
71
|
+
* Positions use the source coordinate system, with an optional RTC center subtracted before float32
|
|
72
|
+
* encoding and stored in the PNTS feature table. Source placement metadata is retained so callers
|
|
73
|
+
* can assemble the output tileset. This function does not transform the source CRS.
|
|
74
|
+
*
|
|
75
|
+
* @param sourceTile - Tile header and decoded Arrow point content from `traversePointCloudSource`.
|
|
76
|
+
* @param options - Optional RTC center, batch-table values, and constant color override.
|
|
77
|
+
* @returns An encoded source tile, or null when the source tile has no content.
|
|
78
|
+
*/
|
|
79
|
+
export function encodePointCloudSourceTile(
|
|
80
|
+
sourceTile: PointCloudSourceTile,
|
|
81
|
+
options: EncodePointCloudTileOptions = {}
|
|
82
|
+
): EncodedPointCloudSourceTile | null {
|
|
83
|
+
const {header, content} = sourceTile;
|
|
84
|
+
if (!content) {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return {
|
|
89
|
+
id: header.id,
|
|
90
|
+
header,
|
|
91
|
+
pnts: encodePointCloudTile(content.data, {
|
|
92
|
+
...options,
|
|
93
|
+
constantRGBA: options.constantRGBA ?? content.constantRGBA
|
|
94
|
+
}),
|
|
95
|
+
pointCount: content.pointCount,
|
|
96
|
+
coordinateSystem: content.coordinateSystem,
|
|
97
|
+
cartographicOrigin: [...content.cartographicOrigin],
|
|
98
|
+
modelMatrix: content.modelMatrix ? Array.from(content.modelMatrix) : undefined,
|
|
99
|
+
spatialReference: content.spatialReference,
|
|
100
|
+
spatialBoundingVolume: content.spatialBoundingVolume || header.spatialBoundingVolume
|
|
101
|
+
};
|
|
102
|
+
}
|