@loaders.gl/tile-converter 5.0.0-alpha.11 → 5.0.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -2
- package/bin/converter-v5.js +2 -0
- package/dist/converter.min.cjs +162 -163
- package/dist/i3s-server/bin/i3s-server.min.cjs +73 -73
- package/dist/index.cjs +1749 -2152
- package/dist/index.js +1748 -2162
- package/dist/v5/adapters.cjs +1550 -128
- package/dist/v5/adapters.d.ts +16 -0
- package/dist/v5/adapters.js +1558 -117
- package/dist/v5/archive-source.d.ts +36 -0
- package/dist/v5/browser-tileset-source.d.ts +55 -11
- package/dist/v5/browser.cjs +18 -0
- package/dist/v5/browser.d.ts +14 -0
- package/dist/v5/browser.js +26 -0
- package/dist/v5/conversion-api.d.ts +9 -0
- package/dist/v5/converter-cli.d.ts +1 -0
- package/dist/v5/converter-cli.js +1214 -0
- package/dist/v5/core.cjs +471 -60
- package/dist/v5/core.d.ts +2 -0
- package/dist/v5/core.js +471 -60
- package/dist/v5/i3s-mesh-conversion.d.ts +4 -0
- package/dist/v5/i3s-selection-source.d.ts +70 -0
- package/dist/v5/index.cjs +10 -0
- package/dist/v5/index.d.ts +8 -0
- package/dist/v5/index.js +14 -0
- package/dist/v5/mesh-conversion.d.ts +18 -0
- package/dist/v5/mesh-hierarchy.d.ts +47 -0
- package/dist/v5/mesh-source-features.d.ts +2 -1
- package/dist/v5/mesh-source-schema.d.ts +12 -0
- package/dist/v5/mesh-source.d.ts +5 -2
- package/dist/v5/mesh-tileset-sink.d.ts +5 -0
- package/dist/v5/node-cli.d.ts +9 -0
- package/dist/v5/node-conversion.d.ts +32 -0
- package/dist/v5/node-feature-schema.d.ts +3 -0
- package/dist/v5/node-i3s-source.d.ts +41 -0
- package/dist/v5/node-source.d.ts +47 -0
- package/dist/v5/node.cjs +900 -0
- package/dist/v5/node.d.ts +6 -0
- package/dist/v5/node.js +877 -0
- package/dist/v5/point-cloud-file.d.ts +83 -0
- package/dist/v5/point-cloud-potree.d.ts +47 -0
- package/dist/v5/point-cloud-tileset.d.ts +44 -0
- package/dist/v5/resource-manifest.d.ts +10 -1
- package/dist/v5/selected-mesh-source.d.ts +53 -0
- package/package.json +33 -25
- package/src/index.ts +4 -1
- package/src/v4/3d-tiles-converter/helpers/load-i3s.ts +2 -4
- package/src/v4/converter-cli.ts +8 -2
- package/src/v4/i3s-converter/helpers/load-3d-tiles.ts +3 -51
- package/src/v4/i3s-converter/helpers/node-index-document.ts +5 -2
- package/src/v4/i3s-converter/helpers/preprocess-3d-tiles.ts +1 -1
- package/src/v4/i3s-converter/json-templates/store.ts +2 -1
- package/src/v5/adapters.ts +48 -0
- package/src/v5/archive-source.ts +235 -0
- package/src/v5/browser-tileset-source.ts +418 -48
- package/src/v5/browser.ts +45 -0
- package/src/v5/conversion-api.ts +69 -9
- package/src/v5/converter-cli.ts +28 -0
- package/src/v5/core.ts +6 -0
- package/src/v5/i3s-mesh-conversion.ts +40 -2
- package/src/v5/i3s-mesh-sink.ts +1 -1
- package/src/v5/i3s-mesh-source.ts +1 -0
- package/src/v5/i3s-selection-source.ts +298 -0
- package/src/v5/index.ts +26 -0
- package/src/v5/mesh-conversion.ts +20 -0
- package/src/v5/mesh-hierarchy.ts +648 -0
- package/src/v5/mesh-source-features.ts +39 -7
- package/src/v5/mesh-source-schema.ts +134 -0
- package/src/v5/mesh-source.ts +16 -4
- package/src/v5/mesh-tileset-sink.ts +1 -1
- package/src/v5/node-cli.ts +197 -0
- package/src/v5/node-conversion.ts +345 -0
- package/src/v5/node-feature-schema.ts +155 -0
- package/src/v5/node-i3s-source.ts +181 -0
- package/src/v5/node-source.ts +328 -0
- package/src/v5/node.ts +16 -0
- package/src/v5/point-cloud-file.ts +306 -0
- package/src/v5/point-cloud-potree.ts +130 -0
- package/src/v5/point-cloud-tileset.ts +137 -0
- package/src/v5/resource-fetcher.ts +4 -1
- package/src/v5/resource-manifest.ts +40 -5
- package/src/v5/selected-mesh-source.ts +384 -0
- package/src/v5/spatial-conversion.ts +88 -17
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {mkdir, open, rm, link, mkdtemp} from 'node:fs/promises';
|
|
6
|
+
import {dirname, join, resolve} from 'node:path';
|
|
7
|
+
import {convertTileset, TileConversionError} from '@loaders.gl/tile-converter/v5/core';
|
|
8
|
+
import type {
|
|
9
|
+
BrowserTileConversionFile,
|
|
10
|
+
BrowserTilesetConversionInspection,
|
|
11
|
+
TileConversionReport
|
|
12
|
+
} from '@loaders.gl/tile-converter/v5/core';
|
|
13
|
+
import {
|
|
14
|
+
createSelectedMeshConversionSource,
|
|
15
|
+
measureMeshBytes,
|
|
16
|
+
SELECTED_MESH_LIMITS,
|
|
17
|
+
createMeshConversionCodec,
|
|
18
|
+
createI3SMeshConversionCodec,
|
|
19
|
+
createMeshTilesetSink,
|
|
20
|
+
createMeshHierarchyTilesetSink,
|
|
21
|
+
createI3SMeshSink,
|
|
22
|
+
encodeTileConversionArchiveInBatches
|
|
23
|
+
} from '@loaders.gl/tile-converter/v5/adapters';
|
|
24
|
+
import type {
|
|
25
|
+
I3SMeshSourceFeatureOptions,
|
|
26
|
+
MeshSourceFeatureOptions
|
|
27
|
+
} from '@loaders.gl/tile-converter/v5/adapters';
|
|
28
|
+
import {createNodeTilesetConversionSource} from './node-source.js';
|
|
29
|
+
import {createNodeI3SConversionSource} from './node-i3s-source.js';
|
|
30
|
+
import type {NodeTilesetConversionSourceOptions} from './node-source.js';
|
|
31
|
+
|
|
32
|
+
/** Explicit targets for the initial bounded Node mesh profile. */
|
|
33
|
+
export type NodeTileConversionOutputFormat = '3d-tiles' | '3tz' | 'slpk';
|
|
34
|
+
|
|
35
|
+
/** Selected static mesh conversion with application-owned Node output. */
|
|
36
|
+
export interface NodeTileConversionOptions
|
|
37
|
+
extends Omit<NodeTilesetConversionSourceOptions, 'format'> {
|
|
38
|
+
/** Explicit 3D Tiles, I3S layer or indexed archive source. */
|
|
39
|
+
readonly format?: '3d-tiles' | '3tz' | 'i3s' | 'slpk';
|
|
40
|
+
/** Explicit I3S attribute mapping, also supplied by the CLI feature-schema file. */
|
|
41
|
+
readonly i3sFeatures?: I3SMeshSourceFeatureOptions;
|
|
42
|
+
/** Required metric source error for I3S to 3D Tiles; no screen-size inference. */
|
|
43
|
+
readonly geometricError?: number;
|
|
44
|
+
/** Retain a complete explicit 3D Tiles tree, including inspected external documents in 3D Tiles/3TZ output. */
|
|
45
|
+
readonly preserveHierarchy?: boolean;
|
|
46
|
+
/** Authorizes IntegratedMesh to 3DObject output with a reported semantic warning. */
|
|
47
|
+
readonly integratedMeshPolicy?: 'as-3d-object';
|
|
48
|
+
/** One to 64 inspected content placement identifiers; no automatic all-content conversion. */
|
|
49
|
+
readonly resourceIds: readonly string[];
|
|
50
|
+
/** Explicit feature class/schema mapping, also supplied by the CLI feature-schema file. */
|
|
51
|
+
readonly features?: MeshSourceFeatureOptions;
|
|
52
|
+
/** Target mesh profile, independent of output filename. */
|
|
53
|
+
readonly outputFormat: NodeTileConversionOutputFormat;
|
|
54
|
+
/** New destination path; omitted for profile validation without writing. */
|
|
55
|
+
readonly output?: string;
|
|
56
|
+
/** Maximum retained files and, separately, archive bytes; at most 32 MiB. */
|
|
57
|
+
readonly maxOutputBytes?: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Runs the qualified browser mesh codecs with confined Node input and optional output. */
|
|
61
|
+
export async function convertNodeTileset(
|
|
62
|
+
options: NodeTileConversionOptions
|
|
63
|
+
): Promise<TileConversionReport> {
|
|
64
|
+
const signal = options.signal ?? new AbortController().signal;
|
|
65
|
+
const maxOutputBytes = options.maxOutputBytes ?? SELECTED_MESH_LIMITS.maxOutputBytes;
|
|
66
|
+
if (
|
|
67
|
+
!['3d-tiles', '3tz', 'slpk'].includes(options.outputFormat) ||
|
|
68
|
+
!Number.isSafeInteger(maxOutputBytes) ||
|
|
69
|
+
maxOutputBytes < 1 ||
|
|
70
|
+
maxOutputBytes > SELECTED_MESH_LIMITS.maxOutputBytes
|
|
71
|
+
)
|
|
72
|
+
throw new TileConversionError(
|
|
73
|
+
'INVALID_NODE_CONVERSION_OPTIONS',
|
|
74
|
+
'Select a supported output profile and a positive output budget at most 32 MiB'
|
|
75
|
+
);
|
|
76
|
+
const i3s = options.format === 'i3s' || options.format === 'slpk';
|
|
77
|
+
if (
|
|
78
|
+
(options.preserveHierarchy !== undefined && typeof options.preserveHierarchy !== 'boolean') ||
|
|
79
|
+
(options.preserveHierarchy && (i3s || options.outputFormat === 'slpk'))
|
|
80
|
+
)
|
|
81
|
+
throw new TileConversionError(
|
|
82
|
+
'MESH_HIERARCHY_UNSUPPORTED',
|
|
83
|
+
'Hierarchy preservation requires 3D Tiles/3TZ input and output'
|
|
84
|
+
);
|
|
85
|
+
if (
|
|
86
|
+
i3s &&
|
|
87
|
+
((options.outputFormat !== 'slpk' && options.geometricError === undefined) ||
|
|
88
|
+
(options.geometricError !== undefined &&
|
|
89
|
+
(typeof options.geometricError !== 'number' ||
|
|
90
|
+
!Number.isFinite(options.geometricError) ||
|
|
91
|
+
options.geometricError < 0)))
|
|
92
|
+
)
|
|
93
|
+
throw new TileConversionError(
|
|
94
|
+
'MESH_GEOMETRIC_ERROR_INVALID',
|
|
95
|
+
'I3S to 3D Tiles requires an explicit nonnegative geometric error in meters'
|
|
96
|
+
);
|
|
97
|
+
if (
|
|
98
|
+
i3s
|
|
99
|
+
? options.features !== undefined
|
|
100
|
+
: options.i3sFeatures !== undefined ||
|
|
101
|
+
options.integratedMeshPolicy !== undefined ||
|
|
102
|
+
options.geometricError !== undefined
|
|
103
|
+
)
|
|
104
|
+
throw new TileConversionError(
|
|
105
|
+
'INVALID_NODE_CONVERSION_OPTIONS',
|
|
106
|
+
'Feature and layer policies must match the selected input format'
|
|
107
|
+
);
|
|
108
|
+
const sourceOptions = {
|
|
109
|
+
...options,
|
|
110
|
+
signal,
|
|
111
|
+
inspectNestedTilesets: true,
|
|
112
|
+
maxTilesetDocuments: options.maxTilesetDocuments ?? SELECTED_MESH_LIMITS.maxTilesetDocuments
|
|
113
|
+
};
|
|
114
|
+
const rawSource = i3s
|
|
115
|
+
? await createNodeI3SConversionSource({
|
|
116
|
+
...sourceOptions,
|
|
117
|
+
format: options.format as 'i3s' | 'slpk'
|
|
118
|
+
})
|
|
119
|
+
: await createNodeTilesetConversionSource({
|
|
120
|
+
...sourceOptions,
|
|
121
|
+
format: options.format as '3d-tiles' | '3tz' | undefined
|
|
122
|
+
});
|
|
123
|
+
let files: readonly BrowserTileConversionFile[];
|
|
124
|
+
let report: TileConversionReport;
|
|
125
|
+
try {
|
|
126
|
+
const inspection = await rawSource.inspect(signal);
|
|
127
|
+
await import('@loaders.gl/polyfills');
|
|
128
|
+
const {getDracoLibraryOptions} = await import('@loaders.gl/draco/bundled');
|
|
129
|
+
const dracoLibraryOptions = getDracoLibraryOptions({
|
|
130
|
+
decoderProfile: 'full',
|
|
131
|
+
useLocalLibraries: true
|
|
132
|
+
});
|
|
133
|
+
const selection =
|
|
134
|
+
'select' in rawSource
|
|
135
|
+
? {
|
|
136
|
+
...(await rawSource.select(
|
|
137
|
+
inspection as Awaited<ReturnType<typeof rawSource.inspect>>,
|
|
138
|
+
options.resourceIds,
|
|
139
|
+
dracoLibraryOptions,
|
|
140
|
+
options.i3sFeatures
|
|
141
|
+
)),
|
|
142
|
+
geometricError: (options.geometricError ?? 0) + SELECTED_MESH_LIMITS.maxPositionError
|
|
143
|
+
}
|
|
144
|
+
: await createSelectedMeshConversionSource(
|
|
145
|
+
inspection as Awaited<ReturnType<typeof rawSource.inspect>>,
|
|
146
|
+
{
|
|
147
|
+
resourceIds: options.resourceIds,
|
|
148
|
+
preserveHierarchy: options.preserveHierarchy,
|
|
149
|
+
features: options.features,
|
|
150
|
+
signal,
|
|
151
|
+
source: rawSource,
|
|
152
|
+
fetcher: rawSource.fetcher,
|
|
153
|
+
dracoLibraryOptions,
|
|
154
|
+
maxInputBytes: options.maxInputBytes
|
|
155
|
+
}
|
|
156
|
+
);
|
|
157
|
+
const {source, spatialContext, geometricError} = selection;
|
|
158
|
+
const common = {
|
|
159
|
+
source,
|
|
160
|
+
signal,
|
|
161
|
+
measureInputBytes: measureMeshBytes,
|
|
162
|
+
maxInputBytes: options.maxInputBytes ?? SELECTED_MESH_LIMITS.maxInputBytes,
|
|
163
|
+
maxOutputBytes,
|
|
164
|
+
maxInputResources: SELECTED_MESH_LIMITS.maxMeshResources,
|
|
165
|
+
maxOutputResources: SELECTED_MESH_LIMITS.maxMeshResources
|
|
166
|
+
};
|
|
167
|
+
const codecOptions = {
|
|
168
|
+
spatialContext,
|
|
169
|
+
autoOrigin: i3s,
|
|
170
|
+
integratedMeshPolicy: options.integratedMeshPolicy,
|
|
171
|
+
dracoLibraryOptions,
|
|
172
|
+
maxPositionError: SELECTED_MESH_LIMITS.maxPositionError
|
|
173
|
+
};
|
|
174
|
+
if (options.outputFormat === 'slpk') {
|
|
175
|
+
const sink = createI3SMeshSink({
|
|
176
|
+
maxTotalBytes: maxOutputBytes,
|
|
177
|
+
maxMeshes: SELECTED_MESH_LIMITS.maxMeshResources,
|
|
178
|
+
maxResourceBytes: maxOutputBytes
|
|
179
|
+
});
|
|
180
|
+
report = await convertTileset({
|
|
181
|
+
...common,
|
|
182
|
+
sink,
|
|
183
|
+
codec: createI3SMeshConversionCodec({...codecOptions, maxResourceBytes: maxOutputBytes}),
|
|
184
|
+
measureOutputBytes: resource =>
|
|
185
|
+
Object.values(resource.files).reduce((total, bytes) => total + bytes.byteLength, 0)
|
|
186
|
+
});
|
|
187
|
+
files = sink.getFiles();
|
|
188
|
+
} else {
|
|
189
|
+
const sink = options.preserveHierarchy
|
|
190
|
+
? createMeshHierarchyTilesetSink({
|
|
191
|
+
inspection: inspection as BrowserTilesetConversionInspection,
|
|
192
|
+
resourceIds: options.resourceIds,
|
|
193
|
+
maxTotalBytes: maxOutputBytes,
|
|
194
|
+
maxPositionError: SELECTED_MESH_LIMITS.maxPositionError
|
|
195
|
+
})
|
|
196
|
+
: createMeshTilesetSink({
|
|
197
|
+
maxTotalBytes: maxOutputBytes,
|
|
198
|
+
maxMeshes: SELECTED_MESH_LIMITS.maxMeshResources,
|
|
199
|
+
geometricError
|
|
200
|
+
});
|
|
201
|
+
report = await convertTileset({
|
|
202
|
+
...common,
|
|
203
|
+
sink,
|
|
204
|
+
codec: createMeshConversionCodec(codecOptions),
|
|
205
|
+
measureOutputBytes: resource => resource.glb.byteLength
|
|
206
|
+
});
|
|
207
|
+
files = sink.getFiles();
|
|
208
|
+
}
|
|
209
|
+
} finally {
|
|
210
|
+
await rawSource.close();
|
|
211
|
+
}
|
|
212
|
+
signal.throwIfAborted();
|
|
213
|
+
// Validate packaging as well as content, including on the write-free validation path.
|
|
214
|
+
if (options.outputFormat !== '3d-tiles' && !options.output) {
|
|
215
|
+
for await (const chunk of encodeTileConversionArchiveInBatches(files, {
|
|
216
|
+
format: options.outputFormat,
|
|
217
|
+
maxArchiveBytes: maxOutputBytes,
|
|
218
|
+
signal
|
|
219
|
+
}))
|
|
220
|
+
void chunk;
|
|
221
|
+
}
|
|
222
|
+
if (options.output)
|
|
223
|
+
await writeNodeTileConversionOutput(
|
|
224
|
+
files,
|
|
225
|
+
options.output,
|
|
226
|
+
options.outputFormat,
|
|
227
|
+
maxOutputBytes,
|
|
228
|
+
signal
|
|
229
|
+
);
|
|
230
|
+
return report;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Saves fixed relative resource names or an archive; existing destinations are never overwritten. */
|
|
234
|
+
export async function writeNodeTileConversionOutput(
|
|
235
|
+
files: readonly BrowserTileConversionFile[],
|
|
236
|
+
output: string,
|
|
237
|
+
format: NodeTileConversionOutputFormat,
|
|
238
|
+
maxOutputBytes: number,
|
|
239
|
+
signal: AbortSignal
|
|
240
|
+
): Promise<void> {
|
|
241
|
+
signal.throwIfAborted();
|
|
242
|
+
if (
|
|
243
|
+
!['3d-tiles', '3tz', 'slpk'].includes(format) ||
|
|
244
|
+
!Number.isSafeInteger(maxOutputBytes) ||
|
|
245
|
+
maxOutputBytes < 1
|
|
246
|
+
)
|
|
247
|
+
throw new TileConversionError(
|
|
248
|
+
'INVALID_NODE_OUTPUT',
|
|
249
|
+
'Output requires a supported format and positive safe byte limit'
|
|
250
|
+
);
|
|
251
|
+
let bytes = 0;
|
|
252
|
+
const names = new Set<string>();
|
|
253
|
+
for (const file of files) {
|
|
254
|
+
const name = file.resourceId;
|
|
255
|
+
if (
|
|
256
|
+
!name ||
|
|
257
|
+
name.includes('\\') ||
|
|
258
|
+
name.includes('\0') ||
|
|
259
|
+
name.includes(':') ||
|
|
260
|
+
name.split('/').some(segment => !segment || segment === '.' || segment === '..') ||
|
|
261
|
+
names.has(name)
|
|
262
|
+
)
|
|
263
|
+
throw new TileConversionError(
|
|
264
|
+
'INVALID_NODE_OUTPUT',
|
|
265
|
+
'Output names must be unique confined relative resource paths'
|
|
266
|
+
);
|
|
267
|
+
names.add(name);
|
|
268
|
+
if (file.blob.size > maxOutputBytes - bytes)
|
|
269
|
+
throw new TileConversionError(
|
|
270
|
+
'OUTPUT_BYTES_LIMIT_EXCEEDED',
|
|
271
|
+
'Output files exceed the byte budget'
|
|
272
|
+
);
|
|
273
|
+
bytes += file.blob.size;
|
|
274
|
+
}
|
|
275
|
+
const destination = resolve(output);
|
|
276
|
+
if (format === '3d-tiles') {
|
|
277
|
+
// Exclusive directory creation makes every subsequently created child application-owned.
|
|
278
|
+
await mkdir(destination);
|
|
279
|
+
try {
|
|
280
|
+
for (const file of files) {
|
|
281
|
+
signal.throwIfAborted();
|
|
282
|
+
const path = join(destination, file.resourceId);
|
|
283
|
+
await mkdir(dirname(path), {recursive: true});
|
|
284
|
+
const handle = await open(path, 'wx');
|
|
285
|
+
try {
|
|
286
|
+
const reader = file.blob.stream().getReader();
|
|
287
|
+
try {
|
|
288
|
+
while (true) {
|
|
289
|
+
signal.throwIfAborted();
|
|
290
|
+
const {value, done} = await reader.read();
|
|
291
|
+
if (done) break;
|
|
292
|
+
await writeFileBytes(handle, value, signal);
|
|
293
|
+
}
|
|
294
|
+
} finally {
|
|
295
|
+
await reader.cancel();
|
|
296
|
+
reader.releaseLock();
|
|
297
|
+
}
|
|
298
|
+
} finally {
|
|
299
|
+
await handle.close();
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
signal.throwIfAborted();
|
|
303
|
+
} catch (error) {
|
|
304
|
+
await rm(destination, {recursive: true, force: true});
|
|
305
|
+
throw error;
|
|
306
|
+
}
|
|
307
|
+
} else {
|
|
308
|
+
const temporaryDirectory = await mkdtemp(join(dirname(destination), '.tile-converter-'));
|
|
309
|
+
try {
|
|
310
|
+
const path = join(temporaryDirectory, 'archive');
|
|
311
|
+
const handle = await open(path, 'wx');
|
|
312
|
+
try {
|
|
313
|
+
for await (const chunk of encodeTileConversionArchiveInBatches(files, {
|
|
314
|
+
format,
|
|
315
|
+
maxArchiveBytes: maxOutputBytes,
|
|
316
|
+
signal
|
|
317
|
+
}))
|
|
318
|
+
await writeFileBytes(handle, chunk, signal);
|
|
319
|
+
} finally {
|
|
320
|
+
await handle.close();
|
|
321
|
+
}
|
|
322
|
+
signal.throwIfAborted();
|
|
323
|
+
// Same-filesystem hard link publishes a complete archive and fails if the destination exists.
|
|
324
|
+
await link(path, destination);
|
|
325
|
+
} finally {
|
|
326
|
+
await rm(temporaryDirectory, {recursive: true, force: true});
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** Completes native typed-array writes, including partial writes and cooperative cancellation. */
|
|
332
|
+
async function writeFileBytes(
|
|
333
|
+
handle: Awaited<ReturnType<typeof open>>,
|
|
334
|
+
bytes: Uint8Array,
|
|
335
|
+
signal: AbortSignal
|
|
336
|
+
): Promise<void> {
|
|
337
|
+
let offset = 0;
|
|
338
|
+
while (offset < bytes.byteLength) {
|
|
339
|
+
signal.throwIfAborted();
|
|
340
|
+
const result = await handle.write(bytes, offset, bytes.byteLength - offset);
|
|
341
|
+
if (!result.bytesWritten)
|
|
342
|
+
throw new TileConversionError('NODE_OUTPUT_WRITE_FAILED', 'Output write made no progress');
|
|
343
|
+
offset += result.bytesWritten;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {constants} from 'node:fs';
|
|
6
|
+
import {open} from 'node:fs/promises';
|
|
7
|
+
import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
|
|
8
|
+
import type {NodeTileConversionOptions} from './node-conversion.js';
|
|
9
|
+
|
|
10
|
+
/** Maximum bytes retained for a local declarative feature mapping. */
|
|
11
|
+
const MAXIMUM_SCHEMA_BYTES = 64 * 1024;
|
|
12
|
+
/** Scalar storage types shared by the qualified mesh feature readers and writers. */
|
|
13
|
+
const FIELD_TYPES = [
|
|
14
|
+
'int8',
|
|
15
|
+
'uint8',
|
|
16
|
+
'int16',
|
|
17
|
+
'uint16',
|
|
18
|
+
'int32',
|
|
19
|
+
'uint32',
|
|
20
|
+
'int64',
|
|
21
|
+
'uint64',
|
|
22
|
+
'float32',
|
|
23
|
+
'float64',
|
|
24
|
+
'utf8'
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
/** Reads a bounded local JSON mapping; source/destination adapters validate data semantics. */
|
|
28
|
+
export async function readFeatureSchema(
|
|
29
|
+
path: string,
|
|
30
|
+
i3s: boolean,
|
|
31
|
+
signal?: AbortSignal
|
|
32
|
+
): Promise<Pick<NodeTileConversionOptions, 'features' | 'i3sFeatures'>> {
|
|
33
|
+
signal?.throwIfAborted();
|
|
34
|
+
// Avoid blocking on special files before the regular-file check.
|
|
35
|
+
const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK);
|
|
36
|
+
let mapping: unknown;
|
|
37
|
+
try {
|
|
38
|
+
if (!(await file.stat()).isFile()) rejectSchema('Feature schema must be a regular local file');
|
|
39
|
+
const bytes = new Uint8Array(MAXIMUM_SCHEMA_BYTES + 1);
|
|
40
|
+
let length = 0;
|
|
41
|
+
while (length < bytes.length) {
|
|
42
|
+
signal?.throwIfAborted();
|
|
43
|
+
const {bytesRead} = await file.read(bytes, length, bytes.length - length, null);
|
|
44
|
+
if (!bytesRead) break;
|
|
45
|
+
length += bytesRead;
|
|
46
|
+
}
|
|
47
|
+
signal?.throwIfAborted();
|
|
48
|
+
if (length > MAXIMUM_SCHEMA_BYTES) rejectSchema('Feature schema exceeds 64 KiB');
|
|
49
|
+
try {
|
|
50
|
+
mapping = JSON.parse(
|
|
51
|
+
new TextDecoder('utf-8', {fatal: true}).decode(bytes.subarray(0, length))
|
|
52
|
+
);
|
|
53
|
+
} catch {
|
|
54
|
+
rejectSchema('Feature schema must contain valid UTF-8 JSON');
|
|
55
|
+
}
|
|
56
|
+
} finally {
|
|
57
|
+
await file.close();
|
|
58
|
+
}
|
|
59
|
+
const options = requireRecord(mapping);
|
|
60
|
+
rejectUnknownKeys(options, [
|
|
61
|
+
'schema',
|
|
62
|
+
'metadataClass',
|
|
63
|
+
'sourceFeatureIdProperty',
|
|
64
|
+
'featureIdField',
|
|
65
|
+
'batchSize',
|
|
66
|
+
'integer64Encoding',
|
|
67
|
+
...(i3s ? ['objectIdProperty', 'maxAttributeBytes'] : [])
|
|
68
|
+
]);
|
|
69
|
+
requireName(options.metadataClass);
|
|
70
|
+
for (const key of ['sourceFeatureIdProperty', 'featureIdField']) {
|
|
71
|
+
if (options[key] !== undefined) requireName(options[key]);
|
|
72
|
+
}
|
|
73
|
+
if (
|
|
74
|
+
options.batchSize !== undefined &&
|
|
75
|
+
(!Number.isSafeInteger(options.batchSize) ||
|
|
76
|
+
Number(options.batchSize) < 1 ||
|
|
77
|
+
Number(options.batchSize) > 65536)
|
|
78
|
+
)
|
|
79
|
+
rejectSchema('batchSize must be an integer from 1 to 65536');
|
|
80
|
+
if (options.integer64Encoding !== undefined && options.integer64Encoding !== 'decimal-string')
|
|
81
|
+
rejectSchema('integer64Encoding must be decimal-string when supplied');
|
|
82
|
+
const schema = requireRecord(options.schema);
|
|
83
|
+
rejectUnknownKeys(schema, ['fields', 'metadata']);
|
|
84
|
+
validateMetadata(schema.metadata);
|
|
85
|
+
if (!Array.isArray(schema.fields) || !schema.fields.length || schema.fields.length > 256)
|
|
86
|
+
rejectSchema('schema.fields must declare between 1 and 256 fields');
|
|
87
|
+
const names = new Set<string>();
|
|
88
|
+
for (const value of schema.fields) {
|
|
89
|
+
const field = requireRecord(value);
|
|
90
|
+
rejectUnknownKeys(field, ['name', 'type', 'nullable', 'metadata']);
|
|
91
|
+
requireName(field.name);
|
|
92
|
+
if (names.has(field.name as string)) rejectSchema('Feature fields must have unique names');
|
|
93
|
+
names.add(field.name as string);
|
|
94
|
+
if (
|
|
95
|
+
typeof field.type !== 'string' ||
|
|
96
|
+
!FIELD_TYPES.includes(field.type) ||
|
|
97
|
+
(field.nullable !== undefined && typeof field.nullable !== 'boolean')
|
|
98
|
+
)
|
|
99
|
+
rejectSchema('Fields require a scalar numeric/utf8 type and an optional boolean nullable');
|
|
100
|
+
validateMetadata(field.metadata);
|
|
101
|
+
}
|
|
102
|
+
const identifier = schema.fields.find(
|
|
103
|
+
value => value.name === (options.featureIdField ?? 'feature_id')
|
|
104
|
+
);
|
|
105
|
+
if (
|
|
106
|
+
!identifier ||
|
|
107
|
+
identifier.nullable === true ||
|
|
108
|
+
!['utf8', ...FIELD_TYPES.filter(type => /^(u?int)/.test(type))].includes(identifier.type)
|
|
109
|
+
)
|
|
110
|
+
rejectSchema('Declare a nonnullable string or integer feature ID field');
|
|
111
|
+
if (i3s) {
|
|
112
|
+
requireName(options.objectIdProperty);
|
|
113
|
+
if (
|
|
114
|
+
!Number.isSafeInteger(options.maxAttributeBytes) ||
|
|
115
|
+
Number(options.maxAttributeBytes) < 1 ||
|
|
116
|
+
Number(options.maxAttributeBytes) > 16 * 1024 * 1024
|
|
117
|
+
)
|
|
118
|
+
rejectSchema('maxAttributeBytes must be a positive integer at most 16 MiB');
|
|
119
|
+
return {
|
|
120
|
+
i3sFeatures: options as unknown as NonNullable<NodeTileConversionOptions['i3sFeatures']>
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
return {features: options as unknown as NonNullable<NodeTileConversionOptions['features']>};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Rejects unknown configuration keys rather than silently ignoring misspelled controls. */
|
|
127
|
+
function rejectUnknownKeys(record: Record<string, unknown>, allowed: readonly string[]): void {
|
|
128
|
+
if (Object.keys(record).some(key => !allowed.includes(key)))
|
|
129
|
+
rejectSchema('Unknown feature-schema option');
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Requires a JSON object before accessing its fields. */
|
|
133
|
+
function requireRecord(value: unknown): Record<string, unknown> {
|
|
134
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
135
|
+
rejectSchema('Feature mappings require JSON objects');
|
|
136
|
+
return value as Record<string, unknown>;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Requires a nonempty class, property or column name. */
|
|
140
|
+
function requireName(value: unknown): void {
|
|
141
|
+
if (typeof value !== 'string' || !value.trim())
|
|
142
|
+
rejectSchema('Feature names must be nonempty strings');
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Retains only string-valued schema/field metadata. */
|
|
146
|
+
function validateMetadata(value: unknown): void {
|
|
147
|
+
if (value === undefined) return;
|
|
148
|
+
if (Object.values(requireRecord(value)).some(entry => typeof entry !== 'string'))
|
|
149
|
+
rejectSchema('Schema metadata values must be strings');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Reports configuration failures before source conversion or output publication. */
|
|
153
|
+
function rejectSchema(message: string): never {
|
|
154
|
+
throw new TileConversionError('CLI_FEATURE_SCHEMA_INVALID', message);
|
|
155
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import {realpath} from 'node:fs/promises';
|
|
6
|
+
import {fileURLToPath} from 'node:url';
|
|
7
|
+
import {resolve} from 'node:path';
|
|
8
|
+
import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
|
|
9
|
+
import {
|
|
10
|
+
createSelectedI3SMeshConversionSource,
|
|
11
|
+
inspectI3SMeshConversionSource,
|
|
12
|
+
createTileConversionArchiveTransport,
|
|
13
|
+
openTileConversionArchive
|
|
14
|
+
} from '@loaders.gl/tile-converter/v5/adapters';
|
|
15
|
+
import type {
|
|
16
|
+
I3SMeshSelectionOptions,
|
|
17
|
+
I3SMeshSourceFeatureOptions
|
|
18
|
+
} from '@loaders.gl/tile-converter/v5/adapters';
|
|
19
|
+
import {createTileConversionResourceFetcher} from './resource-fetcher.js';
|
|
20
|
+
import {NODE_TILESET_LIMITS, openLocalFile} from './node-source.js';
|
|
21
|
+
import type {NodeTilesetConversionSourceOptions} from './node-source.js';
|
|
22
|
+
|
|
23
|
+
/** Bounded I3S layer URL or indexed SLPK input with application-owned transport. */
|
|
24
|
+
export interface NodeI3SConversionSourceOptions
|
|
25
|
+
extends Omit<NodeTilesetConversionSourceOptions, 'format' | 'inspectNestedTilesets'> {
|
|
26
|
+
/** I3S HTTP(S) layer or local/HTTP(S) SLPK archive. */
|
|
27
|
+
readonly format: 'i3s' | 'slpk';
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Opens one lifetime-budgeted I3S source; its owner must close it in finally. */
|
|
31
|
+
export async function createNodeI3SConversionSource(options: NodeI3SConversionSourceOptions) {
|
|
32
|
+
const maxInputBytes = options.maxInputBytes ?? NODE_TILESET_LIMITS.maxInputBytes;
|
|
33
|
+
const maxInputResources = options.maxInputResources ?? NODE_TILESET_LIMITS.maxInputResources;
|
|
34
|
+
if (
|
|
35
|
+
!['i3s', 'slpk'].includes(options.format) ||
|
|
36
|
+
!Number.isSafeInteger(maxInputBytes) ||
|
|
37
|
+
maxInputBytes < 1 ||
|
|
38
|
+
maxInputBytes > NODE_TILESET_LIMITS.maxInputBytes ||
|
|
39
|
+
!Number.isSafeInteger(maxInputResources) ||
|
|
40
|
+
maxInputResources < 1 ||
|
|
41
|
+
maxInputResources > NODE_TILESET_LIMITS.maxInputResources
|
|
42
|
+
)
|
|
43
|
+
throw new TileConversionError(
|
|
44
|
+
'INVALID_NODE_SOURCE_OPTIONS',
|
|
45
|
+
'I3S sources require positive budgets within the bounded input profile'
|
|
46
|
+
);
|
|
47
|
+
const controller = new AbortController();
|
|
48
|
+
const signal = options.signal
|
|
49
|
+
? AbortSignal.any([options.signal, controller.signal])
|
|
50
|
+
: controller.signal;
|
|
51
|
+
signal.throwIfAborted();
|
|
52
|
+
let release: (() => Promise<void>) | undefined;
|
|
53
|
+
let closePromise: Promise<void> | undefined;
|
|
54
|
+
/** Cancels cached/runtime work and releases owned indexed readers exactly once. */
|
|
55
|
+
const close = async (): Promise<void> => {
|
|
56
|
+
controller.abort(new TileConversionError('NODE_SOURCE_CLOSED', 'Node source is closed'));
|
|
57
|
+
closePromise ??= release?.() ?? Promise.resolve();
|
|
58
|
+
await closePromise;
|
|
59
|
+
};
|
|
60
|
+
try {
|
|
61
|
+
const remote = /^https?:/i.test(options.input);
|
|
62
|
+
if (options.format === 'i3s' && !remote)
|
|
63
|
+
throw new TileConversionError(
|
|
64
|
+
'UNSUPPORTED_NODE_SOURCE',
|
|
65
|
+
'I3S input requires an HTTP(S) layer URL; use slpk for a local archive'
|
|
66
|
+
);
|
|
67
|
+
let rootUrl: string;
|
|
68
|
+
let transport: typeof fetch;
|
|
69
|
+
if (options.format === 'slpk') {
|
|
70
|
+
const archive = remote
|
|
71
|
+
? await openTileConversionArchive(
|
|
72
|
+
{input: options.input, format: 'slpk'},
|
|
73
|
+
maxInputBytes,
|
|
74
|
+
signal,
|
|
75
|
+
options.fetcher
|
|
76
|
+
)
|
|
77
|
+
: await createTileConversionArchiveTransport(
|
|
78
|
+
await openLocalFile(
|
|
79
|
+
await realpath(
|
|
80
|
+
/^file:/i.test(options.input)
|
|
81
|
+
? fileURLToPath(options.input)
|
|
82
|
+
: resolve(options.input)
|
|
83
|
+
),
|
|
84
|
+
maxInputBytes,
|
|
85
|
+
signal,
|
|
86
|
+
() => {}
|
|
87
|
+
),
|
|
88
|
+
'slpk',
|
|
89
|
+
maxInputBytes,
|
|
90
|
+
signal
|
|
91
|
+
);
|
|
92
|
+
release = archive.close;
|
|
93
|
+
rootUrl = archive.rootUrl;
|
|
94
|
+
transport = archive.fetcher;
|
|
95
|
+
} else {
|
|
96
|
+
rootUrl = new URL(options.input).href;
|
|
97
|
+
transport = options.fetcher ?? fetch;
|
|
98
|
+
}
|
|
99
|
+
const bounded = createTileConversionResourceFetcher({
|
|
100
|
+
maxInputBytes,
|
|
101
|
+
signal,
|
|
102
|
+
fetcher: transport
|
|
103
|
+
});
|
|
104
|
+
let metadataBytes = 0;
|
|
105
|
+
const root = new URL(rootUrl);
|
|
106
|
+
/** Inherits query credentials only within the selected layer's origin and path. */
|
|
107
|
+
const fetcher: typeof fetch = async (input, requestOptions) => {
|
|
108
|
+
const url = new URL(input instanceof Request ? input.url : String(input));
|
|
109
|
+
if (
|
|
110
|
+
url.origin === root.origin &&
|
|
111
|
+
(url.pathname === root.pathname || url.pathname.startsWith(`${root.pathname}/`))
|
|
112
|
+
)
|
|
113
|
+
for (const [key, value] of root.searchParams)
|
|
114
|
+
if (!url.searchParams.has(key)) url.searchParams.set(key, value);
|
|
115
|
+
const response = await bounded(
|
|
116
|
+
input instanceof Request ? new Request(url, input) : url,
|
|
117
|
+
requestOptions
|
|
118
|
+
);
|
|
119
|
+
if (response.url && response.url !== url.href)
|
|
120
|
+
throw new TileConversionError(
|
|
121
|
+
'I3S_REDIRECT_UNSUPPORTED',
|
|
122
|
+
'Use the final I3S resource URL; redirected resource bases are not qualified'
|
|
123
|
+
);
|
|
124
|
+
const bytes = await response.arrayBuffer();
|
|
125
|
+
metadataBytes += bytes.byteLength;
|
|
126
|
+
const result = new Response(bytes, {status: response.status, headers: response.headers});
|
|
127
|
+
Object.defineProperty(result, 'url', {value: response.url});
|
|
128
|
+
return result;
|
|
129
|
+
};
|
|
130
|
+
// Metadata inspection needs no decoder transport initialization. Conversion injects bundled assets.
|
|
131
|
+
const selectionOptions: I3SMeshSelectionOptions = {
|
|
132
|
+
rootUrl,
|
|
133
|
+
fetcher,
|
|
134
|
+
signal,
|
|
135
|
+
maxInputBytes,
|
|
136
|
+
maxInputResources,
|
|
137
|
+
dracoLibraryOptions: {decoderProfile: 'full' as const}
|
|
138
|
+
};
|
|
139
|
+
return {
|
|
140
|
+
close,
|
|
141
|
+
/** Reads only layer and node metadata, under the same lifetime-wide budget as conversion. */
|
|
142
|
+
async inspect(operationSignal?: AbortSignal) {
|
|
143
|
+
signal.throwIfAborted();
|
|
144
|
+
operationSignal?.throwIfAborted();
|
|
145
|
+
const activeSignal = operationSignal ? AbortSignal.any([signal, operationSignal]) : signal;
|
|
146
|
+
/** Forwards borrowed operation cancellation without closing the owning source. */
|
|
147
|
+
const operationFetcher: typeof fetch = (input, requestOptions) =>
|
|
148
|
+
fetcher(input, {
|
|
149
|
+
...requestOptions,
|
|
150
|
+
signal: requestOptions?.signal
|
|
151
|
+
? AbortSignal.any([activeSignal, requestOptions.signal])
|
|
152
|
+
: activeSignal
|
|
153
|
+
});
|
|
154
|
+
const inspection = await inspectI3SMeshConversionSource({
|
|
155
|
+
...selectionOptions,
|
|
156
|
+
fetcher: operationFetcher,
|
|
157
|
+
signal: activeSignal
|
|
158
|
+
});
|
|
159
|
+
return {...inspection, metadataBytes};
|
|
160
|
+
},
|
|
161
|
+
/** Creates an original-coordinate selected mesh source using application-provided Draco assets. */
|
|
162
|
+
select: (
|
|
163
|
+
inspection: Awaited<ReturnType<typeof inspectI3SMeshConversionSource>>,
|
|
164
|
+
resourceIds: readonly string[],
|
|
165
|
+
dracoLibraryOptions: I3SMeshSelectionOptions['dracoLibraryOptions'],
|
|
166
|
+
features?: I3SMeshSourceFeatureOptions
|
|
167
|
+
) => {
|
|
168
|
+
signal.throwIfAborted();
|
|
169
|
+
return createSelectedI3SMeshConversionSource(inspection, {
|
|
170
|
+
...selectionOptions,
|
|
171
|
+
resourceIds,
|
|
172
|
+
dracoLibraryOptions,
|
|
173
|
+
features
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
} catch (error) {
|
|
178
|
+
await close();
|
|
179
|
+
throw error;
|
|
180
|
+
}
|
|
181
|
+
}
|