3dtiles-inspector 0.2.6 → 0.2.8

package/src/server/splatCrop/index.js

@@ -6,6 +6,7 @@ const { assertPathInsideRoot } = require('./gltfResource');
 const { getRootUpRotationMatrix } = require('./gaussianPrimitives');
 const {
   collectCandidateSplatResources,
+  deleteOrphanedSplatResources,
   readTilesetJson,
   traverseTileset,
 } = require('./traversal');
@@ -35,6 +36,8 @@ async function deleteSplatsInNormalizedSelections(
   if (normalizedScreenSelections.length === 0) {
     return {
       deletedSplats: 0,
+      deletedSplatFiles: 0,
+      failedSplatFileDeletes: 0,
       processedSplatResources: 0,
     };
   }
@@ -45,11 +48,12 @@ async function deleteSplatsInNormalizedSelections(
 
   const { THREE } = await getSplatCropModules();
   const rootTileset = readTilesetJson(tilesetPath);
-  const totalResources = collectCandidateSplatResources({
+  const initialResourcePaths = collectCandidateSplatResources({
     rootDir,
     tileset: rootTileset,
     tilesetPath,
-  }).size;
+  });
+  const totalResources = initialResourcePaths.size;
   if (typeof onProgress === 'function') {
     const readStreamHint = tileReadStreamsClosed
       ? ' Tile read streams closed.'
@@ -77,7 +81,7 @@ async function deleteSplatsInNormalizedSelections(
   const workerPool = new SplatCropWorkerPool(SPLAT_CROP_WORKER_COUNT);
 
   try {
-    return await traverseTileset({
+    const traversalResult = await traverseTileset({
       THREE,
       tilesetPath,
       tileset: rootTileset,
@@ -102,6 +106,20 @@ async function deleteSplatsInNormalizedSelections(
       resourceLocks: new Map(),
       workerPool,
     });
+    const remainingResourcePaths = collectCandidateSplatResources({
+      rootDir,
+      tilesetPath,
+    });
+    const cleanupResult = await deleteOrphanedSplatResources({
+      initialResourcePaths,
+      remainingResourcePaths,
+      rootDir,
+    });
+    return {
+      ...traversalResult,
+      deletedSplatFiles: cleanupResult.deletedFiles.length,
+      failedSplatFileDeletes: cleanupResult.failedFiles.length,
+    };
   } finally {
     await workerPool.close();
   }

package/src/server/splatCrop/traversal.js

@@ -20,7 +20,73 @@ const {
   hasNonGaussianScenePrimitives,
   hasScenePrimitives,
   removeMeshPrimitives,
+  updateGaussianPrimitiveAccessorCounts,
 } = require('./gaussianPrimitives');
+const { SPLAT_CROP_WORKER_COUNT } = require('./workerPool');
+
+class AsyncLimiter {
+  constructor(limit) {
+    this.active = 0;
+    this.limit = normalizeConcurrency(limit);
+    this.queue = [];
+  }
+
+  run(task) {
+    return new Promise((resolve, reject) => {
+      this.queue.push({ reject, resolve, task });
+      this.dispatch();
+    });
+  }
+
+  dispatch() {
+    while (this.active < this.limit && this.queue.length > 0) {
+      const job = this.queue.shift();
+      this.active += 1;
+      Promise.resolve()
+        .then(job.task)
+        .then(job.resolve, job.reject)
+        .finally(() => {
+          this.active -= 1;
+          this.dispatch();
+        });
+    }
+  }
+}
+
+function normalizeConcurrency(value) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0
+    ? Math.max(1, Math.floor(number))
+    : 1;
+}
+
+function createAsyncLimiter(limit = SPLAT_CROP_WORKER_COUNT) {
+  return new AsyncLimiter(limit);
+}
+
+async function mapWithConcurrency(items, concurrency, mapper) {
+  if (!Array.isArray(items) || items.length === 0) {
+    return [];
+  }
+
+  const limit = normalizeConcurrency(concurrency);
+  const results = new Array(items.length);
+  let nextIndex = 0;
+
+  async function worker() {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      results[index] = await mapper(items[index], index);
+    }
+  }
+
+  const workerCount = Math.min(limit, items.length);
+  await Promise.all(
+    Array.from({ length: workerCount }, () => worker()),
+  );
+  return results;
+}
 
 function getContentSlots(tile) {
   const slots = [];
@@ -399,6 +465,126 @@ function collectCandidateSplatResources({
   return resourcePaths;
 }
 
+function collectLocalGltfBufferPaths(filePath, rootDir) {
+  const bufferPaths = new Set();
+  if (path.extname(filePath).toLowerCase() !== '.gltf') {
+    return bufferPaths;
+  }
+  if (!fs.existsSync(filePath)) {
+    return bufferPaths;
+  }
+
+  const json = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  const buffers = Array.isArray(json.buffers) ? json.buffers : [];
+  const resourceDir = path.dirname(filePath);
+  buffers.forEach((buffer, index) => {
+    const uri = buffer?.uri;
+    if (
+      typeof uri !== 'string' ||
+      uri.length === 0 ||
+      /^data:/i.test(uri) ||
+      isRemoteOrProtocolUri(uri)
+    ) {
+      return;
+    }
+
+    bufferPaths.add(
+      resolveLocalUri(
+        resourceDir,
+        rootDir,
+        uri,
+        `${filePath}.buffers[${index}].uri`,
+      ),
+    );
+  });
+  return bufferPaths;
+}
+
+function collectReferencedGltfBufferPaths(resourcePaths, rootDir) {
+  const bufferPaths = new Set();
+  resourcePaths.forEach((resourcePath) => {
+    collectLocalGltfBufferPaths(resourcePath, rootDir).forEach((bufferPath) => {
+      bufferPaths.add(bufferPath);
+    });
+  });
+  return bufferPaths;
+}
+
+async function deleteFileIfSafe(filePath, rootDir) {
+  const resolvedPath = assertPathInsideRoot(
+    filePath,
+    rootDir,
+    'Orphaned splat resource path',
+  );
+  let stats;
+  try {
+    stats = await fs.promises.stat(resolvedPath);
+  } catch (err) {
+    if (err && err.code === 'ENOENT') {
+      return { deleted: false };
+    }
+    return {
+      deleted: false,
+      error: err && err.message ? err.message : String(err),
+    };
+  }
+
+  if (!stats.isFile()) {
+    return { deleted: false };
+  }
+
+  try {
+    await fs.promises.unlink(resolvedPath);
+    return { deleted: true };
+  } catch (err) {
+    return {
+      deleted: false,
+      error: err && err.message ? err.message : String(err),
+    };
+  }
+}
+
+async function deleteOrphanedSplatResources({
+  initialResourcePaths,
+  remainingResourcePaths,
+  rootDir,
+}) {
+  const remainingBuffers = collectReferencedGltfBufferPaths(
+    remainingResourcePaths,
+    rootDir,
+  );
+  const deletePaths = new Set();
+
+  initialResourcePaths.forEach((resourcePath) => {
+    if (remainingResourcePaths.has(resourcePath)) {
+      return;
+    }
+
+    deletePaths.add(resourcePath);
+    collectLocalGltfBufferPaths(resourcePath, rootDir).forEach((bufferPath) => {
+      if (
+        !remainingResourcePaths.has(bufferPath) &&
+        !remainingBuffers.has(bufferPath)
+      ) {
+        deletePaths.add(bufferPath);
+      }
+    });
+  });
+
+  const deletedFiles = [];
+  const failedFiles = [];
+  for (const filePath of deletePaths) {
+    const result = await deleteFileIfSafe(filePath, rootDir);
+    if (result.deleted) {
+      deletedFiles.push(filePath);
+    } else if (result.error) {
+      failedFiles.push({ error: result.error, filePath });
+    }
+  }
+
+  return { deletedFiles, failedFiles };
+}
+
 function markSplatResourceProgress(context, resourcePath) {
   const progress = context.progress;
   if (
@@ -472,36 +658,17 @@ async function processGltfResource({
   const resourceBounds = createBounds();
   let hasResourceBounds = false;
   let deletedSplats = 0;
-  const rewriteTasks = [];
+  let modified = false;
 
   for (const [bufferViewIndex, viewDescriptors] of byBufferView) {
     const slice = getBufferViewSlice(resource, bufferViewIndex);
-    rewriteTasks.push({
-      bufferViewIndex,
-      slice,
-      viewDescriptors,
-      promise: rewriteSpzBytesInWorker({
-        bytes: slice.bytes,
-        descriptors: viewDescriptors,
-        screenSelections,
-        workerPool,
-      }),
+    const rewrite = await rewriteSpzBytesInWorker({
+      bytes: slice.bytes,
+      descriptors: viewDescriptors,
+      screenSelections,
+      workerPool,
     });
-  }
-
-  const rewriteResults = await Promise.all(
-    rewriteTasks.map(async (task) => ({
-      ...task,
-      rewrite: await task.promise,
-    })),
-  );
 
-  for (const {
-    bufferViewIndex,
-    rewrite,
-    slice,
-    viewDescriptors,
-  } of rewriteResults) {
     deletedSplats += rewrite.deleted;
     if (rewrite.bounds) {
       hasResourceBounds =
@@ -511,6 +678,14 @@ async function processGltfResource({
       emptyDescriptors.push(...viewDescriptors);
       continue;
     }
+    if (Number.isInteger(rewrite.survivorCount)) {
+      modified =
+        updateGaussianPrimitiveAccessorCounts(
+          resource,
+          viewDescriptors,
+          rewrite.survivorCount,
+        ) > 0 || modified;
+    }
     if (!rewrite.bytes) {
       continue;
     }
@@ -526,7 +701,6 @@ async function processGltfResource({
     });
   }
 
-  let modified = false;
   if (emptyDescriptors.length > 0) {
     modified = removeMeshPrimitives(resource, emptyDescriptors) > 0 || modified;
   }
@@ -637,11 +811,13 @@ async function processGltfContentSlot(
     context.resourceLocks,
     resourcePath,
     () =>
-      processLockedSplatResource(
-        context,
-        resourcePath,
-        tileSceneMatrix,
-        tileProjectionMatrix,
+      context.resourceLimiter.run(() =>
+        processLockedSplatResource(
+          context,
+          resourcePath,
+          tileSceneMatrix,
+          tileProjectionMatrix,
+        ),
       ),
   );
 
@@ -680,6 +856,8 @@ async function processNestedTilesetContentSlot(
     emptySplatResources: context.emptySplatResources,
     progress: context.progress,
     resourceLocks: context.resourceLocks,
+    resourceLimiter: context.resourceLimiter,
+    traversalConcurrency: context.traversalConcurrency,
     workerPool: context.workerPool,
   });
 
@@ -733,6 +911,14 @@ async function processContentSlot(context, slot, tileTransform, tileProjection)
   return createContentSlotResult(slot, { boundsKnown: false });
 }
 
+async function processContentSlots(context, tile, tileTransform, tileProjection) {
+  return mapWithConcurrency(
+    getContentSlots(tile),
+    context.traversalConcurrency,
+    (slot) => processContentSlot(context, slot, tileTransform, tileProjection),
+  );
+}
+
 function applyContentResultsToTile(context, tile, contentResults) {
   const contentBounds = [];
   let boundsKnown = true;
@@ -764,16 +950,18 @@ async function pruneEmptyChildren(context, tile, tileTransform, tileProjection)
     };
   }
 
-  const children = tile.children.slice();
-  const childResults = await Promise.all(
-    children.map(async (child) => ({
-      child,
-      result: await visitTilesetTile(context, child, tileTransform, false),
-    })),
-  );
   const childBounds = [];
   let boundsKnown = true;
   const keptChildren = [];
+  const childResults = await mapWithConcurrency(
+    tile.children,
+    context.traversalConcurrency,
+    async (child) => ({
+      child,
+      result: await visitTilesetTile(context, child, tileTransform, false),
+    }),
+  );
+
   childResults.forEach(({ child, result }) => {
     if (result.empty) {
       context.tilesetModified = true;
@@ -831,10 +1019,11 @@ async function visitTilesetTile(context, tile, inheritedTransform, isRootTile) {
     isRootTile,
   );
   const tileProjection = getTileProjection(context.THREE, tile);
-  const contentResults = await Promise.all(
-    getContentSlots(tile).map((slot) =>
-      processContentSlot(context, slot, worldTransform, tileProjection),
-    ),
+  const contentResults = await processContentSlots(
+    context,
+    tile,
+    worldTransform,
+    tileProjection,
   );
 
   const contentSummary = applyContentResultsToTile(context, tile, contentResults);
@@ -887,6 +1076,8 @@ async function traverseTileset({
   emptySplatResources,
   progress,
   resourceLocks,
+  resourceLimiter = null,
+  traversalConcurrency = SPLAT_CROP_WORKER_COUNT,
   workerPool,
 }) {
   const resolvedTilesetPath = assertPathInsideRoot(
@@ -910,6 +1101,8 @@ async function traverseTileset({
     throw new InspectorError(`${resolvedTilesetPath} must contain a root object.`);
   }
 
+  const normalizedTraversalConcurrency =
+    normalizeConcurrency(traversalConcurrency);
   const context = {
     THREE,
     deletedSplats: 0,
@@ -918,11 +1111,14 @@ async function traverseTileset({
     processedResources,
     processedSplatResources: 0,
     resourceLocks,
+    resourceLimiter:
+      resourceLimiter || createAsyncLimiter(normalizedTraversalConcurrency),
     rootDir,
     rootTransform,
     screenSelections,
     tilesetDir: path.dirname(resolvedTilesetPath),
     tilesetModified: false,
+    traversalConcurrency: normalizedTraversalConcurrency,
     upRotationMatrix,
     visitedTilesets,
     workerPool,
@@ -948,6 +1144,7 @@ async function traverseTileset({
 
 module.exports = {
   collectCandidateSplatResources,
+  deleteOrphanedSplatResources,
   readTilesetJson,
   traverseTileset,
 };

package/src/server/splatCrop/worker.js

@@ -344,10 +344,24 @@ async function rewriteSpzBytes({
     descriptors,
   );
   if (survivors.length === 0) {
-    return { bounds, bytes: null, deleted, empty: true };
+    return {
+      bounds,
+      bytes: null,
+      deleted,
+      empty: true,
+      splatCount: spz.numSplats,
+      survivorCount: 0,
+    };
   }
   if (deleted === 0) {
-    return { bounds, bytes: null, deleted: 0, empty: false };
+    return {
+      bounds,
+      bytes: null,
+      deleted: 0,
+      empty: false,
+      splatCount: spz.numSplats,
+      survivorCount: survivors.length,
+    };
   }
 
   return {
@@ -355,6 +369,8 @@ async function rewriteSpzBytes({
     bytes: await writeSurvivingSpzBytes(SpzWriter, spz, splatData, survivors),
     deleted,
     empty: false,
+    splatCount: spz.numSplats,
+    survivorCount: survivors.length,
   };
 }
 
@@ -380,6 +396,8 @@ parentPort.on('message', async (message) => {
           bytes,
           deleted: result.deleted,
           empty: result.empty,
+          splatCount: result.splatCount,
+          survivorCount: result.survivorCount,
         },
       },
       transferList,

package/src/server/splatCrop/workerPool.js

@@ -1,11 +1,37 @@
 const path = require('path');
+const os = require('os');
 const { Worker } = require('worker_threads');
 
 const { InspectorError } = require('../../errors');
 
-const SPLAT_CROP_WORKER_COUNT = 4;
+const MAX_SPLAT_CROP_WORKER_COUNT = 8;
 const SPLAT_CROP_WORKER_PATH = path.join(__dirname, 'worker.js');
 
+function getAvailableParallelism() {
+  if (typeof os.availableParallelism === 'function') {
+    try {
+      const parallelism = os.availableParallelism();
+      if (Number.isFinite(parallelism) && parallelism > 0) {
+        return parallelism;
+      }
+    } catch (err) {
+      // Fall back to os.cpus() below.
+    }
+  }
+
+  const cpus = os.cpus();
+  return Array.isArray(cpus) && cpus.length > 0 ? cpus.length : 1;
+}
+
+function getDefaultSplatCropWorkerCount() {
+  return Math.min(
+    MAX_SPLAT_CROP_WORKER_COUNT,
+    Math.max(1, getAvailableParallelism() - 1),
+  );
+}
+
+const SPLAT_CROP_WORKER_COUNT = getDefaultSplatCropWorkerCount();
+
 function deserializeWorkerError(error) {
   const message =
     error && typeof error.message === 'string'
@@ -74,9 +100,17 @@ class SplatCropWorkerPool {
       const result = message.result || {};
       job.resolve({
         bounds: result.bounds || null,
-        bytes: result.bytes ? Buffer.from(result.bytes) : null,
+        bytes: result.bytes
+          ? Buffer.from(
+              result.bytes.buffer,
+              result.bytes.byteOffset,
+              result.bytes.byteLength,
+            )
+          : null,
         deleted: Number(result.deleted || 0),
         empty: !!result.empty,
+        splatCount: Number(result.splatCount || 0),
+        survivorCount: Number(result.survivorCount || 0),
       });
     }
 
@@ -177,6 +211,7 @@ class SplatCropWorkerPool {
 }
 
 module.exports = {
+  getDefaultSplatCropWorkerCount,
   SPLAT_CROP_WORKER_COUNT,
   SplatCropWorkerPool,
 };

package/src/viewer/app.js

@@ -550,10 +550,11 @@ async function saveTransform() {
       const processedSplatResources = Number(
         payload.processedSplatResources || 0,
       );
+      const deletedSplatFiles = Number(payload.deletedSplatFiles || 0);
       cropController.clearAll();
       loadTileset(TILESET_URL, { frameOnLoad: false });
       setStatus(
-        `Saved transform and deleted ${deletedSplats} cropped splats from ${processedSplatResources} splat resource${processedSplatResources === 1 ? '' : 's'}. Reloading tileset.`,
+        `Saved transform and deleted ${deletedSplats} cropped splats from ${processedSplatResources} splat resource${processedSplatResources === 1 ? '' : 's'}${deletedSplatFiles > 0 ? `, removing ${deletedSplatFiles} orphaned file${deletedSplatFiles === 1 ? '' : 's'}` : ''}. Reloading tileset.`,
       );
     } else {
       setStatus(

package/src/viewer/scene/tiles.js

@@ -100,8 +100,8 @@ export function createTerrainGlobeTiles(options) {
 function createGeometricErrorLayerScalePlugin(preprocessNode) {
   return {
     name: 'GeometricErrorLayerScalePlugin',
-    preprocessNode(tile) {
-      preprocessNode(tile);
+    preprocessNode(tile, tilesetDir, parentTile) {
+      preprocessNode(tile, null, parentTile);
     },
   };
 }