@woosh/meep-engine 3.23.0 → 3.25.0

package/src/engine/graphics3/PickingSystem.js

@@ -1,273 +1,413 @@
-import { assert } from "../../core/assert.js";
-import { BVH } from "../../core/bvh2/bvh3/BVH.js";
-import {
-    aabb3_ray_segment_entry_distance
-} from "../../core/geom/3d/aabb/aabb3_ray_segment_entry_distance.js";
-import { BVHQueryIntersectsRay } from "../../core/bvh2/bvh3/query/BVHQueryIntersectsRay.js";
-import { bvh_query_leaves_generic } from "../../core/bvh2/bvh3/query/bvh_query_leaves_generic.js";
-import { System } from "../ecs/System.js";
-import { Transform64 } from "../ecs/transform/Transform64.js";
-import { Pickable } from "./Pickable.js";
-import { PickingResult } from "./PickingResult.js";
-
-/**
- * @type {Float32Array}
- */
-const SCRATCH_BOUNDS = new Float32Array(6);
-
-/**
- * Answers "what is under this ray" for entities that asked to be pickable.
- *
- * **Asynchronous and batched from the first line, on purpose (I-9).** The backend here is a CPU
- * bounding-volume hierarchy and could answer synchronously; the API does not let it, because the
- * backend this is meant to grow into reads entity ids back off the GPU's visibility buffer, and that
- * answer is inherently a frame late and arrives for a whole batch at once. A synchronous entry point
- * would have to be broken to get there, and every caller broken with it.
- *
- * **Bounding boxes, not triangles.** A hit means the ray entered the entity's world bounding box.
- * That is the precision this can offer for free — Shade maintains those boxes as part of drawing —
- * and going finer means keeping CPU-side triangles for every pickable entity, which is the
- * duplication {@link Pickable} exists to keep small. Terrain picking, which is where the game's
- * precision actually matters, does not come through here at all: it is an exact raycast against the
- * terrain's own representation and is untouched by any of this.
- *
- * The index is refreshed **on query rather than per frame**. Nothing pays for picking on a frame where
- * nobody picks, and there is no window in which the index disagrees with the scene, so no class of
- * bug where a pick lands on where something used to be.
- */
-export class PickingSystem extends System {
-    /**
-     * @type {MeshSystem}
-     */
-    #meshes;
-
-    /**
-     * @type {BVH}
-     */
-    #bvh = new BVH();
-
-    /**
-     * The BVH leaf standing for each pickable entity.
-     *
-     * @type {Map<number, number>}
-     */
-    #leaves = new Map();
-
-    /**
-     * Entities that have opted in but have nothing in the scene yet — a model still loading, most
-     * often. They hold no leaf, and every query looks again, which is what lets one appear without
-     * anything having to notice the moment it did.
-     *
-     * @type {Set<number>}
-     */
-    #pending = new Set();
-
-    /**
-     * @param {MeshSystem} meshes what an entity looks like, which is where its bounds come from
-     */
-    constructor(meshes) {
-        super();
-
-        assert.defined(meshes, 'meshes');
-
-        this.#meshes = meshes;
-
-        this.dependencies = [Pickable, Transform64];
-    }
-
-    async startup(entityManager) {
-        this.entityManager = entityManager;
-    }
-
-    async shutdown(entityManager) {
-        for (const entity of Array.from(this.#leaves.keys())) {
-            this.#drop(entity);
-        }
-
-        this.#pending.clear();
-    }
-
-    /**
-     * @param {Pickable} pickable
-     * @param {Transform64} transform
-     * @param {number} entity
-     */
-    link(pickable, transform, entity) {
-        // no bounds are taken here: the model may not have arrived, and the first query is early
-        // enough to find out
-        this.#pending.add(entity);
-    }
-
-    /**
-     * @param {Pickable} pickable
-     * @param {Transform64} transform
-     * @param {number} entity
-     */
-    unlink(pickable, transform, entity) {
-        this.#drop(entity);
-    }
-
-    /**
-     * How many entities the index currently stands for.
-     *
-     * The index is built on query, so this is zero until the first {@link pick} — and stays behind by
-     * however many pickable entities are still waiting on their models.
-     *
-     * @returns {number}
-     */
-    get indexed_count() {
-        return this.#leaves.size;
-    }
-
-    /**
-     * What each of these rays hits first.
-     *
-     * One result per query, in the order the queries were given. A ray that hits nothing still gets a
-     * result — `entity` is {@link PickingResult.NOTHING} — so the two arrays line up by position and a
-     * caller never has to search for the answer to its own question.
-     *
-     * @param {PickingQuery[]} queries
-     * @returns {Promise<PickingResult[]>}
-     */
-    async pick(queries) {
-        assert.isArray(queries, 'queries');
-
-        this.#refresh();
-
-        const results = [];
-
-        for (let i = 0; i < queries.length; i++) {
-            results.push(this.#pick_one(queries[i]));
-        }
-
-        return results;
-    }
-
-    /**
-     * @param {PickingQuery} query
-     * @returns {PickingResult}
-     */
-    #pick_one(query) {
-        const result = new PickingResult();
-
-        const bvh = this.#bvh;
-
-        const origin = query.origin;
-        const direction = query.direction;
-
-        const origin_x = origin.x;
-        const origin_y = origin.y;
-        const origin_z = origin.z;
-
-        const hits = [];
-
-        const hit_count = bvh_query_leaves_generic(
-            hits, 0,
-            bvh, bvh.root,
-            BVHQueryIntersectsRay.from([
-                origin_x, origin_y, origin_z,
-                direction.x, direction.y, direction.z
-            ])
-        );
-
-        // reciprocals once for the whole batch of candidates; the entry-distance test wants them, and
-        // a division component that is zero deliberately produces an infinity it handles
-        const inv_x = 1 / direction.x;
-        const inv_y = 1 / direction.y;
-        const inv_z = 1 / direction.z;
-
-        let best_distance = query.max_distance;
-        let best_entity = PickingResult.NOTHING;
-
-        for (let i = 0; i < hit_count; i++) {
-            const leaf = hits[i];
-
-            bvh.node_get_aabb(leaf, SCRATCH_BOUNDS);
-
-            const distance = aabb3_ray_segment_entry_distance(
-                SCRATCH_BOUNDS[0], SCRATCH_BOUNDS[1], SCRATCH_BOUNDS[2],
-                SCRATCH_BOUNDS[3], SCRATCH_BOUNDS[4], SCRATCH_BOUNDS[5],
-                origin_x, origin_y, origin_z,
-                inv_x, inv_y, inv_z,
-                0, query.max_distance
-            );
-
-            if (distance >= best_distance) {
-                // further than something already found, or a miss the broadphase let through
-                continue;
-            }
-
-            best_distance = distance;
-            best_entity = bvh.node_get_user_data(leaf);
-        }
-
-        if (best_entity === PickingResult.NOTHING) {
-            return result;
-        }
-
-        result.entity = best_entity;
-        result.distance = best_distance;
-
-        result.position.set(
-            origin_x + direction.x * best_distance,
-            origin_y + direction.y * best_distance,
-            origin_z + direction.z * best_distance
-        );
-
-        return result;
-    }
-
-    /**
-     * Bring every leaf in line with where its entity now is, and give a leaf to any entity whose
-     * model has arrived since the last look.
-     */
-    #refresh() {
-        const meshes = this.#meshes;
-
-        for (const [entity, leaf] of this.#leaves) {
-            if (meshes.compute_world_bounds(SCRATCH_BOUNDS, entity)) {
-                this.#bvh.node_move_aabb(leaf, SCRATCH_BOUNDS);
-
-                continue;
-            }
-
-            // the model went away under it — the entity is still pickable, so it goes back to
-            // waiting rather than losing its opt-in
-            this.#drop(entity);
-            this.#pending.add(entity);
-        }
-
-        for (const entity of Array.from(this.#pending)) {
-            if (!meshes.compute_world_bounds(SCRATCH_BOUNDS, entity)) {
-                continue;
-            }
-
-            const leaf = this.#bvh.allocate_node();
-
-            this.#bvh.node_set_aabb(leaf, SCRATCH_BOUNDS);
-            this.#bvh.node_set_user_data(leaf, entity);
-            this.#bvh.insert_leaf(leaf);
-
-            this.#leaves.set(entity, leaf);
-            this.#pending.delete(entity);
-        }
-    }
-
-    /**
-     * @param {number} entity
-     */
-    #drop(entity) {
-        this.#pending.delete(entity);
-
-        const leaf = this.#leaves.get(entity);
-
-        if (leaf === undefined) {
-            return;
-        }
-
-        this.#bvh.remove_leaf(leaf);
-        this.#bvh.release_node(leaf);
-
-        this.#leaves.delete(entity);
-    }
-}
+import { assert } from "../../core/assert.js";
+import { m4_invert } from "../../core/geom/3d/mat4/m4_invert.js";
+import { BufferResourceDescriptor } from "../../shade/device/graph/BufferResourceDescriptor.js";
+import { graph_read_buffer } from "../../shade/renderer/buffer/graph_read_buffer.js";
+import { FramePhase } from "../../shade/renderer/extension/FramePhase.js";
+import { RenderExtension } from "../../shade/renderer/extension/RenderExtension.js";
+import { ViewTextures } from "../../shade/renderer/extension/ViewTextures.js";
+import { VIZ_BUFFER_INVALID_MESH_VALUE } from "../../shade/renderer/rasterize/native/viz/VIZ_BUFFER_INVALID_MESH_VALUE.js";
+import { entity_of_row } from "../../shade/renderer/scene/rows/GPUSceneRows.js";
+import { ParentEntity } from "../ecs/parent/ParentEntity.js";
+import { System } from "../ecs/System.js";
+import { Pickable } from "./Pickable.js";
+import { PickingResult } from "./PickingResult.js";
+import {
+    PICKING_GATHER_STRIDE_WORDS,
+    PICKING_QUERY_OFF_VIEW,
+    shader_picking_gather
+} from "./shader_picking_gather.js";
+
+/**
+ * Bytes a query occupies in the gather buffer: the mesh row and the depth, one word each.
+ *
+ * @type {number}
+ */
+const QUERY_STRIDE = PICKING_GATHER_STRIDE_WORDS * 4;
+
+/**
+ * Answers "what is under this pixel" for entities that asked to be pickable.
+ *
+ * **Off the visibility buffer.** The rasterizer writes every pixel's `meshes` row — an entity's
+ * row, one on — and its depth; a pick reads those two texels out of the frame that drew them and
+ * resolves the row to the entity that drew it, then up its parents to the one that opted in. What
+ * was drawn is what is picked: at triangle precision, under skinning, with nothing kept on the CPU
+ * for it — no bounds, no tree, no per-frame refresh.
+ *
+ * **Asynchronous and batched, of necessity.** The texels are gathered by the frame after the query
+ * was asked and read back when the device has finished it, so an answer is a frame or two late and
+ * arrives for the whole batch at once. A result is a promise for that reason, and nothing this
+ * tick can come to depend on it.
+ *
+ * Terrain picking, which is where the game's precision matters, does not come through here at all:
+ * it is an exact raycast against the terrain's own representation and is untouched by any of this.
+ */
+export class PickingSystem extends System {
+    /**
+     * @type {GraphicsEngine}
+     */
+    #graphics;
+
+    /**
+     * @type {PickingExtension|null}
+     */
+    #extension = null;
+
+    /**
+     * Entities that have opted in.
+     *
+     * @type {Set<number>}
+     */
+    #pickable = new Set();
+
+    /**
+     * Queries asked since the last frame took a batch, with the promise each batch answers.
+     *
+     * @type {{queries: PickingQuery[], resolve: function(PickingResult[]): void, reject: function(Error): void}[]}
+     */
+    #waiting = [];
+
+    /**
+     * @param {GraphicsEngine} graphics whose frames draw the view the picks read
+     */
+    constructor(graphics) {
+        super();
+
+        assert.defined(graphics, 'graphics');
+
+        this.#graphics = graphics;
+
+        this.dependencies = [Pickable];
+    }
+
+    async startup(entityManager) {
+        this.entityManager = entityManager;
+
+        this.#extension = this.#graphics.add_extension(new PickingExtension(this));
+    }
+
+    async shutdown(entityManager) {
+        if (this.#extension !== null) {
+            this.#graphics.remove_extension(this.#extension);
+            this.#extension = null;
+        }
+
+        this.#pickable.clear();
+
+        // nothing will draw a frame for them
+        for (const batch of this.#waiting.splice(0)) {
+            batch.reject(new Error('the picking system shut down before a frame answered'));
+        }
+    }
+
+    /**
+     * @param {Pickable} pickable
+     * @param {number} entity
+     */
+    link(pickable, entity) {
+        this.#pickable.add(entity);
+    }
+
+    /**
+     * @param {Pickable} pickable
+     * @param {number} entity
+     */
+    unlink(pickable, entity) {
+        this.#pickable.delete(entity);
+    }
+
+    /**
+     * How many entities have opted in.
+     *
+     * @returns {number}
+     */
+    get pickable_count() {
+        return this.#pickable.size;
+    }
+
+    /**
+     * Batches asked and not yet answered by a frame.
+     *
+     * @returns {number}
+     */
+    get waiting_count() {
+        return this.#waiting.length;
+    }
+
+    /**
+     * What is drawn under each of these pixels.
+     *
+     * One result per query, in the order the queries were given. A pixel over nothing pickable
+     * still gets a result — `entity` is {@link PickingResult.NOTHING} — so the two arrays line up
+     * by position and a caller never has to search for the answer to its own question.
+     *
+     * Answered by the next frame that draws the view, when the device has finished it.
+     *
+     * @param {PickingQuery[]} queries
+     * @returns {Promise<PickingResult[]>}
+     */
+    pick(queries) {
+        assert.isArray(queries, 'queries');
+
+        if (queries.length === 0) {
+            return Promise.resolve([]);
+        }
+
+        return new Promise((resolve, reject) => {
+            this.#waiting.push({ queries: queries.slice(), resolve, reject });
+        });
+    }
+
+    /**
+     * Take everything asked since the last frame, to be answered by this one.
+     *
+     * @returns {{queries: PickingQuery[], resolve: function(PickingResult[]): void, reject: function(Error): void}[]}
+     */
+    take_waiting() {
+        return this.#waiting.splice(0);
+    }
+
+    /**
+     * The entity a drawn mesh row picks: the one that drew it, or the nearest of its ancestors,
+     * that opted in.
+     *
+     * @param {number} row a `meshes` row, as the visibility buffer holds it
+     * @returns {number} the entity, or {@link PickingResult.NOTHING}
+     */
+    resolve_row(row) {
+        if (row >= VIZ_BUFFER_INVALID_MESH_VALUE) {
+            return PickingResult.NOTHING;
+        }
+
+        const dataset = this.entityManager.dataset;
+        const pickable = this.#pickable;
+
+        let entity = entity_of_row(row);
+
+        while (entity >= 0 && dataset.entityExists(entity)) {
+            if (pickable.has(entity)) {
+                return entity;
+            }
+
+            const parent = dataset.getComponent(entity, ParentEntity);
+
+            entity = parent === undefined || parent === null ? -1 : parent.entity;
+        }
+
+        return PickingResult.NOTHING;
+    }
+}
+
+/**
+ * Gathers the queried texels out of the frame that drew them, after the G-buffer is written and
+ * before anything overwrites the visibility buffer, and answers the batch when they land.
+ */
+export class PickingExtension extends RenderExtension {
+    name = "picking";
+    phase = FramePhase.AfterGBuffer;
+
+    /**
+     * @type {PickingSystem}
+     */
+    #system;
+
+    /**
+     * @param {PickingSystem} system
+     */
+    constructor(system) {
+        super();
+
+        this.#system = system;
+    }
+
+    /**
+     * @param {import("../../shade/renderer/extension/FrameContext.js").FrameContext} frame
+     */
+    record(frame) {
+        const batches = this.#system.take_waiting();
+
+        if (batches.length === 0) {
+            return;
+        }
+
+        // the view's own resolution: what the visibility buffer is sized to, whatever the frame's
+        // target at this phase is
+        const resolution = frame.view.resolution;
+        const width = resolution[0];
+        const height = resolution[1];
+
+        // what the frame projected with, kept for when the answer comes: the frame's camera has
+        // moved on by then
+        const view_projection = new Float32Array(frame.view.camera.view_projection_matrix);
+        const camera = frame.view.camera.camera.transform;
+        const eye = [camera.translation_x, camera.translation_y, camera.translation_z];
+
+        /** @type {PickingQuery[]} every query of every batch, in order */
+        const queries = [];
+
+        for (const batch of batches) {
+            for (const query of batch.queries) {
+                queries.push(query);
+            }
+        }
+
+        /*
+         The pixel each query names, resolved here rather than in the shader: the fraction is the
+         caller's unit and the resolution is the frame's, and neither is the gather's business. A
+         query that is not on the view at all carries `PICKING_QUERY_OFF_VIEW` instead of a pixel.
+        */
+        const pixels = new Uint32Array(queries.length * PICKING_GATHER_STRIDE_WORDS);
+
+        for (let i = 0; i < queries.length; i++) {
+            const query = queries[i];
+            const base = i * PICKING_GATHER_STRIDE_WORDS;
+
+            if (on_view(query)) {
+                pixels[base] = texel(query.u, width);
+                pixels[base + 1] = texel(query.v, height);
+            } else {
+                pixels[base] = PICKING_QUERY_OFF_VIEW;
+            }
+        }
+
+        const textures = frame.get(ViewTextures);
+        const graph = frame.graph;
+
+        const data = {};
+        const pass = graph.add("Picking / gather texels", data, (data, resources, context) => {
+            const encoder = context.encoder;
+
+            shader_picking_gather.dispatchThreads({
+                encoder,
+                bindings: {
+                    query_count: queries.length,
+                    viz_mesh: resources.get(data.mesh),
+                    viz_depth: resources.get(data.depth),
+                    queries: encoder.allocateTransientBufferAndLoad(
+                        pixels.buffer, GPUBufferUsage.STORAGE
+                    ),
+                    results: resources.get(data.results),
+                },
+                thread_count_x: queries.length,
+            });
+        });
+
+        data.mesh = pass.read(textures.visibility_mesh);
+        data.depth = pass.read(textures.depth);
+        data.results = pass.create("picking/results", BufferResourceDescriptor.from(
+            queries.length * QUERY_STRIDE,
+            GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC
+        ));
+
+        const system = this.#system;
+
+        graph_read_buffer({ graph, buffer: data.results })
+            .then(ref => {
+                try {
+                    const words = new Uint32Array(ref.getValue());
+                    const floats = new Float32Array(words.buffer);
+
+                    let index = 0;
+
+                    for (const batch of batches) {
+                        const results = [];
+
+                        for (let i = 0; i < batch.queries.length; i++, index++) {
+                            const query = batch.queries[i];
+                            const result = new PickingResult();
+
+                            results.push(result);
+
+                            if (!on_view(query)) {
+                                // the gather writes an invalid row here too; this is the same
+                                // answer without depending on a dispatch having run
+                                continue;
+                            }
+
+                            const base = index * PICKING_GATHER_STRIDE_WORDS;
+                            const entity = system.resolve_row(words[base]);
+
+                            if (entity === PickingResult.NOTHING) {
+                                continue;
+                            }
+
+                            result.entity = entity;
+
+                            unproject(
+                                result.position, view_projection,
+                                texel(query.u, width), texel(query.v, height), width, height,
+                                floats[base + 1]
+                            );
+
+                            result.distance = Math.hypot(
+                                result.position.x - eye[0],
+                                result.position.y - eye[1],
+                                result.position.z - eye[2]
+                            );
+                        }
+
+                        batch.resolve(results);
+                    }
+                } finally {
+                    ref.release();
+                }
+            })
+            .catch(error => {
+                for (const batch of batches) {
+                    batch.reject(error instanceof Error ? error : new Error('the picking readback failed'));
+                }
+            });
+    }
+}
+
+/**
+ * @param {PickingQuery} query
+ * @returns {boolean} whether the point is on the view at all
+ */
+function on_view(query) {
+    return query.u >= 0 && query.u < 1 && query.v >= 0 && query.v < 1;
+}
+
+/**
+ * The texel a fraction of the view falls in.
+ *
+ * @param {number} fraction in [0, 1)
+ * @param {number} extent texels
+ * @returns {number}
+ */
+function texel(fraction, extent) {
+    return Math.min(extent - 1, Math.floor(fraction * extent));
+}
+
+const SCRATCH_INVERSE = new Float32Array(16);
+
+/**
+ * The world point a pixel's depth stands for, through the inverse of what the frame projected
+ * with. Depth is what the buffer holds — reverse-Z, `1` at the near plane — and WebGPU's clip z
+ * is that same `[0, 1]`, so it goes straight in.
+ *
+ * @param {Vector3} target
+ * @param {Float32Array} view_projection
+ * @param {number} x pixel
+ * @param {number} y pixel
+ * @param {number} width
+ * @param {number} height
+ * @param {number} depth
+ */
+function unproject(target, view_projection, x, y, width, height, depth) {
+    const m = SCRATCH_INVERSE;
+
+    m4_invert(m, view_projection);
+
+    // the pixel's centre, in NDC: x to the right, y up
+    const nx = ((x + 0.5) / width) * 2 - 1;
+    const ny = 1 - ((y + 0.5) / height) * 2;
+    const nz = depth;
+
+    // column-major: column c is m[c * 4 .. c * 4 + 3]
+    const wx = m[0] * nx + m[4] * ny + m[8] * nz + m[12];
+    const wy = m[1] * nx + m[5] * ny + m[9] * nz + m[13];
+    const wz = m[2] * nx + m[6] * ny + m[10] * nz + m[14];
+    const ww = m[3] * nx + m[7] * ny + m[11] * nz + m[15];
+
+    target.set(wx / ww, wy / ww, wz / ww);
+}