@woosh/meep-engine 3.58.0 → 3.60.0

package/src/shade/renderer/shader/chunk/bvh/chunk_bvh_query_aabb_cooperative.js

@@ -1,294 +1,262 @@
-import { WebGPUExtensionType } from "../../../../descriptor/WebGPUExtensionType.js";
-import { CodeChunk } from "../../compiler/CodeChunk.js";
-import { chunk_sort_bitonic_comparator } from "../../../gpu_primitive/sort/chunk_sort_bitonic_comparator.js";
-import { BVH2_NODE_STRUCT } from "./BVH2_NODE_STRUCT.js";
-import { BVH_NULL_NODE_DEFINITION } from "./BVH_NULL_NODE_DEFINITION.js";
-import { chunk_bvh_query_aabb_frontier } from "./chunk_bvh_query_aabb_frontier.js";
-
-/**
- * Every leaf of a BVH whose box overlaps one query box, found by a whole workgroup together and
- * returned as payloads in ascending order.
- *
- * ## Why not a stack per thread
- *
- * The engine's traversals (`chunk_sdf_query_distance`, `chunk_ray_query_*`) walk a stack per
- * thread with no cap on visits, which is right for one ray per thread and wrong for one box per
- * workgroup: the box meets a subtree, not a path, and one thread walking it serially is the slowest
- * thread of the dispatch. This expands the tree a **level at a time** instead, as the vgeo cut does
- * its descent: every node on the current level is tested by its own invocation, the survivors'
- * children are appended to the next level, overlapping leaves to the candidate list, and a barrier
- * separates the levels. Per-invocation work is bounded by the depth of the tree, not by the size of
- * the answer.
- *
- * ## The contract
- *
- * Call `bvh_query_aabb_cooperative(root, query, lane)` from every invocation of a workgroup of
- * exactly `BVH_QUERY_AABB_WORKGROUP_SIZE` invocations, in uniform control flow, with `lane` the
- * `local_invocation_index` and `root` and `query` the same in every invocation. `root` is the
- * tree's root word (word 0 of `BVH_BUFFER_LAYOUT`), and `BVH_NULL_NODE` for an empty tree. `query`
- * is `[min.xyz, max.xyz]`; touching counts as overlapping, as in `aabb3_intersects_aabb3`.
- *
- * It returns a `BVHQueryAABBResult` whose fields are workgroup-uniform, so they can bound the
- * caller's loops. The payloads are `bvh_query_aabb_candidates[0, candidate_count)`, ascending, and
- * visible to every invocation when the call returns. They stay there until the next query.
- *
- * The node layout is `BVH_BUFFER_LAYOUT`'s: a node is a leaf exactly when `child_1` is
- * `BVH_NULL_NODE`, and then `child_2` is its payload.
- *
- * ## The seam: two functions the enclosing shader defines
- *
- * The chunk binds nothing. It reads the tree and filters leaves through two functions the shader
- * that includes it must define, in the manner of the particle VM's `vm_reg_read` and
- * `vm_reg_write`:
- *
- * - `fn bvh_query_aabb_node(index: u32) -> BVH2Node` returns node `index`, where `BVH2Node` is
- *   `BVH2_NODE_STRUCT.wgsl_ref`. A tree bound as `BVH_STRUCT` answers with `tree.nodes[index]`; one
- *   inside a word arena reads its eight words.
- * - `fn bvh_query_aabb_accept_leaf(payload: u32, bounds: array<f32, 6>) -> bool` is asked about
- *   every leaf that overlaps the box, before it takes a place on the list. A caller with nothing to
- *   filter returns `true`. The fluid's resolution cull lives here: a leaf too small for the cascade
- *   is refused before it can crowd out one that is not.
- *
- * ## Deterministic order
- *
- * Appends from different subgroups land in whatever order the hardware runs them, so the list is
- * sorted by payload before it is returned: the bitonic network of `chunk_sort_bitonic_comparator`
- * over the smallest power-of-two prefix that holds the candidates (`sort_bitonic_size`), padded
- * with `0xffffffff`, one compare-exchange per invocation per stage. A payload that is a stable row index then makes "the lowest K" the same K every frame for
- * the same overlaps, however the traversal was scheduled.
- *
- * ## Bounds, and what is counted
- *
- * Nothing is dropped silently. A level whose appends pass `BVH_QUERY_AABB_FRONTIER_CAPACITY`, and a
- * list still holding nodes when `BVH_QUERY_AABB_LEVEL_BUDGET` levels have been expanded, add those
- * nodes to `nodes_dropped`: subtrees never tested. A leaf that passes the filter when the candidate
- * list is full adds to `candidates_dropped`. With both zero the candidates are every overlapping
- * leaf the filter accepted; with either non-zero they are a subset, and the lowest K of them need
- * not be the lowest K overall.
- *
- * Per invocation, at most `BVH_QUERY_AABB_LEVEL_BUDGET * BVH_QUERY_AABB_FRONTIER_ROUNDS` = 51 * 4 =
- * 204 node reads and filter calls, then two padding writes and 28 compare-exchanges. A query that
- * meets nothing costs one level: the root is read, misses, and the loop leaves.
- *
- * Workgroup memory: two lists of 256 node indices, 128 payloads and five words, 2580 bytes.
- *
- * ## Subgroups
- *
- * Appends are aggregated per subgroup: an exclusive add numbers the subgroup's appends, one elected
- * invocation reserves them all with one workgroup `atomicAdd`, and a broadcast hands each
- * invocation its base. Nothing assumes which invocations share a subgroup or that a subgroup's
- * invocations are contiguous in `local_invocation_index`: a subgroup reserves a run of its own
- * wherever it lands. Every subgroup operation is reached in uniform control flow; the loops' trip
- * counts are literals, and the one early exit is on a value read back through
- * `workgroupUniformLoad`.
- *
- * @type {CodeChunk}
- */
-export const chunk_bvh_query_aabb_cooperative = CodeChunk.from(
-    //language=WGSL
-    `
-struct BVHQueryAABBResult {
-    // payloads in bvh_query_aabb_candidates[0, candidate_count), ascending
-    candidate_count: u32,
-    // leaves the filter accepted that found the candidate list full
-    candidates_dropped: u32,
-    // nodes never tested: frontier overflow, and what was left when the level budget ran out
-    nodes_dropped: u32,
-}
-
-// two lists of node indices, alternating by level; bvh_query_aabb_list says which one a level reads
-var<workgroup> bvh_query_aabb_frontier: array<array<u32, BVH_QUERY_AABB_FRONTIER_CAPACITY>, 2>;
-// appends asked of each list; past the capacity they were counted but not stored
-var<workgroup> bvh_query_aabb_frontier_reserved: array<atomic<u32>, 2>;
-var<workgroup> bvh_query_aabb_candidates: array<u32, BVH_QUERY_AABB_CANDIDATE_CAPACITY>;
-var<workgroup> bvh_query_aabb_candidates_reserved: atomic<u32>;
-// written by invocation 0 only
-var<workgroup> bvh_query_aabb_nodes_dropped: u32;
-// what invocation 0 publishes, read back through workgroupUniformLoad
-var<workgroup> bvh_query_aabb_broadcast: u32;
-
-// This invocation's first slot of count appends to a frontier list, reserved for the whole
-// subgroup by one atomic.
-fn bvh_query_aabb_reserve_frontier(list: u32, count: u32) -> u32 {
-    let offset = subgroupExclusiveAdd(count);
-    let total = subgroupAdd(count);
-
-    // elected before the branch, which then holds no subgroup operation and may be non-uniform
-    let elected = subgroupElect();
-
-    var base = 0u;
-
-    if (elected && total > 0u) {
-        base = atomicAdd(&bvh_query_aabb_frontier_reserved[list], total);
-    }
-
-    return subgroupBroadcastFirst(base) + offset;
-}
-
-// The same, for the candidate list.
-fn bvh_query_aabb_reserve_candidates(count: u32) -> u32 {
-    let offset = subgroupExclusiveAdd(count);
-    let total = subgroupAdd(count);
-
-    let elected = subgroupElect();
-
-    var base = 0u;
-
-    if (elected && total > 0u) {
-        base = atomicAdd(&bvh_query_aabb_candidates_reserved, total);
-    }
-
-    return subgroupBroadcastFirst(base) + offset;
-}
-
-// Test this level's list: the children of overlapping internal nodes go on the next list, the
-// payloads of overlapping leaves the filter accepts on the candidate list.
-fn bvh_query_aabb_expand(list: u32, expand_count: u32, query: array<f32, 6>, lane: u32) {
-    let next = list ^ 1u;
-
-    for (var iteration = 0u; iteration < BVH_QUERY_AABB_FRONTIER_ROUNDS; iteration++) {
-        if (iteration * BVH_QUERY_AABB_WORKGROUP_SIZE >= expand_count) {
-            break;
-        }
-
-        let slot = bvh_query_aabb_slot(iteration, lane);
-
-        var node: ${BVH2_NODE_STRUCT.wgsl_ref};
-        var kind = BVH_QUERY_AABB_SKIP;
-
-        if (slot < expand_count) {
-            node = bvh_query_aabb_node(bvh_query_aabb_frontier[list][slot]);
-
-            kind = bvh_query_aabb_classify(node.bounds, node.child_1, query);
-
-            if (kind == BVH_QUERY_AABB_LEAF && !bvh_query_aabb_accept_leaf(node.child_2, node.bounds)) {
-                kind = BVH_QUERY_AABB_SKIP;
-            }
-        }
-
-        // every invocation reserves, holding or not, so the subgroup operations stay uniform
-        let frontier_slot = bvh_query_aabb_reserve_frontier(next, select(0u, 2u, kind == BVH_QUERY_AABB_DESCEND));
-        let candidate_slot = bvh_query_aabb_reserve_candidates(select(0u, 1u, kind == BVH_QUERY_AABB_LEAF));
-
-        if (kind == BVH_QUERY_AABB_DESCEND) {
-            if (frontier_slot < BVH_QUERY_AABB_FRONTIER_CAPACITY) {
-                bvh_query_aabb_frontier[next][frontier_slot] = node.child_1;
-            }
-            if (frontier_slot + 1u < BVH_QUERY_AABB_FRONTIER_CAPACITY) {
-                bvh_query_aabb_frontier[next][frontier_slot + 1u] = node.child_2;
-            }
-        }
-
-        if (kind == BVH_QUERY_AABB_LEAF && candidate_slot < BVH_QUERY_AABB_CANDIDATE_CAPACITY) {
-            bvh_query_aabb_candidates[candidate_slot] = node.child_2;
-        }
-    }
-}
-
-// Order the first count candidates ascending, through the power-of-two prefix that holds them.
-fn bvh_query_aabb_sort(count: u32, lane: u32) {
-    if (count <= 1u) {
-        return;
-    }
-
-    let span = sort_bitonic_size(count);
-
-    // the prefix past the candidates sorts last
-    for (var iteration = 0u; iteration < BVH_QUERY_AABB_CANDIDATE_ROUNDS; iteration++) {
-        let slot = iteration * BVH_QUERY_AABB_WORKGROUP_SIZE + lane;
-
-        if (slot >= count && slot < span) {
-            bvh_query_aabb_candidates[slot] = 0xffffffffu;
-        }
-    }
-
-    // span is workgroup-uniform, so is every bound and exit below
-    for (var k = 2u; k <= BVH_QUERY_AABB_CANDIDATE_CAPACITY; k = k << 1u) {
-        if (k > span) {
-            break;
-        }
-
-        for (var j = k >> 1u; j > 0u; j = j >> 1u) {
-            workgroupBarrier();
-
-            // the smaller of the two belongs in pair.x
-            let pair = sort_bitonic_comparator(lane, k, j);
-
-            // a pair is wholly inside the prefix or wholly past it
-            if (max(pair.x, pair.y) < span) {
-                let a = bvh_query_aabb_candidates[pair.x];
-                let b = bvh_query_aabb_candidates[pair.y];
-
-                if (b < a) {
-                    bvh_query_aabb_candidates[pair.x] = b;
-                    bvh_query_aabb_candidates[pair.y] = a;
-                }
-            }
-        }
-    }
-}
-
-fn bvh_query_aabb_cooperative(root: u32, query: array<f32, 6>, lane: u32) -> BVHQueryAABBResult {
-    // whatever the workgroup did with a previous query's candidates is done before they are reset
-    workgroupBarrier();
-
-    if (lane == 0u) {
-        bvh_query_aabb_frontier[0][0] = root;
-        atomicStore(&bvh_query_aabb_frontier_reserved[0], select(1u, 0u, root == BVH_NULL_NODE));
-        atomicStore(&bvh_query_aabb_frontier_reserved[1], 0u);
-        atomicStore(&bvh_query_aabb_candidates_reserved, 0u);
-        bvh_query_aabb_nodes_dropped = 0u;
-    }
-
-    // one pass past the budget, to count what the last expanded level left on its list; that
-    // pass publishes nothing to expand, so the loop always leaves through the break
-    for (var level = 0u; level <= BVH_QUERY_AABB_LEVEL_BUDGET; level++) {
-        let list = bvh_query_aabb_list(level);
-
-        // every append to this level's list has landed
-        workgroupBarrier();
-
-        if (lane == 0u) {
-            let published = bvh_query_aabb_publish(atomicLoad(&bvh_query_aabb_frontier_reserved[list]), level);
-
-            bvh_query_aabb_broadcast = published.x;
-            bvh_query_aabb_nodes_dropped += published.y;
-
-            // the list this level appends to, read two levels ago and empty again
-            atomicStore(&bvh_query_aabb_frontier_reserved[list ^ 1u], 0u);
-        }
-
-        let expand_count = workgroupUniformLoad(&bvh_query_aabb_broadcast);
-
-        if (expand_count == 0u) {
-            break;
-        }
-
-        bvh_query_aabb_expand(list, expand_count, query, lane);
-    }
-
-    // every invocation has read the last broadcast before invocation 0 writes the next
-    workgroupBarrier();
-
-    if (lane == 0u) {
-        bvh_query_aabb_broadcast = atomicLoad(&bvh_query_aabb_candidates_reserved);
-    }
-
-    let reserved = workgroupUniformLoad(&bvh_query_aabb_broadcast);
-    let count = min(reserved, BVH_QUERY_AABB_CANDIDATE_CAPACITY);
-
-    bvh_query_aabb_sort(count, lane);
-
-    // also the barrier that makes the sorted list visible to every invocation
-    let nodes_dropped = workgroupUniformLoad(&bvh_query_aabb_nodes_dropped);
-
-    return BVHQueryAABBResult(count, reserved - count, nodes_dropped);
-}
-`, [
-    BVH2_NODE_STRUCT.declaration_chunk,
-    BVH_NULL_NODE_DEFINITION,
-    chunk_bvh_query_aabb_frontier,
-    chunk_sort_bitonic_comparator,
-]);
-
-chunk_bvh_query_aabb_cooperative.addExtension(WebGPUExtensionType.Subgroups);
+import { WebGPUExtensionType } from "../../../../descriptor/WebGPUExtensionType.js";
+import { CodeChunk } from "../../compiler/CodeChunk.js";
+import { chunk_sort_bitonic_comparator } from "../../../gpu_primitive/sort/chunk_sort_bitonic_comparator.js";
+import { chunk_subgroup_reserve_workgroup } from "../subgroup/chunk_subgroup_reserve_workgroup.js";
+import { BVH2_NODE_STRUCT } from "./BVH2_NODE_STRUCT.js";
+import { BVH_NULL_NODE_DEFINITION } from "./BVH_NULL_NODE_DEFINITION.js";
+import { chunk_bvh_query_aabb_frontier } from "./chunk_bvh_query_aabb_frontier.js";
+
+/**
+ * Every leaf of a BVH whose box overlaps one query box, found by a whole workgroup together and
+ * returned as payloads in ascending order.
+ *
+ * ## Why not a stack per thread
+ *
+ * The engine's traversals (`chunk_sdf_query_distance`, `chunk_ray_query_*`) walk a stack per
+ * thread with no cap on visits, which is right for one ray per thread and wrong for one box per
+ * workgroup: the box meets a subtree, not a path, and one thread walking it serially is the slowest
+ * thread of the dispatch. This expands the tree a **level at a time** instead, as the vgeo cut does
+ * its descent: every node on the current level is tested by its own invocation, the survivors'
+ * children are appended to the next level, overlapping leaves to the candidate list, and a barrier
+ * separates the levels. Per-invocation work is bounded by the depth of the tree, not by the size of
+ * the answer.
+ *
+ * ## The contract
+ *
+ * Call `bvh_query_aabb_cooperative(root, query, lane)` from every invocation of a workgroup of
+ * exactly `BVH_QUERY_AABB_WORKGROUP_SIZE` invocations, in uniform control flow, with `lane` the
+ * `local_invocation_index` and `root` and `query` the same in every invocation. `root` is the
+ * tree's root word (word 0 of `BVH_BUFFER_LAYOUT`), and `BVH_NULL_NODE` for an empty tree. `query`
+ * is `[min.xyz, max.xyz]`; touching counts as overlapping, as in `aabb3_intersects_aabb3`.
+ *
+ * It returns a `BVHQueryAABBResult` whose fields are workgroup-uniform, so they can bound the
+ * caller's loops. The payloads are `bvh_query_aabb_candidates[0, candidate_count)`, ascending, and
+ * visible to every invocation when the call returns. They stay there until the next query.
+ *
+ * The node layout is `BVH_BUFFER_LAYOUT`'s: a node is a leaf exactly when `child_1` is
+ * `BVH_NULL_NODE`, and then `child_2` is its payload.
+ *
+ * ## The seam: two functions the enclosing shader defines
+ *
+ * The chunk binds nothing. It reads the tree and filters leaves through two functions the shader
+ * that includes it must define, in the manner of the particle VM's `vm_reg_read` and
+ * `vm_reg_write`:
+ *
+ * - `fn bvh_query_aabb_node(index: u32) -> BVH2Node` returns node `index`, where `BVH2Node` is
+ *   `BVH2_NODE_STRUCT.wgsl_ref`. A tree bound as `BVH_STRUCT` answers with `tree.nodes[index]`; one
+ *   inside a word arena reads its eight words.
+ * - `fn bvh_query_aabb_accept_leaf(payload: u32, bounds: array<f32, 6>) -> bool` is asked about
+ *   every leaf that overlaps the box, before it takes a place on the list. A caller with nothing to
+ *   filter returns `true`. The fluid's resolution cull lives here: a leaf too small for the cascade
+ *   is refused before it can crowd out one that is not.
+ *
+ * ## Deterministic order
+ *
+ * Appends from different subgroups land in whatever order the hardware runs them, so the list is
+ * sorted by payload before it is returned: the bitonic network of `chunk_sort_bitonic_comparator`
+ * over the smallest power-of-two prefix that holds the candidates (`sort_bitonic_size`), padded
+ * with `0xffffffff`, one compare-exchange per invocation per stage. A payload that is a stable row index then makes "the lowest K" the same K every frame for
+ * the same overlaps, however the traversal was scheduled.
+ *
+ * ## Bounds, and what is counted
+ *
+ * Nothing is dropped silently. A level whose appends pass `BVH_QUERY_AABB_FRONTIER_CAPACITY`, and a
+ * list still holding nodes when `BVH_QUERY_AABB_LEVEL_BUDGET` levels have been expanded, add those
+ * nodes to `nodes_dropped`: subtrees never tested. A leaf that passes the filter when the candidate
+ * list is full adds to `candidates_dropped`. With both zero the candidates are every overlapping
+ * leaf the filter accepted; with either non-zero they are a subset, and the lowest K of them need
+ * not be the lowest K overall.
+ *
+ * Per invocation, at most `BVH_QUERY_AABB_LEVEL_BUDGET * BVH_QUERY_AABB_FRONTIER_ROUNDS` = 51 * 4 =
+ * 204 node reads and filter calls, then two padding writes and 28 compare-exchanges. A query that
+ * meets nothing costs one level: the root is read, misses, and the loop leaves.
+ *
+ * Workgroup memory: two lists of 256 node indices, 128 payloads and five words, 2580 bytes.
+ *
+ * ## Subgroups
+ *
+ * Appends are aggregated per subgroup (`chunk_subgroup_reserve_workgroup`): an exclusive add numbers
+ * the subgroup's appends, one elected invocation reserves them all with one workgroup `atomicAdd`,
+ * and a broadcast hands each invocation its base. Nothing assumes which invocations share a
+ * subgroup or that a subgroup's invocations are contiguous in `local_invocation_index`: a subgroup
+ * reserves a run of its own wherever it lands. Every subgroup operation is reached in uniform
+ * control flow; the loops' trip counts are literals, and the one early exit is on a value read
+ * back through `workgroupUniformLoad`.
+ *
+ * @type {CodeChunk}
+ */
+export const chunk_bvh_query_aabb_cooperative = CodeChunk.from(
+    //language=WGSL
+    `
+struct BVHQueryAABBResult {
+    // payloads in bvh_query_aabb_candidates[0, candidate_count), ascending
+    candidate_count: u32,
+    // leaves the filter accepted that found the candidate list full
+    candidates_dropped: u32,
+    // nodes never tested: frontier overflow, and what was left when the level budget ran out
+    nodes_dropped: u32,
+}
+
+// two lists of node indices, alternating by level; bvh_query_aabb_list says which one a level reads
+var<workgroup> bvh_query_aabb_frontier: array<array<u32, BVH_QUERY_AABB_FRONTIER_CAPACITY>, 2>;
+// appends asked of each list; past the capacity they were counted but not stored
+var<workgroup> bvh_query_aabb_frontier_reserved: array<atomic<u32>, 2>;
+var<workgroup> bvh_query_aabb_candidates: array<u32, BVH_QUERY_AABB_CANDIDATE_CAPACITY>;
+var<workgroup> bvh_query_aabb_candidates_reserved: atomic<u32>;
+// written by invocation 0 only
+var<workgroup> bvh_query_aabb_nodes_dropped: u32;
+// what invocation 0 publishes, read back through workgroupUniformLoad
+var<workgroup> bvh_query_aabb_broadcast: u32;
+
+// Test this level's list: the children of overlapping internal nodes go on the next list, the
+// payloads of overlapping leaves the filter accepts on the candidate list.
+fn bvh_query_aabb_expand(list: u32, expand_count: u32, query: array<f32, 6>, lane: u32) {
+    let next = list ^ 1u;
+
+    for (var iteration = 0u; iteration < BVH_QUERY_AABB_FRONTIER_ROUNDS; iteration++) {
+        if (iteration * BVH_QUERY_AABB_WORKGROUP_SIZE >= expand_count) {
+            break;
+        }
+
+        let slot = bvh_query_aabb_slot(iteration, lane);
+
+        var node: ${BVH2_NODE_STRUCT.wgsl_ref};
+        var kind = BVH_QUERY_AABB_SKIP;
+
+        if (slot < expand_count) {
+            node = bvh_query_aabb_node(bvh_query_aabb_frontier[list][slot]);
+
+            kind = bvh_query_aabb_classify(node.bounds, node.child_1, query);
+
+            if (kind == BVH_QUERY_AABB_LEAF && !bvh_query_aabb_accept_leaf(node.child_2, node.bounds)) {
+                kind = BVH_QUERY_AABB_SKIP;
+            }
+        }
+
+        // every invocation reserves, holding or not, so the subgroup operations stay uniform
+        let frontier_slot = subgroup_reserve_workgroup(&bvh_query_aabb_frontier_reserved[next], select(0u, 2u, kind == BVH_QUERY_AABB_DESCEND));
+        let candidate_slot = subgroup_reserve_workgroup(&bvh_query_aabb_candidates_reserved, select(0u, 1u, kind == BVH_QUERY_AABB_LEAF));
+
+        if (kind == BVH_QUERY_AABB_DESCEND) {
+            if (frontier_slot < BVH_QUERY_AABB_FRONTIER_CAPACITY) {
+                bvh_query_aabb_frontier[next][frontier_slot] = node.child_1;
+            }
+            if (frontier_slot + 1u < BVH_QUERY_AABB_FRONTIER_CAPACITY) {
+                bvh_query_aabb_frontier[next][frontier_slot + 1u] = node.child_2;
+            }
+        }
+
+        if (kind == BVH_QUERY_AABB_LEAF && candidate_slot < BVH_QUERY_AABB_CANDIDATE_CAPACITY) {
+            bvh_query_aabb_candidates[candidate_slot] = node.child_2;
+        }
+    }
+}
+
+// Order the first count candidates ascending, through the power-of-two prefix that holds them.
+fn bvh_query_aabb_sort(count: u32, lane: u32) {
+    if (count <= 1u) {
+        return;
+    }
+
+    let span = sort_bitonic_size(count);
+
+    // the prefix past the candidates sorts last
+    for (var iteration = 0u; iteration < BVH_QUERY_AABB_CANDIDATE_ROUNDS; iteration++) {
+        let slot = iteration * BVH_QUERY_AABB_WORKGROUP_SIZE + lane;
+
+        if (slot >= count && slot < span) {
+            bvh_query_aabb_candidates[slot] = 0xffffffffu;
+        }
+    }
+
+    // span is workgroup-uniform, so is every bound and exit below
+    for (var k = 2u; k <= BVH_QUERY_AABB_CANDIDATE_CAPACITY; k = k << 1u) {
+        if (k > span) {
+            break;
+        }
+
+        for (var j = k >> 1u; j > 0u; j = j >> 1u) {
+            workgroupBarrier();
+
+            // the smaller of the two belongs in pair.x
+            let pair = sort_bitonic_comparator(lane, k, j);
+
+            // a pair is wholly inside the prefix or wholly past it
+            if (max(pair.x, pair.y) < span) {
+                let a = bvh_query_aabb_candidates[pair.x];
+                let b = bvh_query_aabb_candidates[pair.y];
+
+                if (b < a) {
+                    bvh_query_aabb_candidates[pair.x] = b;
+                    bvh_query_aabb_candidates[pair.y] = a;
+                }
+            }
+        }
+    }
+}
+
+fn bvh_query_aabb_cooperative(root: u32, query: array<f32, 6>, lane: u32) -> BVHQueryAABBResult {
+    // whatever the workgroup did with a previous query's candidates is done before they are reset
+    workgroupBarrier();
+
+    if (lane == 0u) {
+        bvh_query_aabb_frontier[0][0] = root;
+        atomicStore(&bvh_query_aabb_frontier_reserved[0], select(1u, 0u, root == BVH_NULL_NODE));
+        atomicStore(&bvh_query_aabb_frontier_reserved[1], 0u);
+        atomicStore(&bvh_query_aabb_candidates_reserved, 0u);
+        bvh_query_aabb_nodes_dropped = 0u;
+    }
+
+    // one pass past the budget, to count what the last expanded level left on its list; that
+    // pass publishes nothing to expand, so the loop always leaves through the break
+    for (var level = 0u; level <= BVH_QUERY_AABB_LEVEL_BUDGET; level++) {
+        let list = bvh_query_aabb_list(level);
+
+        // every append to this level's list has landed
+        workgroupBarrier();
+
+        if (lane == 0u) {
+            let published = bvh_query_aabb_publish(atomicLoad(&bvh_query_aabb_frontier_reserved[list]), level);
+
+            bvh_query_aabb_broadcast = published.x;
+            bvh_query_aabb_nodes_dropped += published.y;
+
+            // the list this level appends to, read two levels ago and empty again
+            atomicStore(&bvh_query_aabb_frontier_reserved[list ^ 1u], 0u);
+        }
+
+        let expand_count = workgroupUniformLoad(&bvh_query_aabb_broadcast);
+
+        if (expand_count == 0u) {
+            break;
+        }
+
+        bvh_query_aabb_expand(list, expand_count, query, lane);
+    }
+
+    // every invocation has read the last broadcast before invocation 0 writes the next
+    workgroupBarrier();
+
+    if (lane == 0u) {
+        bvh_query_aabb_broadcast = atomicLoad(&bvh_query_aabb_candidates_reserved);
+    }
+
+    let reserved = workgroupUniformLoad(&bvh_query_aabb_broadcast);
+    let count = min(reserved, BVH_QUERY_AABB_CANDIDATE_CAPACITY);
+
+    bvh_query_aabb_sort(count, lane);
+
+    // also the barrier that makes the sorted list visible to every invocation
+    let nodes_dropped = workgroupUniformLoad(&bvh_query_aabb_nodes_dropped);
+
+    return BVHQueryAABBResult(count, reserved - count, nodes_dropped);
+}
+`, [
+    BVH2_NODE_STRUCT.declaration_chunk,
+    BVH_NULL_NODE_DEFINITION,
+    chunk_bvh_query_aabb_frontier,
+    chunk_sort_bitonic_comparator,
+    chunk_subgroup_reserve_workgroup,
+]);
+
+chunk_bvh_query_aabb_cooperative.addExtension(WebGPUExtensionType.Subgroups);

package/src/shade/renderer/shader/chunk/filter/MULTISCALE_MEAN_STRUCT.d.ts

@@ -0,0 +1,9 @@
+/**
+ * A multi-scale mean estimator's state, see {@link chunk_multiscale_mean}. Packed, so its fields are not in the
+ * order written here: build one with {@link WebGPUStruct#make}, never a positional constructor.
+ *
+ * @type {WebGPUStruct}
+ */
+export const MULTISCALE_MEAN_STRUCT: WebGPUStruct;
+import { WebGPUStruct } from "../../type/WebGPUStruct.js";
+//# sourceMappingURL=MULTISCALE_MEAN_STRUCT.d.ts.map

package/src/shade/renderer/shader/chunk/filter/MULTISCALE_MEAN_STRUCT.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"MULTISCALE_MEAN_STRUCT.d.ts","sourceRoot":"","sources":["../../../../../../../src/shade/renderer/shader/chunk/filter/MULTISCALE_MEAN_STRUCT.js"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,qCAFU,YAAY,CAaM;6BAnBC,4BAA4B"}

package/src/shade/renderer/shader/chunk/filter/MULTISCALE_MEAN_STRUCT.js

@@ -0,0 +1,21 @@
+import { f32, vec3f } from "../../type/WEBGPU_WGSL_PRIMITIVE_TYPES.js";
+import { WebGPUStruct } from "../../type/WebGPUStruct.js";
+
+/**
+ * A multi-scale mean estimator's state, see {@link chunk_multiscale_mean}. Packed, so its fields are not in the
+ * order written here: build one with {@link WebGPUStruct#make}, never a positional constructor.
+ *
+ * @type {WebGPUStruct}
+ */
+export const MULTISCALE_MEAN_STRUCT = WebGPUStruct.from({
+    /** the long mean, the estimate */
+    mean: vec3f,
+    /** the short mean, blended by the short blend */
+    short_mean: vec3f,
+    /** a channel's variance about the short mean, blended by the variance blend */
+    variance: vec3f,
+    /** the blend reduction the long mean's catch-up is scaled by */
+    reduction: f32,
+    /** the means' gap over the deviation, smoothed by the short blend */
+    inconsistency: f32,
+}, "MultiscaleMean").pack();