@woosh/meep-engine 3.58.0 → 3.60.0

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_histogram.js

@@ -1,75 +1,49 @@
-import { u32 } from "../../shader/type/WEBGPU_WGSL_PRIMITIVE_TYPES.js";
-import { WebGPUArray } from "../../shader/type/WebGPUArray.js";
 import { CodeChunk } from "../../shader/compiler/CodeChunk.js";
 import { ComputeShader } from "../../shader/ComputeShader.js";
 import { ShaderResourceSetDescriptor } from "../../shader/resource/ShaderResourceSetDescriptor.js";
-import { chunk_sort_digit } from "./chunk_sort_digit.js";
-import { RADIX, SORT_KEYS_PER_THREAD, SORT_TILE_SIZE, SORT_WORKGROUP_SIZE } from "./defs.js";
+import {
+    chunk_sort_histogram_tile,
+    SORT_HISTOGRAM_KEYS_RESOURCE,
+    SORT_HISTOGRAM_SPINE_RESOURCE
+} from "./chunk_sort_histogram_tile.js";
 import { SORT_PASS_SETTINGS_STRUCT } from "./SORT_PASS_SETTINGS_STRUCT.js";
 
 const resources = new ShaderResourceSetDescriptor();
 
-// per-pass, 12 bytes, read-only: immediate data rather than a uniform buffer. A 32-bit key sort
+// per-pass, 16 bytes, read-only: immediate data rather than a uniform buffer. A 32-bit key sort
 // runs four passes, and both this shader and the scatter took a buffer of their own in each.
 resources.setImmediate("settings", SORT_PASS_SETTINGS_STRUCT);
 
 resources.createGroup()
-    .addStorageBuffer("keys", WebGPUArray.from(u32), false)
-    // `{count: u32, elements: array<vec4u>}` as the scan sees it, flat as this pass writes it.
-    // Every cell is produced by exactly one thread of one workgroup, so no atomics: the only
-    // consumer is a later dispatch, and WebGPU orders dispatches against each other.
-    .addStorageBuffer("spine", WebGPUArray.from(u32), true);
+    .add(SORT_HISTOGRAM_KEYS_RESOURCE)
+    .add(SORT_HISTOGRAM_SPINE_RESOURCE);
 
 const body = CodeChunk.from(
     //language=WGSL
     `
-const WORKGROUP_SIZE = ${SORT_WORKGROUP_SIZE}u;
-const KEYS_PER_THREAD = ${SORT_KEYS_PER_THREAD}u;
-const TILE_SIZE = ${SORT_TILE_SIZE}u;
-const RADIX = ${RADIX}u;
-
-// One bin per thread — RADIX and the workgroup size are equal, so the clear and the write-out are
-// both a single unstrided step.
-var<workgroup> wg_histogram: array<atomic<u32>, RADIX>;
-
-@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
+@compute @workgroup_size(SORT_WORKGROUP_SIZE, 1, 1)
 fn main(
     @builtin(local_invocation_index) thread_id: u32,
     @builtin(workgroup_id) workgroup_id: vec3<u32>,
 ) {
     let tile = workgroup_id.x;
 
-    atomicStore(&wg_histogram[thread_id], 0u);
-
-    workgroupBarrier();
-
-    // Striped so that neighbouring threads read neighbouring addresses. The tail tile is partial;
-    // the keys past the end are not counted, which is what lets the scatter recognise them later
-    // by position alone.
-    let tile_base = tile * TILE_SIZE;
-    for (var k = 0u; k < KEYS_PER_THREAD; k += 1u) {
-        let source = tile_base + k * WORKGROUP_SIZE + thread_id;
-
-        if (source < settings.element_count) {
-            atomicAdd(&wg_histogram[sort_extract_digit(keys[source], settings.shift)], 1u);
-        }
-    }
-
-    workgroupBarrier();
-
-    spine[sort_spine_index(thread_id, tile, settings.tile_count)] = atomicLoad(&wg_histogram[thread_id]);
+    sort_histogram_tile(tile, settings.element_count, settings.tile_count, settings.shift, settings.key_mask, thread_id);
 
     // The scan is indirect-dispatched off an element count it reads from word 0 of this buffer.
     if (tile == 0u && thread_id == 0u) {
-        spine[0] = settings.tile_count * RADIX;
+        spine[0] = settings.tile_count * SORT_RADIX;
     }
 }
 `, [
-    chunk_sort_digit,
+    chunk_sort_histogram_tile,
 ]);
 
 /**
- * Per-tile 256-bin digit histogram — stage (a) of one digit pass of {@link graph_radix_sort}.
+ * Per-tile 256-bin digit histogram — stage (a) of one digit pass of {@link graph_radix_sort} over a
+ * host count, which it dispatches directly, a workgroup per tile. The tile's work is
+ * {@link chunk_sort_histogram_tile}; {@link shader_radix_sort_histogram_indirect} is the same work
+ * over a device count.
  *
  * One workgroup per tile; the counts land in the spine bin-major, so that the single scan that
  * follows produces global digit offsets and per-tile offsets in one traversal.

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_histogram_indirect.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Per-tile 256-bin digit histogram — stage (a) of one digit pass of {@link graph_radix_sort} over a
+ * device count. Dispatched indirectly through the command {@link shader_radix_sort_prepare} folded
+ * over the live tile count, it finds its tile with `folded_workgroup_index`, and reads the live size
+ * that pass wrote. The tile's work is {@link chunk_sort_histogram_tile}, as it is
+ * {@link shader_radix_sort_histogram}'s.
+ *
+ * Unlike the host-count histogram it leaves the spine's count word alone: the prepare writes it, once
+ * for every digit pass, because an empty sort runs no tile that could.
+ *
+ * @type {ComputeShader}
+ */
+export const shader_radix_sort_histogram_indirect: ComputeShader;
+import { ComputeShader } from "../../shader/ComputeShader.js";
+//# sourceMappingURL=shader_radix_sort_histogram_indirect.d.ts.map

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_histogram_indirect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shader_radix_sort_histogram_indirect.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/shader_radix_sort_histogram_indirect.js"],"names":[],"mappings":"AA2CA;;;;;;;;;;;GAWG;AACH,mDAFU,aAAa,CAMpB;8BAzD2B,+BAA+B"}

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_histogram_indirect.js

@@ -0,0 +1,60 @@
+import { chunk_folded_workgroup_index } from "../../shader/chunk/grid/chunk_folded_workgroup_index.js";
+import { CodeChunk } from "../../shader/compiler/CodeChunk.js";
+import { ComputeShader } from "../../shader/ComputeShader.js";
+import { ShaderResourceSetDescriptor } from "../../shader/resource/ShaderResourceSetDescriptor.js";
+import {
+    chunk_sort_histogram_tile,
+    SORT_HISTOGRAM_KEYS_RESOURCE,
+    SORT_HISTOGRAM_SPINE_RESOURCE
+} from "./chunk_sort_histogram_tile.js";
+import { SORT_DIGIT_SETTINGS_STRUCT } from "./SORT_DIGIT_SETTINGS_STRUCT.js";
+import { SORT_SIZE_STRUCT } from "./SORT_SIZE_STRUCT.js";
+
+const resources = new ShaderResourceSetDescriptor();
+
+resources.setImmediate("settings", SORT_DIGIT_SETTINGS_STRUCT);
+
+resources.createGroup()
+    .add(SORT_HISTOGRAM_KEYS_RESOURCE)
+    .add(SORT_HISTOGRAM_SPINE_RESOURCE)
+    .addStorageBuffer("sort_size", SORT_SIZE_STRUCT, false);
+
+const body = CodeChunk.from(
+    //language=WGSL
+    `
+@compute @workgroup_size(SORT_WORKGROUP_SIZE, 1, 1)
+fn main(
+    @builtin(local_invocation_index) thread_id: u32,
+    @builtin(workgroup_id) workgroup_id: vec3<u32>,
+) {
+    let tile = folded_workgroup_index(workgroup_id);
+
+    // A folded grid's last row runs past the live tiles. Uniform, the whole workgroup leaves.
+    if (tile >= sort_size.tile_count) {
+        return;
+    }
+
+    sort_histogram_tile(tile, sort_size.element_count, sort_size.tile_count, settings.shift, settings.key_mask, thread_id);
+}
+`, [
+    chunk_folded_workgroup_index,
+    chunk_sort_histogram_tile,
+]);
+
+/**
+ * Per-tile 256-bin digit histogram — stage (a) of one digit pass of {@link graph_radix_sort} over a
+ * device count. Dispatched indirectly through the command {@link shader_radix_sort_prepare} folded
+ * over the live tile count, it finds its tile with `folded_workgroup_index`, and reads the live size
+ * that pass wrote. The tile's work is {@link chunk_sort_histogram_tile}, as it is
+ * {@link shader_radix_sort_histogram}'s.
+ *
+ * Unlike the host-count histogram it leaves the spine's count word alone: the prepare writes it, once
+ * for every digit pass, because an empty sort runs no tile that could.
+ *
+ * @type {ComputeShader}
+ */
+export const shader_radix_sort_histogram_indirect = ComputeShader.from({
+    label: "RadixSort/Histogram (indirect)",
+    resources,
+    body,
+});

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_prepare.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @type {WebGPUStruct}
+ */
+export const SORT_PREPARE_SETTINGS_STRUCT: WebGPUStruct;
+/**
+ * One thread ahead of every digit pass of {@link graph_radix_sort} over a device count: it reads the
+ * count, clamps it to what the memory holds, and writes what the passes after it are dispatched and
+ * bounded by —
+ *
+ * - `sort_size`, the live key and tile counts ({@link SORT_SIZE_STRUCT}), which the histogram and
+ *   scatter read;
+ * - `command`, their indirect dispatch, a workgroup a live tile folded as
+ *   {@link chunk_folded_dispatch} folds it — `(0, 1, 1)` for no keys, which dispatches nothing;
+ * - the spine's count word, the `{count, elements}` header `graph_prefix_scan_csdldf` reads, a
+ *   256-bin histogram a live tile. The spine is laid out bin-major with the live tile count as its
+ *   stride, so the scan covers exactly the cells the histogram writes.
+ *
+ * Every digit pass reads these, and they cannot change between passes, so one thread writes them once.
+ *
+ * @type {ComputeShader}
+ */
+export const shader_radix_sort_prepare: ComputeShader;
+import { WebGPUStruct } from "../../shader/type/WebGPUStruct.js";
+import { ComputeShader } from "../../shader/ComputeShader.js";
+//# sourceMappingURL=shader_radix_sort_prepare.d.ts.map

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_prepare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shader_radix_sort_prepare.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/shader_radix_sort_prepare.js"],"names":[],"mappings":"AAWA;;GAEG;AACH,2CAFU,YAAY,CAWI;AA4C1B;;;;;;;;;;;;;;;;GAgBG;AACH,wCAFU,aAAa,CAMpB;6BAjF0B,mCAAmC;8BAJlC,+BAA+B"}

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_prepare.js

@@ -0,0 +1,89 @@
+import { chunk_folded_dispatch } from "../indirect/chunk_folded_dispatch.js";
+import { chunk_divide_round_up_u32 } from "../../shader/chunk/math/chunk_divide_round_up_u32.js";
+import { CodeChunk } from "../../shader/compiler/CodeChunk.js";
+import { ComputeShader } from "../../shader/ComputeShader.js";
+import { ShaderResourceSetDescriptor } from "../../shader/resource/ShaderResourceSetDescriptor.js";
+import { u32 } from "../../shader/type/WEBGPU_WGSL_PRIMITIVE_TYPES.js";
+import { WebGPUArray } from "../../shader/type/WebGPUArray.js";
+import { WebGPUStruct } from "../../shader/type/WebGPUStruct.js";
+import { chunk_sort_constants } from "./chunk_sort_constants.js";
+import { SORT_SIZE_STRUCT } from "./SORT_SIZE_STRUCT.js";
+
+/**
+ * @type {WebGPUStruct}
+ */
+export const SORT_PREPARE_SETTINGS_STRUCT = WebGPUStruct.from({
+    /**
+     * The `u32` of `count_source` that holds the count: its byte offset over four.
+     */
+    count_word: u32,
+    /**
+     * Keys the sort's memory holds — its spine and scratch, and the caller's buffers.
+     */
+    element_max_count: u32,
+}, "SortPrepareSettings");
+
+const resources = new ShaderResourceSetDescriptor();
+
+resources.setImmediate("settings", SORT_PREPARE_SETTINGS_STRUCT);
+
+resources.createGroup()
+    // the caller's buffer, whole, so that its count can sit at any word of it
+    .addStorageBuffer("count_source", WebGPUArray.from(u32), false)
+    .addStorageBuffer("sort_size", SORT_SIZE_STRUCT, true)
+    .addStorageBuffer("command", WebGPUArray.from(u32), true)
+    // the spine, for its `{count, elements}` header alone
+    .addStorageBuffer("spine", WebGPUArray.from(u32), true);
+
+const body = CodeChunk.from(
+    //language=WGSL
+    `
+@compute @workgroup_size(1)
+fn main() {
+    // A count past what the memory was sized for sorts the first element_max_count keys, rather than
+    // running the histogram and scatter past the ends of the spine and the scratch.
+    let element_count = min(count_source[settings.count_word], settings.element_max_count);
+    let tile_count = divide_round_up_u32(element_count, SORT_TILE_SIZE);
+
+    sort_size.element_count = element_count;
+    sort_size.tile_count = tile_count;
+
+    // a workgroup a tile, folded: x to the limit, then rows of x, then slices of x·y
+    let groups = folded_dispatch(tile_count);
+
+    command[0] = groups.x;
+    command[1] = groups.y;
+    command[2] = groups.z;
+
+    // The spine's element count, which every digit pass's scan is dispatched over. Written here and
+    // not by a histogram tile, because an empty sort runs none.
+    spine[0] = tile_count * SORT_RADIX;
+}
+`, [
+    chunk_sort_constants,
+    chunk_divide_round_up_u32,
+    chunk_folded_dispatch,
+]);
+
+/**
+ * One thread ahead of every digit pass of {@link graph_radix_sort} over a device count: it reads the
+ * count, clamps it to what the memory holds, and writes what the passes after it are dispatched and
+ * bounded by —
+ *
+ * - `sort_size`, the live key and tile counts ({@link SORT_SIZE_STRUCT}), which the histogram and
+ *   scatter read;
+ * - `command`, their indirect dispatch, a workgroup a live tile folded as
+ *   {@link chunk_folded_dispatch} folds it — `(0, 1, 1)` for no keys, which dispatches nothing;
+ * - the spine's count word, the `{count, elements}` header `graph_prefix_scan_csdldf` reads, a
+ *   256-bin histogram a live tile. The spine is laid out bin-major with the live tile count as its
+ *   stride, so the scan covers exactly the cells the histogram writes.
+ *
+ * Every digit pass reads these, and they cannot change between passes, so one thread writes them once.
+ *
+ * @type {ComputeShader}
+ */
+export const shader_radix_sort_prepare = ComputeShader.from({
+    label: "RadixSort/Prepare",
+    resources,
+    body,
+});

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter.d.ts

@@ -1,12 +1,7 @@
 /**
- * Rank-and-scatter — stage (c) of one digit pass of {@link graph_radix_sort}.
- *
- * One workgroup per tile. The tile is sorted by the current digit in workgroup memory (stability is
- * what an LSD radix sort's correctness rests on, and it is also what lets the tail tile's padding
- * be identified by position), the tile's own histogram is re-derived rather than carried over from
- * the histogram pass — the scan is inclusive-only, and re-counting is cheaper than preserving
- * pre-scan counts — and each key is written to
- * `spine_inclusive[bin] - local_count[bin] - bin_start[bin] + sorted_position`.
+ * Rank-and-scatter — stage (c) of one digit pass of {@link graph_radix_sort} over a host count, which
+ * it dispatches directly, a workgroup per tile. The tile's work is {@link chunk_sort_scatter_tile};
+ * {@link shader_radix_sort_scatter_indirect} is the same work over a device count.
  *
  * @type {ComputeShader}
  */

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shader_radix_sort_scatter.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter.js"],"names":[],"mappings":"AAiLA;;;;;;;;;;;GAWG;AACH,wCAFU,aAAa,CAMpB;8BA7L2B,+BAA+B"}
+{"version":3,"file":"shader_radix_sort_scatter.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter.js"],"names":[],"mappings":"AAkDA;;;;;;GAMG;AACH,wCAFU,aAAa,CAMpB;8BA5D2B,+BAA+B"}

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter.js

@@ -1,13 +1,14 @@
-import { WebGPUExtensionType } from "../../../descriptor/WebGPUExtensionType.js";
-import { u32 } from "../../shader/type/WEBGPU_WGSL_PRIMITIVE_TYPES.js";
-import { WebGPUArray } from "../../shader/type/WebGPUArray.js";
 import { CodeChunk } from "../../shader/compiler/CodeChunk.js";
 import { ComputeShader } from "../../shader/ComputeShader.js";
 import { ShaderResourceSetDescriptor } from "../../shader/resource/ShaderResourceSetDescriptor.js";
-import { chunk_sort_digit } from "./chunk_sort_digit.js";
-import { chunk_sort_rank } from "./chunk_sort_rank.js";
-import { chunk_sort_workgroup_scan } from "./chunk_sort_workgroup_scan.js";
-import { RADIX, RADIX_LOG, SORT_KEYS_PER_THREAD, SORT_TILE_SIZE, SORT_WORKGROUP_SIZE } from "./defs.js";
+import {
+    chunk_sort_scatter_tile,
+    SORT_SCATTER_IN_KEYS_RESOURCE,
+    SORT_SCATTER_IN_VALUES_RESOURCE,
+    SORT_SCATTER_OUT_KEYS_RESOURCE,
+    SORT_SCATTER_OUT_VALUES_RESOURCE,
+    SORT_SCATTER_SPINE_RESOURCE
+} from "./chunk_sort_scatter_tile.js";
 import { SORT_PASS_SETTINGS_STRUCT } from "./SORT_PASS_SETTINGS_STRUCT.js";
 
 const resources = new ShaderResourceSetDescriptor();
@@ -16,174 +17,41 @@ const resources = new ShaderResourceSetDescriptor();
 resources.setImmediate("settings", SORT_PASS_SETTINGS_STRUCT);
 
 resources.createGroup()
-    .addStorageBuffer("in_keys", WebGPUArray.from(u32), false)
-    .addStorageBuffer("in_values", WebGPUArray.from(u32), false)
-    .addStorageBuffer("out_keys", WebGPUArray.from(u32), true)
-    .addStorageBuffer("out_values", WebGPUArray.from(u32), true)
-    // scanned spine, read flat past its `{count, elements}` header
-    .addStorageBuffer("spine", WebGPUArray.from(u32), false);
+    .add(SORT_SCATTER_IN_KEYS_RESOURCE)
+    .add(SORT_SCATTER_IN_VALUES_RESOURCE)
+    .add(SORT_SCATTER_OUT_KEYS_RESOURCE)
+    .add(SORT_SCATTER_OUT_VALUES_RESOURCE)
+    .add(SORT_SCATTER_SPINE_RESOURCE);
 
 const body = CodeChunk.from(
     //language=WGSL
     `
-const WORKGROUP_SIZE = ${SORT_WORKGROUP_SIZE}u;
-const KEYS_PER_THREAD = ${SORT_KEYS_PER_THREAD}u;
-const TILE_SIZE = ${SORT_TILE_SIZE}u;
-const RADIX = ${RADIX}u;
-const RADIX_LOG = ${RADIX_LOG}u;
-
-/**
- * The key a slot past the end of the input is filled with. It is the largest u32, so its digit is
- * the largest in every pass and the stable in-tile sort parks every one of them behind every real
- * key — which is what makes "sorted position < the tile's real key count" a sufficient test for
- * whether a slot holds a real key.
- */
-const PADDING_KEY = 0xffffffffu;
-
-var<workgroup> wg_keys: array<u32, TILE_SIZE>;
-var<workgroup> wg_values: array<u32, TILE_SIZE>;
-// one bin per thread; RADIX == WORKGROUP_SIZE
-var<workgroup> wg_histogram: array<atomic<u32>, RADIX>;
-var<workgroup> wg_base: array<u32, RADIX>;
-
-@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
+@compute @workgroup_size(SORT_WORKGROUP_SIZE, 1, 1)
 fn main(
     @builtin(local_invocation_index) thread_id: u32,
     @builtin(workgroup_id) workgroup_id: vec3<u32>,
     @builtin(subgroup_invocation_id) lane_id: u32,
     @builtin(subgroup_size) lane_count: u32,
 ) {
-    let tile = workgroup_id.x;
-    let tile_base = tile * TILE_SIZE;
-    let shift = settings.shift;
-
-    atomicStore(&wg_histogram[thread_id], 0u);
-
-    // Striped load, so neighbouring threads read neighbouring addresses.
-    for (var k = 0u; k < KEYS_PER_THREAD; k += 1u) {
-        let slot = k * WORKGROUP_SIZE + thread_id;
-        let source = tile_base + slot;
-
-        var key = PADDING_KEY;
-        var value = 0u;
-
-        if (source < settings.element_count) {
-            key = in_keys[source];
-            value = in_values[source];
-        }
-
-        wg_keys[slot] = key;
-        wg_values[slot] = value;
-    }
-
-    workgroupBarrier();
-
-    // Local histogram over the real keys only — it has to agree with what the histogram pass
-    // counted, because the spine value this tile reads back was built from that count.
-    for (var k = 0u; k < KEYS_PER_THREAD; k += 1u) {
-        let slot = k * WORKGROUP_SIZE + thread_id;
-
-        if (tile_base + slot < settings.element_count) {
-            atomicAdd(&wg_histogram[sort_extract_digit(wg_keys[slot], shift)], 1u);
-        }
-    }
-
-    workgroupBarrier();
-
-    // this thread's bin, captured before the in-tile sort touches anything
-    let local_count = atomicLoad(&wg_histogram[thread_id]);
-
-    // Order the tile by this pass's digit, one bit at a time, least significant first. Eight stable
-    // binary splits are a sort by the whole 8-bit digit; ranking a key against its same-digit peers
-    // then costs nothing, because the rank IS the sorted position, and the device writes that
-    // follow come out contiguous per bin.
-    for (var bit_index = 0u; bit_index < RADIX_LOG; bit_index += 1u) {
-        let bit = shift + bit_index;
-
-        // Blocked ownership for the permutation: thread t owns the tile's slots
-        // [t*KEYS_PER_THREAD, (t+1)*KEYS_PER_THREAD), so the offsets a thread hands its own keys
-        // are consecutive and the scan below only has to run over one value per thread.
-        var keys: array<u32, KEYS_PER_THREAD>;
-        var values: array<u32, KEYS_PER_THREAD>;
-        var ones = 0u;
-
-        for (var j = 0u; j < KEYS_PER_THREAD; j += 1u) {
-            let slot = thread_id * KEYS_PER_THREAD + j;
-
-            keys[j] = wg_keys[slot];
-            values[j] = wg_values[slot];
-            ones += (keys[j] >> bit) & 1u;
-        }
-
-        // The scan's first barrier is what fences every thread's reads above against the writes
-        // below — by the time it returns, no read of wg_keys/wg_values is outstanding.
-        let scan = sort_workgroup_exclusive_scan(ones, thread_id, lane_id, lane_count);
-
-        let total_zeros = TILE_SIZE - scan.y;
-
-        var ones_before = scan.x;
-        var zeros_before = thread_id * KEYS_PER_THREAD - scan.x;
-
-        for (var j = 0u; j < KEYS_PER_THREAD; j += 1u) {
-            let is_one = ((keys[j] >> bit) & 1u) != 0u;
-            let destination = sort_split_destination(is_one, zeros_before, ones_before, total_zeros);
-
-            if (is_one) {
-                ones_before += 1u;
-            } else {
-                zeros_before += 1u;
-            }
-
-            wg_keys[destination] = keys[j];
-            wg_values[destination] = values[j];
-        }
-
-        workgroupBarrier();
-    }
-
-    // Exclusive prefix of the local histogram — for a tile now sorted by digit this is exactly the
-    // slot each bin's run begins at, and the total is the tile's real key count, because the
-    // padding was never counted.
-    let histogram_scan = sort_workgroup_exclusive_scan(local_count, thread_id, lane_id, lane_count);
-
-    let bin_start = histogram_scan.x;
-    let valid_count = histogram_scan.y;
-
-    let spine_inclusive = spine[sort_spine_index(thread_id, tile, settings.tile_count)];
-
-    wg_base[thread_id] = sort_scatter_base(spine_inclusive, local_count, bin_start);
-
-    workgroupBarrier();
-
-    for (var k = 0u; k < KEYS_PER_THREAD; k += 1u) {
-        let slot = k * WORKGROUP_SIZE + thread_id;
-
-        if (slot < valid_count) {
-            let key = wg_keys[slot];
-            let destination = wg_base[sort_extract_digit(key, shift)] + slot;
-
-            out_keys[destination] = key;
-            out_values[destination] = wg_values[slot];
-        }
-    }
+    sort_scatter_tile(
+        workgroup_id.x,
+        settings.element_count,
+        settings.tile_count,
+        settings.shift,
+        settings.key_mask,
+        thread_id,
+        lane_id,
+        lane_count
+    );
 }
 `, [
-    chunk_sort_digit,
-    chunk_sort_rank,
-    chunk_sort_workgroup_scan,
+    chunk_sort_scatter_tile,
 ]);
 
-body.addExtension(WebGPUExtensionType.Subgroups);
-
 /**
- * Rank-and-scatter — stage (c) of one digit pass of {@link graph_radix_sort}.
- *
- * One workgroup per tile. The tile is sorted by the current digit in workgroup memory (stability is
- * what an LSD radix sort's correctness rests on, and it is also what lets the tail tile's padding
- * be identified by position), the tile's own histogram is re-derived rather than carried over from
- * the histogram pass — the scan is inclusive-only, and re-counting is cheaper than preserving
- * pre-scan counts — and each key is written to
- * `spine_inclusive[bin] - local_count[bin] - bin_start[bin] + sorted_position`.
+ * Rank-and-scatter — stage (c) of one digit pass of {@link graph_radix_sort} over a host count, which
+ * it dispatches directly, a workgroup per tile. The tile's work is {@link chunk_sort_scatter_tile};
+ * {@link shader_radix_sort_scatter_indirect} is the same work over a device count.
  *
  * @type {ComputeShader}
  */

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter_indirect.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Rank-and-scatter — stage (c) of one digit pass of {@link graph_radix_sort} over a device count.
+ * Dispatched indirectly through the command {@link shader_radix_sort_prepare} folded over the live
+ * tile count, it finds its tile with `folded_workgroup_index`, and reads the live size that pass
+ * wrote. The tile's work is {@link chunk_sort_scatter_tile}, as it is
+ * {@link shader_radix_sort_scatter}'s.
+ *
+ * @type {ComputeShader}
+ */
+export const shader_radix_sort_scatter_indirect: ComputeShader;
+import { ComputeShader } from "../../shader/ComputeShader.js";
+//# sourceMappingURL=shader_radix_sort_scatter_indirect.d.ts.map

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter_indirect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shader_radix_sort_scatter_indirect.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter_indirect.js"],"names":[],"mappings":"AA4DA;;;;;;;;GAQG;AACH,iDAFU,aAAa,CAMpB;8BAvE2B,+BAA+B"}

package/src/shade/renderer/gpu_primitive/sort/shader_radix_sort_scatter_indirect.js

@@ -0,0 +1,74 @@
+import { chunk_folded_workgroup_index } from "../../shader/chunk/grid/chunk_folded_workgroup_index.js";
+import { CodeChunk } from "../../shader/compiler/CodeChunk.js";
+import { ComputeShader } from "../../shader/ComputeShader.js";
+import { ShaderResourceSetDescriptor } from "../../shader/resource/ShaderResourceSetDescriptor.js";
+import {
+    chunk_sort_scatter_tile,
+    SORT_SCATTER_IN_KEYS_RESOURCE,
+    SORT_SCATTER_IN_VALUES_RESOURCE,
+    SORT_SCATTER_OUT_KEYS_RESOURCE,
+    SORT_SCATTER_OUT_VALUES_RESOURCE,
+    SORT_SCATTER_SPINE_RESOURCE
+} from "./chunk_sort_scatter_tile.js";
+import { SORT_DIGIT_SETTINGS_STRUCT } from "./SORT_DIGIT_SETTINGS_STRUCT.js";
+import { SORT_SIZE_STRUCT } from "./SORT_SIZE_STRUCT.js";
+
+const resources = new ShaderResourceSetDescriptor();
+
+resources.setImmediate("settings", SORT_DIGIT_SETTINGS_STRUCT);
+
+resources.createGroup()
+    .add(SORT_SCATTER_IN_KEYS_RESOURCE)
+    .add(SORT_SCATTER_IN_VALUES_RESOURCE)
+    .add(SORT_SCATTER_OUT_KEYS_RESOURCE)
+    .add(SORT_SCATTER_OUT_VALUES_RESOURCE)
+    .add(SORT_SCATTER_SPINE_RESOURCE)
+    .addStorageBuffer("sort_size", SORT_SIZE_STRUCT, false);
+
+const body = CodeChunk.from(
+    //language=WGSL
+    `
+@compute @workgroup_size(SORT_WORKGROUP_SIZE, 1, 1)
+fn main(
+    @builtin(local_invocation_index) thread_id: u32,
+    @builtin(workgroup_id) workgroup_id: vec3<u32>,
+    @builtin(subgroup_invocation_id) lane_id: u32,
+    @builtin(subgroup_size) lane_count: u32,
+) {
+    let tile = folded_workgroup_index(workgroup_id);
+
+    // A folded grid's last row runs past the live tiles. Uniform, the whole workgroup leaves.
+    if (tile >= sort_size.tile_count) {
+        return;
+    }
+
+    sort_scatter_tile(
+        tile,
+        sort_size.element_count,
+        sort_size.tile_count,
+        settings.shift,
+        settings.key_mask,
+        thread_id,
+        lane_id,
+        lane_count
+    );
+}
+`, [
+    chunk_folded_workgroup_index,
+    chunk_sort_scatter_tile,
+]);
+
+/**
+ * Rank-and-scatter — stage (c) of one digit pass of {@link graph_radix_sort} over a device count.
+ * Dispatched indirectly through the command {@link shader_radix_sort_prepare} folded over the live
+ * tile count, it finds its tile with `folded_workgroup_index`, and reads the live size that pass
+ * wrote. The tile's work is {@link chunk_sort_scatter_tile}, as it is
+ * {@link shader_radix_sort_scatter}'s.
+ *
+ * @type {ComputeShader}
+ */
+export const shader_radix_sort_scatter_indirect = ComputeShader.from({
+    label: "RadixSort/Scatter (indirect)",
+    resources,
+    body,
+});

package/src/shade/renderer/gpu_primitive/sort/verify_radix_sort.d.ts

@@ -2,7 +2,9 @@
  * On-device verification of {@link graph_radix_sort}: sorts deterministic key/payload sets across
  * tile-boundary element counts and three key distributions, reads the results back, and checks on
  * the CPU that the keys are ascending, that the payload is a permutation of the input indices, that
- * every payload still sits next to ITS key, and that equal keys kept their input order.
+ * every payload still sits next to ITS key, and that equal keys kept their input order. Then the same
+ * by part of the key, with random bits above it, and over counts that live on the device — where the
+ * entries past the count must also come back as they went in.
  *
  * Logs one `[SORT]` PASS/FAIL line per case.
  *

package/src/shade/renderer/gpu_primitive/sort/verify_radix_sort.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"verify_radix_sort.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/verify_radix_sort.js"],"names":[],"mappings":"AA+GA;;;;;;;;;GASG;AACH,mEAkEC"}
+{"version":3,"file":"verify_radix_sort.d.ts","sourceRoot":"","sources":["../../../../../../src/shade/renderer/gpu_primitive/sort/verify_radix_sort.js"],"names":[],"mappings":"AA+QA;;;;;;;;;;;GAWG;AACH,mEAuGC"}