@woosh/meep-engine 3.29.0 → 3.31.0

package/src/shade/renderer/buffer/table/GPUDatabaseReadback.js

@@ -1,1323 +1,1332 @@
-import { assert } from "../../../../core/assert.js";
-import { round_up } from "../../../../core/math/round_up.js";
-import { GPU_DATABASE_BUFFER_ELEMENT_TYPE } from "./GPU_DATABASE_BUFFER_ELEMENT_TYPE.js";
-import { GPU_TABLE_GATHER_DESCRIPTOR_WORDS } from "./GPU_TABLE_GATHER_DESCRIPTOR_WORDS.js";
-import { gpu_table_gather_shader } from "./gpu_table_gather_shader.js";
-import { GPUReadbackStatus } from "./GPUReadbackStatus.js";
-
-/**
- * Bytes per {@link GPU_DATABASE_BUFFER_ELEMENT_TYPE}.
- * @type {number}
- */
-const WORD_SIZE = GPU_DATABASE_BUFFER_ELEMENT_TYPE.size;
-
-/**
- * Largest transient descriptor buffer one dispatch will be given, matching the upload path's.
- * @type {number}
- */
-const DESCRIPTOR_CHUNK_BYTES = 256 * 1024;
-
-/**
- * Granularity the gather scratch buffer grows by.
- * @type {number}
- */
-const GATHER_SIZE_GRANULARITY = 1024;
-
-/**
- * Requests at or above which a batch is gathered by a dispatch rather than copied range by range.
- *
- * **Measured, not chosen**, on a real device: Chrome 141 / Dawn, NVIDIA Lovelace, reading a 64-byte
- * range out of each of N 176-byte rows spread across 4 KiB pages. Best of five blocks of a hundred
- * batches, by `src/shade/playground/readback_crossover/`.
- *
- * | rows | encode copies | encode gather | round trip copies | round trip gather |
- * |-----:|--------------:|--------------:|------------------:|------------------:|
- * |    1 |       5.0 µs  |      17.0 µs  |            63 µs  |            79 µs  |
- * |    8 |       6.0 µs  |      25.0 µs  |            75 µs  |           125 µs  |
- * |   32 |      10.0 µs  |      27.0 µs  |           111 µs  |           130 µs  |
- * |   64 |      14.0 µs  |      33.0 µs  |           135 µs  |           129 µs  |
- * |  256 |      45.0 µs  |      53.0 µs  |           230 µs  |           116 µs  |
- * |  512 |      87.0 µs  |      77.0 µs  |           307 µs  |           115 µs  |
- *
- * Fitted: encoding costs `4.5 µs + 0.161 per row` for copies against `21.9 µs + 0.112` for the
- * gather, which crosses at **352 rows** — on CPU encode cost alone the gather is almost never worth
- * it, because appending a descriptor is not much cheaper than appending a copy command.
- *
- * The round trip is where the difference actually lives: `81 µs + 0.480 per row` for copies against
- * `104 µs + 0.037` for the gather. Submitting and executing N scattered copy commands costs about
- * half a microsecond each, and the gather is flat — 512 rows cost it no more than 8 do. Those cross
- * at **51 rows**, and the measured curves cross between 32 and 64. This is 48: inside the measured
- * interval, and on the side of the path whose cost does not grow.
- *
- * The fixed term of the round trip is mostly the wait for `onSubmittedWorkDone` and is not a cost a
- * frame pays synchronously; the slopes are the part that decides, and they differ by 13×.
- *
- * Those numbers were taken against the first, callback-and-objects version of this service. The
- * same page, re-run on the same adapter after the request became a few integer stores (two idle
- * runs, headless Chrome): encode `5.7–5.9 + 0.14 per row` for copies against `23.5–23.8 + 0.10` for
- * the gather; round trip `67–73 + 0.31` against `83–86 + 0.003–0.010`, crossing at 34 and 62 rows
- * with the curves crossing between 16 and 64. The per-row cost fell on both paths and the slopes
- * that decide are further apart than before; 48 is still inside the interval.
- *
- * @see https://claude.ai/code/artifact/88ee4932-ed98-407e-a25b-d379f3cf852a F5, which asked for this
- * number rather than for "batched is better"
- * @type {number}
- */
-const DEFAULT_GATHER_THRESHOLD = 48;
-
-/**
- * Batches allowed in flight before a frame's requests are refused. A mapping normally lands one or
- * two frames after its copy, so this is only reached when the device has stopped answering, and
- * the right thing to do with a frame's requests then is to say so and drop them: a consumer asks
- * every frame anyway, and a value delivered a dozen frames late is not the value it wanted.
- *
- * @type {number}
- */
-const DEFAULT_MAX_IN_FLIGHT = 4;
-
-/**
- * Request records and batch handle lists start at this many entries and double.
- * @type {number}
- */
-const INITIAL_CAPACITY = 64;
-
-/*
- * One request is RECORD_WORDS words of the flat record array, at handle * RECORD_WORDS. The first
- * six are written by `request`, the last two by `encode`.
- */
-const R_TABLE = 0;
-const R_ROW = 1;
-const R_WORD_OFFSET = 2;
-const R_WORD_COUNT = 3;
-const R_GENERATION = 4;
-const R_TARGET_OFFSET = 5;
-const R_SOURCE_WORD = 6;
-const R_DESTINATION_WORD = 7;
-const RECORD_WORDS = 8;
-
-/**
- * @enum {number}
- */
-const BatchState = {
-    /** taking requests; the one batch in this state is the frame's */
-    Assembling: 0,
-    /** copy recorded, the frame that recorded it not yet known to be submitted */
-    Encoded: 1,
-    /** `mapAsync` outstanding */
-    Mapping: 2,
-    /** mapped, waiting for an apply point */
-    Ready: 3,
-    /** the map was refused; the batch is discarded without delivering */
-    Failed: 4,
-};
-
-/**
- * One frame's worth of requests, travelling together. Pooled: a batch that has been delivered keeps
- * its arrays and takes the next frame's requests, so a steady stream of batches allocates nothing.
- */
-class GPUReadbackBatch {
-    /**
-     * Handles in request order.
-     * @type {Uint32Array}
-     */
-    handles = new Uint32Array(INITIAL_CAPACITY);
-
-    /**
-     * @type {number}
-     */
-    count = 0;
-
-    /**
-     * Requests whose row was resident when the batch was encoded.
-     * @type {number}
-     */
-    live = 0;
-
-    /**
-     * Bytes of the packed image.
-     * @type {number}
-     */
-    byte_size = 0;
-
-    /**
-     * The readback buffer the image was copied into. From {@link GPUBufferReadbackAllocator}, and
-     * given back when the batch is delivered or discarded.
-     *
-     * @type {GPUBuffer|null}
-     */
-    buffer = null;
-
-    /**
-     * The mapped image, valid from the mapping settling until the buffer is given back. Read
-     * directly at the apply point — the buffer stays mapped until then, which is what makes the
-     * copy into the caller's memory the only copy.
-     *
-     * @type {Uint8Array|null}
-     */
-    bytes = null;
-
-    /**
-     * The command context the copy was recorded on. The mapping may not start until it has been
-     * finished, because a map of a buffer whose copy has not been submitted reads the frame before.
-     *
-     * @type {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext|null}
-     */
-    context = null;
-
-    /**
-     * @type {BatchState}
-     */
-    state = BatchState.Assembling;
-
-    /**
-     * Live handles grouped by table, for the gather path's one dispatch per table.
-     * @type {Uint32Array}
-     */
-    order = new Uint32Array(0);
-
-    /**
-     * Reactions to this batch's mapping, bound once per pooled batch rather than once per frame.
-     * @type {function(): void}
-     */
-    on_mapped = null;
-
-    /**
-     * @type {function(): void}
-     */
-    on_failed = null;
-}
-
-/**
- * Reading rows back from a {@link GPUDatabase} in batches: one gather, one copy, one map, and one
- * point in the frame where the answers land — in memory the caller already owns.
- *
- * **Why this is not `debug_table_read_element`.** That resolves one row's byte offset and calls
- * {@link inspect_gpu_buffer_typed}, which creates a `MAP_READ` buffer, copies, submits, maps and
- * destroys it — per call. It is a debug affordance and stays one. Here the buffers persist, the
- * requests of a frame travel together, and the result arrives at a stated moment rather than
- * whenever a promise happens to settle.
- *
- * **The caller names memory, and nothing is called.** A request is a table handle, a row, a byte
- * range of the record, and a `Uint8Array` plus offset to put it in. At the apply point the service
- * copies the bytes there and marks the request's handle in {@link status}; the consumer reads the
- * status when its own system next runs, which is when it wanted the value anyway. There is no
- * callback and no promise, for the same reason: a function called per row is a closure allocated
- * per row and a permanently megamorphic call site, and code running inside the apply point could
- * re-enter this service, or destroy the database, from the middle of `GPUDatabase.update`. The
- * shader packs a batch contiguously in request order, so a consumer's run of rows lands in its
- * destination as one `set` — which a per-row function could never have done.
- *
- * **Gather, then one copy.** Rows sit at scattered offsets across pages, so a compute shader packs
- * the requested ranges contiguously before a single copy into a readback buffer. Packing a *range*
- * rather than a record is what makes it worth doing: an entity's world matrix is 64 bytes of a
- * 176-byte row.
- *
- * **But the request list is itself an upload.** A gather needs its descriptors on the GPU, so every
- * frame with any request writes a buffer, allocates, binds and dispatches. For one row that costs
- * far more than the value retrieved. Below {@link DEFAULT_GATHER_THRESHOLD} — a measured number, see
- * its docblock — the batch degrades to direct `copyBufferToBuffer` calls into the same readback
- * buffer, with no shader at all. A frame with no requests skips the whole path: no buffer, no
- * command.
- *
- * **One apply point, in encode order.** Results are delivered from {@link apply}, at the head of a
- * frame, and never on the timing of the map's own promise. A component that changed value in the
- * middle of an update because a readback happened to settle would be a different world for the
- * systems that ran before it and the ones that ran after. And a batch is delivered only after every
- * older batch has been: `mapAsync` promises are not promised to settle in the order the maps were
- * started, and a consumer asking for the same row every frame must never see a fresh value
- * overwritten by a stale one.
- *
- * **Nothing here may reach simulation state.** A readback is a frame or more late and settles on the
- * GPU's own schedule, so an outcome that depended on one would depend on frame pacing, and the
- * determinism gate with it. This answers tools, sounds and cosmetics. Where the simulation wants a
- * GPU-owned pose it evaluates the chain exactly, on the CPU.
- *
- * @see GPUStateAuthority which says whose value is worth reading back in the first place
- * @see GPUReadbackStatus
- */
-export class GPUDatabaseReadback {
-    /**
-     * @type {import("./GPUDatabase.js").GPUDatabase}
-     */
-    #database;
-
-    /**
-     * @type {GPUDevice}
-     */
-    #device;
-
-    /**
-     * Where readback buffers come from: the graphics context's, shared with every other database
-     * on it, and taken from the first command context this service encodes on — the database is
-     * built against a device, not a context.
-     *
-     * @type {import("../GPUBufferReadbackAllocator.js").GPUBufferReadbackAllocator|null}
-     */
-    #allocator = null;
-
-    /**
-     * Shader-writable scratch the gather packs into, copied to a readback buffer in one go.
-     * Persistent and grown, never per batch, and unlike the readback buffers it needs no pool,
-     * because it is written and read inside one submission.
-     *
-     * @type {GPUBuffer|null}
-     */
-    #gather_buffer = null;
-
-    /**
-     * Request records, {@link RECORD_WORDS} words per handle. A request is a few integer stores
-     * into this array; the target it names is the only thing that is not an integer, and it sits
-     * in {@link #targets} beside it.
-     *
-     * @type {Uint32Array}
-     */
-    #records = new Uint32Array(INITIAL_CAPACITY * RECORD_WORDS);
-
-    /**
-     * Per handle, the memory the bytes are to land in.
-     * @type {(Uint8Array|null)[]}
-     */
-    #targets = new Array(INITIAL_CAPACITY).fill(null);
-
-    /**
-     * Per handle, a {@link GPUReadbackStatus}. Exposed as {@link status}.
-     * @type {Uint8Array}
-     */
-    #status = new Uint8Array(INITIAL_CAPACITY);
-
-    /**
-     * Recycled handles. Never longer than the record capacity, so it never overflows.
-     * @type {Uint32Array}
-     */
-    #free = new Uint32Array(INITIAL_CAPACITY);
-
-    /**
-     * @type {number}
-     */
-    #free_count = 0;
-
-    /**
-     * Handles below this have been handed out at least once.
-     * @type {number}
-     */
-    #fresh = 0;
-
-    /**
-     * The batch taking this frame's requests.
-     * @type {GPUReadbackBatch}
-     */
-    #assembling;
-
-    /**
-     * Encoded batches, oldest first.
-     * @type {GPUReadbackBatch[]}
-     */
-    #in_flight = [];
-
-    /**
-     * Batches delivered or dropped since the last apply point. Their handles stay readable until
-     * the next one, and are recycled there.
-     *
-     * @type {GPUReadbackBatch[]}
-     */
-    #retiring = [];
-
-    /**
-     * Batches with nothing in them, waiting to be the next frame's.
-     * @type {GPUReadbackBatch[]}
-     */
-    #spare = [];
-
-    /**
-     * Scratch for bucketing a batch's live requests by table: counts, then starts.
-     * @type {Uint32Array}
-     */
-    #table_starts;
-
-    /**
-     * @type {number}
-     */
-    #gather_threshold;
-
-    /**
-     * @type {number}
-     */
-    #max_in_flight;
-
-    /**
-     * Descriptor words for one dispatch, reused. Grown to the largest chunk this service has ever
-     * needed and kept.
-     *
-     * @type {Uint32Array}
-     */
-    #descriptor_scratch = new Uint32Array(0);
-
-    /**
-     * Immediate data for the gather dispatch, reused. `dispatchThreads` serialises it at the call,
-     * so one object can carry every dispatch of a frame.
-     *
-     * @type {Object}
-     */
-    #gather_settings = {
-        count: 0,
-        record_size: 0,
-        elements_per_page: 0,
-        page_lookup_address: 0,
-        page_header_words: 0,
-        page_limit: 0,
-    };
-
-    /**
-     * @type {boolean}
-     */
-    #destroyed = false;
-
-    #copies_encoded = 0;
-    #dispatches_encoded = 0;
-    #delivered_count = 0;
-    #dropped_count = 0;
-    #stalled_count = 0;
-
-    /**
-     * @param {Object} options
-     * @param {import("./GPUDatabase.js").GPUDatabase} options.database
-     * @param {GPUDevice} options.device
-     * @param {number} [options.gather_threshold] requests at or above which a dispatch is used
-     * @param {number} [options.max_in_flight] batches in flight before a frame's requests are dropped
-     * @param {string} [options.label]
-     */
-    constructor({
-                    database,
-                    device,
-                    gather_threshold = DEFAULT_GATHER_THRESHOLD,
-                    max_in_flight = DEFAULT_MAX_IN_FLIGHT,
-                    label = 'GPUDatabaseReadback',
-                }) {
-        assert.defined(database, 'database');
-        assert.defined(device, 'device');
-        assert.isNonNegativeInteger(gather_threshold, 'gather_threshold');
-        assert.isPositiveInteger(max_in_flight, 'max_in_flight');
-        assert.isString(label, 'label');
-
-        this.#database = database;
-        this.#device = device;
-        this.#gather_threshold = gather_threshold;
-        this.#max_in_flight = max_in_flight;
-
-        this.#table_starts = new Uint32Array(database.tables.length + 1);
-
-        this.#assembling = this.#make_batch();
-    }
-
-    /**
-     * Requests at or above which a batch is gathered rather than copied range by range.
-     * @returns {number}
-     */
-    get gather_threshold() {
-        return this.#gather_threshold;
-    }
-
-    /**
-     * Batches in flight before a frame's requests are refused.
-     * @returns {number}
-     */
-    get max_in_flight() {
-        return this.#max_in_flight;
-    }
-
-    /**
-     * Per handle, what became of the request — a {@link GPUReadbackStatus}.
-     *
-     * Read it fresh each time rather than keeping a reference: the array is replaced when the
-     * handle space grows. A status is readable for one frame, from the apply point that set it to
-     * the next.
-     *
-     * @returns {Uint8Array}
-     */
-    get status() {
-        return this.#status;
-    }
-
-    /**
-     * How many handles exist. Grows to the working set of a steady stream of requests and stops:
-     * handles are recycled a frame after their result lands.
-     *
-     * @returns {number}
-     */
-    get handle_capacity() {
-        return this.#status.length;
-    }
-
-    /**
-     * Requests made and not yet encoded.
-     * @returns {number}
-     */
-    get pending_count() {
-        return this.#assembling.count;
-    }
-
-    /**
-     * Batches encoded and not yet delivered or discarded.
-     * @returns {number}
-     */
-    get in_flight_count() {
-        return this.#in_flight.length;
-    }
-
-    /**
-     * How many `copyBufferToBuffer` calls this service has encoded, over its life. Read by profiles,
-     * and by the specs that assert an empty frame encodes nothing.
-     *
-     * @returns {number}
-     */
-    get copies_encoded() {
-        return this.#copies_encoded;
-    }
-
-    /**
-     * How many gather dispatches this service has encoded, over its life.
-     * @returns {number}
-     */
-    get dispatches_encoded() {
-        return this.#dispatches_encoded;
-    }
-
-    /**
-     * Requests whose bytes reached their target.
-     * @returns {number}
-     */
-    get delivered_count() {
-        return this.#delivered_count;
-    }
-
-    /**
-     * Requests thrown away because of the row — its page was not resident when the batch was
-     * encoded, it was not occupied, or it was freed before the answer landed.
-     *
-     * Reported rather than swallowed: a consumer whose readbacks are all being dropped is asking
-     * about rows that keep dying, and silence reads downstream as "the value did not change".
-     *
-     * @returns {number}
-     */
-    get dropped_count() {
-        return this.#dropped_count;
-    }
-
-    /**
-     * Requests thrown away because of the pipeline — {@link max_in_flight} batches were still
-     * waiting on the device when the frame's batch was due, or a mapping was refused. Separate
-     * from {@link dropped_count} because the cure is different: those rows died, these the device
-     * stopped answering about.
-     *
-     * @returns {number}
-     */
-    get stalled_count() {
-        return this.#stalled_count;
-    }
-
-    /**
-     * Resolve a table name to the handle {@link request} takes. Once, at setup: a string lookup
-     * per row per frame is what this exists to avoid.
-     *
-     * From here on the table counts how many times each of its rows is freed, which is what lets
-     * an answer about a row that died and was reused be told from an answer about the row asked
-     * for.
-     *
-     * @param {string} name
-     * @returns {number}
-     */
-    table(name) {
-        assert.isString(name, 'name');
-
-        const table = this.#database.get(name);
-
-        if (table === undefined) {
-            throw new Error(`Table '${name}' does not exist`);
-        }
-
-        table.track_generations();
-
-        return table.descriptor.index;
-    }
-
-    /**
-     * Ask for a byte range of a row to be copied into `target`, at a later apply point.
-     *
-     * Positional, and a few integer stores: this is called once per row per frame by anything that
-     * tracks rows, so it takes no options object and allocates nothing.
-     *
-     * @param {number} table from {@link table}
-     * @param {number} row
-     * @param {number} byte_offset within the record, word-aligned
-     * @param {number} byte_size a positive multiple of the word size
-     * @param {Uint8Array} target memory the caller owns and keeps; the bytes are written here at
-     *        the apply point. A byte view on purpose — a consumer that reads floats holds a
-     *        `Float32Array` over the same buffer, made once
-     * @param {number} [target_byte_offset]
-     * @returns {number} a handle into {@link status}, valid until the frame after its result lands
-     */
-    request(table, row, byte_offset, byte_size, target, target_byte_offset = 0) {
-        assert.isNonNegativeInteger(table, 'table');
-        assert.isNonNegativeInteger(row, 'row');
-        assert.isNonNegativeInteger(byte_offset, 'byte_offset');
-        assert.isPositiveInteger(byte_size, 'byte_size');
-        assert.isNonNegativeInteger(target_byte_offset, 'target_byte_offset');
-
-        const tables = this.#database.tables;
-
-        assert.lessThan(table, tables.length, 'table is not a handle from table()');
-        assert.equal(byte_offset % WORD_SIZE, 0, 'byte_offset must be word-aligned');
-        assert.equal(byte_size % WORD_SIZE, 0, 'byte_size must be a whole number of words');
-        assert.ok(target instanceof Uint8Array, 'target must be a Uint8Array');
-
-        // constant messages: these run per row per frame, and a template literal built to describe a
-        // failure that does not happen is the cost on the path being measured
-        assert.lessThanOrEqual(byte_offset + byte_size, tables[table].descriptor.packed_element_size_bytes, 'range reaches past the record');
-        assert.lessThanOrEqual(target_byte_offset + byte_size, target.length, 'range reaches past the target');
-
-        const handle = this.#allocate_handle();
-        const base = handle * RECORD_WORDS;
-        const records = this.#records;
-
-        records[base + R_TABLE] = table;
-        records[base + R_ROW] = row;
-        records[base + R_WORD_OFFSET] = byte_offset / WORD_SIZE;
-        records[base + R_WORD_COUNT] = byte_size / WORD_SIZE;
-        records[base + R_GENERATION] = tables[table].generation_of(row);
-        records[base + R_TARGET_OFFSET] = target_byte_offset;
-
-        this.#targets[handle] = target;
-        this.#status[handle] = GPUReadbackStatus.Pending;
-
-        const batch = this.#assembling;
-
-        if (batch.count === batch.handles.length) {
-            const grown = new Uint32Array(batch.handles.length * 2);
-
-            grown.set(batch.handles);
-
-            batch.handles = grown;
-        }
-
-        batch.handles[batch.count++] = handle;
-
-        return handle;
-    }
-
-    /**
-     * Deliver every batch whose bytes have arrived, oldest first and stopping at the first that has
-     * not, then start the map for every batch whose frame has been submitted.
-     *
-     * **This is the apply point.** Call it at the head of a frame, before any system runs. A batch
-     * therefore reaches its targets at the first frame head at which its mapping — and every older
-     * batch's — has resolved, which is the frame after the one that encoded it at the earliest:
-     * never inside an update, and never at two different moments for two requests of the same
-     * batch. No caller-supplied code runs here.
-     *
-     * @returns {number} requests delivered
-     */
-    apply() {
-        if (this.#destroyed) {
-            return 0;
-        }
-
-        // The handles delivered or dropped at the previous apply point have had their frame of
-        // being readable; they go back to the free list before anything is delivered, so nothing
-        // delivered now can be recycled before it has been read.
-        const retiring = this.#retiring;
-
-        for (let i = 0; i < retiring.length; i++) {
-            this.#recycle(retiring[i]);
-        }
-
-        retiring.length = 0;
-
-        const in_flight = this.#in_flight;
-
-        let delivered = 0;
-
-        while (in_flight.length > 0) {
-            const batch = in_flight[0];
-
-            if (batch.state === BatchState.Ready) {
-                delivered += this.#deliver(batch);
-            } else if (batch.state === BatchState.Failed) {
-                this.#drop_pending(batch, true);
-            } else {
-                // still on the device, and nothing younger may overtake it
-                break;
-            }
-
-            this.#allocator.release(batch.buffer);
-
-            batch.buffer = null;
-            batch.bytes = null;
-            batch.context = null;
-
-            in_flight.shift();
-            retiring.push(batch);
-        }
-
-        // Start the maps only now, and only for batches whose frame has been submitted. `encode`
-        // records a copy into a command context the caller has not finished yet, and mapping a
-        // buffer whose copy has not been submitted reads the frame before.
-        for (let i = 0; i < in_flight.length; i++) {
-            const batch = in_flight[i];
-
-            if (batch.state === BatchState.Encoded && batch.context.finished) {
-                this.#start_map(batch);
-            }
-        }
-
-        return delivered;
-    }
-
-    /**
-     * Record this frame's batch: a gather and one copy, or a handful of copies, or nothing at all.
-     *
-     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
-     * @returns {boolean} whether anything was encoded
-     */
-    encode(encoder) {
-        if (this.#destroyed) {
-            return false;
-        }
-
-        const batch = this.#assembling;
-
-        if (batch.count === 0) {
-            // The point of the whole arrangement: a frame nobody asked anything of costs one
-            // comparison. No buffer is taken, no command is recorded.
-            return false;
-        }
-
-        if (this.#allocator === null) {
-            const allocator = encoder.graphics.buffer_allocator_readback;
-
-            assert.defined(allocator, 'encoder.graphics.buffer_allocator_readback');
-
-            this.#allocator = allocator;
-        }
-
-        if (this.#in_flight.length >= this.#max_in_flight) {
-            // The device has stopped answering. Refusing the frame's requests, with a status the
-            // consumer can see, bounds everything: nothing queues, nothing is re-resolved next
-            // frame, and no buffer is taken for bytes that would be a dozen frames stale.
-            this.#drop_pending(batch, true);
-            this.#retire(batch);
-
-            return false;
-        }
-
-        const live = this.#resolve(batch);
-
-        if (live === 0) {
-            // every row asked about has gone; nothing to copy and nothing to deliver
-            this.#retire(batch);
-
-            return false;
-        }
-
-        batch.buffer = this.#allocator.get(batch.byte_size);
-
-        if (live < this.#gather_threshold) {
-            this.#encode_copies(encoder, batch);
-        } else {
-            this.#encode_gather(encoder, batch);
-        }
-
-        batch.context = encoder;
-        batch.state = BatchState.Encoded;
-
-        this.#in_flight.push(batch);
-        this.#assembling = this.#take_batch();
-
-        return true;
-    }
-
-    /**
-     * Where each request's range starts in the database buffer and where it lands in the image,
-     * for every request whose row is there to be read. A request whose page is not resident, or
-     * whose row is not occupied, is dropped here: "the page was gone" is a fact about the moment
-     * the batch was encoded and cannot be recovered from the bytes afterwards.
-     *
-     * The same arithmetic {@link GPUDatabase#debug_table_read_element} does, and the same the gather
-     * shader does — the CPU needs it for the copy path, and the residency answer for both.
-     *
-     * One pass, in request order, so the image is packed in request order and a consumer's run of
-     * consecutive rows is one contiguous span of it. Requests cluster by table and by page, so the
-     * table and the page are looked up when they change rather than per request. Live requests are
-     * counted per table on the way through, which is all the gather path's bucketing needs.
-     *
-     * @param {GPUReadbackBatch} batch
-     * @returns {number} live requests
-     */
-    #resolve(batch) {
-        const database = this.#database;
-        const tables = database.tables;
-        const data_start = database.data_start_offset;
-
-        const records = this.#records;
-        const status = this.#status;
-        const handles = batch.handles;
-        const count = batch.count;
-
-        const starts = this.#table_starts;
-
-        starts.fill(0);
-
-        let destination = 0;
-        let live = 0;
-        let dropped = 0;
-
-        let table_index = -1;
-        let table = null;
-        let descriptor = null;
-        let elements_per_page = 0;
-        let record_words = 0;
-
-        let page_index = -1;
-        let page_base = -1;
-
-        for (let i = 0; i < count; i++) {
-            const handle = handles[i];
-            const base = handle * RECORD_WORDS;
-
-            const t = records[base + R_TABLE];
-            const row = records[base + R_ROW];
-
-            if (t !== table_index) {
-                table_index = t;
-                table = tables[t];
-                descriptor = table.descriptor;
-                elements_per_page = descriptor.elements_per_page;
-                record_words = descriptor.packed_element_size_bytes / WORD_SIZE;
-                page_index = -1;
-            }
-
-            const p = Math.floor(row / elements_per_page);
-
-            if (p !== page_index) {
-                page_index = p;
-
-                const page = table.pages.get(p);
-
-                page_base = (page === undefined || page.slot_offset === -1)
-                    ? -1
-                    : page.slot_offset + data_start + descriptor.page_header_words;
-            }
-
-            if (page_base === -1 || !table.occupancy.get(row)) {
-                // the page is not on the GPU, or the row was removed and its bytes are whatever the
-                // last occupant left
-                status[handle] = GPUReadbackStatus.Dropped;
-                dropped++;
-
-                continue;
-            }
-
-            records[base + R_SOURCE_WORD] = page_base
-                + (row - page_index * elements_per_page) * record_words
-                + records[base + R_WORD_OFFSET];
-            records[base + R_DESTINATION_WORD] = destination;
-
-            destination += records[base + R_WORD_COUNT];
-
-            live++;
-            starts[t + 1]++;
-        }
-
-        this.#dropped_count += dropped;
-
-        batch.live = live;
-        batch.byte_size = destination * WORD_SIZE;
-
-        return live;
-    }
-
-    /**
-     * Copy each range straight out of the database into its place in the image. No shader, no
-     * descriptor upload, no bind group — which below the threshold is cheaper than all three.
-     *
-     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
-     * @param {GPUReadbackBatch} batch
-     */
-    #encode_copies(encoder, batch) {
-        const source = this.#database.buffer;
-        const destination = batch.buffer;
-
-        const records = this.#records;
-        const status = this.#status;
-        const handles = batch.handles;
-        const count = batch.count;
-
-        let copies = 0;
-
-        for (let i = 0; i < count; i++) {
-            const handle = handles[i];
-
-            if (status[handle] !== GPUReadbackStatus.Pending) {
-                continue;
-            }
-
-            const base = handle * RECORD_WORDS;
-
-            encoder.copyBufferToBuffer(
-                source,
-                records[base + R_SOURCE_WORD] * WORD_SIZE,
-                destination,
-                records[base + R_DESTINATION_WORD] * WORD_SIZE,
-                records[base + R_WORD_COUNT] * WORD_SIZE,
-            );
-
-            copies++;
-        }
-
-        this.#copies_encoded += copies;
-    }
-
-    /**
-     * Pack every range into the scratch buffer with one dispatch per table, then copy the whole
-     * image across in one go.
-     *
-     * One dispatch per table rather than one for the batch, because the page geometry a thread needs
-     * — record size, elements per page, where the table's page addresses live — is per table, and
-     * carrying it per descriptor would cost more words than the descriptor has.
-     *
-     * The live handles are bucketed by table with a counting sort off the per-table counts
-     * {@link #resolve} left behind: one pass to scatter, then each table's run is emitted in turn.
-     * Linear in the requests plus the tables, where scanning every request once per table was
-     * their product.
-     *
-     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
-     * @param {GPUReadbackBatch} batch
-     */
-    #encode_gather(encoder, batch) {
-        const live = batch.live;
-        const byte_size = batch.byte_size;
-
-        const gather_buffer = this.#fit_gather_buffer(byte_size);
-
-        const tables = this.#database.tables;
-        const table_count = tables.length;
-
-        const records = this.#records;
-        const status = this.#status;
-        const handles = batch.handles;
-        const count = batch.count;
-
-        // counts to starts
-        const starts = this.#table_starts;
-
-        for (let t = 1; t <= table_count; t++) {
-            starts[t] += starts[t - 1];
-        }
-
-        if (batch.order.length < live) {
-            batch.order = new Uint32Array(Math.max(live, batch.order.length * 2));
-        }
-
-        const order = batch.order;
-
-        // scatter, in request order within each table; afterwards starts[t] is the end of table t
-        for (let i = 0; i < count; i++) {
-            const handle = handles[i];
-
-            if (status[handle] !== GPUReadbackStatus.Pending) {
-                continue;
-            }
-
-            order[starts[records[handle * RECORD_WORDS + R_TABLE]]++] = handle;
-        }
-
-        const descriptor_stride = GPU_TABLE_GATHER_DESCRIPTOR_WORDS;
-        const max_per_chunk = Math.max(1, Math.floor(DESCRIPTOR_CHUNK_BYTES / (descriptor_stride * WORD_SIZE)));
-
-        const scratch_words = Math.min(live, max_per_chunk) * descriptor_stride;
-
-        if (this.#descriptor_scratch.length < scratch_words) {
-            this.#descriptor_scratch = new Uint32Array(scratch_words);
-        }
-
-        const scratch = this.#descriptor_scratch;
-        const settings = this.#gather_settings;
-
-        let run_start = 0;
-
-        for (let t = 0; t < table_count; t++) {
-            const run_end = starts[t];
-
-            if (run_end === run_start) {
-                continue;
-            }
-
-            const table_descriptor = tables[t].descriptor;
-
-            settings.record_size = table_descriptor.packed_element_size_bytes / WORD_SIZE;
-            settings.elements_per_page = table_descriptor.elements_per_page;
-            settings.page_lookup_address = table_descriptor.page_lookup_address;
-            settings.page_header_words = table_descriptor.page_header_words;
-            settings.page_limit = table_descriptor.page_limit;
-
-            let filled = 0;
-
-            for (let i = run_start; i < run_end; i++) {
-                const base = order[i] * RECORD_WORDS;
-                const d = filled * descriptor_stride;
-
-                scratch[d] = records[base + R_ROW];
-                scratch[d + 1] = records[base + R_WORD_OFFSET];
-                scratch[d + 2] = records[base + R_WORD_COUNT];
-                scratch[d + 3] = records[base + R_DESTINATION_WORD];
-
-                filled++;
-
-                if (filled === max_per_chunk) {
-                    this.#dispatch_gather(encoder, scratch, filled, settings, gather_buffer);
-                    filled = 0;
-                }
-            }
-
-            if (filled > 0) {
-                this.#dispatch_gather(encoder, scratch, filled, settings, gather_buffer);
-            }
-
-            run_start = run_end;
-        }
-
-        encoder.copyBufferToBuffer(gather_buffer, 0, batch.buffer, 0, byte_size);
-
-        this.#copies_encoded++;
-    }
-
-    /**
-     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
-     * @param {Uint32Array} scratch
-     * @param {number} count
-     * @param {Object} settings
-     * @param {GPUBuffer} gather_buffer
-     */
-    #dispatch_gather(encoder, scratch, count, settings, gather_buffer) {
-        settings.count = count;
-
-        const descriptors = encoder.allocateTransientBufferAndLoad(
-            scratch.buffer,
-            GPUBufferUsage.STORAGE,
-            0,
-            count * GPU_TABLE_GATHER_DESCRIPTOR_WORDS * WORD_SIZE,
-        );
-
-        gpu_table_gather_shader.dispatchThreads({
-            encoder,
-            bindings: {
-                settings,
-                descriptors,
-                database: this.#database.buffer,
-                gathered: gather_buffer,
-            },
-            thread_count_x: count,
-        });
-
-        this.#dispatches_encoded++;
-    }
-
-    /**
-     * @param {number} byte_size
-     * @returns {GPUBuffer}
-     */
-    #fit_gather_buffer(byte_size) {
-        const existing = this.#gather_buffer;
-
-        if (existing !== null && existing.size >= byte_size) {
-            return existing;
-        }
-
-        if (existing !== null) {
-            existing.destroy();
-        }
-
-        const buffer = this.#device.createBuffer({
-            label: 'GPUDatabaseReadback/gather',
-            size: round_up(byte_size, GATHER_SIZE_GRANULARITY),
-            usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
-            mappedAtCreation: false,
-        });
-
-        this.#gather_buffer = buffer;
-
-        return buffer;
-    }
-
-    /**
-     * @param {GPUReadbackBatch} batch
-     */
-    #start_map(batch) {
-        batch.state = BatchState.Mapping;
-
-        batch.buffer.mapAsync(GPUMapMode.READ, 0, batch.byte_size).then(batch.on_mapped, batch.on_failed);
-    }
-
-    /**
-     * Copy a mapped batch into its targets.
-     *
-     * The image is packed in request order, so consecutive live requests are consecutive in it;
-     * when they also name consecutive bytes of one target, the run is one `set`. A consumer that
-     * asks for a hundred rows into one array gets one copy, not a hundred.
-     *
-     * @param {GPUReadbackBatch} batch
-     * @returns {number} requests delivered
-     */
-    #deliver(batch) {
-        const records = this.#records;
-        const status = this.#status;
-        const targets = this.#targets;
-        const tables = this.#database.tables;
-
-        const bytes = batch.bytes;
-        const handles = batch.handles;
-        const count = batch.count;
-
-        let delivered = 0;
-        let dropped = 0;
-
-        let run_target = null;
-        let run_source = 0;
-        let run_target_offset = 0;
-        let run_length = 0;
-
-        for (let i = 0; i < count; i++) {
-            const handle = handles[i];
-
-            if (status[handle] !== GPUReadbackStatus.Pending) {
-                // dropped at encode; it has no bytes in the image
-                continue;
-            }
-
-            const base = handle * RECORD_WORDS;
-            const row = records[base + R_ROW];
-
-            if (tables[records[base + R_TABLE]].generation_of(row) !== records[base + R_GENERATION]) {
-                // the row was freed while this was in flight, and an id freed is an id already
-                // handed out again — answering would be answering about a stranger
-                status[handle] = GPUReadbackStatus.Dropped;
-                dropped++;
-
-                continue;
-            }
-
-            const source = records[base + R_DESTINATION_WORD] * WORD_SIZE;
-            const length = records[base + R_WORD_COUNT] * WORD_SIZE;
-            const target = targets[handle];
-            const target_offset = records[base + R_TARGET_OFFSET];
-
-            if (
-                target === run_target
-                && target_offset === run_target_offset + run_length
-                && source === run_source + run_length
-            ) {
-                run_length += length;
-            } else {
-                if (run_length > 0) {
-                    run_target.set(bytes.subarray(run_source, run_source + run_length), run_target_offset);
-                }
-
-                run_target = target;
-                run_source = source;
-                run_target_offset = target_offset;
-                run_length = length;
-            }
-
-            status[handle] = GPUReadbackStatus.Delivered;
-            delivered++;
-        }
-
-        if (run_length > 0) {
-            run_target.set(bytes.subarray(run_source, run_source + run_length), run_target_offset);
-        }
-
-        this.#delivered_count += delivered;
-        this.#dropped_count += dropped;
-
-        return delivered;
-    }
-
-    /**
-     * Mark every request of a batch still waiting as dropped.
-     *
-     * @param {GPUReadbackBatch} batch
-     * @param {boolean} stalled whether the pipeline, rather than the rows, is the reason
-     */
-    #drop_pending(batch, stalled) {
-        const status = this.#status;
-        const handles = batch.handles;
-        const count = batch.count;
-
-        let dropped = 0;
-
-        for (let i = 0; i < count; i++) {
-            const handle = handles[i];
-
-            if (status[handle] === GPUReadbackStatus.Pending) {
-                status[handle] = GPUReadbackStatus.Dropped;
-                dropped++;
-            }
-        }
-
-        if (stalled) {
-            this.#stalled_count += dropped;
-        } else {
-            this.#dropped_count += dropped;
-        }
-    }
-
-    /**
-     * Send the assembling batch straight to retiring — nothing of it is in flight — and take a
-     * fresh one for the requests that follow.
-     *
-     * @param {GPUReadbackBatch} batch
-     */
-    #retire(batch) {
-        this.#retiring.push(batch);
-        this.#assembling = this.#take_batch();
-    }
-
-    /**
-     * Free a retired batch's handles and put the batch back in the pool.
-     *
-     * @param {GPUReadbackBatch} batch
-     */
-    #recycle(batch) {
-        const status = this.#status;
-        const targets = this.#targets;
-        const free = this.#free;
-        const handles = batch.handles;
-        const count = batch.count;
-
-        let free_count = this.#free_count;
-
-        for (let i = 0; i < count; i++) {
-            const handle = handles[i];
-
-            status[handle] = GPUReadbackStatus.None;
-            targets[handle] = null;
-
-            free[free_count++] = handle;
-        }
-
-        this.#free_count = free_count;
-
-        batch.count = 0;
-        batch.live = 0;
-        batch.byte_size = 0;
-        batch.state = BatchState.Assembling;
-
-        this.#spare.push(batch);
-    }
-
-    /**
-     * @returns {GPUReadbackBatch}
-     */
-    #take_batch() {
-        const spare = this.#spare;
-
-        if (spare.length > 0) {
-            return spare.pop();
-        }
-
-        return this.#make_batch();
-    }
-
-    /**
-     * @returns {GPUReadbackBatch}
-     */
-    #make_batch() {
-        const batch = new GPUReadbackBatch();
-
-        batch.on_mapped = () => {
-            if (this.#destroyed || batch.state !== BatchState.Mapping) {
-                // torn down while the map was out; the buffer went with it
-                return;
-            }
-
-            batch.bytes = new Uint8Array(batch.buffer.getMappedRange(0, batch.byte_size));
-            batch.state = BatchState.Ready;
-        };
-
-        batch.on_failed = () => {
-            if (batch.state === BatchState.Mapping) {
-                // device lost, buffer destroyed — discarded at the next apply point
-                batch.state = BatchState.Failed;
-            }
-        };
-
-        return batch;
-    }
-
-    /**
-     * @returns {number}
-     */
-    #allocate_handle() {
-        if (this.#free_count > 0) {
-            return this.#free[--this.#free_count];
-        }
-
-        if (this.#fresh === this.#status.length) {
-            this.#grow_handles();
-        }
-
-        return this.#fresh++;
-    }
-
-    /**
-     * Double the handle space. Handles are indices, so nothing moves; a caller keeps the ones it
-     * has.
-     */
-    #grow_handles() {
-        const capacity = this.#status.length * 2;
-
-        const records = new Uint32Array(capacity * RECORD_WORDS);
-        records.set(this.#records);
-        this.#records = records;
-
-        const status = new Uint8Array(capacity);
-        status.set(this.#status);
-        this.#status = status;
-
-        const free = new Uint32Array(capacity);
-        free.set(this.#free);
-        this.#free = free;
-
-        const targets = this.#targets;
-
-        for (let i = targets.length; i < capacity; i++) {
-            targets.push(null);
-        }
-    }
-
-    destroy() {
-        this.#destroyed = true;
-
-        const in_flight = this.#in_flight;
-
-        for (let i = 0; i < in_flight.length; i++) {
-            const batch = in_flight[i];
-
-            if (batch.buffer !== null) {
-                // may be mid-map, which WebGPU lets destroy() abort but not unmap()
-                this.#allocator.discard(batch.buffer);
-
-                batch.buffer = null;
-            }
-        }
-
-        in_flight.length = 0;
-        this.#retiring.length = 0;
-        this.#spare.length = 0;
-        this.#assembling.count = 0;
-
-        if (this.#gather_buffer !== null) {
-            this.#gather_buffer.destroy();
-            this.#gather_buffer = null;
-        }
-    }
-}
+import { assert } from "../../../../core/assert.js";
+import { round_up } from "../../../../core/math/round_up.js";
+import { GPU_DATABASE_BUFFER_ELEMENT_TYPE } from "./GPU_DATABASE_BUFFER_ELEMENT_TYPE.js";
+import { GPU_TABLE_GATHER_DESCRIPTOR_WORDS } from "./GPU_TABLE_GATHER_DESCRIPTOR_WORDS.js";
+import { gpu_table_gather_shader } from "./gpu_table_gather_shader.js";
+import { GPUReadbackStatus } from "./GPUReadbackStatus.js";
+
+/**
+ * Bytes per {@link GPU_DATABASE_BUFFER_ELEMENT_TYPE}.
+ * @type {number}
+ */
+const WORD_SIZE = GPU_DATABASE_BUFFER_ELEMENT_TYPE.size;
+
+/**
+ * Largest transient descriptor buffer one dispatch will be given, matching the upload path's.
+ * @type {number}
+ */
+const DESCRIPTOR_CHUNK_BYTES = 256 * 1024;
+
+/**
+ * Granularity the gather scratch buffer grows by.
+ * @type {number}
+ */
+const GATHER_SIZE_GRANULARITY = 1024;
+
+/**
+ * Requests at or above which a batch is gathered by a dispatch rather than copied range by range.
+ *
+ * **Measured, not chosen**, on a real device: Chrome 141 / Dawn, NVIDIA Lovelace, reading a 64-byte
+ * range out of each of N 176-byte rows spread across 4 KiB pages. Best of five blocks of a hundred
+ * batches, by `src/shade/playground/readback_crossover/`.
+ *
+ * | rows | encode copies | encode gather | round trip copies | round trip gather |
+ * |-----:|--------------:|--------------:|------------------:|------------------:|
+ * |    1 |       5.0 µs  |      17.0 µs  |            63 µs  |            79 µs  |
+ * |    8 |       6.0 µs  |      25.0 µs  |            75 µs  |           125 µs  |
+ * |   32 |      10.0 µs  |      27.0 µs  |           111 µs  |           130 µs  |
+ * |   64 |      14.0 µs  |      33.0 µs  |           135 µs  |           129 µs  |
+ * |  256 |      45.0 µs  |      53.0 µs  |           230 µs  |           116 µs  |
+ * |  512 |      87.0 µs  |      77.0 µs  |           307 µs  |           115 µs  |
+ *
+ * Fitted: encoding costs `4.5 µs + 0.161 per row` for copies against `21.9 µs + 0.112` for the
+ * gather, which crosses at **352 rows** — on CPU encode cost alone the gather is almost never worth
+ * it, because appending a descriptor is not much cheaper than appending a copy command.
+ *
+ * The round trip is where the difference actually lives: `81 µs + 0.480 per row` for copies against
+ * `104 µs + 0.037` for the gather. Submitting and executing N scattered copy commands costs about
+ * half a microsecond each, and the gather is flat — 512 rows cost it no more than 8 do. Those cross
+ * at **51 rows**, and the measured curves cross between 32 and 64. This is 48: inside the measured
+ * interval, and on the side of the path whose cost does not grow.
+ *
+ * The fixed term of the round trip is mostly the wait for `onSubmittedWorkDone` and is not a cost a
+ * frame pays synchronously; the slopes are the part that decides, and they differ by 13×.
+ *
+ * Those numbers were taken against the first, callback-and-objects version of this service. The
+ * same page, re-run on the same adapter after the request became a few integer stores (two idle
+ * runs, headless Chrome): encode `5.7–5.9 + 0.14 per row` for copies against `23.5–23.8 + 0.10` for
+ * the gather; round trip `67–73 + 0.31` against `83–86 + 0.003–0.010`, crossing at 34 and 62 rows
+ * with the curves crossing between 16 and 64. The per-row cost fell on both paths and the slopes
+ * that decide are further apart than before; 48 is still inside the interval.
+ *
+ * @see https://claude.ai/code/artifact/88ee4932-ed98-407e-a25b-d379f3cf852a F5, which asked for this
+ * number rather than for "batched is better"
+ * @type {number}
+ */
+const DEFAULT_GATHER_THRESHOLD = 48;
+
+/**
+ * Batches allowed in flight before a frame's requests are refused. A mapping normally lands one or
+ * two frames after its copy, so this is only reached when the device has stopped answering, and
+ * the right thing to do with a frame's requests then is to say so and drop them: a consumer asks
+ * every frame anyway, and a value delivered a dozen frames late is not the value it wanted.
+ *
+ * @type {number}
+ */
+const DEFAULT_MAX_IN_FLIGHT = 4;
+
+/**
+ * Request records and batch handle lists start at this many entries and double.
+ * @type {number}
+ */
+const INITIAL_CAPACITY = 64;
+
+/*
+ * One request is RECORD_WORDS words of the flat record array, at handle * RECORD_WORDS. The first
+ * six are written by `request`, the last two by `encode`.
+ */
+const R_TABLE = 0;
+const R_ROW = 1;
+const R_WORD_OFFSET = 2;
+const R_WORD_COUNT = 3;
+const R_GENERATION = 4;
+const R_TARGET_OFFSET = 5;
+const R_SOURCE_WORD = 6;
+const R_DESTINATION_WORD = 7;
+const RECORD_WORDS = 8;
+
+/**
+ * @enum {number}
+ */
+const BatchState = {
+    /** taking requests; the one batch in this state is the frame's */
+    Assembling: 0,
+    /** copy recorded, the frame that recorded it not yet known to be submitted */
+    Encoded: 1,
+    /** `mapAsync` outstanding */
+    Mapping: 2,
+    /** mapped, waiting for an apply point */
+    Ready: 3,
+    /** the map was refused; the batch is discarded without delivering */
+    Failed: 4,
+};
+
+/**
+ * One frame's worth of requests, travelling together. Pooled: a batch that has been delivered keeps
+ * its arrays and takes the next frame's requests, so a steady stream of batches allocates nothing.
+ */
+class GPUReadbackBatch {
+    /**
+     * Handles in request order.
+     * @type {Uint32Array}
+     */
+    handles = new Uint32Array(INITIAL_CAPACITY);
+
+    /**
+     * @type {number}
+     */
+    count = 0;
+
+    /**
+     * Requests whose row was resident when the batch was encoded.
+     * @type {number}
+     */
+    live = 0;
+
+    /**
+     * Bytes of the packed image.
+     * @type {number}
+     */
+    byte_size = 0;
+
+    /**
+     * The readback buffer the image was copied into. From {@link GPUBufferReadbackAllocator}, and
+     * given back when the batch is delivered or discarded.
+     *
+     * @type {GPUBuffer|null}
+     */
+    buffer = null;
+
+    /**
+     * The mapped image, valid from the mapping settling until the buffer is given back. Read
+     * directly at the apply point — the buffer stays mapped until then, which is what makes the
+     * copy into the caller's memory the only copy.
+     *
+     * @type {Uint8Array|null}
+     */
+    bytes = null;
+
+    /**
+     * The command context the copy was recorded on. The mapping may not start until it has been
+     * finished, because a map of a buffer whose copy has not been submitted reads the frame before.
+     *
+     * @type {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext|null}
+     */
+    context = null;
+
+    /**
+     * @type {BatchState}
+     */
+    state = BatchState.Assembling;
+
+    /**
+     * Live handles grouped by table, for the gather path's one dispatch per table.
+     * @type {Uint32Array}
+     */
+    order = new Uint32Array(0);
+
+    /**
+     * Reactions to this batch's mapping, bound once per pooled batch rather than once per frame.
+     * @type {function(): void}
+     */
+    on_mapped = null;
+
+    /**
+     * @type {function(): void}
+     */
+    on_failed = null;
+}
+
+/**
+ * Reading rows back from a {@link GPUDatabase} in batches: one gather, one copy, one map, and one
+ * point in the frame where the answers land — in memory the caller already owns.
+ *
+ * **Why this is not `debug_table_read_element`.** That resolves one row's byte offset and calls
+ * {@link inspect_gpu_buffer_typed}, which creates a `MAP_READ` buffer, copies, submits, maps and
+ * destroys it — per call. It is a debug affordance and stays one. Here the buffers persist, the
+ * requests of a frame travel together, and the result arrives at a stated moment rather than
+ * whenever a promise happens to settle.
+ *
+ * **The caller names memory, and nothing is called.** A request is a table handle, a row, a byte
+ * range of the record, and a `Uint8Array` plus offset to put it in. At the apply point the service
+ * copies the bytes there and marks the request's handle in {@link status}; the consumer reads the
+ * status when its own system next runs, which is when it wanted the value anyway. There is no
+ * callback and no promise, for the same reason: a function called per row is a closure allocated
+ * per row and a permanently megamorphic call site, and code running inside the apply point could
+ * re-enter this service, or destroy the database, from the middle of `GPUDatabase.update`. The
+ * shader packs a batch contiguously in request order, so a consumer's run of rows lands in its
+ * destination as one `set` — which a per-row function could never have done.
+ *
+ * **Gather, then one copy.** Rows sit at scattered offsets across pages, so a compute shader packs
+ * the requested ranges contiguously before a single copy into a readback buffer. Packing a *range*
+ * rather than a record is what makes it worth doing: an entity's world matrix is 64 bytes of a
+ * 176-byte row.
+ *
+ * **But the request list is itself an upload.** A gather needs its descriptors on the GPU, so every
+ * frame with any request writes a buffer, allocates, binds and dispatches. For one row that costs
+ * far more than the value retrieved. Below {@link DEFAULT_GATHER_THRESHOLD} — a measured number, see
+ * its docblock — the batch degrades to direct `copyBufferToBuffer` calls into the same readback
+ * buffer, with no shader at all. A frame with no requests skips the whole path: no buffer, no
+ * command.
+ *
+ * **One apply point, in encode order.** Results are delivered from {@link apply}, at the head of a
+ * frame, and never on the timing of the map's own promise. A component that changed value in the
+ * middle of an update because a readback happened to settle would be a different world for the
+ * systems that ran before it and the ones that ran after. And a batch is delivered only after every
+ * older batch has been: `mapAsync` promises are not promised to settle in the order the maps were
+ * started, and a consumer asking for the same row every frame must never see a fresh value
+ * overwritten by a stale one.
+ *
+ * **Nothing here may reach simulation state.** A readback is a frame or more late and settles on the
+ * GPU's own schedule, so an outcome that depended on one would depend on frame pacing, and the
+ * determinism gate with it. This answers tools, sounds and cosmetics. Where the simulation wants a
+ * GPU-owned pose it evaluates the chain exactly, on the CPU.
+ *
+ * @see GPUStateAuthority which says whose value is worth reading back in the first place
+ * @see GPUReadbackStatus
+ */
+export class GPUDatabaseReadback {
+    /**
+     * @type {import("./GPUDatabase.js").GPUDatabase}
+     */
+    #database;
+
+    /**
+     * @type {GPUDevice}
+     */
+    #device;
+
+    /**
+     * Where readback buffers come from: the graphics context's, shared with every other database
+     * on it, and taken from the first command context this service encodes on — the database is
+     * built against a device, not a context.
+     *
+     * @type {import("../GPUBufferReadbackAllocator.js").GPUBufferReadbackAllocator|null}
+     */
+    #allocator = null;
+
+    /**
+     * Shader-writable scratch the gather packs into, copied to a readback buffer in one go.
+     * Persistent and grown, never per batch, and unlike the readback buffers it needs no pool,
+     * because it is written and read inside one submission.
+     *
+     * @type {GPUBuffer|null}
+     */
+    #gather_buffer = null;
+
+    /**
+     * Request records, {@link RECORD_WORDS} words per handle. A request is a few integer stores
+     * into this array; the target it names is the only thing that is not an integer, and it sits
+     * in {@link #targets} beside it.
+     *
+     * @type {Uint32Array}
+     */
+    #records = new Uint32Array(INITIAL_CAPACITY * RECORD_WORDS);
+
+    /**
+     * Per handle, the memory the bytes are to land in.
+     * @type {(Uint8Array|null)[]}
+     */
+    #targets = new Array(INITIAL_CAPACITY).fill(null);
+
+    /**
+     * Per handle, a {@link GPUReadbackStatus}. Exposed as {@link status}.
+     * @type {Uint8Array}
+     */
+    #status = new Uint8Array(INITIAL_CAPACITY);
+
+    /**
+     * Recycled handles. Never longer than the record capacity, so it never overflows.
+     * @type {Uint32Array}
+     */
+    #free = new Uint32Array(INITIAL_CAPACITY);
+
+    /**
+     * @type {number}
+     */
+    #free_count = 0;
+
+    /**
+     * Handles below this have been handed out at least once.
+     * @type {number}
+     */
+    #fresh = 0;
+
+    /**
+     * The batch taking this frame's requests.
+     * @type {GPUReadbackBatch}
+     */
+    #assembling;
+
+    /**
+     * Encoded batches, oldest first.
+     * @type {GPUReadbackBatch[]}
+     */
+    #in_flight = [];
+
+    /**
+     * Batches delivered or dropped since the last apply point. Their handles stay readable until
+     * the next one, and are recycled there.
+     *
+     * @type {GPUReadbackBatch[]}
+     */
+    #retiring = [];
+
+    /**
+     * Batches with nothing in them, waiting to be the next frame's.
+     * @type {GPUReadbackBatch[]}
+     */
+    #spare = [];
+
+    /**
+     * Scratch for bucketing a batch's live requests by table: counts, then starts.
+     * @type {Uint32Array}
+     */
+    #table_starts;
+
+    /**
+     * @type {number}
+     */
+    #gather_threshold;
+
+    /**
+     * @type {number}
+     */
+    #max_in_flight;
+
+    /**
+     * Descriptor words for one dispatch, reused. Grown to the largest chunk this service has ever
+     * needed and kept.
+     *
+     * @type {Uint32Array}
+     */
+    #descriptor_scratch = new Uint32Array(0);
+
+    /**
+     * Immediate data for the gather dispatch, reused. `dispatchThreads` serialises it at the call,
+     * so one object can carry every dispatch of a frame.
+     *
+     * @type {Object}
+     */
+    #gather_settings = {
+        count: 0,
+        record_size: 0,
+        elements_per_page: 0,
+        page_lookup_address: 0,
+        page_header_words: 0,
+        page_limit: 0,
+    };
+
+    /**
+     * @type {boolean}
+     */
+    #destroyed = false;
+
+    #copies_encoded = 0;
+    #dispatches_encoded = 0;
+    #delivered_count = 0;
+    #dropped_count = 0;
+    #stalled_count = 0;
+
+    /**
+     * @param {Object} options
+     * @param {import("./GPUDatabase.js").GPUDatabase} options.database
+     * @param {GPUDevice} options.device
+     * @param {number} [options.gather_threshold] requests at or above which a dispatch is used
+     * @param {number} [options.max_in_flight] batches in flight before a frame's requests are dropped
+     * @param {string} [options.label]
+     */
+    constructor({
+                    database,
+                    device,
+                    gather_threshold = DEFAULT_GATHER_THRESHOLD,
+                    max_in_flight = DEFAULT_MAX_IN_FLIGHT,
+                    label = 'GPUDatabaseReadback',
+                }) {
+        assert.defined(database, 'database');
+        assert.defined(device, 'device');
+        assert.isNonNegativeInteger(gather_threshold, 'gather_threshold');
+        assert.isPositiveInteger(max_in_flight, 'max_in_flight');
+        assert.isString(label, 'label');
+
+        this.#database = database;
+        this.#device = device;
+        this.#gather_threshold = gather_threshold;
+        this.#max_in_flight = max_in_flight;
+
+        this.#table_starts = new Uint32Array(database.tables.length + 1);
+
+        this.#assembling = this.#make_batch();
+    }
+
+    /**
+     * Requests at or above which a batch is gathered rather than copied range by range.
+     * @returns {number}
+     */
+    get gather_threshold() {
+        return this.#gather_threshold;
+    }
+
+    /**
+     * Batches in flight before a frame's requests are refused.
+     * @returns {number}
+     */
+    get max_in_flight() {
+        return this.#max_in_flight;
+    }
+
+    /**
+     * Per handle, what became of the request — a {@link GPUReadbackStatus}.
+     *
+     * Read it fresh each time rather than keeping a reference: the array is replaced when the
+     * handle space grows. A status is readable for one frame, from the apply point that set it to
+     * the next.
+     *
+     * @returns {Uint8Array}
+     */
+    get status() {
+        return this.#status;
+    }
+
+    /**
+     * How many handles exist. Grows to the working set of a steady stream of requests and stops:
+     * handles are recycled a frame after their result lands.
+     *
+     * @returns {number}
+     */
+    get handle_capacity() {
+        return this.#status.length;
+    }
+
+    /**
+     * Requests made and not yet encoded.
+     * @returns {number}
+     */
+    get pending_count() {
+        return this.#assembling.count;
+    }
+
+    /**
+     * Batches encoded and not yet delivered or discarded.
+     * @returns {number}
+     */
+    get in_flight_count() {
+        return this.#in_flight.length;
+    }
+
+    /**
+     * How many `copyBufferToBuffer` calls this service has encoded, over its life. Read by profiles,
+     * and by the specs that assert an empty frame encodes nothing.
+     *
+     * @returns {number}
+     */
+    get copies_encoded() {
+        return this.#copies_encoded;
+    }
+
+    /**
+     * How many gather dispatches this service has encoded, over its life.
+     * @returns {number}
+     */
+    get dispatches_encoded() {
+        return this.#dispatches_encoded;
+    }
+
+    /**
+     * Requests whose bytes reached their target.
+     * @returns {number}
+     */
+    get delivered_count() {
+        return this.#delivered_count;
+    }
+
+    /**
+     * Requests thrown away because of the row — its page was not resident when the batch was
+     * encoded, it was not occupied, or it was freed before the answer landed.
+     *
+     * Reported rather than swallowed: a consumer whose readbacks are all being dropped is asking
+     * about rows that keep dying, and silence reads downstream as "the value did not change".
+     *
+     * @returns {number}
+     */
+    get dropped_count() {
+        return this.#dropped_count;
+    }
+
+    /**
+     * Requests thrown away because of the pipeline — {@link max_in_flight} batches were still
+     * waiting on the device when the frame's batch was due, or a mapping was refused. Separate
+     * from {@link dropped_count} because the cure is different: those rows died, these the device
+     * stopped answering about.
+     *
+     * @returns {number}
+     */
+    get stalled_count() {
+        return this.#stalled_count;
+    }
+
+    /**
+     * Resolve a table name to the handle {@link request} takes. Once, at setup: a string lookup
+     * per row per frame is what this exists to avoid.
+     *
+     * From here on the table counts how many times each of its rows is freed, which is what lets
+     * an answer about a row that died and was reused be told from an answer about the row asked
+     * for.
+     *
+     * @param {string} name
+     * @returns {number}
+     */
+    table(name) {
+        assert.isString(name, 'name');
+
+        const table = this.#database.get(name);
+
+        if (table === undefined) {
+            throw new Error(`Table '${name}' does not exist`);
+        }
+
+        table.track_generations();
+
+        return table.descriptor.index;
+    }
+
+    /**
+     * Ask for a byte range of a row to be copied into `target`, at a later apply point.
+     *
+     * Positional, and a few integer stores: this is called once per row per frame by anything that
+     * tracks rows, so it takes no options object and allocates nothing.
+     *
+     * @param {number} table from {@link table}
+     * @param {number} row
+     * @param {number} byte_offset within the record, word-aligned
+     * @param {number} byte_size a positive multiple of the word size
+     * @param {Uint8Array} target memory the caller owns and keeps; the bytes are written here at
+     *        the apply point. A byte view on purpose — a consumer that reads floats holds a
+     *        `Float32Array` over the same buffer, made once
+     * @param {number} [target_byte_offset]
+     * @returns {number} a handle into {@link status}, valid until the frame after its result lands
+     */
+    request(table, row, byte_offset, byte_size, target, target_byte_offset = 0) {
+        assert.isNonNegativeInteger(table, 'table');
+        assert.isNonNegativeInteger(row, 'row');
+        assert.isNonNegativeInteger(byte_offset, 'byte_offset');
+        assert.isPositiveInteger(byte_size, 'byte_size');
+        assert.isNonNegativeInteger(target_byte_offset, 'target_byte_offset');
+
+        const tables = this.#database.tables;
+
+        assert.lessThan(table, tables.length, 'table is not a handle from table()');
+        assert.equal(byte_offset % WORD_SIZE, 0, 'byte_offset must be word-aligned');
+        assert.equal(byte_size % WORD_SIZE, 0, 'byte_size must be a whole number of words');
+        assert.ok(target instanceof Uint8Array, 'target must be a Uint8Array');
+
+        // constant messages: these run per row per frame, and a template literal built to describe a
+        // failure that does not happen is the cost on the path being measured
+        assert.lessThanOrEqual(byte_offset + byte_size, tables[table].descriptor.packed_element_size_bytes, 'range reaches past the record');
+        assert.lessThanOrEqual(target_byte_offset + byte_size, target.length, 'range reaches past the target');
+
+        const handle = this.#allocate_handle();
+        const base = handle * RECORD_WORDS;
+        const records = this.#records;
+
+        records[base + R_TABLE] = table;
+        records[base + R_ROW] = row;
+        records[base + R_WORD_OFFSET] = byte_offset / WORD_SIZE;
+        records[base + R_WORD_COUNT] = byte_size / WORD_SIZE;
+        records[base + R_GENERATION] = tables[table].generation_of(row);
+        records[base + R_TARGET_OFFSET] = target_byte_offset;
+
+        this.#targets[handle] = target;
+        this.#status[handle] = GPUReadbackStatus.Pending;
+
+        const batch = this.#assembling;
+
+        if (batch.count === batch.handles.length) {
+            const grown = new Uint32Array(batch.handles.length * 2);
+
+            grown.set(batch.handles);
+
+            batch.handles = grown;
+        }
+
+        batch.handles[batch.count++] = handle;
+
+        return handle;
+    }
+
+    /**
+     * Deliver every batch whose bytes have arrived, oldest first and stopping at the first that has
+     * not, then start the map for every batch whose frame has been submitted.
+     *
+     * **This is the apply point.** Call it at the head of a frame, before any system runs. A batch
+     * therefore reaches its targets at the first frame head at which its mapping — and every older
+     * batch's — has resolved, which is the frame after the one that encoded it at the earliest:
+     * never inside an update, and never at two different moments for two requests of the same
+     * batch. No caller-supplied code runs here.
+     *
+     * @returns {number} requests delivered
+     */
+    apply() {
+        if (this.#destroyed) {
+            return 0;
+        }
+
+        // The handles delivered or dropped at the previous apply point have had their frame of
+        // being readable; they go back to the free list before anything is delivered, so nothing
+        // delivered now can be recycled before it has been read.
+        const retiring = this.#retiring;
+
+        for (let i = 0; i < retiring.length; i++) {
+            this.#recycle(retiring[i]);
+        }
+
+        retiring.length = 0;
+
+        const in_flight = this.#in_flight;
+
+        let delivered = 0;
+
+        while (in_flight.length > 0) {
+            const batch = in_flight[0];
+
+            if (batch.state === BatchState.Ready) {
+                delivered += this.#deliver(batch);
+
+                this.#allocator.release(batch.buffer);
+            } else if (batch.state === BatchState.Failed) {
+                this.#drop_pending(batch, true);
+
+                /*
+                 A refused mapping is the one case the allocator has discard for, and this used to
+                 release alongside the delivered ones. release only inspects mapState, and a
+                 destroyed buffer reports "unmapped", so the corpse went back on the free list in
+                 size order and the next batch of the same size was handed it — a validation error
+                 on every copy into it, for as long as the pool kept handing it out.
+                */
+                this.#allocator.discard(batch.buffer);
+            } else {
+                // still on the device, and nothing younger may overtake it
+                break;
+            }
+
+            batch.buffer = null;
+            batch.bytes = null;
+            batch.context = null;
+
+            in_flight.shift();
+            retiring.push(batch);
+        }
+
+        // Start the maps only now, and only for batches whose frame has been submitted. `encode`
+        // records a copy into a command context the caller has not finished yet, and mapping a
+        // buffer whose copy has not been submitted reads the frame before.
+        for (let i = 0; i < in_flight.length; i++) {
+            const batch = in_flight[i];
+
+            if (batch.state === BatchState.Encoded && batch.context.finished) {
+                this.#start_map(batch);
+            }
+        }
+
+        return delivered;
+    }
+
+    /**
+     * Record this frame's batch: a gather and one copy, or a handful of copies, or nothing at all.
+     *
+     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
+     * @returns {boolean} whether anything was encoded
+     */
+    encode(encoder) {
+        if (this.#destroyed) {
+            return false;
+        }
+
+        const batch = this.#assembling;
+
+        if (batch.count === 0) {
+            // The point of the whole arrangement: a frame nobody asked anything of costs one
+            // comparison. No buffer is taken, no command is recorded.
+            return false;
+        }
+
+        if (this.#allocator === null) {
+            const allocator = encoder.graphics.buffer_allocator_readback;
+
+            assert.defined(allocator, 'encoder.graphics.buffer_allocator_readback');
+
+            this.#allocator = allocator;
+        }
+
+        if (this.#in_flight.length >= this.#max_in_flight) {
+            // The device has stopped answering. Refusing the frame's requests, with a status the
+            // consumer can see, bounds everything: nothing queues, nothing is re-resolved next
+            // frame, and no buffer is taken for bytes that would be a dozen frames stale.
+            this.#drop_pending(batch, true);
+            this.#retire(batch);
+
+            return false;
+        }
+
+        const live = this.#resolve(batch);
+
+        if (live === 0) {
+            // every row asked about has gone; nothing to copy and nothing to deliver
+            this.#retire(batch);
+
+            return false;
+        }
+
+        batch.buffer = this.#allocator.get(batch.byte_size);
+
+        if (live < this.#gather_threshold) {
+            this.#encode_copies(encoder, batch);
+        } else {
+            this.#encode_gather(encoder, batch);
+        }
+
+        batch.context = encoder;
+        batch.state = BatchState.Encoded;
+
+        this.#in_flight.push(batch);
+        this.#assembling = this.#take_batch();
+
+        return true;
+    }
+
+    /**
+     * Where each request's range starts in the database buffer and where it lands in the image,
+     * for every request whose row is there to be read. A request whose page is not resident, or
+     * whose row is not occupied, is dropped here: "the page was gone" is a fact about the moment
+     * the batch was encoded and cannot be recovered from the bytes afterwards.
+     *
+     * The same arithmetic {@link GPUDatabase#debug_table_read_element} does, and the same the gather
+     * shader does — the CPU needs it for the copy path, and the residency answer for both.
+     *
+     * One pass, in request order, so the image is packed in request order and a consumer's run of
+     * consecutive rows is one contiguous span of it. Requests cluster by table and by page, so the
+     * table and the page are looked up when they change rather than per request. Live requests are
+     * counted per table on the way through, which is all the gather path's bucketing needs.
+     *
+     * @param {GPUReadbackBatch} batch
+     * @returns {number} live requests
+     */
+    #resolve(batch) {
+        const database = this.#database;
+        const tables = database.tables;
+        const data_start = database.data_start_offset;
+
+        const records = this.#records;
+        const status = this.#status;
+        const handles = batch.handles;
+        const count = batch.count;
+
+        const starts = this.#table_starts;
+
+        starts.fill(0);
+
+        let destination = 0;
+        let live = 0;
+        let dropped = 0;
+
+        let table_index = -1;
+        let table = null;
+        let descriptor = null;
+        let elements_per_page = 0;
+        let record_words = 0;
+
+        let page_index = -1;
+        let page_base = -1;
+
+        for (let i = 0; i < count; i++) {
+            const handle = handles[i];
+            const base = handle * RECORD_WORDS;
+
+            const t = records[base + R_TABLE];
+            const row = records[base + R_ROW];
+
+            if (t !== table_index) {
+                table_index = t;
+                table = tables[t];
+                descriptor = table.descriptor;
+                elements_per_page = descriptor.elements_per_page;
+                record_words = descriptor.packed_element_size_bytes / WORD_SIZE;
+                page_index = -1;
+            }
+
+            const p = Math.floor(row / elements_per_page);
+
+            if (p !== page_index) {
+                page_index = p;
+
+                const page = table.pages.get(p);
+
+                page_base = (page === undefined || page.slot_offset === -1)
+                    ? -1
+                    : page.slot_offset + data_start + descriptor.page_header_words;
+            }
+
+            if (page_base === -1 || !table.occupancy.get(row)) {
+                // the page is not on the GPU, or the row was removed and its bytes are whatever the
+                // last occupant left
+                status[handle] = GPUReadbackStatus.Dropped;
+                dropped++;
+
+                continue;
+            }
+
+            records[base + R_SOURCE_WORD] = page_base
+                + (row - page_index * elements_per_page) * record_words
+                + records[base + R_WORD_OFFSET];
+            records[base + R_DESTINATION_WORD] = destination;
+
+            destination += records[base + R_WORD_COUNT];
+
+            live++;
+            starts[t + 1]++;
+        }
+
+        this.#dropped_count += dropped;
+
+        batch.live = live;
+        batch.byte_size = destination * WORD_SIZE;
+
+        return live;
+    }
+
+    /**
+     * Copy each range straight out of the database into its place in the image. No shader, no
+     * descriptor upload, no bind group — which below the threshold is cheaper than all three.
+     *
+     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
+     * @param {GPUReadbackBatch} batch
+     */
+    #encode_copies(encoder, batch) {
+        const source = this.#database.buffer;
+        const destination = batch.buffer;
+
+        const records = this.#records;
+        const status = this.#status;
+        const handles = batch.handles;
+        const count = batch.count;
+
+        let copies = 0;
+
+        for (let i = 0; i < count; i++) {
+            const handle = handles[i];
+
+            if (status[handle] !== GPUReadbackStatus.Pending) {
+                continue;
+            }
+
+            const base = handle * RECORD_WORDS;
+
+            encoder.copyBufferToBuffer(
+                source,
+                records[base + R_SOURCE_WORD] * WORD_SIZE,
+                destination,
+                records[base + R_DESTINATION_WORD] * WORD_SIZE,
+                records[base + R_WORD_COUNT] * WORD_SIZE,
+            );
+
+            copies++;
+        }
+
+        this.#copies_encoded += copies;
+    }
+
+    /**
+     * Pack every range into the scratch buffer with one dispatch per table, then copy the whole
+     * image across in one go.
+     *
+     * One dispatch per table rather than one for the batch, because the page geometry a thread needs
+     * — record size, elements per page, where the table's page addresses live — is per table, and
+     * carrying it per descriptor would cost more words than the descriptor has.
+     *
+     * The live handles are bucketed by table with a counting sort off the per-table counts
+     * {@link #resolve} left behind: one pass to scatter, then each table's run is emitted in turn.
+     * Linear in the requests plus the tables, where scanning every request once per table was
+     * their product.
+     *
+     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
+     * @param {GPUReadbackBatch} batch
+     */
+    #encode_gather(encoder, batch) {
+        const live = batch.live;
+        const byte_size = batch.byte_size;
+
+        const gather_buffer = this.#fit_gather_buffer(byte_size);
+
+        const tables = this.#database.tables;
+        const table_count = tables.length;
+
+        const records = this.#records;
+        const status = this.#status;
+        const handles = batch.handles;
+        const count = batch.count;
+
+        // counts to starts
+        const starts = this.#table_starts;
+
+        for (let t = 1; t <= table_count; t++) {
+            starts[t] += starts[t - 1];
+        }
+
+        if (batch.order.length < live) {
+            batch.order = new Uint32Array(Math.max(live, batch.order.length * 2));
+        }
+
+        const order = batch.order;
+
+        // scatter, in request order within each table; afterwards starts[t] is the end of table t
+        for (let i = 0; i < count; i++) {
+            const handle = handles[i];
+
+            if (status[handle] !== GPUReadbackStatus.Pending) {
+                continue;
+            }
+
+            order[starts[records[handle * RECORD_WORDS + R_TABLE]]++] = handle;
+        }
+
+        const descriptor_stride = GPU_TABLE_GATHER_DESCRIPTOR_WORDS;
+        const max_per_chunk = Math.max(1, Math.floor(DESCRIPTOR_CHUNK_BYTES / (descriptor_stride * WORD_SIZE)));
+
+        const scratch_words = Math.min(live, max_per_chunk) * descriptor_stride;
+
+        if (this.#descriptor_scratch.length < scratch_words) {
+            this.#descriptor_scratch = new Uint32Array(scratch_words);
+        }
+
+        const scratch = this.#descriptor_scratch;
+        const settings = this.#gather_settings;
+
+        let run_start = 0;
+
+        for (let t = 0; t < table_count; t++) {
+            const run_end = starts[t];
+
+            if (run_end === run_start) {
+                continue;
+            }
+
+            const table_descriptor = tables[t].descriptor;
+
+            settings.record_size = table_descriptor.packed_element_size_bytes / WORD_SIZE;
+            settings.elements_per_page = table_descriptor.elements_per_page;
+            settings.page_lookup_address = table_descriptor.page_lookup_address;
+            settings.page_header_words = table_descriptor.page_header_words;
+            settings.page_limit = table_descriptor.page_limit;
+
+            let filled = 0;
+
+            for (let i = run_start; i < run_end; i++) {
+                const base = order[i] * RECORD_WORDS;
+                const d = filled * descriptor_stride;
+
+                scratch[d] = records[base + R_ROW];
+                scratch[d + 1] = records[base + R_WORD_OFFSET];
+                scratch[d + 2] = records[base + R_WORD_COUNT];
+                scratch[d + 3] = records[base + R_DESTINATION_WORD];
+
+                filled++;
+
+                if (filled === max_per_chunk) {
+                    this.#dispatch_gather(encoder, scratch, filled, settings, gather_buffer);
+                    filled = 0;
+                }
+            }
+
+            if (filled > 0) {
+                this.#dispatch_gather(encoder, scratch, filled, settings, gather_buffer);
+            }
+
+            run_start = run_end;
+        }
+
+        encoder.copyBufferToBuffer(gather_buffer, 0, batch.buffer, 0, byte_size);
+
+        this.#copies_encoded++;
+    }
+
+    /**
+     * @param {import("../../../device/ShadeGPUCommandContext.js").ShadeGPUCommandContext} encoder
+     * @param {Uint32Array} scratch
+     * @param {number} count
+     * @param {Object} settings
+     * @param {GPUBuffer} gather_buffer
+     */
+    #dispatch_gather(encoder, scratch, count, settings, gather_buffer) {
+        settings.count = count;
+
+        const descriptors = encoder.allocateTransientBufferAndLoad(
+            scratch.buffer,
+            GPUBufferUsage.STORAGE,
+            0,
+            count * GPU_TABLE_GATHER_DESCRIPTOR_WORDS * WORD_SIZE,
+        );
+
+        gpu_table_gather_shader.dispatchThreads({
+            encoder,
+            bindings: {
+                settings,
+                descriptors,
+                database: this.#database.buffer,
+                gathered: gather_buffer,
+            },
+            thread_count_x: count,
+        });
+
+        this.#dispatches_encoded++;
+    }
+
+    /**
+     * @param {number} byte_size
+     * @returns {GPUBuffer}
+     */
+    #fit_gather_buffer(byte_size) {
+        const existing = this.#gather_buffer;
+
+        if (existing !== null && existing.size >= byte_size) {
+            return existing;
+        }
+
+        if (existing !== null) {
+            existing.destroy();
+        }
+
+        const buffer = this.#device.createBuffer({
+            label: 'GPUDatabaseReadback/gather',
+            size: round_up(byte_size, GATHER_SIZE_GRANULARITY),
+            usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
+            mappedAtCreation: false,
+        });
+
+        this.#gather_buffer = buffer;
+
+        return buffer;
+    }
+
+    /**
+     * @param {GPUReadbackBatch} batch
+     */
+    #start_map(batch) {
+        batch.state = BatchState.Mapping;
+
+        batch.buffer.mapAsync(GPUMapMode.READ, 0, batch.byte_size).then(batch.on_mapped, batch.on_failed);
+    }
+
+    /**
+     * Copy a mapped batch into its targets.
+     *
+     * The image is packed in request order, so consecutive live requests are consecutive in it;
+     * when they also name consecutive bytes of one target, the run is one `set`. A consumer that
+     * asks for a hundred rows into one array gets one copy, not a hundred.
+     *
+     * @param {GPUReadbackBatch} batch
+     * @returns {number} requests delivered
+     */
+    #deliver(batch) {
+        const records = this.#records;
+        const status = this.#status;
+        const targets = this.#targets;
+        const tables = this.#database.tables;
+
+        const bytes = batch.bytes;
+        const handles = batch.handles;
+        const count = batch.count;
+
+        let delivered = 0;
+        let dropped = 0;
+
+        let run_target = null;
+        let run_source = 0;
+        let run_target_offset = 0;
+        let run_length = 0;
+
+        for (let i = 0; i < count; i++) {
+            const handle = handles[i];
+
+            if (status[handle] !== GPUReadbackStatus.Pending) {
+                // dropped at encode; it has no bytes in the image
+                continue;
+            }
+
+            const base = handle * RECORD_WORDS;
+            const row = records[base + R_ROW];
+
+            if (tables[records[base + R_TABLE]].generation_of(row) !== records[base + R_GENERATION]) {
+                // the row was freed while this was in flight, and an id freed is an id already
+                // handed out again — answering would be answering about a stranger
+                status[handle] = GPUReadbackStatus.Dropped;
+                dropped++;
+
+                continue;
+            }
+
+            const source = records[base + R_DESTINATION_WORD] * WORD_SIZE;
+            const length = records[base + R_WORD_COUNT] * WORD_SIZE;
+            const target = targets[handle];
+            const target_offset = records[base + R_TARGET_OFFSET];
+
+            if (
+                target === run_target
+                && target_offset === run_target_offset + run_length
+                && source === run_source + run_length
+            ) {
+                run_length += length;
+            } else {
+                if (run_length > 0) {
+                    run_target.set(bytes.subarray(run_source, run_source + run_length), run_target_offset);
+                }
+
+                run_target = target;
+                run_source = source;
+                run_target_offset = target_offset;
+                run_length = length;
+            }
+
+            status[handle] = GPUReadbackStatus.Delivered;
+            delivered++;
+        }
+
+        if (run_length > 0) {
+            run_target.set(bytes.subarray(run_source, run_source + run_length), run_target_offset);
+        }
+
+        this.#delivered_count += delivered;
+        this.#dropped_count += dropped;
+
+        return delivered;
+    }
+
+    /**
+     * Mark every request of a batch still waiting as dropped.
+     *
+     * @param {GPUReadbackBatch} batch
+     * @param {boolean} stalled whether the pipeline, rather than the rows, is the reason
+     */
+    #drop_pending(batch, stalled) {
+        const status = this.#status;
+        const handles = batch.handles;
+        const count = batch.count;
+
+        let dropped = 0;
+
+        for (let i = 0; i < count; i++) {
+            const handle = handles[i];
+
+            if (status[handle] === GPUReadbackStatus.Pending) {
+                status[handle] = GPUReadbackStatus.Dropped;
+                dropped++;
+            }
+        }
+
+        if (stalled) {
+            this.#stalled_count += dropped;
+        } else {
+            this.#dropped_count += dropped;
+        }
+    }
+
+    /**
+     * Send the assembling batch straight to retiring — nothing of it is in flight — and take a
+     * fresh one for the requests that follow.
+     *
+     * @param {GPUReadbackBatch} batch
+     */
+    #retire(batch) {
+        this.#retiring.push(batch);
+        this.#assembling = this.#take_batch();
+    }
+
+    /**
+     * Free a retired batch's handles and put the batch back in the pool.
+     *
+     * @param {GPUReadbackBatch} batch
+     */
+    #recycle(batch) {
+        const status = this.#status;
+        const targets = this.#targets;
+        const free = this.#free;
+        const handles = batch.handles;
+        const count = batch.count;
+
+        let free_count = this.#free_count;
+
+        for (let i = 0; i < count; i++) {
+            const handle = handles[i];
+
+            status[handle] = GPUReadbackStatus.None;
+            targets[handle] = null;
+
+            free[free_count++] = handle;
+        }
+
+        this.#free_count = free_count;
+
+        batch.count = 0;
+        batch.live = 0;
+        batch.byte_size = 0;
+        batch.state = BatchState.Assembling;
+
+        this.#spare.push(batch);
+    }
+
+    /**
+     * @returns {GPUReadbackBatch}
+     */
+    #take_batch() {
+        const spare = this.#spare;
+
+        if (spare.length > 0) {
+            return spare.pop();
+        }
+
+        return this.#make_batch();
+    }
+
+    /**
+     * @returns {GPUReadbackBatch}
+     */
+    #make_batch() {
+        const batch = new GPUReadbackBatch();
+
+        batch.on_mapped = () => {
+            if (this.#destroyed || batch.state !== BatchState.Mapping) {
+                // torn down while the map was out; the buffer went with it
+                return;
+            }
+
+            batch.bytes = new Uint8Array(batch.buffer.getMappedRange(0, batch.byte_size));
+            batch.state = BatchState.Ready;
+        };
+
+        batch.on_failed = () => {
+            if (batch.state === BatchState.Mapping) {
+                // device lost, buffer destroyed — discarded at the next apply point
+                batch.state = BatchState.Failed;
+            }
+        };
+
+        return batch;
+    }
+
+    /**
+     * @returns {number}
+     */
+    #allocate_handle() {
+        if (this.#free_count > 0) {
+            return this.#free[--this.#free_count];
+        }
+
+        if (this.#fresh === this.#status.length) {
+            this.#grow_handles();
+        }
+
+        return this.#fresh++;
+    }
+
+    /**
+     * Double the handle space. Handles are indices, so nothing moves; a caller keeps the ones it
+     * has.
+     */
+    #grow_handles() {
+        const capacity = this.#status.length * 2;
+
+        const records = new Uint32Array(capacity * RECORD_WORDS);
+        records.set(this.#records);
+        this.#records = records;
+
+        const status = new Uint8Array(capacity);
+        status.set(this.#status);
+        this.#status = status;
+
+        const free = new Uint32Array(capacity);
+        free.set(this.#free);
+        this.#free = free;
+
+        const targets = this.#targets;
+
+        for (let i = targets.length; i < capacity; i++) {
+            targets.push(null);
+        }
+    }
+
+    destroy() {
+        this.#destroyed = true;
+
+        const in_flight = this.#in_flight;
+
+        for (let i = 0; i < in_flight.length; i++) {
+            const batch = in_flight[i];
+
+            if (batch.buffer !== null) {
+                // may be mid-map, which WebGPU lets destroy() abort but not unmap()
+                this.#allocator.discard(batch.buffer);
+
+                batch.buffer = null;
+            }
+        }
+
+        in_flight.length = 0;
+        this.#retiring.length = 0;
+        this.#spare.length = 0;
+        this.#assembling.count = 0;
+
+        if (this.#gather_buffer !== null) {
+            this.#gather_buffer.destroy();
+            this.#gather_buffer = null;
+        }
+    }
+}