@driftengine/core 3.62.0 → 4.0.0

package/src/render/gpudriven/twoPhase.ts

@@ -0,0 +1,214 @@
+/**
+ * Occlusion tested against depth from **this** frame rather than the last one.
+ *
+ * A hierarchical depth buffer is only an occluder if something has already been drawn into it, and
+ * the cheap way to have one at the start of a frame is to keep last frame's. That is wrong exactly
+ * when the camera moved: the pyramid describes a view nobody is looking from, and geometry the old
+ * view happened to hide is culled out of the new one. What it looks like is a wall or a floor
+ * disappearing for a frame or two after a cut, a teleport, or a fast turn — and it is intermittent,
+ * so it is reported as a streaming fault.
+ *
+ * **So the frame is drawn in two halves.** Phase one draws only what was visible last frame, which
+ * on a continuous camera is nearly everything and is a good occluder for the rest. The pyramid is
+ * reduced from *that* depth. Phase two then tests everything else against it and draws whatever
+ * phase one missed. Both halves write the same colour and depth, because they are one frame.
+ *
+ * **The history decides which phase a cluster is judged in and never whether it is judged.** Every
+ * cluster is in exactly one of the two lists, whatever the history holds, so a history that is
+ * stale — which after a teleport is all of it — costs a phase-two test and never a hole. That
+ * invariant is what makes the history safe to keep across a cut, and it is the one this file's
+ * tests assert most directly.
+ *
+ * **One history per view.** A shadow pass and the main camera see different things, and sharing a
+ * bit set between them puts one view's visibility into the other's phase one — which is the same
+ * defect as a stale history, arriving every frame instead of after a cut.
+ *
+ * The pass recording here is the frame graph's data rather than device calls: `phaseOnePasses` and
+ * `phaseTwoPasses` record what each half reads and writes, and the edge from the reduction's write
+ * to phase two's read *is* the guarantee in the first paragraph. Wiring them to a real encoder is
+ * Task 14's, against a consumer, for the reason `ARCHITECTURE.md` §3 gives about rendering changes
+ * nobody has looked at.
+ */
+
+import { recordDeps } from '../frame/deps.ts';
+import type { Deps } from '../frame/deps.ts';
+
+/** Which clusters a view drew last frame, one bit each. */
+export interface VisibilityHistory {
+  /** Thirty-two clusters a word, cluster `n` at bit `n & 31` of word `n >> 5`. */
+  bits: Uint32Array;
+  /** How many clusters it has room for. A wider one is allocated rather than the old one read. */
+  capacity: number;
+}
+
+export function createVisibilityHistory(capacity: number): VisibilityHistory {
+  return { bits: new Uint32Array(Math.max(1, Math.ceil(capacity / 32))), capacity };
+}
+
+/**
+ * Whether a cluster was drawn last frame.
+ *
+ * **Out of range answers false rather than throwing or wrapping.** A cluster index past the end is
+ * a scene that grew since the history was made, and the honest answer for one nobody has seen is
+ * "not last frame" — which puts it in phase two, where it is tested rather than assumed. Wrapping
+ * would put an unrelated cluster's bit in its place, which is a visibility answer for the wrong
+ * geometry and is invisible until it is wrong.
+ *
+ * **A perturbation survives this check and it is kept anyway.** `setHistory` refuses the same
+ * range, so no bit above `capacity` can ever be set, and a read past the end of the word array is
+ * `undefined` — which `&` coerces to zero and therefore answers false already. What it decides is
+ * the case where the two guards disagree, and the reason both are here rather than one is that
+ * they are read independently: the write side is the one a test can catch, and it is caught. The
+ * bits between `capacity` and the end of the last word are the ones that make the write side
+ * matter — forty clusters is two words, so twenty-four of them belong to nobody.
+ */
+export function historyHas(history: VisibilityHistory, cluster: number): boolean {
+  if (cluster < 0 || cluster >= history.capacity) return false;
+  return ((history.bits[cluster >> 5] as number) & (1 << (cluster & 31))) !== 0;
+}
+
+export function setHistory(history: VisibilityHistory, cluster: number, drawn: boolean): void {
+  if (cluster < 0 || cluster >= history.capacity) return;
+  const word = cluster >> 5;
+  const mask = 1 << (cluster & 31);
+  const held = history.bits[word] as number;
+  history.bits[word] = drawn ? held | mask : held & ~mask;
+}
+
+export function clearHistory(history: VisibilityHistory): void {
+  history.bits.fill(0);
+}
+
+/** How many of the first `count` clusters the history holds. */
+export function historyCount(history: VisibilityHistory, count: number): number {
+  let total = 0;
+  for (let cluster = 0; cluster < count; cluster += 1) if (historyHas(history, cluster)) total += 1;
+  return total;
+}
+
+/** The clusters phase one draws: exactly what was visible last frame, in index order. */
+export function phaseOneClusters(
+  history: VisibilityHistory,
+  count: number,
+  out: Uint32Array,
+): number {
+  let at = 0;
+  for (let cluster = 0; cluster < count; cluster += 1) {
+    if (!historyHas(history, cluster)) continue;
+    out[at] = cluster;
+    at += 1;
+  }
+  return at;
+}
+
+/** The clusters phase two tests: everything the history does not hold. */
+export function phaseTwoClusters(
+  history: VisibilityHistory,
+  count: number,
+  out: Uint32Array,
+): number {
+  let at = 0;
+  for (let cluster = 0; cluster < count; cluster += 1) {
+    if (historyHas(history, cluster)) continue;
+    out[at] = cluster;
+    at += 1;
+  }
+  return at;
+}
+
+/**
+ * Replace the history with what this frame actually drew.
+ *
+ * **Replace rather than accumulate**, and that is the half that is easy to leave out: a cluster
+ * kept in phase one and then culled in phase two has to leave the set, or it is drawn every frame
+ * for ever on the strength of having once been visible. `keep` is the flag buffer
+ * `cullClusters` writes, so this is one pass over the same array the draw used.
+ */
+export function updateHistory(history: VisibilityHistory, keep: Uint32Array, count: number): void {
+  for (let cluster = 0; cluster < count; cluster += 1) {
+    setHistory(history, cluster, (keep[cluster] ?? 0) !== 0);
+  }
+}
+
+/** Every view's history, keyed by whatever identifier the caller gives its views. */
+export interface VisibilityHistories {
+  readonly byView: Map<number, VisibilityHistory>;
+}
+
+export function createVisibilityHistories(): VisibilityHistories {
+  return { byView: new Map() };
+}
+
+/**
+ * This view's history, widened in place if the scene grew.
+ *
+ * **Widened rather than replaced**, because replacing it throws away a frame of visibility every
+ * time a cluster is added and puts the whole scene through phase two — which is correct and is
+ * also the slow path this feature exists to avoid.
+ */
+export function historyFor(
+  store: VisibilityHistories,
+  view: number,
+  capacity: number,
+): VisibilityHistory {
+  const held = store.byView.get(view);
+  if (held === undefined) {
+    const made = createVisibilityHistory(capacity);
+    store.byView.set(view, made);
+    return made;
+  }
+  if (capacity > held.capacity) {
+    const widened = new Uint32Array(Math.max(1, Math.ceil(capacity / 32)));
+    widened.set(held.bits);
+    held.bits = widened;
+    held.capacity = capacity;
+  }
+  return held;
+}
+
+/** The virtual resources a two-phase frame moves between its halves. */
+export interface TwoPhaseResources {
+  readonly colour: number;
+  readonly depth: number;
+  readonly hzb: number;
+  readonly clusters: number;
+  readonly keep: number;
+}
+
+/** Where phase one's two nodes landed in the dependency table. */
+export interface PhaseOneNodes {
+  readonly draw: number;
+  readonly reduce: number;
+}
+
+/** Where phase two's two nodes landed. */
+export interface PhaseTwoNodes {
+  readonly cull: number;
+  readonly draw: number;
+}
+
+/**
+ * Phase one: draw what was visible, then reduce the depth it produced into the pyramid.
+ *
+ * The reduction reads the depth the draw wrote, so nothing can reorder them and nothing can alias
+ * the depth away between them — which is exactly what `lifetime.ts` and `alias.ts` are reading
+ * these edges for.
+ */
+export function phaseOnePasses(deps: Deps, res: TwoPhaseResources): PhaseOneNodes {
+  const draw = recordDeps(deps, [res.clusters], [res.colour, res.depth]);
+  const reduce = recordDeps(deps, [res.depth], [res.hzb]);
+  return { draw, reduce };
+}
+
+/**
+ * Phase two: cull everything else against that pyramid, then draw what survived.
+ *
+ * `cull` reads `hzb`, which `phaseOnePasses` wrote. That one edge is the whole claim of this
+ * module — the depth being tested against belongs to this frame — expressed where the scheduler
+ * can enforce it rather than in a comment somebody has to obey.
+ */
+export function phaseTwoPasses(deps: Deps, res: TwoPhaseResources): PhaseTwoNodes {
+  const cull = recordDeps(deps, [res.hzb, res.clusters], [res.keep]);
+  const draw = recordDeps(deps, [res.keep, res.clusters], [res.colour, res.depth]);
+  return { cull, draw };
+}

package/src/render/gpudriven/visbuffer.ts

@@ -0,0 +1,41 @@
+/**
+ * What a pixel records: which triangle of which cluster of which instance covered it.
+ *
+ * **One 32-bit target, and its ceiling is stated here rather than discovered.** Seven bits carry
+ * the triangle, which is the 128 a cluster may hold, and the remaining twenty-five carry the
+ * cluster-in-instance identifier — 33.5 million of them, far past what a streaming budget permits.
+ * If a scene ever reaches it the answer is a second target, not a repack.
+ *
+ * **Zero is a legal value, so the buffer is cleared to a sentinel instead.** Cleared to zero, an
+ * uncovered pixel is indistinguishable from triangle 0 of cluster 0 of instance 0, which is a real
+ * triangle in every scene that has one.
+ */
+
+/** Triangles a cluster may hold, and therefore what the low bits carry. */
+export const VIS_TRIANGLE_BITS = 7;
+export const VIS_MAX_TRIANGLES = 1 << VIS_TRIANGLE_BITS;
+/** Cluster-in-instance identifiers the remaining bits carry. */
+export const VIS_MAX_CLUSTERS = 1 << (32 - VIS_TRIANGLE_BITS);
+/** What an uncovered pixel holds. Not zero — see the header. */
+export const VIS_EMPTY = 0xffffffff;
+
+export function packVisibility(cluster: number, triangle: number): number {
+  if (triangle < 0 || triangle >= VIS_MAX_TRIANGLES) {
+    throw new RangeError(`triangle ${triangle} does not fit in ${VIS_TRIANGLE_BITS} bits`);
+  }
+  if (cluster < 0 || cluster >= VIS_MAX_CLUSTERS) {
+    throw new RangeError(`cluster ${cluster} does not fit in ${32 - VIS_TRIANGLE_BITS} bits`);
+  }
+  return ((cluster << VIS_TRIANGLE_BITS) | triangle) >>> 0;
+}
+
+export function unpackVisibility(packed: number): { cluster: number; triangle: number } {
+  return {
+    cluster: packed >>> VIS_TRIANGLE_BITS,
+    triangle: packed & (VIS_MAX_TRIANGLES - 1),
+  };
+}
+
+export function visibilityCovered(packed: number): boolean {
+  return packed !== VIS_EMPTY;
+}

package/src/render/halfFloat.ts

@@ -5,6 +5,16 @@ const conversion = new ArrayBuffer(4);
 const asFloat = new Float32Array(conversion);
 const asUint = new Uint32Array(conversion);
 
+/**
+ * The nearest half to `value`, ties to even — IEEE 754's default, and what the network runner's
+ * half-precision weights need, since the reference they are checked against rounds the same way.
+ *
+ * **This truncated until 2026-09-19**, so every value moved toward zero by up to a whole unit in the
+ * last place: an environment cube lost radiance on every texel rather than rounding on each, and a
+ * weight disagreed with the reference's by as much. Rounding costs a few operations per value and
+ * runs once per upload. The carry a round produces is added into the packed bits, so a mantissa that
+ * overflows lifts the exponent — and past 65,504 lands on infinity — with no case of its own.
+ */
 export function halfBits(value: number): number {
   asFloat[0] = value;
   const bits = asUint[0] ?? 0;
@@ -18,17 +28,25 @@ export function halfBits(value: number): number {
   /* Rebias 127 to 15. Above half's range saturates to infinity rather than wrapping to zero. */
   exponent = exponent - 127 + 15;
   if (exponent >= 0x1f) return sign | 0x7c00;
+  let shift = 13;
+  let packed: number;
   if (exponent <= 0) {
     /*
      * Subnormal. An environment's darkest texels live here, and flushing them to zero is what
-     * makes a night sky's gradient band — so they are shifted rather than discarded.
+     * makes a night sky's gradient band — so they are shifted rather than discarded. Below half the
+     * smallest subnormal, nothing is left to round up.
      */
     if (exponent < -10) return sign;
     mantissa |= 0x800000;
-    const shift = 14 - exponent;
-    return sign | (mantissa >>> shift);
+    shift = 14 - exponent;
+    packed = mantissa >>> shift;
+  } else {
+    packed = (exponent << 10) | (mantissa >>> 13);
   }
-  return sign | (exponent << 10) | (mantissa >>> 13);
+  const halfway = 1 << (shift - 1);
+  const rest = mantissa & ((1 << shift) - 1);
+  const up = rest > halfway || (rest === halfway && (packed & 1) === 1);
+  return sign | (packed + (up ? 1 : 0));
 }
 
 /** A whole array of floats as binary16, which is what a `r16float` or `rgba16float` upload wants. */

package/src/render/inference/attentionTiled.ts

@@ -0,0 +1,132 @@
+/**
+ * Attention with keys and values staged in workgroup memory: many queries against many keys, at
+ * head widths that would spill an invocation's accumulator from its registers.
+ *
+ * **A workgroup takes one window's head and as many of its queries as 256 threads hold**, each
+ * thread one query and 16 of the head's channels — so what a thread keeps is 16 values of its output
+ * and as many of its query, where the other two forms keep the whole head; a head of 256 is 16
+ * queries a workgroup, and a head of 64 is 64, and every key staged serves each of them. Keys come
+ * in tiles of 8: the tile is staged, each thread's share of every query-key dot is written out, a
+ * dot's shares are summed, and the same memory then holds the tile's values for the weighted sum. A
+ * query's softmax runs key by key with the running maximum, exactly as the other forms run it, so
+ * the bound that holds for them holds here.
+ *
+ * **What it gives up is barriers**: four a tile, so 2,000 for 4,096 keys, which is why it is chosen
+ * where the work is large enough to pay for them. The memory is under 16 KB at any width — at a
+ * head of 256, all of what a workgroup is promised — and a head wider than that, or one sixteen does
+ * not divide, takes one of the other forms.
+ */
+import {
+  type KernelRequest,
+  type KernelSource,
+  float,
+  header,
+  num,
+  perRow,
+  read,
+} from './kernelKit.ts';
+
+const THREADS = 256;
+const KEYS = 8;
+
+/*
+ * The groups a query's channels split into: 8, or 16 above a head of 128, which measured best on an
+ * RX 9070 XT — 8 took a head of 64 from 1.37 ms at 16 to 1.0 and a head of 96 from 7.3 to 4.8, and
+ * 16 took a head of 256 from 22.4 at 8 to 12.2 (`tools/capture-weights/opbudget.ts`).
+ */
+export function tiledGroups(d: number): number {
+  return d > 128 ? 16 : 8;
+}
+
+/** Whether this form can take a head of `d` channels in `groups` groups. */
+export function tiledAttentionFits(d: number, groups = tiledGroups(d)): boolean {
+  return d % groups === 0 && d <= 256;
+}
+
+/**
+ * The tiled form, each query's channels in `groups` groups and so `256 / groups` queries a
+ * workgroup. `attentionKernel` chooses it; the argument is how `opbudget.ts` measured the choice.
+ */
+export function tiledAttention(request: KernelRequest, groups?: number): KernelSource {
+  const queryShape = request.inputShapes[0] as readonly number[];
+  const [queries, channels] = queryShape.slice(-2) as [number, number];
+  const keys = (request.inputShapes[1] as readonly number[]).at(-2) as number;
+  const batch = queryShape.length === 3 ? (queryShape[0] as number) : 1;
+  const heads = num(request.attributes, 'heads', 1);
+  const d = channels / heads;
+  const GROUPS = groups ?? tiledGroups(d);
+  const share = d / GROUPS;
+  const QUERIES = THREADS / GROUPS;
+  const threads = THREADS;
+  const blocks = Math.ceil(queries / QUERIES);
+  const grid = perRow(batch * heads * blocks);
+  const bias =
+    request.inputShapes.length > 3
+      ? ` + ${read(request, 3, `(head * ${queries}u + min(query, ${queries - 1}u)) * ${keys}u + key`)}`
+      : '';
+  const stage = (
+    input: number,
+  ): string => `  for (var i = local.x; i < ${KEYS * d}u; i = i + ${threads}u) {
+    let key = start + i / ${d}u;
+    tile[i] = select(0.0, ${read(request, input, `(keyRow + min(key, ${keys - 1}u)) * ${channels}u + first + i % ${d}u`)}, key < ${keys}u);
+  }
+  workgroupBarrier();`;
+  return {
+    code: `${header(request)}
+
+var<workgroup> tile: array<f32, ${KEYS * d}>;
+var<workgroup> partial: array<f32, ${QUERIES * KEYS * GROUPS}>;
+
+@compute @workgroup_size(${threads})
+fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(local_invocation_id) local: vec3<u32>) {
+  let block = ${grid.row};
+  if (block >= ${batch * heads * blocks}u) { return; }
+  let window = block / ${heads * blocks}u;
+  let head = (block / ${blocks}u) % ${heads}u;
+  let r = local.x / ${GROUPS}u;
+  let g = local.x % ${GROUPS}u;
+  let query = (block % ${blocks}u) * ${QUERIES}u + r;
+  let row = window * ${queries}u + min(query, ${queries - 1}u);
+  let keyRow = window * ${keys}u;
+  let first = head * ${d}u;
+  let own = g * ${share}u;
+  var mine: array<f32, ${share}>;
+  var gathered: array<f32, ${share}>;
+  for (var c = 0u; c < ${share}u; c = c + 1u) { mine[c] = ${read(request, 0, `row * ${channels}u + first + own + c`)}; }
+  var largest = -3.4028234e38;
+  var total = 0.0;
+  for (var start = 0u; start < ${keys}u; start = start + ${KEYS}u) {
+${stage(1)}
+    for (var k = 0u; k < ${KEYS}u; k = k + 1u) {
+      var dot = 0.0;
+      for (var c = 0u; c < ${share}u; c = c + 1u) { dot = dot + mine[c] * tile[k * ${d}u + own + c]; }
+      partial[(r * ${KEYS}u + k) * ${GROUPS}u + g] = dot;
+    }
+    workgroupBarrier();
+    for (var k = g; k < ${KEYS}u; k = k + ${GROUPS}u) {
+      var sum = 0.0;
+      for (var j = 0u; j < ${GROUPS}u; j = j + 1u) { sum = sum + partial[(r * ${KEYS}u + k) * ${GROUPS}u + j]; }
+      partial[(r * ${KEYS}u + k) * ${GROUPS}u] = sum;
+    }
+${stage(2)}
+    for (var k = 0u; k < ${KEYS}u; k = k + 1u) {
+      let key = start + k;
+      if (key >= ${keys}u) { break; }
+      let score = partial[(r * ${KEYS}u + k) * ${GROUPS}u] * ${float(1 / Math.sqrt(d))}${bias};
+      let top = max(largest, score);
+      let rescale = exp(largest - top);
+      let weight = exp(score - top);
+      total = total * rescale + weight;
+      for (var c = 0u; c < ${share}u; c = c + 1u) { gathered[c] = gathered[c] * rescale + weight * tile[k * ${d}u + own + c]; }
+      largest = top;
+    }
+    workgroupBarrier();
+  }
+  if (query < ${queries}u) {
+    for (var c = 0u; c < ${share}u; c = c + 1u) { output[row * ${channels}u + first + own + c] = gathered[c] / total; }
+  }
+}
+`,
+    workgroups: grid.workgroups,
+  };
+}