@aardworx/wombat.rendering 0.9.33 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/renderObject.d.ts +16 -0
- package/dist/core/renderObject.d.ts.map +1 -1
- package/dist/runtime/derivedModes/kernelCodegen.d.ts +90 -0
- package/dist/runtime/derivedModes/kernelCodegen.d.ts.map +1 -0
- package/dist/runtime/derivedModes/kernelCodegen.js +623 -0
- package/dist/runtime/derivedModes/kernelCodegen.js.map +1 -0
- package/dist/runtime/derivedModes/modeKeyCpu.d.ts.map +1 -1
- package/dist/runtime/derivedModes/modeKeyCpu.js +23 -67
- package/dist/runtime/derivedModes/modeKeyCpu.js.map +1 -1
- package/dist/runtime/derivedModes/partitionDispatcher.d.ts +79 -0
- package/dist/runtime/derivedModes/partitionDispatcher.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionDispatcher.js +303 -0
- package/dist/runtime/derivedModes/partitionDispatcher.js.map +1 -0
- package/dist/runtime/derivedModes/partitionKernel.d.ts +12 -0
- package/dist/runtime/derivedModes/partitionKernel.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionKernel.js +127 -0
- package/dist/runtime/derivedModes/partitionKernel.js.map +1 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.d.ts +4 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.js +24 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.js.map +1 -0
- package/dist/runtime/derivedModes/rule.d.ts +38 -86
- package/dist/runtime/derivedModes/rule.d.ts.map +1 -1
- package/dist/runtime/derivedModes/rule.js +34 -74
- package/dist/runtime/derivedModes/rule.js.map +1 -1
- package/dist/runtime/derivedUniforms/dispatch.d.ts +10 -8
- package/dist/runtime/derivedUniforms/dispatch.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/dispatch.js +31 -18
- package/dist/runtime/derivedUniforms/dispatch.js.map +1 -1
- package/dist/runtime/derivedUniforms/marker.d.ts +90 -11
- package/dist/runtime/derivedUniforms/marker.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/marker.js +161 -15
- package/dist/runtime/derivedUniforms/marker.js.map +1 -1
- package/dist/runtime/derivedUniforms/sceneIntegration.d.ts +25 -6
- package/dist/runtime/derivedUniforms/sceneIntegration.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/sceneIntegration.js +61 -20
- package/dist/runtime/derivedUniforms/sceneIntegration.js.map +1 -1
- package/dist/runtime/heapAdapter.d.ts.map +1 -1
- package/dist/runtime/heapAdapter.js +17 -0
- package/dist/runtime/heapAdapter.js.map +1 -1
- package/dist/runtime/heapEligibility.d.ts.map +1 -1
- package/dist/runtime/heapEligibility.js +22 -2
- package/dist/runtime/heapEligibility.js.map +1 -1
- package/dist/runtime/heapScene/chunkedArena.d.ts +76 -0
- package/dist/runtime/heapScene/chunkedArena.d.ts.map +1 -0
- package/dist/runtime/heapScene/chunkedArena.js +223 -0
- package/dist/runtime/heapScene/chunkedArena.js.map +1 -0
- package/dist/runtime/heapScene/freelist.d.ts +55 -0
- package/dist/runtime/heapScene/freelist.d.ts.map +1 -0
- package/dist/runtime/heapScene/freelist.js +188 -0
- package/dist/runtime/heapScene/freelist.js.map +1 -0
- package/dist/runtime/heapScene/growBuffer.d.ts +23 -2
- package/dist/runtime/heapScene/growBuffer.d.ts.map +1 -1
- package/dist/runtime/heapScene/growBuffer.js +35 -3
- package/dist/runtime/heapScene/growBuffer.js.map +1 -1
- package/dist/runtime/heapScene/pools.d.ts +84 -42
- package/dist/runtime/heapScene/pools.d.ts.map +1 -1
- package/dist/runtime/heapScene/pools.js +309 -134
- package/dist/runtime/heapScene/pools.js.map +1 -1
- package/dist/runtime/heapScene.d.ts +24 -0
- package/dist/runtime/heapScene.d.ts.map +1 -1
- package/dist/runtime/heapScene.js +1158 -92
- package/dist/runtime/heapScene.js.map +1 -1
- package/dist/runtime/hybridScene.d.ts +15 -0
- package/dist/runtime/hybridScene.d.ts.map +1 -1
- package/dist/runtime/hybridScene.js +16 -0
- package/dist/runtime/hybridScene.js.map +1 -1
- package/dist/runtime/index.d.ts +4 -3
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +4 -3
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/renderTask.d.ts +5 -0
- package/dist/runtime/renderTask.d.ts.map +1 -1
- package/dist/runtime/renderTask.js +1 -0
- package/dist/runtime/renderTask.js.map +1 -1
- package/dist/runtime/runtime.d.ts +6 -0
- package/dist/runtime/runtime.d.ts.map +1 -1
- package/dist/runtime/runtime.js +1 -0
- package/dist/runtime/runtime.js.map +1 -1
- package/dist/runtime/textureAtlas/atlasPool.d.ts +12 -0
- package/dist/runtime/textureAtlas/atlasPool.d.ts.map +1 -1
- package/dist/runtime/textureAtlas/atlasPool.js +20 -0
- package/dist/runtime/textureAtlas/atlasPool.js.map +1 -1
- package/package.json +1 -1
- package/src/core/renderObject.ts +16 -0
- package/src/runtime/derivedModes/kernelCodegen.ts +673 -0
- package/src/runtime/derivedModes/modeKeyCpu.ts +22 -60
- package/src/runtime/derivedModes/partitionDispatcher.ts +349 -0
- package/src/runtime/derivedModes/partitionKernelLayout.ts +25 -0
- package/src/runtime/derivedModes/rule.ts +66 -145
- package/src/runtime/derivedUniforms/dispatch.ts +48 -21
- package/src/runtime/derivedUniforms/marker.ts +210 -16
- package/src/runtime/derivedUniforms/sceneIntegration.ts +67 -20
- package/src/runtime/heapAdapter.ts +17 -0
- package/src/runtime/heapEligibility.ts +22 -1
- package/src/runtime/heapScene/chunkedArena.ts +238 -0
- package/src/runtime/heapScene/freelist.ts +196 -0
- package/src/runtime/heapScene/growBuffer.ts +39 -2
- package/src/runtime/heapScene/pools.ts +350 -143
- package/src/runtime/heapScene.ts +1350 -88
- package/src/runtime/hybridScene.ts +31 -0
- package/src/runtime/index.ts +12 -9
- package/src/runtime/renderTask.ts +6 -0
- package/src/runtime/runtime.ts +7 -0
- package/src/runtime/textureAtlas/atlasPool.ts +19 -0
- package/src/runtime/derivedModes/gpuDispatcher.ts +0 -226
- package/src/runtime/derivedModes/gpuKernel.ts +0 -98
package/src/runtime/heapScene.ts
CHANGED
|
@@ -85,7 +85,7 @@ import {
|
|
|
85
85
|
} from "./heapScene/growBuffer.js";
|
|
86
86
|
import {
|
|
87
87
|
UniformPool, IndexPool, DrawHeap,
|
|
88
|
-
AttributeArena, IndexAllocator,
|
|
88
|
+
AttributeArena, IndexAllocator,
|
|
89
89
|
buildArenaState, arenaBytes, writeAttribute,
|
|
90
90
|
asAval, isBufferView, asFloat32,
|
|
91
91
|
ALLOC_HEADER_BYTES, ALLOC_HEADER_PAD_TO,
|
|
@@ -94,7 +94,21 @@ import {
|
|
|
94
94
|
} from "./heapScene/pools.js";
|
|
95
95
|
import { encodeModeKey, type PipelineStateDescriptor } from "./pipelineCache/index.js";
|
|
96
96
|
import { snapshotDescriptor, ModeKeyTracker } from "./derivedModes/modeKeyCpu.js";
|
|
97
|
+
import { GpuPartitionScene } from "./derivedModes/partitionDispatcher.js";
|
|
98
|
+
import {
|
|
99
|
+
emitPartitionKernel,
|
|
100
|
+
substituteReadInputInStmt,
|
|
101
|
+
collectUniformReadsInStmt,
|
|
102
|
+
rewriteOutputsToSlotIndices,
|
|
103
|
+
} from "./derivedModes/kernelCodegen.js";
|
|
104
|
+
import { type DerivedModeRule, type ModeAxis } from "./derivedModes/rule.js";
|
|
105
|
+
import type { CullMode, FrontFace, Topology } from "./pipelineCache/index.js";
|
|
97
106
|
import { addMarkingCallback } from "@aardworx/wombat.adaptive";
|
|
107
|
+
import {
|
|
108
|
+
analyseOutputSet, evaluateStructural, evaluateStructuralSet,
|
|
109
|
+
stableStringify,
|
|
110
|
+
type Expr, type Stmt,
|
|
111
|
+
} from "@aardworx/wombat.shader/ir";
|
|
98
112
|
|
|
99
113
|
// GrowBuffer + ALIGN16 + MIN_BUFFER_BYTES + POW2 live in ./heapScene/growBuffer.
|
|
100
114
|
// Packers (WGSL-type → JS-value → arena bytes) live in ./heapScene/packers.
|
|
@@ -160,6 +174,10 @@ interface BucketSlot {
|
|
|
160
174
|
pipeline: GPURenderPipeline;
|
|
161
175
|
/** modeKey this slot covers — the bitfield-encoded descriptor. */
|
|
162
176
|
modeKey: bigint;
|
|
177
|
+
/** Snapshotted full descriptor the slot was created with. Used by
|
|
178
|
+
* CPU→GPU promotion to synthesise per-distinct-descriptor combos
|
|
179
|
+
* for pre-existing CPU-routed ROs. */
|
|
180
|
+
cpuDescriptor?: PipelineStateDescriptor;
|
|
163
181
|
/** Render bind group (drawTable + firstDrawInTile are per-slot). */
|
|
164
182
|
bindGroup?: GPUBindGroup;
|
|
165
183
|
|
|
@@ -190,10 +208,57 @@ interface BucketSlot {
|
|
|
190
208
|
scanDirty: boolean;
|
|
191
209
|
}
|
|
192
210
|
|
|
211
|
+
/**
|
|
212
|
+
* One distinct derived-mode rule registered on a GPU-routed bucket,
|
|
213
|
+
* keyed by axis. Multiple ROs in the same bucket may carry different
|
|
214
|
+
* rules across different axes; each axis maintains its own
|
|
215
|
+
* `axisRuleId` namespace. A bucket-unique `comboId` (a chosen
|
|
216
|
+
* tuple of (axis → axisRuleId, or "no rule") per RO) is baked into
|
|
217
|
+
* master-pool records and selects the right composer fn in the
|
|
218
|
+
* partition kernel.
|
|
219
|
+
*/
|
|
220
|
+
interface AxisRuleEntry {
|
|
221
|
+
readonly axis: ModeAxis;
|
|
222
|
+
readonly axisRuleId: number;
|
|
223
|
+
readonly contentHash: string;
|
|
224
|
+
readonly rule: DerivedModeRule<ModeAxis>;
|
|
225
|
+
readonly body: Stmt;
|
|
226
|
+
/** Last declared u32 we re-specialised against — drives the
|
|
227
|
+
* dispatch-time "needs re-specialise?" check. */
|
|
228
|
+
lastDeclaredU32: number | undefined;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* One combination of rules carried by ROs in a GPU-routed bucket.
|
|
233
|
+
* A combo specifies at most one rule per axis (axes absent from the
|
|
234
|
+
* map fall back to baseDescriptor at a compile-time-constant per-
|
|
235
|
+
* axis index). Master records store `comboId` — the partition
|
|
236
|
+
* kernel emits one `combo_<id>` fn per combo composing per-axis
|
|
237
|
+
* indices into a global slot index via mixed-radix encoding.
|
|
238
|
+
*/
|
|
239
|
+
interface ComboEntry {
|
|
240
|
+
readonly comboId: number;
|
|
241
|
+
readonly comboKey: string;
|
|
242
|
+
/** axis → axisRuleId for axes this combo has a rule on. */
|
|
243
|
+
readonly axisRules: ReadonlyMap<ModeAxis, number>;
|
|
244
|
+
/** axis → explicit fixed value for axes the combo has no rule on
|
|
245
|
+
* but wants to override baseDescriptor's value (e.g. pre-existing
|
|
246
|
+
* CPU-routed ROs whose pipelineState differs from baseDescriptor
|
|
247
|
+
* on a non-ruled axis). Axes absent from both maps fall back to
|
|
248
|
+
* baseDescriptor at codegen time. */
|
|
249
|
+
readonly axisFixedValues: ReadonlyMap<ModeAxis, unknown>;
|
|
250
|
+
}
|
|
251
|
+
|
|
193
252
|
interface Bucket {
|
|
194
253
|
readonly label: string;
|
|
195
254
|
readonly textures: HeapTextureSet | undefined;
|
|
196
255
|
readonly layout: BucketLayout;
|
|
256
|
+
/** §3: which arena chunk this bucket's ROs live in. The bucket's
|
|
257
|
+
* bind groups reference `arena.attrs.chunk(chunkIdx).buffer` and
|
|
258
|
+
* `arena.indices.chunk(chunkIdx).buffer`. Two ROs sharing an
|
|
259
|
+
* effect that landed in different chunks live in DIFFERENT
|
|
260
|
+
* buckets — encode emits a draw per (effect, chunkIdx). */
|
|
261
|
+
readonly chunkIdx: number;
|
|
197
262
|
/**
|
|
198
263
|
* One entry per distinct pipeline state. Phase 5a: always length 1
|
|
199
264
|
* (per-RO pipeline-state still keyed at the bucket level).
|
|
@@ -276,6 +341,60 @@ interface Bucket {
|
|
|
276
341
|
* for slots without an atlas source aval.
|
|
277
342
|
*/
|
|
278
343
|
readonly localAtlasArrIdx: (number | undefined)[];
|
|
344
|
+
|
|
345
|
+
// ─── Phase 5c.3 GPU-routed bucket state ─────────────────────────
|
|
346
|
+
/**
|
|
347
|
+
* True when this bucket's first RO carried a GPU-flavoured mode
|
|
348
|
+
* rule (e.g. `gpuFlipCullByDeterminant`). The partition kernel
|
|
349
|
+
* decides per-record routing each frame; CPU never reslots.
|
|
350
|
+
*/
|
|
351
|
+
gpuRouted: boolean;
|
|
352
|
+
/**
|
|
353
|
+
* Per-bucket partition dispatcher. Owns the master record buffer
|
|
354
|
+
* + per-slot atomic counters + params. Slot draw buffers are
|
|
355
|
+
* externally owned (point at `bucket.slots[i].drawTableBuf.buffer`).
|
|
356
|
+
*/
|
|
357
|
+
partitionScene: GpuPartitionScene | undefined;
|
|
358
|
+
/**
|
|
359
|
+
* drawId → master record index (for swap-pop on removeDraw).
|
|
360
|
+
*/
|
|
361
|
+
drawIdToRecord: Map<number, number> | undefined;
|
|
362
|
+
/**
|
|
363
|
+
* master record index → drawId (for fixup when swap-pop moves the
|
|
364
|
+
* tail record into the freed slot).
|
|
365
|
+
*/
|
|
366
|
+
recordToDrawId: number[] | undefined;
|
|
367
|
+
/**
|
|
368
|
+
* Per-axis distinct rules registered on this bucket, keyed by
|
|
369
|
+
* content hash (axis + expr.id + declared identity). `axisRuleId`
|
|
370
|
+
* is unique within an axis. ROs carry an arbitrary subset of axis
|
|
371
|
+
* rules; combos (below) dedupe distinct subsets across ROs.
|
|
372
|
+
*/
|
|
373
|
+
rulesByAxis: Map<ModeAxis, Map<string, AxisRuleEntry>> | undefined;
|
|
374
|
+
/** Per-axis ordered list of registered rule hashes; index =
|
|
375
|
+
* `axisRuleId` for that axis. */
|
|
376
|
+
axisRuleOrder: Map<ModeAxis, string[]> | undefined;
|
|
377
|
+
/** Bucket-wide ordered list of arena uniform names read by any
|
|
378
|
+
* rule body. Each entry maps to a `ref<i>: u32` field in the
|
|
379
|
+
* master-record uniform tail; appendRecord packs them per RO. */
|
|
380
|
+
uniformOrder: string[] | undefined;
|
|
381
|
+
uniformIdx: Map<string, number> | undefined;
|
|
382
|
+
/** Distinct combos appearing on ROs in this bucket. */
|
|
383
|
+
combosByKey: Map<string, ComboEntry> | undefined;
|
|
384
|
+
/** `comboId` → comboKey. Index = comboId. */
|
|
385
|
+
comboOrder: string[] | undefined;
|
|
386
|
+
/** drawId → comboId. Master records carry the comboId; this is
|
|
387
|
+
* the CPU-side mirror used when an RO is removed. */
|
|
388
|
+
drawIdToComboId: Map<number, number> | undefined;
|
|
389
|
+
/** Seed PipelineStateDescriptor (the descriptor of the first GPU-
|
|
390
|
+
* routed RO). Used as the base for slot descriptors. */
|
|
391
|
+
baseDescriptor: PipelineStateDescriptor | undefined;
|
|
392
|
+
/**
|
|
393
|
+
* Has the partition kernel's inputs (records or rule deps)
|
|
394
|
+
* changed since last dispatch? Set true on addRO / removeDraw /
|
|
395
|
+
* any tracker-leaf mark. Reset after dispatch.
|
|
396
|
+
*/
|
|
397
|
+
partitionDirty: boolean;
|
|
279
398
|
}
|
|
280
399
|
|
|
281
400
|
// ---------------------------------------------------------------------------
|
|
@@ -319,6 +438,18 @@ export interface HeapDrawSpec {
|
|
|
319
438
|
* computes `indexCount * instanceCount` per record.
|
|
320
439
|
*/
|
|
321
440
|
readonly instanceCount?: aval<number> | number;
|
|
441
|
+
/**
|
|
442
|
+
* Optional visibility gate. When ticks to `false`, this RO emits
|
|
443
|
+
* 0 vertices that frame — its drawTable record gets effective
|
|
444
|
+
* `indexCount = 0`, the scan kernel's prefix sum skips past it,
|
|
445
|
+
* and no fragment shader invocations fire. Pool allocations,
|
|
446
|
+
* arena uploads, and bucket membership all stay intact, so the
|
|
447
|
+
* `true → false → true` round-trip is a pair of small drawTable
|
|
448
|
+
* writes plus a scan-pass re-issue — no `addDraw`/`removeDraw`
|
|
449
|
+
* churn. Designed for `<Sg Active={cval}>` driven by LOD walkers
|
|
450
|
+
* that flip many tiles per camera move.
|
|
451
|
+
*/
|
|
452
|
+
readonly active?: aval<boolean>;
|
|
322
453
|
/**
|
|
323
454
|
* Index buffer for this draw. Indices live in their own `INDEX`-
|
|
324
455
|
* usage `GPUBuffer` (WebGPU forces this), separate from the arena.
|
|
@@ -570,6 +701,18 @@ export interface BuildHeapSceneOptions {
|
|
|
570
701
|
* effect's `spec.inputs[name]` like any other uniform.
|
|
571
702
|
*/
|
|
572
703
|
readonly enableDerivedUniforms?: boolean;
|
|
704
|
+
/**
|
|
705
|
+
* §3 — per-arena-chunk size cap in bytes. When omitted, falls
|
|
706
|
+
* back to `min(device.limits.maxStorageBufferBindingSize,
|
|
707
|
+
* DEFAULT_MAX_BUFFER_BYTES = 256 MB)` so chunks grow as far as
|
|
708
|
+
* the hardware allows. Demos / tests can pass a small value
|
|
709
|
+
* (e.g. 4 MB) to exercise multi-chunk routing without realistic
|
|
710
|
+
* workload pressure. Each chunk lives in its own GPUBuffer;
|
|
711
|
+
* buckets bind exactly one chunk and emit one draw call per
|
|
712
|
+
* (effect, chunkIdx). Shared-aval avals that land in different
|
|
713
|
+
* chunks duplicate their pool entries (accepted trade-off).
|
|
714
|
+
*/
|
|
715
|
+
readonly maxChunkBytes?: number;
|
|
573
716
|
}
|
|
574
717
|
|
|
575
718
|
export function buildHeapScene(
|
|
@@ -607,6 +750,7 @@ export function buildHeapScene(
|
|
|
607
750
|
const arena = buildArenaState(
|
|
608
751
|
device, 64 * 1024, 16 * 1024, "heapScene",
|
|
609
752
|
GPUBufferUsage.STORAGE,
|
|
753
|
+
opts.maxChunkBytes,
|
|
610
754
|
);
|
|
611
755
|
|
|
612
756
|
// ─── Per-draw global bookkeeping (sparse, indexed by drawId) ──────
|
|
@@ -642,6 +786,90 @@ export function buildHeapScene(
|
|
|
642
786
|
const modeAvalToDrawIds = new Map<aval<unknown>, Set<number>>();
|
|
643
787
|
const modeAvalCallbacks = new Map<aval<unknown>, IDisposable>();
|
|
644
788
|
|
|
789
|
+
// ─── HeapDrawSpec.active subscription ────────────────────────────────
|
|
790
|
+
// Per-drawId state for the "skip without re-allocating" path. When a
|
|
791
|
+
// tile (or any RO) flips visibility many times per second (e.g. an
|
|
792
|
+
// LOD walker culling/uncrossing the frustum), pool churn would be
|
|
793
|
+
// prohibitive. Instead we keep the RO in the bucket and flip its
|
|
794
|
+
// drawTable record's `indexCount` field between the original count
|
|
795
|
+
// (visible) and 0 (hidden). The scan kernel's prefix sum naturally
|
|
796
|
+
// skips a record with 0 emit count, so this costs one drawTable
|
|
797
|
+
// write + a re-scan dispatch — no arena/freelist activity.
|
|
798
|
+
const drawIdToActiveAval = new Map<number, aval<boolean>>();
|
|
799
|
+
const drawIdToOrigIndexCount = new Map<number, number>();
|
|
800
|
+
const drawIdToActiveCallback = new Map<number, IDisposable>();
|
|
801
|
+
const activeAvalToDrawIds = new Map<aval<boolean>, Set<number>>();
|
|
802
|
+
const activeDirty = new Set<number>();
|
|
803
|
+
|
|
804
|
+
function subscribeActive(av: aval<boolean>, drawId: number): void {
|
|
805
|
+
let set = activeAvalToDrawIds.get(av);
|
|
806
|
+
if (set === undefined) {
|
|
807
|
+
set = new Set<number>();
|
|
808
|
+
activeAvalToDrawIds.set(av, set);
|
|
809
|
+
}
|
|
810
|
+
set.add(drawId);
|
|
811
|
+
const cb = addMarkingCallback(av, () => { activeDirty.add(drawId); });
|
|
812
|
+
drawIdToActiveCallback.set(drawId, cb);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function unsubscribeActive(drawId: number): void {
|
|
816
|
+
const av = drawIdToActiveAval.get(drawId);
|
|
817
|
+
if (av === undefined) return;
|
|
818
|
+
const cb = drawIdToActiveCallback.get(drawId);
|
|
819
|
+
cb?.dispose();
|
|
820
|
+
drawIdToActiveCallback.delete(drawId);
|
|
821
|
+
const set = activeAvalToDrawIds.get(av);
|
|
822
|
+
if (set !== undefined) {
|
|
823
|
+
set.delete(drawId);
|
|
824
|
+
if (set.size === 0) activeAvalToDrawIds.delete(av);
|
|
825
|
+
}
|
|
826
|
+
drawIdToActiveAval.delete(drawId);
|
|
827
|
+
drawIdToOrigIndexCount.delete(drawId);
|
|
828
|
+
activeDirty.delete(drawId);
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/** Update the drawTable record's effective indexCount in place
|
|
832
|
+
* (GPU-routed: masterShadow + partition re-dispatch; legacy:
|
|
833
|
+
* drawTableShadow + scan re-dispatch). Common helper used by the
|
|
834
|
+
* active-aval drain in `update()`. */
|
|
835
|
+
function setEffectiveIndexCount(drawId: number, newIndexCount: number): void {
|
|
836
|
+
const bucket = drawIdToBucket[drawId];
|
|
837
|
+
if (bucket === undefined) return;
|
|
838
|
+
if (bucket.gpuRouted) {
|
|
839
|
+
const partition = bucket.partitionScene!;
|
|
840
|
+
const recIdx = bucket.drawIdToRecord!.get(drawId);
|
|
841
|
+
if (recIdx === undefined) return;
|
|
842
|
+
const ru32 = partition.recordU32;
|
|
843
|
+
const oldIndexCount = partition.masterShadow[recIdx * ru32 + 3]!;
|
|
844
|
+
const instanceCount = partition.masterShadow[recIdx * ru32 + 4]!;
|
|
845
|
+
partition.masterShadow[recIdx * ru32 + 3] = newIndexCount >>> 0;
|
|
846
|
+
const delta = (newIndexCount - oldIndexCount) * instanceCount;
|
|
847
|
+
bucket.partitionDirty = true;
|
|
848
|
+
for (const s of bucket.slots) {
|
|
849
|
+
s.totalEmitEstimate = Math.max(0, s.totalEmitEstimate + delta);
|
|
850
|
+
s.scanDirty = true;
|
|
851
|
+
}
|
|
852
|
+
} else {
|
|
853
|
+
const slotIdx = drawIdToSlotIdx[drawId];
|
|
854
|
+
const slot = slotIdx !== undefined ? bucket.slots[slotIdx] : bucket.slots[0];
|
|
855
|
+
if (slot === undefined) return;
|
|
856
|
+
const localSlot = drawIdToLocalSlot[drawId];
|
|
857
|
+
if (localSlot === undefined) return;
|
|
858
|
+
const recIdx = slot.slotToRecord[localSlot];
|
|
859
|
+
if (recIdx === undefined || recIdx < 0) return;
|
|
860
|
+
const shadow = slot.drawTableShadow!;
|
|
861
|
+
const oldIndexCount = shadow[recIdx * RECORD_U32 + 3]!;
|
|
862
|
+
const instanceCount = shadow[recIdx * RECORD_U32 + 4]!;
|
|
863
|
+
shadow[recIdx * RECORD_U32 + 3] = newIndexCount >>> 0;
|
|
864
|
+
const byteOff = recIdx * RECORD_BYTES + 3 * 4;
|
|
865
|
+
if (byteOff < slot.drawTableDirtyMin) slot.drawTableDirtyMin = byteOff;
|
|
866
|
+
if (byteOff + 4 > slot.drawTableDirtyMax) slot.drawTableDirtyMax = byteOff + 4;
|
|
867
|
+
const delta = (newIndexCount - oldIndexCount) * instanceCount;
|
|
868
|
+
slot.totalEmitEstimate = Math.max(0, slot.totalEmitEstimate + delta);
|
|
869
|
+
slot.scanDirty = true;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
|
|
645
873
|
function subscribeModeLeaf(av: aval<unknown>, drawId: number): void {
|
|
646
874
|
let set = modeAvalToDrawIds.get(av);
|
|
647
875
|
if (set === undefined) {
|
|
@@ -867,7 +1095,7 @@ export function buildHeapScene(
|
|
|
867
1095
|
override inputChanged(_t: unknown, o: IAdaptiveObject): void {
|
|
868
1096
|
// Pool avals are stored as `aval<unknown>`; the IAdaptiveObject
|
|
869
1097
|
// identity matches.
|
|
870
|
-
if (pool.
|
|
1098
|
+
if (pool.hasAny(o as unknown as aval<unknown>)) {
|
|
871
1099
|
allocDirty.add(o as unknown as aval<unknown>);
|
|
872
1100
|
return;
|
|
873
1101
|
}
|
|
@@ -892,11 +1120,21 @@ export function buildHeapScene(
|
|
|
892
1120
|
// `enableDerivedUniforms: false` to opt out.
|
|
893
1121
|
const enableDerivedUniforms = opts.enableDerivedUniforms !== false;
|
|
894
1122
|
const derivedScene: DerivedUniformsScene | undefined = enableDerivedUniforms
|
|
895
|
-
? new DerivedUniformsScene(device,
|
|
1123
|
+
? new DerivedUniformsScene(device, {
|
|
896
1124
|
// Larger initial capacity reduces grow-during-first-frame churn.
|
|
897
1125
|
initialConstituentSlots: 4096,
|
|
898
1126
|
})
|
|
899
1127
|
: undefined;
|
|
1128
|
+
// §3: register each arena chunk's main-heap buffer with the
|
|
1129
|
+
// derived-uniforms scene; refresh the binding when chunks open or
|
|
1130
|
+
// a chunk's GrowBuffer reallocates.
|
|
1131
|
+
if (derivedScene !== undefined) {
|
|
1132
|
+
const wireChunk = (idx: number): void => {
|
|
1133
|
+
derivedScene.setMainHeapForChunk(idx, () => arena.attrs.chunk(idx).buffer);
|
|
1134
|
+
};
|
|
1135
|
+
for (let i = 0; i < arena.attrs.chunkCount; i++) wireChunk(i);
|
|
1136
|
+
arena.attrs.onChunkAdded(wireChunk);
|
|
1137
|
+
}
|
|
900
1138
|
/** Per-RO §7 registration handles, keyed by global drawId.
|
|
901
1139
|
* Drained on removeDraw to release slots + records. */
|
|
902
1140
|
const derivedByDrawId = new Map<number, RoRegistration>();
|
|
@@ -1412,11 +1650,12 @@ export function buildHeapScene(
|
|
|
1412
1650
|
// SAME global arena GPUBuffer (emscripten-style aliasing). The
|
|
1413
1651
|
// WGSL prelude declares one binding per view; the shader picks
|
|
1414
1652
|
// whichever matches its read.
|
|
1653
|
+
const attrsBuf = arena.attrs.chunk(bucket.chunkIdx).buffer;
|
|
1415
1654
|
const entries: GPUBindGroupEntry[] = [
|
|
1416
|
-
{ binding: 0, resource: { buffer:
|
|
1655
|
+
{ binding: 0, resource: { buffer: attrsBuf } }, // heapU32
|
|
1417
1656
|
{ binding: 1, resource: { buffer: bucket.drawHeap.buffer } }, // headersU32
|
|
1418
|
-
{ binding: 2, resource: { buffer:
|
|
1419
|
-
{ binding: 3, resource: { buffer:
|
|
1657
|
+
{ binding: 2, resource: { buffer: attrsBuf } }, // heapF32
|
|
1658
|
+
{ binding: 3, resource: { buffer: attrsBuf } }, // heapV4f
|
|
1420
1659
|
];
|
|
1421
1660
|
{
|
|
1422
1661
|
const s = bucket.slots[slotIdx]!;
|
|
@@ -1431,7 +1670,7 @@ export function buildHeapScene(
|
|
|
1431
1670
|
const dtBytes = Math.max(RECORD_BYTES, s.recordCount * RECORD_BYTES);
|
|
1432
1671
|
entries.push(
|
|
1433
1672
|
{ binding: 4, resource: { buffer: s.drawTableBuf.buffer, offset: 0, size: dtBytes } },
|
|
1434
|
-
{ binding: 5, resource: { buffer: arena.indices.buffer } },
|
|
1673
|
+
{ binding: 5, resource: { buffer: arena.indices.chunk(bucket.chunkIdx).buffer } },
|
|
1435
1674
|
{ binding: 6, resource: { buffer: s.firstDrawInTileBuf!.buffer } },
|
|
1436
1675
|
);
|
|
1437
1676
|
s.renderBoundRecordCount = s.recordCount;
|
|
@@ -1541,16 +1780,37 @@ export function buildHeapScene(
|
|
|
1541
1780
|
}
|
|
1542
1781
|
}
|
|
1543
1782
|
|
|
1544
|
-
arena.attrs.
|
|
1545
|
-
|
|
1546
|
-
|
|
1783
|
+
arena.attrs.onAnyResize((resizedChunkIdx) => {
|
|
1784
|
+
// Only rebuild bind groups for buckets that bind THIS chunk —
|
|
1785
|
+
// a resize in chunk K doesn't affect a bucket reading chunk N≠K.
|
|
1786
|
+
for (const b of buckets) {
|
|
1787
|
+
if (b.chunkIdx !== resizedChunkIdx) continue;
|
|
1788
|
+
for (let i = 0; i < b.slots.length; i++) {
|
|
1789
|
+
b.slots[i]!.bindGroup = buildBucketBindGroup(b, i);
|
|
1790
|
+
}
|
|
1791
|
+
if (b.gpuRouted && b.partitionScene !== undefined) {
|
|
1792
|
+
b.partitionScene.rebindSlotDrawBufs(b.slots.map(s => s.drawTableBuf!.buffer));
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
// §7: each chunk's main-heap getter is registered via
|
|
1796
|
+
// `derivedScene.setMainHeapForChunk` (at construction + on
|
|
1797
|
+
// onChunkAdded). The getter returns the chunk's current
|
|
1798
|
+
// GPUBuffer; the dispatcher's bind groups consult it per
|
|
1799
|
+
// dispatch, so no explicit rebind needed here.
|
|
1547
1800
|
});
|
|
1548
1801
|
if (atlasPool !== undefined) {
|
|
1549
1802
|
for (const f of ATLAS_PAGE_FORMATS) {
|
|
1550
1803
|
atlasPool.onPageAdded(f, () => rebuildAllBindGroups(true));
|
|
1551
1804
|
}
|
|
1552
1805
|
}
|
|
1553
|
-
arena.indices.
|
|
1806
|
+
arena.indices.onAnyResize((resizedChunkIdx) => {
|
|
1807
|
+
for (const b of buckets) {
|
|
1808
|
+
if (b.chunkIdx !== resizedChunkIdx) continue;
|
|
1809
|
+
for (let i = 0; i < b.slots.length; i++) {
|
|
1810
|
+
b.slots[i]!.bindGroup = buildBucketBindGroup(b, i);
|
|
1811
|
+
}
|
|
1812
|
+
}
|
|
1813
|
+
});
|
|
1554
1814
|
|
|
1555
1815
|
// ─── findOrCreateBucket ───────────────────────────────────────────
|
|
1556
1816
|
// Slice 3c: the bucket key collapses to (familyId, pipelineState).
|
|
@@ -1604,6 +1864,7 @@ export function buildHeapScene(
|
|
|
1604
1864
|
|
|
1605
1865
|
const slot: BucketSlot = {
|
|
1606
1866
|
pipeline, modeKey,
|
|
1867
|
+
cpuDescriptor: descriptor,
|
|
1607
1868
|
drawTableDirtyMin: Infinity, drawTableDirtyMax: 0,
|
|
1608
1869
|
recordCount: 0, slotToRecord: [], recordToSlot: [],
|
|
1609
1870
|
totalEmitEstimate: 0, scanDirty: false,
|
|
@@ -1632,6 +1893,11 @@ export function buildHeapScene(
|
|
|
1632
1893
|
rebuildScanBg();
|
|
1633
1894
|
slot.drawTableDirtyMin = 0;
|
|
1634
1895
|
slot.drawTableDirtyMax = slot.recordCount * RECORD_BYTES;
|
|
1896
|
+
// Phase 5c.3: partition's bind group references every slot's
|
|
1897
|
+
// drawTable. When ANY slot's GPUBuffer reallocates we rebind all.
|
|
1898
|
+
if (bucket.gpuRouted && bucket.partitionScene !== undefined) {
|
|
1899
|
+
bucket.partitionScene.rebindSlotDrawBufs(bucket.slots.map(s => s.drawTableBuf!.buffer));
|
|
1900
|
+
}
|
|
1635
1901
|
});
|
|
1636
1902
|
blockSumsBuf.onResize(rebuildScanBg);
|
|
1637
1903
|
blockOffsetsBuf.onResize(rebuildScanBg);
|
|
@@ -1648,10 +1914,721 @@ export function buildHeapScene(
|
|
|
1648
1914
|
/** Side map from Bucket → family. Populated by findOrCreateBucket. */
|
|
1649
1915
|
const familyForBucket = new WeakMap<Bucket, ReturnType<typeof familyFor>>();
|
|
1650
1916
|
|
|
1917
|
+
/** Canonical axis order — every part of the GPU routing path uses
|
|
1918
|
+
* this ordering so the kernel's params layout, the per-slot mixed-
|
|
1919
|
+
* radix mode key, and the declared-aval lookup stay in lockstep. */
|
|
1920
|
+
const AXIS_ORDER: ReadonlyArray<ModeAxis> = [
|
|
1921
|
+
"cull", "frontFace", "topology",
|
|
1922
|
+
"depthCompare", "depthWrite", "alphaToCoverage",
|
|
1923
|
+
];
|
|
1924
|
+
|
|
1925
|
+
function collectRules(modeRules: NonNullable<HeapDrawSpec["modeRules"]>): DerivedModeRule<ModeAxis>[] {
|
|
1926
|
+
const out: DerivedModeRule<ModeAxis>[] = [];
|
|
1927
|
+
for (const axis of AXIS_ORDER) {
|
|
1928
|
+
const r = (modeRules as Record<string, DerivedModeRule<ModeAxis> | undefined>)[axis];
|
|
1929
|
+
if (r !== undefined) out.push(r);
|
|
1930
|
+
}
|
|
1931
|
+
return out;
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
/** Apply a per-axis enum value to a `PipelineStateDescriptor` and
|
|
1935
|
+
* return the patched copy. Used to materialise one descriptor per
|
|
1936
|
+
* slot (= one per cartesian-product point across the rule domains). */
|
|
1937
|
+
function patchDescriptor(
|
|
1938
|
+
base: PipelineStateDescriptor,
|
|
1939
|
+
axis: ModeAxis,
|
|
1940
|
+
value: unknown,
|
|
1941
|
+
): PipelineStateDescriptor {
|
|
1942
|
+
switch (axis) {
|
|
1943
|
+
case "cull": return { ...base, cullMode: value as CullMode };
|
|
1944
|
+
case "frontFace": return { ...base, frontFace: value as FrontFace };
|
|
1945
|
+
case "topology": {
|
|
1946
|
+
const topology = value as Topology;
|
|
1947
|
+
return { ...base, topology, stripIndexFormat: stripFormatForTopology(topology) };
|
|
1948
|
+
}
|
|
1949
|
+
case "depthCompare":
|
|
1950
|
+
return base.depth !== undefined
|
|
1951
|
+
? { ...base, depth: { ...base.depth, compare: value as GPUCompareFunction } }
|
|
1952
|
+
: base;
|
|
1953
|
+
case "depthWrite":
|
|
1954
|
+
return base.depth !== undefined
|
|
1955
|
+
? { ...base, depth: { ...base.depth, write: value as boolean } }
|
|
1956
|
+
: base;
|
|
1957
|
+
case "alphaToCoverage": return { ...base, alphaToCoverage: value as boolean };
|
|
1958
|
+
case "blend": {
|
|
1959
|
+
// Blend is per-attachment; apply the resolved AttachmentBlend
|
|
1960
|
+
// to attachment 0 (the main color target). Multi-attachment
|
|
1961
|
+
// structured rules can extend this when needed.
|
|
1962
|
+
const blend = value as import("./pipelineCache/index.js").AttachmentBlend;
|
|
1963
|
+
const atts = base.attachments.slice();
|
|
1964
|
+
if (atts.length === 0) atts.push(blend);
|
|
1965
|
+
else atts[0] = blend;
|
|
1966
|
+
return { ...base, attachments: atts };
|
|
1967
|
+
}
|
|
1968
|
+
default: return base;
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
function stripFormatForTopology(t: Topology): GPUIndexFormat | undefined {
|
|
1973
|
+
return t === "line-strip" || t === "triangle-strip" ? "uint32" : undefined;
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/** Per-axis enum order. The rule body produces u32 indices into
|
|
1977
|
+
* this array, and the heap renderer uses the same order to map
|
|
1978
|
+
* resolved-set integers back to pipeline descriptor values. The
|
|
1979
|
+
* ordering MUST match the bitfield encoder in pipelineCache. */
|
|
1980
|
+
const AXIS_ENUM_TABLE: { readonly [A in ModeAxis]: ReadonlyArray<unknown> } = {
|
|
1981
|
+
cull: ["none", "front", "back"],
|
|
1982
|
+
frontFace: ["ccw", "cw"],
|
|
1983
|
+
topology: ["point-list", "line-list", "line-strip", "triangle-list", "triangle-strip"],
|
|
1984
|
+
depthCompare: ["never", "less", "equal", "less-equal", "greater", "not-equal", "greater-equal", "always"],
|
|
1985
|
+
depthWrite: [false, true],
|
|
1986
|
+
alphaToCoverage: [false, true],
|
|
1987
|
+
// Blend has no canonical enum table — the rule's `resolve` /
|
|
1988
|
+
// `values` callback MUST be supplied so the heap renderer knows
|
|
1989
|
+
// how to map u32 outputs to AttachmentBlend objects.
|
|
1990
|
+
blend: [],
|
|
1991
|
+
};
|
|
1992
|
+
|
|
1993
|
+
/**
|
|
1994
|
+
* Map a resolved slot value (from `evaluateStructuralSet`) to the
|
|
1995
|
+
* axis-shape value used for descriptor patching.
|
|
1996
|
+
*
|
|
1997
|
+
* - If the resolved value is a JS object: it's already the full
|
|
1998
|
+
* axis value (e.g. an `AttachmentBlend` from a blend rule's
|
|
1999
|
+
* object-literal return) — use it directly.
|
|
2000
|
+
* - If it's a number: for enum axes, look up the canonical
|
|
2001
|
+
* table; for structured axes without a table, pass through.
|
|
2002
|
+
*/
|
|
2003
|
+
function resolveAxisValue<A extends ModeAxis>(
|
|
2004
|
+
rule: DerivedModeRule<A>,
|
|
2005
|
+
value: unknown,
|
|
2006
|
+
): unknown {
|
|
2007
|
+
if (typeof value === "object" && value !== null) return value;
|
|
2008
|
+
if (typeof value === "number") {
|
|
2009
|
+
const table = AXIS_ENUM_TABLE[rule.axis];
|
|
2010
|
+
if (table.length > 0) {
|
|
2011
|
+
const v = table[value];
|
|
2012
|
+
if (v !== undefined) return v;
|
|
2013
|
+
}
|
|
2014
|
+
return value;
|
|
2015
|
+
}
|
|
2016
|
+
return value;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
/**
|
|
2020
|
+
* Resolve an axis's `declared` value to a u32 index.
|
|
2021
|
+
*
|
|
2022
|
+
* - omitted: `0` (the rule body either doesn't reference `declared`
|
|
2023
|
+
* or wants the canonical zero of its axis as the default).
|
|
2024
|
+
* - number: passed through directly (e.g. `declared: 2` for cull "back").
|
|
2025
|
+
* - aval / string / boolean: looked up in the canonical enum table.
|
|
2026
|
+
*/
|
|
2027
|
+
function resolveDeclaredU32<A extends ModeAxis>(rule: DerivedModeRule<A>, tok: AdaptiveToken): number {
|
|
2028
|
+
const d = rule.declared;
|
|
2029
|
+
if (d === undefined) return 0;
|
|
2030
|
+
const v = (typeof d === "object" && d !== null && "getValue" in (d as object))
|
|
2031
|
+
? (d as aval<unknown>).getValue(tok)
|
|
2032
|
+
: d;
|
|
2033
|
+
if (typeof v === "number") return v >>> 0;
|
|
2034
|
+
const table = AXIS_ENUM_TABLE[rule.axis];
|
|
2035
|
+
if (table.length === 0) {
|
|
2036
|
+
throw new Error(
|
|
2037
|
+
`heapScene/gpuRouting: axis '${rule.axis}' has no canonical enum table; ` +
|
|
2038
|
+
`pass \`declared\` as a u32 index or omit it`,
|
|
2039
|
+
);
|
|
2040
|
+
}
|
|
2041
|
+
const i = table.indexOf(v);
|
|
2042
|
+
if (i < 0) {
|
|
2043
|
+
throw new Error(
|
|
2044
|
+
`heapScene/gpuRouting: declared value '${String(v)}' for axis '${rule.axis}' ` +
|
|
2045
|
+
`not in canonical enum table ${JSON.stringify(table)}`,
|
|
2046
|
+
);
|
|
2047
|
+
}
|
|
2048
|
+
return i;
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
/** Extract the rule body Stmt from a RuleExpr's Module. The plugin
|
|
2052
|
+
* emits one `Entry` ValueDef whose body is the user's closure body
|
|
2053
|
+
* (with the synthetic compute-stage scaffolding stripped). */
|
|
2054
|
+
function ruleBodyOf(rule: DerivedModeRule<ModeAxis>): Stmt {
|
|
2055
|
+
const tmpl = rule.expr.template as { values: ReadonlyArray<{ kind: string; entry?: { body: Stmt } }> };
|
|
2056
|
+
for (const v of tmpl.values) {
|
|
2057
|
+
if (v.kind === "Entry" && v.entry !== undefined) return v.entry.body;
|
|
2058
|
+
}
|
|
2059
|
+
throw new Error(
|
|
2060
|
+
`heapScene/gpuRouting: rule for axis '${rule.axis}' has no Entry in its template (id=${rule.expr.id})`,
|
|
2061
|
+
);
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
/**
|
|
2065
|
+
* Specialise a rule for the current declared u32 value:
|
|
2066
|
+
* 1. Bake `declared` as a Const in the rule body (so the kernel
|
|
2067
|
+
* has no per-dispatch uniform — declared marks trigger a
|
|
2068
|
+
* re-specialise + cached-kernel swap).
|
|
2069
|
+
* 2. Run `analyseOutputSet` to collect distinct return Exprs.
|
|
2070
|
+
* 3. `evaluateStructuralSet` folds each Expr to a JS value (a
|
|
2071
|
+
* number for enum axes; a full object for structured axes
|
|
2072
|
+
* like blend). Distinct values define the slot count.
|
|
2073
|
+
* 4. Build a stableStringify-keyed `exprToSlot` map.
|
|
2074
|
+
* 5. Rewrite the body so every ReturnValue emits a Const u32
|
|
2075
|
+
* slot index.
|
|
2076
|
+
* 6. emitPartitionKernel produces the WGSL.
|
|
2077
|
+
*/
|
|
2078
|
+
/** WeakMap-based stable identity for aval objects. Two `cval`s
|
|
2079
|
+
* have different identity; the same cval observed twice has the
|
|
2080
|
+
* same identity. Used in `ruleContentHash` so rules differing
|
|
2081
|
+
* only in their `declared` aval are recognised as distinct. */
|
|
2082
|
+
const avalIdentityMap = new WeakMap<object, number>();
|
|
2083
|
+
let nextAvalIdentity = 0;
|
|
2084
|
+
function avalIdentity(av: object): number {
|
|
2085
|
+
let id = avalIdentityMap.get(av);
|
|
2086
|
+
if (id === undefined) {
|
|
2087
|
+
id = ++nextAvalIdentity;
|
|
2088
|
+
avalIdentityMap.set(av, id);
|
|
2089
|
+
}
|
|
2090
|
+
return id;
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
/** Content hash for rule identity within a bucket. Two rules
|
|
2094
|
+
* hashing the same can share a `ruleId` and one body; rules
|
|
2095
|
+
* hashing differently get distinct ids. */
|
|
2096
|
+
function ruleContentHash(rule: DerivedModeRule<ModeAxis>): string {
|
|
2097
|
+
const dec = rule.declared;
|
|
2098
|
+
let dKey: string;
|
|
2099
|
+
if (dec === undefined) {
|
|
2100
|
+
dKey = "u";
|
|
2101
|
+
} else if (typeof dec === "object" && dec !== null && "getValue" in (dec as object)) {
|
|
2102
|
+
dKey = `a${avalIdentity(dec as object)}`;
|
|
2103
|
+
} else {
|
|
2104
|
+
dKey = `v${String(dec)}`;
|
|
2105
|
+
}
|
|
2106
|
+
return `${rule.axis}|${rule.expr.id}|${dKey}`;
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2109
|
+
/** Read baseDescriptor's value for a given axis (the value used by
|
|
2110
|
+
* combos that have no rule on that axis). */
|
|
2111
|
+
function axisValueOfDesc(desc: PipelineStateDescriptor, axis: ModeAxis): unknown {
|
|
2112
|
+
switch (axis) {
|
|
2113
|
+
case "cull": return desc.cullMode;
|
|
2114
|
+
case "frontFace": return desc.frontFace;
|
|
2115
|
+
case "topology": return desc.topology;
|
|
2116
|
+
case "depthCompare": return desc.depth?.compare;
|
|
2117
|
+
case "depthWrite": return desc.depth?.write;
|
|
2118
|
+
case "alphaToCoverage": return desc.alphaToCoverage;
|
|
2119
|
+
case "blend": return desc.attachments[0];
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
/**
|
|
2124
|
+
* Specialise the WHOLE bucket — N-D cartesian over per-axis
|
|
2125
|
+
* unions:
|
|
2126
|
+
* 1. Per (axis, rule): bake declared as Const, analyse the body,
|
|
2127
|
+
* `evaluateStructuralSet` to fold each return Expr to a JS
|
|
2128
|
+
* value. Collect the per-axis union (dedup by stableStringify,
|
|
2129
|
+
* sorted deterministically). If any combo lacks a rule on
|
|
2130
|
+
* this axis, also include baseDesc's value for that axis.
|
|
2131
|
+
* 2. Active axes = axes with ≥1 registered rule, in AXIS_ORDER.
|
|
2132
|
+
* Cardinalities Cₐ = |axisValues[a]|; totalSlots = ∏ Cₐ.
|
|
2133
|
+
* 3. Per axis-rule: rewrite the body so every `ReturnValue`
|
|
2134
|
+
* emits a `Const u32` PER-AXIS index (into axisValues[axis]).
|
|
2135
|
+
* 4. Per combo: assemble a ComboCodegenSpec (per-axis source =
|
|
2136
|
+
* rule or compile-time const index).
|
|
2137
|
+
* 5. Emit the kernel.
|
|
2138
|
+
*
|
|
2139
|
+
* Returns the cartesian slot descriptors + the kernel WGSL.
|
|
2140
|
+
*/
|
|
2141
|
+
function specialiseBucket(
|
|
2142
|
+
bucket: Bucket,
|
|
2143
|
+
): { slotDescs: ReadonlyArray<PipelineStateDescriptor>; kernelWGSL: string } {
|
|
2144
|
+
const Tu32 = { kind: "Int", signed: false, width: 32 } as const;
|
|
2145
|
+
const rulesByAxis = bucket.rulesByAxis!;
|
|
2146
|
+
const axisRuleOrder = bucket.axisRuleOrder!;
|
|
2147
|
+
const combosByKey = bucket.combosByKey!;
|
|
2148
|
+
const comboOrder = bucket.comboOrder!;
|
|
2149
|
+
if (comboOrder.length === 0) {
|
|
2150
|
+
throw new Error("heapScene/gpuRouting: specialiseBucket called with no registered combos");
|
|
2151
|
+
}
|
|
2152
|
+
const base = bucket.baseDescriptor!;
|
|
2153
|
+
// An axis is "active" (gets cartesian-product cardinality) if it
|
|
2154
|
+
// has at least one rule, OR any combo overrides baseDesc with a
|
|
2155
|
+
// distinct fixed value. Both grow the per-axis value union.
|
|
2156
|
+
const baseAxisValues = new Map<ModeAxis, unknown>(AXIS_ORDER.map(a => [a, axisValueOfDesc(base, a)] as const));
|
|
2157
|
+
const activeAxes: ModeAxis[] = AXIS_ORDER.filter(a => {
|
|
2158
|
+
if ((rulesByAxis.get(a)?.size ?? 0) > 0) return true;
|
|
2159
|
+
const baseVal = baseAxisValues.get(a);
|
|
2160
|
+
const baseKey = stableStringify(baseVal);
|
|
2161
|
+
for (const k of comboOrder) {
|
|
2162
|
+
const c = combosByKey.get(k)!;
|
|
2163
|
+
const fixed = c.axisFixedValues.get(a);
|
|
2164
|
+
if (fixed !== undefined && stableStringify(fixed) !== baseKey) return true;
|
|
2165
|
+
}
|
|
2166
|
+
return false;
|
|
2167
|
+
});
|
|
2168
|
+
|
|
2169
|
+
// Step 1: per-axis-rule analyse + collect per-axis union.
|
|
2170
|
+
interface PerRule {
|
|
2171
|
+
entry: AxisRuleEntry;
|
|
2172
|
+
bakedBody: Stmt;
|
|
2173
|
+
resolved: ReadonlyArray<unknown>;
|
|
2174
|
+
declaredU32: number;
|
|
2175
|
+
}
|
|
2176
|
+
const perAxisRules = new Map<ModeAxis, PerRule[]>();
|
|
2177
|
+
for (const axis of activeAxes) {
|
|
2178
|
+
const list: PerRule[] = [];
|
|
2179
|
+
for (const hash of axisRuleOrder.get(axis)!) {
|
|
2180
|
+
const entry = rulesByAxis.get(axis)!.get(hash)!;
|
|
2181
|
+
const declaredU32 = resolveDeclaredU32(entry.rule, AdaptiveToken.top);
|
|
2182
|
+
const declaredConst: Expr = {
|
|
2183
|
+
kind: "Const", type: Tu32 as never,
|
|
2184
|
+
value: { kind: "Int", value: declaredU32 >>> 0, signed: false } as never,
|
|
2185
|
+
} as Expr;
|
|
2186
|
+
const bakedBody = substituteReadInputInStmt(entry.body, "Uniform", "declared", declaredConst);
|
|
2187
|
+
const symbolic = analyseOutputSet(bakedBody);
|
|
2188
|
+
const env = new Map<string, number>([["declared", declaredU32]]);
|
|
2189
|
+
const { resolved, unresolvedCount } = evaluateStructuralSet(symbolic, env, new Map());
|
|
2190
|
+
let outputs = resolved.slice();
|
|
2191
|
+
if (outputs.length === 0 || unresolvedCount > 0) {
|
|
2192
|
+
const table = AXIS_ENUM_TABLE[entry.rule.axis];
|
|
2193
|
+
if (table.length > 0 && unresolvedCount > 0) {
|
|
2194
|
+
outputs = table.map((_, i) => i);
|
|
2195
|
+
} else if (outputs.length === 0) {
|
|
2196
|
+
throw new Error(
|
|
2197
|
+
`heapScene/gpuRouting: rule on axis '${entry.rule.axis}' could not be ` +
|
|
2198
|
+
`statically resolved AND the axis has no canonical enum table. ` +
|
|
2199
|
+
`Simplify the rule body so analyseOutputSet can fold each branch.`,
|
|
2200
|
+
);
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
list.push({ entry, bakedBody, resolved: outputs, declaredU32 });
|
|
2204
|
+
entry.lastDeclaredU32 = declaredU32;
|
|
2205
|
+
}
|
|
2206
|
+
perAxisRules.set(axis, list);
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
// Per axis: union of resolved values across rules + each combo's
|
|
2210
|
+
// effective value when it has no rule on this axis. Combos that
|
|
2211
|
+
// declare an explicit `axisFixedValues[axis]` contribute that
|
|
2212
|
+
// value; combos without an entry default to baseDesc.value.
|
|
2213
|
+
interface AxisInfo {
|
|
2214
|
+
readonly axis: ModeAxis;
|
|
2215
|
+
readonly values: ReadonlyArray<unknown>; // resolved axis values, sorted
|
|
2216
|
+
readonly keyToIdx: ReadonlyMap<string, number>; // stableStringify(value) → idx
|
|
2217
|
+
readonly baseValue: unknown; // baseDesc's value for this axis
|
|
2218
|
+
readonly baseIdx: number; // index of baseValue in values
|
|
2219
|
+
}
|
|
2220
|
+
const axisInfos: AxisInfo[] = [];
|
|
2221
|
+
for (const axis of activeAxes) {
|
|
2222
|
+
const list = perAxisRules.get(axis) ?? [];
|
|
2223
|
+
const dummyRule = { axis } as DerivedModeRule<ModeAxis>;
|
|
2224
|
+
const byKey = new Map<string, unknown>();
|
|
2225
|
+
for (const p of list) {
|
|
2226
|
+
for (const v of p.resolved) {
|
|
2227
|
+
const av = resolveAxisValue(dummyRule, v);
|
|
2228
|
+
byKey.set(stableStringify(av), av);
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
const baseValue = baseAxisValues.get(axis);
|
|
2232
|
+
// Include every combo's effective value for this axis (so
|
|
2233
|
+
// const-source idx lookups always succeed).
|
|
2234
|
+
for (const k of comboOrder) {
|
|
2235
|
+
const c = combosByKey.get(k)!;
|
|
2236
|
+
if (c.axisRules.has(axis)) continue;
|
|
2237
|
+
const effective = c.axisFixedValues.has(axis) ? c.axisFixedValues.get(axis) : baseValue;
|
|
2238
|
+
byKey.set(stableStringify(effective), effective);
|
|
2239
|
+
}
|
|
2240
|
+
const sortedKeys = [...byKey.keys()].sort();
|
|
2241
|
+
const values = sortedKeys.map(k => byKey.get(k)!);
|
|
2242
|
+
const keyToIdx = new Map(sortedKeys.map((k, i) => [k, i] as const));
|
|
2243
|
+
const baseIdx = keyToIdx.get(stableStringify(baseValue)) ?? 0;
|
|
2244
|
+
axisInfos.push({ axis, values, keyToIdx, baseValue, baseIdx });
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
// Step 2: cartesian shape.
|
|
2248
|
+
const cards = axisInfos.map(a => a.values.length);
|
|
2249
|
+
const totalSlots = cards.reduce((m, n) => m * n, 1);
|
|
2250
|
+
|
|
2251
|
+
// Step 3: rewrite each axis-rule body to emit per-axis indices.
|
|
2252
|
+
const ruleSpecs: import("./derivedModes/kernelCodegen.js").RuleCodegenSpec[] = [];
|
|
2253
|
+
for (const info of axisInfos) {
|
|
2254
|
+
const list = perAxisRules.get(info.axis)!;
|
|
2255
|
+
const dummyRule = { axis: info.axis } as DerivedModeRule<ModeAxis>;
|
|
2256
|
+
for (const p of list) {
|
|
2257
|
+
const env = new Map<string, number>([["declared", p.declaredU32]]);
|
|
2258
|
+
const exprToAxisIdx = (e: Expr): number | undefined => {
|
|
2259
|
+
const v = evaluateStructural(e, env, new Map());
|
|
2260
|
+
if (v === undefined) return undefined;
|
|
2261
|
+
const av = resolveAxisValue(dummyRule, v);
|
|
2262
|
+
return info.keyToIdx.get(stableStringify(av));
|
|
2263
|
+
};
|
|
2264
|
+
const rewrittenBody = rewriteOutputsToSlotIndices(p.bakedBody, exprToAxisIdx);
|
|
2265
|
+
ruleSpecs.push({
|
|
2266
|
+
axis: info.axis,
|
|
2267
|
+
axisRuleId: p.entry.axisRuleId,
|
|
2268
|
+
body: rewrittenBody,
|
|
2269
|
+
intrinsics: new Map(),
|
|
2270
|
+
helpersWGSL: "",
|
|
2271
|
+
inputUniforms: ["ModelTrafo"],
|
|
2272
|
+
});
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
// Step 4: per-combo axis sources. Rule axes → rule source.
|
|
2277
|
+
// Non-rule axes → const idx of (axisFixedValues.get(axis)
|
|
2278
|
+
// ?? baseValue) within the axis's sorted union.
|
|
2279
|
+
const comboSpecs: import("./derivedModes/kernelCodegen.js").ComboCodegenSpec[] = [];
|
|
2280
|
+
for (const comboKey of comboOrder) {
|
|
2281
|
+
const combo = combosByKey.get(comboKey)!;
|
|
2282
|
+
const axes = axisInfos.map(info => {
|
|
2283
|
+
const axisRuleId = combo.axisRules.get(info.axis);
|
|
2284
|
+
if (axisRuleId !== undefined) {
|
|
2285
|
+
return {
|
|
2286
|
+
axis: info.axis,
|
|
2287
|
+
cardinality: info.values.length,
|
|
2288
|
+
source: { kind: "rule" as const, axisRuleId },
|
|
2289
|
+
};
|
|
2290
|
+
}
|
|
2291
|
+
const effective = combo.axisFixedValues.has(info.axis)
|
|
2292
|
+
? combo.axisFixedValues.get(info.axis)
|
|
2293
|
+
: info.baseValue;
|
|
2294
|
+
const idx = info.keyToIdx.get(stableStringify(effective)) ?? info.baseIdx;
|
|
2295
|
+
return {
|
|
2296
|
+
axis: info.axis,
|
|
2297
|
+
cardinality: info.values.length,
|
|
2298
|
+
source: { kind: "const" as const, idx },
|
|
2299
|
+
};
|
|
2300
|
+
});
|
|
2301
|
+
comboSpecs.push({ comboId: combo.comboId, axes });
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
// Step 5: enumerate cartesian → slot descriptors (mixed-radix
|
|
2305
|
+
// decode in the same axis order the kernel uses).
|
|
2306
|
+
const slotDescs: PipelineStateDescriptor[] = [];
|
|
2307
|
+
for (let slotIdx = 0; slotIdx < totalSlots; slotIdx++) {
|
|
2308
|
+
let desc = base;
|
|
2309
|
+
let rem = slotIdx;
|
|
2310
|
+
// Match kernel: strides[i] = ∏_{j>i} cards[j]. Decode:
|
|
2311
|
+
// idx[i] = (slotIdx / strides[i]) % cards[i].
|
|
2312
|
+
const strides: number[] = new Array(axisInfos.length).fill(1);
|
|
2313
|
+
for (let i = axisInfos.length - 2; i >= 0; i--) strides[i] = strides[i + 1]! * cards[i + 1]!;
|
|
2314
|
+
for (let i = 0; i < axisInfos.length; i++) {
|
|
2315
|
+
const info = axisInfos[i]!;
|
|
2316
|
+
const ai = Math.floor(rem / strides[i]!) % cards[i]!;
|
|
2317
|
+
rem -= ai * strides[i]!;
|
|
2318
|
+
desc = patchDescriptor(desc, info.axis, info.values[ai]);
|
|
2319
|
+
}
|
|
2320
|
+
slotDescs.push(desc);
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
const kernelWGSL = emitPartitionKernel({
|
|
2324
|
+
rules: ruleSpecs,
|
|
2325
|
+
combos: comboSpecs,
|
|
2326
|
+
totalSlots,
|
|
2327
|
+
uniformOrder: bucket.uniformOrder ?? [],
|
|
2328
|
+
});
|
|
2329
|
+
return { slotDescs, kernelWGSL };
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
/** Build pipeline + slot for each cartesian slot descriptor. */
|
|
2333
|
+
function ensureSlotsForResolved(
|
|
2334
|
+
bucket: Bucket,
|
|
2335
|
+
slotDescs: ReadonlyArray<PipelineStateDescriptor>,
|
|
2336
|
+
): void {
|
|
2337
|
+
const fam = familyForBucket.get(bucket)!;
|
|
2338
|
+
for (let slotIdx = 0; slotIdx < slotDescs.length; slotIdx++) {
|
|
2339
|
+
const desc = slotDescs[slotIdx]!;
|
|
2340
|
+
const key = encodeModeKey(desc);
|
|
2341
|
+
if (slotIdx < bucket.slots.length) {
|
|
2342
|
+
const slot = bucket.slots[slotIdx]!;
|
|
2343
|
+
if (slot.modeKey !== key) {
|
|
2344
|
+
slot.pipeline = getOrCreatePipeline(fam, bucket.layout, bucket.isAtlasBucket, desc);
|
|
2345
|
+
slot.modeKey = key;
|
|
2346
|
+
}
|
|
2347
|
+
} else {
|
|
2348
|
+
createSlot(bucket, desc);
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
/** Register a combo (set of axis rules, ≤ one per axis) on a GPU-
|
|
2354
|
+
* routed bucket (or promote the bucket on first call). Returns
|
|
2355
|
+
* the bucket-unique `comboId` to bake into master-pool records. */
|
|
2356
|
+
function registerCombo(
|
|
2357
|
+
bucket: Bucket,
|
|
2358
|
+
axisRules: ReadonlyArray<DerivedModeRule<ModeAxis>>,
|
|
2359
|
+
baseDesc: PipelineStateDescriptor,
|
|
2360
|
+
): number {
|
|
2361
|
+
// CPU→GPU promotion: capture pre-existing CPU-routed ROs so they
|
|
2362
|
+
// can be migrated into the master pool after the partition is up.
|
|
2363
|
+
// Each RO is tagged with its CPU descriptor (read off the slot's
|
|
2364
|
+
// `cpuDescriptor` snapshot); we synthesise a const-source combo
|
|
2365
|
+
// per distinct descriptor below, with `axisFixedValues` set on
|
|
2366
|
+
// any axis the descriptor differs from baseDescriptor on.
|
|
2367
|
+
interface CpuMigration {
|
|
2368
|
+
localSlot: number;
|
|
2369
|
+
drawId: number;
|
|
2370
|
+
firstIndex: number;
|
|
2371
|
+
indexCount: number;
|
|
2372
|
+
instanceCount: number;
|
|
2373
|
+
perDrawRefs: Map<string, number>;
|
|
2374
|
+
descriptor: PipelineStateDescriptor;
|
|
2375
|
+
/** Filled in below after synthetic combos are registered. */
|
|
2376
|
+
targetComboId: number;
|
|
2377
|
+
}
|
|
2378
|
+
let cpuMigration: CpuMigration[] | undefined;
|
|
2379
|
+
if (!bucket.gpuRouted && bucket.drawSlots.size > 0) {
|
|
2380
|
+
cpuMigration = [];
|
|
2381
|
+
for (const ls of bucket.drawSlots) {
|
|
2382
|
+
const e = bucket.localEntries[ls]!;
|
|
2383
|
+
const drawId = bucket.localToDrawId[ls]!;
|
|
2384
|
+
const slotIdx = drawIdToSlotIdx[drawId];
|
|
2385
|
+
const slot = slotIdx !== undefined ? bucket.slots[slotIdx] : undefined;
|
|
2386
|
+
const descriptor = slot?.cpuDescriptor;
|
|
2387
|
+
if (descriptor === undefined) {
|
|
2388
|
+
throw new Error(
|
|
2389
|
+
`heapScene/gpuRouting: pre-existing CPU RO (drawId=${drawId}, localSlot=${ls}) ` +
|
|
2390
|
+
`has no captured descriptor — bucket promotion cannot synthesise a migration combo.`,
|
|
2391
|
+
);
|
|
2392
|
+
}
|
|
2393
|
+
cpuMigration.push({
|
|
2394
|
+
localSlot: ls,
|
|
2395
|
+
drawId,
|
|
2396
|
+
firstIndex: e.firstIndex,
|
|
2397
|
+
indexCount: e.indexCount,
|
|
2398
|
+
instanceCount: e.instanceCount,
|
|
2399
|
+
perDrawRefs: bucket.localPerDrawRefs[ls]!,
|
|
2400
|
+
descriptor,
|
|
2401
|
+
targetComboId: 0,
|
|
2402
|
+
});
|
|
2403
|
+
}
|
|
2404
|
+
}
|
|
2405
|
+
if (!bucket.gpuRouted) {
|
|
2406
|
+
bucket.baseDescriptor = baseDesc;
|
|
2407
|
+
bucket.rulesByAxis = new Map();
|
|
2408
|
+
bucket.axisRuleOrder = new Map();
|
|
2409
|
+
bucket.uniformOrder = [];
|
|
2410
|
+
bucket.uniformIdx = new Map();
|
|
2411
|
+
bucket.combosByKey = new Map();
|
|
2412
|
+
bucket.comboOrder = [];
|
|
2413
|
+
bucket.drawIdToComboId = new Map();
|
|
2414
|
+
bucket.drawIdToRecord = new Map();
|
|
2415
|
+
bucket.recordToDrawId = [];
|
|
2416
|
+
// Synthesise one combo per distinct pre-existing CPU
|
|
2417
|
+
// descriptor. Each carries `axisFixedValues` on axes where the
|
|
2418
|
+
// descriptor differs from baseDescriptor; specialiseBucket
|
|
2419
|
+
// expands the per-axis union to include each variant value
|
|
2420
|
+
// and emits a const-source combo fn that routes to the right
|
|
2421
|
+
// cartesian slot.
|
|
2422
|
+
if (cpuMigration !== undefined) {
|
|
2423
|
+
const baseVals = new Map<ModeAxis, unknown>(AXIS_ORDER.map(a => [a, axisValueOfDesc(baseDesc, a)] as const));
|
|
2424
|
+
const synthByKey = new Map<string, number>();
|
|
2425
|
+
for (const m of cpuMigration) {
|
|
2426
|
+
const axisFixedValues = new Map<ModeAxis, unknown>();
|
|
2427
|
+
const keyParts: string[] = [];
|
|
2428
|
+
for (const a of AXIS_ORDER) {
|
|
2429
|
+
const v = axisValueOfDesc(m.descriptor, a);
|
|
2430
|
+
const baseKey = stableStringify(baseVals.get(a));
|
|
2431
|
+
const vKey = stableStringify(v);
|
|
2432
|
+
if (vKey !== baseKey) {
|
|
2433
|
+
axisFixedValues.set(a, v);
|
|
2434
|
+
keyParts.push(`${a}:_:fixed:${vKey}`);
|
|
2435
|
+
} else {
|
|
2436
|
+
keyParts.push(`${a}:_`);
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
const comboKey = keyParts.join("|");
|
|
2440
|
+
let comboId = synthByKey.get(comboKey);
|
|
2441
|
+
if (comboId === undefined) {
|
|
2442
|
+
comboId = bucket.comboOrder.length;
|
|
2443
|
+
bucket.combosByKey.set(comboKey, {
|
|
2444
|
+
comboId, comboKey,
|
|
2445
|
+
axisRules: new Map(),
|
|
2446
|
+
axisFixedValues,
|
|
2447
|
+
});
|
|
2448
|
+
bucket.comboOrder.push(comboKey);
|
|
2449
|
+
synthByKey.set(comboKey, comboId);
|
|
2450
|
+
}
|
|
2451
|
+
m.targetComboId = comboId;
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
// Per-axis: dedupe rule by content hash; assign axisRuleId.
|
|
2456
|
+
const perAxisRuleId = new Map<ModeAxis, number>();
|
|
2457
|
+
let respec = false;
|
|
2458
|
+
for (const rule of axisRules) {
|
|
2459
|
+
if (perAxisRuleId.has(rule.axis)) {
|
|
2460
|
+
throw new Error(
|
|
2461
|
+
`heapScene/gpuRouting: combo has two rules on the same axis '${rule.axis}'`,
|
|
2462
|
+
);
|
|
2463
|
+
}
|
|
2464
|
+
const hash = ruleContentHash(rule);
|
|
2465
|
+
let axisMap = bucket.rulesByAxis!.get(rule.axis);
|
|
2466
|
+
if (axisMap === undefined) {
|
|
2467
|
+
axisMap = new Map();
|
|
2468
|
+
bucket.rulesByAxis!.set(rule.axis, axisMap);
|
|
2469
|
+
bucket.axisRuleOrder!.set(rule.axis, []);
|
|
2470
|
+
}
|
|
2471
|
+
let entry = axisMap.get(hash);
|
|
2472
|
+
if (entry === undefined) {
|
|
2473
|
+
const order = bucket.axisRuleOrder!.get(rule.axis)!;
|
|
2474
|
+
const body = ruleBodyOf(rule);
|
|
2475
|
+
entry = {
|
|
2476
|
+
axis: rule.axis,
|
|
2477
|
+
axisRuleId: order.length,
|
|
2478
|
+
contentHash: hash,
|
|
2479
|
+
rule,
|
|
2480
|
+
body,
|
|
2481
|
+
lastDeclaredU32: undefined,
|
|
2482
|
+
};
|
|
2483
|
+
axisMap.set(hash, entry);
|
|
2484
|
+
order.push(hash);
|
|
2485
|
+
// Discover the arena uniforms this rule reads — append unseen
|
|
2486
|
+
// ones to the bucket-wide uniformOrder so master records
|
|
2487
|
+
// carry refs for them. `declared` is a CPU-baked Const, not
|
|
2488
|
+
// an arena uniform; skip it.
|
|
2489
|
+
for (const uName of collectUniformReadsInStmt(body)) {
|
|
2490
|
+
if (uName === "declared") continue;
|
|
2491
|
+
if (!bucket.uniformIdx!.has(uName)) {
|
|
2492
|
+
const idx = bucket.uniformOrder!.length;
|
|
2493
|
+
bucket.uniformOrder!.push(uName);
|
|
2494
|
+
bucket.uniformIdx!.set(uName, idx);
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
respec = true;
|
|
2498
|
+
}
|
|
2499
|
+
perAxisRuleId.set(rule.axis, entry.axisRuleId);
|
|
2500
|
+
}
|
|
2501
|
+
// Combo key = canonical-axis-order list of (axis, contentHash).
|
|
2502
|
+
const comboKeyParts: string[] = [];
|
|
2503
|
+
for (const axis of AXIS_ORDER) {
|
|
2504
|
+
const axisRuleId = perAxisRuleId.get(axis);
|
|
2505
|
+
if (axisRuleId === undefined) {
|
|
2506
|
+
comboKeyParts.push(`${axis}:_`);
|
|
2507
|
+
} else {
|
|
2508
|
+
comboKeyParts.push(`${axis}:${bucket.axisRuleOrder!.get(axis)![axisRuleId]!}`);
|
|
2509
|
+
}
|
|
2510
|
+
}
|
|
2511
|
+
const comboKey = comboKeyParts.join("|");
|
|
2512
|
+
let combo = bucket.combosByKey!.get(comboKey);
|
|
2513
|
+
if (combo === undefined) {
|
|
2514
|
+
const comboId = bucket.comboOrder!.length;
|
|
2515
|
+
const axisRules = new Map<ModeAxis, number>();
|
|
2516
|
+
for (const [axis, id] of perAxisRuleId) axisRules.set(axis, id);
|
|
2517
|
+
combo = { comboId, comboKey, axisRules, axisFixedValues: new Map() };
|
|
2518
|
+
bucket.combosByKey!.set(comboKey, combo);
|
|
2519
|
+
bucket.comboOrder!.push(comboKey);
|
|
2520
|
+
respec = true;
|
|
2521
|
+
}
|
|
2522
|
+
// First time gpuRouted? Build the partition scene now.
|
|
2523
|
+
if (!bucket.gpuRouted) {
|
|
2524
|
+
const { slotDescs, kernelWGSL } = specialiseBucket(bucket);
|
|
2525
|
+
ensureSlotsForResolved(bucket, slotDescs);
|
|
2526
|
+
const partition = new GpuPartitionScene(device, `${bucket.label}/partition`, {
|
|
2527
|
+
totalSlots: slotDescs.length,
|
|
2528
|
+
slotDrawBufs: bucket.slots.map(s => s.drawTableBuf!.buffer),
|
|
2529
|
+
kernelWGSL,
|
|
2530
|
+
initialRecords: Math.max(16, (cpuMigration?.length ?? 0) + 16),
|
|
2531
|
+
numUniforms: bucket.uniformOrder!.length,
|
|
2532
|
+
});
|
|
2533
|
+
bucket.gpuRouted = true;
|
|
2534
|
+
bucket.partitionScene = partition;
|
|
2535
|
+
bucket.partitionDirty = true;
|
|
2536
|
+
// Migrate any captured CPU records into the master pool, each
|
|
2537
|
+
// using the synthetic combo registered for its CPU descriptor.
|
|
2538
|
+
if (cpuMigration !== undefined) {
|
|
2539
|
+
let totalEmit = 0;
|
|
2540
|
+
for (const m of cpuMigration) {
|
|
2541
|
+
const uRefs = bucket.uniformOrder!.map(n => m.perDrawRefs.get(n) ?? 0);
|
|
2542
|
+
const recIdx = partition.appendRecord(
|
|
2543
|
+
m.localSlot, m.firstIndex, m.indexCount, m.instanceCount, m.targetComboId, uRefs,
|
|
2544
|
+
);
|
|
2545
|
+
bucket.drawIdToRecord!.set(m.drawId, recIdx);
|
|
2546
|
+
bucket.recordToDrawId![recIdx] = m.drawId;
|
|
2547
|
+
bucket.drawIdToComboId!.set(m.drawId, m.targetComboId);
|
|
2548
|
+
totalEmit += m.indexCount * m.instanceCount;
|
|
2549
|
+
}
|
|
2550
|
+
const numRecords = partition.numRecords;
|
|
2551
|
+
const recBytes = numRecords * RECORD_BYTES;
|
|
2552
|
+
const needBlocks = Math.max(1, Math.ceil(numRecords / SCAN_TILE_SIZE));
|
|
2553
|
+
for (const s of bucket.slots) {
|
|
2554
|
+
s.drawTableBuf!.ensureCapacity(recBytes);
|
|
2555
|
+
s.drawTableBuf!.setUsed(Math.max(s.drawTableBuf!.usedBytes, recBytes));
|
|
2556
|
+
s.blockSumsBuf!.ensureCapacity(needBlocks * 4);
|
|
2557
|
+
s.blockOffsetsBuf!.ensureCapacity(needBlocks * 4);
|
|
2558
|
+
s.recordCount = numRecords;
|
|
2559
|
+
s.totalEmitEstimate += totalEmit;
|
|
2560
|
+
const newNumTiles = Math.max(1, Math.ceil(s.totalEmitEstimate / TILE_K));
|
|
2561
|
+
s.firstDrawInTileBuf!.ensureCapacity((newNumTiles + 1) * 4);
|
|
2562
|
+
s.scanDirty = true;
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2565
|
+
} else if (respec) {
|
|
2566
|
+
const { slotDescs, kernelWGSL } = specialiseBucket(bucket);
|
|
2567
|
+
ensureSlotsForResolved(bucket, slotDescs);
|
|
2568
|
+
const newCount = bucket.slots.length;
|
|
2569
|
+
const draws = bucket.slots.map(s => s.drawTableBuf!.buffer);
|
|
2570
|
+
if (newCount > bucket.partitionScene!.totalSlots) {
|
|
2571
|
+
bucket.partitionScene!.growSlots(newCount, draws);
|
|
2572
|
+
} else {
|
|
2573
|
+
bucket.partitionScene!.rebindSlotDrawBufs(draws);
|
|
2574
|
+
}
|
|
2575
|
+
// Bucket-wide uniformOrder may have grown (a new rule reads a
|
|
2576
|
+
// previously-unseen uniform). Widen master records before
|
|
2577
|
+
// rebuilding the kernel — the WGSL struct width changes too.
|
|
2578
|
+
if (bucket.uniformOrder!.length > bucket.partitionScene!.numUniforms) {
|
|
2579
|
+
bucket.partitionScene!.growUniforms(bucket.uniformOrder!.length);
|
|
2580
|
+
}
|
|
2581
|
+
bucket.partitionScene!.rebuildKernel(kernelWGSL);
|
|
2582
|
+
bucket.partitionDirty = true;
|
|
2583
|
+
}
|
|
2584
|
+
return combo.comboId;
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
/**
|
|
2588
|
+
* Per-frame partition dispatch. Detects per-rule `declared`
|
|
2589
|
+
* changes and re-specialises the whole bucket if any rule's
|
|
2590
|
+
* declared value differs from what was used at the last
|
|
2591
|
+
* dispatch (cheap — analysis runs on cached IR).
|
|
2592
|
+
*/
|
|
2593
|
+
function dispatchPartition(bucket: Bucket, enc: GPUCommandEncoder, tok: AdaptiveToken): void {
|
|
2594
|
+
if (!bucket.gpuRouted || bucket.partitionScene === undefined) return;
|
|
2595
|
+
let needRespec = false;
|
|
2596
|
+
outer: for (const axisMap of bucket.rulesByAxis!.values()) {
|
|
2597
|
+
for (const entry of axisMap.values()) {
|
|
2598
|
+
const declaredU32 = resolveDeclaredU32(entry.rule, tok);
|
|
2599
|
+
if (declaredU32 !== entry.lastDeclaredU32) { needRespec = true; break outer; }
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
if (needRespec) {
|
|
2603
|
+
const { slotDescs, kernelWGSL } = specialiseBucket(bucket);
|
|
2604
|
+
ensureSlotsForResolved(bucket, slotDescs);
|
|
2605
|
+
const newCount = bucket.slots.length;
|
|
2606
|
+
const draws = bucket.slots.map(s => s.drawTableBuf!.buffer);
|
|
2607
|
+
if (newCount > bucket.partitionScene.totalSlots) {
|
|
2608
|
+
bucket.partitionScene.growSlots(newCount, draws);
|
|
2609
|
+
} else {
|
|
2610
|
+
bucket.partitionScene.rebindSlotDrawBufs(draws);
|
|
2611
|
+
}
|
|
2612
|
+
bucket.partitionScene.rebuildKernel(kernelWGSL);
|
|
2613
|
+
}
|
|
2614
|
+
bucket.partitionScene.flush();
|
|
2615
|
+
bucket.partitionScene.dispatch(arena.attrs.chunk(bucket.chunkIdx).buffer, enc);
|
|
2616
|
+
for (let i = 0; i < bucket.slots.length; i++) {
|
|
2617
|
+
enc.copyBufferToBuffer(
|
|
2618
|
+
bucket.partitionScene.slotCountBufs[i]!, 0,
|
|
2619
|
+
bucket.slots[i]!.paramsBuf!, 0, 4,
|
|
2620
|
+
);
|
|
2621
|
+
bucket.slots[i]!.scanDirty = true;
|
|
2622
|
+
}
|
|
2623
|
+
bucket.partitionDirty = false;
|
|
2624
|
+
}
|
|
2625
|
+
|
|
2626
|
+
|
|
1651
2627
|
function findOrCreateBucket(
|
|
1652
2628
|
effect: Effect,
|
|
1653
2629
|
_textures: HeapTextureSet | undefined,
|
|
1654
2630
|
pipelineState: PipelineState | undefined,
|
|
2631
|
+
chunkIdx: number,
|
|
1655
2632
|
/**
|
|
1656
2633
|
* Optional precomputed descriptor. addDrawImpl passes this when
|
|
1657
2634
|
* the spec carries `modeRules` so the bucket key reflects the
|
|
@@ -1664,12 +2641,13 @@ export function buildHeapScene(
|
|
|
1664
2641
|
throw new Error("heapScene: findOrCreateBucket called before family build");
|
|
1665
2642
|
}
|
|
1666
2643
|
const fam = familyFor(effect);
|
|
1667
|
-
//
|
|
1668
|
-
//
|
|
1669
|
-
//
|
|
1670
|
-
//
|
|
1671
|
-
//
|
|
1672
|
-
|
|
2644
|
+
// §3: bucket key includes `chunkIdx` — two ROs with the same
|
|
2645
|
+
// effect/family that landed in different arena chunks live in
|
|
2646
|
+
// separate buckets so each bucket can bind exactly one chunk's
|
|
2647
|
+
// GPUBuffers in its draw call. The shaders are unchanged
|
|
2648
|
+
// (refs are byte offsets within whichever chunk's buffer is
|
|
2649
|
+
// currently bound).
|
|
2650
|
+
const bk = `family#${fam.schema.id}/c${chunkIdx}`;
|
|
1673
2651
|
const existing = bucketByKey.get(bk);
|
|
1674
2652
|
if (existing !== undefined) return existing;
|
|
1675
2653
|
|
|
@@ -1682,7 +2660,7 @@ export function buildHeapScene(
|
|
|
1682
2660
|
const drawHeap = new DrawHeap(drawHeapBuf, layout.drawHeaderBytes);
|
|
1683
2661
|
|
|
1684
2662
|
const bucket: Bucket = {
|
|
1685
|
-
label: bk, textures: undefined, layout,
|
|
2663
|
+
label: bk, textures: undefined, layout, chunkIdx,
|
|
1686
2664
|
slots: [], // populated lazily by ensureSlot per modeKey
|
|
1687
2665
|
bindGroup: null as unknown as GPUBindGroup, // legacy field; slot.bindGroup is the real one
|
|
1688
2666
|
drawHeap,
|
|
@@ -1696,6 +2674,19 @@ export function buildHeapScene(
|
|
|
1696
2674
|
localAtlasReleases: [],
|
|
1697
2675
|
localAtlasTextures: [],
|
|
1698
2676
|
localAtlasArrIdx: [],
|
|
2677
|
+
gpuRouted: false,
|
|
2678
|
+
partitionScene: undefined,
|
|
2679
|
+
drawIdToRecord: undefined,
|
|
2680
|
+
recordToDrawId: undefined,
|
|
2681
|
+
rulesByAxis: undefined,
|
|
2682
|
+
axisRuleOrder: undefined,
|
|
2683
|
+
uniformOrder: undefined,
|
|
2684
|
+
uniformIdx: undefined,
|
|
2685
|
+
combosByKey: undefined,
|
|
2686
|
+
comboOrder: undefined,
|
|
2687
|
+
drawIdToComboId: undefined,
|
|
2688
|
+
baseDescriptor: undefined,
|
|
2689
|
+
partitionDirty: false,
|
|
1699
2690
|
};
|
|
1700
2691
|
familyForBucket.set(bucket, fam);
|
|
1701
2692
|
|
|
@@ -1890,6 +2881,11 @@ export function buildHeapScene(
|
|
|
1890
2881
|
// inside a single outer evaluateAlways and invoke addDrawImpl
|
|
1891
2882
|
// directly with their token — collapsing 1000× nested
|
|
1892
2883
|
// evaluateAlways into 1× outer.
|
|
2884
|
+
// DEBUG counters.
|
|
2885
|
+
let __addDrawCalls = 0;
|
|
2886
|
+
let __removeDrawCalls = 0;
|
|
2887
|
+
(sceneObj as unknown as { __addDrawCalls: () => number }).__addDrawCalls = () => __addDrawCalls;
|
|
2888
|
+
(sceneObj as unknown as { __removeDrawCalls: () => number }).__removeDrawCalls = () => __removeDrawCalls;
|
|
1893
2889
|
function addDraw(spec: HeapDrawSpec): number {
|
|
1894
2890
|
let id = -1;
|
|
1895
2891
|
sceneObj.evaluateAlways(AdaptiveToken.top, (tok) => {
|
|
@@ -1898,6 +2894,7 @@ export function buildHeapScene(
|
|
|
1898
2894
|
return id;
|
|
1899
2895
|
}
|
|
1900
2896
|
function addDrawImpl(spec: HeapDrawSpec, outerTok: AdaptiveToken): number {
|
|
2897
|
+
__addDrawCalls++;
|
|
1901
2898
|
const drawId = nextDrawId++;
|
|
1902
2899
|
// Family-merge (slice 3c): build the family lazily from this
|
|
1903
2900
|
// single spec when no batched lazy-build occurred earlier (e.g.
|
|
@@ -1929,12 +2926,59 @@ export function buildHeapScene(
|
|
|
1929
2926
|
token: outerTok,
|
|
1930
2927
|
});
|
|
1931
2928
|
}
|
|
1932
|
-
|
|
2929
|
+
// §3 chunk routing — pick which arena chunk this RO's allocations
|
|
2930
|
+
// land in. Preference order:
|
|
2931
|
+
// 1. If any of the RO's bindings is an aval that's already
|
|
2932
|
+
// allocated in some chunk via the uniform pool, prefer that
|
|
2933
|
+
// chunk to avoid duplication.
|
|
2934
|
+
// 2. Otherwise, use the newest open chunk (highest chunkIdx).
|
|
2935
|
+
// ChunkedArena.alloc spills to the next chunk automatically
|
|
2936
|
+
// when this one is full; the spill ends up driving the
|
|
2937
|
+
// pool's `finalChunk` selection so all of the RO's
|
|
2938
|
+
// allocations land in one chunk consistently.
|
|
2939
|
+
let chunkIdx: number;
|
|
2940
|
+
{
|
|
2941
|
+
let preferred: number | undefined;
|
|
2942
|
+
for (const [, val] of Object.entries(spec.inputs)) {
|
|
2943
|
+
const av = (typeof val === "object" && val !== null && "getValue" in (val as object))
|
|
2944
|
+
? (val as aval<unknown>) : undefined;
|
|
2945
|
+
if (av !== undefined) {
|
|
2946
|
+
const where = pool.firstChunkContaining(av);
|
|
2947
|
+
if (where !== undefined) { preferred = where; break; }
|
|
2948
|
+
}
|
|
2949
|
+
}
|
|
2950
|
+
chunkIdx = preferred ?? Math.max(0, arena.attrs.chunkCount - 1);
|
|
2951
|
+
}
|
|
2952
|
+
const bucket = findOrCreateBucket(spec.effect, spec.textures, spec.pipelineState, chunkIdx, precomputedDescriptor);
|
|
2953
|
+
// Phase 5c.3 — every RO carrying a derived-mode rule is registered
|
|
2954
|
+
// on the bucket (lazy-promoting it to GPU-routed on the first
|
|
2955
|
+
// call). ROs may carry DIFFERENT rules even when they share the
|
|
2956
|
+
// bucket's `(effect, textureSet)` — each gets its own `ruleId`
|
|
2957
|
+
// baked into the master-pool record and the partition kernel's
|
|
2958
|
+
// dispatch switch picks the right body per record.
|
|
2959
|
+
const collectedRules = spec.modeRules !== undefined ? collectRules(spec.modeRules) : [];
|
|
2960
|
+
const roDescriptor = precomputedDescriptor ?? snapshotDescriptor(spec.pipelineState, sig);
|
|
2961
|
+
let roComboId = 0;
|
|
2962
|
+
if (collectedRules.length > 0) {
|
|
2963
|
+
roComboId = registerCombo(bucket, collectedRules, roDescriptor);
|
|
2964
|
+
} else if (bucket.gpuRouted) {
|
|
2965
|
+
// An unruled RO joining an already-GPU-routed bucket needs a
|
|
2966
|
+
// real combo to land on. Register the trivial all-fixed combo
|
|
2967
|
+
// (every axis falls back to baseDescriptor) so this record's
|
|
2968
|
+
// `comboId` doesn't silently alias to combo 0 (which is
|
|
2969
|
+
// whichever combo was registered first — almost never the
|
|
2970
|
+
// "no rules" intent).
|
|
2971
|
+
roComboId = registerCombo(bucket, [], roDescriptor);
|
|
2972
|
+
}
|
|
1933
2973
|
// Phase 5c.2: route this RO to the bucket's slot covering its
|
|
1934
2974
|
// current descriptor. ensureSlot creates a new slot on miss; the
|
|
1935
|
-
// pipeline lookup hits the scene-level cache.
|
|
1936
|
-
|
|
1937
|
-
|
|
2975
|
+
// pipeline lookup hits the scene-level cache. For GPU-routed
|
|
2976
|
+
// buckets the slot decision is made each frame on the GPU; we
|
|
2977
|
+
// still pin the RO to slot 0 (declared) CPU-side so existing
|
|
2978
|
+
// accounting paths (slotToRecord, dirty bookkeeping) keep
|
|
2979
|
+
// working. The partition kernel overwrites both slots' draw
|
|
2980
|
+
// tables every frame from the master.
|
|
2981
|
+
const roSlot = bucket.gpuRouted ? bucket.slots[0]! : ensureSlot(bucket, roDescriptor);
|
|
1938
2982
|
const roSlotIdx = bucket.slots.indexOf(roSlot);
|
|
1939
2983
|
const fam = familyFor(spec.effect);
|
|
1940
2984
|
const effectFields = fam.fieldsForEffect.get(spec.effect.id)!;
|
|
@@ -1944,7 +2988,7 @@ export function buildHeapScene(
|
|
|
1944
2988
|
// index allocation + one upload.
|
|
1945
2989
|
const indicesAval = asAval(spec.indices) as aval<Uint32Array>;
|
|
1946
2990
|
const indicesArr = readPlain(spec.indices) as Uint32Array;
|
|
1947
|
-
const idxAlloc = indexPool.acquire(device, arena.indices, indicesAval, indicesArr);
|
|
2991
|
+
const idxAlloc = indexPool.acquire(device, arena.indices, indicesAval, bucket.chunkIdx, indicesArr);
|
|
1948
2992
|
|
|
1949
2993
|
const localSlot = bucket.drawHeap.alloc();
|
|
1950
2994
|
// Per-RO instancing: read `spec.instanceCount` (defaults to 1).
|
|
@@ -1977,7 +3021,7 @@ export function buildHeapScene(
|
|
|
1977
3021
|
if (ruleForUniform(spec, f.name) !== undefined) {
|
|
1978
3022
|
const dummyAval = AVal.constant<M44d>(M44d.zero);
|
|
1979
3023
|
const ref = pool.acquire(
|
|
1980
|
-
device, arena.attrs, dummyAval, M44d.zero,
|
|
3024
|
+
device, arena.attrs, dummyAval, bucket.chunkIdx, M44d.zero,
|
|
1981
3025
|
PACKER_MAT4.dataBytes, PACKER_MAT4.typeId, 1, PACKER_MAT4.pack,
|
|
1982
3026
|
);
|
|
1983
3027
|
perDrawRefs.set(f.name, ref);
|
|
@@ -2026,7 +3070,7 @@ export function buildHeapScene(
|
|
|
2026
3070
|
: poolPlacementFor(f, value);
|
|
2027
3071
|
}
|
|
2028
3072
|
const ref = pool.acquire(
|
|
2029
|
-
device, arena.attrs, av, value,
|
|
3073
|
+
device, arena.attrs, av, bucket.chunkIdx, value,
|
|
2030
3074
|
placement.dataBytes, placement.typeId, placement.length, placement.pack,
|
|
2031
3075
|
);
|
|
2032
3076
|
perDrawRefs.set(f.name, ref);
|
|
@@ -2049,6 +3093,51 @@ export function buildHeapScene(
|
|
|
2049
3093
|
packBucketHeader(bucket, localSlot, perDrawRefs, layoutId);
|
|
2050
3094
|
if (bucket.isAtlasBucket && spec.textures !== undefined && spec.textures.kind === "atlas") {
|
|
2051
3095
|
packAtlasTextureFields(bucket, localSlot, spec.textures);
|
|
3096
|
+
// Pair this addDraw with one atlas refcount bump. The spec's
|
|
3097
|
+
// release closure (stored below, fired in removeDraw) decrements
|
|
3098
|
+
// exactly once per add/remove cycle; without this incRef the
|
|
3099
|
+
// refcount underflows when the same cached spec is re-introduced
|
|
3100
|
+
// across multiple aset add/remove cycles (e.g., a tile flipping
|
|
3101
|
+
// in and out of the heap-eligible subset).
|
|
3102
|
+
//
|
|
3103
|
+
// Falls back to a full re-acquire if the entry was already
|
|
3104
|
+
// evicted (incRef returns false): in that case the spec's stored
|
|
3105
|
+
// (pageId, origin, size) are stale and the new acquire's values
|
|
3106
|
+
// overwrite them before packAtlasTextureFields is re-called.
|
|
3107
|
+
if (atlasPool !== undefined) {
|
|
3108
|
+
const ok = atlasPool.incRef(spec.textures.poolRef);
|
|
3109
|
+
if (!ok) {
|
|
3110
|
+
// Entry was evicted while the cached spec was idle in heap-
|
|
3111
|
+
// remove state. Acquire a fresh sub-rect, redirect the spec
|
|
3112
|
+
// (and the per-aval cell behind its release closure) to the
|
|
3113
|
+
// new ref, and re-emit the drawHeader fields.
|
|
3114
|
+
const acq = atlasPool.acquire(
|
|
3115
|
+
spec.textures.page.format,
|
|
3116
|
+
spec.textures.sourceAval as aval<ITexture>,
|
|
3117
|
+
spec.textures.size.x, spec.textures.size.y,
|
|
3118
|
+
{ wantsMips: spec.textures.numMips > 1 },
|
|
3119
|
+
);
|
|
3120
|
+
(spec.textures as { pageId: number }).pageId = acq.pageId;
|
|
3121
|
+
(spec.textures as { origin: V2f }).origin = acq.origin;
|
|
3122
|
+
(spec.textures as { size: V2f }).size = acq.size;
|
|
3123
|
+
(spec.textures as { numMips: number }).numMips = acq.numMips;
|
|
3124
|
+
(spec.textures as { page: AtlasPage }).page = acq.page;
|
|
3125
|
+
(spec.textures as { poolRef: number }).poolRef = acq.ref;
|
|
3126
|
+
// The release closure captures a per-aval cell whose `ref`
|
|
3127
|
+
// must track the LATEST acquired sub-rect; otherwise the
|
|
3128
|
+
// closure releases a long-evicted ref (no-op leak).
|
|
3129
|
+
if (spec.textures.repack !== undefined) {
|
|
3130
|
+
// repack updates the per-aval cell to the new ref as a
|
|
3131
|
+
// side effect of its own atlas-mark drain path. Re-using
|
|
3132
|
+
// it here would free the entry we just acquired; instead
|
|
3133
|
+
// mirror the same effect with a tiny helper attached to
|
|
3134
|
+
// the spec at adapter time (`__retarget`).
|
|
3135
|
+
}
|
|
3136
|
+
const retarget = (spec.textures as unknown as { __retarget?: (ref: number) => void }).__retarget;
|
|
3137
|
+
if (retarget !== undefined) retarget(acq.ref);
|
|
3138
|
+
packAtlasTextureFields(bucket, localSlot, spec.textures);
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
2052
3141
|
bucket.localAtlasReleases[localSlot] = spec.textures.release;
|
|
2053
3142
|
bucket.localAtlasTextures[localSlot] = spec.textures;
|
|
2054
3143
|
// Reactivity wire-up: subscribe to `sourceAval` (so sceneObj.inputChanged
|
|
@@ -2072,7 +3161,53 @@ export function buildHeapScene(
|
|
|
2072
3161
|
const end = byteOff + bucket.layout.drawHeaderBytes;
|
|
2073
3162
|
if (end > bucket.headerDirtyMax) bucket.headerDirtyMax = end;
|
|
2074
3163
|
|
|
2075
|
-
{
|
|
3164
|
+
if (bucket.gpuRouted) {
|
|
3165
|
+
// Phase 5c.3 GPU routing: master pool is authoritative.
|
|
3166
|
+
// Both slot[0] and slot[1] get the SAME accounting (counts,
|
|
3167
|
+
// emit estimates, buffer capacities) — partition overwrites
|
|
3168
|
+
// their GPU drawTables every frame, so per-slot CPU shadows
|
|
3169
|
+
// are irrelevant. We still grow their buffers in lockstep so
|
|
3170
|
+
// the scan dispatch shape covers every possible record.
|
|
3171
|
+
const partition = bucket.partitionScene!;
|
|
3172
|
+
const recIdx = partition.numRecords;
|
|
3173
|
+
if (recIdx >= SCAN_MAX_RECORDS) {
|
|
3174
|
+
throw new Error(
|
|
3175
|
+
`heapScene: GPU-routed bucket exceeds SCAN_MAX_RECORDS (${SCAN_MAX_RECORDS})`,
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
// Pack the bucket's per-RO uniform refs (arena byte offsets) in
|
|
3179
|
+
// the same order as `bucket.uniformOrder`. ROs whose perDrawRefs
|
|
3180
|
+
// don't carry a given name (e.g. their effect doesn't declare
|
|
3181
|
+
// it) get 0 — the kernel only reads refs through combo fns that
|
|
3182
|
+
// gate on `r.comboId`, so an unread 0 is harmless.
|
|
3183
|
+
const uniformRefs: number[] = (bucket.uniformOrder ?? []).map(
|
|
3184
|
+
name => perDrawRefs.get(name) ?? 0,
|
|
3185
|
+
);
|
|
3186
|
+
partition.appendRecord(localSlot, idxAlloc.firstIndex, idxAlloc.count, instanceCount, roComboId, uniformRefs);
|
|
3187
|
+
bucket.drawIdToRecord!.set(drawId, recIdx);
|
|
3188
|
+
bucket.recordToDrawId![recIdx] = drawId;
|
|
3189
|
+
bucket.drawIdToComboId!.set(drawId, roComboId);
|
|
3190
|
+
// Master grew? Invalidate partition's bind group so it picks
|
|
3191
|
+
// up the new masterBuf. (rebindSlotDrawBufs nulls bindGroup.)
|
|
3192
|
+
// grow() inside appendRecord already does this when capacity
|
|
3193
|
+
// is exceeded; nothing to do here.
|
|
3194
|
+
const numRecords = recIdx + 1;
|
|
3195
|
+
const recBytes = numRecords * RECORD_BYTES;
|
|
3196
|
+
const needBlocks = Math.max(1, Math.ceil(numRecords / SCAN_TILE_SIZE));
|
|
3197
|
+
const emitDelta = idxAlloc.count * instanceCount;
|
|
3198
|
+
for (const s of bucket.slots) {
|
|
3199
|
+
s.drawTableBuf!.ensureCapacity(recBytes);
|
|
3200
|
+
s.drawTableBuf!.setUsed(Math.max(s.drawTableBuf!.usedBytes, recBytes));
|
|
3201
|
+
s.blockSumsBuf!.ensureCapacity(needBlocks * 4);
|
|
3202
|
+
s.blockOffsetsBuf!.ensureCapacity(needBlocks * 4);
|
|
3203
|
+
s.recordCount = numRecords;
|
|
3204
|
+
s.totalEmitEstimate += emitDelta;
|
|
3205
|
+
const newNumTiles = Math.max(1, Math.ceil(s.totalEmitEstimate / TILE_K));
|
|
3206
|
+
s.firstDrawInTileBuf!.ensureCapacity((newNumTiles + 1) * 4);
|
|
3207
|
+
s.scanDirty = true;
|
|
3208
|
+
}
|
|
3209
|
+
bucket.partitionDirty = true;
|
|
3210
|
+
} else {
|
|
2076
3211
|
const dtBuf = roSlot.drawTableBuf!;
|
|
2077
3212
|
const recIdx = roSlot.recordCount;
|
|
2078
3213
|
if (recIdx >= SCAN_MAX_RECORDS) {
|
|
@@ -2111,6 +3246,23 @@ export function buildHeapScene(
|
|
|
2111
3246
|
drawIdToSlotIdx[drawId] = roSlotIdx;
|
|
2112
3247
|
drawIdToIndexAval[drawId] = indicesAval;
|
|
2113
3248
|
|
|
3249
|
+
// ─── HeapDrawSpec.active wire-up ──────────────────────────────────
|
|
3250
|
+
// Track the visibility aval (if any) and apply its initial value
|
|
3251
|
+
// immediately. After this point, ticks on the aval are routed via
|
|
3252
|
+
// `subscribeActive` → `activeDirty` and drained in `update()`.
|
|
3253
|
+
if (spec.active !== undefined) {
|
|
3254
|
+
drawIdToActiveAval.set(drawId, spec.active);
|
|
3255
|
+
drawIdToOrigIndexCount.set(drawId, idxAlloc.count);
|
|
3256
|
+
const initial = spec.active.getValue(outerTok);
|
|
3257
|
+
subscribeActive(spec.active, drawId);
|
|
3258
|
+
if (initial === false) {
|
|
3259
|
+
// Apply the off state synchronously so the very first frame
|
|
3260
|
+
// already skips this RO (no flash of un-gated geometry while
|
|
3261
|
+
// we wait for the next update tick).
|
|
3262
|
+
setEffectiveIndexCount(drawId, 0);
|
|
3263
|
+
}
|
|
3264
|
+
}
|
|
3265
|
+
|
|
2114
3266
|
// ─── Reactive rebucket: track PS-modeKey changes ────────────────
|
|
2115
3267
|
// When any mode-axis aval marks (e.g. cullCval.value = 'front'),
|
|
2116
3268
|
// schedule this RO for rebucket on the next update(). Today's
|
|
@@ -2144,34 +3296,18 @@ export function buildHeapScene(
|
|
|
2144
3296
|
drawIdToModeTracker[drawId] = tracker;
|
|
2145
3297
|
tracker.forEachLeaf((av) => subscribeModeLeaf(av, drawId));
|
|
2146
3298
|
|
|
2147
|
-
// ───
|
|
2148
|
-
//
|
|
2149
|
-
//
|
|
2150
|
-
//
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
subscribeModeLeaf(gpuRule.declared as aval<unknown>, drawId);
|
|
2160
|
-
}
|
|
2161
|
-
}
|
|
2162
|
-
|
|
2163
|
-
// ─── Pipeline pre-warm for derived-mode rules (5c.2) ──────────
|
|
2164
|
-
// Enumerate the rule's possible outputs and pre-build the
|
|
2165
|
-
// matching pipelines in the scene-level cache. Future cullModeC
|
|
2166
|
-
// flips become cache hits — zero `createRenderPipeline` calls
|
|
2167
|
-
// on the hot path.
|
|
2168
|
-
if (cullRule !== undefined && cullRule.domain !== undefined && tracker.descriptor !== undefined) {
|
|
2169
|
-
const fam = familyFor(spec.effect);
|
|
2170
|
-
const baseDesc = tracker.descriptor;
|
|
2171
|
-
for (const mode of cullRule.domain) {
|
|
2172
|
-
if (mode === baseDesc.cullMode) continue; // already built
|
|
2173
|
-
const variantDesc: PipelineStateDescriptor = { ...baseDesc, cullMode: mode };
|
|
2174
|
-
getOrCreatePipeline(fam, bucket.layout, bucket.isAtlasBucket, variantDesc);
|
|
3299
|
+
// ─── Derived-mode rule registration ────────────────────────────
|
|
3300
|
+
// Subscribe to each rule's `declared` aval so the bucket's
|
|
3301
|
+
// partition kernel re-dispatches when it changes. Slot pipelines
|
|
3302
|
+
// for every (axis, declared) combination are pre-built eagerly
|
|
3303
|
+
// in `initGpuRouting` (one per cartesian-product point), so
|
|
3304
|
+
// declared flips are pipeline-cache hits.
|
|
3305
|
+
if (spec.modeRules !== undefined) {
|
|
3306
|
+
for (const rule of Object.values(spec.modeRules) as Array<DerivedModeRule<ModeAxis> | undefined>) {
|
|
3307
|
+
if (rule === undefined) continue;
|
|
3308
|
+
const d = rule.declared;
|
|
3309
|
+
if (typeof d === "object" && d !== null && "getValue" in (d as object)) {
|
|
3310
|
+
subscribeModeLeaf(d as aval<unknown>, drawId);
|
|
2175
3311
|
}
|
|
2176
3312
|
}
|
|
2177
3313
|
}
|
|
@@ -2213,16 +3349,13 @@ export function buildHeapScene(
|
|
|
2213
3349
|
const reg = registerRoDerivations(derivedScene, {}, {
|
|
2214
3350
|
rules: ruleSubset,
|
|
2215
3351
|
trafoAvals,
|
|
2216
|
-
// A non-trafo rule leaf reads the uniform's data straight from this RO's
|
|
2217
|
-
// drawHeader region in the arena (= its pool ref + the alloc-header pad).
|
|
2218
|
-
// Only resolvable if the uniform was actually packed (i.e. it's a drawHeader
|
|
2219
|
-
// field of the effect too) — undefined otherwise, which `resolveSource` rejects.
|
|
2220
3352
|
hostUniformOffset: (n) => {
|
|
2221
3353
|
const r = perDrawRefs.get(n);
|
|
2222
3354
|
return r === undefined ? undefined : r + ALLOC_HEADER_PAD_TO;
|
|
2223
3355
|
},
|
|
2224
3356
|
outputOffset: (n) => outOffsetByName.get(n),
|
|
2225
3357
|
drawHeaderBaseByte: 0,
|
|
3358
|
+
chunkIdx: bucket.chunkIdx,
|
|
2226
3359
|
});
|
|
2227
3360
|
derivedByDrawId.set(drawId, reg);
|
|
2228
3361
|
}
|
|
@@ -2234,9 +3367,13 @@ export function buildHeapScene(
|
|
|
2234
3367
|
}
|
|
2235
3368
|
|
|
2236
3369
|
function removeDraw(drawId: number): void {
|
|
3370
|
+
__removeDrawCalls++;
|
|
2237
3371
|
const bucket = drawIdToBucket[drawId];
|
|
2238
3372
|
const localSlot = drawIdToLocalSlot[drawId];
|
|
2239
3373
|
if (bucket === undefined || localSlot === undefined) return;
|
|
3374
|
+
// Tear down `active` subscription (if any) so its callback won't
|
|
3375
|
+
// fire after the underlying drawTable record is gone.
|
|
3376
|
+
unsubscribeActive(drawId);
|
|
2240
3377
|
// §7: deregister this RO's derivation records and release slots.
|
|
2241
3378
|
if (derivedScene !== undefined) {
|
|
2242
3379
|
const reg = derivedByDrawId.get(drawId);
|
|
@@ -2245,13 +3382,54 @@ export function buildHeapScene(
|
|
|
2245
3382
|
derivedByDrawId.delete(drawId);
|
|
2246
3383
|
}
|
|
2247
3384
|
}
|
|
2248
|
-
{
|
|
3385
|
+
if (bucket.gpuRouted) {
|
|
3386
|
+
const partition = bucket.partitionScene!;
|
|
3387
|
+
const recIdx = bucket.drawIdToRecord!.get(drawId);
|
|
3388
|
+
const removedEntry = bucket.localEntries[localSlot];
|
|
3389
|
+
// Read the EFFECTIVE indexCount from masterShadow (already 0 for
|
|
3390
|
+
// an inactive draw via setEffectiveIndexCount). Using the original
|
|
3391
|
+
// count from localEntries would double-subtract for inactive draws
|
|
3392
|
+
// (whose contribution was already removed when active flipped false).
|
|
3393
|
+
const effectiveIndexCount = recIdx !== undefined
|
|
3394
|
+
? partition.masterShadow[recIdx * partition.recordU32 + 3]!
|
|
3395
|
+
: (removedEntry?.indexCount ?? 0);
|
|
3396
|
+
const instanceCount = removedEntry?.instanceCount ?? 0;
|
|
3397
|
+
const removedCount = effectiveIndexCount * instanceCount;
|
|
3398
|
+
if (recIdx !== undefined) {
|
|
3399
|
+
const moved = partition.removeRecord(recIdx);
|
|
3400
|
+
if (moved >= 0) {
|
|
3401
|
+
// The record at `moved` (the old tail) now lives at recIdx.
|
|
3402
|
+
// Fix maps for the moved drawId.
|
|
3403
|
+
const movedDrawId = bucket.recordToDrawId![moved]!;
|
|
3404
|
+
bucket.drawIdToRecord!.set(movedDrawId, recIdx);
|
|
3405
|
+
bucket.recordToDrawId![recIdx] = movedDrawId;
|
|
3406
|
+
bucket.recordToDrawId![moved] = -1;
|
|
3407
|
+
} else {
|
|
3408
|
+
bucket.recordToDrawId![recIdx] = -1;
|
|
3409
|
+
}
|
|
3410
|
+
bucket.drawIdToRecord!.delete(drawId);
|
|
3411
|
+
}
|
|
3412
|
+
// Mirror the count + emit-estimate decrement across every slot
|
|
3413
|
+
// (both share the master's accounting).
|
|
3414
|
+
for (const s of bucket.slots) {
|
|
3415
|
+
s.recordCount = partition.numRecords;
|
|
3416
|
+
s.totalEmitEstimate = Math.max(0, s.totalEmitEstimate - removedCount);
|
|
3417
|
+
s.scanDirty = true;
|
|
3418
|
+
}
|
|
3419
|
+
bucket.partitionDirty = true;
|
|
3420
|
+
} else {
|
|
2249
3421
|
const slotIdx = drawIdToSlotIdx[drawId];
|
|
2250
3422
|
const slot = slotIdx !== undefined ? bucket.slots[slotIdx]! : bucket.slots[0]!;
|
|
2251
3423
|
const removedEntry = bucket.localEntries[localSlot];
|
|
2252
|
-
const
|
|
2253
|
-
|
|
2254
|
-
|
|
3424
|
+
const localSlotRec = slot.slotToRecord[localSlot];
|
|
3425
|
+
// Effective (post-active) indexCount, read from shadow so that
|
|
3426
|
+
// inactive draws contribute 0 here (their contribution was
|
|
3427
|
+
// already removed when active flipped false).
|
|
3428
|
+
const effectiveIndexCount = localSlotRec !== undefined && localSlotRec >= 0
|
|
3429
|
+
? slot.drawTableShadow![localSlotRec * RECORD_U32 + 3]!
|
|
3430
|
+
: (removedEntry?.indexCount ?? 0);
|
|
3431
|
+
const instanceCount = removedEntry?.instanceCount ?? 0;
|
|
3432
|
+
const removedCount = effectiveIndexCount * instanceCount;
|
|
2255
3433
|
slot.totalEmitEstimate = Math.max(0, slot.totalEmitEstimate - removedCount);
|
|
2256
3434
|
// Swap-pop: move the last record into the freed slot, decrement
|
|
2257
3435
|
// recordCount. firstEmit is GPU-rewritten by the next scan, so
|
|
@@ -2282,9 +3460,9 @@ export function buildHeapScene(
|
|
|
2282
3460
|
|
|
2283
3461
|
// Release pool entries — refcount drops; if zero, allocation freed.
|
|
2284
3462
|
const avals = bucket.localPerDrawAvals[localSlot];
|
|
2285
|
-
if (avals !== undefined) for (const av of avals) pool.release(arena.attrs, av);
|
|
3463
|
+
if (avals !== undefined) for (const av of avals) pool.release(arena.attrs, av, bucket.chunkIdx);
|
|
2286
3464
|
const idxAval = drawIdToIndexAval[drawId];
|
|
2287
|
-
if (idxAval !== undefined) indexPool.release(arena.indices, idxAval);
|
|
3465
|
+
if (idxAval !== undefined) indexPool.release(arena.indices, idxAval, bucket.chunkIdx);
|
|
2288
3466
|
const atlasRel = bucket.localAtlasReleases[localSlot];
|
|
2289
3467
|
if (atlasRel !== undefined) atlasRel();
|
|
2290
3468
|
// Drop atlas-aval ref (if any). When the last ref is dropped we
|
|
@@ -2458,11 +3636,19 @@ export function buildHeapScene(
|
|
|
2458
3636
|
for (const drawId of dirty) {
|
|
2459
3637
|
const tracker = drawIdToModeTracker[drawId];
|
|
2460
3638
|
if (tracker === undefined) continue;
|
|
3639
|
+
const bucket = drawIdToBucket[drawId];
|
|
3640
|
+
// Phase 5c.3: GPU-routed buckets never reslot CPU-side.
|
|
3641
|
+
// The partition kernel handles re-routing per frame.
|
|
3642
|
+
// Just mark the bucket so the next encodeComputePrep
|
|
3643
|
+
// dispatches partition.
|
|
3644
|
+
if (bucket !== undefined && bucket.gpuRouted) {
|
|
3645
|
+
bucket.partitionDirty = true;
|
|
3646
|
+
continue;
|
|
3647
|
+
}
|
|
2461
3648
|
const oldKey = tracker.modeKey;
|
|
2462
3649
|
if (!tracker.recompute()) continue;
|
|
2463
3650
|
if (tracker.modeKey === oldKey) continue;
|
|
2464
3651
|
|
|
2465
|
-
const bucket = drawIdToBucket[drawId];
|
|
2466
3652
|
const localSlot = drawIdToLocalSlot[drawId];
|
|
2467
3653
|
const oldSlotIdx = drawIdToSlotIdx[drawId];
|
|
2468
3654
|
if (bucket === undefined || localSlot === undefined || oldSlotIdx === undefined) continue;
|
|
@@ -2534,12 +3720,27 @@ export function buildHeapScene(
|
|
|
2534
3720
|
if (allocDirty.size > 0) {
|
|
2535
3721
|
for (const av of allocDirty) {
|
|
2536
3722
|
pool.repack(device, arena.attrs, av, av.getValue(tok));
|
|
2537
|
-
|
|
2538
|
-
if (e !== undefined) totalDirtyBytes += e.dataBytes;
|
|
3723
|
+
totalDirtyBytes += pool.totalDataBytes(av);
|
|
2539
3724
|
}
|
|
2540
3725
|
allocDirty.clear();
|
|
2541
3726
|
}
|
|
2542
3727
|
|
|
3728
|
+
// 1c. HeapDrawSpec.active drain — flip drawTable.indexCount
|
|
3729
|
+
// between origIndexCount (visible) and 0 (hidden) for any RO
|
|
3730
|
+
// whose active aval ticked since last frame. Set is small in
|
|
3731
|
+
// steady state (only the ROs whose visibility actually
|
|
3732
|
+
// changed). No pool / arena / freelist activity.
|
|
3733
|
+
if (activeDirty.size > 0) {
|
|
3734
|
+
for (const did of activeDirty) {
|
|
3735
|
+
const av = drawIdToActiveAval.get(did);
|
|
3736
|
+
if (av === undefined) continue;
|
|
3737
|
+
const a = av.getValue(tok);
|
|
3738
|
+
const orig = drawIdToOrigIndexCount.get(did) ?? 0;
|
|
3739
|
+
setEffectiveIndexCount(did, a ? orig : 0);
|
|
3740
|
+
}
|
|
3741
|
+
activeDirty.clear();
|
|
3742
|
+
}
|
|
3743
|
+
|
|
2543
3744
|
// 1b. Atlas-texture aval reactivity: an `aval<ITexture>` that
|
|
2544
3745
|
// drives an atlas placement was marked. Repack the pool entry
|
|
2545
3746
|
// and rewrite the drawHeader fields of every (bucket, slot)
|
|
@@ -2625,6 +3826,9 @@ export function buildHeapScene(
|
|
|
2625
3826
|
}
|
|
2626
3827
|
// Phase 5c.2: each slot has its own drawTable + shadow. Upload
|
|
2627
3828
|
// the dirty range of every slot, not just slots[0].
|
|
3829
|
+
// Phase 5c.3: GPU-routed buckets skip this entirely — the
|
|
3830
|
+
// partition kernel writes slot drawTables directly each frame.
|
|
3831
|
+
if (bucket.gpuRouted) continue;
|
|
2628
3832
|
for (const slot of bucket.slots) {
|
|
2629
3833
|
if (slot.drawTableDirtyMax <= slot.drawTableDirtyMin) continue;
|
|
2630
3834
|
const shadow = slot.drawTableShadow!;
|
|
@@ -2698,15 +3902,24 @@ export function buildHeapScene(
|
|
|
2698
3902
|
stats.derivedPullMs = _t1 - _t0;
|
|
2699
3903
|
stats.derivedUploadMs = _t2 - _t1;
|
|
2700
3904
|
stats.derivedEncodeMs = _t3 - _t2;
|
|
2701
|
-
stats.derivedRecords = derivedScene.
|
|
3905
|
+
stats.derivedRecords = derivedScene.totalRecordCount;
|
|
2702
3906
|
}
|
|
2703
3907
|
let anyDirty = false;
|
|
2704
3908
|
outer: for (const b of buckets) {
|
|
3909
|
+
if (b.gpuRouted && b.partitionDirty) { anyDirty = true; break outer; }
|
|
2705
3910
|
for (const s of b.slots) {
|
|
2706
3911
|
if (s.scanDirty) { anyDirty = true; break outer; }
|
|
2707
3912
|
}
|
|
2708
3913
|
}
|
|
2709
3914
|
if (!anyDirty) return;
|
|
3915
|
+
// Phase 5c.3: per GPU-routed bucket — dispatch partition (its own
|
|
3916
|
+
// compute pass), then copyBufferToBuffer slot counts into scan
|
|
3917
|
+
// paramsBuf. Must happen BEFORE the shared scan pass below.
|
|
3918
|
+
for (const b of buckets) {
|
|
3919
|
+
if (b.gpuRouted && b.partitionDirty) {
|
|
3920
|
+
dispatchPartition(b, enc, token);
|
|
3921
|
+
}
|
|
3922
|
+
}
|
|
2710
3923
|
const pass = enc.beginComputePass({ label: "heapScene/scan" });
|
|
2711
3924
|
for (const b of buckets) {
|
|
2712
3925
|
for (let i = 0; i < b.slots.length; i++) {
|
|
@@ -2867,8 +4080,18 @@ export function buildHeapScene(
|
|
|
2867
4080
|
let tilesChecked = 0, tilesBad = 0;
|
|
2868
4081
|
let vidChecks = 0, vidBad = 0;
|
|
2869
4082
|
const push = (s: string) => { if (issues.length < 60) issues.push(s); };
|
|
2870
|
-
|
|
2871
|
-
|
|
4083
|
+
// §3 v1: validateHeap currently only diagnoses chunk 0 — a
|
|
4084
|
+
// multi-chunk-aware validator needs per-bucket-chunkIdx
|
|
4085
|
+
// staging which is a bigger refactor; assert for now so
|
|
4086
|
+
// multi-chunk callers don't get false-passes.
|
|
4087
|
+
if (arena.attrs.chunkCount > 1 || arena.indices.chunkCount > 1) {
|
|
4088
|
+
throw new Error(
|
|
4089
|
+
"heapScene/validateHeap: multi-chunk arenas not yet supported by this validator (chunk-0-only). " +
|
|
4090
|
+
"Run with the default chunk cap (no second chunk opened) for now.",
|
|
4091
|
+
);
|
|
4092
|
+
}
|
|
4093
|
+
const arenaSize = arena.attrs.chunk(0).buffer.size;
|
|
4094
|
+
const indicesSize = arena.indices.chunk(0).buffer.size;
|
|
2872
4095
|
|
|
2873
4096
|
const enc = device.createCommandEncoder({ label: "validateHeap" });
|
|
2874
4097
|
const stage = (src: GPUBuffer, size: number): GPUBuffer => {
|
|
@@ -2880,8 +4103,8 @@ export function buildHeapScene(
|
|
|
2880
4103
|
return c;
|
|
2881
4104
|
};
|
|
2882
4105
|
|
|
2883
|
-
const arenaCopy = stage(arena.attrs.buffer, arenaSize);
|
|
2884
|
-
const indicesCopy = indicesSize > 0 ? stage(arena.indices.buffer, indicesSize) : undefined;
|
|
4106
|
+
const arenaCopy = stage(arena.attrs.chunk(0).buffer, arenaSize);
|
|
4107
|
+
const indicesCopy = indicesSize > 0 ? stage(arena.indices.chunk(0).buffer, indicesSize) : undefined;
|
|
2885
4108
|
type DC = {
|
|
2886
4109
|
bucket: typeof buckets[number];
|
|
2887
4110
|
drawHeap: GPUBuffer;
|
|
@@ -2967,7 +4190,26 @@ export function buildHeapScene(
|
|
|
2967
4190
|
const length = arenaU32[refU32 + 1]!;
|
|
2968
4191
|
|
|
2969
4192
|
if (!KNOWN_TYPE_IDS.has(typeId)) {
|
|
2970
|
-
|
|
4193
|
+
// Dump the first 8 u32s of the alloc to help diagnose
|
|
4194
|
+
// what wrote garbage onto the header. Also peek the
|
|
4195
|
+
// FOUR u32s BEFORE the alloc to see if a neighbour
|
|
4196
|
+
// overran into us.
|
|
4197
|
+
const before = refU32 >= 4
|
|
4198
|
+
? `prev=[${arenaU32[refU32 - 4]},${arenaU32[refU32 - 3]},${arenaU32[refU32 - 2]},${arenaU32[refU32 - 1]}] `
|
|
4199
|
+
: "";
|
|
4200
|
+
const here = `here=[${arenaU32[refU32]},${arenaU32[refU32 + 1]},${arenaU32[refU32 + 2]},${arenaU32[refU32 + 3]},${arenaU32[refU32 + 4]},${arenaU32[refU32 + 5]},${arenaU32[refU32 + 6]},${arenaU32[refU32 + 7]}]`;
|
|
4201
|
+
// Compare CPU shadow vs GPU read-back. Differ → GPU
|
|
4202
|
+
// write corrupted; match → bug is in CPU-side write path
|
|
4203
|
+
// OR the alloc never went through pool.acquire.
|
|
4204
|
+
const shadowChunk = arena.attrs.chunk(0);
|
|
4205
|
+
const shadowU32 = shadowChunk.peekShadowU32(ref, 8);
|
|
4206
|
+
const shadowStr = `shadow=[${shadowU32[0]},${shadowU32[1]},${shadowU32[2]},${shadowU32[3]},${shadowU32[4]},${shadowU32[5]},${shadowU32[6]},${shadowU32[7]}]`;
|
|
4207
|
+
const cpuMatches = shadowU32[0] === arenaU32[refU32]
|
|
4208
|
+
&& shadowU32[1] === arenaU32[refU32 + 1]
|
|
4209
|
+
&& shadowU32[2] === arenaU32[refU32 + 2]
|
|
4210
|
+
&& shadowU32[3] === arenaU32[refU32 + 3];
|
|
4211
|
+
const verdict = cpuMatches ? "GPU=CPU (likely CPU-write bug)" : "GPU≠CPU (likely GPU-write corruption)";
|
|
4212
|
+
push(`bucket#${bucketIdx} slot=${slot} field='${f.name}' alloc@0x${ref.toString(16)} typeId=${typeId} unknown ${verdict} ${before}${here} ${shadowStr}`);
|
|
2971
4213
|
attrAllocsBad++;
|
|
2972
4214
|
continue;
|
|
2973
4215
|
}
|
|
@@ -3075,11 +4317,18 @@ export function buildHeapScene(
|
|
|
3075
4317
|
tilesBad++;
|
|
3076
4318
|
}
|
|
3077
4319
|
}
|
|
4320
|
+
// The scan kernel intentionally writes `numRecords - 1` as the
|
|
4321
|
+
// sentinel (not `numRecords`) — the render VS's binary search
|
|
4322
|
+
// uses `hi = firstDrawInTile[_tileIdx + 1u]` as an *inclusive*
|
|
4323
|
+
// upper bound, so the sentinel must point at the last valid
|
|
4324
|
+
// record slot. See scanKernel.ts `buildTileIndex` for the
|
|
4325
|
+
// full rationale.
|
|
4326
|
+
const expectedSentinel = recordCount === 0 ? 0 : recordCount - 1;
|
|
3078
4327
|
const sentinel = fdt[dc.numTiles]!;
|
|
3079
4328
|
tilesChecked++;
|
|
3080
|
-
if (sentinel !==
|
|
4329
|
+
if (sentinel !== expectedSentinel) {
|
|
3081
4330
|
push(`bucket#${bucketIdx} firstDrawInTile[${dc.numTiles}] sentinel ` +
|
|
3082
|
-
`got=${sentinel} expected=${
|
|
4331
|
+
`got=${sentinel} expected=${expectedSentinel}`);
|
|
3083
4332
|
tilesBad++;
|
|
3084
4333
|
}
|
|
3085
4334
|
dc.firstDrawInTile.unmap();
|
|
@@ -3170,8 +4419,12 @@ export function buildHeapScene(
|
|
|
3170
4419
|
let emitsChecked = 0;
|
|
3171
4420
|
const push = (s: string) => { if (issues.length < 30) issues.push(s); };
|
|
3172
4421
|
|
|
3173
|
-
|
|
3174
|
-
|
|
4422
|
+
// §3 v1: chunk-0-only like validateHeap.
|
|
4423
|
+
if (arena.attrs.chunkCount > 1 || arena.indices.chunkCount > 1) {
|
|
4424
|
+
throw new Error("heapScene/simulateDraws: multi-chunk arenas not yet supported (chunk-0-only).");
|
|
4425
|
+
}
|
|
4426
|
+
const arenaSize = arena.attrs.chunk(0).buffer.size;
|
|
4427
|
+
const indicesSize = arena.indices.chunk(0).buffer.size;
|
|
3175
4428
|
|
|
3176
4429
|
// Stage all buffers we'll need.
|
|
3177
4430
|
const enc = device.createCommandEncoder({ label: "simulateDraws" });
|
|
@@ -3182,8 +4435,8 @@ export function buildHeapScene(
|
|
|
3182
4435
|
enc.copyBufferToBuffer(src, 0, c, 0, size);
|
|
3183
4436
|
return c;
|
|
3184
4437
|
};
|
|
3185
|
-
const arenaCopy = stage(arena.attrs.buffer, arenaSize);
|
|
3186
|
-
const indicesCopy = indicesSize > 0 ? stage(arena.indices.buffer, indicesSize) : undefined;
|
|
4438
|
+
const arenaCopy = stage(arena.attrs.chunk(0).buffer, arenaSize);
|
|
4439
|
+
const indicesCopy = indicesSize > 0 ? stage(arena.indices.chunk(0).buffer, indicesSize) : undefined;
|
|
3187
4440
|
type DC = {
|
|
3188
4441
|
bucket: typeof buckets[number];
|
|
3189
4442
|
drawHeap: GPUBuffer;
|
|
@@ -3453,19 +4706,25 @@ export function buildHeapScene(
|
|
|
3453
4706
|
// cross-RO triangle symptom.
|
|
3454
4707
|
const allocClaims: { ref: number; bytes: number; owner: string }[] = [];
|
|
3455
4708
|
|
|
4709
|
+
// §3 v1: chunk-0-only (see validateHeap).
|
|
4710
|
+
if (arena.attrs.chunkCount > 1 || arena.indices.chunkCount > 1) {
|
|
4711
|
+
throw new Error("heapScene/checkTriangleCoherence: multi-chunk arenas not yet supported (chunk-0-only).");
|
|
4712
|
+
}
|
|
4713
|
+
const arenaBuf0 = arena.attrs.chunk(0).buffer;
|
|
4714
|
+
const indicesBuf0 = arena.indices.chunk(0).buffer;
|
|
3456
4715
|
// Stage arena + arena.indices once (shared across buckets).
|
|
3457
4716
|
const arenaCopy = device.createBuffer({
|
|
3458
|
-
size:
|
|
4717
|
+
size: arenaBuf0.size,
|
|
3459
4718
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3460
4719
|
});
|
|
3461
|
-
const indicesCopy =
|
|
3462
|
-
size:
|
|
4720
|
+
const indicesCopy = indicesBuf0.size > 0 ? device.createBuffer({
|
|
4721
|
+
size: indicesBuf0.size,
|
|
3463
4722
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3464
4723
|
}) : undefined;
|
|
3465
4724
|
{
|
|
3466
4725
|
const enc = device.createCommandEncoder({ label: "checkTriangleCoherence.arena" });
|
|
3467
|
-
enc.copyBufferToBuffer(
|
|
3468
|
-
if (indicesCopy !== undefined) enc.copyBufferToBuffer(
|
|
4726
|
+
enc.copyBufferToBuffer(arenaBuf0, 0, arenaCopy, 0, arenaBuf0.size);
|
|
4727
|
+
if (indicesCopy !== undefined) enc.copyBufferToBuffer(indicesBuf0, 0, indicesCopy, 0, indicesBuf0.size);
|
|
3469
4728
|
device.queue.submit([enc.finish()]);
|
|
3470
4729
|
}
|
|
3471
4730
|
await arenaCopy.mapAsync(GPUMapMode.READ);
|
|
@@ -3898,7 +5157,7 @@ export function buildHeapScene(
|
|
|
3898
5157
|
{ binding: 0, resource: { buffer: target.slots[0]!.drawTableBuf!.buffer } },
|
|
3899
5158
|
{ binding: 1, resource: { buffer: target.slots[0]!.firstDrawInTileBuf!.buffer } },
|
|
3900
5159
|
{ binding: 2, resource: { buffer: sampleBuf } },
|
|
3901
|
-
{ binding: 3, resource: { buffer: arena.indices.buffer } },
|
|
5160
|
+
{ binding: 3, resource: { buffer: arena.indices.chunk(target.chunkIdx).buffer } },
|
|
3902
5161
|
{ binding: 4, resource: { buffer: outBuf } },
|
|
3903
5162
|
{ binding: 5, resource: { buffer: paramBuf } },
|
|
3904
5163
|
],
|
|
@@ -3921,12 +5180,13 @@ export function buildHeapScene(
|
|
|
3921
5180
|
sampleBuf.destroy(); outBuf.destroy(); paramBuf.destroy();
|
|
3922
5181
|
|
|
3923
5182
|
// Read drawTable + indices for CPU expectation.
|
|
5183
|
+
const idxBuf = arena.indices.chunk(target.chunkIdx).buffer;
|
|
3924
5184
|
const indicesCopy2 = device.createBuffer({
|
|
3925
|
-
size:
|
|
5185
|
+
size: idxBuf.size,
|
|
3926
5186
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3927
5187
|
});
|
|
3928
5188
|
const enc2 = device.createCommandEncoder();
|
|
3929
|
-
enc2.copyBufferToBuffer(
|
|
5189
|
+
enc2.copyBufferToBuffer(idxBuf, 0, indicesCopy2, 0, idxBuf.size);
|
|
3930
5190
|
device.queue.submit([enc2.finish()]);
|
|
3931
5191
|
await indicesCopy2.mapAsync(GPUMapMode.READ);
|
|
3932
5192
|
const indicesU32 = new Uint32Array(indicesCopy2.getMappedRange().slice(0));
|
|
@@ -3980,5 +5240,7 @@ export function buildHeapScene(
|
|
|
3980
5240
|
},
|
|
3981
5241
|
};
|
|
3982
5242
|
|
|
3983
|
-
|
|
5243
|
+
const __addDrawCallsGetter = (): number => __addDrawCalls;
|
|
5244
|
+
const __removeDrawCallsGetter = (): number => __removeDrawCalls;
|
|
5245
|
+
return { frame, update, encodeIntoPass, encodeComputePrep, addDraw, removeDraw, stats, dispose, _debug, __addDrawCalls: __addDrawCallsGetter, __removeDrawCalls: __removeDrawCallsGetter } as HeapScene;
|
|
3984
5246
|
}
|