@aardworx/wombat.rendering 0.9.32 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime/derivedModes/kernelCodegen.d.ts +90 -0
- package/dist/runtime/derivedModes/kernelCodegen.d.ts.map +1 -0
- package/dist/runtime/derivedModes/kernelCodegen.js +623 -0
- package/dist/runtime/derivedModes/kernelCodegen.js.map +1 -0
- package/dist/runtime/derivedModes/modeKeyCpu.d.ts.map +1 -1
- package/dist/runtime/derivedModes/modeKeyCpu.js +23 -67
- package/dist/runtime/derivedModes/modeKeyCpu.js.map +1 -1
- package/dist/runtime/derivedModes/partitionDispatcher.d.ts +79 -0
- package/dist/runtime/derivedModes/partitionDispatcher.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionDispatcher.js +303 -0
- package/dist/runtime/derivedModes/partitionDispatcher.js.map +1 -0
- package/dist/runtime/derivedModes/partitionKernel.d.ts +12 -0
- package/dist/runtime/derivedModes/partitionKernel.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionKernel.js +127 -0
- package/dist/runtime/derivedModes/partitionKernel.js.map +1 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.d.ts +4 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.js +24 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.js.map +1 -0
- package/dist/runtime/derivedModes/rule.d.ts +38 -86
- package/dist/runtime/derivedModes/rule.d.ts.map +1 -1
- package/dist/runtime/derivedModes/rule.js +34 -74
- package/dist/runtime/derivedModes/rule.js.map +1 -1
- package/dist/runtime/derivedUniforms/dispatch.d.ts +10 -8
- package/dist/runtime/derivedUniforms/dispatch.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/dispatch.js +31 -18
- package/dist/runtime/derivedUniforms/dispatch.js.map +1 -1
- package/dist/runtime/derivedUniforms/marker.d.ts +90 -11
- package/dist/runtime/derivedUniforms/marker.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/marker.js +161 -15
- package/dist/runtime/derivedUniforms/marker.js.map +1 -1
- package/dist/runtime/derivedUniforms/sceneIntegration.d.ts +25 -6
- package/dist/runtime/derivedUniforms/sceneIntegration.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/sceneIntegration.js +61 -20
- package/dist/runtime/derivedUniforms/sceneIntegration.js.map +1 -1
- package/dist/runtime/heapEligibility.d.ts.map +1 -1
- package/dist/runtime/heapEligibility.js +22 -2
- package/dist/runtime/heapEligibility.js.map +1 -1
- package/dist/runtime/heapScene/chunkedArena.d.ts +76 -0
- package/dist/runtime/heapScene/chunkedArena.d.ts.map +1 -0
- package/dist/runtime/heapScene/chunkedArena.js +223 -0
- package/dist/runtime/heapScene/chunkedArena.js.map +1 -0
- package/dist/runtime/heapScene/freelist.d.ts +50 -0
- package/dist/runtime/heapScene/freelist.d.ts.map +1 -0
- package/dist/runtime/heapScene/freelist.js +160 -0
- package/dist/runtime/heapScene/freelist.js.map +1 -0
- package/dist/runtime/heapScene/growBuffer.d.ts +23 -2
- package/dist/runtime/heapScene/growBuffer.d.ts.map +1 -1
- package/dist/runtime/heapScene/growBuffer.js +35 -3
- package/dist/runtime/heapScene/growBuffer.js.map +1 -1
- package/dist/runtime/heapScene/pools.d.ts +65 -42
- package/dist/runtime/heapScene/pools.d.ts.map +1 -1
- package/dist/runtime/heapScene/pools.js +206 -134
- package/dist/runtime/heapScene/pools.js.map +1 -1
- package/dist/runtime/heapScene.d.ts +12 -0
- package/dist/runtime/heapScene.d.ts.map +1 -1
- package/dist/runtime/heapScene.js +949 -167
- package/dist/runtime/heapScene.js.map +1 -1
- package/dist/runtime/hybridScene.d.ts +7 -0
- package/dist/runtime/hybridScene.d.ts.map +1 -1
- package/dist/runtime/hybridScene.js +1 -0
- package/dist/runtime/hybridScene.js.map +1 -1
- package/dist/runtime/index.d.ts +4 -3
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +4 -3
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/renderTask.d.ts +5 -0
- package/dist/runtime/renderTask.d.ts.map +1 -1
- package/dist/runtime/renderTask.js +1 -0
- package/dist/runtime/renderTask.js.map +1 -1
- package/dist/runtime/runtime.d.ts +6 -0
- package/dist/runtime/runtime.d.ts.map +1 -1
- package/dist/runtime/runtime.js +1 -0
- package/dist/runtime/runtime.js.map +1 -1
- package/package.json +1 -1
- package/src/runtime/derivedModes/kernelCodegen.ts +673 -0
- package/src/runtime/derivedModes/modeKeyCpu.ts +22 -60
- package/src/runtime/derivedModes/partitionDispatcher.ts +349 -0
- package/src/runtime/derivedModes/partitionKernelLayout.ts +25 -0
- package/src/runtime/derivedModes/rule.ts +66 -145
- package/src/runtime/derivedUniforms/dispatch.ts +48 -21
- package/src/runtime/derivedUniforms/marker.ts +210 -16
- package/src/runtime/derivedUniforms/sceneIntegration.ts +67 -20
- package/src/runtime/heapEligibility.ts +22 -1
- package/src/runtime/heapScene/chunkedArena.ts +238 -0
- package/src/runtime/heapScene/freelist.ts +161 -0
- package/src/runtime/heapScene/growBuffer.ts +39 -2
- package/src/runtime/heapScene/pools.ts +231 -143
- package/src/runtime/heapScene.ts +1134 -161
- package/src/runtime/hybridScene.ts +8 -0
- package/src/runtime/index.ts +12 -9
- package/src/runtime/renderTask.ts +6 -0
- package/src/runtime/runtime.ts +7 -0
- package/src/runtime/derivedModes/gpuDispatcher.ts +0 -226
- package/src/runtime/derivedModes/gpuKernel.ts +0 -98
package/src/runtime/heapScene.ts
CHANGED
|
@@ -85,7 +85,7 @@ import {
|
|
|
85
85
|
} from "./heapScene/growBuffer.js";
|
|
86
86
|
import {
|
|
87
87
|
UniformPool, IndexPool, DrawHeap,
|
|
88
|
-
AttributeArena, IndexAllocator,
|
|
88
|
+
AttributeArena, IndexAllocator,
|
|
89
89
|
buildArenaState, arenaBytes, writeAttribute,
|
|
90
90
|
asAval, isBufferView, asFloat32,
|
|
91
91
|
ALLOC_HEADER_BYTES, ALLOC_HEADER_PAD_TO,
|
|
@@ -94,9 +94,21 @@ import {
|
|
|
94
94
|
} from "./heapScene/pools.js";
|
|
95
95
|
import { encodeModeKey, type PipelineStateDescriptor } from "./pipelineCache/index.js";
|
|
96
96
|
import { snapshotDescriptor, ModeKeyTracker } from "./derivedModes/modeKeyCpu.js";
|
|
97
|
-
import {
|
|
98
|
-
import {
|
|
97
|
+
import { GpuPartitionScene } from "./derivedModes/partitionDispatcher.js";
|
|
98
|
+
import {
|
|
99
|
+
emitPartitionKernel,
|
|
100
|
+
substituteReadInputInStmt,
|
|
101
|
+
collectUniformReadsInStmt,
|
|
102
|
+
rewriteOutputsToSlotIndices,
|
|
103
|
+
} from "./derivedModes/kernelCodegen.js";
|
|
104
|
+
import { type DerivedModeRule, type ModeAxis } from "./derivedModes/rule.js";
|
|
105
|
+
import type { CullMode, FrontFace, Topology } from "./pipelineCache/index.js";
|
|
99
106
|
import { addMarkingCallback } from "@aardworx/wombat.adaptive";
|
|
107
|
+
import {
|
|
108
|
+
analyseOutputSet, evaluateStructural, evaluateStructuralSet,
|
|
109
|
+
stableStringify,
|
|
110
|
+
type Expr, type Stmt,
|
|
111
|
+
} from "@aardworx/wombat.shader/ir";
|
|
100
112
|
|
|
101
113
|
// GrowBuffer + ALIGN16 + MIN_BUFFER_BYTES + POW2 live in ./heapScene/growBuffer.
|
|
102
114
|
// Packers (WGSL-type → JS-value → arena bytes) live in ./heapScene/packers.
|
|
@@ -162,6 +174,10 @@ interface BucketSlot {
|
|
|
162
174
|
pipeline: GPURenderPipeline;
|
|
163
175
|
/** modeKey this slot covers — the bitfield-encoded descriptor. */
|
|
164
176
|
modeKey: bigint;
|
|
177
|
+
/** Snapshotted full descriptor the slot was created with. Used by
|
|
178
|
+
* CPU→GPU promotion to synthesise per-distinct-descriptor combos
|
|
179
|
+
* for pre-existing CPU-routed ROs. */
|
|
180
|
+
cpuDescriptor?: PipelineStateDescriptor;
|
|
165
181
|
/** Render bind group (drawTable + firstDrawInTile are per-slot). */
|
|
166
182
|
bindGroup?: GPUBindGroup;
|
|
167
183
|
|
|
@@ -192,10 +208,57 @@ interface BucketSlot {
|
|
|
192
208
|
scanDirty: boolean;
|
|
193
209
|
}
|
|
194
210
|
|
|
211
|
+
/**
|
|
212
|
+
* One distinct derived-mode rule registered on a GPU-routed bucket,
|
|
213
|
+
* keyed by axis. Multiple ROs in the same bucket may carry different
|
|
214
|
+
* rules across different axes; each axis maintains its own
|
|
215
|
+
* `axisRuleId` namespace. A bucket-unique `comboId` (a chosen
|
|
216
|
+
* tuple of (axis → axisRuleId, or "no rule") per RO) is baked into
|
|
217
|
+
* master-pool records and selects the right composer fn in the
|
|
218
|
+
* partition kernel.
|
|
219
|
+
*/
|
|
220
|
+
interface AxisRuleEntry {
|
|
221
|
+
readonly axis: ModeAxis;
|
|
222
|
+
readonly axisRuleId: number;
|
|
223
|
+
readonly contentHash: string;
|
|
224
|
+
readonly rule: DerivedModeRule<ModeAxis>;
|
|
225
|
+
readonly body: Stmt;
|
|
226
|
+
/** Last declared u32 we re-specialised against — drives the
|
|
227
|
+
* dispatch-time "needs re-specialise?" check. */
|
|
228
|
+
lastDeclaredU32: number | undefined;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* One combination of rules carried by ROs in a GPU-routed bucket.
|
|
233
|
+
* A combo specifies at most one rule per axis (axes absent from the
|
|
234
|
+
* map fall back to baseDescriptor at a compile-time-constant per-
|
|
235
|
+
* axis index). Master records store `comboId` — the partition
|
|
236
|
+
* kernel emits one `combo_<id>` fn per combo composing per-axis
|
|
237
|
+
* indices into a global slot index via mixed-radix encoding.
|
|
238
|
+
*/
|
|
239
|
+
interface ComboEntry {
|
|
240
|
+
readonly comboId: number;
|
|
241
|
+
readonly comboKey: string;
|
|
242
|
+
/** axis → axisRuleId for axes this combo has a rule on. */
|
|
243
|
+
readonly axisRules: ReadonlyMap<ModeAxis, number>;
|
|
244
|
+
/** axis → explicit fixed value for axes the combo has no rule on
|
|
245
|
+
* but wants to override baseDescriptor's value (e.g. pre-existing
|
|
246
|
+
* CPU-routed ROs whose pipelineState differs from baseDescriptor
|
|
247
|
+
* on a non-ruled axis). Axes absent from both maps fall back to
|
|
248
|
+
* baseDescriptor at codegen time. */
|
|
249
|
+
readonly axisFixedValues: ReadonlyMap<ModeAxis, unknown>;
|
|
250
|
+
}
|
|
251
|
+
|
|
195
252
|
interface Bucket {
|
|
196
253
|
readonly label: string;
|
|
197
254
|
readonly textures: HeapTextureSet | undefined;
|
|
198
255
|
readonly layout: BucketLayout;
|
|
256
|
+
/** §3: which arena chunk this bucket's ROs live in. The bucket's
|
|
257
|
+
* bind groups reference `arena.attrs.chunk(chunkIdx).buffer` and
|
|
258
|
+
* `arena.indices.chunk(chunkIdx).buffer`. Two ROs sharing an
|
|
259
|
+
* effect that landed in different chunks live in DIFFERENT
|
|
260
|
+
* buckets — encode emits a draw per (effect, chunkIdx). */
|
|
261
|
+
readonly chunkIdx: number;
|
|
199
262
|
/**
|
|
200
263
|
* One entry per distinct pipeline state. Phase 5a: always length 1
|
|
201
264
|
* (per-RO pipeline-state still keyed at the bucket level).
|
|
@@ -278,6 +341,60 @@ interface Bucket {
|
|
|
278
341
|
* for slots without an atlas source aval.
|
|
279
342
|
*/
|
|
280
343
|
readonly localAtlasArrIdx: (number | undefined)[];
|
|
344
|
+
|
|
345
|
+
// ─── Phase 5c.3 GPU-routed bucket state ─────────────────────────
|
|
346
|
+
/**
|
|
347
|
+
* True when this bucket's first RO carried a GPU-flavoured mode
|
|
348
|
+
* rule (e.g. `gpuFlipCullByDeterminant`). The partition kernel
|
|
349
|
+
* decides per-record routing each frame; CPU never reslots.
|
|
350
|
+
*/
|
|
351
|
+
gpuRouted: boolean;
|
|
352
|
+
/**
|
|
353
|
+
* Per-bucket partition dispatcher. Owns the master record buffer
|
|
354
|
+
* + per-slot atomic counters + params. Slot draw buffers are
|
|
355
|
+
* externally owned (point at `bucket.slots[i].drawTableBuf.buffer`).
|
|
356
|
+
*/
|
|
357
|
+
partitionScene: GpuPartitionScene | undefined;
|
|
358
|
+
/**
|
|
359
|
+
* drawId → master record index (for swap-pop on removeDraw).
|
|
360
|
+
*/
|
|
361
|
+
drawIdToRecord: Map<number, number> | undefined;
|
|
362
|
+
/**
|
|
363
|
+
* master record index → drawId (for fixup when swap-pop moves the
|
|
364
|
+
* tail record into the freed slot).
|
|
365
|
+
*/
|
|
366
|
+
recordToDrawId: number[] | undefined;
|
|
367
|
+
/**
|
|
368
|
+
* Per-axis distinct rules registered on this bucket, keyed by
|
|
369
|
+
* content hash (axis + expr.id + declared identity). `axisRuleId`
|
|
370
|
+
* is unique within an axis. ROs carry an arbitrary subset of axis
|
|
371
|
+
* rules; combos (below) dedupe distinct subsets across ROs.
|
|
372
|
+
*/
|
|
373
|
+
rulesByAxis: Map<ModeAxis, Map<string, AxisRuleEntry>> | undefined;
|
|
374
|
+
/** Per-axis ordered list of registered rule hashes; index =
|
|
375
|
+
* `axisRuleId` for that axis. */
|
|
376
|
+
axisRuleOrder: Map<ModeAxis, string[]> | undefined;
|
|
377
|
+
/** Bucket-wide ordered list of arena uniform names read by any
|
|
378
|
+
* rule body. Each entry maps to a `ref<i>: u32` field in the
|
|
379
|
+
* master-record uniform tail; appendRecord packs them per RO. */
|
|
380
|
+
uniformOrder: string[] | undefined;
|
|
381
|
+
uniformIdx: Map<string, number> | undefined;
|
|
382
|
+
/** Distinct combos appearing on ROs in this bucket. */
|
|
383
|
+
combosByKey: Map<string, ComboEntry> | undefined;
|
|
384
|
+
/** `comboId` → comboKey. Index = comboId. */
|
|
385
|
+
comboOrder: string[] | undefined;
|
|
386
|
+
/** drawId → comboId. Master records carry the comboId; this is
|
|
387
|
+
* the CPU-side mirror used when an RO is removed. */
|
|
388
|
+
drawIdToComboId: Map<number, number> | undefined;
|
|
389
|
+
/** Seed PipelineStateDescriptor (the descriptor of the first GPU-
|
|
390
|
+
* routed RO). Used as the base for slot descriptors. */
|
|
391
|
+
baseDescriptor: PipelineStateDescriptor | undefined;
|
|
392
|
+
/**
|
|
393
|
+
* Has the partition kernel's inputs (records or rule deps)
|
|
394
|
+
* changed since last dispatch? Set true on addRO / removeDraw /
|
|
395
|
+
* any tracker-leaf mark. Reset after dispatch.
|
|
396
|
+
*/
|
|
397
|
+
partitionDirty: boolean;
|
|
281
398
|
}
|
|
282
399
|
|
|
283
400
|
// ---------------------------------------------------------------------------
|
|
@@ -572,6 +689,18 @@ export interface BuildHeapSceneOptions {
|
|
|
572
689
|
* effect's `spec.inputs[name]` like any other uniform.
|
|
573
690
|
*/
|
|
574
691
|
readonly enableDerivedUniforms?: boolean;
|
|
692
|
+
/**
|
|
693
|
+
* §3 — per-arena-chunk size cap in bytes. When omitted, falls
|
|
694
|
+
* back to `min(device.limits.maxStorageBufferBindingSize,
|
|
695
|
+
* DEFAULT_MAX_BUFFER_BYTES = 256 MB)` so chunks grow as far as
|
|
696
|
+
* the hardware allows. Demos / tests can pass a small value
|
|
697
|
+
* (e.g. 4 MB) to exercise multi-chunk routing without realistic
|
|
698
|
+
* workload pressure. Each chunk lives in its own GPUBuffer;
|
|
699
|
+
* buckets bind exactly one chunk and emit one draw call per
|
|
700
|
+
* (effect, chunkIdx). Shared-aval avals that land in different
|
|
701
|
+
* chunks duplicate their pool entries (accepted trade-off).
|
|
702
|
+
*/
|
|
703
|
+
readonly maxChunkBytes?: number;
|
|
575
704
|
}
|
|
576
705
|
|
|
577
706
|
export function buildHeapScene(
|
|
@@ -609,6 +738,7 @@ export function buildHeapScene(
|
|
|
609
738
|
const arena = buildArenaState(
|
|
610
739
|
device, 64 * 1024, 16 * 1024, "heapScene",
|
|
611
740
|
GPUBufferUsage.STORAGE,
|
|
741
|
+
opts.maxChunkBytes,
|
|
612
742
|
);
|
|
613
743
|
|
|
614
744
|
// ─── Per-draw global bookkeeping (sparse, indexed by drawId) ──────
|
|
@@ -651,12 +781,6 @@ export function buildHeapScene(
|
|
|
651
781
|
modeAvalToDrawIds.set(av, set);
|
|
652
782
|
modeAvalCallbacks.set(av, addMarkingCallback(av, () => {
|
|
653
783
|
for (const id of set!) dirtyModeKeyDrawIds.add(id);
|
|
654
|
-
// Mark the GPU mode-rules dispatcher dirty too — at least one
|
|
655
|
-
// of those drawIds may carry a GPU rule whose input changed.
|
|
656
|
-
// The dispatch path checks consumeDirty() before encoding, so
|
|
657
|
-
// an unrelated aval mark (e.g. Color change on hover) leaves
|
|
658
|
-
// the kernel quiet.
|
|
659
|
-
gpuModesScene?.markDirty();
|
|
660
784
|
}));
|
|
661
785
|
}
|
|
662
786
|
set.add(drawId);
|
|
@@ -875,7 +999,7 @@ export function buildHeapScene(
|
|
|
875
999
|
override inputChanged(_t: unknown, o: IAdaptiveObject): void {
|
|
876
1000
|
// Pool avals are stored as `aval<unknown>`; the IAdaptiveObject
|
|
877
1001
|
// identity matches.
|
|
878
|
-
if (pool.
|
|
1002
|
+
if (pool.hasAny(o as unknown as aval<unknown>)) {
|
|
879
1003
|
allocDirty.add(o as unknown as aval<unknown>);
|
|
880
1004
|
return;
|
|
881
1005
|
}
|
|
@@ -900,22 +1024,31 @@ export function buildHeapScene(
|
|
|
900
1024
|
// `enableDerivedUniforms: false` to opt out.
|
|
901
1025
|
const enableDerivedUniforms = opts.enableDerivedUniforms !== false;
|
|
902
1026
|
const derivedScene: DerivedUniformsScene | undefined = enableDerivedUniforms
|
|
903
|
-
? new DerivedUniformsScene(device,
|
|
1027
|
+
? new DerivedUniformsScene(device, {
|
|
904
1028
|
// Larger initial capacity reduces grow-during-first-frame churn.
|
|
905
1029
|
initialConstituentSlots: 4096,
|
|
906
1030
|
})
|
|
907
1031
|
: undefined;
|
|
1032
|
+
// §3: register each arena chunk's main-heap buffer with the
|
|
1033
|
+
// derived-uniforms scene; refresh the binding when chunks open or
|
|
1034
|
+
// a chunk's GrowBuffer reallocates.
|
|
1035
|
+
if (derivedScene !== undefined) {
|
|
1036
|
+
const wireChunk = (idx: number): void => {
|
|
1037
|
+
derivedScene.setMainHeapForChunk(idx, () => arena.attrs.chunk(idx).buffer);
|
|
1038
|
+
};
|
|
1039
|
+
for (let i = 0; i < arena.attrs.chunkCount; i++) wireChunk(i);
|
|
1040
|
+
arena.attrs.onChunkAdded(wireChunk);
|
|
1041
|
+
}
|
|
908
1042
|
/** Per-RO §7 registration handles, keyed by global drawId.
|
|
909
1043
|
* Drained on removeDraw to release slots + records. */
|
|
910
1044
|
const derivedByDrawId = new Map<number, RoRegistration>();
|
|
911
1045
|
|
|
912
|
-
//
|
|
913
|
-
//
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
const gpuModesLastKey = new Map<number, number>();
|
|
1046
|
+
// (Phase 5c.2: the per-RO routing decision is CPU-driven via
|
|
1047
|
+
// tracker.recompute(). The previous GpuDerivedModesScene dispatch +
|
|
1048
|
+
// mapAsync readback path is gone — its output was never consumed
|
|
1049
|
+
// after the rebucket-via-spec-patch approach was removed. If a
|
|
1050
|
+
// future rule's inputs only exist GPU-side, a partition-kernel-
|
|
1051
|
+
// driven scatter would put routing on the GPU; see Phase 5c.3.)
|
|
919
1052
|
|
|
920
1053
|
/** The derived-uniform rule for a uniform binding on this RO, if any: an explicit
|
|
921
1054
|
* `derivedUniform(...)` value in `spec.inputs`, or a standard trafo recipe by name
|
|
@@ -1421,11 +1554,12 @@ export function buildHeapScene(
|
|
|
1421
1554
|
// SAME global arena GPUBuffer (emscripten-style aliasing). The
|
|
1422
1555
|
// WGSL prelude declares one binding per view; the shader picks
|
|
1423
1556
|
// whichever matches its read.
|
|
1557
|
+
const attrsBuf = arena.attrs.chunk(bucket.chunkIdx).buffer;
|
|
1424
1558
|
const entries: GPUBindGroupEntry[] = [
|
|
1425
|
-
{ binding: 0, resource: { buffer:
|
|
1559
|
+
{ binding: 0, resource: { buffer: attrsBuf } }, // heapU32
|
|
1426
1560
|
{ binding: 1, resource: { buffer: bucket.drawHeap.buffer } }, // headersU32
|
|
1427
|
-
{ binding: 2, resource: { buffer:
|
|
1428
|
-
{ binding: 3, resource: { buffer:
|
|
1561
|
+
{ binding: 2, resource: { buffer: attrsBuf } }, // heapF32
|
|
1562
|
+
{ binding: 3, resource: { buffer: attrsBuf } }, // heapV4f
|
|
1429
1563
|
];
|
|
1430
1564
|
{
|
|
1431
1565
|
const s = bucket.slots[slotIdx]!;
|
|
@@ -1440,7 +1574,7 @@ export function buildHeapScene(
|
|
|
1440
1574
|
const dtBytes = Math.max(RECORD_BYTES, s.recordCount * RECORD_BYTES);
|
|
1441
1575
|
entries.push(
|
|
1442
1576
|
{ binding: 4, resource: { buffer: s.drawTableBuf.buffer, offset: 0, size: dtBytes } },
|
|
1443
|
-
{ binding: 5, resource: { buffer: arena.indices.buffer } },
|
|
1577
|
+
{ binding: 5, resource: { buffer: arena.indices.chunk(bucket.chunkIdx).buffer } },
|
|
1444
1578
|
{ binding: 6, resource: { buffer: s.firstDrawInTileBuf!.buffer } },
|
|
1445
1579
|
);
|
|
1446
1580
|
s.renderBoundRecordCount = s.recordCount;
|
|
@@ -1550,16 +1684,37 @@ export function buildHeapScene(
|
|
|
1550
1684
|
}
|
|
1551
1685
|
}
|
|
1552
1686
|
|
|
1553
|
-
arena.attrs.
|
|
1554
|
-
|
|
1555
|
-
|
|
1687
|
+
arena.attrs.onAnyResize((resizedChunkIdx) => {
|
|
1688
|
+
// Only rebuild bind groups for buckets that bind THIS chunk —
|
|
1689
|
+
// a resize in chunk K doesn't affect a bucket reading chunk N≠K.
|
|
1690
|
+
for (const b of buckets) {
|
|
1691
|
+
if (b.chunkIdx !== resizedChunkIdx) continue;
|
|
1692
|
+
for (let i = 0; i < b.slots.length; i++) {
|
|
1693
|
+
b.slots[i]!.bindGroup = buildBucketBindGroup(b, i);
|
|
1694
|
+
}
|
|
1695
|
+
if (b.gpuRouted && b.partitionScene !== undefined) {
|
|
1696
|
+
b.partitionScene.rebindSlotDrawBufs(b.slots.map(s => s.drawTableBuf!.buffer));
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
// §7: each chunk's main-heap getter is registered via
|
|
1700
|
+
// `derivedScene.setMainHeapForChunk` (at construction + on
|
|
1701
|
+
// onChunkAdded). The getter returns the chunk's current
|
|
1702
|
+
// GPUBuffer; the dispatcher's bind groups consult it per
|
|
1703
|
+
// dispatch, so no explicit rebind needed here.
|
|
1556
1704
|
});
|
|
1557
1705
|
if (atlasPool !== undefined) {
|
|
1558
1706
|
for (const f of ATLAS_PAGE_FORMATS) {
|
|
1559
1707
|
atlasPool.onPageAdded(f, () => rebuildAllBindGroups(true));
|
|
1560
1708
|
}
|
|
1561
1709
|
}
|
|
1562
|
-
arena.indices.
|
|
1710
|
+
arena.indices.onAnyResize((resizedChunkIdx) => {
|
|
1711
|
+
for (const b of buckets) {
|
|
1712
|
+
if (b.chunkIdx !== resizedChunkIdx) continue;
|
|
1713
|
+
for (let i = 0; i < b.slots.length; i++) {
|
|
1714
|
+
b.slots[i]!.bindGroup = buildBucketBindGroup(b, i);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
});
|
|
1563
1718
|
|
|
1564
1719
|
// ─── findOrCreateBucket ───────────────────────────────────────────
|
|
1565
1720
|
// Slice 3c: the bucket key collapses to (familyId, pipelineState).
|
|
@@ -1613,6 +1768,7 @@ export function buildHeapScene(
|
|
|
1613
1768
|
|
|
1614
1769
|
const slot: BucketSlot = {
|
|
1615
1770
|
pipeline, modeKey,
|
|
1771
|
+
cpuDescriptor: descriptor,
|
|
1616
1772
|
drawTableDirtyMin: Infinity, drawTableDirtyMax: 0,
|
|
1617
1773
|
recordCount: 0, slotToRecord: [], recordToSlot: [],
|
|
1618
1774
|
totalEmitEstimate: 0, scanDirty: false,
|
|
@@ -1641,6 +1797,11 @@ export function buildHeapScene(
|
|
|
1641
1797
|
rebuildScanBg();
|
|
1642
1798
|
slot.drawTableDirtyMin = 0;
|
|
1643
1799
|
slot.drawTableDirtyMax = slot.recordCount * RECORD_BYTES;
|
|
1800
|
+
// Phase 5c.3: partition's bind group references every slot's
|
|
1801
|
+
// drawTable. When ANY slot's GPUBuffer reallocates we rebind all.
|
|
1802
|
+
if (bucket.gpuRouted && bucket.partitionScene !== undefined) {
|
|
1803
|
+
bucket.partitionScene.rebindSlotDrawBufs(bucket.slots.map(s => s.drawTableBuf!.buffer));
|
|
1804
|
+
}
|
|
1644
1805
|
});
|
|
1645
1806
|
blockSumsBuf.onResize(rebuildScanBg);
|
|
1646
1807
|
blockOffsetsBuf.onResize(rebuildScanBg);
|
|
@@ -1657,10 +1818,721 @@ export function buildHeapScene(
|
|
|
1657
1818
|
/** Side map from Bucket → family. Populated by findOrCreateBucket. */
|
|
1658
1819
|
const familyForBucket = new WeakMap<Bucket, ReturnType<typeof familyFor>>();
|
|
1659
1820
|
|
|
1821
|
+
/** Canonical axis order — every part of the GPU routing path uses
|
|
1822
|
+
* this ordering so the kernel's params layout, the per-slot mixed-
|
|
1823
|
+
* radix mode key, and the declared-aval lookup stay in lockstep. */
|
|
1824
|
+
const AXIS_ORDER: ReadonlyArray<ModeAxis> = [
|
|
1825
|
+
"cull", "frontFace", "topology",
|
|
1826
|
+
"depthCompare", "depthWrite", "alphaToCoverage",
|
|
1827
|
+
];
|
|
1828
|
+
|
|
1829
|
+
function collectRules(modeRules: NonNullable<HeapDrawSpec["modeRules"]>): DerivedModeRule<ModeAxis>[] {
|
|
1830
|
+
const out: DerivedModeRule<ModeAxis>[] = [];
|
|
1831
|
+
for (const axis of AXIS_ORDER) {
|
|
1832
|
+
const r = (modeRules as Record<string, DerivedModeRule<ModeAxis> | undefined>)[axis];
|
|
1833
|
+
if (r !== undefined) out.push(r);
|
|
1834
|
+
}
|
|
1835
|
+
return out;
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
/** Apply a per-axis enum value to a `PipelineStateDescriptor` and
|
|
1839
|
+
* return the patched copy. Used to materialise one descriptor per
|
|
1840
|
+
* slot (= one per cartesian-product point across the rule domains). */
|
|
1841
|
+
function patchDescriptor(
|
|
1842
|
+
base: PipelineStateDescriptor,
|
|
1843
|
+
axis: ModeAxis,
|
|
1844
|
+
value: unknown,
|
|
1845
|
+
): PipelineStateDescriptor {
|
|
1846
|
+
switch (axis) {
|
|
1847
|
+
case "cull": return { ...base, cullMode: value as CullMode };
|
|
1848
|
+
case "frontFace": return { ...base, frontFace: value as FrontFace };
|
|
1849
|
+
case "topology": {
|
|
1850
|
+
const topology = value as Topology;
|
|
1851
|
+
return { ...base, topology, stripIndexFormat: stripFormatForTopology(topology) };
|
|
1852
|
+
}
|
|
1853
|
+
case "depthCompare":
|
|
1854
|
+
return base.depth !== undefined
|
|
1855
|
+
? { ...base, depth: { ...base.depth, compare: value as GPUCompareFunction } }
|
|
1856
|
+
: base;
|
|
1857
|
+
case "depthWrite":
|
|
1858
|
+
return base.depth !== undefined
|
|
1859
|
+
? { ...base, depth: { ...base.depth, write: value as boolean } }
|
|
1860
|
+
: base;
|
|
1861
|
+
case "alphaToCoverage": return { ...base, alphaToCoverage: value as boolean };
|
|
1862
|
+
case "blend": {
|
|
1863
|
+
// Blend is per-attachment; apply the resolved AttachmentBlend
|
|
1864
|
+
// to attachment 0 (the main color target). Multi-attachment
|
|
1865
|
+
// structured rules can extend this when needed.
|
|
1866
|
+
const blend = value as import("./pipelineCache/index.js").AttachmentBlend;
|
|
1867
|
+
const atts = base.attachments.slice();
|
|
1868
|
+
if (atts.length === 0) atts.push(blend);
|
|
1869
|
+
else atts[0] = blend;
|
|
1870
|
+
return { ...base, attachments: atts };
|
|
1871
|
+
}
|
|
1872
|
+
default: return base;
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
function stripFormatForTopology(t: Topology): GPUIndexFormat | undefined {
|
|
1877
|
+
return t === "line-strip" || t === "triangle-strip" ? "uint32" : undefined;
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
/** Per-axis enum order. The rule body produces u32 indices into
|
|
1881
|
+
* this array, and the heap renderer uses the same order to map
|
|
1882
|
+
* resolved-set integers back to pipeline descriptor values. The
|
|
1883
|
+
* ordering MUST match the bitfield encoder in pipelineCache. */
|
|
1884
|
+
const AXIS_ENUM_TABLE: { readonly [A in ModeAxis]: ReadonlyArray<unknown> } = {
|
|
1885
|
+
cull: ["none", "front", "back"],
|
|
1886
|
+
frontFace: ["ccw", "cw"],
|
|
1887
|
+
topology: ["point-list", "line-list", "line-strip", "triangle-list", "triangle-strip"],
|
|
1888
|
+
depthCompare: ["never", "less", "equal", "less-equal", "greater", "not-equal", "greater-equal", "always"],
|
|
1889
|
+
depthWrite: [false, true],
|
|
1890
|
+
alphaToCoverage: [false, true],
|
|
1891
|
+
// Blend has no canonical enum table — the rule's `resolve` /
|
|
1892
|
+
// `values` callback MUST be supplied so the heap renderer knows
|
|
1893
|
+
// how to map u32 outputs to AttachmentBlend objects.
|
|
1894
|
+
blend: [],
|
|
1895
|
+
};
|
|
1896
|
+
|
|
1897
|
+
/**
|
|
1898
|
+
* Map a resolved slot value (from `evaluateStructuralSet`) to the
|
|
1899
|
+
* axis-shape value used for descriptor patching.
|
|
1900
|
+
*
|
|
1901
|
+
* - If the resolved value is a JS object: it's already the full
|
|
1902
|
+
* axis value (e.g. an `AttachmentBlend` from a blend rule's
|
|
1903
|
+
* object-literal return) — use it directly.
|
|
1904
|
+
* - If it's a number: for enum axes, look up the canonical
|
|
1905
|
+
* table; for structured axes without a table, pass through.
|
|
1906
|
+
*/
|
|
1907
|
+
function resolveAxisValue<A extends ModeAxis>(
|
|
1908
|
+
rule: DerivedModeRule<A>,
|
|
1909
|
+
value: unknown,
|
|
1910
|
+
): unknown {
|
|
1911
|
+
if (typeof value === "object" && value !== null) return value;
|
|
1912
|
+
if (typeof value === "number") {
|
|
1913
|
+
const table = AXIS_ENUM_TABLE[rule.axis];
|
|
1914
|
+
if (table.length > 0) {
|
|
1915
|
+
const v = table[value];
|
|
1916
|
+
if (v !== undefined) return v;
|
|
1917
|
+
}
|
|
1918
|
+
return value;
|
|
1919
|
+
}
|
|
1920
|
+
return value;
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
/**
|
|
1924
|
+
* Resolve an axis's `declared` value to a u32 index.
|
|
1925
|
+
*
|
|
1926
|
+
* - omitted: `0` (the rule body either doesn't reference `declared`
|
|
1927
|
+
* or wants the canonical zero of its axis as the default).
|
|
1928
|
+
* - number: passed through directly (e.g. `declared: 2` for cull "back").
|
|
1929
|
+
* - aval / string / boolean: looked up in the canonical enum table.
|
|
1930
|
+
*/
|
|
1931
|
+
function resolveDeclaredU32<A extends ModeAxis>(rule: DerivedModeRule<A>, tok: AdaptiveToken): number {
|
|
1932
|
+
const d = rule.declared;
|
|
1933
|
+
if (d === undefined) return 0;
|
|
1934
|
+
const v = (typeof d === "object" && d !== null && "getValue" in (d as object))
|
|
1935
|
+
? (d as aval<unknown>).getValue(tok)
|
|
1936
|
+
: d;
|
|
1937
|
+
if (typeof v === "number") return v >>> 0;
|
|
1938
|
+
const table = AXIS_ENUM_TABLE[rule.axis];
|
|
1939
|
+
if (table.length === 0) {
|
|
1940
|
+
throw new Error(
|
|
1941
|
+
`heapScene/gpuRouting: axis '${rule.axis}' has no canonical enum table; ` +
|
|
1942
|
+
`pass \`declared\` as a u32 index or omit it`,
|
|
1943
|
+
);
|
|
1944
|
+
}
|
|
1945
|
+
const i = table.indexOf(v);
|
|
1946
|
+
if (i < 0) {
|
|
1947
|
+
throw new Error(
|
|
1948
|
+
`heapScene/gpuRouting: declared value '${String(v)}' for axis '${rule.axis}' ` +
|
|
1949
|
+
`not in canonical enum table ${JSON.stringify(table)}`,
|
|
1950
|
+
);
|
|
1951
|
+
}
|
|
1952
|
+
return i;
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
/** Extract the rule body Stmt from a RuleExpr's Module. The plugin
|
|
1956
|
+
* emits one `Entry` ValueDef whose body is the user's closure body
|
|
1957
|
+
* (with the synthetic compute-stage scaffolding stripped). */
|
|
1958
|
+
function ruleBodyOf(rule: DerivedModeRule<ModeAxis>): Stmt {
|
|
1959
|
+
const tmpl = rule.expr.template as { values: ReadonlyArray<{ kind: string; entry?: { body: Stmt } }> };
|
|
1960
|
+
for (const v of tmpl.values) {
|
|
1961
|
+
if (v.kind === "Entry" && v.entry !== undefined) return v.entry.body;
|
|
1962
|
+
}
|
|
1963
|
+
throw new Error(
|
|
1964
|
+
`heapScene/gpuRouting: rule for axis '${rule.axis}' has no Entry in its template (id=${rule.expr.id})`,
|
|
1965
|
+
);
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
/**
|
|
1969
|
+
* Specialise a rule for the current declared u32 value:
|
|
1970
|
+
* 1. Bake `declared` as a Const in the rule body (so the kernel
|
|
1971
|
+
* has no per-dispatch uniform — declared marks trigger a
|
|
1972
|
+
* re-specialise + cached-kernel swap).
|
|
1973
|
+
* 2. Run `analyseOutputSet` to collect distinct return Exprs.
|
|
1974
|
+
* 3. `evaluateStructuralSet` folds each Expr to a JS value (a
|
|
1975
|
+
* number for enum axes; a full object for structured axes
|
|
1976
|
+
* like blend). Distinct values define the slot count.
|
|
1977
|
+
* 4. Build a stableStringify-keyed `exprToSlot` map.
|
|
1978
|
+
* 5. Rewrite the body so every ReturnValue emits a Const u32
|
|
1979
|
+
* slot index.
|
|
1980
|
+
* 6. emitPartitionKernel produces the WGSL.
|
|
1981
|
+
*/
|
|
1982
|
+
/** WeakMap-based stable identity for aval objects. Two `cval`s
|
|
1983
|
+
* have different identity; the same cval observed twice has the
|
|
1984
|
+
* same identity. Used in `ruleContentHash` so rules differing
|
|
1985
|
+
* only in their `declared` aval are recognised as distinct. */
|
|
1986
|
+
const avalIdentityMap = new WeakMap<object, number>();
|
|
1987
|
+
let nextAvalIdentity = 0;
|
|
1988
|
+
function avalIdentity(av: object): number {
|
|
1989
|
+
let id = avalIdentityMap.get(av);
|
|
1990
|
+
if (id === undefined) {
|
|
1991
|
+
id = ++nextAvalIdentity;
|
|
1992
|
+
avalIdentityMap.set(av, id);
|
|
1993
|
+
}
|
|
1994
|
+
return id;
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
/** Content hash for rule identity within a bucket. Two rules
|
|
1998
|
+
* hashing the same can share a `ruleId` and one body; rules
|
|
1999
|
+
* hashing differently get distinct ids. */
|
|
2000
|
+
function ruleContentHash(rule: DerivedModeRule<ModeAxis>): string {
|
|
2001
|
+
const dec = rule.declared;
|
|
2002
|
+
let dKey: string;
|
|
2003
|
+
if (dec === undefined) {
|
|
2004
|
+
dKey = "u";
|
|
2005
|
+
} else if (typeof dec === "object" && dec !== null && "getValue" in (dec as object)) {
|
|
2006
|
+
dKey = `a${avalIdentity(dec as object)}`;
|
|
2007
|
+
} else {
|
|
2008
|
+
dKey = `v${String(dec)}`;
|
|
2009
|
+
}
|
|
2010
|
+
return `${rule.axis}|${rule.expr.id}|${dKey}`;
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
/** Read baseDescriptor's value for a given axis (the value used by
|
|
2014
|
+
* combos that have no rule on that axis). */
|
|
2015
|
+
function axisValueOfDesc(desc: PipelineStateDescriptor, axis: ModeAxis): unknown {
|
|
2016
|
+
switch (axis) {
|
|
2017
|
+
case "cull": return desc.cullMode;
|
|
2018
|
+
case "frontFace": return desc.frontFace;
|
|
2019
|
+
case "topology": return desc.topology;
|
|
2020
|
+
case "depthCompare": return desc.depth?.compare;
|
|
2021
|
+
case "depthWrite": return desc.depth?.write;
|
|
2022
|
+
case "alphaToCoverage": return desc.alphaToCoverage;
|
|
2023
|
+
case "blend": return desc.attachments[0];
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
/**
|
|
2028
|
+
* Specialise the WHOLE bucket — N-D cartesian over per-axis
|
|
2029
|
+
* unions:
|
|
2030
|
+
* 1. Per (axis, rule): bake declared as Const, analyse the body,
|
|
2031
|
+
* `evaluateStructuralSet` to fold each return Expr to a JS
|
|
2032
|
+
* value. Collect the per-axis union (dedup by stableStringify,
|
|
2033
|
+
* sorted deterministically). If any combo lacks a rule on
|
|
2034
|
+
* this axis, also include baseDesc's value for that axis.
|
|
2035
|
+
* 2. Active axes = axes with ≥1 registered rule, in AXIS_ORDER.
|
|
2036
|
+
* Cardinalities Cₐ = |axisValues[a]|; totalSlots = ∏ Cₐ.
|
|
2037
|
+
* 3. Per axis-rule: rewrite the body so every `ReturnValue`
|
|
2038
|
+
* emits a `Const u32` PER-AXIS index (into axisValues[axis]).
|
|
2039
|
+
* 4. Per combo: assemble a ComboCodegenSpec (per-axis source =
|
|
2040
|
+
* rule or compile-time const index).
|
|
2041
|
+
* 5. Emit the kernel.
|
|
2042
|
+
*
|
|
2043
|
+
* Returns the cartesian slot descriptors + the kernel WGSL.
|
|
2044
|
+
*/
|
|
2045
|
+
function specialiseBucket(
|
|
2046
|
+
bucket: Bucket,
|
|
2047
|
+
): { slotDescs: ReadonlyArray<PipelineStateDescriptor>; kernelWGSL: string } {
|
|
2048
|
+
const Tu32 = { kind: "Int", signed: false, width: 32 } as const;
|
|
2049
|
+
const rulesByAxis = bucket.rulesByAxis!;
|
|
2050
|
+
const axisRuleOrder = bucket.axisRuleOrder!;
|
|
2051
|
+
const combosByKey = bucket.combosByKey!;
|
|
2052
|
+
const comboOrder = bucket.comboOrder!;
|
|
2053
|
+
if (comboOrder.length === 0) {
|
|
2054
|
+
throw new Error("heapScene/gpuRouting: specialiseBucket called with no registered combos");
|
|
2055
|
+
}
|
|
2056
|
+
const base = bucket.baseDescriptor!;
|
|
2057
|
+
// An axis is "active" (gets cartesian-product cardinality) if it
|
|
2058
|
+
// has at least one rule, OR any combo overrides baseDesc with a
|
|
2059
|
+
// distinct fixed value. Both grow the per-axis value union.
|
|
2060
|
+
const baseAxisValues = new Map<ModeAxis, unknown>(AXIS_ORDER.map(a => [a, axisValueOfDesc(base, a)] as const));
|
|
2061
|
+
const activeAxes: ModeAxis[] = AXIS_ORDER.filter(a => {
|
|
2062
|
+
if ((rulesByAxis.get(a)?.size ?? 0) > 0) return true;
|
|
2063
|
+
const baseVal = baseAxisValues.get(a);
|
|
2064
|
+
const baseKey = stableStringify(baseVal);
|
|
2065
|
+
for (const k of comboOrder) {
|
|
2066
|
+
const c = combosByKey.get(k)!;
|
|
2067
|
+
const fixed = c.axisFixedValues.get(a);
|
|
2068
|
+
if (fixed !== undefined && stableStringify(fixed) !== baseKey) return true;
|
|
2069
|
+
}
|
|
2070
|
+
return false;
|
|
2071
|
+
});
|
|
2072
|
+
|
|
2073
|
+
// Step 1: per-axis-rule analyse + collect per-axis union.
|
|
2074
|
+
interface PerRule {
|
|
2075
|
+
entry: AxisRuleEntry;
|
|
2076
|
+
bakedBody: Stmt;
|
|
2077
|
+
resolved: ReadonlyArray<unknown>;
|
|
2078
|
+
declaredU32: number;
|
|
2079
|
+
}
|
|
2080
|
+
const perAxisRules = new Map<ModeAxis, PerRule[]>();
|
|
2081
|
+
for (const axis of activeAxes) {
|
|
2082
|
+
const list: PerRule[] = [];
|
|
2083
|
+
for (const hash of axisRuleOrder.get(axis)!) {
|
|
2084
|
+
const entry = rulesByAxis.get(axis)!.get(hash)!;
|
|
2085
|
+
const declaredU32 = resolveDeclaredU32(entry.rule, AdaptiveToken.top);
|
|
2086
|
+
const declaredConst: Expr = {
|
|
2087
|
+
kind: "Const", type: Tu32 as never,
|
|
2088
|
+
value: { kind: "Int", value: declaredU32 >>> 0, signed: false } as never,
|
|
2089
|
+
} as Expr;
|
|
2090
|
+
const bakedBody = substituteReadInputInStmt(entry.body, "Uniform", "declared", declaredConst);
|
|
2091
|
+
const symbolic = analyseOutputSet(bakedBody);
|
|
2092
|
+
const env = new Map<string, number>([["declared", declaredU32]]);
|
|
2093
|
+
const { resolved, unresolvedCount } = evaluateStructuralSet(symbolic, env, new Map());
|
|
2094
|
+
let outputs = resolved.slice();
|
|
2095
|
+
if (outputs.length === 0 || unresolvedCount > 0) {
|
|
2096
|
+
const table = AXIS_ENUM_TABLE[entry.rule.axis];
|
|
2097
|
+
if (table.length > 0 && unresolvedCount > 0) {
|
|
2098
|
+
outputs = table.map((_, i) => i);
|
|
2099
|
+
} else if (outputs.length === 0) {
|
|
2100
|
+
throw new Error(
|
|
2101
|
+
`heapScene/gpuRouting: rule on axis '${entry.rule.axis}' could not be ` +
|
|
2102
|
+
`statically resolved AND the axis has no canonical enum table. ` +
|
|
2103
|
+
`Simplify the rule body so analyseOutputSet can fold each branch.`,
|
|
2104
|
+
);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
list.push({ entry, bakedBody, resolved: outputs, declaredU32 });
|
|
2108
|
+
entry.lastDeclaredU32 = declaredU32;
|
|
2109
|
+
}
|
|
2110
|
+
perAxisRules.set(axis, list);
|
|
2111
|
+
}
|
|
2112
|
+
|
|
2113
|
+
// Per axis: union of resolved values across rules + each combo's
|
|
2114
|
+
// effective value when it has no rule on this axis. Combos that
|
|
2115
|
+
// declare an explicit `axisFixedValues[axis]` contribute that
|
|
2116
|
+
// value; combos without an entry default to baseDesc.value.
|
|
2117
|
+
interface AxisInfo {
|
|
2118
|
+
readonly axis: ModeAxis;
|
|
2119
|
+
readonly values: ReadonlyArray<unknown>; // resolved axis values, sorted
|
|
2120
|
+
readonly keyToIdx: ReadonlyMap<string, number>; // stableStringify(value) → idx
|
|
2121
|
+
readonly baseValue: unknown; // baseDesc's value for this axis
|
|
2122
|
+
readonly baseIdx: number; // index of baseValue in values
|
|
2123
|
+
}
|
|
2124
|
+
const axisInfos: AxisInfo[] = [];
|
|
2125
|
+
for (const axis of activeAxes) {
|
|
2126
|
+
const list = perAxisRules.get(axis) ?? [];
|
|
2127
|
+
const dummyRule = { axis } as DerivedModeRule<ModeAxis>;
|
|
2128
|
+
const byKey = new Map<string, unknown>();
|
|
2129
|
+
for (const p of list) {
|
|
2130
|
+
for (const v of p.resolved) {
|
|
2131
|
+
const av = resolveAxisValue(dummyRule, v);
|
|
2132
|
+
byKey.set(stableStringify(av), av);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
const baseValue = baseAxisValues.get(axis);
|
|
2136
|
+
// Include every combo's effective value for this axis (so
|
|
2137
|
+
// const-source idx lookups always succeed).
|
|
2138
|
+
for (const k of comboOrder) {
|
|
2139
|
+
const c = combosByKey.get(k)!;
|
|
2140
|
+
if (c.axisRules.has(axis)) continue;
|
|
2141
|
+
const effective = c.axisFixedValues.has(axis) ? c.axisFixedValues.get(axis) : baseValue;
|
|
2142
|
+
byKey.set(stableStringify(effective), effective);
|
|
2143
|
+
}
|
|
2144
|
+
const sortedKeys = [...byKey.keys()].sort();
|
|
2145
|
+
const values = sortedKeys.map(k => byKey.get(k)!);
|
|
2146
|
+
const keyToIdx = new Map(sortedKeys.map((k, i) => [k, i] as const));
|
|
2147
|
+
const baseIdx = keyToIdx.get(stableStringify(baseValue)) ?? 0;
|
|
2148
|
+
axisInfos.push({ axis, values, keyToIdx, baseValue, baseIdx });
|
|
2149
|
+
}
|
|
2150
|
+
|
|
2151
|
+
// Step 2: cartesian shape.
|
|
2152
|
+
const cards = axisInfos.map(a => a.values.length);
|
|
2153
|
+
const totalSlots = cards.reduce((m, n) => m * n, 1);
|
|
2154
|
+
|
|
2155
|
+
// Step 3: rewrite each axis-rule body to emit per-axis indices.
|
|
2156
|
+
const ruleSpecs: import("./derivedModes/kernelCodegen.js").RuleCodegenSpec[] = [];
|
|
2157
|
+
for (const info of axisInfos) {
|
|
2158
|
+
const list = perAxisRules.get(info.axis)!;
|
|
2159
|
+
const dummyRule = { axis: info.axis } as DerivedModeRule<ModeAxis>;
|
|
2160
|
+
for (const p of list) {
|
|
2161
|
+
const env = new Map<string, number>([["declared", p.declaredU32]]);
|
|
2162
|
+
const exprToAxisIdx = (e: Expr): number | undefined => {
|
|
2163
|
+
const v = evaluateStructural(e, env, new Map());
|
|
2164
|
+
if (v === undefined) return undefined;
|
|
2165
|
+
const av = resolveAxisValue(dummyRule, v);
|
|
2166
|
+
return info.keyToIdx.get(stableStringify(av));
|
|
2167
|
+
};
|
|
2168
|
+
const rewrittenBody = rewriteOutputsToSlotIndices(p.bakedBody, exprToAxisIdx);
|
|
2169
|
+
ruleSpecs.push({
|
|
2170
|
+
axis: info.axis,
|
|
2171
|
+
axisRuleId: p.entry.axisRuleId,
|
|
2172
|
+
body: rewrittenBody,
|
|
2173
|
+
intrinsics: new Map(),
|
|
2174
|
+
helpersWGSL: "",
|
|
2175
|
+
inputUniforms: ["ModelTrafo"],
|
|
2176
|
+
});
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
// Step 4: per-combo axis sources. Rule axes → rule source.
|
|
2181
|
+
// Non-rule axes → const idx of (axisFixedValues.get(axis)
|
|
2182
|
+
// ?? baseValue) within the axis's sorted union.
|
|
2183
|
+
const comboSpecs: import("./derivedModes/kernelCodegen.js").ComboCodegenSpec[] = [];
|
|
2184
|
+
for (const comboKey of comboOrder) {
|
|
2185
|
+
const combo = combosByKey.get(comboKey)!;
|
|
2186
|
+
const axes = axisInfos.map(info => {
|
|
2187
|
+
const axisRuleId = combo.axisRules.get(info.axis);
|
|
2188
|
+
if (axisRuleId !== undefined) {
|
|
2189
|
+
return {
|
|
2190
|
+
axis: info.axis,
|
|
2191
|
+
cardinality: info.values.length,
|
|
2192
|
+
source: { kind: "rule" as const, axisRuleId },
|
|
2193
|
+
};
|
|
2194
|
+
}
|
|
2195
|
+
const effective = combo.axisFixedValues.has(info.axis)
|
|
2196
|
+
? combo.axisFixedValues.get(info.axis)
|
|
2197
|
+
: info.baseValue;
|
|
2198
|
+
const idx = info.keyToIdx.get(stableStringify(effective)) ?? info.baseIdx;
|
|
2199
|
+
return {
|
|
2200
|
+
axis: info.axis,
|
|
2201
|
+
cardinality: info.values.length,
|
|
2202
|
+
source: { kind: "const" as const, idx },
|
|
2203
|
+
};
|
|
2204
|
+
});
|
|
2205
|
+
comboSpecs.push({ comboId: combo.comboId, axes });
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
// Step 5: enumerate cartesian → slot descriptors (mixed-radix
|
|
2209
|
+
// decode in the same axis order the kernel uses).
|
|
2210
|
+
const slotDescs: PipelineStateDescriptor[] = [];
|
|
2211
|
+
for (let slotIdx = 0; slotIdx < totalSlots; slotIdx++) {
|
|
2212
|
+
let desc = base;
|
|
2213
|
+
let rem = slotIdx;
|
|
2214
|
+
// Match kernel: strides[i] = ∏_{j>i} cards[j]. Decode:
|
|
2215
|
+
// idx[i] = (slotIdx / strides[i]) % cards[i].
|
|
2216
|
+
const strides: number[] = new Array(axisInfos.length).fill(1);
|
|
2217
|
+
for (let i = axisInfos.length - 2; i >= 0; i--) strides[i] = strides[i + 1]! * cards[i + 1]!;
|
|
2218
|
+
for (let i = 0; i < axisInfos.length; i++) {
|
|
2219
|
+
const info = axisInfos[i]!;
|
|
2220
|
+
const ai = Math.floor(rem / strides[i]!) % cards[i]!;
|
|
2221
|
+
rem -= ai * strides[i]!;
|
|
2222
|
+
desc = patchDescriptor(desc, info.axis, info.values[ai]);
|
|
2223
|
+
}
|
|
2224
|
+
slotDescs.push(desc);
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
const kernelWGSL = emitPartitionKernel({
|
|
2228
|
+
rules: ruleSpecs,
|
|
2229
|
+
combos: comboSpecs,
|
|
2230
|
+
totalSlots,
|
|
2231
|
+
uniformOrder: bucket.uniformOrder ?? [],
|
|
2232
|
+
});
|
|
2233
|
+
return { slotDescs, kernelWGSL };
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
/** Build pipeline + slot for each cartesian slot descriptor. */
|
|
2237
|
+
function ensureSlotsForResolved(
|
|
2238
|
+
bucket: Bucket,
|
|
2239
|
+
slotDescs: ReadonlyArray<PipelineStateDescriptor>,
|
|
2240
|
+
): void {
|
|
2241
|
+
const fam = familyForBucket.get(bucket)!;
|
|
2242
|
+
for (let slotIdx = 0; slotIdx < slotDescs.length; slotIdx++) {
|
|
2243
|
+
const desc = slotDescs[slotIdx]!;
|
|
2244
|
+
const key = encodeModeKey(desc);
|
|
2245
|
+
if (slotIdx < bucket.slots.length) {
|
|
2246
|
+
const slot = bucket.slots[slotIdx]!;
|
|
2247
|
+
if (slot.modeKey !== key) {
|
|
2248
|
+
slot.pipeline = getOrCreatePipeline(fam, bucket.layout, bucket.isAtlasBucket, desc);
|
|
2249
|
+
slot.modeKey = key;
|
|
2250
|
+
}
|
|
2251
|
+
} else {
|
|
2252
|
+
createSlot(bucket, desc);
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
/** Register a combo (set of axis rules, ≤ one per axis) on a GPU-
|
|
2258
|
+
* routed bucket (or promote the bucket on first call). Returns
|
|
2259
|
+
* the bucket-unique `comboId` to bake into master-pool records. */
|
|
2260
|
+
function registerCombo(
|
|
2261
|
+
bucket: Bucket,
|
|
2262
|
+
axisRules: ReadonlyArray<DerivedModeRule<ModeAxis>>,
|
|
2263
|
+
baseDesc: PipelineStateDescriptor,
|
|
2264
|
+
): number {
|
|
2265
|
+
// CPU→GPU promotion: capture pre-existing CPU-routed ROs so they
|
|
2266
|
+
// can be migrated into the master pool after the partition is up.
|
|
2267
|
+
// Each RO is tagged with its CPU descriptor (read off the slot's
|
|
2268
|
+
// `cpuDescriptor` snapshot); we synthesise a const-source combo
|
|
2269
|
+
// per distinct descriptor below, with `axisFixedValues` set on
|
|
2270
|
+
// any axis the descriptor differs from baseDescriptor on.
|
|
2271
|
+
interface CpuMigration {
|
|
2272
|
+
localSlot: number;
|
|
2273
|
+
drawId: number;
|
|
2274
|
+
firstIndex: number;
|
|
2275
|
+
indexCount: number;
|
|
2276
|
+
instanceCount: number;
|
|
2277
|
+
perDrawRefs: Map<string, number>;
|
|
2278
|
+
descriptor: PipelineStateDescriptor;
|
|
2279
|
+
/** Filled in below after synthetic combos are registered. */
|
|
2280
|
+
targetComboId: number;
|
|
2281
|
+
}
|
|
2282
|
+
let cpuMigration: CpuMigration[] | undefined;
|
|
2283
|
+
if (!bucket.gpuRouted && bucket.drawSlots.size > 0) {
|
|
2284
|
+
cpuMigration = [];
|
|
2285
|
+
for (const ls of bucket.drawSlots) {
|
|
2286
|
+
const e = bucket.localEntries[ls]!;
|
|
2287
|
+
const drawId = bucket.localToDrawId[ls]!;
|
|
2288
|
+
const slotIdx = drawIdToSlotIdx[drawId];
|
|
2289
|
+
const slot = slotIdx !== undefined ? bucket.slots[slotIdx] : undefined;
|
|
2290
|
+
const descriptor = slot?.cpuDescriptor;
|
|
2291
|
+
if (descriptor === undefined) {
|
|
2292
|
+
throw new Error(
|
|
2293
|
+
`heapScene/gpuRouting: pre-existing CPU RO (drawId=${drawId}, localSlot=${ls}) ` +
|
|
2294
|
+
`has no captured descriptor — bucket promotion cannot synthesise a migration combo.`,
|
|
2295
|
+
);
|
|
2296
|
+
}
|
|
2297
|
+
cpuMigration.push({
|
|
2298
|
+
localSlot: ls,
|
|
2299
|
+
drawId,
|
|
2300
|
+
firstIndex: e.firstIndex,
|
|
2301
|
+
indexCount: e.indexCount,
|
|
2302
|
+
instanceCount: e.instanceCount,
|
|
2303
|
+
perDrawRefs: bucket.localPerDrawRefs[ls]!,
|
|
2304
|
+
descriptor,
|
|
2305
|
+
targetComboId: 0,
|
|
2306
|
+
});
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
if (!bucket.gpuRouted) {
|
|
2310
|
+
bucket.baseDescriptor = baseDesc;
|
|
2311
|
+
bucket.rulesByAxis = new Map();
|
|
2312
|
+
bucket.axisRuleOrder = new Map();
|
|
2313
|
+
bucket.uniformOrder = [];
|
|
2314
|
+
bucket.uniformIdx = new Map();
|
|
2315
|
+
bucket.combosByKey = new Map();
|
|
2316
|
+
bucket.comboOrder = [];
|
|
2317
|
+
bucket.drawIdToComboId = new Map();
|
|
2318
|
+
bucket.drawIdToRecord = new Map();
|
|
2319
|
+
bucket.recordToDrawId = [];
|
|
2320
|
+
// Synthesise one combo per distinct pre-existing CPU
|
|
2321
|
+
// descriptor. Each carries `axisFixedValues` on axes where the
|
|
2322
|
+
// descriptor differs from baseDescriptor; specialiseBucket
|
|
2323
|
+
// expands the per-axis union to include each variant value
|
|
2324
|
+
// and emits a const-source combo fn that routes to the right
|
|
2325
|
+
// cartesian slot.
|
|
2326
|
+
if (cpuMigration !== undefined) {
|
|
2327
|
+
const baseVals = new Map<ModeAxis, unknown>(AXIS_ORDER.map(a => [a, axisValueOfDesc(baseDesc, a)] as const));
|
|
2328
|
+
const synthByKey = new Map<string, number>();
|
|
2329
|
+
for (const m of cpuMigration) {
|
|
2330
|
+
const axisFixedValues = new Map<ModeAxis, unknown>();
|
|
2331
|
+
const keyParts: string[] = [];
|
|
2332
|
+
for (const a of AXIS_ORDER) {
|
|
2333
|
+
const v = axisValueOfDesc(m.descriptor, a);
|
|
2334
|
+
const baseKey = stableStringify(baseVals.get(a));
|
|
2335
|
+
const vKey = stableStringify(v);
|
|
2336
|
+
if (vKey !== baseKey) {
|
|
2337
|
+
axisFixedValues.set(a, v);
|
|
2338
|
+
keyParts.push(`${a}:_:fixed:${vKey}`);
|
|
2339
|
+
} else {
|
|
2340
|
+
keyParts.push(`${a}:_`);
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
const comboKey = keyParts.join("|");
|
|
2344
|
+
let comboId = synthByKey.get(comboKey);
|
|
2345
|
+
if (comboId === undefined) {
|
|
2346
|
+
comboId = bucket.comboOrder.length;
|
|
2347
|
+
bucket.combosByKey.set(comboKey, {
|
|
2348
|
+
comboId, comboKey,
|
|
2349
|
+
axisRules: new Map(),
|
|
2350
|
+
axisFixedValues,
|
|
2351
|
+
});
|
|
2352
|
+
bucket.comboOrder.push(comboKey);
|
|
2353
|
+
synthByKey.set(comboKey, comboId);
|
|
2354
|
+
}
|
|
2355
|
+
m.targetComboId = comboId;
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
// Per-axis: dedupe rule by content hash; assign axisRuleId.
|
|
2360
|
+
const perAxisRuleId = new Map<ModeAxis, number>();
|
|
2361
|
+
let respec = false;
|
|
2362
|
+
for (const rule of axisRules) {
|
|
2363
|
+
if (perAxisRuleId.has(rule.axis)) {
|
|
2364
|
+
throw new Error(
|
|
2365
|
+
`heapScene/gpuRouting: combo has two rules on the same axis '${rule.axis}'`,
|
|
2366
|
+
);
|
|
2367
|
+
}
|
|
2368
|
+
const hash = ruleContentHash(rule);
|
|
2369
|
+
let axisMap = bucket.rulesByAxis!.get(rule.axis);
|
|
2370
|
+
if (axisMap === undefined) {
|
|
2371
|
+
axisMap = new Map();
|
|
2372
|
+
bucket.rulesByAxis!.set(rule.axis, axisMap);
|
|
2373
|
+
bucket.axisRuleOrder!.set(rule.axis, []);
|
|
2374
|
+
}
|
|
2375
|
+
let entry = axisMap.get(hash);
|
|
2376
|
+
if (entry === undefined) {
|
|
2377
|
+
const order = bucket.axisRuleOrder!.get(rule.axis)!;
|
|
2378
|
+
const body = ruleBodyOf(rule);
|
|
2379
|
+
entry = {
|
|
2380
|
+
axis: rule.axis,
|
|
2381
|
+
axisRuleId: order.length,
|
|
2382
|
+
contentHash: hash,
|
|
2383
|
+
rule,
|
|
2384
|
+
body,
|
|
2385
|
+
lastDeclaredU32: undefined,
|
|
2386
|
+
};
|
|
2387
|
+
axisMap.set(hash, entry);
|
|
2388
|
+
order.push(hash);
|
|
2389
|
+
// Discover the arena uniforms this rule reads — append unseen
|
|
2390
|
+
// ones to the bucket-wide uniformOrder so master records
|
|
2391
|
+
// carry refs for them. `declared` is a CPU-baked Const, not
|
|
2392
|
+
// an arena uniform; skip it.
|
|
2393
|
+
for (const uName of collectUniformReadsInStmt(body)) {
|
|
2394
|
+
if (uName === "declared") continue;
|
|
2395
|
+
if (!bucket.uniformIdx!.has(uName)) {
|
|
2396
|
+
const idx = bucket.uniformOrder!.length;
|
|
2397
|
+
bucket.uniformOrder!.push(uName);
|
|
2398
|
+
bucket.uniformIdx!.set(uName, idx);
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
respec = true;
|
|
2402
|
+
}
|
|
2403
|
+
perAxisRuleId.set(rule.axis, entry.axisRuleId);
|
|
2404
|
+
}
|
|
2405
|
+
// Combo key = canonical-axis-order list of (axis, contentHash).
|
|
2406
|
+
const comboKeyParts: string[] = [];
|
|
2407
|
+
for (const axis of AXIS_ORDER) {
|
|
2408
|
+
const axisRuleId = perAxisRuleId.get(axis);
|
|
2409
|
+
if (axisRuleId === undefined) {
|
|
2410
|
+
comboKeyParts.push(`${axis}:_`);
|
|
2411
|
+
} else {
|
|
2412
|
+
comboKeyParts.push(`${axis}:${bucket.axisRuleOrder!.get(axis)![axisRuleId]!}`);
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
const comboKey = comboKeyParts.join("|");
|
|
2416
|
+
let combo = bucket.combosByKey!.get(comboKey);
|
|
2417
|
+
if (combo === undefined) {
|
|
2418
|
+
const comboId = bucket.comboOrder!.length;
|
|
2419
|
+
const axisRules = new Map<ModeAxis, number>();
|
|
2420
|
+
for (const [axis, id] of perAxisRuleId) axisRules.set(axis, id);
|
|
2421
|
+
combo = { comboId, comboKey, axisRules, axisFixedValues: new Map() };
|
|
2422
|
+
bucket.combosByKey!.set(comboKey, combo);
|
|
2423
|
+
bucket.comboOrder!.push(comboKey);
|
|
2424
|
+
respec = true;
|
|
2425
|
+
}
|
|
2426
|
+
// First time gpuRouted? Build the partition scene now.
|
|
2427
|
+
if (!bucket.gpuRouted) {
|
|
2428
|
+
const { slotDescs, kernelWGSL } = specialiseBucket(bucket);
|
|
2429
|
+
ensureSlotsForResolved(bucket, slotDescs);
|
|
2430
|
+
const partition = new GpuPartitionScene(device, `${bucket.label}/partition`, {
|
|
2431
|
+
totalSlots: slotDescs.length,
|
|
2432
|
+
slotDrawBufs: bucket.slots.map(s => s.drawTableBuf!.buffer),
|
|
2433
|
+
kernelWGSL,
|
|
2434
|
+
initialRecords: Math.max(16, (cpuMigration?.length ?? 0) + 16),
|
|
2435
|
+
numUniforms: bucket.uniformOrder!.length,
|
|
2436
|
+
});
|
|
2437
|
+
bucket.gpuRouted = true;
|
|
2438
|
+
bucket.partitionScene = partition;
|
|
2439
|
+
bucket.partitionDirty = true;
|
|
2440
|
+
// Migrate any captured CPU records into the master pool, each
|
|
2441
|
+
// using the synthetic combo registered for its CPU descriptor.
|
|
2442
|
+
if (cpuMigration !== undefined) {
|
|
2443
|
+
let totalEmit = 0;
|
|
2444
|
+
for (const m of cpuMigration) {
|
|
2445
|
+
const uRefs = bucket.uniformOrder!.map(n => m.perDrawRefs.get(n) ?? 0);
|
|
2446
|
+
const recIdx = partition.appendRecord(
|
|
2447
|
+
m.localSlot, m.firstIndex, m.indexCount, m.instanceCount, m.targetComboId, uRefs,
|
|
2448
|
+
);
|
|
2449
|
+
bucket.drawIdToRecord!.set(m.drawId, recIdx);
|
|
2450
|
+
bucket.recordToDrawId![recIdx] = m.drawId;
|
|
2451
|
+
bucket.drawIdToComboId!.set(m.drawId, m.targetComboId);
|
|
2452
|
+
totalEmit += m.indexCount * m.instanceCount;
|
|
2453
|
+
}
|
|
2454
|
+
const numRecords = partition.numRecords;
|
|
2455
|
+
const recBytes = numRecords * RECORD_BYTES;
|
|
2456
|
+
const needBlocks = Math.max(1, Math.ceil(numRecords / SCAN_TILE_SIZE));
|
|
2457
|
+
for (const s of bucket.slots) {
|
|
2458
|
+
s.drawTableBuf!.ensureCapacity(recBytes);
|
|
2459
|
+
s.drawTableBuf!.setUsed(Math.max(s.drawTableBuf!.usedBytes, recBytes));
|
|
2460
|
+
s.blockSumsBuf!.ensureCapacity(needBlocks * 4);
|
|
2461
|
+
s.blockOffsetsBuf!.ensureCapacity(needBlocks * 4);
|
|
2462
|
+
s.recordCount = numRecords;
|
|
2463
|
+
s.totalEmitEstimate += totalEmit;
|
|
2464
|
+
const newNumTiles = Math.max(1, Math.ceil(s.totalEmitEstimate / TILE_K));
|
|
2465
|
+
s.firstDrawInTileBuf!.ensureCapacity((newNumTiles + 1) * 4);
|
|
2466
|
+
s.scanDirty = true;
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2469
|
+
} else if (respec) {
|
|
2470
|
+
const { slotDescs, kernelWGSL } = specialiseBucket(bucket);
|
|
2471
|
+
ensureSlotsForResolved(bucket, slotDescs);
|
|
2472
|
+
const newCount = bucket.slots.length;
|
|
2473
|
+
const draws = bucket.slots.map(s => s.drawTableBuf!.buffer);
|
|
2474
|
+
if (newCount > bucket.partitionScene!.totalSlots) {
|
|
2475
|
+
bucket.partitionScene!.growSlots(newCount, draws);
|
|
2476
|
+
} else {
|
|
2477
|
+
bucket.partitionScene!.rebindSlotDrawBufs(draws);
|
|
2478
|
+
}
|
|
2479
|
+
// Bucket-wide uniformOrder may have grown (a new rule reads a
|
|
2480
|
+
// previously-unseen uniform). Widen master records before
|
|
2481
|
+
// rebuilding the kernel — the WGSL struct width changes too.
|
|
2482
|
+
if (bucket.uniformOrder!.length > bucket.partitionScene!.numUniforms) {
|
|
2483
|
+
bucket.partitionScene!.growUniforms(bucket.uniformOrder!.length);
|
|
2484
|
+
}
|
|
2485
|
+
bucket.partitionScene!.rebuildKernel(kernelWGSL);
|
|
2486
|
+
bucket.partitionDirty = true;
|
|
2487
|
+
}
|
|
2488
|
+
return combo.comboId;
|
|
2489
|
+
}
|
|
2490
|
+
|
|
2491
|
+
/**
|
|
2492
|
+
* Per-frame partition dispatch. Detects per-rule `declared`
|
|
2493
|
+
* changes and re-specialises the whole bucket if any rule's
|
|
2494
|
+
* declared value differs from what was used at the last
|
|
2495
|
+
* dispatch (cheap — analysis runs on cached IR).
|
|
2496
|
+
*/
|
|
2497
|
+
function dispatchPartition(bucket: Bucket, enc: GPUCommandEncoder, tok: AdaptiveToken): void {
|
|
2498
|
+
if (!bucket.gpuRouted || bucket.partitionScene === undefined) return;
|
|
2499
|
+
let needRespec = false;
|
|
2500
|
+
outer: for (const axisMap of bucket.rulesByAxis!.values()) {
|
|
2501
|
+
for (const entry of axisMap.values()) {
|
|
2502
|
+
const declaredU32 = resolveDeclaredU32(entry.rule, tok);
|
|
2503
|
+
if (declaredU32 !== entry.lastDeclaredU32) { needRespec = true; break outer; }
|
|
2504
|
+
}
|
|
2505
|
+
}
|
|
2506
|
+
if (needRespec) {
|
|
2507
|
+
const { slotDescs, kernelWGSL } = specialiseBucket(bucket);
|
|
2508
|
+
ensureSlotsForResolved(bucket, slotDescs);
|
|
2509
|
+
const newCount = bucket.slots.length;
|
|
2510
|
+
const draws = bucket.slots.map(s => s.drawTableBuf!.buffer);
|
|
2511
|
+
if (newCount > bucket.partitionScene.totalSlots) {
|
|
2512
|
+
bucket.partitionScene.growSlots(newCount, draws);
|
|
2513
|
+
} else {
|
|
2514
|
+
bucket.partitionScene.rebindSlotDrawBufs(draws);
|
|
2515
|
+
}
|
|
2516
|
+
bucket.partitionScene.rebuildKernel(kernelWGSL);
|
|
2517
|
+
}
|
|
2518
|
+
bucket.partitionScene.flush();
|
|
2519
|
+
bucket.partitionScene.dispatch(arena.attrs.chunk(bucket.chunkIdx).buffer, enc);
|
|
2520
|
+
for (let i = 0; i < bucket.slots.length; i++) {
|
|
2521
|
+
enc.copyBufferToBuffer(
|
|
2522
|
+
bucket.partitionScene.slotCountBufs[i]!, 0,
|
|
2523
|
+
bucket.slots[i]!.paramsBuf!, 0, 4,
|
|
2524
|
+
);
|
|
2525
|
+
bucket.slots[i]!.scanDirty = true;
|
|
2526
|
+
}
|
|
2527
|
+
bucket.partitionDirty = false;
|
|
2528
|
+
}
|
|
2529
|
+
|
|
2530
|
+
|
|
1660
2531
|
function findOrCreateBucket(
|
|
1661
2532
|
effect: Effect,
|
|
1662
2533
|
_textures: HeapTextureSet | undefined,
|
|
1663
2534
|
pipelineState: PipelineState | undefined,
|
|
2535
|
+
chunkIdx: number,
|
|
1664
2536
|
/**
|
|
1665
2537
|
* Optional precomputed descriptor. addDrawImpl passes this when
|
|
1666
2538
|
* the spec carries `modeRules` so the bucket key reflects the
|
|
@@ -1673,12 +2545,13 @@ export function buildHeapScene(
|
|
|
1673
2545
|
throw new Error("heapScene: findOrCreateBucket called before family build");
|
|
1674
2546
|
}
|
|
1675
2547
|
const fam = familyFor(effect);
|
|
1676
|
-
//
|
|
1677
|
-
//
|
|
1678
|
-
//
|
|
1679
|
-
//
|
|
1680
|
-
//
|
|
1681
|
-
|
|
2548
|
+
// §3: bucket key includes `chunkIdx` — two ROs with the same
|
|
2549
|
+
// effect/family that landed in different arena chunks live in
|
|
2550
|
+
// separate buckets so each bucket can bind exactly one chunk's
|
|
2551
|
+
// GPUBuffers in its draw call. The shaders are unchanged
|
|
2552
|
+
// (refs are byte offsets within whichever chunk's buffer is
|
|
2553
|
+
// currently bound).
|
|
2554
|
+
const bk = `family#${fam.schema.id}/c${chunkIdx}`;
|
|
1682
2555
|
const existing = bucketByKey.get(bk);
|
|
1683
2556
|
if (existing !== undefined) return existing;
|
|
1684
2557
|
|
|
@@ -1691,7 +2564,7 @@ export function buildHeapScene(
|
|
|
1691
2564
|
const drawHeap = new DrawHeap(drawHeapBuf, layout.drawHeaderBytes);
|
|
1692
2565
|
|
|
1693
2566
|
const bucket: Bucket = {
|
|
1694
|
-
label: bk, textures: undefined, layout,
|
|
2567
|
+
label: bk, textures: undefined, layout, chunkIdx,
|
|
1695
2568
|
slots: [], // populated lazily by ensureSlot per modeKey
|
|
1696
2569
|
bindGroup: null as unknown as GPUBindGroup, // legacy field; slot.bindGroup is the real one
|
|
1697
2570
|
drawHeap,
|
|
@@ -1705,6 +2578,19 @@ export function buildHeapScene(
|
|
|
1705
2578
|
localAtlasReleases: [],
|
|
1706
2579
|
localAtlasTextures: [],
|
|
1707
2580
|
localAtlasArrIdx: [],
|
|
2581
|
+
gpuRouted: false,
|
|
2582
|
+
partitionScene: undefined,
|
|
2583
|
+
drawIdToRecord: undefined,
|
|
2584
|
+
recordToDrawId: undefined,
|
|
2585
|
+
rulesByAxis: undefined,
|
|
2586
|
+
axisRuleOrder: undefined,
|
|
2587
|
+
uniformOrder: undefined,
|
|
2588
|
+
uniformIdx: undefined,
|
|
2589
|
+
combosByKey: undefined,
|
|
2590
|
+
comboOrder: undefined,
|
|
2591
|
+
drawIdToComboId: undefined,
|
|
2592
|
+
baseDescriptor: undefined,
|
|
2593
|
+
partitionDirty: false,
|
|
1708
2594
|
};
|
|
1709
2595
|
familyForBucket.set(bucket, fam);
|
|
1710
2596
|
|
|
@@ -1938,16 +2824,60 @@ export function buildHeapScene(
|
|
|
1938
2824
|
token: outerTok,
|
|
1939
2825
|
});
|
|
1940
2826
|
}
|
|
1941
|
-
|
|
2827
|
+
// §3 chunk routing — pick which arena chunk this RO's allocations
|
|
2828
|
+
// land in. Preference order:
|
|
2829
|
+
// 1. If any of the RO's bindings is an aval that's already
|
|
2830
|
+
// allocated in some chunk via the uniform pool, prefer that
|
|
2831
|
+
// chunk to avoid duplication.
|
|
2832
|
+
// 2. Otherwise, use the newest open chunk (highest chunkIdx).
|
|
2833
|
+
// ChunkedArena.alloc spills to the next chunk automatically
|
|
2834
|
+
// when this one is full; the spill ends up driving the
|
|
2835
|
+
// pool's `finalChunk` selection so all of the RO's
|
|
2836
|
+
// allocations land in one chunk consistently.
|
|
2837
|
+
let chunkIdx: number;
|
|
2838
|
+
{
|
|
2839
|
+
let preferred: number | undefined;
|
|
2840
|
+
for (const [, val] of Object.entries(spec.inputs)) {
|
|
2841
|
+
const av = (typeof val === "object" && val !== null && "getValue" in (val as object))
|
|
2842
|
+
? (val as aval<unknown>) : undefined;
|
|
2843
|
+
if (av !== undefined) {
|
|
2844
|
+
const where = pool.firstChunkContaining(av);
|
|
2845
|
+
if (where !== undefined) { preferred = where; break; }
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2848
|
+
chunkIdx = preferred ?? Math.max(0, arena.attrs.chunkCount - 1);
|
|
2849
|
+
}
|
|
2850
|
+
const bucket = findOrCreateBucket(spec.effect, spec.textures, spec.pipelineState, chunkIdx, precomputedDescriptor);
|
|
2851
|
+
// Phase 5c.3 — every RO carrying a derived-mode rule is registered
|
|
2852
|
+
// on the bucket (lazy-promoting it to GPU-routed on the first
|
|
2853
|
+
// call). ROs may carry DIFFERENT rules even when they share the
|
|
2854
|
+
// bucket's `(effect, textureSet)` — each gets its own `ruleId`
|
|
2855
|
+
// baked into the master-pool record and the partition kernel's
|
|
2856
|
+
// dispatch switch picks the right body per record.
|
|
2857
|
+
const collectedRules = spec.modeRules !== undefined ? collectRules(spec.modeRules) : [];
|
|
2858
|
+
const roDescriptor = precomputedDescriptor ?? snapshotDescriptor(spec.pipelineState, sig);
|
|
2859
|
+
let roComboId = 0;
|
|
2860
|
+
if (collectedRules.length > 0) {
|
|
2861
|
+
roComboId = registerCombo(bucket, collectedRules, roDescriptor);
|
|
2862
|
+
} else if (bucket.gpuRouted) {
|
|
2863
|
+
// An unruled RO joining an already-GPU-routed bucket needs a
|
|
2864
|
+
// real combo to land on. Register the trivial all-fixed combo
|
|
2865
|
+
// (every axis falls back to baseDescriptor) so this record's
|
|
2866
|
+
// `comboId` doesn't silently alias to combo 0 (which is
|
|
2867
|
+
// whichever combo was registered first — almost never the
|
|
2868
|
+
// "no rules" intent).
|
|
2869
|
+
roComboId = registerCombo(bucket, [], roDescriptor);
|
|
2870
|
+
}
|
|
1942
2871
|
// Phase 5c.2: route this RO to the bucket's slot covering its
|
|
1943
2872
|
// current descriptor. ensureSlot creates a new slot on miss; the
|
|
1944
|
-
// pipeline lookup hits the scene-level cache.
|
|
1945
|
-
|
|
1946
|
-
|
|
2873
|
+
// pipeline lookup hits the scene-level cache. For GPU-routed
|
|
2874
|
+
// buckets the slot decision is made each frame on the GPU; we
|
|
2875
|
+
// still pin the RO to slot 0 (declared) CPU-side so existing
|
|
2876
|
+
// accounting paths (slotToRecord, dirty bookkeeping) keep
|
|
2877
|
+
// working. The partition kernel overwrites both slots' draw
|
|
2878
|
+
// tables every frame from the master.
|
|
2879
|
+
const roSlot = bucket.gpuRouted ? bucket.slots[0]! : ensureSlot(bucket, roDescriptor);
|
|
1947
2880
|
const roSlotIdx = bucket.slots.indexOf(roSlot);
|
|
1948
|
-
if (spec.modeRules !== undefined && drawId < 8) {
|
|
1949
|
-
console.debug(`[heapScene] addDraw drawId=${drawId} bucket=${bucket.label} slotIdx=${roSlotIdx}/${bucket.slots.length} cullMode=${roDescriptor.cullMode} recordCount=${roSlot.recordCount}`);
|
|
1950
|
-
}
|
|
1951
2881
|
const fam = familyFor(spec.effect);
|
|
1952
2882
|
const effectFields = fam.fieldsForEffect.get(spec.effect.id)!;
|
|
1953
2883
|
|
|
@@ -1956,7 +2886,7 @@ export function buildHeapScene(
|
|
|
1956
2886
|
// index allocation + one upload.
|
|
1957
2887
|
const indicesAval = asAval(spec.indices) as aval<Uint32Array>;
|
|
1958
2888
|
const indicesArr = readPlain(spec.indices) as Uint32Array;
|
|
1959
|
-
const idxAlloc = indexPool.acquire(device, arena.indices, indicesAval, indicesArr);
|
|
2889
|
+
const idxAlloc = indexPool.acquire(device, arena.indices, indicesAval, bucket.chunkIdx, indicesArr);
|
|
1960
2890
|
|
|
1961
2891
|
const localSlot = bucket.drawHeap.alloc();
|
|
1962
2892
|
// Per-RO instancing: read `spec.instanceCount` (defaults to 1).
|
|
@@ -1989,7 +2919,7 @@ export function buildHeapScene(
|
|
|
1989
2919
|
if (ruleForUniform(spec, f.name) !== undefined) {
|
|
1990
2920
|
const dummyAval = AVal.constant<M44d>(M44d.zero);
|
|
1991
2921
|
const ref = pool.acquire(
|
|
1992
|
-
device, arena.attrs, dummyAval, M44d.zero,
|
|
2922
|
+
device, arena.attrs, dummyAval, bucket.chunkIdx, M44d.zero,
|
|
1993
2923
|
PACKER_MAT4.dataBytes, PACKER_MAT4.typeId, 1, PACKER_MAT4.pack,
|
|
1994
2924
|
);
|
|
1995
2925
|
perDrawRefs.set(f.name, ref);
|
|
@@ -2038,7 +2968,7 @@ export function buildHeapScene(
|
|
|
2038
2968
|
: poolPlacementFor(f, value);
|
|
2039
2969
|
}
|
|
2040
2970
|
const ref = pool.acquire(
|
|
2041
|
-
device, arena.attrs, av, value,
|
|
2971
|
+
device, arena.attrs, av, bucket.chunkIdx, value,
|
|
2042
2972
|
placement.dataBytes, placement.typeId, placement.length, placement.pack,
|
|
2043
2973
|
);
|
|
2044
2974
|
perDrawRefs.set(f.name, ref);
|
|
@@ -2084,7 +3014,53 @@ export function buildHeapScene(
|
|
|
2084
3014
|
const end = byteOff + bucket.layout.drawHeaderBytes;
|
|
2085
3015
|
if (end > bucket.headerDirtyMax) bucket.headerDirtyMax = end;
|
|
2086
3016
|
|
|
2087
|
-
{
|
|
3017
|
+
if (bucket.gpuRouted) {
|
|
3018
|
+
// Phase 5c.3 GPU routing: master pool is authoritative.
|
|
3019
|
+
// Both slot[0] and slot[1] get the SAME accounting (counts,
|
|
3020
|
+
// emit estimates, buffer capacities) — partition overwrites
|
|
3021
|
+
// their GPU drawTables every frame, so per-slot CPU shadows
|
|
3022
|
+
// are irrelevant. We still grow their buffers in lockstep so
|
|
3023
|
+
// the scan dispatch shape covers every possible record.
|
|
3024
|
+
const partition = bucket.partitionScene!;
|
|
3025
|
+
const recIdx = partition.numRecords;
|
|
3026
|
+
if (recIdx >= SCAN_MAX_RECORDS) {
|
|
3027
|
+
throw new Error(
|
|
3028
|
+
`heapScene: GPU-routed bucket exceeds SCAN_MAX_RECORDS (${SCAN_MAX_RECORDS})`,
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
// Pack the bucket's per-RO uniform refs (arena byte offsets) in
|
|
3032
|
+
// the same order as `bucket.uniformOrder`. ROs whose perDrawRefs
|
|
3033
|
+
// don't carry a given name (e.g. their effect doesn't declare
|
|
3034
|
+
// it) get 0 — the kernel only reads refs through combo fns that
|
|
3035
|
+
// gate on `r.comboId`, so an unread 0 is harmless.
|
|
3036
|
+
const uniformRefs: number[] = (bucket.uniformOrder ?? []).map(
|
|
3037
|
+
name => perDrawRefs.get(name) ?? 0,
|
|
3038
|
+
);
|
|
3039
|
+
partition.appendRecord(localSlot, idxAlloc.firstIndex, idxAlloc.count, instanceCount, roComboId, uniformRefs);
|
|
3040
|
+
bucket.drawIdToRecord!.set(drawId, recIdx);
|
|
3041
|
+
bucket.recordToDrawId![recIdx] = drawId;
|
|
3042
|
+
bucket.drawIdToComboId!.set(drawId, roComboId);
|
|
3043
|
+
// Master grew? Invalidate partition's bind group so it picks
|
|
3044
|
+
// up the new masterBuf. (rebindSlotDrawBufs nulls bindGroup.)
|
|
3045
|
+
// grow() inside appendRecord already does this when capacity
|
|
3046
|
+
// is exceeded; nothing to do here.
|
|
3047
|
+
const numRecords = recIdx + 1;
|
|
3048
|
+
const recBytes = numRecords * RECORD_BYTES;
|
|
3049
|
+
const needBlocks = Math.max(1, Math.ceil(numRecords / SCAN_TILE_SIZE));
|
|
3050
|
+
const emitDelta = idxAlloc.count * instanceCount;
|
|
3051
|
+
for (const s of bucket.slots) {
|
|
3052
|
+
s.drawTableBuf!.ensureCapacity(recBytes);
|
|
3053
|
+
s.drawTableBuf!.setUsed(Math.max(s.drawTableBuf!.usedBytes, recBytes));
|
|
3054
|
+
s.blockSumsBuf!.ensureCapacity(needBlocks * 4);
|
|
3055
|
+
s.blockOffsetsBuf!.ensureCapacity(needBlocks * 4);
|
|
3056
|
+
s.recordCount = numRecords;
|
|
3057
|
+
s.totalEmitEstimate += emitDelta;
|
|
3058
|
+
const newNumTiles = Math.max(1, Math.ceil(s.totalEmitEstimate / TILE_K));
|
|
3059
|
+
s.firstDrawInTileBuf!.ensureCapacity((newNumTiles + 1) * 4);
|
|
3060
|
+
s.scanDirty = true;
|
|
3061
|
+
}
|
|
3062
|
+
bucket.partitionDirty = true;
|
|
3063
|
+
} else {
|
|
2088
3064
|
const dtBuf = roSlot.drawTableBuf!;
|
|
2089
3065
|
const recIdx = roSlot.recordCount;
|
|
2090
3066
|
if (recIdx >= SCAN_MAX_RECORDS) {
|
|
@@ -2156,44 +3132,18 @@ export function buildHeapScene(
|
|
|
2156
3132
|
drawIdToModeTracker[drawId] = tracker;
|
|
2157
3133
|
tracker.forEachLeaf((av) => subscribeModeLeaf(av, drawId));
|
|
2158
3134
|
|
|
2159
|
-
// ───
|
|
2160
|
-
//
|
|
2161
|
-
//
|
|
2162
|
-
//
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
// If declared is reactive (e.g. a shared cullModeC cval),
|
|
2172
|
-
// subscribe so flipping it marks the dispatcher dirty.
|
|
2173
|
-
// Plain values: no subscription needed.
|
|
2174
|
-
if (typeof gpuRule.declared !== "string") {
|
|
2175
|
-
subscribeModeLeaf(gpuRule.declared as aval<unknown>, drawId);
|
|
2176
|
-
}
|
|
2177
|
-
if (drawId < 3 || drawId % 20 === 0) {
|
|
2178
|
-
console.debug(`[heapScene] GPU rule registered drawId=${drawId} input='${gpuRule.inputUniform}' ref=${inputRef} declared=${typeof gpuRule.declared === "string" ? gpuRule.declared : "<aval>"}`);
|
|
2179
|
-
}
|
|
2180
|
-
} else {
|
|
2181
|
-
console.warn(`[heapScene] GPU rule input '${gpuRule.inputUniform}' not in spec.inputs for drawId=${drawId}; rule disabled`);
|
|
2182
|
-
}
|
|
2183
|
-
}
|
|
2184
|
-
|
|
2185
|
-
// ─── Pipeline pre-warm for derived-mode rules (5c.2) ──────────
|
|
2186
|
-
// Enumerate the rule's possible outputs and pre-build the
|
|
2187
|
-
// matching pipelines in the scene-level cache. Future cullModeC
|
|
2188
|
-
// flips become cache hits — zero `createRenderPipeline` calls
|
|
2189
|
-
// on the hot path.
|
|
2190
|
-
if (cullRule !== undefined && cullRule.domain !== undefined && tracker.descriptor !== undefined) {
|
|
2191
|
-
const fam = familyFor(spec.effect);
|
|
2192
|
-
const baseDesc = tracker.descriptor;
|
|
2193
|
-
for (const mode of cullRule.domain) {
|
|
2194
|
-
if (mode === baseDesc.cullMode) continue; // already built
|
|
2195
|
-
const variantDesc: PipelineStateDescriptor = { ...baseDesc, cullMode: mode };
|
|
2196
|
-
getOrCreatePipeline(fam, bucket.layout, bucket.isAtlasBucket, variantDesc);
|
|
3135
|
+
// ─── Derived-mode rule registration ────────────────────────────
|
|
3136
|
+
// Subscribe to each rule's `declared` aval so the bucket's
|
|
3137
|
+
// partition kernel re-dispatches when it changes. Slot pipelines
|
|
3138
|
+
// for every (axis, declared) combination are pre-built eagerly
|
|
3139
|
+
// in `initGpuRouting` (one per cartesian-product point), so
|
|
3140
|
+
// declared flips are pipeline-cache hits.
|
|
3141
|
+
if (spec.modeRules !== undefined) {
|
|
3142
|
+
for (const rule of Object.values(spec.modeRules) as Array<DerivedModeRule<ModeAxis> | undefined>) {
|
|
3143
|
+
if (rule === undefined) continue;
|
|
3144
|
+
const d = rule.declared;
|
|
3145
|
+
if (typeof d === "object" && d !== null && "getValue" in (d as object)) {
|
|
3146
|
+
subscribeModeLeaf(d as aval<unknown>, drawId);
|
|
2197
3147
|
}
|
|
2198
3148
|
}
|
|
2199
3149
|
}
|
|
@@ -2235,16 +3185,13 @@ export function buildHeapScene(
|
|
|
2235
3185
|
const reg = registerRoDerivations(derivedScene, {}, {
|
|
2236
3186
|
rules: ruleSubset,
|
|
2237
3187
|
trafoAvals,
|
|
2238
|
-
// A non-trafo rule leaf reads the uniform's data straight from this RO's
|
|
2239
|
-
// drawHeader region in the arena (= its pool ref + the alloc-header pad).
|
|
2240
|
-
// Only resolvable if the uniform was actually packed (i.e. it's a drawHeader
|
|
2241
|
-
// field of the effect too) — undefined otherwise, which `resolveSource` rejects.
|
|
2242
3188
|
hostUniformOffset: (n) => {
|
|
2243
3189
|
const r = perDrawRefs.get(n);
|
|
2244
3190
|
return r === undefined ? undefined : r + ALLOC_HEADER_PAD_TO;
|
|
2245
3191
|
},
|
|
2246
3192
|
outputOffset: (n) => outOffsetByName.get(n),
|
|
2247
3193
|
drawHeaderBaseByte: 0,
|
|
3194
|
+
chunkIdx: bucket.chunkIdx,
|
|
2248
3195
|
});
|
|
2249
3196
|
derivedByDrawId.set(drawId, reg);
|
|
2250
3197
|
}
|
|
@@ -2267,7 +3214,36 @@ export function buildHeapScene(
|
|
|
2267
3214
|
derivedByDrawId.delete(drawId);
|
|
2268
3215
|
}
|
|
2269
3216
|
}
|
|
2270
|
-
{
|
|
3217
|
+
if (bucket.gpuRouted) {
|
|
3218
|
+
const partition = bucket.partitionScene!;
|
|
3219
|
+
const recIdx = bucket.drawIdToRecord!.get(drawId);
|
|
3220
|
+
const removedEntry = bucket.localEntries[localSlot];
|
|
3221
|
+
const removedCount = removedEntry !== undefined
|
|
3222
|
+
? removedEntry.indexCount * removedEntry.instanceCount
|
|
3223
|
+
: 0;
|
|
3224
|
+
if (recIdx !== undefined) {
|
|
3225
|
+
const moved = partition.removeRecord(recIdx);
|
|
3226
|
+
if (moved >= 0) {
|
|
3227
|
+
// The record at `moved` (the old tail) now lives at recIdx.
|
|
3228
|
+
// Fix maps for the moved drawId.
|
|
3229
|
+
const movedDrawId = bucket.recordToDrawId![moved]!;
|
|
3230
|
+
bucket.drawIdToRecord!.set(movedDrawId, recIdx);
|
|
3231
|
+
bucket.recordToDrawId![recIdx] = movedDrawId;
|
|
3232
|
+
bucket.recordToDrawId![moved] = -1;
|
|
3233
|
+
} else {
|
|
3234
|
+
bucket.recordToDrawId![recIdx] = -1;
|
|
3235
|
+
}
|
|
3236
|
+
bucket.drawIdToRecord!.delete(drawId);
|
|
3237
|
+
}
|
|
3238
|
+
// Mirror the count + emit-estimate decrement across every slot
|
|
3239
|
+
// (both share the master's accounting).
|
|
3240
|
+
for (const s of bucket.slots) {
|
|
3241
|
+
s.recordCount = partition.numRecords;
|
|
3242
|
+
s.totalEmitEstimate = Math.max(0, s.totalEmitEstimate - removedCount);
|
|
3243
|
+
s.scanDirty = true;
|
|
3244
|
+
}
|
|
3245
|
+
bucket.partitionDirty = true;
|
|
3246
|
+
} else {
|
|
2271
3247
|
const slotIdx = drawIdToSlotIdx[drawId];
|
|
2272
3248
|
const slot = slotIdx !== undefined ? bucket.slots[slotIdx]! : bucket.slots[0]!;
|
|
2273
3249
|
const removedEntry = bucket.localEntries[localSlot];
|
|
@@ -2304,9 +3280,9 @@ export function buildHeapScene(
|
|
|
2304
3280
|
|
|
2305
3281
|
// Release pool entries — refcount drops; if zero, allocation freed.
|
|
2306
3282
|
const avals = bucket.localPerDrawAvals[localSlot];
|
|
2307
|
-
if (avals !== undefined) for (const av of avals) pool.release(arena.attrs, av);
|
|
3283
|
+
if (avals !== undefined) for (const av of avals) pool.release(arena.attrs, av, bucket.chunkIdx);
|
|
2308
3284
|
const idxAval = drawIdToIndexAval[drawId];
|
|
2309
|
-
if (idxAval !== undefined) indexPool.release(arena.indices, idxAval);
|
|
3285
|
+
if (idxAval !== undefined) indexPool.release(arena.indices, idxAval, bucket.chunkIdx);
|
|
2310
3286
|
const atlasRel = bucket.localAtlasReleases[localSlot];
|
|
2311
3287
|
if (atlasRel !== undefined) atlasRel();
|
|
2312
3288
|
// Drop atlas-aval ref (if any). When the last ref is dropped we
|
|
@@ -2361,10 +3337,6 @@ export function buildHeapScene(
|
|
|
2361
3337
|
}
|
|
2362
3338
|
drawIdToModeTracker[drawId] = undefined;
|
|
2363
3339
|
dirtyModeKeyDrawIds.delete(drawId);
|
|
2364
|
-
if (gpuModesScene !== undefined) {
|
|
2365
|
-
gpuModesScene.deregisterRo(drawId);
|
|
2366
|
-
gpuModesLastKey.delete(drawId);
|
|
2367
|
-
}
|
|
2368
3340
|
|
|
2369
3341
|
stats.totalDraws--;
|
|
2370
3342
|
stats.geometryBytes = arenaBytes(arena);
|
|
@@ -2484,11 +3456,19 @@ export function buildHeapScene(
|
|
|
2484
3456
|
for (const drawId of dirty) {
|
|
2485
3457
|
const tracker = drawIdToModeTracker[drawId];
|
|
2486
3458
|
if (tracker === undefined) continue;
|
|
3459
|
+
const bucket = drawIdToBucket[drawId];
|
|
3460
|
+
// Phase 5c.3: GPU-routed buckets never reslot CPU-side.
|
|
3461
|
+
// The partition kernel handles re-routing per frame.
|
|
3462
|
+
// Just mark the bucket so the next encodeComputePrep
|
|
3463
|
+
// dispatches partition.
|
|
3464
|
+
if (bucket !== undefined && bucket.gpuRouted) {
|
|
3465
|
+
bucket.partitionDirty = true;
|
|
3466
|
+
continue;
|
|
3467
|
+
}
|
|
2487
3468
|
const oldKey = tracker.modeKey;
|
|
2488
3469
|
if (!tracker.recompute()) continue;
|
|
2489
3470
|
if (tracker.modeKey === oldKey) continue;
|
|
2490
3471
|
|
|
2491
|
-
const bucket = drawIdToBucket[drawId];
|
|
2492
3472
|
const localSlot = drawIdToLocalSlot[drawId];
|
|
2493
3473
|
const oldSlotIdx = drawIdToSlotIdx[drawId];
|
|
2494
3474
|
if (bucket === undefined || localSlot === undefined || oldSlotIdx === undefined) continue;
|
|
@@ -2560,8 +3540,7 @@ export function buildHeapScene(
|
|
|
2560
3540
|
if (allocDirty.size > 0) {
|
|
2561
3541
|
for (const av of allocDirty) {
|
|
2562
3542
|
pool.repack(device, arena.attrs, av, av.getValue(tok));
|
|
2563
|
-
|
|
2564
|
-
if (e !== undefined) totalDirtyBytes += e.dataBytes;
|
|
3543
|
+
totalDirtyBytes += pool.totalDataBytes(av);
|
|
2565
3544
|
}
|
|
2566
3545
|
allocDirty.clear();
|
|
2567
3546
|
}
|
|
@@ -2651,6 +3630,9 @@ export function buildHeapScene(
|
|
|
2651
3630
|
}
|
|
2652
3631
|
// Phase 5c.2: each slot has its own drawTable + shadow. Upload
|
|
2653
3632
|
// the dirty range of every slot, not just slots[0].
|
|
3633
|
+
// Phase 5c.3: GPU-routed buckets skip this entirely — the
|
|
3634
|
+
// partition kernel writes slot drawTables directly each frame.
|
|
3635
|
+
if (bucket.gpuRouted) continue;
|
|
2654
3636
|
for (const slot of bucket.slots) {
|
|
2655
3637
|
if (slot.drawTableDirtyMax <= slot.drawTableDirtyMin) continue;
|
|
2656
3638
|
const shadow = slot.drawTableShadow!;
|
|
@@ -2671,26 +3653,23 @@ export function buildHeapScene(
|
|
|
2671
3653
|
* begin/end — caller owns the pass. Used by both the convenience
|
|
2672
3654
|
* `frame()` below and (eventually) the hybrid render task.
|
|
2673
3655
|
*/
|
|
2674
|
-
let _encodeLogged = false;
|
|
2675
3656
|
function encodeIntoPass(pass: GPURenderPassEncoder): void {
|
|
2676
3657
|
let curBg: GPUBindGroup | null = null;
|
|
2677
|
-
let drawCount = 0;
|
|
2678
|
-
const slotInfo: string[] = [];
|
|
2679
3658
|
for (const b of buckets) {
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
3659
|
+
// Phase 5c.2: iterate the bucket's slots, drawIndirect per
|
|
3660
|
+
// non-empty slot. Empty slots draw zero (skipped to save the
|
|
3661
|
+
// setPipeline/setBindGroup overhead).
|
|
3662
|
+
for (const slot of b.slots) {
|
|
2683
3663
|
if (slot.recordCount === 0) continue;
|
|
2684
3664
|
const bg = slot.bindGroup;
|
|
2685
|
-
if (bg !== undefined && bg !== curBg) {
|
|
3665
|
+
if (bg !== undefined && bg !== curBg) {
|
|
3666
|
+
pass.setBindGroup(0, bg);
|
|
3667
|
+
curBg = bg;
|
|
3668
|
+
}
|
|
2686
3669
|
pass.setPipeline(slot.pipeline);
|
|
2687
|
-
if (slot.indirectBuf !== undefined)
|
|
3670
|
+
if (slot.indirectBuf !== undefined) pass.drawIndirect(slot.indirectBuf, 0);
|
|
2688
3671
|
}
|
|
2689
3672
|
}
|
|
2690
|
-
if (!_encodeLogged) {
|
|
2691
|
-
console.debug(`[heapScene] encode: ${drawCount} drawIndirect; slots:`, slotInfo);
|
|
2692
|
-
_encodeLogged = true;
|
|
2693
|
-
}
|
|
2694
3673
|
}
|
|
2695
3674
|
|
|
2696
3675
|
/**
|
|
@@ -2704,41 +3683,6 @@ export function buildHeapScene(
|
|
|
2704
3683
|
* we just swap the bind group + dispatch shape per bucket.
|
|
2705
3684
|
*/
|
|
2706
3685
|
function encodeComputePrep(enc: GPUCommandEncoder, token: AdaptiveToken): void {
|
|
2707
|
-
// GPU-eval derived-mode rules (Task 2 Phase 5): dispatch the
|
|
2708
|
-
// hardcoded determinant-flip-cull kernel + start readback.
|
|
2709
|
-
// Resolved on the next frame's update() — one-frame lag for the
|
|
2710
|
-
// first appearance of a state transition. CPU fallback runs at
|
|
2711
|
-
// addDraw time so the initial bucket assignment is still correct.
|
|
2712
|
-
if (gpuModesScene !== undefined
|
|
2713
|
-
&& gpuModesScene.registered > 0
|
|
2714
|
-
&& gpuModesScene.consumeDirty()) {
|
|
2715
|
-
// Dirty-gate: skip the kernel + readback when no rule input
|
|
2716
|
-
// changed since the last dispatch. Unrelated avals (Color on
|
|
2717
|
-
// hover, etc.) don't trigger dirty, so steady state has zero
|
|
2718
|
-
// GPU mode-rule work per frame.
|
|
2719
|
-
const numROs = nextDrawId; // upper bound; sparse entries are skipped via 0xFFFFFFFF sentinel
|
|
2720
|
-
gpuModesScene.dispatch(arena.attrs.buffer, numROs, enc);
|
|
2721
|
-
// mapAsync the staging buffer AFTER the caller's submit. The
|
|
2722
|
-
// microtask queued by Promise.resolve() runs after the current
|
|
2723
|
-
// synchronous block — by which time `frame()` has already
|
|
2724
|
-
// submitted the enc. Without this deferral, mapAsync fires
|
|
2725
|
-
// before submit and WebGPU rejects the next dispatch's
|
|
2726
|
-
// copy-to-staging with "buffer used in submit while mapped".
|
|
2727
|
-
// No readback: v1's patch-spec.modeRules approach broke
|
|
2728
|
-
// reactivity (a second cullModeC flip stopped re-bucketing
|
|
2729
|
-
// because the patched constant closure never produces a new
|
|
2730
|
-
// modeKey). The CPU fallback is authoritative for now —
|
|
2731
|
-
// tracker.recompute drives the rebucket flow correctly.
|
|
2732
|
-
//
|
|
2733
|
-
// Phase 5c.2 (no-readback by design): pre-warm all slot
|
|
2734
|
-
// pipelines at scene-build via static analysis; kernel writes
|
|
2735
|
-
// per-RO slot index; partition kernel produces per-slot
|
|
2736
|
-
// indirect args; encode iterates fixed slots. The
|
|
2737
|
-
// GpuDerivedModesScene kernel dispatch stays here as a
|
|
2738
|
-
// demonstration of the GPU path — its output is currently
|
|
2739
|
-
// unused but proves the kernel runs against arena data.
|
|
2740
|
-
}
|
|
2741
|
-
|
|
2742
3686
|
// §7: derived-uniforms dispatch first — writes into per-bucket
|
|
2743
3687
|
// drawHeap regions before the scan reads anything. One dispatcher
|
|
2744
3688
|
// per bucket; constituents are shared so dirty state propagates
|
|
@@ -2762,15 +3706,24 @@ export function buildHeapScene(
|
|
|
2762
3706
|
stats.derivedPullMs = _t1 - _t0;
|
|
2763
3707
|
stats.derivedUploadMs = _t2 - _t1;
|
|
2764
3708
|
stats.derivedEncodeMs = _t3 - _t2;
|
|
2765
|
-
stats.derivedRecords = derivedScene.
|
|
3709
|
+
stats.derivedRecords = derivedScene.totalRecordCount;
|
|
2766
3710
|
}
|
|
2767
3711
|
let anyDirty = false;
|
|
2768
3712
|
outer: for (const b of buckets) {
|
|
3713
|
+
if (b.gpuRouted && b.partitionDirty) { anyDirty = true; break outer; }
|
|
2769
3714
|
for (const s of b.slots) {
|
|
2770
3715
|
if (s.scanDirty) { anyDirty = true; break outer; }
|
|
2771
3716
|
}
|
|
2772
3717
|
}
|
|
2773
3718
|
if (!anyDirty) return;
|
|
3719
|
+
// Phase 5c.3: per GPU-routed bucket — dispatch partition (its own
|
|
3720
|
+
// compute pass), then copyBufferToBuffer slot counts into scan
|
|
3721
|
+
// paramsBuf. Must happen BEFORE the shared scan pass below.
|
|
3722
|
+
for (const b of buckets) {
|
|
3723
|
+
if (b.gpuRouted && b.partitionDirty) {
|
|
3724
|
+
dispatchPartition(b, enc, token);
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
2774
3727
|
const pass = enc.beginComputePass({ label: "heapScene/scan" });
|
|
2775
3728
|
for (const b of buckets) {
|
|
2776
3729
|
for (let i = 0; i < b.slots.length; i++) {
|
|
@@ -2870,7 +3823,6 @@ export function buildHeapScene(
|
|
|
2870
3823
|
arena.attrs.destroy();
|
|
2871
3824
|
arena.indices.destroy();
|
|
2872
3825
|
for (const b of buckets) destroyBucketResources(b);
|
|
2873
|
-
gpuModesScene?.dispose();
|
|
2874
3826
|
}
|
|
2875
3827
|
|
|
2876
3828
|
// Test-only escape hatch for inspecting megacall bucket state. Not
|
|
@@ -2932,8 +3884,18 @@ export function buildHeapScene(
|
|
|
2932
3884
|
let tilesChecked = 0, tilesBad = 0;
|
|
2933
3885
|
let vidChecks = 0, vidBad = 0;
|
|
2934
3886
|
const push = (s: string) => { if (issues.length < 60) issues.push(s); };
|
|
2935
|
-
|
|
2936
|
-
|
|
3887
|
+
// §3 v1: validateHeap currently only diagnoses chunk 0 — a
|
|
3888
|
+
// multi-chunk-aware validator needs per-bucket-chunkIdx
|
|
3889
|
+
// staging which is a bigger refactor; assert for now so
|
|
3890
|
+
// multi-chunk callers don't get false-passes.
|
|
3891
|
+
if (arena.attrs.chunkCount > 1 || arena.indices.chunkCount > 1) {
|
|
3892
|
+
throw new Error(
|
|
3893
|
+
"heapScene/validateHeap: multi-chunk arenas not yet supported by this validator (chunk-0-only). " +
|
|
3894
|
+
"Run with the default chunk cap (no second chunk opened) for now.",
|
|
3895
|
+
);
|
|
3896
|
+
}
|
|
3897
|
+
const arenaSize = arena.attrs.chunk(0).buffer.size;
|
|
3898
|
+
const indicesSize = arena.indices.chunk(0).buffer.size;
|
|
2937
3899
|
|
|
2938
3900
|
const enc = device.createCommandEncoder({ label: "validateHeap" });
|
|
2939
3901
|
const stage = (src: GPUBuffer, size: number): GPUBuffer => {
|
|
@@ -2945,8 +3907,8 @@ export function buildHeapScene(
|
|
|
2945
3907
|
return c;
|
|
2946
3908
|
};
|
|
2947
3909
|
|
|
2948
|
-
const arenaCopy = stage(arena.attrs.buffer, arenaSize);
|
|
2949
|
-
const indicesCopy = indicesSize > 0 ? stage(arena.indices.buffer, indicesSize) : undefined;
|
|
3910
|
+
const arenaCopy = stage(arena.attrs.chunk(0).buffer, arenaSize);
|
|
3911
|
+
const indicesCopy = indicesSize > 0 ? stage(arena.indices.chunk(0).buffer, indicesSize) : undefined;
|
|
2950
3912
|
type DC = {
|
|
2951
3913
|
bucket: typeof buckets[number];
|
|
2952
3914
|
drawHeap: GPUBuffer;
|
|
@@ -3235,8 +4197,12 @@ export function buildHeapScene(
|
|
|
3235
4197
|
let emitsChecked = 0;
|
|
3236
4198
|
const push = (s: string) => { if (issues.length < 30) issues.push(s); };
|
|
3237
4199
|
|
|
3238
|
-
|
|
3239
|
-
|
|
4200
|
+
// §3 v1: chunk-0-only like validateHeap.
|
|
4201
|
+
if (arena.attrs.chunkCount > 1 || arena.indices.chunkCount > 1) {
|
|
4202
|
+
throw new Error("heapScene/simulateDraws: multi-chunk arenas not yet supported (chunk-0-only).");
|
|
4203
|
+
}
|
|
4204
|
+
const arenaSize = arena.attrs.chunk(0).buffer.size;
|
|
4205
|
+
const indicesSize = arena.indices.chunk(0).buffer.size;
|
|
3240
4206
|
|
|
3241
4207
|
// Stage all buffers we'll need.
|
|
3242
4208
|
const enc = device.createCommandEncoder({ label: "simulateDraws" });
|
|
@@ -3247,8 +4213,8 @@ export function buildHeapScene(
|
|
|
3247
4213
|
enc.copyBufferToBuffer(src, 0, c, 0, size);
|
|
3248
4214
|
return c;
|
|
3249
4215
|
};
|
|
3250
|
-
const arenaCopy = stage(arena.attrs.buffer, arenaSize);
|
|
3251
|
-
const indicesCopy = indicesSize > 0 ? stage(arena.indices.buffer, indicesSize) : undefined;
|
|
4216
|
+
const arenaCopy = stage(arena.attrs.chunk(0).buffer, arenaSize);
|
|
4217
|
+
const indicesCopy = indicesSize > 0 ? stage(arena.indices.chunk(0).buffer, indicesSize) : undefined;
|
|
3252
4218
|
type DC = {
|
|
3253
4219
|
bucket: typeof buckets[number];
|
|
3254
4220
|
drawHeap: GPUBuffer;
|
|
@@ -3518,19 +4484,25 @@ export function buildHeapScene(
|
|
|
3518
4484
|
// cross-RO triangle symptom.
|
|
3519
4485
|
const allocClaims: { ref: number; bytes: number; owner: string }[] = [];
|
|
3520
4486
|
|
|
4487
|
+
// §3 v1: chunk-0-only (see validateHeap).
|
|
4488
|
+
if (arena.attrs.chunkCount > 1 || arena.indices.chunkCount > 1) {
|
|
4489
|
+
throw new Error("heapScene/checkTriangleCoherence: multi-chunk arenas not yet supported (chunk-0-only).");
|
|
4490
|
+
}
|
|
4491
|
+
const arenaBuf0 = arena.attrs.chunk(0).buffer;
|
|
4492
|
+
const indicesBuf0 = arena.indices.chunk(0).buffer;
|
|
3521
4493
|
// Stage arena + arena.indices once (shared across buckets).
|
|
3522
4494
|
const arenaCopy = device.createBuffer({
|
|
3523
|
-
size:
|
|
4495
|
+
size: arenaBuf0.size,
|
|
3524
4496
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3525
4497
|
});
|
|
3526
|
-
const indicesCopy =
|
|
3527
|
-
size:
|
|
4498
|
+
const indicesCopy = indicesBuf0.size > 0 ? device.createBuffer({
|
|
4499
|
+
size: indicesBuf0.size,
|
|
3528
4500
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3529
4501
|
}) : undefined;
|
|
3530
4502
|
{
|
|
3531
4503
|
const enc = device.createCommandEncoder({ label: "checkTriangleCoherence.arena" });
|
|
3532
|
-
enc.copyBufferToBuffer(
|
|
3533
|
-
if (indicesCopy !== undefined) enc.copyBufferToBuffer(
|
|
4504
|
+
enc.copyBufferToBuffer(arenaBuf0, 0, arenaCopy, 0, arenaBuf0.size);
|
|
4505
|
+
if (indicesCopy !== undefined) enc.copyBufferToBuffer(indicesBuf0, 0, indicesCopy, 0, indicesBuf0.size);
|
|
3534
4506
|
device.queue.submit([enc.finish()]);
|
|
3535
4507
|
}
|
|
3536
4508
|
await arenaCopy.mapAsync(GPUMapMode.READ);
|
|
@@ -3963,7 +4935,7 @@ export function buildHeapScene(
|
|
|
3963
4935
|
{ binding: 0, resource: { buffer: target.slots[0]!.drawTableBuf!.buffer } },
|
|
3964
4936
|
{ binding: 1, resource: { buffer: target.slots[0]!.firstDrawInTileBuf!.buffer } },
|
|
3965
4937
|
{ binding: 2, resource: { buffer: sampleBuf } },
|
|
3966
|
-
{ binding: 3, resource: { buffer: arena.indices.buffer } },
|
|
4938
|
+
{ binding: 3, resource: { buffer: arena.indices.chunk(target.chunkIdx).buffer } },
|
|
3967
4939
|
{ binding: 4, resource: { buffer: outBuf } },
|
|
3968
4940
|
{ binding: 5, resource: { buffer: paramBuf } },
|
|
3969
4941
|
],
|
|
@@ -3986,12 +4958,13 @@ export function buildHeapScene(
|
|
|
3986
4958
|
sampleBuf.destroy(); outBuf.destroy(); paramBuf.destroy();
|
|
3987
4959
|
|
|
3988
4960
|
// Read drawTable + indices for CPU expectation.
|
|
4961
|
+
const idxBuf = arena.indices.chunk(target.chunkIdx).buffer;
|
|
3989
4962
|
const indicesCopy2 = device.createBuffer({
|
|
3990
|
-
size:
|
|
4963
|
+
size: idxBuf.size,
|
|
3991
4964
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
3992
4965
|
});
|
|
3993
4966
|
const enc2 = device.createCommandEncoder();
|
|
3994
|
-
enc2.copyBufferToBuffer(
|
|
4967
|
+
enc2.copyBufferToBuffer(idxBuf, 0, indicesCopy2, 0, idxBuf.size);
|
|
3995
4968
|
device.queue.submit([enc2.finish()]);
|
|
3996
4969
|
await indicesCopy2.mapAsync(GPUMapMode.READ);
|
|
3997
4970
|
const indicesU32 = new Uint32Array(indicesCopy2.getMappedRange().slice(0));
|