@aardworx/wombat.rendering 0.9.32 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime/derivedModes/kernelCodegen.d.ts +90 -0
- package/dist/runtime/derivedModes/kernelCodegen.d.ts.map +1 -0
- package/dist/runtime/derivedModes/kernelCodegen.js +623 -0
- package/dist/runtime/derivedModes/kernelCodegen.js.map +1 -0
- package/dist/runtime/derivedModes/modeKeyCpu.d.ts.map +1 -1
- package/dist/runtime/derivedModes/modeKeyCpu.js +23 -67
- package/dist/runtime/derivedModes/modeKeyCpu.js.map +1 -1
- package/dist/runtime/derivedModes/partitionDispatcher.d.ts +79 -0
- package/dist/runtime/derivedModes/partitionDispatcher.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionDispatcher.js +303 -0
- package/dist/runtime/derivedModes/partitionDispatcher.js.map +1 -0
- package/dist/runtime/derivedModes/partitionKernel.d.ts +12 -0
- package/dist/runtime/derivedModes/partitionKernel.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionKernel.js +127 -0
- package/dist/runtime/derivedModes/partitionKernel.js.map +1 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.d.ts +4 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.d.ts.map +1 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.js +24 -0
- package/dist/runtime/derivedModes/partitionKernelLayout.js.map +1 -0
- package/dist/runtime/derivedModes/rule.d.ts +38 -86
- package/dist/runtime/derivedModes/rule.d.ts.map +1 -1
- package/dist/runtime/derivedModes/rule.js +34 -74
- package/dist/runtime/derivedModes/rule.js.map +1 -1
- package/dist/runtime/derivedUniforms/dispatch.d.ts +10 -8
- package/dist/runtime/derivedUniforms/dispatch.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/dispatch.js +31 -18
- package/dist/runtime/derivedUniforms/dispatch.js.map +1 -1
- package/dist/runtime/derivedUniforms/marker.d.ts +90 -11
- package/dist/runtime/derivedUniforms/marker.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/marker.js +161 -15
- package/dist/runtime/derivedUniforms/marker.js.map +1 -1
- package/dist/runtime/derivedUniforms/sceneIntegration.d.ts +25 -6
- package/dist/runtime/derivedUniforms/sceneIntegration.d.ts.map +1 -1
- package/dist/runtime/derivedUniforms/sceneIntegration.js +61 -20
- package/dist/runtime/derivedUniforms/sceneIntegration.js.map +1 -1
- package/dist/runtime/heapEligibility.d.ts.map +1 -1
- package/dist/runtime/heapEligibility.js +22 -2
- package/dist/runtime/heapEligibility.js.map +1 -1
- package/dist/runtime/heapScene/chunkedArena.d.ts +76 -0
- package/dist/runtime/heapScene/chunkedArena.d.ts.map +1 -0
- package/dist/runtime/heapScene/chunkedArena.js +223 -0
- package/dist/runtime/heapScene/chunkedArena.js.map +1 -0
- package/dist/runtime/heapScene/freelist.d.ts +50 -0
- package/dist/runtime/heapScene/freelist.d.ts.map +1 -0
- package/dist/runtime/heapScene/freelist.js +160 -0
- package/dist/runtime/heapScene/freelist.js.map +1 -0
- package/dist/runtime/heapScene/growBuffer.d.ts +23 -2
- package/dist/runtime/heapScene/growBuffer.d.ts.map +1 -1
- package/dist/runtime/heapScene/growBuffer.js +35 -3
- package/dist/runtime/heapScene/growBuffer.js.map +1 -1
- package/dist/runtime/heapScene/pools.d.ts +65 -42
- package/dist/runtime/heapScene/pools.d.ts.map +1 -1
- package/dist/runtime/heapScene/pools.js +206 -134
- package/dist/runtime/heapScene/pools.js.map +1 -1
- package/dist/runtime/heapScene.d.ts +12 -0
- package/dist/runtime/heapScene.d.ts.map +1 -1
- package/dist/runtime/heapScene.js +949 -167
- package/dist/runtime/heapScene.js.map +1 -1
- package/dist/runtime/hybridScene.d.ts +7 -0
- package/dist/runtime/hybridScene.d.ts.map +1 -1
- package/dist/runtime/hybridScene.js +1 -0
- package/dist/runtime/hybridScene.js.map +1 -1
- package/dist/runtime/index.d.ts +4 -3
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/index.js +4 -3
- package/dist/runtime/index.js.map +1 -1
- package/dist/runtime/renderTask.d.ts +5 -0
- package/dist/runtime/renderTask.d.ts.map +1 -1
- package/dist/runtime/renderTask.js +1 -0
- package/dist/runtime/renderTask.js.map +1 -1
- package/dist/runtime/runtime.d.ts +6 -0
- package/dist/runtime/runtime.d.ts.map +1 -1
- package/dist/runtime/runtime.js +1 -0
- package/dist/runtime/runtime.js.map +1 -1
- package/package.json +1 -1
- package/src/runtime/derivedModes/kernelCodegen.ts +673 -0
- package/src/runtime/derivedModes/modeKeyCpu.ts +22 -60
- package/src/runtime/derivedModes/partitionDispatcher.ts +349 -0
- package/src/runtime/derivedModes/partitionKernelLayout.ts +25 -0
- package/src/runtime/derivedModes/rule.ts +66 -145
- package/src/runtime/derivedUniforms/dispatch.ts +48 -21
- package/src/runtime/derivedUniforms/marker.ts +210 -16
- package/src/runtime/derivedUniforms/sceneIntegration.ts +67 -20
- package/src/runtime/heapEligibility.ts +22 -1
- package/src/runtime/heapScene/chunkedArena.ts +238 -0
- package/src/runtime/heapScene/freelist.ts +161 -0
- package/src/runtime/heapScene/growBuffer.ts +39 -2
- package/src/runtime/heapScene/pools.ts +231 -143
- package/src/runtime/heapScene.ts +1134 -161
- package/src/runtime/hybridScene.ts +8 -0
- package/src/runtime/index.ts +12 -9
- package/src/runtime/renderTask.ts +6 -0
- package/src/runtime/runtime.ts +7 -0
- package/src/runtime/derivedModes/gpuDispatcher.ts +0 -226
- package/src/runtime/derivedModes/gpuKernel.ts +0 -98
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
// §7 v2 — heap-scene integration.
|
|
2
2
|
//
|
|
3
|
-
// One DerivedUniformsScene per heap scene: owns the rule registry,
|
|
4
|
-
//
|
|
5
|
-
// constituents GPU buffer, and the dispatcher.
|
|
6
|
-
//
|
|
7
|
-
// (
|
|
3
|
+
// One DerivedUniformsScene per heap scene: owns the rule registry, a per-chunk
|
|
4
|
+
// records buffer (§3), the constituent-slots heap (df32 trafo halves, scene-
|
|
5
|
+
// wide), the constituents GPU buffer, and the dispatcher.
|
|
6
|
+
//
|
|
7
|
+
// Multi-chunk (§3): each arena chunk has its own GPUBuffer for drawHeaders.
|
|
8
|
+
// Derived-uniform records reference HostHeap slots by absolute byte offset
|
|
9
|
+
// within ONE buffer — so a record must be bound to its bucket's chunk at
|
|
10
|
+
// dispatch. We partition records by chunkIdx: one RecordsBuffer per chunk,
|
|
11
|
+
// one dispatch per chunk binding that chunk's main heap.
|
|
8
12
|
//
|
|
9
13
|
// Per frame, inside the heap scene's `evaluateAlways(token, …)` scope:
|
|
10
14
|
// const dirty = scene.pullDirty(token); // drains changed trafo avals (re-subscribes)
|
|
11
15
|
// scene.uploadDirty(dirty); // O(changed) value uploads
|
|
12
|
-
// scene.encode(enc); // one compute dispatch
|
|
16
|
+
// scene.encode(enc); // one compute dispatch per chunk
|
|
13
17
|
// Clean frames: zero CPU work, records buffer static.
|
|
14
18
|
// See docs/derived-uniforms-extensible.md.
|
|
15
19
|
|
|
@@ -33,16 +37,20 @@ export class DerivedUniformsScene {
|
|
|
33
37
|
readonly device: GPUDevice;
|
|
34
38
|
readonly constituents: ConstituentSlots;
|
|
35
39
|
readonly registry = new DerivedUniformRegistry();
|
|
36
|
-
/**
|
|
37
|
-
|
|
40
|
+
/** Per-chunk records buffers (§3). Constituents + registry are
|
|
41
|
+
* scene-wide; records partition by chunk so each dispatch binds
|
|
42
|
+
* the right chunk's main heap. */
|
|
43
|
+
private readonly recordsByChunk = new Map<number, RecordsBuffer>();
|
|
38
44
|
readonly dispatcher: DerivedUniformsDispatcher;
|
|
39
45
|
/** Constituents storage GPU buffer (`array<vec2<f32>>` — df32 mat4 halves). */
|
|
40
46
|
constituentsBuf: GPUBuffer;
|
|
41
|
-
|
|
47
|
+
/** Per-chunk main-heap GPUBuffer getters. Populated as chunks are
|
|
48
|
+
* registered via `setMainHeapForChunk` from the scene factory. */
|
|
49
|
+
private readonly mainHeapByChunk = new Map<number, () => GPUBuffer>();
|
|
42
50
|
/** Bumped each time a shared GPU buffer is replaced. */
|
|
43
51
|
bufferEpoch = 0;
|
|
44
52
|
|
|
45
|
-
constructor(device: GPUDevice,
|
|
53
|
+
constructor(device: GPUDevice, opts?: DerivedUniformsSceneOptions) {
|
|
46
54
|
this.device = device;
|
|
47
55
|
const initial = opts?.initialConstituentSlots ?? 64;
|
|
48
56
|
this.constituents = new ConstituentSlots(() => {}, initial);
|
|
@@ -51,15 +59,34 @@ export class DerivedUniformsScene {
|
|
|
51
59
|
size: initial * DF32_MAT4_BYTES,
|
|
52
60
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
53
61
|
});
|
|
54
|
-
this.mainHeapRef = { current: mainHeapBuf };
|
|
55
62
|
this.dispatcher = new DerivedUniformsDispatcher(device, {
|
|
56
63
|
constituentsBuf: () => this.constituentsBuf,
|
|
57
|
-
mainHeapBuf: () => this.mainHeapRef.current,
|
|
58
64
|
});
|
|
59
65
|
}
|
|
60
66
|
|
|
61
|
-
|
|
62
|
-
|
|
67
|
+
/** Register (or replace) the GPUBuffer getter for a chunk's main
|
|
68
|
+
* heap. Called by the scene factory at construction and whenever
|
|
69
|
+
* a new chunk opens or an existing chunk's buffer reallocates. */
|
|
70
|
+
setMainHeapForChunk(chunkIdx: number, getter: () => GPUBuffer): void {
|
|
71
|
+
this.mainHeapByChunk.set(chunkIdx, getter);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Get-or-create the records buffer for `chunkIdx`. */
|
|
75
|
+
recordsFor(chunkIdx: number): RecordsBuffer {
|
|
76
|
+
let r = this.recordsByChunk.get(chunkIdx);
|
|
77
|
+
if (r === undefined) {
|
|
78
|
+
r = new RecordsBuffer();
|
|
79
|
+
this.recordsByChunk.set(chunkIdx, r);
|
|
80
|
+
}
|
|
81
|
+
return r;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Total record count summed across every chunk's records buffer.
|
|
85
|
+
* Used by the scene's stats / diagnostics. */
|
|
86
|
+
get totalRecordCount(): number {
|
|
87
|
+
let n = 0;
|
|
88
|
+
for (const r of this.recordsByChunk.values()) n += r.recordCount;
|
|
89
|
+
return n;
|
|
63
90
|
}
|
|
64
91
|
|
|
65
92
|
ensureConstituentsCapacity(requiredBytes: number): void {
|
|
@@ -102,9 +129,23 @@ export class DerivedUniformsScene {
|
|
|
102
129
|
uploadConstituentsRange(this.device, this.constituentsBuf, this.constituents.cpuMirror, dirty);
|
|
103
130
|
}
|
|
104
131
|
|
|
105
|
-
/** Run the uber-kernel
|
|
132
|
+
/** Run the uber-kernel once per chunk that holds records. No-op
|
|
133
|
+
* when no chunk has records. Each dispatch binds that chunk's
|
|
134
|
+
* main heap buffer. */
|
|
106
135
|
encode(enc: GPUCommandEncoder): boolean {
|
|
107
|
-
|
|
136
|
+
let any = false;
|
|
137
|
+
for (const [chunkIdx, records] of this.recordsByChunk) {
|
|
138
|
+
if (records.recordCount === 0) continue;
|
|
139
|
+
const heapGetter = this.mainHeapByChunk.get(chunkIdx);
|
|
140
|
+
if (heapGetter === undefined) {
|
|
141
|
+
throw new Error(
|
|
142
|
+
`derivedUniforms.encode: no main-heap binding registered for chunkIdx=${chunkIdx}. ` +
|
|
143
|
+
`Call setMainHeapForChunk(idx, getter) when each chunk opens.`,
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
if (this.dispatcher.encodeChunk(enc, this.registry, records, heapGetter)) any = true;
|
|
147
|
+
}
|
|
148
|
+
return any;
|
|
108
149
|
}
|
|
109
150
|
|
|
110
151
|
dispose(): void {
|
|
@@ -126,6 +167,9 @@ export interface RoDerivedRequest {
|
|
|
126
167
|
readonly outputOffset: (name: string) => number | undefined;
|
|
127
168
|
/** Absolute byte offset of this RO's drawHeader within the main heap. */
|
|
128
169
|
readonly drawHeaderBaseByte: number;
|
|
170
|
+
/** §3: which arena chunk this RO lives in. Records are partitioned
|
|
171
|
+
* by chunk so each dispatch binds the right main-heap buffer. */
|
|
172
|
+
readonly chunkIdx: number;
|
|
129
173
|
}
|
|
130
174
|
|
|
131
175
|
export interface RoRegistration {
|
|
@@ -134,6 +178,8 @@ export interface RoRegistration {
|
|
|
134
178
|
readonly constituentAvals: readonly aval<Trafo3d>[];
|
|
135
179
|
/** Flattened-rule hashes — for `registry.release` on teardown. */
|
|
136
180
|
readonly ruleHashes: readonly string[];
|
|
181
|
+
/** Chunk this registration was placed into — used by deregister. */
|
|
182
|
+
readonly chunkIdx: number;
|
|
137
183
|
}
|
|
138
184
|
|
|
139
185
|
export function registerRoDerivations(
|
|
@@ -141,7 +187,8 @@ export function registerRoDerivations(
|
|
|
141
187
|
owner: object,
|
|
142
188
|
req: RoDerivedRequest,
|
|
143
189
|
): RoRegistration {
|
|
144
|
-
if (req.rules.size === 0) return { owner, constituentAvals: [], ruleHashes: [] };
|
|
190
|
+
if (req.rules.size === 0) return { owner, constituentAvals: [], ruleHashes: [], chunkIdx: req.chunkIdx };
|
|
191
|
+
const records = scene.recordsFor(req.chunkIdx);
|
|
145
192
|
|
|
146
193
|
const acquiredAvals: aval<Trafo3d>[] = [];
|
|
147
194
|
const ruleHashes: string[] = [];
|
|
@@ -192,14 +239,14 @@ export function registerRoDerivations(
|
|
|
192
239
|
if (outOff === undefined) {
|
|
193
240
|
throw new Error(`derivedUniforms: no drawHeader slot for derived uniform '${outName}' on this RO`);
|
|
194
241
|
}
|
|
195
|
-
|
|
242
|
+
records.add(owner, id, makeHandle(SlotTag.HostHeap, req.drawHeaderBaseByte + outOff), inSlots);
|
|
196
243
|
}
|
|
197
244
|
scene.ensureConstituentsCapacity(scene.constituents.slotCount * DF32_MAT4_BYTES);
|
|
198
|
-
return { owner, constituentAvals: acquiredAvals, ruleHashes };
|
|
245
|
+
return { owner, constituentAvals: acquiredAvals, ruleHashes, chunkIdx: req.chunkIdx };
|
|
199
246
|
}
|
|
200
247
|
|
|
201
248
|
export function deregisterRoDerivations(scene: DerivedUniformsScene, reg: RoRegistration): void {
|
|
202
|
-
scene.
|
|
249
|
+
scene.recordsFor(reg.chunkIdx).removeAllForOwner(reg.owner);
|
|
203
250
|
for (const h of reg.ruleHashes) scene.registry.release(h);
|
|
204
251
|
for (const av of reg.constituentAvals) scene.constituents.release(av);
|
|
205
252
|
}
|
|
@@ -51,6 +51,19 @@ import { asAttributeProvider } from "../core/provider.js";
|
|
|
51
51
|
*/
|
|
52
52
|
const LEGACY_MAX_DIM = 4096;
|
|
53
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Total per-RO payload (attribute + instance + index bytes) above
|
|
56
|
+
* which the heap path ejects the RO to the legacy renderer. The
|
|
57
|
+
* heap path's value is collapsing many small ROs into one
|
|
58
|
+
* drawIndirect; a single-RO bucket emits the same drawIndexed it
|
|
59
|
+
* would have under legacy, AND a multi-MB allocation forces the
|
|
60
|
+
* arena's pow2 GrowBuffer to allocate (and re-copy on grow) a
|
|
61
|
+
* giant contiguous GPUBuffer. Default 16 MB — picked so typical
|
|
62
|
+
* UI/SG geometry (a few hundred kB) sails through and only
|
|
63
|
+
* terrain/photogrammetry/dense-pointcloud meshes eject.
|
|
64
|
+
*/
|
|
65
|
+
const HEAP_PAYLOAD_EJECT_BYTES = 16 * 1024 * 1024;
|
|
66
|
+
|
|
54
67
|
const isHostBuffer = (b: IBuffer): boolean => b.kind === "host";
|
|
55
68
|
// Samplers: accept both descriptor-form (`kind: "host"`) and resolved
|
|
56
69
|
// (`kind: "gpu"`). The heap atlas path uses ONE shared GPU sampler for
|
|
@@ -204,7 +217,15 @@ export function isHeapEligible(ro: RenderObject): aval<boolean> {
|
|
|
204
217
|
// can flip; if they violate heap constraints the RO routes to the
|
|
205
218
|
// legacy path that frame).
|
|
206
219
|
return AVal.custom(token => {
|
|
207
|
-
|
|
220
|
+
let payloadBytes = 0;
|
|
221
|
+
for (const av of buffers) {
|
|
222
|
+
const b = av.getValue(token);
|
|
223
|
+
if (!isHostBuffer(b)) return false;
|
|
224
|
+
payloadBytes += b.sizeBytes;
|
|
225
|
+
// Early-out if any single RO already busts the budget — avoids
|
|
226
|
+
// walking the rest of the buffer list for a guaranteed reject.
|
|
227
|
+
if (payloadBytes > HEAP_PAYLOAD_EJECT_BYTES) return false;
|
|
228
|
+
}
|
|
208
229
|
for (const av of textures) if (!isHeapServableTexture(av.getValue(token))) return false;
|
|
209
230
|
for (const av of samplers) if (!isHeapServableSampler(av.getValue(token))) return false;
|
|
210
231
|
const dc = ro.drawCall.getValue(token);
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
// Multi-chunk arena allocators — §3 of `docs/heap-future-work.md`.
|
|
2
|
+
//
|
|
3
|
+
// Each chunk owns a separate GPUBuffer with its own pow2-grow + best-
|
|
4
|
+
// fit Freelist. When the newest chunk hits its per-chunk cap, the
|
|
5
|
+
// allocator opens a fresh chunk and routes subsequent allocations
|
|
6
|
+
// there. Each (chunkIdx, offset) is the address; refs in master
|
|
7
|
+
// records / drawHeaders carry only the offset — chunkIdx is implicit
|
|
8
|
+
// in which chunk's bind group the current draw call is using.
|
|
9
|
+
//
|
|
10
|
+
// Cap policy: `maxChunkBytes` ≤ `device.limits.maxStorageBufferBindingSize`
|
|
11
|
+
// per chunk. Default: `min(adapter, DEFAULT_MAX_BUFFER_BYTES)`. Tests
|
|
12
|
+
// can pass a tiny cap (e.g. 4 MB) to exercise multi-chunk behaviour
|
|
13
|
+
// without massive workloads.
|
|
14
|
+
|
|
15
|
+
import type { IDisposable } from "@aardworx/wombat.adaptive";
|
|
16
|
+
import { GrowBuffer, DEFAULT_MAX_BUFFER_BYTES } from "./growBuffer.js";
|
|
17
|
+
import { AttributeArena, IndexAllocator } from "./pools.js";
|
|
18
|
+
|
|
19
|
+
/** Address inside a chunked arena. */
|
|
20
|
+
export interface ChunkedRef {
|
|
21
|
+
readonly chunkIdx: number;
|
|
22
|
+
readonly off: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Multi-chunk wrapper around `AttributeArena`. Each chunk is one
|
|
27
|
+
* GrowBuffer / Freelist pair, capped at `maxChunkBytes`. Caller
|
|
28
|
+
* tracks chunkIdx alongside the offset — refs in shaders are just
|
|
29
|
+
* the offset, because shaders bind one chunk at a time.
|
|
30
|
+
*/
|
|
31
|
+
export class ChunkedAttributeArena {
|
|
32
|
+
private readonly _chunks: AttributeArena[] = [];
|
|
33
|
+
private readonly onChunkAddedCbs = new Set<(chunkIdx: number) => void>();
|
|
34
|
+
|
|
35
|
+
constructor(
|
|
36
|
+
private readonly device: GPUDevice,
|
|
37
|
+
private readonly label: string,
|
|
38
|
+
private readonly usage: GPUBufferUsageFlags,
|
|
39
|
+
private readonly initialChunkBytes: number,
|
|
40
|
+
private readonly maxChunkBytes: number = DEFAULT_MAX_BUFFER_BYTES,
|
|
41
|
+
) {
|
|
42
|
+
this.openChunk();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
get chunkCount(): number { return this._chunks.length; }
|
|
46
|
+
get chunks(): ReadonlyArray<AttributeArena> { return this._chunks; }
|
|
47
|
+
chunk(i: number): AttributeArena { return this._chunks[i]!; }
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Allocate `dataBytes` of attribute storage. Tries the `hint`
|
|
51
|
+
* chunk first when given (caller passes the RO's chunk to keep all
|
|
52
|
+
* uniforms in the same buffer); falls back to the newest chunk;
|
|
53
|
+
* opens a new chunk when none fits.
|
|
54
|
+
*/
|
|
55
|
+
alloc(dataBytes: number, hint?: number): ChunkedRef {
|
|
56
|
+
if (hint !== undefined && hint >= 0 && hint < this._chunks.length) {
|
|
57
|
+
const off = this._chunks[hint]!.tryAlloc(dataBytes);
|
|
58
|
+
if (off !== undefined) return { chunkIdx: hint, off };
|
|
59
|
+
}
|
|
60
|
+
// Newest-first ordering keeps freshly-opened chunks busy and
|
|
61
|
+
// older chunks naturally drain as their ROs release (helping
|
|
62
|
+
// future drop-empty-chunks cleanup land cheaply).
|
|
63
|
+
for (let i = this._chunks.length - 1; i >= 0; i--) {
|
|
64
|
+
if (i === hint) continue;
|
|
65
|
+
const off = this._chunks[i]!.tryAlloc(dataBytes);
|
|
66
|
+
if (off !== undefined) return { chunkIdx: i, off };
|
|
67
|
+
}
|
|
68
|
+
const newIdx = this.openChunk();
|
|
69
|
+
const off = this._chunks[newIdx]!.tryAlloc(dataBytes);
|
|
70
|
+
if (off === undefined) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`ChunkedAttributeArena '${this.label}': allocation of ${dataBytes}B exceeds maxChunkBytes ${this.maxChunkBytes}`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
return { chunkIdx: newIdx, off };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
release(chunkIdx: number, off: number, dataBytes: number): void {
|
|
79
|
+
const c = this._chunks[chunkIdx];
|
|
80
|
+
if (c === undefined) return;
|
|
81
|
+
c.release(off, dataBytes);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Push the dirty CPU shadow of every chunk to its GPU buffer. */
|
|
85
|
+
flush(device: GPUDevice): void {
|
|
86
|
+
for (const c of this._chunks) c.flush(device);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Write into a specific chunk's CPU shadow. */
|
|
90
|
+
write(chunkIdx: number, dst: number, data: Uint8Array): void {
|
|
91
|
+
this._chunks[chunkIdx]!.write(dst, data);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Subscribe to "a new chunk was opened" events. Used by the
|
|
95
|
+
* bucket-rebuild path to widen its per-chunk bookkeeping. */
|
|
96
|
+
onChunkAdded(cb: (chunkIdx: number) => void): IDisposable {
|
|
97
|
+
this.onChunkAddedCbs.add(cb);
|
|
98
|
+
return { dispose: () => { this.onChunkAddedCbs.delete(cb); } };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Subscribe to "any chunk resized (incl. brand-new chunk)" events.
|
|
102
|
+
* Wires the callback to every current chunk's `onResize` AND
|
|
103
|
+
* fires it for each future-added chunk after subscribing.
|
|
104
|
+
* Used by the scene-level rebuild path that follows arena growth. */
|
|
105
|
+
onAnyResize(cb: (chunkIdx: number) => void): IDisposable {
|
|
106
|
+
const disposables: IDisposable[] = [];
|
|
107
|
+
for (let i = 0; i < this._chunks.length; i++) {
|
|
108
|
+
const idx = i;
|
|
109
|
+
disposables.push(this._chunks[i]!.onResize(() => cb(idx)));
|
|
110
|
+
}
|
|
111
|
+
disposables.push(this.onChunkAdded((idx) => {
|
|
112
|
+
disposables.push(this._chunks[idx]!.onResize(() => cb(idx)));
|
|
113
|
+
// Also fire once for the new chunk so listeners can wire bind
|
|
114
|
+
// groups to it.
|
|
115
|
+
cb(idx);
|
|
116
|
+
}));
|
|
117
|
+
return { dispose: () => { for (const d of disposables) d.dispose(); } };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Total bytes used across all chunks (high-watermark sum). */
|
|
121
|
+
totalUsedBytes(): number {
|
|
122
|
+
let s = 0;
|
|
123
|
+
for (const c of this._chunks) s += c.usedBytes;
|
|
124
|
+
return s;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
destroy(): void {
|
|
128
|
+
for (const c of this._chunks) c.destroy();
|
|
129
|
+
this._chunks.length = 0;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
private openChunk(): number {
|
|
133
|
+
const idx = this._chunks.length;
|
|
134
|
+
const buf = new GrowBuffer(
|
|
135
|
+
this.device, `${this.label}/c${idx}`, this.usage,
|
|
136
|
+
this.initialChunkBytes, this.maxChunkBytes,
|
|
137
|
+
);
|
|
138
|
+
this._chunks.push(new AttributeArena(buf));
|
|
139
|
+
for (const cb of this.onChunkAddedCbs) cb(idx);
|
|
140
|
+
return idx;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Multi-chunk wrapper around `IndexAllocator`. Element-bump units
|
|
145
|
+
* are u32. Same chunk-open / hint / fallback semantics as
|
|
146
|
+
* `ChunkedAttributeArena`. */
|
|
147
|
+
export class ChunkedIndexAllocator {
|
|
148
|
+
private readonly _chunks: IndexAllocator[] = [];
|
|
149
|
+
private readonly onChunkAddedCbs = new Set<(chunkIdx: number) => void>();
|
|
150
|
+
|
|
151
|
+
constructor(
|
|
152
|
+
private readonly device: GPUDevice,
|
|
153
|
+
private readonly label: string,
|
|
154
|
+
private readonly usage: GPUBufferUsageFlags,
|
|
155
|
+
private readonly initialChunkBytes: number,
|
|
156
|
+
private readonly maxChunkBytes: number = DEFAULT_MAX_BUFFER_BYTES,
|
|
157
|
+
) {
|
|
158
|
+
this.openChunk();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
get chunkCount(): number { return this._chunks.length; }
|
|
162
|
+
get chunks(): ReadonlyArray<IndexAllocator> { return this._chunks; }
|
|
163
|
+
chunk(i: number): IndexAllocator { return this._chunks[i]!; }
|
|
164
|
+
|
|
165
|
+
alloc(elements: number, hint?: number): ChunkedRef {
|
|
166
|
+
if (hint !== undefined && hint >= 0 && hint < this._chunks.length) {
|
|
167
|
+
const off = this._chunks[hint]!.tryAlloc(elements);
|
|
168
|
+
if (off !== undefined) return { chunkIdx: hint, off };
|
|
169
|
+
}
|
|
170
|
+
for (let i = this._chunks.length - 1; i >= 0; i--) {
|
|
171
|
+
if (i === hint) continue;
|
|
172
|
+
const off = this._chunks[i]!.tryAlloc(elements);
|
|
173
|
+
if (off !== undefined) return { chunkIdx: i, off };
|
|
174
|
+
}
|
|
175
|
+
const newIdx = this.openChunk();
|
|
176
|
+
const off = this._chunks[newIdx]!.tryAlloc(elements);
|
|
177
|
+
if (off === undefined) {
|
|
178
|
+
throw new Error(
|
|
179
|
+
`ChunkedIndexAllocator '${this.label}': allocation of ${elements} elements exceeds maxChunkBytes/4 ${this.maxChunkBytes / 4}`,
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
return { chunkIdx: newIdx, off };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
release(chunkIdx: number, off: number, elements: number): void {
|
|
186
|
+
const c = this._chunks[chunkIdx];
|
|
187
|
+
if (c === undefined) return;
|
|
188
|
+
c.release(off, elements);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
flush(device: GPUDevice): void {
|
|
192
|
+
for (const c of this._chunks) c.flush(device);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
write(chunkIdx: number, dst: number, data: Uint8Array): void {
|
|
196
|
+
this._chunks[chunkIdx]!.write(dst, data);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
onChunkAdded(cb: (chunkIdx: number) => void): IDisposable {
|
|
200
|
+
this.onChunkAddedCbs.add(cb);
|
|
201
|
+
return { dispose: () => { this.onChunkAddedCbs.delete(cb); } };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
onAnyResize(cb: (chunkIdx: number) => void): IDisposable {
|
|
205
|
+
const disposables: IDisposable[] = [];
|
|
206
|
+
for (let i = 0; i < this._chunks.length; i++) {
|
|
207
|
+
const idx = i;
|
|
208
|
+
disposables.push(this._chunks[i]!.onResize(() => cb(idx)));
|
|
209
|
+
}
|
|
210
|
+
disposables.push(this.onChunkAdded((idx) => {
|
|
211
|
+
disposables.push(this._chunks[idx]!.onResize(() => cb(idx)));
|
|
212
|
+
cb(idx);
|
|
213
|
+
}));
|
|
214
|
+
return { dispose: () => { for (const d of disposables) d.dispose(); } };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
totalUsedElements(): number {
|
|
218
|
+
let s = 0;
|
|
219
|
+
for (const c of this._chunks) s += c.usedElements;
|
|
220
|
+
return s;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
destroy(): void {
|
|
224
|
+
for (const c of this._chunks) c.destroy();
|
|
225
|
+
this._chunks.length = 0;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
private openChunk(): number {
|
|
229
|
+
const idx = this._chunks.length;
|
|
230
|
+
const buf = new GrowBuffer(
|
|
231
|
+
this.device, `${this.label}/c${idx}`, this.usage,
|
|
232
|
+
this.initialChunkBytes, this.maxChunkBytes,
|
|
233
|
+
);
|
|
234
|
+
this._chunks.push(new IndexAllocator(buf));
|
|
235
|
+
for (const cb of this.onChunkAddedCbs) cb(idx);
|
|
236
|
+
return idx;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Freelist — best-fit free-block manager used by AttributeArena and
|
|
2
|
+
// IndexAllocator. Replaces the earlier `{off,size}[]` sorted-by-off
|
|
3
|
+
// freelist (whose alloc was O(N) first-fit and release was O(log N)
|
|
4
|
+
// binary-search-then-coalesce).
|
|
5
|
+
//
|
|
6
|
+
// Design:
|
|
7
|
+
//
|
|
8
|
+
// bySize: size → Set<off> — exact-size buckets
|
|
9
|
+
// sizes: sorted ascending array of distinct sizes — for lower_bound
|
|
10
|
+
// byStart: off → size — for right-coalesce
|
|
11
|
+
// byEnd: off+size → off — for left-coalesce
|
|
12
|
+
//
|
|
13
|
+
// `alloc(size)` → lower_bound(sizes, size) picks the smallest bucket
|
|
14
|
+
// with size ≥ requested. O(log K) lookup over distinct
|
|
15
|
+
// sizes K (typically dozens), then constant-time pop
|
|
16
|
+
// from the size's offset set. If the bucket size
|
|
17
|
+
// exceeds the request, the remainder reinserts at
|
|
18
|
+
// (off+size, leftover).
|
|
19
|
+
// `release(o,s)` → check `byEnd[o]` (left neighbour ends here) and
|
|
20
|
+
// `byStart[o+s]` (right neighbour starts here) for
|
|
21
|
+
// O(1) adjacency lookups. Coalesce both sides then
|
|
22
|
+
// reinsert one merged block.
|
|
23
|
+
//
|
|
24
|
+
// On realistic heap-renderer workloads the number of *distinct*
|
|
25
|
+
// free-block sizes is typically much smaller than the number of
|
|
26
|
+
// blocks (allocations cluster around the schema's drawHeader
|
|
27
|
+
// fields: mat4=64B, vec4=16B, …), so the K-sized sorted array's
|
|
28
|
+
// splice cost stays cheap even at thousands of live free blocks.
|
|
29
|
+
// For truly heterogeneous workloads, swap `sizes` for a balanced
|
|
30
|
+
// tree — none of the API changes.
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* A best-fit free-block manager. All offsets and sizes are in the
|
|
34
|
+
* caller's units (bytes for the attribute arena, u32 elements for
|
|
35
|
+
* the index allocator); the freelist doesn't interpret them.
|
|
36
|
+
*/
|
|
37
|
+
export class Freelist {
|
|
38
|
+
/** size → set of starting offsets currently free at that size. */
|
|
39
|
+
private readonly bySize = new Map<number, Set<number>>();
|
|
40
|
+
/** Sorted ascending list of every size present in `bySize`. */
|
|
41
|
+
private readonly sizes: number[] = [];
|
|
42
|
+
/** off → size, for the right-coalesce probe in `release`. */
|
|
43
|
+
private readonly byStart = new Map<number, number>();
|
|
44
|
+
/** off+size (= end offset) → off, for the left-coalesce probe. */
|
|
45
|
+
private readonly byEnd = new Map<number, number>();
|
|
46
|
+
|
|
47
|
+
/** Number of distinct free blocks currently tracked. */
|
|
48
|
+
get blockCount(): number { return this.byStart.size; }
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Return any offset of a free block whose size is ≥ `size`, splitting
|
|
52
|
+
* off any leftover bytes back into the freelist. Returns `undefined`
|
|
53
|
+
* when no block fits.
|
|
54
|
+
*/
|
|
55
|
+
alloc(size: number): number | undefined {
|
|
56
|
+
const idx = lowerBound(this.sizes, size);
|
|
57
|
+
if (idx >= this.sizes.length) return undefined;
|
|
58
|
+
const foundSize = this.sizes[idx]!;
|
|
59
|
+
const set = this.bySize.get(foundSize)!;
|
|
60
|
+
// Set iteration order is insertion order — using the first inserted
|
|
61
|
+
// gives deterministic alloc patterns under matched test workloads.
|
|
62
|
+
const off = set.values().next().value as number;
|
|
63
|
+
this.removeBlock(off, foundSize);
|
|
64
|
+
if (foundSize > size) this.addBlock(off + size, foundSize - size);
|
|
65
|
+
return off;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Reintroduce a `(off, size)` block, coalescing with any immediate
|
|
70
|
+
* neighbour on the left (some other block ending at `off`) or right
|
|
71
|
+
* (some other block starting at `off + size`).
|
|
72
|
+
*/
|
|
73
|
+
release(off: number, size: number): void {
|
|
74
|
+
let mergeOff = off;
|
|
75
|
+
let mergeSize = size;
|
|
76
|
+
// Left neighbour: a block whose end offset is exactly `off`.
|
|
77
|
+
const leftStart = this.byEnd.get(mergeOff);
|
|
78
|
+
if (leftStart !== undefined) {
|
|
79
|
+
const leftSize = this.byStart.get(leftStart)!;
|
|
80
|
+
this.removeBlock(leftStart, leftSize);
|
|
81
|
+
mergeOff = leftStart;
|
|
82
|
+
mergeSize += leftSize;
|
|
83
|
+
}
|
|
84
|
+
// Right neighbour: a block whose start offset is exactly `off + size`.
|
|
85
|
+
const rightStart = mergeOff + mergeSize;
|
|
86
|
+
const rightSize = this.byStart.get(rightStart);
|
|
87
|
+
if (rightSize !== undefined) {
|
|
88
|
+
this.removeBlock(rightStart, rightSize);
|
|
89
|
+
mergeSize += rightSize;
|
|
90
|
+
}
|
|
91
|
+
this.addBlock(mergeOff, mergeSize);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Take and remove the free block whose end offset is exactly
|
|
96
|
+
* `endOff`, if any exists. Used by `AttributeArena` /
|
|
97
|
+
* `IndexAllocator` to shrink the bump cursor back when releases
|
|
98
|
+
* expose a free tail (§5 — cursor-shrink hygiene for long-lived
|
|
99
|
+
* high-churn scenes).
|
|
100
|
+
*/
|
|
101
|
+
takeBlockEndingAt(endOff: number): { off: number; size: number } | undefined {
|
|
102
|
+
const off = this.byEnd.get(endOff);
|
|
103
|
+
if (off === undefined) return undefined;
|
|
104
|
+
const size = this.byStart.get(off)!;
|
|
105
|
+
this.removeBlock(off, size);
|
|
106
|
+
return { off, size };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Drop every block. */
|
|
110
|
+
clear(): void {
|
|
111
|
+
this.bySize.clear();
|
|
112
|
+
this.sizes.length = 0;
|
|
113
|
+
this.byStart.clear();
|
|
114
|
+
this.byEnd.clear();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Snapshot for tests / debugging — sorted by offset. */
|
|
118
|
+
toArray(): Array<{ off: number; size: number }> {
|
|
119
|
+
const out: Array<{ off: number; size: number }> = [];
|
|
120
|
+
for (const [off, size] of this.byStart) out.push({ off, size });
|
|
121
|
+
out.sort((a, b) => a.off - b.off);
|
|
122
|
+
return out;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
private addBlock(off: number, size: number): void {
|
|
126
|
+
let set = this.bySize.get(size);
|
|
127
|
+
if (set === undefined) {
|
|
128
|
+
set = new Set();
|
|
129
|
+
this.bySize.set(size, set);
|
|
130
|
+
const idx = lowerBound(this.sizes, size);
|
|
131
|
+
this.sizes.splice(idx, 0, size);
|
|
132
|
+
}
|
|
133
|
+
set.add(off);
|
|
134
|
+
this.byStart.set(off, size);
|
|
135
|
+
this.byEnd.set(off + size, off);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private removeBlock(off: number, size: number): void {
|
|
139
|
+
const set = this.bySize.get(size);
|
|
140
|
+
if (set === undefined) return;
|
|
141
|
+
set.delete(off);
|
|
142
|
+
if (set.size === 0) {
|
|
143
|
+
this.bySize.delete(size);
|
|
144
|
+
const idx = lowerBound(this.sizes, size);
|
|
145
|
+
if (this.sizes[idx] === size) this.sizes.splice(idx, 1);
|
|
146
|
+
}
|
|
147
|
+
this.byStart.delete(off);
|
|
148
|
+
this.byEnd.delete(off + size);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Index of the first element of `arr` that is `>= target`. */
|
|
153
|
+
function lowerBound(arr: ReadonlyArray<number>, target: number): number {
|
|
154
|
+
let lo = 0, hi = arr.length;
|
|
155
|
+
while (lo < hi) {
|
|
156
|
+
const mid = (lo + hi) >>> 1;
|
|
157
|
+
if (arr[mid]! < target) lo = mid + 1;
|
|
158
|
+
else hi = mid;
|
|
159
|
+
}
|
|
160
|
+
return lo;
|
|
161
|
+
}
|
|
@@ -26,18 +26,38 @@ export const ALIGN16 = (n: number) => (n + 15) & ~15;
|
|
|
26
26
|
* it allocates, and `ensureCapacity` grows when required. This
|
|
27
27
|
* separates allocation policy from grow policy.
|
|
28
28
|
*/
|
|
29
|
+
/**
|
|
30
|
+
* Default chunk capacity cap when none is provided. We pick 256 MB as
|
|
31
|
+
* a safety ceiling that's well within typical adapter
|
|
32
|
+
* `maxStorageBufferBindingSize` (WebGPU spec minimum is 128 MB; most
|
|
33
|
+
* desktop GPUs expose 2 GiB+, mobile/integrated ≥ 256 MB). Callers
|
|
34
|
+
* that know the adapter limit should pass it explicitly so chunks
|
|
35
|
+
* can grow to whatever the hardware supports — see §3 in
|
|
36
|
+
* `docs/heap-future-work.md` and `runtime/heapScene.ts`'s scene
|
|
37
|
+
* factory where the device limit is threaded in.
|
|
38
|
+
*/
|
|
39
|
+
export const DEFAULT_MAX_BUFFER_BYTES = 256 * 1024 * 1024;
|
|
40
|
+
|
|
29
41
|
export class GrowBuffer {
|
|
30
42
|
private buf: GPUBuffer;
|
|
31
43
|
private cap: number;
|
|
32
44
|
private used = 0;
|
|
33
45
|
private readonly listeners = new Set<() => void>();
|
|
46
|
+
private readonly maxBytes: number;
|
|
34
47
|
constructor(
|
|
35
48
|
private readonly device: GPUDevice,
|
|
36
49
|
private readonly label: string,
|
|
37
50
|
private readonly usage: GPUBufferUsageFlags,
|
|
38
51
|
initialBytes: number,
|
|
52
|
+
maxBytes: number = DEFAULT_MAX_BUFFER_BYTES,
|
|
39
53
|
) {
|
|
40
54
|
this.cap = Math.max(MIN_BUFFER_BYTES, POW2(initialBytes));
|
|
55
|
+
this.maxBytes = Math.max(this.cap, maxBytes);
|
|
56
|
+
if (this.cap > this.maxBytes) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`GrowBuffer '${label}': initial capacity ${this.cap} exceeds maxBytes ${this.maxBytes}`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
41
61
|
this.buf = device.createBuffer({
|
|
42
62
|
size: this.cap,
|
|
43
63
|
usage: usage | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC,
|
|
@@ -46,16 +66,33 @@ export class GrowBuffer {
|
|
|
46
66
|
}
|
|
47
67
|
get buffer(): GPUBuffer { return this.buf; }
|
|
48
68
|
get capacity(): number { return this.cap; }
|
|
69
|
+
get maxCapacity(): number { return this.maxBytes; }
|
|
49
70
|
get usedBytes(): number { return this.used; }
|
|
50
71
|
setUsed(n: number): void { this.used = n; }
|
|
51
72
|
onResize(cb: () => void): IDisposable {
|
|
52
73
|
this.listeners.add(cb);
|
|
53
74
|
return { dispose: () => { this.listeners.delete(cb); } };
|
|
54
75
|
}
|
|
55
|
-
/**
|
|
76
|
+
/**
|
|
77
|
+
* Ensure the buffer is at least `bytes` capacity. Grows by pow2 +
|
|
78
|
+
* copies live tail. Throws if `bytes` exceeds the configured
|
|
79
|
+
* `maxBytes` cap — caller is expected to either open a new chunk
|
|
80
|
+
* (full §3 multi-draw-call path, deferred) or reject the
|
|
81
|
+
* allocation upstream (see `isHeapEligible`'s §2 large-object
|
|
82
|
+
* eject, which keeps multi-MB single ROs out of the heap arena
|
|
83
|
+
* entirely).
|
|
84
|
+
*/
|
|
56
85
|
ensureCapacity(bytes: number): void {
|
|
57
86
|
if (bytes <= this.cap) return;
|
|
58
|
-
|
|
87
|
+
if (bytes > this.maxBytes) {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`GrowBuffer '${this.label}': requested ${bytes} bytes exceeds maxBytes ${this.maxBytes}. ` +
|
|
90
|
+
`This is the adapter's per-binding storage limit — landing larger workloads needs the ` +
|
|
91
|
+
`multi-chunk path (§3 in docs/heap-future-work.md) or upstream eviction via §2.`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
let newCap = POW2(bytes);
|
|
95
|
+
if (newCap > this.maxBytes) newCap = this.maxBytes;
|
|
59
96
|
const newBuf = this.device.createBuffer({
|
|
60
97
|
size: newCap,
|
|
61
98
|
usage: this.usage | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC,
|