@volter/blender-engine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +724 -0
- package/README.md +48 -0
- package/browser/blender-emscripten-engine.mts +289 -0
- package/browser/blender-engine.mts +412 -0
- package/browser/blender-wali-engine.mts +362 -0
- package/browser/index.ts +7 -0
- package/browser/protocol.ts +202 -0
- package/browser/rna.ts +697 -0
- package/browser/runtime.ts +511 -0
- package/browser/session-frame.mts +169 -0
- package/browser/session.py +4166 -0
- package/browser/three/agx-base-srgb.lut +0 -0
- package/browser/three/agx-look-medium-high-contrast.lut +0 -0
- package/browser/three/agx-look-punchy.lut +0 -0
- package/browser/three/attach-presenter.ts +140 -0
- package/browser/three/blender-agx.ts +235 -0
- package/browser/three/blender-base64.ts +42 -0
- package/browser/three/blender-corner-normals.ts +432 -0
- package/browser/three/blender-display-lut.ts +145 -0
- package/browser/three/blender-filmic.ts +49 -0
- package/browser/three/blender-frame-columns.ts +100 -0
- package/browser/three/blender-gradient-texture.ts +57 -0
- package/browser/three/blender-runtime-armature.ts +528 -0
- package/browser/three/blender-runtime-frame.ts +39 -0
- package/browser/three/blender-runtime-geometry.ts +342 -0
- package/browser/three/blender-runtime-lighting.ts +829 -0
- package/browser/three/blender-runtime-shadows.ts +107 -0
- package/browser/three/blender-runtime-view.ts +1481 -0
- package/browser/three/blender-runtime-volume.ts +128 -0
- package/browser/three/blender-runtime-weights.ts +306 -0
- package/browser/three/blender-sky.ts +461 -0
- package/browser/three/blender-standard.ts +68 -0
- package/browser/three/blender-triangulate.ts +181 -0
- package/browser/three/filmic-srgb.lut +0 -0
- package/browser/three/presenter.ts +265 -0
- package/browser/three/release.ts +27 -0
- package/browser/three/sky-precompute-worker.ts +45 -0
- package/browser/three/sky-worker.ts +79 -0
- package/browser/three/world-field-sampler.ts +358 -0
- package/browser/three/world-math.ts +59 -0
- package/browser/vgai_three.py +554 -0
- package/browser/worker.ts +648 -0
- package/package.json +48 -0
- package/wasm/BUNDLE.json +65 -0
- package/wasm/DEPENDENCY-LICENSES.txt +4879 -0
- package/wasm/blender_browser.data.br +0 -0
- package/wasm/blender_browser.js +2 -0
- package/wasm/blender_browser.wasm.br +0 -0
|
@@ -0,0 +1,1481 @@
|
|
|
1
|
+
/** Disposable Three.js presentation of an authoritative Python Blender session.
|
|
2
|
+
* No modeling operations or source builds run here. Stable datablock addresses
|
|
3
|
+
* retain objects/resources; changed meshes replace only their draw geometry.
|
|
4
|
+
*/
|
|
5
|
+
import * as THREE from 'three';
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
import { bytesFromBase64 } from './blender-base64';
|
|
8
|
+
import { ArmatureOverlay, armatureSchema } from './blender-runtime-armature';
|
|
9
|
+
import { UNKNOWN_GEOMETRY, UNKNOWN_IMAGE } from './blender-runtime-frame';
|
|
10
|
+
import {
|
|
11
|
+
drawArraysFromColumns,
|
|
12
|
+
drawRuntimeGeometry,
|
|
13
|
+
geometryFromDrawArrays,
|
|
14
|
+
} from './blender-runtime-geometry';
|
|
15
|
+
import {
|
|
16
|
+
aimLight,
|
|
17
|
+
buildLight,
|
|
18
|
+
fitShadow,
|
|
19
|
+
lightingReady,
|
|
20
|
+
lightSchema,
|
|
21
|
+
ViewportLighting,
|
|
22
|
+
WorldBackground,
|
|
23
|
+
worldSchema,
|
|
24
|
+
} from './blender-runtime-lighting';
|
|
25
|
+
import { fitModelDirectionalShadow, visibleShadowReceivers } from './blender-runtime-shadows';
|
|
26
|
+
import { volumeMesh, volumeSchema } from './blender-runtime-volume';
|
|
27
|
+
import { WeightOverlay, weightsSchema } from './blender-runtime-weights';
|
|
28
|
+
|
|
29
|
+
const scalar = z.number().finite();
|
|
30
|
+
const point = z.tuple([scalar, scalar, scalar]);
|
|
31
|
+
const edge = z.tuple([z.number().int().nonnegative(), z.number().int().nonnegative()]);
|
|
32
|
+
const attributeFields = { name: z.string(), domain: z.enum(['POINT', 'EDGE', 'FACE', 'CORNER']) };
|
|
33
|
+
const attributeSchema = z.discriminatedUnion('type', [
|
|
34
|
+
z
|
|
35
|
+
.object({
|
|
36
|
+
...attributeFields,
|
|
37
|
+
type: z.literal('FLOAT2'),
|
|
38
|
+
data: z.array(z.tuple([scalar, scalar])),
|
|
39
|
+
})
|
|
40
|
+
.strict(),
|
|
41
|
+
z.object({ ...attributeFields, type: z.literal('FLOAT_VECTOR'), data: z.array(point) }).strict(),
|
|
42
|
+
z
|
|
43
|
+
.object({
|
|
44
|
+
...attributeFields,
|
|
45
|
+
type: z.literal('FLOAT_COLOR'),
|
|
46
|
+
data: z.array(z.tuple([scalar, scalar, scalar, scalar])),
|
|
47
|
+
})
|
|
48
|
+
.strict(),
|
|
49
|
+
z
|
|
50
|
+
.object({
|
|
51
|
+
...attributeFields,
|
|
52
|
+
type: z.literal('BYTE_COLOR'),
|
|
53
|
+
data: z.array(z.tuple([scalar, scalar, scalar, scalar])),
|
|
54
|
+
})
|
|
55
|
+
.strict(),
|
|
56
|
+
z
|
|
57
|
+
.object({
|
|
58
|
+
...attributeFields,
|
|
59
|
+
type: z.literal('QUATERNION'),
|
|
60
|
+
data: z.array(z.tuple([scalar, scalar, scalar, scalar])),
|
|
61
|
+
})
|
|
62
|
+
.strict(),
|
|
63
|
+
z
|
|
64
|
+
.object({
|
|
65
|
+
...attributeFields,
|
|
66
|
+
type: z.literal('FLOAT4'),
|
|
67
|
+
data: z.array(z.tuple([scalar, scalar, scalar, scalar])),
|
|
68
|
+
})
|
|
69
|
+
.strict(),
|
|
70
|
+
z
|
|
71
|
+
.object({
|
|
72
|
+
...attributeFields,
|
|
73
|
+
type: z.literal('FLOAT4X4'),
|
|
74
|
+
data: z.array(z.array(scalar).length(16)),
|
|
75
|
+
})
|
|
76
|
+
.strict(),
|
|
77
|
+
z
|
|
78
|
+
.object({
|
|
79
|
+
...attributeFields,
|
|
80
|
+
type: z.literal('INT16_2D'),
|
|
81
|
+
data: z.array(z.tuple([z.number().int(), z.number().int()])),
|
|
82
|
+
})
|
|
83
|
+
.strict(),
|
|
84
|
+
z
|
|
85
|
+
.object({
|
|
86
|
+
...attributeFields,
|
|
87
|
+
type: z.literal('INT32_2D'),
|
|
88
|
+
data: z.array(z.tuple([z.number().int(), z.number().int()])),
|
|
89
|
+
})
|
|
90
|
+
.strict(),
|
|
91
|
+
z.object({ ...attributeFields, type: z.literal('FLOAT'), data: z.array(scalar) }).strict(),
|
|
92
|
+
z
|
|
93
|
+
.object({ ...attributeFields, type: z.literal('INT'), data: z.array(z.number().int()) })
|
|
94
|
+
.strict(),
|
|
95
|
+
z
|
|
96
|
+
.object({ ...attributeFields, type: z.literal('INT8'), data: z.array(z.number().int()) })
|
|
97
|
+
.strict(),
|
|
98
|
+
z.object({ ...attributeFields, type: z.literal('BOOLEAN'), data: z.array(z.boolean()) }).strict(),
|
|
99
|
+
z.object({ ...attributeFields, type: z.literal('STRING'), data: z.array(z.string()) }).strict(),
|
|
100
|
+
]);
|
|
101
|
+
const drawArraysSchema = z
|
|
102
|
+
.object({
|
|
103
|
+
positions: z.instanceof(Float32Array),
|
|
104
|
+
normals: z.instanceof(Float32Array).nullable(),
|
|
105
|
+
uv: z.instanceof(Float32Array).nullable(),
|
|
106
|
+
indices: z.instanceof(Uint32Array),
|
|
107
|
+
groups: z.array(
|
|
108
|
+
z
|
|
109
|
+
.object({
|
|
110
|
+
start: z.number().int().nonnegative(),
|
|
111
|
+
count: z.number().int().nonnegative(),
|
|
112
|
+
materialIndex: z.number().int().nonnegative(),
|
|
113
|
+
})
|
|
114
|
+
.strict(),
|
|
115
|
+
),
|
|
116
|
+
hash: z.string(),
|
|
117
|
+
})
|
|
118
|
+
.strict();
|
|
119
|
+
const meshJsonSchema = z
|
|
120
|
+
.object({
|
|
121
|
+
v: z.array(point),
|
|
122
|
+
f: z.array(z.array(z.number().int().nonnegative())),
|
|
123
|
+
e: z.array(edge).optional(),
|
|
124
|
+
edge_order: z.array(edge).optional(),
|
|
125
|
+
m: z.array(z.number().int().nonnegative()).optional(),
|
|
126
|
+
s: z.array(z.boolean()).optional(),
|
|
127
|
+
sharp: z.array(edge).optional(),
|
|
128
|
+
seams: z.array(edge).optional(),
|
|
129
|
+
creases: z.array(z.tuple([scalar, scalar, scalar])).optional(),
|
|
130
|
+
attributes: z.array(attributeSchema).optional(),
|
|
131
|
+
active_uv: z.string().nullable().optional(),
|
|
132
|
+
render_uv: z.string().nullable().optional(),
|
|
133
|
+
})
|
|
134
|
+
.strict();
|
|
135
|
+
/** A mesh the presenter already holds: the worker sends the reference rather
|
|
136
|
+
* than the columns when the store's revision has not moved since the last
|
|
137
|
+
* frame it sent (`blender-runtime-frame.ts`). A Blender session names no
|
|
138
|
+
* store -- its revision is the datablock's, accumulated from
|
|
139
|
+
* `depsgraph_update_post` -- so `store` is optional here. */
|
|
140
|
+
const unchangedMeshSchema = z
|
|
141
|
+
.object({
|
|
142
|
+
store: z.number().int().optional(),
|
|
143
|
+
revision: z.number().int(),
|
|
144
|
+
unchanged: z.literal(true),
|
|
145
|
+
})
|
|
146
|
+
.strict();
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* BLENDER'S OWN ARRAYS, as the columns the draw reads.
|
|
150
|
+
*
|
|
151
|
+
* This is the shape the export door emits -- `session.py`'s `foreach_get`
|
|
152
|
+
* reader today, the C++ door later -- and `drawArraysFromColumns` is what
|
|
153
|
+
* draws it, so the presenter takes the columns rather than a drawn mesh. The
|
|
154
|
+
* buffers arrive transferred from the worker (`session-frame.mts` reads them
|
|
155
|
+
* out of the module filesystem), which is why every one is a typed array and
|
|
156
|
+
* none is an array of numbers.
|
|
157
|
+
*/
|
|
158
|
+
const columnsSchema = z
|
|
159
|
+
.object({
|
|
160
|
+
co: z.instanceof(Float64Array),
|
|
161
|
+
faceStart: z.instanceof(Uint32Array),
|
|
162
|
+
corner: z.instanceof(Uint32Array),
|
|
163
|
+
cornerEdge: z.instanceof(Int32Array),
|
|
164
|
+
edge: z.instanceof(Uint32Array),
|
|
165
|
+
edgeSharp: z.instanceof(Uint8Array),
|
|
166
|
+
edgeSeam: z.instanceof(Uint8Array),
|
|
167
|
+
edgeCrease: z.instanceof(Float32Array),
|
|
168
|
+
material: z.instanceof(Uint32Array),
|
|
169
|
+
smooth: z.instanceof(Uint8Array),
|
|
170
|
+
vertSelect: z.instanceof(Uint8Array),
|
|
171
|
+
vertHide: z.instanceof(Uint8Array),
|
|
172
|
+
edgeSelect: z.instanceof(Uint8Array),
|
|
173
|
+
edgeHide: z.instanceof(Uint8Array),
|
|
174
|
+
faceSelect: z.instanceof(Uint8Array),
|
|
175
|
+
faceHide: z.instanceof(Uint8Array),
|
|
176
|
+
})
|
|
177
|
+
.strict();
|
|
178
|
+
const columnAttributeSchema = z
|
|
179
|
+
.object({
|
|
180
|
+
name: z.string(),
|
|
181
|
+
domain: z.enum(['POINT', 'EDGE', 'FACE', 'CORNER']),
|
|
182
|
+
type: z.string(),
|
|
183
|
+
data: z.custom<ArrayBufferView>((value) => ArrayBuffer.isView(value)),
|
|
184
|
+
})
|
|
185
|
+
.strict();
|
|
186
|
+
const exportedMeshSchema = z
|
|
187
|
+
.object({
|
|
188
|
+
columns: columnsSchema,
|
|
189
|
+
attributes: z.array(columnAttributeSchema),
|
|
190
|
+
activeUv: z.string().nullable(),
|
|
191
|
+
renderUv: z.string().nullable(),
|
|
192
|
+
counts: z
|
|
193
|
+
.object({
|
|
194
|
+
verts: z.number().int().nonnegative(),
|
|
195
|
+
edges: z.number().int().nonnegative(),
|
|
196
|
+
faces: z.number().int().nonnegative(),
|
|
197
|
+
corners: z.number().int().nonnegative(),
|
|
198
|
+
})
|
|
199
|
+
.strict(),
|
|
200
|
+
revision: z.number().int().nonnegative(),
|
|
201
|
+
})
|
|
202
|
+
.strict();
|
|
203
|
+
/** A mesh arrives drawn (the worker's buffers, keyed by the store's content
|
|
204
|
+
* hash), as the boundary JSON the node lane still sends, or as the reference
|
|
205
|
+
* above when nothing about it changed. */
|
|
206
|
+
const meshSchema = z.union([
|
|
207
|
+
drawArraysSchema,
|
|
208
|
+
exportedMeshSchema,
|
|
209
|
+
meshJsonSchema,
|
|
210
|
+
unchangedMeshSchema,
|
|
211
|
+
]);
|
|
212
|
+
/** Texture decodes still in flight. A RENDER MUST NOT START BEFORE THEY LAND:
|
|
213
|
+
* `createImageBitmap` is asynchronous, so a frame drawn in the same turn the
|
|
214
|
+
* material arrived would use an empty texture and look exactly like the
|
|
215
|
+
* untextured render this whole path exists to stop. (A RASTER needs no wait:
|
|
216
|
+
* a `DataTexture` holds its bytes the moment it is built.) */
|
|
217
|
+
const pendingTextures = new Set<Promise<void>>();
|
|
218
|
+
|
|
219
|
+
export async function texturesReady(): Promise<void> {
|
|
220
|
+
while (pendingTextures.size > 0) await Promise.all([...pendingTextures]);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** A PNG's bytes as a three texture, decoded by the browser's own decoder. */
|
|
224
|
+
function loadPngTexture(png: Uint8Array): { texture: THREE.Texture; ready: Promise<void> } {
|
|
225
|
+
const blob = new Blob([png as BlobPart], { type: 'image/png' });
|
|
226
|
+
const texture = new THREE.Texture();
|
|
227
|
+
let disposed = false;
|
|
228
|
+
texture.addEventListener('dispose', () => {
|
|
229
|
+
disposed = true;
|
|
230
|
+
texture.image?.close?.();
|
|
231
|
+
});
|
|
232
|
+
// DECODED BOTTOM ROW FIRST, because that is the row Blender's v=0 is. A PNG
|
|
233
|
+
// stores its rows top-down -- this one is a FILE-backed image's own bytes,
|
|
234
|
+
// sent as they sit on disk -- while a UV's v=0 in Blender samples the
|
|
235
|
+
// image's BOTTOM row. A bitmap decoded in the PNG's own order therefore drew
|
|
236
|
+
// every textured surface upside down: measured on `20-rigged-courier`, the
|
|
237
|
+
// courier's painted face came back with the brows below the eyes and the
|
|
238
|
+
// mouth under the hat.
|
|
239
|
+
//
|
|
240
|
+
// The flip has to be asked for HERE and cannot be asked for on the texture:
|
|
241
|
+
// `Texture.flipY` (and `premultiplyAlpha`) are `UNPACK_FLIP_Y_WEBGL` state,
|
|
242
|
+
// which WebGL ignores for an ImageBitmap source -- three says so itself at
|
|
243
|
+
// `three/src/textures/Texture.js:274` (0.180.0), "this property has no
|
|
244
|
+
// effect when using `ImageBitmap`. You need to configure the flip on bitmap
|
|
245
|
+
// creation instead."
|
|
246
|
+
const decoding = createImageBitmap(blob, { imageOrientation: 'flipY' })
|
|
247
|
+
.then((bitmap) => {
|
|
248
|
+
if (disposed) {
|
|
249
|
+
bitmap.close();
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
texture.image = bitmap;
|
|
253
|
+
texture.needsUpdate = true;
|
|
254
|
+
})
|
|
255
|
+
.finally(() => {
|
|
256
|
+
pendingTextures.delete(decoding);
|
|
257
|
+
});
|
|
258
|
+
pendingTextures.add(decoding);
|
|
259
|
+
// Keep a rejection handled even if a document closes before its first
|
|
260
|
+
// render. The owner's ready promise still reports that same failure.
|
|
261
|
+
void decoding.catch(() => {});
|
|
262
|
+
return { texture, ready: decoding };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** A RASTER as a three texture, uploaded with no decode and no flip.
|
|
266
|
+
*
|
|
267
|
+
* The bytes are Blender's own buffer in Blender's own row order -- v=0 first
|
|
268
|
+
* -- and GL's texture origin is bottom-left, so unflipped data puts v=0 where
|
|
269
|
+
* Blender puts it. That is the same place the PNG path has to FLIP to reach,
|
|
270
|
+
* because a PNG stores its rows the other way up. */
|
|
271
|
+
function rasterTexture(
|
|
272
|
+
rgba: Uint8Array,
|
|
273
|
+
width: number,
|
|
274
|
+
height: number,
|
|
275
|
+
colorspace: 'sRGB' | 'data',
|
|
276
|
+
): THREE.DataTexture {
|
|
277
|
+
const texture = new THREE.DataTexture(
|
|
278
|
+
rgba,
|
|
279
|
+
width,
|
|
280
|
+
height,
|
|
281
|
+
THREE.RGBAFormat,
|
|
282
|
+
THREE.UnsignedByteType,
|
|
283
|
+
);
|
|
284
|
+
texture.flipY = false;
|
|
285
|
+
// Blender's own colour space for the picture: an sRGB image is linearised
|
|
286
|
+
// by the sampler, a Non-Color one (a roughness map, a baked remap) is data.
|
|
287
|
+
texture.colorSpace = colorspace === 'data' ? THREE.NoColorSpace : THREE.SRGBColorSpace;
|
|
288
|
+
texture.needsUpdate = true;
|
|
289
|
+
return texture;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** One image the frame carries in full, in the only two shapes it comes in.
|
|
293
|
+
*
|
|
294
|
+
* A RASTER states its size, because a `DataTexture` cannot be built without
|
|
295
|
+
* one; a FILE's own PNG does not, because the decoder reads it off the bitmap
|
|
296
|
+
* and asking Blender would have decoded the file just to answer
|
|
297
|
+
* (`_image_values.frame_image`). Both carry the REVISION those bytes are. The
|
|
298
|
+
* `*Base64` spellings are the same bytes on a channel with no transfer list
|
|
299
|
+
* (`dispatch.mts`, `runtime_present`). */
|
|
300
|
+
const rasterImageSchema = z
|
|
301
|
+
.object({
|
|
302
|
+
width: z.number().int().positive(),
|
|
303
|
+
height: z.number().int().positive(),
|
|
304
|
+
revision: z.number().int().nonnegative(),
|
|
305
|
+
rgba: z.instanceof(Uint8Array).optional(),
|
|
306
|
+
rgbaBase64: z.string().optional(),
|
|
307
|
+
/** Blender's `colorspace_settings.name`, reduced to the two the sampler
|
|
308
|
+
* tells apart. Absent means sRGB. */
|
|
309
|
+
colorspace: z.enum(['sRGB', 'data']).optional(),
|
|
310
|
+
})
|
|
311
|
+
.strict();
|
|
312
|
+
const pngImageSchema = z
|
|
313
|
+
.object({
|
|
314
|
+
revision: z.number().int().nonnegative(),
|
|
315
|
+
png: z.instanceof(Uint8Array).optional(),
|
|
316
|
+
pngBase64: z.string().optional(),
|
|
317
|
+
})
|
|
318
|
+
.strict();
|
|
319
|
+
const frameImageSchema = z.union([rasterImageSchema, pngImageSchema]);
|
|
320
|
+
|
|
321
|
+
const textureReferenceSchema = z
|
|
322
|
+
.object({
|
|
323
|
+
image: z.object({ name: z.string(), revision: z.number().int().nonnegative() }).strict(),
|
|
324
|
+
/** Blender's `extension`; CLIP never arrives -- the reducer refuses it. */
|
|
325
|
+
extension: z.enum(['REPEAT', 'EXTEND', 'MIRROR']).default('REPEAT'),
|
|
326
|
+
uv: z.string(),
|
|
327
|
+
/** A constant the texture is multiplied by -- Blender's MULTIPLY mix at
|
|
328
|
+
* full factor, which is what `map * color` already is. */
|
|
329
|
+
tint: z.tuple([scalar, scalar, scalar]).optional(),
|
|
330
|
+
})
|
|
331
|
+
.strict();
|
|
332
|
+
const materialSchema = z
|
|
333
|
+
.object({
|
|
334
|
+
name: z.string(),
|
|
335
|
+
color: z.tuple([scalar, scalar, scalar, scalar]),
|
|
336
|
+
roughness: scalar,
|
|
337
|
+
metallic: scalar,
|
|
338
|
+
transmission: scalar,
|
|
339
|
+
ior: scalar,
|
|
340
|
+
/** A Base Color image, NAMED. Its bytes travel in the frame's `images`
|
|
341
|
+
* map, once per `(name, revision)`. */
|
|
342
|
+
texture: textureReferenceSchema.optional(),
|
|
343
|
+
/** A Roughness image, the same way; `roughness` is then the multiplier
|
|
344
|
+
* (the session sends 1). A linear Map Range on the way arrives already
|
|
345
|
+
* baked into the picture. */
|
|
346
|
+
roughness_texture: textureReferenceSchema.optional(),
|
|
347
|
+
/** Blender's emission: an Emission surface, or Principled Emission
|
|
348
|
+
* Color and Strength -- three's `emissive` and `emissiveIntensity`. */
|
|
349
|
+
emission: z
|
|
350
|
+
.object({ color: z.tuple([scalar, scalar, scalar]), strength: scalar })
|
|
351
|
+
.strict()
|
|
352
|
+
.optional(),
|
|
353
|
+
})
|
|
354
|
+
.strict();
|
|
355
|
+
/** THE FRAME CONTRACT, and the reason it is exported: the C++ export door
|
|
356
|
+
* (`bpy_web_export.cc`) emits this shape and its gate parses every frame it
|
|
357
|
+
* produces through THIS schema rather than a transcription of it. Strict in
|
|
358
|
+
* both directions -- an unrecognized key is rejected and a missing column is
|
|
359
|
+
* rejected -- so a door/presenter mismatch is a loud failure at the door's
|
|
360
|
+
* own gate instead of a quiet one on the page. A mismatch is fixed in the
|
|
361
|
+
* door, never by loosening this. */
|
|
362
|
+
export const frameSchema = z
|
|
363
|
+
.object({
|
|
364
|
+
session: z.string(),
|
|
365
|
+
revision: z.number().int().nonnegative(),
|
|
366
|
+
volumes: z.record(z.string(), volumeSchema).default({}),
|
|
367
|
+
lights: z.record(z.string(), lightSchema).default({}),
|
|
368
|
+
world: worldSchema.nullable().default(null),
|
|
369
|
+
meshes: z.record(z.string(), meshSchema),
|
|
370
|
+
materials: z.record(z.string(), materialSchema),
|
|
371
|
+
images: z.record(z.string(), frameImageSchema).default({}),
|
|
372
|
+
objects: z.array(
|
|
373
|
+
z
|
|
374
|
+
.object({
|
|
375
|
+
id: z.string(),
|
|
376
|
+
name: z.string(),
|
|
377
|
+
type: z.enum([
|
|
378
|
+
'MESH',
|
|
379
|
+
'CURVE',
|
|
380
|
+
'SURFACE',
|
|
381
|
+
'FONT',
|
|
382
|
+
'META',
|
|
383
|
+
'ARMATURE',
|
|
384
|
+
'LATTICE',
|
|
385
|
+
'VOLUME',
|
|
386
|
+
'EMPTY',
|
|
387
|
+
'CAMERA',
|
|
388
|
+
'LIGHT',
|
|
389
|
+
'SPEAKER',
|
|
390
|
+
'GREASEPENCIL',
|
|
391
|
+
'POINTCLOUD',
|
|
392
|
+
'LIGHT_PROBE',
|
|
393
|
+
]),
|
|
394
|
+
mesh: z.string().nullable(),
|
|
395
|
+
volume: z.string().nullable().optional(),
|
|
396
|
+
light: z.string().nullable().optional(),
|
|
397
|
+
materials: z.array(z.string().nullable()),
|
|
398
|
+
matrix: z.array(z.tuple([scalar, scalar, scalar, scalar])).length(4),
|
|
399
|
+
visible: z.boolean(),
|
|
400
|
+
render_visible: z.boolean().default(true),
|
|
401
|
+
selected: z.boolean(),
|
|
402
|
+
parent: z.string().nullable(),
|
|
403
|
+
})
|
|
404
|
+
.strict(),
|
|
405
|
+
),
|
|
406
|
+
active: z.string().nullable(),
|
|
407
|
+
mode: z.string(),
|
|
408
|
+
/** The scene's cameras, so a render can be framed through the one the
|
|
409
|
+
* scene names (`session.py`'s `draw_camera`). */
|
|
410
|
+
cameras: z.record(z.string(), z.record(z.string(), z.unknown())).default({}),
|
|
411
|
+
/** Capabilities the export could not reach, named. Never a refusal: the
|
|
412
|
+
* mesh or material falls back to what it would have had. */
|
|
413
|
+
warnings: z.array(z.string()).default([]),
|
|
414
|
+
/** The mesh ids this frame carries columns or a reference for. */
|
|
415
|
+
updated: z.array(z.string()).default([]),
|
|
416
|
+
/** `scene.frame_current`. */
|
|
417
|
+
frame: z.number().int().default(0),
|
|
418
|
+
/**
|
|
419
|
+
* THE INSPECTION OVERLAYS (WORK.md §Blender in the tab is Blender,
|
|
420
|
+
* "Inspection parity", I4), keyed by armature OBJECT name. What Blender's
|
|
421
|
+
* own overlay engine READS, never what it draws: per bone a pose matrix, a
|
|
422
|
+
* length, a parent and its flags, plus the armature's display type and
|
|
423
|
+
* `show_in_front`. `blender-runtime-armature.ts` is the drawing.
|
|
424
|
+
*/
|
|
425
|
+
armatures: z.record(z.string(), armatureSchema).default({}),
|
|
426
|
+
/** The active object's active vertex group, per vertex
|
|
427
|
+
* (`blender-runtime-weights.ts`); null when nothing is painted. */
|
|
428
|
+
weights: weightsSchema.nullable().default(null),
|
|
429
|
+
})
|
|
430
|
+
.strict();
|
|
431
|
+
type Frame = z.infer<typeof frameSchema>;
|
|
432
|
+
/** One render's two poses, in Blender's own (Z-up) frame. */
|
|
433
|
+
export interface PhotographRecord {
|
|
434
|
+
/** `scene.camera.matrix_world`, as `session.py::_photograph` sent it. */
|
|
435
|
+
sent: { position: number[]; target: number[]; up: number[] };
|
|
436
|
+
/** The camera the photograph was taken through, carried back through the
|
|
437
|
+
* model root's world matrix. Equal to `sent`, exactly, or the Blender ->
|
|
438
|
+
* document conversion is wrong. */
|
|
439
|
+
photographed: { position: number[]; target: number[]; up: number[] };
|
|
440
|
+
/** The render this was taken for: what was asked, not what came out. */
|
|
441
|
+
render: { width: number; height: number; fov: number; orthographic: boolean };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
export class BlenderRuntimeView {
|
|
445
|
+
readonly root = new THREE.Group();
|
|
446
|
+
private readonly objects = new Map<string, THREE.Object3D>();
|
|
447
|
+
private readonly meshes = new Map<
|
|
448
|
+
string,
|
|
449
|
+
{ signature: string; geometry: THREE.BufferGeometry }
|
|
450
|
+
>();
|
|
451
|
+
private readonly volumes = new Map<
|
|
452
|
+
string,
|
|
453
|
+
{ signature: string; mesh: ReturnType<typeof volumeMesh> }
|
|
454
|
+
>();
|
|
455
|
+
private readonly materials = new Map<string, THREE.MeshPhysicalMaterial>();
|
|
456
|
+
private readonly lights = new Map<string, THREE.Light>();
|
|
457
|
+
private readonly lighting = new ViewportLighting();
|
|
458
|
+
private readonly world = new WorldBackground();
|
|
459
|
+
/**
|
|
460
|
+
* THE INSPECTION OVERLAYS (I4). They are NOT children of {@link root}, and
|
|
461
|
+
* that is the whole design: an overlay is EDITOR FURNITURE, so its
|
|
462
|
+
* visibility belongs to the viewport's Helpers menu rather than to this
|
|
463
|
+
* presenter, and the door for that is the host's own — a document hands
|
|
464
|
+
* each group to its stage's `setHelper(kind, object)`
|
|
465
|
+
* (`@vgai/editor-sdk/host`, `EditorHostStage`), which marks it
|
|
466
|
+
* `editorHelper`, keeps it out of the hierarchy and the raycast, and turns
|
|
467
|
+
* it on and off with the kind's checkbox. `blender-runtime.document.tsx`
|
|
468
|
+
* is where they are handed over.
|
|
469
|
+
*
|
|
470
|
+
* They therefore carry the Blender → document permutation THEMSELVES
|
|
471
|
+
* ({@link overlayRoots}), because a helper is a scene-root object and does
|
|
472
|
+
* not inherit this root's matrix.
|
|
473
|
+
*/
|
|
474
|
+
private readonly armatureOverlay = new ArmatureOverlay();
|
|
475
|
+
private readonly weightOverlay = new WeightOverlay();
|
|
476
|
+
/** The two groups the stage is handed: the Helpers menu owns THEIR
|
|
477
|
+
* `visible`, and the inner group is what a RENDER stands down (an overlay
|
|
478
|
+
* is modeling chrome and never appears in a photograph — the same
|
|
479
|
+
* distinction `applyVisibility` draws for the scene's own objects). */
|
|
480
|
+
private readonly armatureRoot = new THREE.Group();
|
|
481
|
+
private readonly weightRoot = new THREE.Group();
|
|
482
|
+
private rendered = false;
|
|
483
|
+
private readonly fallback = new THREE.MeshStandardMaterial({ color: 0xb9bec6, roughness: 0.72 });
|
|
484
|
+
/** Base Color images, by image name -- ONE texture per image however many
|
|
485
|
+
* materials read it, and the cache OWNS it: a material points at one and
|
|
486
|
+
* never disposes it. Held with the size and revision the resident bytes
|
|
487
|
+
* are, because a repaint at the same size is an upload into this texture
|
|
488
|
+
* while a resize is a new one every material has to be re-pointed at. */
|
|
489
|
+
private readonly textures = new Map<
|
|
490
|
+
string,
|
|
491
|
+
{
|
|
492
|
+
texture: THREE.Texture;
|
|
493
|
+
width: number;
|
|
494
|
+
height: number;
|
|
495
|
+
revision: number;
|
|
496
|
+
png?: Uint8Array;
|
|
497
|
+
ready?: Promise<void>;
|
|
498
|
+
}
|
|
499
|
+
>();
|
|
500
|
+
/** Which image each material's `map` is currently pointing at. */
|
|
501
|
+
private readonly textureNames = new Map<string, string>();
|
|
502
|
+
/** Which runtime image each material's roughness map is, by material id. */
|
|
503
|
+
private readonly roughnessTextureNames = new Map<string, string>();
|
|
504
|
+
private frame: Frame | null = null;
|
|
505
|
+
private readonly retiredSessions = new Set<string>();
|
|
506
|
+
private geometryBuilds = 0;
|
|
507
|
+
/** WHAT THE SESSION SUBMITTED, as the description the worker sent with the
|
|
508
|
+
* frame (`@volter/blender-engine/browser/protocol.ts`): the same JSON with every
|
|
509
|
+
* column replaced by `{dtype, length, sha256}`. Kept per mesh id, and
|
|
510
|
+
* MERGED ACROSS FRAMES, because a present ships columns only for the meshes
|
|
511
|
+
* whose revision moved and a bare reference for the rest -- so the last
|
|
512
|
+
* frame alone describes almost nothing, while what is DISPLAYED is the
|
|
513
|
+
* accumulation. The map follows exactly the rule `this.meshes` follows: a
|
|
514
|
+
* description is replaced when its columns are re-sent and dropped when the
|
|
515
|
+
* frame stops naming it. */
|
|
516
|
+
private readonly submittedMeshes = new Map<string, unknown>();
|
|
517
|
+
private submitted: Record<string, unknown> | null = null;
|
|
518
|
+
private submittedSession: string | null = null;
|
|
519
|
+
/** EVERY PHOTOGRAPH THIS TAB TOOK FOR A RENDER: the pose the session sent in
|
|
520
|
+
* Blender's frame, and the pose the photograph was actually taken from,
|
|
521
|
+
* back in that same frame (`blender-runtime-host.ts`). Python asserts the
|
|
522
|
+
* two are equal on every render; this is the record a HARNESS grades, and
|
|
523
|
+
* it is written before that assertion can throw, so a conversion defect
|
|
524
|
+
* leaves both poses behind instead of only an exception. */
|
|
525
|
+
private readonly photographs: PhotographRecord[] = [];
|
|
526
|
+
/** WHO WANTS TO KNOW THE MODEL MOVED. The Properties sections read the
|
|
527
|
+
* engine through the RNA door, and a frame is the one signal in the tab
|
|
528
|
+
* that says the answers are stale — every mutation presents
|
|
529
|
+
* (`session.py::dispatch`), so a present is "re-read what you are
|
|
530
|
+
* showing". Fired after the frame is applied, so a listener that reads
|
|
531
|
+
* the graph sees the new one. */
|
|
532
|
+
private readonly frameListeners = new Set<() => void>();
|
|
533
|
+
|
|
534
|
+
/** Subscribe to frames. Returns the unsubscribe. */
|
|
535
|
+
subscribeFrames(listener: () => void): () => void {
|
|
536
|
+
this.frameListeners.add(listener);
|
|
537
|
+
return () => {
|
|
538
|
+
this.frameListeners.delete(listener);
|
|
539
|
+
};
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** The Blender datablock NAME of a presented object, or null when this
|
|
543
|
+
* object is not one of the engine's (an editor helper, a light three.js
|
|
544
|
+
* parents under one). It is a lookup by IDENTITY, never by `object.name`:
|
|
545
|
+
* the frame's own table is what makes an answer here an answer about the
|
|
546
|
+
* engine rather than about a string that happens to match. */
|
|
547
|
+
blenderObjectName(object: THREE.Object3D): string | null {
|
|
548
|
+
for (const [id, held] of this.objects)
|
|
549
|
+
if (held === object) return this.frame?.objects.find((o) => o.id === id)?.name ?? null;
|
|
550
|
+
return null;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* BLENDER'S OWN SELECTION, AS THIS FRAME CARRIES IT — the per-object
|
|
555
|
+
* `selected` flag and the scene's `active`, by Blender datablock NAME.
|
|
556
|
+
*
|
|
557
|
+
* The session exports both on every present, which is what makes the FRAME
|
|
558
|
+
* the state rather than a copy of it: an agent's `select_set` through the
|
|
559
|
+
* script door, an operator that activates what it just created, and a
|
|
560
|
+
* person's click in this editor all land in one place and come back the
|
|
561
|
+
* same way. Nothing in the tab holds a selection of its own —
|
|
562
|
+
* `blender-outliner-authoring.ts` reads this on every frame and overwrites
|
|
563
|
+
* the cache its panels render from.
|
|
564
|
+
*
|
|
565
|
+
* THE ACTIVE OBJECT IS REPORTED SEPARATELY because Blender's own state
|
|
566
|
+
* allows an active object that is not selected, and the Properties editor
|
|
567
|
+
* follows the ACTIVE one while the outline follows the selected set.
|
|
568
|
+
*/
|
|
569
|
+
blenderSelection(): { readonly selected: readonly string[]; readonly active: string | null } {
|
|
570
|
+
const frame = this.frame;
|
|
571
|
+
if (frame === null) return { selected: [], active: null };
|
|
572
|
+
return {
|
|
573
|
+
selected: frame.objects.filter((object) => object.selected).map((object) => object.name),
|
|
574
|
+
active: frame.active,
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* BLENDER'S MODE, PUBLISHED ON THE DOCUMENT'S OWN CONTEXT.
|
|
580
|
+
*
|
|
581
|
+
* `vgai.stage.mode` (U6's context key, `editor-host-door.ts`'s `stage()`) had
|
|
582
|
+
* no reporter at all: Edit Mesh went with the mesh kit and the Model document
|
|
583
|
+
* had no mode strip, so every `when` clause that reads it was answering
|
|
584
|
+
* `null`. The engine HAS a mode — `bpy.context.mode`, and `Object.mode` on the
|
|
585
|
+
* active object, which is what the Outliner's own pose rows key on
|
|
586
|
+
* (`tree_element_pose.cc`: the channels exist only in pose mode) — and the
|
|
587
|
+
* TREE door reports both. So the document publishes it here, on the context it
|
|
588
|
+
* already publishes (`publishContext(view)`), rather than through a second
|
|
589
|
+
* door: `blender-outliner-model.ts` sets it on every read, and the host's
|
|
590
|
+
* `stage()` asks the ACTIVE document's context for it.
|
|
591
|
+
*/
|
|
592
|
+
private mode: string | null = null;
|
|
593
|
+
|
|
594
|
+
setStageMode(mode: string | null): void {
|
|
595
|
+
this.mode = mode;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
stageMode(): string | null {
|
|
599
|
+
return this.mode;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** The inverse: the presented object for a Blender datablock NAME, or null
|
|
603
|
+
* when this tab is not showing one (a camera, an excluded collection's
|
|
604
|
+
* member, an object type the presenter draws nothing for). The OUTLINER
|
|
605
|
+
* needs it — its rows come from the engine by name, and every one that our
|
|
606
|
+
* viewport can select has to find its way back to a three object through
|
|
607
|
+
* the frame's own table, never through `object.name`. */
|
|
608
|
+
objectForBlenderName(name: string): THREE.Object3D | null {
|
|
609
|
+
const entry = this.frame?.objects.find((o) => o.name === name);
|
|
610
|
+
return entry === undefined ? null : (this.objects.get(entry.id) ?? null);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* Put `next` where `previous` stood — same parent, same id in the object
|
|
615
|
+
* table, same place among its siblings' children.
|
|
616
|
+
*
|
|
617
|
+
* THE ONE CALLER IS THE SKIN (`blender-runtime-skin.ts`): a mesh that turns
|
|
618
|
+
* out to be deformed by an armature has to become a `THREE.SkinnedMesh`, and
|
|
619
|
+
* three has no way to promote a `Mesh` in place. It goes through this method
|
|
620
|
+
* rather than the map directly because the table's KEY is the frame's object
|
|
621
|
+
* id and only this class knows it — a swap that missed the table would be
|
|
622
|
+
* replaced again by the next frame's reuse check, every frame, forever.
|
|
623
|
+
*
|
|
624
|
+
* A `SkinnedMesh` still answers `isMesh`, so the next frame's check
|
|
625
|
+
* (`isMesh === (obj.mesh !== null)`) keeps it and merely re-points its
|
|
626
|
+
* geometry and materials, which is exactly right.
|
|
627
|
+
*/
|
|
628
|
+
replacePresentedObject(previous: THREE.Object3D, next: THREE.Object3D): void {
|
|
629
|
+
for (const [id, object] of this.objects)
|
|
630
|
+
if (object === previous) {
|
|
631
|
+
const parent = previous.parent;
|
|
632
|
+
const children = [...previous.children];
|
|
633
|
+
previous.removeFromParent();
|
|
634
|
+
for (const child of children) next.add(child);
|
|
635
|
+
parent?.add(next);
|
|
636
|
+
this.objects.set(id, next);
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
constructor() {
|
|
642
|
+
this.root.name = 'Model';
|
|
643
|
+
// BLENDER IS Z-UP, THREE IS Y-UP: (x, y, z) -> (x, z, -y). Written as the
|
|
644
|
+
// matrix rather than `rotation.x = -Math.PI / 2`, because that Euler is a
|
|
645
|
+
// FLOAT quarter turn -- `Math.cos(-Math.PI / 2)` is 6.12e-17, not 0 -- and
|
|
646
|
+
// this matrix is the one a render inverts to answer with the pose it
|
|
647
|
+
// photographed from (`blender-runtime-host.ts`). An exact signed axis
|
|
648
|
+
// permutation inverts exactly, so that round trip is checkable with no
|
|
649
|
+
// tolerance; the float quarter turn is not, and it also leaves every model
|
|
650
|
+
// rotated by 89.999999999999996 degrees for nothing.
|
|
651
|
+
this.root.matrixAutoUpdate = false;
|
|
652
|
+
this.root.matrix.set(1, 0, 0, 0, 0, 0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1);
|
|
653
|
+
this.root.updateMatrixWorld(true);
|
|
654
|
+
// The overlays sit beside the model, not under it (see the fields), so
|
|
655
|
+
// each carries the same permutation. `matrix.copy` rather than a second
|
|
656
|
+
// literal: one spelling of the permutation, and it stays one.
|
|
657
|
+
for (const [group, overlay] of [
|
|
658
|
+
[this.armatureRoot, this.armatureOverlay.group],
|
|
659
|
+
[this.weightRoot, this.weightOverlay.group],
|
|
660
|
+
] as const) {
|
|
661
|
+
group.matrixAutoUpdate = false;
|
|
662
|
+
group.matrix.copy(this.root.matrix);
|
|
663
|
+
group.add(overlay);
|
|
664
|
+
group.updateMatrixWorld(true);
|
|
665
|
+
}
|
|
666
|
+
this.armatureRoot.name = 'BlenderBones';
|
|
667
|
+
this.weightRoot.name = 'BlenderWeights';
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* THE OVERLAY GROUPS, for the document to hand to its stage.
|
|
672
|
+
*
|
|
673
|
+
* `kind` is the `HelperVisibility` member whose checkbox owns it: `skeletons`
|
|
674
|
+
* is the Helpers menu's existing Skeletons row, which is Blender's viewport
|
|
675
|
+
* overlay "Bones" checkbox (`View3DOverlay.show_bones`,
|
|
676
|
+
* `rna_space.cc:5125-5129`; `space_view3d.py:7161`) under this editor's own
|
|
677
|
+
* noun; `weights` is a member this unit added, because nothing in the set
|
|
678
|
+
* stood for a vertex-group weight display.
|
|
679
|
+
*/
|
|
680
|
+
/**
|
|
681
|
+
* BLENDER'S SOLID-MODE STUDIO, for the document to hand to its stage as
|
|
682
|
+
* `dressing.viewLocked`.
|
|
683
|
+
*
|
|
684
|
+
* It is NOT under the model root, and that is the whole point: Blender's
|
|
685
|
+
* four solid lights are stated in VIEW space and turn with the camera, so
|
|
686
|
+
* the group belongs to the camera the stage draws with, not to the model's
|
|
687
|
+
* own frame. The stage owns that parenting and its teardown; this side only
|
|
688
|
+
* builds the lights and stands them down for a render
|
|
689
|
+
* (`blender-runtime-lighting.ts`).
|
|
690
|
+
*/
|
|
691
|
+
studioLights(): THREE.Object3D {
|
|
692
|
+
return this.lighting.group;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
overlayGroups(): readonly { kind: string; object: THREE.Object3D }[] {
|
|
696
|
+
return [
|
|
697
|
+
{ kind: 'skeletons', object: this.armatureRoot },
|
|
698
|
+
{ kind: 'weights', object: this.weightRoot },
|
|
699
|
+
];
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* Light the model the way a RENDER is lit — by the scene's own lights —
|
|
704
|
+
* rather than the way modeling is, by the studio key. `bpy.ops.render.render()`
|
|
705
|
+
* turns this on around its one capture and off again, which is the same
|
|
706
|
+
* distinction Blender draws between its solid viewport and a render.
|
|
707
|
+
*/
|
|
708
|
+
async setRendered(rendered: boolean, camera?: THREE.Camera): Promise<void> {
|
|
709
|
+
// An area light cannot be DRAWN until its lookup tables are uploaded, and a
|
|
710
|
+
// render is one photograph with no second chance at it.
|
|
711
|
+
// A sky is derived off the main thread now, so a render waits for it the
|
|
712
|
+
// same way it waits for area-light tables and image decodes: one
|
|
713
|
+
// photograph, no second chance at it.
|
|
714
|
+
this.rendered = rendered;
|
|
715
|
+
this.lighting.setRendered(rendered);
|
|
716
|
+
// The scene's world is what a render sees past the geometry AND its
|
|
717
|
+
// ambient light; modeling keeps the document's own backdrop and fill.
|
|
718
|
+
if (rendered) this.world.apply(this.root, this.frame?.world ?? null, camera);
|
|
719
|
+
else this.world.clear();
|
|
720
|
+
this.applyVisibility();
|
|
721
|
+
this.applyShadows(rendered, camera);
|
|
722
|
+
// AFTER the applies, because they are what REGISTERS the work. Awaiting
|
|
723
|
+
// first made every one of these a no-op on the first render: `apply` had
|
|
724
|
+
// not started the sky yet, so `worldReady()` saw nothing pending and the
|
|
725
|
+
// photograph went out with an empty sky texture. Measured -- a sky the page
|
|
726
|
+
// had never derived returned a render in 0.08s, which is not fast, it is
|
|
727
|
+
// wrong. One photograph, no second chance at it.
|
|
728
|
+
if (rendered) {
|
|
729
|
+
await lightingReady();
|
|
730
|
+
await Promise.all([...this.textures.values()].map((held) => held.ready));
|
|
731
|
+
await this.world.ready();
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/**
|
|
736
|
+
* Who is drawn, which is a different question in each state.
|
|
737
|
+
*
|
|
738
|
+
* MODELING reads the viewport's visibility and lights nothing the scene
|
|
739
|
+
* owns; a RENDER reads `hide_render` — the flag the Cycles path's own
|
|
740
|
+
* snapshot skipped objects by — and lights exactly the scene's lights.
|
|
741
|
+
*/
|
|
742
|
+
/**
|
|
743
|
+
* Shadows, for a render only.
|
|
744
|
+
*
|
|
745
|
+
* The renderer's shadow map is already on (`StageHost`); what is
|
|
746
|
+
* missing is anything opting into it, because MODELING wants none — a
|
|
747
|
+
* shadow across the model is in the way while it is being built, and the
|
|
748
|
+
* viewport has never cast one. So the meshes opt in for the photograph and
|
|
749
|
+
* back out afterwards, and every shadow camera is fitted to the model,
|
|
750
|
+
* whose size no default could know.
|
|
751
|
+
*/
|
|
752
|
+
private applyShadows(rendered: boolean, camera?: THREE.Camera): void {
|
|
753
|
+
this.root.updateMatrixWorld(true);
|
|
754
|
+
const boxes: THREE.Box3[] = [];
|
|
755
|
+
for (const object of this.objects.values()) {
|
|
756
|
+
const mesh = object as THREE.Mesh;
|
|
757
|
+
if (!mesh.isMesh) continue;
|
|
758
|
+
mesh.castShadow = rendered;
|
|
759
|
+
mesh.receiveShadow = rendered;
|
|
760
|
+
if (rendered && mesh.visible) {
|
|
761
|
+
const bounds = new THREE.Box3().setFromObject(mesh);
|
|
762
|
+
if (!bounds.isEmpty()) boxes.push(bounds);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
if (!rendered || !boxes.length) return;
|
|
766
|
+
const box = boxes.reduce((all, bounds) => all.union(bounds), new THREE.Box3());
|
|
767
|
+
const sphere = box.getBoundingSphere(new THREE.Sphere());
|
|
768
|
+
const frustum = camera
|
|
769
|
+
? new THREE.Frustum().setFromProjectionMatrix(
|
|
770
|
+
new THREE.Matrix4().multiplyMatrices(camera.projectionMatrix, camera.matrixWorldInverse),
|
|
771
|
+
)
|
|
772
|
+
: null;
|
|
773
|
+
const receivers = frustum
|
|
774
|
+
? boxes.flatMap((bounds) => visibleShadowReceivers(bounds, frustum))
|
|
775
|
+
: [];
|
|
776
|
+
for (const light of this.lights.values()) {
|
|
777
|
+
if ((light as THREE.DirectionalLight).isDirectionalLight && camera)
|
|
778
|
+
fitModelDirectionalShadow(light as THREE.DirectionalLight, receivers, boxes);
|
|
779
|
+
else fitShadow(light, sphere.center, sphere.radius);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* Draw the frame's overlays, and answer with whatever they could not honour.
|
|
785
|
+
*
|
|
786
|
+
* The armature drawing takes the armature object's matrix from the FRAME
|
|
787
|
+
* rather than from the presented object: a presented object is reparented
|
|
788
|
+
* and premultiplied by its parent's inverse (`applyFrame`), while a pose
|
|
789
|
+
* matrix is in the armature's own object space, so the frame's own absolute
|
|
790
|
+
* matrix is the one that composes.
|
|
791
|
+
*/
|
|
792
|
+
private applyOverlays(next: Frame): readonly string[] {
|
|
793
|
+
// THE FRAME'S OWN MATRIX, never the presented object's `matrixWorld`: an
|
|
794
|
+
// object is reparented here and premultiplied by its parent's inverse, and
|
|
795
|
+
// its world matrix also carries the model root's permutation, which both
|
|
796
|
+
// overlay groups carry themselves. Blender's absolute matrix is what
|
|
797
|
+
// composes with a pose matrix and with these groups.
|
|
798
|
+
const blenderMatrix = (name: string): THREE.Matrix4 | null => {
|
|
799
|
+
const entry = next.objects.find((obj) => obj.name === name);
|
|
800
|
+
return entry === undefined
|
|
801
|
+
? null
|
|
802
|
+
: new THREE.Matrix4().set(...(entry.matrix.flat() as Parameters<THREE.Matrix4['set']>));
|
|
803
|
+
};
|
|
804
|
+
const warnings = [...this.armatureOverlay.apply(next.armatures, blenderMatrix)];
|
|
805
|
+
const painted = next.weights
|
|
806
|
+
? (this.objects.get(
|
|
807
|
+
next.objects.find((obj) => obj.name === next.weights?.object)?.id ?? '',
|
|
808
|
+
) as THREE.Mesh | undefined)
|
|
809
|
+
: undefined;
|
|
810
|
+
const weightWarning = this.weightOverlay.apply(
|
|
811
|
+
next.weights,
|
|
812
|
+
painted ?? null,
|
|
813
|
+
next.weights ? blenderMatrix(next.weights.object) : null,
|
|
814
|
+
);
|
|
815
|
+
if (weightWarning !== null) warnings.push(weightWarning);
|
|
816
|
+
return warnings;
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
private applyVisibility(): void {
|
|
820
|
+
for (const obj of this.frame?.objects ?? []) {
|
|
821
|
+
const object = this.objects.get(obj.id);
|
|
822
|
+
if (object) object.visible = this.rendered ? obj.render_visible : obj.visible;
|
|
823
|
+
if (!obj.light) continue;
|
|
824
|
+
const light = this.lights.get(obj.light);
|
|
825
|
+
if (light) light.visible = this.rendered && obj.render_visible;
|
|
826
|
+
}
|
|
827
|
+
// AN OVERLAY IS MODELING CHROME AND IS NEVER PHOTOGRAPHED. The same
|
|
828
|
+
// distinction the loop above draws between what the viewport shows and
|
|
829
|
+
// what a render shows: bones and weight colours are drawn while the model
|
|
830
|
+
// is being looked at and stand down for the one capture. It is the INNER
|
|
831
|
+
// group that yields, never the group the Helpers menu owns — the two facts
|
|
832
|
+
// are separate and must not overwrite each other.
|
|
833
|
+
this.armatureOverlay.group.visible = !this.rendered;
|
|
834
|
+
this.weightOverlay.group.visible = !this.rendered;
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
applyFrame(input: unknown) {
|
|
838
|
+
// WHAT THIS PRESENTER HELD BEFORE THIS FRAME, read before anything is
|
|
839
|
+
// applied, and carried back to the session in the present's answer.
|
|
840
|
+
//
|
|
841
|
+
// THE PRESENTER IS THE AUTHORITY ON WHAT IT HOLDS -- the session only has a
|
|
842
|
+
// RECORD of what it sent, and the two come apart without either side
|
|
843
|
+
// failing: the worker and its Python session are the host's
|
|
844
|
+
// (`blender-runtime-host.ts` keeps one `BlenderRuntime` for the tab) while
|
|
845
|
+
// this view belongs to the Model DOCUMENT, so closing and reopening that
|
|
846
|
+
// document builds a new view holding nothing while the session's tables
|
|
847
|
+
// still say every mesh and picture crossed. The next frame then ships
|
|
848
|
+
// references to bytes that are gone, and the session learns it only from a
|
|
849
|
+
// refusal it may be unable to satisfy. Reporting the holding is what lets
|
|
850
|
+
// the session correct its own tables BEFORE it decides what to ship.
|
|
851
|
+
const held =
|
|
852
|
+
this.frame === null ? null : { session: this.frame.session, revision: this.frame.revision };
|
|
853
|
+
const next = frameSchema.parse(input);
|
|
854
|
+
if (this.retiredSessions.has(next.session))
|
|
855
|
+
throw new Error(
|
|
856
|
+
'The runtime session was replaced; its delayed frame cannot overwrite the active model',
|
|
857
|
+
);
|
|
858
|
+
if (this.frame?.session === next.session && next.revision < this.frame.revision)
|
|
859
|
+
throw new Error('The runtime frame is older than the displayed model');
|
|
860
|
+
const ids = new Set(next.objects.map((obj) => obj.id));
|
|
861
|
+
if (ids.size !== next.objects.length) throw new Error('Runtime object IDs must be unique');
|
|
862
|
+
const byId = new Map(next.objects.map((obj) => [obj.id, obj]));
|
|
863
|
+
for (const obj of next.objects) {
|
|
864
|
+
if (obj.mesh !== null && !next.meshes[obj.mesh])
|
|
865
|
+
throw new Error(`Missing runtime mesh ${obj.mesh}`);
|
|
866
|
+
if (obj.volume && (!next.volumes[obj.volume] || obj.mesh !== null))
|
|
867
|
+
throw new Error(`Invalid runtime volume ${obj.volume}`);
|
|
868
|
+
if (obj.light && (!next.lights[obj.light] || obj.mesh !== null))
|
|
869
|
+
throw new Error(`Invalid runtime light ${obj.light}`);
|
|
870
|
+
for (const mat of obj.materials)
|
|
871
|
+
if (mat !== null && !next.materials[mat])
|
|
872
|
+
throw new Error(`Missing runtime material ${mat}`);
|
|
873
|
+
const ancestors = new Set([obj.id]);
|
|
874
|
+
let parent = obj.parent;
|
|
875
|
+
while (parent !== null) {
|
|
876
|
+
if (ancestors.has(parent)) throw new Error('Runtime parent cycle');
|
|
877
|
+
ancestors.add(parent);
|
|
878
|
+
const ancestor = byId.get(parent);
|
|
879
|
+
if (!ancestor) throw new Error(`Missing runtime parent ${parent}`);
|
|
880
|
+
parent = ancestor.parent;
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
// Prepare all changed geometry before replacing any displayed resource.
|
|
884
|
+
const prepared = new Map<string, { signature: string; geometry: THREE.BufferGeometry }>();
|
|
885
|
+
const preparedVolumes = new Map<
|
|
886
|
+
string,
|
|
887
|
+
{ signature: string; mesh: ReturnType<typeof volumeMesh> }
|
|
888
|
+
>();
|
|
889
|
+
try {
|
|
890
|
+
for (const [id, data] of Object.entries(next.volumes)) {
|
|
891
|
+
const signature = JSON.stringify(data);
|
|
892
|
+
if (this.frame?.session === next.session && this.volumes.get(id)?.signature === signature)
|
|
893
|
+
continue;
|
|
894
|
+
preparedVolumes.set(id, { signature, mesh: volumeMesh(data) });
|
|
895
|
+
}
|
|
896
|
+
for (const [id, data] of Object.entries(next.meshes)) {
|
|
897
|
+
if ('unchanged' in data) {
|
|
898
|
+
// A REFERENCE, not geometry: the worker says this store has not been
|
|
899
|
+
// written since the frame it last sent, so the resident
|
|
900
|
+
// BufferGeometry stands. If it is not resident the worker's record
|
|
901
|
+
// is ahead of this presenter — a page reloaded under a still-running
|
|
902
|
+
// session — and saying so by name is what makes it re-send the frame
|
|
903
|
+
// in full (`dispatch.mts`, `runtime_present`).
|
|
904
|
+
if (this.frame?.session === next.session && this.meshes.has(id)) continue;
|
|
905
|
+
throw new Error(
|
|
906
|
+
`${UNKNOWN_GEOMETRY}: the presenter does not hold runtime mesh ${id} (store ${data.store}, revision ${data.revision})`,
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
const drawn = 'hash' in data;
|
|
910
|
+
const exported = 'columns' in data;
|
|
911
|
+
// A COLUMN FRAME'S SIGNATURE IS ITS REVISION, not a hash of its
|
|
912
|
+
// buffers: the revision is what the session accumulated from
|
|
913
|
+
// `depsgraph_update_post`, and hashing megabytes to re-derive a fact
|
|
914
|
+
// the engine already stated is the cost the revision exists to avoid.
|
|
915
|
+
const signature = exported
|
|
916
|
+
? `revision:${data.revision}`
|
|
917
|
+
: drawn
|
|
918
|
+
? data.hash
|
|
919
|
+
: JSON.stringify(data);
|
|
920
|
+
if (this.frame?.session === next.session && this.meshes.get(id)?.signature === signature)
|
|
921
|
+
continue;
|
|
922
|
+
prepared.set(id, {
|
|
923
|
+
signature,
|
|
924
|
+
geometry: exported
|
|
925
|
+
? geometryFromDrawArrays(
|
|
926
|
+
drawArraysFromColumns(
|
|
927
|
+
{
|
|
928
|
+
...data.columns,
|
|
929
|
+
attributes: data.attributes as never,
|
|
930
|
+
activeUv: data.activeUv,
|
|
931
|
+
renderUv: data.renderUv,
|
|
932
|
+
},
|
|
933
|
+
signature,
|
|
934
|
+
),
|
|
935
|
+
)
|
|
936
|
+
: drawn
|
|
937
|
+
? geometryFromDrawArrays(data)
|
|
938
|
+
: drawRuntimeGeometry(data),
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
} catch (error) {
|
|
942
|
+
for (const value of prepared.values()) value.geometry.dispose();
|
|
943
|
+
for (const { mesh } of preparedVolumes.values()) {
|
|
944
|
+
mesh.geometry.dispose();
|
|
945
|
+
mesh.material.dispose();
|
|
946
|
+
}
|
|
947
|
+
throw error;
|
|
948
|
+
}
|
|
949
|
+
if (this.frame?.session !== next.session) {
|
|
950
|
+
if (this.frame) this.retiredSessions.add(this.frame.session);
|
|
951
|
+
this.clear();
|
|
952
|
+
}
|
|
953
|
+
for (const [id, value] of preparedVolumes) {
|
|
954
|
+
const old = this.volumes.get(id)?.mesh;
|
|
955
|
+
if (old) {
|
|
956
|
+
old.removeFromParent();
|
|
957
|
+
old.geometry.dispose();
|
|
958
|
+
old.material.dispose();
|
|
959
|
+
}
|
|
960
|
+
this.volumes.set(id, value);
|
|
961
|
+
}
|
|
962
|
+
for (const [id, value] of prepared) {
|
|
963
|
+
this.meshes.get(id)?.geometry.dispose();
|
|
964
|
+
this.meshes.set(id, value);
|
|
965
|
+
this.geometryBuilds++;
|
|
966
|
+
}
|
|
967
|
+
// THE PICTURES THE FRAME BROUGHT, before any material can ask for one. An
|
|
968
|
+
// entry is here only because this session has not sent these bytes at this
|
|
969
|
+
// revision yet (`runtime_session._stage_images`); everything else the
|
|
970
|
+
// materials name is already resident, which is the whole point of the
|
|
971
|
+
// revision.
|
|
972
|
+
for (const [name, data] of Object.entries(next.images)) {
|
|
973
|
+
const held = this.textures.get(name);
|
|
974
|
+
if ('width' in data) {
|
|
975
|
+
const rgba = data.rgba ?? (data.rgbaBase64 ? bytesFromBase64(data.rgbaBase64) : undefined);
|
|
976
|
+
if (!rgba) throw new Error(`Runtime image ${name} carries no raster`);
|
|
977
|
+
// A REPAINT AT THE SAME SIZE IS AN UPLOAD, not a new texture: the
|
|
978
|
+
// bytes go into the resident image and three re-uploads it, so every
|
|
979
|
+
// material pointing at it keeps pointing at it. Native does exactly
|
|
980
|
+
// this with the imbuf.
|
|
981
|
+
if (held && held.width === data.width && held.height === data.height) {
|
|
982
|
+
const image = held.texture.image as { data?: Uint8Array };
|
|
983
|
+
if (image.data) {
|
|
984
|
+
image.data.set(rgba);
|
|
985
|
+
held.texture.needsUpdate = true;
|
|
986
|
+
held.revision = data.revision;
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
held?.texture.dispose();
|
|
991
|
+
this.textures.set(name, {
|
|
992
|
+
texture: rasterTexture(rgba, data.width, data.height, data.colorspace ?? 'sRGB'),
|
|
993
|
+
width: data.width,
|
|
994
|
+
height: data.height,
|
|
995
|
+
revision: data.revision,
|
|
996
|
+
});
|
|
997
|
+
continue;
|
|
998
|
+
}
|
|
999
|
+
// A FILE'S OWN PNG, decoded once here. Its size is the bitmap's, so the
|
|
1000
|
+
// cache records none: the next raster for this name, whatever its size,
|
|
1001
|
+
// is a rebuild rather than an upload into a texture whose dimensions
|
|
1002
|
+
// this side never learned.
|
|
1003
|
+
const png = data.png ?? (data.pngBase64 ? bytesFromBase64(data.pngBase64) : undefined);
|
|
1004
|
+
if (!png) throw new Error(`Runtime image ${name} carries no bytes`);
|
|
1005
|
+
held?.texture.dispose();
|
|
1006
|
+
const { texture, ready } = loadPngTexture(png);
|
|
1007
|
+
texture.colorSpace = THREE.SRGBColorSpace;
|
|
1008
|
+
this.textures.set(name, {
|
|
1009
|
+
texture,
|
|
1010
|
+
ready,
|
|
1011
|
+
width: 0,
|
|
1012
|
+
height: 0,
|
|
1013
|
+
revision: data.revision,
|
|
1014
|
+
png: png.slice(),
|
|
1015
|
+
});
|
|
1016
|
+
}
|
|
1017
|
+
for (const [id, data] of Object.entries(next.materials)) {
|
|
1018
|
+
const material = this.materials.get(id) ?? new THREE.MeshPhysicalMaterial();
|
|
1019
|
+
material.name = data.name;
|
|
1020
|
+
// A LINKED BASE COLOUR REPLACES THE SOCKET'S VALUE, it does not multiply
|
|
1021
|
+
// it. Python's `_reduce` fills `color` from the Base Color socket's
|
|
1022
|
+
// default even when the socket is LINKED (the socket keeps its last
|
|
1023
|
+
// constant, and native never draws it), while three multiplies `map` by
|
|
1024
|
+
// `material.color` -- so carrying that constant alongside a map tinted
|
|
1025
|
+
// every textured surface by a leftover. Measured on `20-rigged-courier`:
|
|
1026
|
+
// the painted face's skin photographed (107,59,41) against native's
|
|
1027
|
+
// (154,108,77) at the same pixel, from a texture whose own skin is
|
|
1028
|
+
// (168,118,84) on both sides.
|
|
1029
|
+
//
|
|
1030
|
+
// With a map, the only legitimate multiplier is the `tint` a MULTIPLY mix
|
|
1031
|
+
// carries; without one it is white. Only a material with NO map draws the
|
|
1032
|
+
// constant.
|
|
1033
|
+
const tint = data.texture?.tint;
|
|
1034
|
+
if (data.texture !== undefined) {
|
|
1035
|
+
material.color.setRGB(
|
|
1036
|
+
tint?.[0] ?? 1,
|
|
1037
|
+
tint?.[1] ?? 1,
|
|
1038
|
+
tint?.[2] ?? 1,
|
|
1039
|
+
THREE.LinearSRGBColorSpace,
|
|
1040
|
+
);
|
|
1041
|
+
} else {
|
|
1042
|
+
material.color.setRGB(
|
|
1043
|
+
data.color[0],
|
|
1044
|
+
data.color[1],
|
|
1045
|
+
data.color[2],
|
|
1046
|
+
THREE.LinearSRGBColorSpace,
|
|
1047
|
+
);
|
|
1048
|
+
}
|
|
1049
|
+
// Alpha is the socket's own fourth channel either way: the reduction
|
|
1050
|
+
// carries no alpha on a texture, and Blender's Alpha is a separate
|
|
1051
|
+
// Principled socket.
|
|
1052
|
+
material.opacity = data.color[3];
|
|
1053
|
+
const transparent = material.opacity < 1;
|
|
1054
|
+
if (material.transparent !== transparent) {
|
|
1055
|
+
material.transparent = transparent;
|
|
1056
|
+
material.needsUpdate = true;
|
|
1057
|
+
}
|
|
1058
|
+
material.roughness = data.roughness;
|
|
1059
|
+
material.metalness = data.metallic;
|
|
1060
|
+
material.transmission = data.transmission;
|
|
1061
|
+
material.ior = data.ior;
|
|
1062
|
+
if (data.emission) {
|
|
1063
|
+
material.emissive.setRGB(
|
|
1064
|
+
data.emission.color[0],
|
|
1065
|
+
data.emission.color[1],
|
|
1066
|
+
data.emission.color[2],
|
|
1067
|
+
THREE.LinearSRGBColorSpace,
|
|
1068
|
+
);
|
|
1069
|
+
material.emissiveIntensity = data.emission.strength;
|
|
1070
|
+
} else {
|
|
1071
|
+
material.emissive.setRGB(0, 0, 0);
|
|
1072
|
+
material.emissiveIntensity = 1;
|
|
1073
|
+
}
|
|
1074
|
+
// THE ROUGHNESS MAP, the same picture cache and the same wrap rule as
|
|
1075
|
+
// the Base Color map below; three reads its green channel, and the
|
|
1076
|
+
// session bakes a grey raster.
|
|
1077
|
+
const roughnessWanted = data.roughness_texture?.image.name ?? null;
|
|
1078
|
+
if (roughnessWanted === null) {
|
|
1079
|
+
if (this.roughnessTextureNames.has(id)) {
|
|
1080
|
+
material.roughnessMap = null;
|
|
1081
|
+
this.roughnessTextureNames.delete(id);
|
|
1082
|
+
material.needsUpdate = true;
|
|
1083
|
+
}
|
|
1084
|
+
} else {
|
|
1085
|
+
const heldRoughness = this.textures.get(roughnessWanted);
|
|
1086
|
+
if (!heldRoughness)
|
|
1087
|
+
throw new Error(
|
|
1088
|
+
`${UNKNOWN_IMAGE}: the presenter does not hold runtime image ${roughnessWanted} ` +
|
|
1089
|
+
`(revision ${data.roughness_texture!.image.revision})`,
|
|
1090
|
+
);
|
|
1091
|
+
const roughnessMap = heldRoughness.texture;
|
|
1092
|
+
const roughnessWrap =
|
|
1093
|
+
data.roughness_texture!.extension === 'EXTEND'
|
|
1094
|
+
? THREE.ClampToEdgeWrapping
|
|
1095
|
+
: data.roughness_texture!.extension === 'MIRROR'
|
|
1096
|
+
? THREE.MirroredRepeatWrapping
|
|
1097
|
+
: THREE.RepeatWrapping;
|
|
1098
|
+
if (roughnessMap.wrapS !== roughnessWrap || roughnessMap.wrapT !== roughnessWrap) {
|
|
1099
|
+
roughnessMap.wrapS = roughnessWrap;
|
|
1100
|
+
roughnessMap.wrapT = roughnessWrap;
|
|
1101
|
+
if (roughnessMap.image) roughnessMap.needsUpdate = true;
|
|
1102
|
+
}
|
|
1103
|
+
if (material.roughnessMap !== roughnessMap) {
|
|
1104
|
+
material.roughnessMap = roughnessMap;
|
|
1105
|
+
this.roughnessTextureNames.set(id, roughnessWanted);
|
|
1106
|
+
material.needsUpdate = true;
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
// THE AUTHORED TEXTURE. Without it a Base Color link rendered as the
|
|
1110
|
+
// socket's default and the image never reached a pixel -- measured as a
|
|
1111
|
+
// byte-identical render with and without one. Blender's Base Color is an
|
|
1112
|
+
// sRGB-encoded image, so the texture says so and three linearises it.
|
|
1113
|
+
const wanted = data.texture?.image.name ?? null;
|
|
1114
|
+
if (wanted === null) {
|
|
1115
|
+
if (this.textureNames.has(id)) {
|
|
1116
|
+
material.map = null;
|
|
1117
|
+
this.textureNames.delete(id);
|
|
1118
|
+
material.needsUpdate = true;
|
|
1119
|
+
}
|
|
1120
|
+
} else {
|
|
1121
|
+
// The picture belongs to the CACHE, put there by the `images` loop
|
|
1122
|
+
// above from an entry this frame carried. Not holding it is a refusal
|
|
1123
|
+
// by name, which the session answers by staging every image again
|
|
1124
|
+
// (`runtime_session.present`); the case it exists for is a page that
|
|
1125
|
+
// reloaded while its Python session kept running.
|
|
1126
|
+
const held = this.textures.get(wanted);
|
|
1127
|
+
if (!held)
|
|
1128
|
+
throw new Error(
|
|
1129
|
+
`${UNKNOWN_IMAGE}: the presenter does not hold runtime image ${wanted} ` +
|
|
1130
|
+
`(revision ${data.texture!.image.revision})`,
|
|
1131
|
+
);
|
|
1132
|
+
const map = held.texture;
|
|
1133
|
+
// REPEAT, because that is what Blender's Image Texture node does:
|
|
1134
|
+
// its `extension` defaults to REPEAT (asked the oracle), while
|
|
1135
|
+
// three's wrapping defaults to ClampToEdge. Any UV outside [0,1]
|
|
1136
|
+
// then smears the edge pixel across the whole surface, which is the
|
|
1137
|
+
// "weirdly stretched" texture the owner saw on the courtyard.
|
|
1138
|
+
//
|
|
1139
|
+
// `extension`, as the description carries it. CLIP never arrives:
|
|
1140
|
+
// the reducer refuses it by name, which is why three modes are
|
|
1141
|
+
// mapped here and not four.
|
|
1142
|
+
const wrap =
|
|
1143
|
+
data.texture!.extension === 'EXTEND'
|
|
1144
|
+
? THREE.ClampToEdgeWrapping
|
|
1145
|
+
: data.texture!.extension === 'MIRROR'
|
|
1146
|
+
? THREE.MirroredRepeatWrapping
|
|
1147
|
+
: THREE.RepeatWrapping;
|
|
1148
|
+
if (map.wrapS !== wrap || map.wrapT !== wrap) {
|
|
1149
|
+
map.wrapS = wrap;
|
|
1150
|
+
map.wrapT = wrap;
|
|
1151
|
+
// ONLY IF THERE IS AN IMAGE TO RE-UPLOAD. A wrap change is a sampler
|
|
1152
|
+
// parameter and three re-reads it when the texture's version moves,
|
|
1153
|
+
// so an already-resident map needs the bump -- but bumping one whose
|
|
1154
|
+
// `createImageBitmap` is still in flight is exactly what three warns
|
|
1155
|
+
// about, and it did, 34 times on one courtyard replay:
|
|
1156
|
+
// `THREE.WebGLRenderer: Texture marked for update but no image data
|
|
1157
|
+
// found`. The decode sets `needsUpdate` itself when the bitmap
|
|
1158
|
+
// lands, and it carries whatever wrap is on the texture by then.
|
|
1159
|
+
if (map.image) map.needsUpdate = true;
|
|
1160
|
+
}
|
|
1161
|
+
// RE-POINTED WHENEVER THE TEXTURE OBJECT CHANGED, which is how a
|
|
1162
|
+
// RESIZED image reaches a material: the cache disposes the old texture
|
|
1163
|
+
// and builds a new one. A repaint at the same size keeps this exact
|
|
1164
|
+
// object and uploads into it, so there is nothing to do here.
|
|
1165
|
+
if (material.map !== map) {
|
|
1166
|
+
material.map = map;
|
|
1167
|
+
this.textureNames.set(id, wanted);
|
|
1168
|
+
material.needsUpdate = true;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
this.materials.set(id, material);
|
|
1172
|
+
}
|
|
1173
|
+
for (const obj of next.objects) {
|
|
1174
|
+
let object = this.objects.get(obj.id);
|
|
1175
|
+
const volume = obj.volume ? this.volumes.get(obj.volume)!.mesh : null;
|
|
1176
|
+
if (
|
|
1177
|
+
!object ||
|
|
1178
|
+
(volume
|
|
1179
|
+
? object !== volume
|
|
1180
|
+
: Boolean((object as THREE.Mesh).isMesh) !== (obj.mesh !== null))
|
|
1181
|
+
) {
|
|
1182
|
+
object?.removeFromParent();
|
|
1183
|
+
object =
|
|
1184
|
+
volume ??
|
|
1185
|
+
(obj.mesh !== null
|
|
1186
|
+
? new THREE.Mesh(this.meshes.get(obj.mesh)!.geometry, this.fallback)
|
|
1187
|
+
: new THREE.Object3D());
|
|
1188
|
+
this.objects.set(obj.id, object);
|
|
1189
|
+
}
|
|
1190
|
+
object.name = obj.name;
|
|
1191
|
+
object.matrixAutoUpdate = false;
|
|
1192
|
+
object.matrix.set(...(obj.matrix.flat() as Parameters<THREE.Matrix4['set']>));
|
|
1193
|
+
if (obj.parent) {
|
|
1194
|
+
const parent = new THREE.Matrix4().set(
|
|
1195
|
+
...(byId.get(obj.parent)!.matrix.flat() as Parameters<THREE.Matrix4['set']>),
|
|
1196
|
+
);
|
|
1197
|
+
object.matrix.premultiply(parent.invert());
|
|
1198
|
+
}
|
|
1199
|
+
object.matrix.decompose(object.position, object.quaternion, object.scale);
|
|
1200
|
+
if (obj.mesh !== null) {
|
|
1201
|
+
const mesh = object as THREE.Mesh;
|
|
1202
|
+
mesh.geometry = this.meshes.get(obj.mesh)!.geometry;
|
|
1203
|
+
mesh.material = obj.materials.length
|
|
1204
|
+
? obj.materials.map((id) => (id === null ? this.fallback : this.materials.get(id)!))
|
|
1205
|
+
: this.fallback;
|
|
1206
|
+
}
|
|
1207
|
+
if (obj.light) {
|
|
1208
|
+
// The light hangs on its object's own node, so the object's matrix
|
|
1209
|
+
// aims it: a Blender light emits down its local -Z, exactly like a
|
|
1210
|
+
// camera, and `aimLight` puts the three.js target on that axis.
|
|
1211
|
+
const previous = this.lights.get(obj.light) ?? null;
|
|
1212
|
+
const light = buildLight(next.lights[obj.light]!, previous);
|
|
1213
|
+
if (previous && previous !== light) {
|
|
1214
|
+
previous.removeFromParent();
|
|
1215
|
+
previous.dispose();
|
|
1216
|
+
}
|
|
1217
|
+
if (light.parent !== object) object.add(light);
|
|
1218
|
+
aimLight(light, object);
|
|
1219
|
+
this.lights.set(obj.light, light);
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
for (const obj of next.objects) {
|
|
1223
|
+
const object = this.objects.get(obj.id)!;
|
|
1224
|
+
const parent = obj.parent === null ? this.root : this.objects.get(obj.parent)!;
|
|
1225
|
+
if (object.parent !== parent) parent.add(object);
|
|
1226
|
+
}
|
|
1227
|
+
for (const [id, object] of this.objects)
|
|
1228
|
+
if (!ids.has(id)) {
|
|
1229
|
+
object.removeFromParent();
|
|
1230
|
+
this.objects.delete(id);
|
|
1231
|
+
}
|
|
1232
|
+
for (const [id, value] of this.volumes)
|
|
1233
|
+
if (!next.volumes[id]) {
|
|
1234
|
+
value.mesh.geometry.dispose();
|
|
1235
|
+
value.mesh.material.dispose();
|
|
1236
|
+
this.volumes.delete(id);
|
|
1237
|
+
}
|
|
1238
|
+
for (const [id, value] of this.meshes)
|
|
1239
|
+
if (!next.meshes[id]) {
|
|
1240
|
+
value.geometry.dispose();
|
|
1241
|
+
this.meshes.delete(id);
|
|
1242
|
+
}
|
|
1243
|
+
for (const [id, material] of this.materials)
|
|
1244
|
+
if (!next.materials[id]) {
|
|
1245
|
+
material.dispose();
|
|
1246
|
+
this.materials.delete(id);
|
|
1247
|
+
}
|
|
1248
|
+
for (const [id, light] of this.lights)
|
|
1249
|
+
if (!next.lights[id]) {
|
|
1250
|
+
light.removeFromParent();
|
|
1251
|
+
light.dispose();
|
|
1252
|
+
this.lights.delete(id);
|
|
1253
|
+
}
|
|
1254
|
+
this.root.updateMatrixWorld(true);
|
|
1255
|
+
// Keep the frame's identity and object table, not its geometry payloads:
|
|
1256
|
+
// a presented mesh is already resident as its BufferGeometry, and the
|
|
1257
|
+
// nested number arrays it arrived as cost ~15x their JSON (the blower's
|
|
1258
|
+
// last frame held 850 MB of main-thread heap this way, measured
|
|
1259
|
+
// 2026-09-13). Reuse checks compare the per-mesh signatures kept above.
|
|
1260
|
+
this.frame = { ...next, meshes: {}, volumes: {} };
|
|
1261
|
+
// THE OVERLAYS, after the graph stands: the weight drawing is laid over
|
|
1262
|
+
// the presented mesh's own geometry and the bones over the armature
|
|
1263
|
+
// OBJECT's Blender matrix, so both need this frame's objects in place.
|
|
1264
|
+
const overlayWarnings = this.applyOverlays(next);
|
|
1265
|
+
if (overlayWarnings.length)
|
|
1266
|
+
this.frame = { ...this.frame, warnings: [...this.frame.warnings, ...overlayWarnings] };
|
|
1267
|
+
// Visibility is read off the frame, so it is applied once the frame stands.
|
|
1268
|
+
this.applyVisibility();
|
|
1269
|
+
// The model moved: whoever is READING the engine (the Properties sections
|
|
1270
|
+
// through the RNA door) re-reads now, with the new graph already standing.
|
|
1271
|
+
for (const listener of [...this.frameListeners]) listener();
|
|
1272
|
+
return { ...this.inspect(), held };
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
inspect() {
|
|
1276
|
+
return {
|
|
1277
|
+
session: this.frame?.session,
|
|
1278
|
+
revision: this.frame?.revision,
|
|
1279
|
+
active: this.frame?.active,
|
|
1280
|
+
mode: this.frame?.mode,
|
|
1281
|
+
geometryBuilds: this.geometryBuilds,
|
|
1282
|
+
objects: [...this.objects].map(([id, object]) => ({
|
|
1283
|
+
id,
|
|
1284
|
+
name: object.name,
|
|
1285
|
+
uuid: object.uuid,
|
|
1286
|
+
geometry: (object as THREE.Mesh).geometry?.uuid,
|
|
1287
|
+
visible: object.visible,
|
|
1288
|
+
})),
|
|
1289
|
+
};
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
snapshot() {
|
|
1293
|
+
return this.frame;
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
/**
|
|
1297
|
+
* A revision-owned render presenter, never a clone of the mounted scene.
|
|
1298
|
+
* Copy the resident draw arrays and image bytes synchronously, then rebuild
|
|
1299
|
+
* through the same frame contract. No live objects, callbacks, textures or
|
|
1300
|
+
* disposable geometry cross this boundary; delta-only frames are sufficient.
|
|
1301
|
+
* The caller owns this snapshot and must dispose it on every outcome.
|
|
1302
|
+
*/
|
|
1303
|
+
captureSnapshot() {
|
|
1304
|
+
const source = this.frame;
|
|
1305
|
+
if (!source) throw new Error('Blender capture requires a presented frame');
|
|
1306
|
+
// Inspection overlays are not render content. In particular weight-paint
|
|
1307
|
+
// overlays reference source vertices, which the evaluated draw no longer
|
|
1308
|
+
// needs and a render snapshot deliberately does not copy.
|
|
1309
|
+
const frame: Frame = structuredClone({ ...source, images: {}, armatures: {}, weights: null });
|
|
1310
|
+
for (const [id, { signature, geometry }] of this.meshes) {
|
|
1311
|
+
const attribute = (name: string): Float32Array<ArrayBuffer> | null => {
|
|
1312
|
+
const value = geometry.getAttribute(name);
|
|
1313
|
+
if (!value) return null;
|
|
1314
|
+
if (!(value instanceof THREE.BufferAttribute))
|
|
1315
|
+
throw new Error(`Blender capture cannot copy interleaved ${name}`);
|
|
1316
|
+
return new Float32Array(value.array);
|
|
1317
|
+
};
|
|
1318
|
+
const positions = attribute('position');
|
|
1319
|
+
if (!positions) throw new Error(`Blender capture mesh ${id} has no positions`);
|
|
1320
|
+
const index = geometry.getIndex();
|
|
1321
|
+
frame.meshes[id] = {
|
|
1322
|
+
hash: signature,
|
|
1323
|
+
positions,
|
|
1324
|
+
normals: attribute('normal'),
|
|
1325
|
+
uv: attribute('uv'),
|
|
1326
|
+
indices: index
|
|
1327
|
+
? new Uint32Array(index.array)
|
|
1328
|
+
: Uint32Array.from({ length: positions.length / 3 }, (_, i) => i),
|
|
1329
|
+
groups: geometry.groups.map((group) => ({
|
|
1330
|
+
start: group.start,
|
|
1331
|
+
count: group.count,
|
|
1332
|
+
materialIndex: group.materialIndex ?? 0,
|
|
1333
|
+
})),
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
for (const [id, { signature }] of this.volumes)
|
|
1337
|
+
frame.volumes[id] = volumeSchema.parse(JSON.parse(signature));
|
|
1338
|
+
const images = new Set(
|
|
1339
|
+
Object.values(frame.materials).flatMap((material) =>
|
|
1340
|
+
[material.texture?.image.name, material.roughness_texture?.image.name].filter(
|
|
1341
|
+
(name): name is string => name !== undefined,
|
|
1342
|
+
),
|
|
1343
|
+
),
|
|
1344
|
+
);
|
|
1345
|
+
for (const name of images) {
|
|
1346
|
+
const held = this.textures.get(name);
|
|
1347
|
+
if (!held) throw new Error(`Blender capture image ${name} is not resident`);
|
|
1348
|
+
if (held.png) {
|
|
1349
|
+
frame.images[name] = { revision: held.revision, png: held.png.slice() };
|
|
1350
|
+
} else {
|
|
1351
|
+
const data = held.texture.image?.data;
|
|
1352
|
+
if (!(data instanceof Uint8Array))
|
|
1353
|
+
throw new Error(`Blender capture image ${name} has no resident raster`);
|
|
1354
|
+
frame.images[name] = {
|
|
1355
|
+
revision: held.revision,
|
|
1356
|
+
width: held.width,
|
|
1357
|
+
height: held.height,
|
|
1358
|
+
rgba: data.slice(),
|
|
1359
|
+
colorspace: held.texture.colorSpace === THREE.NoColorSpace ? 'data' : 'sRGB',
|
|
1360
|
+
};
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
const detached = new BlenderRuntimeView();
|
|
1364
|
+
let disposed = false;
|
|
1365
|
+
const dispose = () => {
|
|
1366
|
+
if (disposed) return;
|
|
1367
|
+
disposed = true;
|
|
1368
|
+
detached.dispose();
|
|
1369
|
+
detached.root.removeFromParent();
|
|
1370
|
+
};
|
|
1371
|
+
try {
|
|
1372
|
+
detached.applyFrame(frame);
|
|
1373
|
+
this.root.updateWorldMatrix(true, false);
|
|
1374
|
+
detached.root.matrix.copy(this.root.matrixWorld);
|
|
1375
|
+
detached.root.updateMatrixWorld(true);
|
|
1376
|
+
} catch (error) {
|
|
1377
|
+
dispose();
|
|
1378
|
+
throw error;
|
|
1379
|
+
}
|
|
1380
|
+
return {
|
|
1381
|
+
root: detached.root,
|
|
1382
|
+
session: source.session,
|
|
1383
|
+
revision: source.revision,
|
|
1384
|
+
prepare: (camera: THREE.Camera) => {
|
|
1385
|
+
if (disposed) throw new Error('Blender capture snapshot is disposed');
|
|
1386
|
+
return detached.setRendered(true, camera);
|
|
1387
|
+
},
|
|
1388
|
+
dispose,
|
|
1389
|
+
};
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
/** Keep the description that arrived with this frame. Called by the tab's
|
|
1393
|
+
* Blender host right after `applyFrame`, so a refused frame records
|
|
1394
|
+
* nothing: the record must describe what is DISPLAYED. */
|
|
1395
|
+
recordPresentation(description: unknown): void {
|
|
1396
|
+
if (typeof description !== 'object' || description === null) return;
|
|
1397
|
+
const next = description as Record<string, unknown>;
|
|
1398
|
+
const meshes = next['meshes'];
|
|
1399
|
+
if (typeof meshes !== 'object' || meshes === null) return;
|
|
1400
|
+
const session = typeof next['session'] === 'string' ? next['session'] : null;
|
|
1401
|
+
if (session !== this.submittedSession) {
|
|
1402
|
+
// A NEW SESSION IS A NEW MODEL. Both records go with the old one, the
|
|
1403
|
+
// same way the displayed objects do (`clear`).
|
|
1404
|
+
this.submittedMeshes.clear();
|
|
1405
|
+
this.photographs.length = 0;
|
|
1406
|
+
this.submittedSession = session;
|
|
1407
|
+
}
|
|
1408
|
+
const held = meshes as Record<string, unknown>;
|
|
1409
|
+
for (const [id, mesh] of Object.entries(held))
|
|
1410
|
+
if (typeof mesh === 'object' && mesh !== null && !('unchanged' in mesh))
|
|
1411
|
+
this.submittedMeshes.set(id, mesh);
|
|
1412
|
+
for (const id of [...this.submittedMeshes.keys()])
|
|
1413
|
+
if (!(id in held)) this.submittedMeshes.delete(id);
|
|
1414
|
+
this.submitted = { ...next, meshes: Object.fromEntries(this.submittedMeshes) };
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
/** Keep what a render photographed. One entry per render, in order. */
|
|
1418
|
+
recordPhotograph(record: PhotographRecord): void {
|
|
1419
|
+
this.photographs.push(record);
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
/** THE OBSERVATION DOOR onto both records, for whoever is grading this tab
|
|
1423
|
+
* from outside (`packages/blender-engine/bench/battery/capture_live_model.mts`
|
|
1424
|
+
* reads it through the editor's document REPL and writes the two files a
|
|
1425
|
+
* replay compares). Nothing in the product reads it. */
|
|
1426
|
+
observations() {
|
|
1427
|
+
return { presentation: this.submitted, photographs: this.photographs };
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
private clear() {
|
|
1431
|
+
for (const light of this.lights.values()) {
|
|
1432
|
+
light.removeFromParent();
|
|
1433
|
+
light.dispose();
|
|
1434
|
+
}
|
|
1435
|
+
this.lights.clear();
|
|
1436
|
+
for (const object of this.objects.values()) object.removeFromParent();
|
|
1437
|
+
for (const value of this.meshes.values()) value.geometry.dispose();
|
|
1438
|
+
for (const material of this.materials.values()) material.dispose();
|
|
1439
|
+
for (const { mesh } of this.volumes.values()) {
|
|
1440
|
+
mesh.geometry.dispose();
|
|
1441
|
+
mesh.material.dispose();
|
|
1442
|
+
}
|
|
1443
|
+
// The pictures go with the session that sent them: the cache OWNS every
|
|
1444
|
+
// texture in it (a material only points at one), so this is the one place
|
|
1445
|
+
// they are disposed, beside the meshes for the same reason.
|
|
1446
|
+
for (const { texture } of this.textures.values()) texture.dispose();
|
|
1447
|
+
this.textures.clear();
|
|
1448
|
+
this.textureNames.clear();
|
|
1449
|
+
this.volumes.clear();
|
|
1450
|
+
this.objects.clear();
|
|
1451
|
+
this.meshes.clear();
|
|
1452
|
+
this.materials.clear();
|
|
1453
|
+
}
|
|
1454
|
+
|
|
1455
|
+
dispose() {
|
|
1456
|
+
this.clear();
|
|
1457
|
+
this.fallback.dispose();
|
|
1458
|
+
this.lighting.dispose();
|
|
1459
|
+
this.world.dispose();
|
|
1460
|
+
this.armatureOverlay.dispose();
|
|
1461
|
+
this.weightOverlay.dispose();
|
|
1462
|
+
this.frame = null;
|
|
1463
|
+
this.retiredSessions.clear();
|
|
1464
|
+
this.submittedMeshes.clear();
|
|
1465
|
+
this.submitted = null;
|
|
1466
|
+
this.submittedSession = null;
|
|
1467
|
+
this.photographs.length = 0;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
|
|
1471
|
+
/**
|
|
1472
|
+
* THE MODEL DOCUMENT'S ONE PRESENTATION, module-scoped because the Python
|
|
1473
|
+
* session outlives workspace switches and document remounts and its presented
|
|
1474
|
+
* graph must too (`blender-runtime.document.tsx` says so where it mounts it).
|
|
1475
|
+
*
|
|
1476
|
+
* It is exported so the TIMELINE can reach the same graph: binding a skeleton
|
|
1477
|
+
* is a change to the presented objects, and there is exactly one set of those.
|
|
1478
|
+
* A new Python session replaces its contents through `applyFrame`'s session
|
|
1479
|
+
* address, not by constructing a second view.
|
|
1480
|
+
*/
|
|
1481
|
+
export const blenderModelView = new BlenderRuntimeView();
|