@volter/blender-engine 0.1.49 → 0.1.51

package/browser/blender-emscripten-engine.mts

@@ -27,6 +27,7 @@
  *    `session.py` chmods its own root as its first act.
  */
 
+import { writeStreamedFile, type StreamFileSystem } from './stream-file.mts';
 import {
   artifactUrl,
   type BlenderArtifactStatus,
@@ -44,7 +45,7 @@ import sessionPython from './session.py?raw';
 
 /** What the module factory is, in the only shape this file uses. */
 interface BlenderModule {
-  FS: {
+  FS: StreamFileSystem & {
     readFile(path: string): Uint8Array;
     writeFile(path: string, data: string | Uint8Array): void;
     mkdirTree(path: string): void;
@@ -115,6 +116,7 @@ function moduleFiles(module: BlenderModule): BlenderFiles {
     readFile: async (path) => FS.readFile(path),
     // Not `async`: a write that throws throws here, so `request` never starts `.done` after a failed `.json`.
     writeFile: (path, data) => { FS.writeFile(path, data); return Promise.resolve(); },
+    writeFileStream: (path, body, size) => writeStreamedFile(FS, path, body, size),
     mkdirTree: async (path) => FS.mkdirTree(path),
     readdir: async (path) => FS.readdir(path),
     stat: async (path) => {
@@ -198,6 +200,21 @@ export async function startEmscriptenBlenderEngine(
     if (text.startsWith('@@VOLTER-READY ')) readyLine = text.slice('@@VOLTER-READY '.length);
     options.log(sessionLevel(text), text);
   };
+  // THE HEAP ITSELF, not a view of it. Blender runs on a pthread, so when its heap grows there
+  // this worker's `module.HEAPU8` keeps the length it had until the glue next refreshes it -- and
+  // no export the door calls does. MEASURED 2026-09-29 on the Stoneguard scene from
+  // game-benchmarks main: a mesh's columns at 1.14 GB were read through a 433 MB view, and the
+  // pull failed "outside the export arena". The glue builds its memory with `new
+  // WebAssembly.Memory` and exports no handle to it, so that constructor is observed while the
+  // factory runs; every read below takes a fresh view of `memory.buffer`.
+  const Memory = WebAssembly.Memory;
+  let memory: WebAssembly.Memory | null = null;
+  (WebAssembly as { Memory: unknown }).Memory = function (descriptor: WebAssembly.MemoryDescriptor) {
+    const made = new Memory(descriptor);
+    memory ??= made;
+    return made;
+  } as unknown as typeof WebAssembly.Memory;
+  const heap = (): Uint8Array => (memory ? new Uint8Array(memory.buffer) : module.HEAPU8);
   const module = await factory({
     arguments: ['--background', '--factory-startup', '--python', SESSION_SCRIPT],
     locateFile: (file: string) => artifactUrl(file),
@@ -253,6 +270,8 @@ export async function startEmscriptenBlenderEngine(
         );
       },
     ],
+  }).finally(() => {
+    (WebAssembly as { Memory: unknown }).Memory = Memory;
   });
   // `main()` runs on a pthread (`-sPROXY_TO_PTHREAD`), so the factory resolves
   // long before the session exists. The ready line is what says it does.
@@ -329,13 +348,15 @@ export async function startEmscriptenBlenderEngine(
     // A VIEW, NOT A COPY. The arena is wasm linear memory and under
     // `-sPROXY_TO_PTHREAD` that memory is SHARED, so `session-frame.mts`
     // slices every column it reads out of here; this call itself is free.
-    readArena: async () =>
-      module.HEAPU8.subarray(
-        module._blender_web_export_buffer(),
-        module._blender_web_export_buffer() + module._blender_web_export_buffer_size(),
-      ),
+    // A DIRECT ARENA (address 0) IS THE WHOLE HEAP, at its length now: the size the door reports
+    // is `emscripten_get_heap_size()`, which is Blender's pthread's own view of the heap and is as
+    // stale as this worker's was (measured: 431 MB while a column lay at 2.0 GB).
+    readArena: async () => {
+      const base = module._blender_web_export_buffer();
+      return base === 0 ? heap() : heap().subarray(base, base + module._blender_web_export_buffer_size());
+    },
     bootMs,
-    memoryBytes: () => module.HEAPU8.length,
+    memoryBytes: () => heap().length,
     releasedPayloadBytes,
   };
 }

package/browser/blender-engine.mts

@@ -85,6 +85,9 @@ export interface BlenderFileStat {
 export interface BlenderFiles {
   readFile(path: string): Promise<Uint8Array>;
   writeFile(path: string, data: string | Uint8Array): Promise<void>;
+  /** Stream a known-size file into the engine without a second whole-file JS buffer.
+   * A failed transfer leaves the previous file intact. */
+  writeFileStream?(path: string, body: ReadableStream<Uint8Array>, size: number): Promise<void>;
   mkdirTree(path: string): Promise<void>;
   readdir(path: string): Promise<string[]>;
   /** Null when the path is not there. */

package/browser/frame-transfer.mts

@@ -0,0 +1,19 @@
+/** Buffers at every depth of a frame move to the presenter exactly once. */
+export function frameTransferBuffers(frame: unknown): ArrayBuffer[] {
+  const buffers = new Set<ArrayBuffer>();
+  const visited = new Set<object>();
+  const walk = (value: unknown): void => {
+    if (value === null || typeof value !== 'object' || visited.has(value)) return;
+    visited.add(value);
+    if (value instanceof ArrayBuffer) {
+      buffers.add(value);
+    } else if (ArrayBuffer.isView(value)) {
+      // Shared Wasm memory cannot be transferred; arena readers make owned copies.
+      if (value.buffer instanceof ArrayBuffer) buffers.add(value.buffer);
+    } else {
+      for (const child of Object.values(value)) walk(child);
+    }
+  };
+  walk(frame);
+  return [...buffers];
+}

package/browser/session.py

@@ -162,6 +162,8 @@ def _mark(label):
     MEMORY_MARKS.append([label] + [round(reading.get(key, 0) / 1048576)
                                    for key in ("heap", "mallocInUse", "mallocFree", "guarded", "guardedPeak")])
     del MEMORY_MARKS[:-64]
+    # Also said, so an engine that dies of memory names in its last output the stage it reached.
+    _say("@@VOLTER-MEMORY %s heap=%d inUse=%d free=%d" % tuple(MEMORY_MARKS[-1][:4]))
 _real_stderr = sys.stderr
 
 
@@ -1611,7 +1613,7 @@ class Session:
         images = [name for name, image in frame["images"].items() if image.get("deferred")]
         if not meshes and not images:
             return False
-        ask({"hold": frame})
+        _asked({"hold": frame})
         options = {"buffer_path": EXPORT_BUFFER_PATH} if EXPORT_BUFFER_PATH else {}
         for kind, keys, door in (("mesh", meshes, _blender_web.export_mesh),
                                  ("image", images, _blender_web.export_image)):
@@ -1626,7 +1628,7 @@ class Session:
                 if piece[kind] is None:
                     del frame["images"][key]
                     continue
-                ask({kind: key, "piece": piece[kind]})
+                _asked({kind: key, "piece": piece[kind]})
         return True
 
     def _send_encoded_image(self, key, deferred):
@@ -1666,7 +1668,7 @@ class Session:
         with open(path, "wb") as fh:
             fh.write(data)
         try:
-            ask({"image": key, "piece": {
+            _asked({"image": key, "piece": {
                 "revision": deferred["revision"],
                 "mime": mime,
                 "colorspace": "sRGB" if image.colorspace_settings.name == "sRGB" else "data",
@@ -2014,6 +2016,17 @@ def _pause(since):
     time.sleep(0.0002 if time.monotonic() - since < _BRISK_SECONDS else 0.002)
 
 
+def _asked(payload):
+    """`ask`, failing as the tab failed: a piece the worker could not take comes back as
+    `{"error": ...}`, and a pull that read past it presented a frame missing that piece, refused
+    later under another name ("deferred and never pulled")."""
+    answer = ask(payload)
+    if isinstance(answer, dict) and answer.get("error"):
+        raise RuntimeError("The tab refused %s: %s" % (
+            next((key for key in ("mesh", "image", "hold") if key in payload), "an ask"), answer["error"]))
+    return answer
+
+
 def ask(payload):
     """Block until the tab answers. The worker's own thread is free while this
     waits; only the Blender pthread is held, which is where the operator is.

package/browser/stream-file.mts

@@ -0,0 +1,54 @@
+/** File imports publish only after every declared byte arrived. WasmFS stores the one final
+ * backing allocation; each write and its temporary Wasm copy are bounded to one MiB. */
+export interface StreamFileSystem {
+  open(path: string, flags: string): { fd: number };
+  close(stream: { fd: number }): void;
+  write(stream: { fd: number }, data: Uint8Array, offset: number, length: number, position: number): number;
+  truncate(path: string, length: number): void;
+  rename(from: string, to: string): void;
+  unlink(path: string): void;
+}
+
+export async function writeStreamedFile(
+  FS: StreamFileSystem,
+  path: string,
+  body: ReadableStream<Uint8Array>,
+  size: number,
+): Promise<void> {
+  if (!Number.isSafeInteger(size) || size < 0) throw new Error(`${path}: invalid file size ${size}`);
+  const temporary = `${path}.volter-stage-${crypto.randomUUID()}`;
+  const reader = body.getReader();
+  let stream: { fd: number } | undefined;
+  let complete = false;
+  try {
+    stream = FS.open(temporary, 'w');
+    // Reserve the exact backing once: growing WasmFS's vector while importing can otherwise
+    // hold both the old and the new allocation in the non-shrinking Wasm heap.
+    FS.truncate(temporary, size);
+    let position = 0;
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (position + value.byteLength > size) throw new Error(`${path}: file grew during import`);
+      for (let offset = 0; offset < value.byteLength;) {
+        const length = Math.min(1024 * 1024, value.byteLength - offset);
+        const written = FS.write(stream, value, offset, length, position);
+        if (written <= 0 || written > length) throw new Error(`${path}: invalid file write (${written})`);
+        offset += written;
+        position += written;
+      }
+    }
+    if (position !== size) throw new Error(`${path}: expected ${size} bytes, received ${position}`);
+    FS.close(stream);
+    stream = undefined;
+    FS.rename(temporary, path);
+    complete = true;
+  } finally {
+    if (!complete) await reader.cancel().catch(() => {});
+    reader.releaseLock();
+    if (stream) FS.close(stream);
+    if (!complete) {
+      try { FS.unlink(temporary); } catch { /* open may have failed before creating the file */ }
+    }
+  }
+}

package/browser/worker.ts

@@ -39,6 +39,7 @@
 
 import { type BlenderEngine, type BlenderFiles, startBlenderEngine } from './blender-engine.mts';
 import type { CaptureRequest, FileEntry, WorkerReply, WorkerRequest } from './protocol';
+import { frameTransferBuffers } from './frame-transfer.mts';
 import { documentChunks } from './document-chunks.mts';
 import { columnsToTypedArrays, describeFrame, isColumnDescriptor } from './session-frame.mts';
 
@@ -75,35 +76,24 @@ function presentToTab(
   const id = ++presentId;
   // A drawn mesh's buffers move to the tab rather than being copied: the
   // frame is the tab's from here on.
-  const transfer: ArrayBuffer[] = [];
-  // A picture travels the same way and for the same reason: an entry in the
-  // frame's `images` carries a raw RGBA raster the tab uploads into a
-  // `DataTexture` (`blender-runtime-view.ts`), once per image per revision.
-  const parts = frame as {
-    meshes?: Record<string, unknown>;
-    images?: Record<string, unknown>;
-  } | null;
-  for (const carrier of [parts?.meshes ?? {}, parts?.images ?? {}])
-    for (const held of Object.values(carrier))
-      for (const value of Object.values(held as Record<string, unknown>))
-        if (
-          ArrayBuffer.isView(value) &&
-          value.buffer instanceof ArrayBuffer &&
-          !transfer.includes(value.buffer)
-        )
-          transfer.push(value.buffer);
+  const transfer = frameTransferBuffers(frame);
   return new Promise((resolve, reject) => {
     pendingPresents.set(id, { resolve, reject });
-    (self as unknown as Worker).postMessage(
-      {
-        op: 'present',
-        id,
-        frame,
-        description,
-        ...(capture ? { capture } : {}),
-      } satisfies WorkerReply,
-      transfer,
-    );
+    try {
+      (self as unknown as Worker).postMessage(
+        {
+          op: 'present',
+          id,
+          frame,
+          description,
+          ...(capture ? { capture } : {}),
+        } satisfies WorkerReply,
+        transfer,
+      );
+    } catch (error) {
+      pendingPresents.delete(id);
+      reject(error);
+    }
   });
 }
 
@@ -535,7 +525,11 @@ async function stageProjectFiles(files_: BlenderFiles, project: string): Promise
     }
     const dir = path.slice(0, path.lastIndexOf('/'));
     if (dir) await files_.mkdirTree(dir);
-    await files_.writeFile(path, new Uint8Array(await answer.arrayBuffer()));
+    if (files_.writeFileStream && answer.body) {
+      await files_.writeFileStream(path, answer.body, file.size);
+    } else {
+      await files_.writeFile(path, new Uint8Array(await answer.arrayBuffer()));
+    }
     staged.set(path, { host: stamp, engine: await stampOf(files_, path) });
   }
 }

package/package.json

@@ -2,7 +2,7 @@
   "name": "@volter/blender-engine",
   "author": "Volter AI, Inc.",
   "license": "GPL-3.0-or-later",
-  "version": "0.1.49",
+  "version": "0.1.51",
   "publishConfig": {
     "access": "public"
   },

package/wasm/BUNDLE.json

@@ -1,12 +1,10 @@
 {
   "blender": "5.2.0 LTS",
-  "source": "1b2547991d6841a6ea4c1394b8d648086b9af6c9",
-  "reviewedSource": "1b2547991d6841a6ea4c1394b8d648086b9af6c9",
+  "source": "e5d7eec67572dfc7c73f8f4d3f601d03442546f9",
+  "reviewedSource": "e5d7eec67572dfc7c73f8f4d3f601d03442546f9",
   "upstream": "fbe6228777e7d9afefcd61a413844e790ae75db7",
   "sourceRepository": "https://github.com/volter-ai/blender",
-  "sourceRecipe": "https://github.com/volter-ai/blender/blob/1b2547991d6841a6ea4c1394b8d648086b9af6c9/build_files/emscripten/RELEASE.md",
-  "sourceRelease": "https://github.com/volter-ai/blender/releases/tag/blender-5.2.0-wasm.4",
-  "verification": "https://github.com/volter-ai/blender/releases/download/blender-5.2.0-wasm.4/VERIFICATION.json",
+  "sourceRecipe": "https://github.com/volter-ai/blender/blob/e5d7eec67572dfc7c73f8f4d3f601d03442546f9/build_files/emscripten/RELEASE.md",
   "factoryEngine": "BLENDER_EEVEE",
   "files": {
     "blender_browser.js": "raw",
@@ -19,8 +17,8 @@
       "sha256": "0c63c29e7d4b1ba1fce4e94057778b0757767b25ea4f6f01d329e99351568f30"
     },
     "blender_browser.wasm": {
-      "bytes": 86318165,
-      "sha256": "722b181cec94f1bdf2f088e0ae92f04ffa5a79c267bf5b04af7260c16e34eb0f"
+      "bytes": 86317655,
+      "sha256": "0ba94ac8388de893bcec4da70516aea69128addaa490ef27f17ca06262ee6546"
     },
     "blender_browser.data": {
       "bytes": 52698568,
@@ -33,8 +31,8 @@
       "sha256": "0c63c29e7d4b1ba1fce4e94057778b0757767b25ea4f6f01d329e99351568f30"
     },
     "blender_browser.wasm.br": {
-      "bytes": 18597884,
-      "sha256": "55ef6f9d63a4c8afa7e7aec4388d1f8c462f5294fcbba8bfbbabe5c41a13f8f9"
+      "bytes": 18596913,
+      "sha256": "64af23954e1911ffcb879f69a9e8c9bbd83b5531939b586228f44fcea8211e3b"
     },
     "blender_browser.data.br": {
       "bytes": 8056563,
@@ -45,9 +43,9 @@
       "sha256": "4085e9336266cf66f061d4d5c962a5929cc96d5732c746b5c13ea05cd104a75b"
     }
   },
-  "note": "Official Blender v5.2.0 plus our headless wasm changes and our existing export/session integration. Dependencies are rebuilt from pinned official sources. No third-party Blender port commits or build scripts are inputs. PROXY_TO_PTHREAD, WasmFS, 512 MiB initial / 4 GiB maximum memory, twelve-worker pool. Build information is disabled. The source release includes the exact source tree, materialized startup.blend, dependency archives, pure Python wheels, and notices.",
+  "note": "Unreleased Stoneguard candidate. Clean source build at the named commit; Release -O3 objects and -O2 link, Emscripten 6.0.5. Focused geometry regression matches shipped engine across original, vertex edit and subdivision-level edit. Full historical battery is absent from current editor checkout; no full-battery claim.",
   "callTrampoline": "CPython 3.13.13 is compiled with PY_CALL_TRAMPOLINE for Blender CPython slot arity casts. This applies to CPython dispatch, not arbitrary indirect calls.",
   "engineIds": "The existing session registers Python render engines under CYCLES, BLENDER_EEVEE and BLENDER_WORKBENCH; the headless build permits built-in IDs to be shadowed.",
-  "exportDoor": "The source-owned bpy_web_export.cc exports Principled physical inputs, normal images, eight named UV layers and evaluated corner normals, and takes graph_materials/graph_images/graph_generated (a graph-drawn shader mix reduced to the one Principled BSDF it reaches, graph images at the door revisions with every UDIM tile, orco for deformed meshes whose graph reads Generated coordinates). A frame can defer its changed meshes and pictures (defer) for the caller to pull one at a time (export_mesh, export_image), each call freeing the previous one's columns and a picture it decoded; an undo step keeps the revisions when Blender kept the depsgraph (session_undo); Blender's CPU subdivision descriptor is released once a mesh is read; the memory door reports malloc in-use and free. Datablock revisions are keyed by session_uid. writefile.cc writes one copy of the document per save in the browser build.",
+  "exportDoor": "The source-owned bpy_web_export.cc exports Principled physical inputs, normal images, eight named UV layers and evaluated corner normals, and takes graph_materials/graph_images/graph_generated (a graph-drawn shader mix reduced to the one Principled BSDF it reaches, graph images at the door revisions with every UDIM tile, orco for deformed meshes whose graph reads Generated coordinates). A frame can defer its changed meshes and pictures (defer) for the caller to pull one at a time (export_mesh, export_image), each call freeing the previous one's columns and a picture it decoded; an undo step keeps the revisions when Blender kept the depsgraph (session_undo); Blender's CPU subdivision descriptor is released once a mesh is read; the memory door reports malloc in-use and free. Datablock revisions are keyed by session_uid. writefile.cc writes one copy of the document per save in the browser build. CPU evaluators adopt stencil tables without cloning; browser subdivision descriptors are released immediately after CPU tessellation/deformation.",
   "preloadedPayload": "releasePreloadedFileData releases the JS preload buffer after session ready. The measured working payload exclusions are retained."
 }