@driftengine/capture 4.0.0

package/dist/delight.js

@@ -0,0 +1,351 @@
+/**
+ * A captured surface's material, separated from the light it was photographed under.
+ *
+ * ## What this is, said plainly
+ *
+ * **Inverse rendering is ill-posed and this does not pretend otherwise.** A photograph of a grey
+ * wall in shadow and a photograph of a dark wall in light are the same photograph. Nothing recovers
+ * the difference from pixels alone; what recovers it is an *assumption*, and the whole of the
+ * honesty in this file is in saying which assumptions and where they fail.
+ *
+ * **Shadow baked into albedo is the field's named failure.** A capture delit badly looks correct
+ * until it is relit — and then the old light is still painted on, a shadow follows an object that
+ * has moved, and the scene reads as flat and wrong in a way nobody can point at. Every measurement
+ * below is aimed at that one failure.
+ *
+ * **No learned model ships here, and the reason is licensing rather than quality.** The models that
+ * do this well are trained on datasets whose terms do not permit redistribution in a product, and
+ * a weight file a consumer cannot lawfully ship is worse than no weight file at all. So this is
+ * multi-view inverse rendering with a classical separation, and `tools/capture-prior/` is where a
+ * prior the project owns would be trained if one is wanted — on scenes DriftEngine itself renders,
+ * which is data nobody else has terms on.
+ *
+ * ## The two assumptions, and what each is worth
+ *
+ * **A Lambertian point looks the same from everywhere; a highlight does not.** That is what
+ * *multi-view* buys, and it buys exactly one thing: the view-dependent part. A point seen from six
+ * cameras gives six observations whose *median* rejects a highlight in any minority of them, and
+ * whose spread above that median is the specular energy — which is where the roughness estimate
+ * comes from. **What it cannot do is remove a cast shadow**, and that is worth stating because it
+ * is the assumption people expect it to carry: a shadowed point is shadowed from every camera, so
+ * every observation agrees, and agreement is not evidence of albedo.
+ *
+ * **A shadow changes intensity and not colour; a material change usually changes colour.** That is
+ * Retinex, and it is what separates a cast shadow from a dark patch of paint. Over the mesh's own
+ * edges, an edge where the intensity jumps and the chromaticity holds is called shading and the
+ * jump is given to the light; an edge where the chromaticity moves is called reflectance and the
+ * jump is given to the albedo. **Where it fails is a grey object on a grey floor** — no chromaticity
+ * to tell them apart — and that is what the confidence is for.
+ *
+ * ## What it achieves, measured
+ *
+ * On the synthetic scene in `delight.test.ts`: a cast shadow that makes one half of a floor **4.8
+ * times darker than the other in the photographs** comes back **2.2 times darker in the albedo** —
+ * half of it removed, in the logarithm. A matte floor's albedo lands within about a tenth of the
+ * truth, and the whole scene relit under a light it never saw is within a tenth of a channel of the
+ * renderer's own answer.
+ *
+ * **Half a shadow is a real result and not a finished one**, and the number is written here rather
+ * than rounded up because it is the number a learned prior would have to beat to be worth its
+ * weights. What is left is the part a chromaticity cannot see: where a shadow's edge falls inside a
+ * single material, the only thing separating shading from paint is a *prior about scenes*, and this
+ * has none.
+ *
+ * ## The confidence is the point, not a decoration
+ *
+ * Every surface here gets a number saying how much of this to believe, and the test that makes it
+ * mean something **correlates it against the actual error**. A channel that does not track the error
+ * is worse than none, because a consumer trusts it. Low confidence is the honest answer for glass,
+ * for a surface one camera saw, and for a grey object on a grey floor — and a pipeline that knows
+ * which surfaces those are can leave them alone rather than delighting them wrongly.
+ */
+import { exactExp, exactLog } from '@driftengine/core';
+import { cameraCentre, nearestHit } from './delightScene.js';
+const DEFAULT_MINIMUM_VIEWS = 2;
+const DEFAULT_CHROMATICITY_STEP = 0.05;
+const DEFAULT_ITERATIONS = 120;
+/** Where in the sorted shading the scene's "fully lit" is taken to be. */
+const ANCHOR_PERCENTILE = 0.9;
+/** Views past which another one adds little: four sample a hemisphere well enough to reject by. */
+const ENOUGH_VIEWS = 4;
+export function createDelightOut(vertices) {
+    return {
+        albedo: new Float32Array(vertices * 3),
+        roughness: new Float32Array(vertices),
+        normal: new Float32Array(vertices * 3),
+        confidence: new Float32Array(vertices),
+    };
+}
+/** `mesh`'s material, as seen in `views`. */
+export function delight(views, mesh, options, out) {
+    const vertices = mesh.positions.length / 3;
+    const minimumViews = options.minimumViews ?? DEFAULT_MINIMUM_VIEWS;
+    const step = options.chromaticityStep ?? DEFAULT_CHROMATICITY_STEP;
+    const iterations = options.iterations ?? DEFAULT_ITERATIONS;
+    out.normal.set(mesh.normals.subarray(0, vertices * 3));
+    /* What each view saw of each vertex, where it saw it at all. */
+    const observed = new Float64Array(vertices * 3);
+    const seen = new Int32Array(vertices);
+    const spread = new Float64Array(vertices);
+    gather(views, mesh, observed, seen, spread);
+    /*
+     * The intensity a point returned and the colour it returned it in, kept apart. Retinex works on
+     * the first and leaves the second alone, because a shadow is the one and a material is the other.
+     */
+    const logObserved = new Float64Array(vertices * 3);
+    const chroma = new Float64Array(vertices * 3);
+    for (let vertex = 0; vertex < vertices; vertex += 1) {
+        let total = 0;
+        for (let c = 0; c < 3; c += 1)
+            total += observed[vertex * 3 + c];
+        for (let c = 0; c < 3; c += 1) {
+            logObserved[vertex * 3 + c] = exactLog(Math.max(observed[vertex * 3 + c], 1e-4));
+            chroma[vertex * 3 + c] = observed[vertex * 3 + c] / Math.max(total, 1e-6);
+        }
+    }
+    /*
+     * **One solve per channel, sharing one classification.** An intensity-only separation cannot tell
+     * a warm key light from a neutral ambient: a surface in shadow is lit by one and out of it by the
+     * other, so its recovered *colour* carries the difference even when its brightness does not.
+     * Measured on the synthetic scene, that alone was most of what was left — the shadowed and lit
+     * halves of one floor came back 35% apart. The decision about which edges are shading is still
+     * made once, on chromaticity, because that is what a shadow does not change.
+     */
+    const channelShading = [];
+    const channel = new Float64Array(vertices);
+    for (let c = 0; c < 3; c += 1) {
+        for (let vertex = 0; vertex < vertices; vertex += 1) {
+            channel[vertex] = logObserved[vertex * 3 + c];
+        }
+        channelShading.push(solveShading(mesh, channel, chroma, step, iterations, seen, minimumViews));
+    }
+    for (let vertex = 0; vertex < vertices; vertex += 1) {
+        /* What is left of each channel once its own light is taken out. */
+        for (let c = 0; c < 3; c += 1) {
+            const shading = channelShading[c][vertex];
+            const value = exactExp(logObserved[vertex * 3 + c] - shading);
+            out.albedo[vertex * 3 + c] = value > 1 ? 1 : value;
+        }
+        /*
+         * **The spread above the median is specular energy, and a wider lobe means a rougher surface.**
+         * A mirror puts all of it in one view and a chalk surface in none, so the roughness falls as
+         * the disagreement rises. Coarse by construction: six views sample a lobe badly, and a number
+         * that says "rough" or "polished" is what a capture can honestly carry.
+         */
+        const excess = spread[vertex];
+        out.roughness[vertex] = Math.min(1, Math.max(0.05, 1 - Math.min(1, excess * 4)));
+        out.confidence[vertex] = confidenceAt(seen[vertex], minimumViews, excess, views.length);
+    }
+}
+/**
+ * Every view's opinion of every vertex: the median colour, how many saw it, and their disagreement.
+ *
+ * **Visibility is a ray back to the camera against the mesh itself.** A vertex behind a wall is one
+ * the camera never saw, and reading the pixel it projects to reads the wall's colour as the
+ * surface's — which is not a subtle error, it is another object's paint.
+ */
+function gather(views, mesh, observed, seen, spread) {
+    const vertices = mesh.positions.length / 3;
+    const samples = [];
+    for (let vertex = 0; vertex < vertices; vertex += 1)
+        samples.push([]);
+    for (const view of views) {
+        const { width, height, intrinsics } = view.camera;
+        const [fx, fy, cx, cy] = intrinsics;
+        const m = view.camera.worldToCamera;
+        const eye = cameraCentre(m);
+        for (let vertex = 0; vertex < vertices; vertex += 1) {
+            const point = [
+                mesh.positions[vertex * 3],
+                mesh.positions[vertex * 3 + 1],
+                mesh.positions[vertex * 3 + 2],
+            ];
+            /* Facing away from the camera is not seen, whatever the depth says. */
+            let facing = 0;
+            for (let k = 0; k < 3; k += 1) {
+                facing += mesh.normals[vertex * 3 + k] * (eye[k] - point[k]);
+            }
+            if (!(facing > 0))
+                continue;
+            let viewZ = m[11];
+            let viewX = m[3];
+            let viewY = m[7];
+            for (let k = 0; k < 3; k += 1) {
+                viewX += m[k] * point[k];
+                viewY += m[4 + k] * point[k];
+                viewZ += m[8 + k] * point[k];
+            }
+            if (!(viewZ > 0))
+                continue;
+            const px = Math.floor((fx * viewX) / viewZ + cx);
+            const py = Math.floor((fy * viewY) / viewZ + cy);
+            if (px < 0 || px >= width || py < 0 || py >= height)
+                continue;
+            /* Occluded by the surface itself: the ray to the camera meets something first. */
+            const toEye = [
+                eye[0] - point[0],
+                eye[1] - point[1],
+                eye[2] - point[2],
+            ];
+            const distance = Math.sqrt(toEye[0] * toEye[0] + toEye[1] * toEye[1] + toEye[2] * toEye[2]);
+            const direction = [
+                toEye[0] / distance,
+                toEye[1] / distance,
+                toEye[2] / distance,
+            ];
+            if (nearestHit(mesh, point, direction, 1e-3, distance - 1e-3) !== null)
+                continue;
+            const pixel = (py * width + px) * 4;
+            samples[vertex].push(view.frame[pixel], view.frame[pixel + 1], view.frame[pixel + 2]);
+        }
+    }
+    for (let vertex = 0; vertex < vertices; vertex += 1) {
+        const held = samples[vertex];
+        const count = held.length / 3;
+        seen[vertex] = count;
+        if (count === 0)
+            continue;
+        /* The median a channel at a time: a highlight in a minority of views is rejected by it. */
+        for (let c = 0; c < 3; c += 1) {
+            const column = [];
+            for (let at = 0; at < count; at += 1)
+                column.push(held[at * 3 + c]);
+            column.sort((a, b) => a - b);
+            observed[vertex * 3 + c] = column[Math.floor((count - 1) / 2)];
+        }
+        /*
+         * **How far the views stand from their own median, in either direction.** A view brighter than
+         * the rest is a highlight and one darker is an occlusion or an inter-reflection, and both say
+         * the same thing about the answer: the median is standing in for observations that did not
+         * agree. The mean of those distances and the worst of them were both measured here and are
+         * interchangeable on these scenes, so it is the mean — the plainer statistic, and the one that
+         * does not turn on a single camera.
+         */
+        let apart = 0;
+        for (let at = 0; at < count; at += 1) {
+            for (let c = 0; c < 3; c += 1) {
+                apart += Math.abs(held[at * 3 + c] - observed[vertex * 3 + c]);
+            }
+        }
+        spread[vertex] = apart / (count * 3);
+    }
+}
+/**
+ * The light, as a field over the mesh that explains every intensity jump a colour jump does not.
+ *
+ * A sweep at a time: each vertex takes the average of what its neighbours imply, where a shading
+ * edge implies the neighbour's shading plus the observed jump and a reflectance edge implies the
+ * neighbour's shading unchanged.
+ *
+ * **The constant is assumed rather than solved for, and which assumption matters.** Nothing in a
+ * photograph says whether the light was bright and the paint dark or the other way round, so one
+ * end has to be fixed. The textbook choice is *the brightest surface is white*, and it is wrong for
+ * a capture: a room of brown floor and blue wall has no white in it, and the recovery comes back
+ * about sixty per cent too bright — measured on the synthetic scene, which then failed its
+ * relighting gate by a third of a channel. What is assumed here instead is **the brightest
+ * illumination in the scene is full illumination**, which is a statement about the light rather
+ * than about the paint, and is true of any scene with a surface facing the light.
+ */
+function solveShading(mesh, logIntensity, chroma, step, iterations, seen, minimumViews) {
+    const vertices = logIntensity.length;
+    const neighbours = adjacency(mesh, vertices);
+    const shading = new Float64Array(vertices);
+    const next = new Float64Array(vertices);
+    for (let sweep = 0; sweep < iterations; sweep += 1) {
+        for (let vertex = 0; vertex < vertices; vertex += 1) {
+            const around = neighbours[vertex];
+            if (around.length === 0) {
+                next[vertex] = shading[vertex];
+                continue;
+            }
+            let total = 0;
+            for (const other of around) {
+                let apart = 0;
+                for (let c = 0; c < 3; c += 1) {
+                    apart += Math.abs(chroma[vertex * 3 + c] - chroma[other * 3 + c]);
+                }
+                const materialChange = apart > step;
+                const jump = materialChange
+                    ? 0
+                    : logIntensity[vertex] - logIntensity[other];
+                total += shading[other] + jump;
+            }
+            next[vertex] = total / around.length;
+        }
+        shading.set(next);
+    }
+    /*
+     * The most-lit surface anybody saw properly is taken as fully lit, which fixes the constant.
+     *
+     * **At a high percentile rather than at the maximum**, and that choice is the load-bearing part:
+     * one vertex is enough to set a maximum, and the one that sets it is whichever caught a highlight
+     * the views could not average away — the whole scene is then scaled against that vertex.
+     * Measured on the synthetic scene, a block's four-vertex top did exactly that and left a matte
+     * floor half again too bright.
+     *
+     * **The shift itself is usually small**, because the solve starts from zero everywhere and its
+     * iteration has no absolute term, so it settles around a shading of nought without being told to.
+     * That is an accident of the method rather than a decision, which is why the constant is set
+     * here explicitly instead of being left to it.
+     */
+    const levels = [];
+    for (let vertex = 0; vertex < vertices; vertex += 1) {
+        if (seen[vertex] < minimumViews)
+            continue;
+        levels.push(shading[vertex]);
+    }
+    if (levels.length > 0) {
+        levels.sort((a, b) => a - b);
+        const brightest = levels[Math.floor((levels.length - 1) * ANCHOR_PERCENTILE)];
+        for (let vertex = 0; vertex < vertices; vertex += 1) {
+            shading[vertex] = shading[vertex] - brightest;
+        }
+    }
+    return shading;
+}
+/** Which vertices share an edge with which, by position so a split seam is still one surface. */
+function adjacency(mesh, vertices) {
+    const out = [];
+    for (let vertex = 0; vertex < vertices; vertex += 1)
+        out.push([]);
+    const triangles = mesh.indices.length / 3;
+    const seen = new Set();
+    for (let face = 0; face < triangles; face += 1) {
+        for (let slot = 0; slot < 3; slot += 1) {
+            const a = mesh.indices[face * 3 + slot];
+            const b = mesh.indices[face * 3 + ((slot + 1) % 3)];
+            if (a === b)
+                continue;
+            const key = a < b ? a * vertices + b : b * vertices + a;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            out[a].push(b);
+            out[b].push(a);
+        }
+    }
+    return out;
+}
+/**
+ * How much of this vertex to believe.
+ *
+ * Three things make an answer weak and they are multiplied rather than averaged, because any one of
+ * them alone is enough to make the answer wrong: too few views to reject anything, a surface whose
+ * views disagree wildly — glass, a mirror, water — and a surface nobody saw at all.
+ */
+function confidenceAt(seen, minimumViews, excess, views) {
+    if (seen === 0)
+        return 0;
+    if (seen < minimumViews)
+        return 0.1;
+    /*
+     * Every view that saw it is evidence, with diminishing returns past a handful — **counted
+     * absolutely, not against how many cameras there happened to be**. Dividing by the number
+     * supplied made two views of two look exactly as good as seven of seven, which is the opposite
+     * of what the number is for: a point two cameras saw is one where nothing could be rejected.
+     */
+    const coverage = Math.min(1, seen / ENOUGH_VIEWS);
+    /* And disagreement across views is the tell for a surface that is not diffuse at all. */
+    const agreement = 1 / (1 + excess * 12);
+    return Math.max(0, Math.min(1, coverage * agreement));
+}

package/dist/delightScene.d.ts

@@ -0,0 +1,37 @@
+import type { MeshData } from '@driftengine/drft';
+import type { RasterCamera } from './gaussians/project.ts';
+export interface LitScene {
+    readonly mesh: MeshData;
+    /** Three per vertex, linear: what the recovery is measured against. */
+    readonly albedo: Float32Array;
+    /** One per vertex: the same. */
+    readonly roughness: Float32Array;
+}
+export interface Light {
+    /** Towards the light, unit length. */
+    readonly direction: readonly [number, number, number];
+    readonly colour: readonly [number, number, number];
+    /** What arrives from everywhere else, so a shadowed point is dark rather than black. */
+    readonly ambient: number;
+}
+/**
+ * `scene` seen from `camera`, into `out` as linear premultiplied RGBA.
+ *
+ * Each pixel is one ray: the nearest triangle, its interpolated normal and albedo, Lambert against
+ * the light, a shadow ray back to it, and a Blinn–Phong lobe whose width is the roughness. What a
+ * real capture would have instead of the shadow ray is a shadow in the photograph; what matters for
+ * the test is that the shadow is *there* and that the caller knows where.
+ */
+export declare function renderLit(scene: LitScene, camera: RasterCamera, light: Light, out: Float32Array): void;
+/** Where a camera stands, from its 3 × 4 world-to-camera. */
+export declare function cameraCentre(worldToCamera: Float64Array): [number, number, number];
+export interface MeshHit {
+    readonly triangle: number;
+    readonly distance: number;
+    /** Barycentric weights of the three corners, in order. */
+    readonly weights: readonly [number, number, number];
+}
+/** The nearest triangle a ray meets, by Möller–Trumbore over every one of them. */
+export declare function nearestHit(mesh: MeshData, origin: readonly [number, number, number], direction: readonly [number, number, number], near: number, far: number): MeshHit | null;
+/** A per-vertex attribute at a hit, by its barycentric weights. */
+export declare function interpolate(mesh: MeshData, values: ArrayLike<number>, hit: MeshHit, stride: number): number[];

package/dist/delightScene.js

@@ -0,0 +1,183 @@
+/**
+ * A lit scene with the answer known: what a delighting test needs and nothing else.
+ *
+ * **A fixture, not a renderer.** `testScene.ts` ray casts analytic boxes and planes for the
+ * geometry stages; this renders a *triangle mesh* with a light in it, because delighting is asked
+ * about a mesh and about shading — a cast shadow, a highlight, and an albedo that the caller knows
+ * because it put it there. It is deliberately the simplest thing that can pose the question: one
+ * directional light, Lambert plus a Blinn–Phong lobe, and a shadow by testing the ray to the light
+ * against the same triangles.
+ *
+ * **It lives beside the code rather than in the test file** because three tests need it and because
+ * the *relighting* gate needs to render the same scene under a second light — which is the one
+ * measurement that says whether a recovered material is a material or a picture of one.
+ */
+import { exactExp, exactLog } from '@driftengine/core';
+/**
+ * `scene` seen from `camera`, into `out` as linear premultiplied RGBA.
+ *
+ * Each pixel is one ray: the nearest triangle, its interpolated normal and albedo, Lambert against
+ * the light, a shadow ray back to it, and a Blinn–Phong lobe whose width is the roughness. What a
+ * real capture would have instead of the shadow ray is a shadow in the photograph; what matters for
+ * the test is that the shadow is *there* and that the caller knows where.
+ */
+export function renderLit(scene, camera, light, out) {
+    const { width, height, intrinsics } = camera;
+    const [fx, fy, cx, cy] = intrinsics;
+    const m = camera.worldToCamera;
+    const eye = cameraCentre(m);
+    out.fill(0);
+    for (let y = 0; y < height; y += 1) {
+        for (let x = 0; x < width; x += 1) {
+            const ahead = toWorld(m, [(x + 0.5 - cx) / fx, (y + 0.5 - cy) / fy, 1]);
+            const direction = normalise([
+                ahead[0] - eye[0],
+                ahead[1] - eye[1],
+                ahead[2] - eye[2],
+            ]);
+            const hit = nearestHit(scene.mesh, eye, direction, 1e-4, Infinity);
+            const pixel = (y * width + x) * 4;
+            if (hit === null) {
+                out[pixel + 3] = 1;
+                continue;
+            }
+            const point = [0, 1, 2].map((k) => eye[k] + hit.distance * direction[k]);
+            const normal = interpolate(scene.mesh, scene.mesh.normals, hit, 3);
+            const albedo = interpolate(scene.mesh, scene.albedo, hit, 3);
+            const rough = interpolate(scene.mesh, scene.roughness, hit, 1)[0];
+            const facing = dot(normal, light.direction);
+            /* A ray back to the light, against the same triangles: a hard shadow with a hard edge. */
+            const lit = facing > 0 &&
+                nearestHit(scene.mesh, point, light.direction, 1e-3, 40) ===
+                    null;
+            const toEye = normalise([
+                eye[0] - point[0],
+                eye[1] - point[1],
+                eye[2] - point[2],
+            ]);
+            const half = normalise([
+                toEye[0] + light.direction[0],
+                toEye[1] + light.direction[1],
+                toEye[2] + light.direction[2],
+            ]);
+            /*
+             * **A Blinn–Phong lobe that widens *and weakens* as the surface roughens.** The exponent
+             * alone is not enough: with a strength that does not fall, a rough surface comes out with an
+             * enormous broad highlight over the whole of it, which is not a surface anybody has seen. It
+             * also made this fixture's floor 1.77 times too bright through a delighting that was working
+             * correctly — the light really was mostly specular, so the recovery was right about a scene
+             * nobody would build.
+             */
+            const square = rough * rough;
+            const shininess = 2 / Math.max(1e-4, square * square) - 2;
+            /*
+             * **Normalised, which is the difference between a rough surface and a bright one.** A lobe
+             * that widens without losing energy puts a broad sheen over the whole of a rough surface,
+             * and it is not a small effect: measured here, the dark channels of a matte blue floor came
+             * back nearly twice as bright through a delighting that was working correctly. The peak of a
+             * Blinn–Phong lobe scales as (n + 2) / 8π, and 0.04 is a dielectric's reflectance head-on.
+             */
+            const strength = (0.04 * (shininess + 2)) / (8 * Math.PI);
+            /*
+             * The power by the engine's own reproducible pair rather than `Math.pow`, which the
+             * determinism gate refuses — two engines may differ by a unit in the last place, and a
+             * fixture that renders differently on two machines is a test nobody can compare.
+             */
+            const facingHalf = Math.max(0, dot(normal, half));
+            const lobe = facingHalf > 0 ? exactExp(shininess * exactLog(facingHalf)) : 0;
+            const specular = lit ? lobe * strength : 0;
+            for (let c = 0; c < 3; c += 1) {
+                const irradiance = (lit ? facing : 0) * light.colour[c] + light.ambient;
+                out[pixel + c] =
+                    albedo[c] * irradiance + specular * light.colour[c];
+            }
+            out[pixel + 3] = 1;
+        }
+    }
+}
+/** Where a camera stands, from its 3 × 4 world-to-camera. */
+export function cameraCentre(worldToCamera) {
+    const out = [0, 0, 0];
+    for (let r = 0; r < 3; r += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            out[c] -= worldToCamera[r * 4 + c] * worldToCamera[r * 4 + 3];
+        }
+    }
+    return out;
+}
+/** A camera-space point in the world: the projection run backwards. */
+function toWorld(m, view) {
+    const out = [0, 0, 0];
+    for (let r = 0; r < 3; r += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            out[c] += m[r * 4 + c] * (view[r] - m[r * 4 + 3]);
+        }
+    }
+    return out;
+}
+/** The nearest triangle a ray meets, by Möller–Trumbore over every one of them. */
+export function nearestHit(mesh, origin, direction, near, far) {
+    let best = null;
+    const triangles = mesh.indices.length / 3;
+    for (let face = 0; face < triangles; face += 1) {
+        const a = mesh.indices[face * 3];
+        const b = mesh.indices[face * 3 + 1];
+        const c = mesh.indices[face * 3 + 2];
+        const e1 = corner(mesh, b, a);
+        const e2 = corner(mesh, c, a);
+        const p = cross(direction, e2);
+        const determinant = dot(e1, p);
+        if (Math.abs(determinant) < 1e-12)
+            continue;
+        const inverse = 1 / determinant;
+        const t = [0, 1, 2].map((k) => origin[k] - mesh.positions[a * 3 + k]);
+        const u = dot(t, p) * inverse;
+        if (u < 0 || u > 1)
+            continue;
+        const q = cross(t, e1);
+        const v = dot(direction, q) * inverse;
+        if (v < 0 || u + v > 1)
+            continue;
+        const distance = dot(e2, q) * inverse;
+        if (distance < near || distance > far)
+            continue;
+        if (best !== null && distance >= best.distance)
+            continue;
+        best = { triangle: face, distance, weights: [1 - u - v, u, v] };
+    }
+    return best;
+}
+/** A per-vertex attribute at a hit, by its barycentric weights. */
+export function interpolate(mesh, values, hit, stride) {
+    const out = new Array(stride).fill(0);
+    for (let corner = 0; corner < 3; corner += 1) {
+        const vertex = mesh.indices[hit.triangle * 3 + corner];
+        const weight = hit.weights[corner];
+        for (let k = 0; k < stride; k += 1) {
+            out[k] = out[k] + weight * values[vertex * stride + k];
+        }
+    }
+    return out;
+}
+function corner(mesh, to, from) {
+    return [0, 1, 2].map((k) => mesh.positions[to * 3 + k] - mesh.positions[from * 3 + k]);
+}
+function cross(a, b) {
+    return [
+        a[1] * b[2] - a[2] * b[1],
+        a[2] * b[0] - a[0] * b[2],
+        a[0] * b[1] - a[1] * b[0],
+    ];
+}
+function dot(a, b) {
+    return (a[0] * b[0] +
+        a[1] * b[1] +
+        a[2] * b[2]);
+}
+function normalise(v) {
+    const length = Math.sqrt(v[0] * v[0] +
+        v[1] * v[1] +
+        v[2] * v[2]);
+    const scale = length > 0 ? 1 / length : 0;
+    return [v[0] * scale, v[1] * scale, v[2] * scale];
+}

package/dist/depth.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Depth, its confidence and a camera per view, from a clip's frames rather than from tensors.
+ *
+ * **The frames arrive as a host hands them over** — RGBA bytes at whatever size the clip is — and
+ * this prepares them as Depth Anything 3's own processor does, runs the views together so the
+ * model's across-view blocks see each other, and decodes what comes back. A graph bakes its shapes,
+ * so one is built per count of views and prepared size and kept: a clip's frames are all the same
+ * size, so the second estimate over the same clip builds nothing.
+ *
+ * **It answers with its own arrays rather than filling the caller's**, which the engine's hot paths
+ * never do: the prepared size is not known until the frames are prepared, so a caller cannot have
+ * sized anything, and this runs once a clip rather than once a frame.
+ */
+import { type WeightSource } from '@driftengine/texture';
+import { type DepthAnything3Config } from './models/depthAnything3.ts';
+import type { GraphRun } from './run.ts';
+/** A frame as a host hands it over: RGBA bytes, and the size they are. */
+export interface RawFrame {
+    readonly pixels: Uint8Array;
+    readonly width: number;
+    readonly height: number;
+}
+export interface DepthView {
+    /** `[height, width]` metres, and one confidence each. */
+    readonly depth: Float32Array;
+    readonly confidence: Float32Array;
+    /** 3 × 4 row-major, world to camera. */
+    readonly worldToCamera: Float32Array;
+    /** 3 × 3 row-major. */
+    readonly intrinsics: Float32Array;
+}
+export interface DepthEstimate {
+    /** The size the frames were prepared at, which the maps and the cameras are in. */
+    readonly width: number;
+    readonly height: number;
+    readonly views: readonly DepthView[];
+}
+export interface DepthEstimator {
+    estimate(frames: readonly RawFrame[]): Promise<DepthEstimate>;
+}
+/** An estimator over one model's weights, running its graphs through `run`. */
+export declare function createDepthEstimator(config: DepthAnything3Config, weights: WeightSource, run: GraphRun, options?: {
+    readonly size?: number;
+}): DepthEstimator;

package/dist/depth.js

@@ -0,0 +1,53 @@
+/**
+ * Depth, its confidence and a camera per view, from a clip's frames rather than from tensors.
+ *
+ * **The frames arrive as a host hands them over** — RGBA bytes at whatever size the clip is — and
+ * this prepares them as Depth Anything 3's own processor does, runs the views together so the
+ * model's across-view blocks see each other, and decodes what comes back. A graph bakes its shapes,
+ * so one is built per count of views and prepared size and kept: a clip's frames are all the same
+ * size, so the second estimate over the same clip builds nothing.
+ *
+ * **It answers with its own arrays rather than filling the caller's**, which the engine's hot paths
+ * never do: the prepared size is not known until the frames are prepared, so a caller cannot have
+ * sized anything, and this runs once a clip rather than once a frame.
+ */
+import { graphFromWeights } from '@driftengine/texture';
+import { decodeCamera, decodeDepth, depthAnything3, } from './models/depthAnything3.js';
+import { prepareDepthFrame } from './prepare.js';
+/** An estimator over one model's weights, running its graphs through `run`. */
+export function createDepthEstimator(config, weights, run, options = {}) {
+    const size = options.size ?? 504;
+    const graphs = new Map();
+    return {
+        async estimate(frames) {
+            if (frames.length === 0)
+                throw new RangeError('a depth estimate needs a frame');
+            const prepared = frames.map((frame) => prepareDepthFrame(frame.pixels, frame.width, frame.height, size));
+            const { width, height } = prepared[0];
+            for (const [at, one] of prepared.entries()) {
+                if (one.width !== width || one.height !== height) {
+                    throw new RangeError(`frame ${at} prepares to ${one.width}×${one.height} and the first to ${width}×${height}` +
+                        ': one estimate is one clip, at one size');
+                }
+            }
+            const key = `${frames.length}.${height}x${width}`;
+            let graph = graphs.get(key);
+            if (graph === undefined) {
+                graph = graphFromWeights(weights, depthAnything3(config, frames.length, height, width));
+                graphs.set(key, graph);
+            }
+            const out = await run(graph, new Map(prepared.map((one, view) => [`image${view}`, one.pixels])));
+            const pose = out.get('pose');
+            const views = prepared.map((_, view) => {
+                const depth = new Float32Array(width * height);
+                const confidence = new Float32Array(width * height);
+                decodeDepth(out.get(`logits${view}`), depth, confidence);
+                const worldToCamera = new Float32Array(12);
+                const intrinsics = new Float32Array(9);
+                decodeCamera(pose, view, height, width, worldToCamera, intrinsics);
+                return { depth, confidence, worldToCamera, intrinsics };
+            });
+            return { width, height, views };
+        },
+    };
+}