@driftengine/capture 4.0.0

package/dist/masks.js

@@ -0,0 +1,165 @@
+const DEFAULT_OVERLAP = 0.25;
+import { planeOf, regionOf } from './segment.js';
+/**
+ * A lattice of prompt points over an image, seeded.
+ *
+ * **A lattice rather than a scatter**, because a scatter leaves gaps a small object falls into and
+ * a lattice does not; the seeded jitter is there so that a repeated object does not land on the
+ * same part of every instance, which is how a grid misses the same feature twice. Two per pixel
+ * pair — `x` then `y` — in reading order, which is the order a model's masks come back in.
+ */
+export function promptGrid(width, height, across, down, random) {
+    const out = new Float32Array(across * down * 2);
+    for (let j = 0; j < down; j += 1) {
+        for (let i = 0; i < across; i += 1) {
+            const at = (j * across + i) * 2;
+            const cellX = (width * (i + 0.5)) / across;
+            const cellY = (height * (j + 0.5)) / down;
+            /*
+             * A sixth of a cell either way. A centre sits half a cell from each of its edges, so a point
+             * cannot leave the cell it belongs to however the seed falls — which is what keeps this a
+             * lattice, and is why nothing here clamps to the frame.
+             */
+            const jitterX = ((random() - 0.5) * width) / across / 3;
+            const jitterY = ((random() - 0.5) * height) / down / 3;
+            out[at] = cellX + jitterX;
+            out[at + 1] = cellY + jitterY;
+        }
+    }
+    return out;
+}
+/**
+ * Masks from several views lifted onto the mesh, by what most of the views that saw a triangle say.
+ *
+ * **Voting rather than taking the best view**, because a mask boundary is where a model is least
+ * sure and every view puts it somewhere slightly different. A triangle no view claimed is left
+ * unassigned rather than given to its neighbour, which is what keeps an unlabelled region
+ * unlabelled.
+ */
+export function liftMasks(views, mesh, labels) {
+    const triangles = mesh.indices.length / 3;
+    const planes = new Float64Array(triangles * 4);
+    const areas = new Float64Array(triangles);
+    for (let face = 0; face < triangles; face += 1)
+        planeOf(mesh, face, planes, areas);
+    const votes = [];
+    for (let face = 0; face < triangles; face += 1)
+        votes.push(new Map());
+    const middle = new Float64Array(3);
+    for (const view of views) {
+        const [fx, fy, cx, cy] = view.intrinsics;
+        const m = view.worldToCamera;
+        for (let face = 0; face < triangles; face += 1) {
+            centroid(mesh, face, middle);
+            let x = m[3];
+            let y = m[7];
+            let z = m[11];
+            for (let k = 0; k < 3; k += 1) {
+                x += m[k] * middle[k];
+                y += m[4 + k] * middle[k];
+                z += m[8 + k] * middle[k];
+            }
+            if (!(z > 0))
+                continue;
+            const px = Math.floor((fx * x) / z + cx);
+            const py = Math.floor((fy * y) / z + cy);
+            if (px < 0 || px >= view.width || py < 0 || py >= view.height)
+                continue;
+            const mask = view.mask[py * view.width + px];
+            if (mask < 0)
+                continue;
+            const held = votes[face];
+            held.set(mask, (held.get(mask) ?? 0) + 1);
+        }
+    }
+    const members = new Map();
+    for (let face = 0; face < triangles; face += 1) {
+        const held = votes[face];
+        let best = -1;
+        let most = 0;
+        /* Ties go to the lower mask index, so two views disagreeing give the same answer every run. */
+        for (const [mask, count] of [...held.entries()].sort((a, b) => a[0] - b[0])) {
+            if (count > most) {
+                most = count;
+                best = mask;
+            }
+        }
+        if (best < 0)
+            continue;
+        const list = members.get(best);
+        if (list === undefined)
+            members.set(best, [face]);
+        else
+            list.push(face);
+    }
+    const out = [];
+    for (const mask of [...members.keys()].sort((a, b) => a - b)) {
+        const region = regionOf(mesh, members.get(mask), planes, areas);
+        out.push({ ...region, label: labels[mask] ?? null });
+    }
+    return out;
+}
+/**
+ * A name for each mask index, from a detector's boxes.
+ *
+ * **The join between the two models a capture runs**: one says *where the things are* and the other
+ * says *what they are called*, and neither can do the other's half. A mask takes the name of the
+ * detection it agrees with most, measured as intersection over union between the mask's own pixels
+ * and the box — the mask's pixels rather than the box around them, because the box around an
+ * L-shaped mask is mostly not the mask.
+ *
+ * **A mask that agrees with nothing gets no name, and that is a result.** `null` here reaches
+ * `proposeEntities` as scenery, which is the safe answer; a name taken from the nearest box
+ * whatever the overlap is how a chair becomes a table. `overlap` is where a caller draws that line.
+ */
+export function labelMasks(view, detections, queries, overlap = DEFAULT_OVERLAP) {
+    let masks = 0;
+    for (const mask of view.mask)
+        if (mask + 1 > masks)
+            masks = mask + 1;
+    const out = new Array(masks).fill(null);
+    const counts = new Int32Array(masks);
+    for (const mask of view.mask)
+        if (mask >= 0)
+            counts[mask] = counts[mask] + 1;
+    /* The best agreement each mask has found so far, so a tie keeps the earlier detection. */
+    const best = new Float64Array(masks);
+    const inside = new Int32Array(masks);
+    for (let at = 0; at < detections.length; at += 1) {
+        const box = detections[at].box;
+        inside.fill(0);
+        for (let y = 0; y < view.height; y += 1) {
+            for (let x = 0; x < view.width; x += 1) {
+                const mask = view.mask[y * view.width + x];
+                if (mask < 0)
+                    continue;
+                if (x + 0.5 < box[0] || x + 0.5 >= box[2])
+                    continue;
+                if (y + 0.5 < box[1] || y + 0.5 >= box[3])
+                    continue;
+                inside[mask] = inside[mask] + 1;
+            }
+        }
+        const area = (box[2] - box[0]) * (box[3] - box[1]);
+        for (let mask = 0; mask < masks; mask += 1) {
+            const met = inside[mask];
+            const union = counts[mask] + area - met;
+            const score = union > 0 ? met / union : 0;
+            if (score < overlap || score <= best[mask])
+                continue;
+            best[mask] = score;
+            out[mask] = queries[detections[at].label] ?? null;
+        }
+    }
+    return out;
+}
+function centroid(mesh, face, out) {
+    for (let k = 0; k < 3; k += 1)
+        out[k] = 0;
+    for (let slot = 0; slot < 3; slot += 1) {
+        const vertex = mesh.indices[face * 3 + slot];
+        for (let k = 0; k < 3; k += 1) {
+            out[k] = out[k] + mesh.positions[vertex * 3 + k] / 3;
+        }
+    }
+}

package/dist/math/dense.d.ts

@@ -0,0 +1,68 @@
+/**
+ * The dense decompositions structure from motion needs, in double precision: a singular value
+ * decomposition, a symmetric eigendecomposition, a Cholesky factor, and Levenberg–Marquardt over a
+ * least-squares problem.
+ *
+ * **Written here rather than depended on.** Each is a few dozen lines against a dependency this
+ * engine would carry into every game that captures a scene, and each is held to its own identity —
+ * a decomposition that reconstructs what it decomposed, a factor whose square is the matrix — which
+ * is a stronger check than agreeing with another implementation.
+ *
+ * **Both decompositions are Jacobi's**, rotating away one off-diagonal at a time. It is not the
+ * fastest way to a large spectrum and it is the most accurate on the small, nearly rank-deficient
+ * matrices this is for — a homography from four points, an essential matrix, a 3 × 3 covariance —
+ * where the answer's smallest value is the one that matters. **What would make it wrong** is a
+ * matrix of thousands of rows, which wants a bidiagonal reduction instead; nothing here has one.
+ *
+ * **Everything is `Float64Array`.** Geometry accumulates: a pose error of a millionth is visible
+ * once a hundred frames carry it. Only arithmetic and `Math.sqrt` are reached for, which IEEE 754
+ * fixes, so a capture reproduces on any engine — `scripts/determinism.mjs` holds this package to
+ * that.
+ */
+/**
+ * One-sided Jacobi: `m`, `rows × cols` row-major with `rows ≥ cols`, into `u` (`rows × cols`),
+ * `s` (`cols`, falling) and `v` (`cols × cols`), where `m = u · diag(s) · vᵀ`.
+ */
+export declare function svdN(m: Float64Array, rows: number, cols: number, u: Float64Array, s: Float64Array, v: Float64Array): void;
+/** The same for the 3 × 3 matrices geometry is full of. */
+export declare function svd3(m: Float64Array, u: Float64Array, s: Float64Array, v: Float64Array): void;
+/**
+ * Jacobi's eigendecomposition of a symmetric `m`, `n × n`: `values` in falling order and `vectors`
+ * by column, so `m · vectors[:, k] = values[k] · vectors[:, k]`.
+ */
+export declare function symmetricEigen(m: Float64Array, n: number, values: Float64Array, vectors: Float64Array): void;
+/**
+ * The lower triangle `l` with `l · lᵀ = m`, or false where `m` is not positive definite — which is
+ * how a caller learns that its normal equations are singular, rather than by finding a NaN later.
+ *
+ * **`out` may be `m`**: only the lower triangle is read, each element before it is written, so a
+ * caller with one matrix to spare factors it in place. Zeroing the whole of `out` first would have
+ * erased the matrix in that case, which is a silent wrong answer rather than a refusal.
+ */
+export declare function cholesky(m: Float64Array, n: number, out: Float64Array): boolean;
+/** Solves `l · lᵀ · x = b` in place over `x`, given the Cholesky factor. */
+export declare function choleskySolve(l: Float64Array, n: number, x: Float64Array): void;
+/** A least-squares problem: how many residuals, how many parameters, and both at a point. */
+export interface LeastSquares {
+    readonly residuals: number;
+    readonly parameters: number;
+    /** The residuals at `x`, and — unless `jacobian` is null — their derivatives, row by residual. */
+    evaluate(x: Float64Array, residuals: Float64Array, jacobian: Float64Array | null): void;
+}
+export interface FitOptions {
+    /** The first damping; **zero is plain Gauss–Newton**, steps taken whatever they cost. */
+    readonly damping?: number;
+    readonly maxIterations?: number;
+    /** Stop once a step moves every parameter less than this. */
+    readonly tolerance?: number;
+}
+/**
+ * Levenberg–Marquardt over `x`, in place: the Gauss–Newton step with `damping · diag(JᵀJ)` added,
+ * the damping raised where a step would cost more and lowered where it pays, which is the whole
+ * reason it is not Gauss–Newton — **a step that overshoots is rejected rather than taken**, and a
+ * start far from the answer is where that decides between converging and diverging.
+ */
+export declare function levenbergMarquardt(problem: LeastSquares, x: Float64Array, options?: FitOptions): {
+    readonly iterations: number;
+    readonly cost: number;
+};

package/dist/math/dense.js

@@ -0,0 +1,294 @@
+/**
+ * The dense decompositions structure from motion needs, in double precision: a singular value
+ * decomposition, a symmetric eigendecomposition, a Cholesky factor, and Levenberg–Marquardt over a
+ * least-squares problem.
+ *
+ * **Written here rather than depended on.** Each is a few dozen lines against a dependency this
+ * engine would carry into every game that captures a scene, and each is held to its own identity —
+ * a decomposition that reconstructs what it decomposed, a factor whose square is the matrix — which
+ * is a stronger check than agreeing with another implementation.
+ *
+ * **Both decompositions are Jacobi's**, rotating away one off-diagonal at a time. It is not the
+ * fastest way to a large spectrum and it is the most accurate on the small, nearly rank-deficient
+ * matrices this is for — a homography from four points, an essential matrix, a 3 × 3 covariance —
+ * where the answer's smallest value is the one that matters. **What would make it wrong** is a
+ * matrix of thousands of rows, which wants a bidiagonal reduction instead; nothing here has one.
+ *
+ * **Everything is `Float64Array`.** Geometry accumulates: a pose error of a millionth is visible
+ * once a hundred frames carry it. Only arithmetic and `Math.sqrt` are reached for, which IEEE 754
+ * fixes, so a capture reproduces on any engine — `scripts/determinism.mjs` holds this package to
+ * that.
+ */
+/** How far off the diagonal a Jacobi sweep tolerates before it stops. */
+const SETTLED = 1e-15;
+const SWEEPS = 60;
+/**
+ * One-sided Jacobi: `m`, `rows × cols` row-major with `rows ≥ cols`, into `u` (`rows × cols`),
+ * `s` (`cols`, falling) and `v` (`cols × cols`), where `m = u · diag(s) · vᵀ`.
+ */
+export function svdN(m, rows, cols, u, s, v) {
+    if (rows < cols) {
+        throw new RangeError(`a ${rows}×${cols} matrix has more columns than rows; transpose it first`);
+    }
+    const a = Float64Array.from(m);
+    v.fill(0);
+    for (let i = 0; i < cols; i += 1)
+        v[i * cols + i] = 1;
+    for (let sweep = 0; sweep < SWEEPS; sweep += 1) {
+        let off = 0;
+        for (let p = 0; p < cols - 1; p += 1) {
+            for (let q = p + 1; q < cols; q += 1) {
+                let alpha = 0;
+                let beta = 0;
+                let gamma = 0;
+                for (let r = 0; r < rows; r += 1) {
+                    const left = a[r * cols + p];
+                    const right = a[r * cols + q];
+                    alpha += left * left;
+                    beta += right * right;
+                    gamma += left * right;
+                }
+                if (gamma === 0)
+                    continue;
+                const scale = Math.sqrt(alpha * beta);
+                off = Math.max(off, scale === 0 ? 0 : Math.abs(gamma) / scale);
+                if (scale === 0 || Math.abs(gamma) <= SETTLED * scale)
+                    continue;
+                /* The rotation that makes the two columns orthogonal, by the stable root. */
+                const zeta = (beta - alpha) / (2 * gamma);
+                const t = (zeta >= 0 ? 1 : -1) / (Math.abs(zeta) + Math.sqrt(1 + zeta * zeta));
+                const c = 1 / Math.sqrt(1 + t * t);
+                const sn = c * t;
+                for (let r = 0; r < rows; r += 1) {
+                    const left = a[r * cols + p];
+                    const right = a[r * cols + q];
+                    a[r * cols + p] = c * left - sn * right;
+                    a[r * cols + q] = sn * left + c * right;
+                }
+                for (let r = 0; r < cols; r += 1) {
+                    const left = v[r * cols + p];
+                    const right = v[r * cols + q];
+                    v[r * cols + p] = c * left - sn * right;
+                    v[r * cols + q] = sn * left + c * right;
+                }
+            }
+        }
+        if (off <= SETTLED)
+            break;
+    }
+    /* What is left of each column is its singular value times a unit vector. */
+    const order = Array.from({ length: cols }, (_, i) => i);
+    const lengths = new Float64Array(cols);
+    for (let c = 0; c < cols; c += 1) {
+        let sum = 0;
+        for (let r = 0; r < rows; r += 1) {
+            const value = a[r * cols + c];
+            sum += value * value;
+        }
+        lengths[c] = Math.sqrt(sum);
+    }
+    order.sort((p, q) => lengths[q] - lengths[p]);
+    const held = Float64Array.from(v);
+    for (const [at, from] of order.entries()) {
+        const length = lengths[from];
+        s[at] = length;
+        for (let r = 0; r < rows; r += 1) {
+            u[r * cols + at] = length === 0 ? 0 : a[r * cols + from] / length;
+        }
+        for (let r = 0; r < cols; r += 1)
+            v[r * cols + at] = held[r * cols + from];
+    }
+}
+/** The same for the 3 × 3 matrices geometry is full of. */
+export function svd3(m, u, s, v) {
+    svdN(m, 3, 3, u, s, v);
+}
+/**
+ * Jacobi's eigendecomposition of a symmetric `m`, `n × n`: `values` in falling order and `vectors`
+ * by column, so `m · vectors[:, k] = values[k] · vectors[:, k]`.
+ */
+export function symmetricEigen(m, n, values, vectors) {
+    const a = Float64Array.from(m);
+    vectors.fill(0);
+    for (let i = 0; i < n; i += 1)
+        vectors[i * n + i] = 1;
+    for (let sweep = 0; sweep < SWEEPS; sweep += 1) {
+        let off = 0;
+        for (let p = 0; p < n - 1; p += 1) {
+            for (let q = p + 1; q < n; q += 1)
+                off = Math.max(off, Math.abs(a[p * n + q]));
+        }
+        if (off <= SETTLED)
+            break;
+        for (let p = 0; p < n - 1; p += 1) {
+            for (let q = p + 1; q < n; q += 1) {
+                const pivot = a[p * n + q];
+                if (Math.abs(pivot) <= SETTLED * off)
+                    continue;
+                const zeta = (a[q * n + q] - a[p * n + p]) / (2 * pivot);
+                const t = (zeta >= 0 ? 1 : -1) / (Math.abs(zeta) + Math.sqrt(1 + zeta * zeta));
+                const c = 1 / Math.sqrt(1 + t * t);
+                const sn = c * t;
+                for (let k = 0; k < n; k += 1) {
+                    const left = a[k * n + p];
+                    const right = a[k * n + q];
+                    a[k * n + p] = c * left - sn * right;
+                    a[k * n + q] = sn * left + c * right;
+                }
+                for (let k = 0; k < n; k += 1) {
+                    const left = a[p * n + k];
+                    const right = a[q * n + k];
+                    a[p * n + k] = c * left - sn * right;
+                    a[q * n + k] = sn * left + c * right;
+                }
+                for (let k = 0; k < n; k += 1) {
+                    const left = vectors[k * n + p];
+                    const right = vectors[k * n + q];
+                    vectors[k * n + p] = c * left - sn * right;
+                    vectors[k * n + q] = sn * left + c * right;
+                }
+            }
+        }
+    }
+    const order = Array.from({ length: n }, (_, i) => i);
+    order.sort((p, q) => a[q * n + q] - a[p * n + p]);
+    const held = Float64Array.from(vectors);
+    for (const [at, from] of order.entries()) {
+        values[at] = a[from * n + from];
+        for (let r = 0; r < n; r += 1)
+            vectors[r * n + at] = held[r * n + from];
+    }
+}
+/**
+ * The lower triangle `l` with `l · lᵀ = m`, or false where `m` is not positive definite — which is
+ * how a caller learns that its normal equations are singular, rather than by finding a NaN later.
+ *
+ * **`out` may be `m`**: only the lower triangle is read, each element before it is written, so a
+ * caller with one matrix to spare factors it in place. Zeroing the whole of `out` first would have
+ * erased the matrix in that case, which is a silent wrong answer rather than a refusal.
+ */
+export function cholesky(m, n, out) {
+    for (let r = 0; r < n; r += 1) {
+        for (let c = r + 1; c < n; c += 1)
+            out[r * n + c] = 0;
+        for (let c = 0; c <= r; c += 1) {
+            let sum = m[r * n + c];
+            for (let k = 0; k < c; k += 1)
+                sum -= out[r * n + k] * out[c * n + k];
+            if (r === c) {
+                if (!(sum > 0))
+                    return false;
+                out[r * n + c] = Math.sqrt(sum);
+            }
+            else {
+                out[r * n + c] = sum / out[c * n + c];
+            }
+        }
+    }
+    return true;
+}
+/** Solves `l · lᵀ · x = b` in place over `x`, given the Cholesky factor. */
+export function choleskySolve(l, n, x) {
+    for (let r = 0; r < n; r += 1) {
+        let sum = x[r];
+        for (let k = 0; k < r; k += 1)
+            sum -= l[r * n + k] * x[k];
+        x[r] = sum / l[r * n + r];
+    }
+    for (let r = n - 1; r >= 0; r -= 1) {
+        let sum = x[r];
+        for (let k = r + 1; k < n; k += 1)
+            sum -= l[k * n + r] * x[k];
+        x[r] = sum / l[r * n + r];
+    }
+}
+/**
+ * Levenberg–Marquardt over `x`, in place: the Gauss–Newton step with `damping · diag(JᵀJ)` added,
+ * the damping raised where a step would cost more and lowered where it pays, which is the whole
+ * reason it is not Gauss–Newton — **a step that overshoots is rejected rather than taken**, and a
+ * start far from the answer is where that decides between converging and diverging.
+ */
+export function levenbergMarquardt(problem, x, options = {}) {
+    const { residuals: rows, parameters: columns } = problem;
+    const maxIterations = options.maxIterations ?? 100;
+    const tolerance = options.tolerance ?? 1e-14;
+    let damping = options.damping ?? 1e-3;
+    const undamped = damping === 0;
+    const residual = new Float64Array(rows);
+    const jacobian = new Float64Array(rows * columns);
+    const normals = new Float64Array(columns * columns);
+    const factor = new Float64Array(columns * columns);
+    const gradient = new Float64Array(columns);
+    const step = new Float64Array(columns);
+    const trial = new Float64Array(columns);
+    const costOf = (values) => {
+        let sum = 0;
+        for (let i = 0; i < values.length; i += 1)
+            sum += values[i] * values[i];
+        return sum;
+    };
+    problem.evaluate(x, residual, null);
+    let cost = costOf(residual);
+    let iterations = 0;
+    while (iterations < maxIterations) {
+        iterations += 1;
+        problem.evaluate(x, residual, jacobian);
+        normals.fill(0);
+        gradient.fill(0);
+        for (let r = 0; r < rows; r += 1) {
+            for (let p = 0; p < columns; p += 1) {
+                const value = jacobian[r * columns + p];
+                gradient[p] = gradient[p] - value * residual[r];
+                for (let q = p; q < columns; q += 1) {
+                    normals[p * columns + q] =
+                        normals[p * columns + q] + value * jacobian[r * columns + q];
+                }
+            }
+        }
+        for (let p = 0; p < columns; p += 1) {
+            for (let q = 0; q < p; q += 1)
+                normals[p * columns + q] = normals[q * columns + p];
+        }
+        let taken = false;
+        for (let attempt = 0; attempt < 12 && !taken; attempt += 1) {
+            factor.set(normals);
+            for (let p = 0; p < columns; p += 1) {
+                /* Marquardt's scaling: the damping rides each parameter's own curvature. */
+                const diagonal = normals[p * columns + p];
+                factor[p * columns + p] = diagonal * (1 + damping) + damping;
+            }
+            if (!cholesky(factor, columns, factor)) {
+                if (undamped)
+                    return { iterations, cost };
+                damping *= 10;
+                continue;
+            }
+            step.set(gradient);
+            choleskySolve(factor, columns, step);
+            let moved = 0;
+            for (let p = 0; p < columns; p += 1) {
+                trial[p] = x[p] + step[p];
+                moved = Math.max(moved, Math.abs(step[p]));
+            }
+            problem.evaluate(trial, residual, null);
+            const next = costOf(residual);
+            if (undamped || next < cost) {
+                x.set(trial);
+                cost = next;
+                taken = true;
+                if (!undamped)
+                    damping = Math.max(damping / 10, 1e-12);
+                if (moved <= tolerance)
+                    return { iterations, cost };
+            }
+            else {
+                damping *= 10;
+                if (damping > 1e12)
+                    return { iterations, cost };
+            }
+        }
+        if (!taken)
+            return { iterations, cost };
+    }
+    return { iterations, cost };
+}

package/dist/math/schur.d.ts

@@ -0,0 +1,23 @@
+/** The normal equations in blocks: cameras, points of three, and one link per observation. */
+export interface BlockNormals {
+    readonly cameras: number;
+    /** Parameters a camera; the points are three each. */
+    readonly cameraSize: number;
+    readonly points: number;
+    /** `cameras · cameraSize²`, each symmetric and row-major. */
+    readonly cameraBlocks: Float64Array;
+    /** `points · 9`, each symmetric and row-major. */
+    readonly pointBlocks: Float64Array;
+    /** `links · cameraSize · 3`, row-major: a camera's rows against a point's columns. */
+    readonly links: Float64Array;
+    readonly linkCamera: Int32Array;
+    readonly linkPoint: Int32Array;
+    /** The right-hand side: `cameras · cameraSize` and `points · 3`. */
+    readonly cameraGradient: Float64Array;
+    readonly pointGradient: Float64Array;
+}
+/**
+ * The step for `damping`, into `cameraStep` (`cameras · cameraSize`) and `pointStep`
+ * (`points · 3`); false where a point's block or the reduced system cannot be factored.
+ */
+export declare function schurSolve(normals: BlockNormals, damping: number, cameraStep: Float64Array, pointStep: Float64Array): boolean;