@driftengine/capture 4.0.0

package/dist/gaussians/optimise.d.ts

@@ -0,0 +1,50 @@
+/**
+ * A cloud of Gaussians fitted to a clip's frames, from a camera path and a sparse start.
+ *
+ * **The engine already knows how to draw the answer.** What comes out is `@driftengine/splats`'
+ * `SplatSource` — the same shape its `.ply` and `.sog` readers produce — so a fitted capture goes
+ * through `packSplats` and into a `.drft` beside anything else, and is drawn by the shader this
+ * package's rasteriser was written to match.
+ *
+ * **Two parameters are fitted through a transform, and it is what keeps the fit standing.** A
+ * scale is optimised as its logarithm and an opacity as its logit, so neither can be stepped past
+ * the end of its own range: a negative standard deviation is a covariance that is not a covariance,
+ * and an opacity above one is light arriving from a Gaussian that was never there. The gradients
+ * this receives are with respect to the real quantities, and the chain rule back to the transformed
+ * ones is one multiplication each.
+ *
+ * **The frames are linear and carry their own coverage.** A synthetic frame comes out of the
+ * rasteriser with both; a photograph has to be decoded from its display transform and given an
+ * alpha of one by whoever read it, because a photograph *is* fully covered and saying so is the
+ * caller's statement about their clip rather than this module's guess. A masked capture says
+ * something narrower with the same field, and the fit needs no further telling.
+ */
+import type { SplatSource } from '@driftengine/splats';
+import { type Frame } from './rasterise.ts';
+export interface OptimiseOptions {
+    /** `fx`, `fy`, `cx`, `cy`, shared by every frame, as the path was solved with. */
+    readonly intrinsics: readonly [number, number, number, number];
+    readonly width: number;
+    readonly height: number;
+    /** Three per point: where the fit starts, usually the path's own sparse reconstruction. */
+    readonly seeds: Float64Array;
+    /** The most Gaussians the answer may carry. */
+    readonly budget: number;
+    readonly iterations: number;
+    /** The engine's seeded generator: the same one twice gives the same capture. */
+    readonly random: () => number;
+    /**
+     * The standard deviation every Gaussian starts at, in metres.
+     *
+     * Left out, it is the distance from each seed to its nearest neighbour — the spacing of the
+     * cloud, which is the size at which the seeds tile what they were sampled from. **A cloud of one
+     * seed has no spacing**, and is refused rather than given a number out of the air.
+     */
+    readonly initialScale?: number;
+    /** Fit the degree-1 band as well as the constant term. */
+    readonly viewDependent?: boolean;
+    /** How often the cloud is grown and pruned. */
+    readonly refineEvery?: number;
+}
+/** A clip fitted to Gaussians. Nothing here is the caller's array; everything is freshly cut. */
+export declare function optimiseGaussians(targets: readonly Frame[], poses: readonly Float64Array[], options: OptimiseOptions): SplatSource;

package/dist/gaussians/optimise.js

@@ -0,0 +1,125 @@
+import { adamStep, createAdamState } from './adam.js';
+import { accumulateGradients, createGradients, imageLoss } from './gradients.js';
+import { chain, clear, createCloud, harvest, materialise, start, viewOf, SPLIT_FRACTION, } from './parameters.js';
+import { rasteriseGaussians } from './rasterise.js';
+import { refineCloud } from './refine.js';
+/** How often the cloud is grown and pruned, where the caller does not say. */
+const DEFAULT_REFINE_EVERY = 50;
+/** A clip fitted to Gaussians. Nothing here is the caller's array; everything is freshly cut. */
+export function optimiseGaussians(targets, poses, options) {
+    const { intrinsics, width, height, seeds, budget, iterations, random } = options;
+    if (targets.length === 0 || targets.length !== poses.length) {
+        throw new RangeError(`capture: ${targets.length} frames against ${poses.length} poses, and a fit needs one of each`);
+    }
+    const seeded = Math.floor(seeds.length / 3);
+    if (seeded === 0)
+        throw new RangeError('capture: a fit needs at least one seed to start from');
+    if (seeded > budget) {
+        throw new RangeError(`capture: ${seeded} seeds do not fit a budget of ${budget} Gaussians`);
+    }
+    const cameras = poses.map((worldToCamera) => ({
+        width,
+        height,
+        intrinsics,
+        worldToCamera,
+    }));
+    const extent = sceneExtent(seeds, seeded, cameras);
+    const scale = options.initialScale ?? seedSpacing(seeds, seeded);
+    const cloud = createCloud(budget, extent, options.viewDependent === true);
+    start(cloud, seeds, seeded, scale);
+    const pixels = width * height * 4;
+    const rendered = new Float64Array(pixels);
+    const dPixels = new Float64Array(pixels);
+    const gradients = createGradients(budget);
+    const adam = createAdamState();
+    const refineEvery = options.refineEvery ?? DEFAULT_REFINE_EVERY;
+    const set = viewOf(cloud, budget);
+    for (let iteration = 0; iteration < iterations; iteration += 1) {
+        materialise(cloud);
+        set.count = cloud.count;
+        const view = iteration % targets.length;
+        rasteriseGaussians(set, cameras[view], rendered);
+        imageLoss(rendered, targets[view], dPixels);
+        clear(gradients, cloud.count);
+        accumulateGradients(set, cameras[view], dPixels, gradients);
+        chain(cloud, gradients);
+        adamStep(cloud.families, cloud.count, adam);
+        /*
+         * Grown and pruned between steps, never on the last one: a refinement leaves fresh Gaussians
+         * that have taken no step, and a cloud handed back in that state is one the caller is drawing
+         * before the fit has looked at it.
+         */
+        if (iteration + 1 < iterations && (iteration + 1) % refineEvery === 0) {
+            materialise(cloud);
+            refineCloud(cloud, { budget, splitAbove: SPLIT_FRACTION * extent, random });
+        }
+    }
+    materialise(cloud);
+    return harvest(cloud);
+}
+/**
+ * How big the capture is: the radius of the sphere holding the seeds and the cameras.
+ *
+ * **The cameras are in it deliberately.** A position's step is in metres, so it has to be a
+ * fraction of something real, and a seed cloud can be a single point — which has no size of its own
+ * and is still a scene, because the cameras looking at it are a distance away.
+ */
+function sceneExtent(seeds, count, cameras) {
+    const points = [];
+    for (let at = 0; at < count; at += 1) {
+        points.push([
+            seeds[at * 3],
+            seeds[at * 3 + 1],
+            seeds[at * 3 + 2],
+        ]);
+    }
+    for (const camera of cameras) {
+        const m = camera.worldToCamera;
+        const centre = [0, 0, 0];
+        for (let r = 0; r < 3; r += 1) {
+            for (let c = 0; c < 3; c += 1) {
+                centre[c] = centre[c] - m[r * 4 + c] * m[r * 4 + 3];
+            }
+        }
+        points.push(centre);
+    }
+    const middle = [0, 0, 0];
+    for (const point of points) {
+        for (let c = 0; c < 3; c += 1)
+            middle[c] = middle[c] + point[c];
+    }
+    for (let c = 0; c < 3; c += 1)
+        middle[c] = middle[c] / points.length;
+    let furthest = 0;
+    for (const point of points) {
+        let square = 0;
+        for (let c = 0; c < 3; c += 1) {
+            const off = point[c] - middle[c];
+            square += off * off;
+        }
+        furthest = square > furthest ? square : furthest;
+    }
+    return Math.sqrt(furthest);
+}
+/** The mean distance from a seed to its nearest neighbour: the spacing the cloud was sampled at. */
+function seedSpacing(seeds, count) {
+    if (count < 2)
+        return 0;
+    let total = 0;
+    for (let at = 0; at < count; at += 1) {
+        let nearest = Infinity;
+        for (let other = 0; other < count; other += 1) {
+            if (other === at)
+                continue;
+            let square = 0;
+            for (let c = 0; c < 3; c += 1) {
+                const off = seeds[at * 3 + c] - seeds[other * 3 + c];
+                square += off * off;
+            }
+            if (square < nearest)
+                nearest = square;
+        }
+        total += Math.sqrt(nearest);
+    }
+    return total / count;
+}

package/dist/gaussians/parameters.d.ts

@@ -0,0 +1,35 @@
+import type { SplatSource } from '@driftengine/splats';
+import { type Family } from './adam.ts';
+import { type GaussianGradients } from './gradients.ts';
+import type { GaussianSet } from './project.ts';
+import type { RefineCloud } from './refine.ts';
+/** A Gaussian wider than this fraction of the scene is split rather than cloned. */
+export declare const SPLIT_FRACTION = 0.01;
+/** The families, and the scratch the refinement rewrites them through. */
+export interface Cloud extends RefineCloud {
+    readonly colors: Family;
+    readonly sh1: Family | null;
+    readonly logits: Family;
+    readonly scales: Float64Array;
+    readonly budget: number;
+}
+export declare function createCloud(budget: number, extent: number, viewDependent: boolean): Cloud;
+/** Every seed as one grey, faint, round Gaussian pointing the way the world does. */
+export declare function start(cloud: Cloud, seeds: Float64Array, count: number, scale: number): void;
+/** The transformed parameters as the rasteriser wants them: real scales and real opacities. */
+export declare function materialise(cloud: Cloud): void;
+/** One `GaussianSet` over the cloud's own arrays, whose `count` moves as the cloud does. */
+export declare function viewOf(cloud: Cloud, budget: number): GaussianSet;
+export declare function clear(gradients: GaussianGradients, count: number): void;
+/**
+ * The render's gradients onto the parameters that are actually stepped.
+ *
+ * Position, rotation, colour and the band are themselves; a scale arrives as a derivative by the
+ * metre and is wanted by the logarithm, which is one factor of the scale; an opacity arrives by the
+ * proportion and is wanted by the logit, which is the logistic's own derivative. **This is also
+ * where each Gaussian's pull is recorded** — how hard the frames are asking it to move, which is
+ * what the refinement grows the busiest of.
+ */
+export declare function chain(cloud: Cloud, gradients: GaussianGradients): void;
+/** The fitted cloud as a capture: single precision, cut to what it holds. */
+export declare function harvest(cloud: Cloud): SplatSource;

package/dist/gaussians/parameters.js

@@ -0,0 +1,165 @@
+/**
+ * What the fit actually steps, and how it becomes what the rasteriser draws.
+ *
+ * **Two of the six families are not the quantity they stand for.** A scale is held as its
+ * logarithm and an opacity as its logit, so that no step can carry either past the end of its own
+ * range — a negative standard deviation is not a covariance, and an opacity above one is light
+ * from a Gaussian that was never there. Everything downstream works in the real quantities, so
+ * this is where the two are made and where the gradients come back through them.
+ *
+ * **The chain back is one multiplication each and is nearly invisible to Adam**, which divides a
+ * step by the size of its own gradients and so cannot see a positive factor applied to every one
+ * of them. That is exactly why it is tested here by its own arithmetic rather than through a fit:
+ * a fit descends about as well with the factor missing, and the gradient is still wrong.
+ */
+import { exactExp, exactLog } from '@driftengine/core';
+import { createFamily } from './adam.js';
+import { SCREEN_GRADIENTS } from './gradients.js';
+import { SH1_COEFFICIENTS } from './harmonics.js';
+/*
+ * The step sizes, as fractions of each family's own scale — which is what Adam makes them. They are
+ * the reference implementation's, and they are *decisions*: a fit that comes out soft wants the
+ * position rate lower and the opacity rate higher, and no test here asserts any of them.
+ *
+ * **The position rate is the one in real units**, so it is a fraction of the scene rather than a
+ * number: the same fit of a room and of a tabletop wants the same *relative* step.
+ */
+const POSITION_RATE = 1.6e-4;
+const SCALE_RATE = 5e-3;
+const ROTATION_RATE = 1e-3;
+const COLOUR_RATE = 2.5e-3;
+/** The band moves at a twentieth of the constant term, as the reference has it. */
+const SH1_RATE = COLOUR_RATE / 20;
+const OPACITY_RATE = 5e-2;
+/** Where a Gaussian starts: faint enough that the fit adds light rather than removing it. */
+const INITIAL_OPACITY = 0.1;
+/** A Gaussian wider than this fraction of the scene is split rather than cloned. */
+export const SPLIT_FRACTION = 0.01;
+export function createCloud(budget, extent, viewDependent) {
+    const positions = createFamily(3, POSITION_RATE * extent, budget);
+    const logScales = createFamily(3, SCALE_RATE, budget);
+    const rotations = createFamily(4, ROTATION_RATE, budget);
+    const colors = createFamily(3, COLOUR_RATE, budget);
+    const sh1 = viewDependent ? createFamily(SH1_COEFFICIENTS, SH1_RATE, budget) : null;
+    const logits = createFamily(1, OPACITY_RATE, budget);
+    const families = [positions, logScales, rotations, colors, logits];
+    if (sh1 !== null)
+        families.push(sh1);
+    return {
+        count: 0,
+        families,
+        positions,
+        logScales,
+        rotations,
+        colors,
+        sh1,
+        logits,
+        scales: new Float64Array(budget * 3),
+        opacities: new Float64Array(budget),
+        motion: new Float64Array(budget),
+        sources: new Int32Array(budget),
+        reverse: new Int32Array(budget),
+        fresh: new Uint8Array(budget),
+        scratch: new Float64Array(budget * SH1_COEFFICIENTS),
+        budget,
+    };
+}
+/** Every seed as one grey, faint, round Gaussian pointing the way the world does. */
+export function start(cloud, seeds, count, scale) {
+    if (!(scale > 0)) {
+        throw new RangeError('capture: a fit needs a starting scale, and a single seed has no spacing');
+    }
+    const logScale = exactLog(scale);
+    const logit = exactLog(INITIAL_OPACITY / (1 - INITIAL_OPACITY));
+    for (let at = 0; at < count; at += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            cloud.positions.values[at * 3 + c] = seeds[at * 3 + c];
+            cloud.logScales.values[at * 3 + c] = logScale;
+            cloud.colors.values[at * 3 + c] = 0.5;
+        }
+        cloud.rotations.values[at * 4 + 3] = 1;
+        cloud.logits.values[at] = logit;
+    }
+    cloud.count = count;
+}
+/** The transformed parameters as the rasteriser wants them: real scales and real opacities. */
+export function materialise(cloud) {
+    for (let at = 0; at < cloud.count; at += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            cloud.scales[at * 3 + c] = exactExp(cloud.logScales.values[at * 3 + c]);
+        }
+        cloud.opacities[at] = 1 / (1 + exactExp(-cloud.logits.values[at]));
+    }
+}
+/** One `GaussianSet` over the cloud's own arrays, whose `count` moves as the cloud does. */
+export function viewOf(cloud, budget) {
+    const set = {
+        count: 0,
+        positions: cloud.positions.values,
+        scales: cloud.scales,
+        rotations: cloud.rotations.values,
+        colors: cloud.colors.values,
+        opacities: cloud.opacities,
+    };
+    return cloud.sh1 === null ? set : { ...set, sh1: cloud.sh1.values.subarray(0, budget * 9) };
+}
+export function clear(gradients, count) {
+    gradients.positions.fill(0, 0, count * 3);
+    gradients.scales.fill(0, 0, count * 3);
+    gradients.rotations.fill(0, 0, count * 4);
+    gradients.colors.fill(0, 0, count * 3);
+    gradients.sh1.fill(0, 0, count * SH1_COEFFICIENTS);
+    gradients.screen.fill(0, 0, count * SCREEN_GRADIENTS);
+    gradients.opacities.fill(0, 0, count);
+}
+/**
+ * The render's gradients onto the parameters that are actually stepped.
+ *
+ * Position, rotation, colour and the band are themselves; a scale arrives as a derivative by the
+ * metre and is wanted by the logarithm, which is one factor of the scale; an opacity arrives by the
+ * proportion and is wanted by the logit, which is the logistic's own derivative. **This is also
+ * where each Gaussian's pull is recorded** — how hard the frames are asking it to move, which is
+ * what the refinement grows the busiest of.
+ */
+export function chain(cloud, gradients) {
+    for (let at = 0; at < cloud.count; at += 1) {
+        let pull = 0;
+        for (let c = 0; c < 3; c += 1) {
+            const move = gradients.positions[at * 3 + c];
+            cloud.positions.gradient[at * 3 + c] = move;
+            pull += move * move;
+            cloud.logScales.gradient[at * 3 + c] =
+                gradients.scales[at * 3 + c] * cloud.scales[at * 3 + c];
+            cloud.colors.gradient[at * 3 + c] = gradients.colors[at * 3 + c];
+        }
+        cloud.motion[at] = cloud.motion[at] + Math.sqrt(pull);
+        for (let c = 0; c < 4; c += 1) {
+            cloud.rotations.gradient[at * 4 + c] = gradients.rotations[at * 4 + c];
+        }
+        const opacity = cloud.opacities[at];
+        cloud.logits.gradient[at] = gradients.opacities[at] * opacity * (1 - opacity);
+        if (cloud.sh1 === null)
+            continue;
+        for (let c = 0; c < SH1_COEFFICIENTS; c += 1) {
+            cloud.sh1.gradient[at * SH1_COEFFICIENTS + c] = gradients.sh1[at * SH1_COEFFICIENTS + c];
+        }
+    }
+}
+/** The fitted cloud as a capture: single precision, cut to what it holds. */
+export function harvest(cloud) {
+    const { count } = cloud;
+    const source = {
+        count,
+        positions: Float32Array.from(cloud.positions.values.subarray(0, count * 3)),
+        scales: Float32Array.from(cloud.scales.subarray(0, count * 3)),
+        rotations: Float32Array.from(cloud.rotations.values.subarray(0, count * 4)),
+        colors: Float32Array.from(cloud.colors.values.subarray(0, count * 3)),
+        opacities: Float32Array.from(cloud.opacities.subarray(0, count)),
+    };
+    if (cloud.sh1 === null)
+        return source;
+    return {
+        ...source,
+        sh1: Float32Array.from(cloud.sh1.values.subarray(0, count * SH1_COEFFICIENTS)),
+    };
+}

package/dist/gaussians/project.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Where a Gaussian lands on screen: the projection `@driftengine/splats`' shader performs, in
+ * JavaScript, and the order a frame draws them in.
+ *
+ * **The same model, in the capture's own frame.** A perspective divide is not linear, so a
+ * three-dimensional Gaussian does not project to a two-dimensional one; the standard treatment,
+ * which that shader uses and this repeats, linearises about the splat's centre — `Σ' = J·R·Σ·Rᵀ·Jᵀ`
+ * — and adds a low-pass of 0.3 to the screen variances so that a splat smaller than a pixel still
+ * covers one. The one difference is handedness: the shader works in the renderer's view space,
+ * where z runs away from the camera negatively, and a capture's cameras look down positive z. The
+ * arithmetic is the same and the frame is the caller's.
+ *
+ * **It exists to be differentiated.** `projectGradient.ts` is this chain run backwards, and
+ * `gradients.test.ts` holds the two together by a central difference of one against the other.
+ */
+export interface GaussianSet {
+    count: number;
+    /** Three per Gaussian, in the capture's world. */
+    readonly positions: Float64Array;
+    /** Three per Gaussian: a standard deviation along each of its own axes, in metres. */
+    readonly scales: Float64Array;
+    /** Four per Gaussian, xyzw, as the engine's quaternions are. */
+    readonly rotations: Float64Array;
+    /** Three per Gaussian, linear, which is a spherical harmonic's constant term. */
+    readonly colors: Float64Array;
+    /** Nine per Gaussian for the degree-1 band, or absent for a cloud with no view dependence. */
+    readonly sh1?: Float64Array;
+    /** One per Gaussian, 0 to 1. */
+    readonly opacities: Float64Array;
+}
+/**
+ * A frame this rasteriser writes: **single precision where it stands in for a render target, double
+ * where the caller is differentiating through it.** The fitting loop accumulates its loss in double
+ * and a frame quantised to single is a floor under every difference taken through one — measured at
+ * about a part in five hundred of the slope, against a part in ten million when the frame is double.
+ */
+export type Frame = Float32Array | Float64Array;
+export interface RasterCamera {
+    readonly width: number;
+    readonly height: number;
+    /** `fx`, `fy`, `cx`, `cy`. */
+    readonly intrinsics: readonly [number, number, number, number];
+    /** 3 × 4 row-major world-to-camera, as `estimatePoses` answers. */
+    readonly worldToCamera: Float64Array;
+}
+/** The low-pass the shader adds to the screen variances, in square pixels. */
+export declare const LOW_PASS = 0.3;
+/** How many standard deviations of a splat are drawn, and the largest radius in pixels. */
+export declare const REACH = 3;
+/** One Gaussian as the screen sees it. */
+export interface Projected {
+    /** Where its centre lands, in pixels. */
+    readonly x: number;
+    readonly y: number;
+    /** Its screen covariance, `[a, b, d]` for `[[a, b], [b, d]]`, the low-pass included. */
+    readonly a: number;
+    readonly b: number;
+    readonly d: number;
+    /** Along the camera's own axis, which is what the painter's order is by. */
+    readonly depth: number;
+    /** The half-width of its axis-aligned box, in pixels. */
+    readonly radius: number;
+}
+/** The rotation a quaternion stands for, into `out` (3 × 3 row-major). */
+export declare function rotationFromQuaternion(q: ArrayLike<number>, at: number, out: Float64Array): void;
+/** One Gaussian projected, or null where it is behind the camera or too flat to draw. */
+export declare function projectGaussian(set: GaussianSet, at: number, camera: RasterCamera): Projected | null;
+/** The Gaussians a camera sees, nearest first, and where each lands. */
+export declare function visibleGaussians(set: GaussianSet, camera: RasterCamera): {
+    readonly order: number[];
+    readonly projected: (Projected | null)[];
+};

package/dist/gaussians/project.js

@@ -0,0 +1,137 @@
+/** The low-pass the shader adds to the screen variances, in square pixels. */
+export const LOW_PASS = 0.3;
+/** How many standard deviations of a splat are drawn, and the largest radius in pixels. */
+export const REACH = 3;
+const MAX_RADIUS = 256;
+/** The rotation a quaternion stands for, into `out` (3 × 3 row-major). */
+export function rotationFromQuaternion(q, at, out) {
+    const x = q[at * 4];
+    const y = q[at * 4 + 1];
+    const z = q[at * 4 + 2];
+    const w = q[at * 4 + 3];
+    const length = Math.sqrt(x * x + y * y + z * z + w * w) || 1;
+    const nx = x / length;
+    const ny = y / length;
+    const nz = z / length;
+    const nw = w / length;
+    out[0] = 1 - 2 * (ny * ny + nz * nz);
+    out[1] = 2 * (nx * ny - nz * nw);
+    out[2] = 2 * (nx * nz + ny * nw);
+    out[3] = 2 * (nx * ny + nz * nw);
+    out[4] = 1 - 2 * (nx * nx + nz * nz);
+    out[5] = 2 * (ny * nz - nx * nw);
+    out[6] = 2 * (nx * nz - ny * nw);
+    out[7] = 2 * (ny * nz + nx * nw);
+    out[8] = 1 - 2 * (nx * nx + ny * ny);
+}
+/** One Gaussian projected, or null where it is behind the camera or too flat to draw. */
+export function projectGaussian(set, at, camera) {
+    const [fx, fy, cx, cy] = camera.intrinsics;
+    const m = camera.worldToCamera;
+    const view = [0, 0, 0];
+    for (let r = 0; r < 3; r += 1) {
+        view[r] =
+            m[r * 4] * set.positions[at * 3] +
+                m[r * 4 + 1] * set.positions[at * 3 + 1] +
+                m[r * 4 + 2] * set.positions[at * 3 + 2] +
+                m[r * 4 + 3];
+    }
+    const z = view[2];
+    if (!(z > 1e-3))
+        return null;
+    /* The world covariance: R · diag(s²) · Rᵀ, turned into the camera's frame by the view rotation. */
+    const rotation = new Float64Array(9);
+    rotationFromQuaternion(set.rotations, at, rotation);
+    const scaled = new Float64Array(9);
+    for (let r = 0; r < 3; r += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            const s = set.scales[at * 3 + c];
+            scaled[r * 3 + c] = rotation[r * 3 + c] * s;
+        }
+    }
+    const world = new Float64Array(9);
+    for (let r = 0; r < 3; r += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            let sum = 0;
+            for (let k = 0; k < 3; k += 1)
+                sum += scaled[r * 3 + k] * scaled[c * 3 + k];
+            world[r * 3 + c] = sum;
+        }
+    }
+    const inCamera = new Float64Array(9);
+    for (let r = 0; r < 3; r += 1) {
+        for (let c = 0; c < 3; c += 1) {
+            let sum = 0;
+            for (let k = 0; k < 3; k += 1) {
+                for (let j = 0; j < 3; j += 1) {
+                    sum += m[r * 4 + k] * world[k * 3 + j] * m[c * 4 + j];
+                }
+            }
+            inCamera[r * 3 + c] = sum;
+        }
+    }
+    /* The projection's Jacobian at this point, in pixels. */
+    const invZ = 1 / z;
+    const invZ2 = invZ * invZ;
+    const j = [
+        [fx * invZ, 0, -fx * view[0] * invZ2],
+        [0, fy * invZ, -fy * view[1] * invZ2],
+    ];
+    const screen = [0, 0, 0, 0];
+    for (let r = 0; r < 2; r += 1) {
+        for (let c = 0; c < 2; c += 1) {
+            let sum = 0;
+            for (let k = 0; k < 3; k += 1) {
+                for (let l = 0; l < 3; l += 1) {
+                    sum +=
+                        j[r][k] *
+                            inCamera[k * 3 + l] *
+                            j[c][l];
+                }
+            }
+            screen[r * 2 + c] = sum;
+        }
+    }
+    const a = screen[0] + LOW_PASS;
+    const b = screen[1];
+    const d = screen[3] + LOW_PASS;
+    const determinant = a * d - b * b;
+    if (!(determinant > 1e-12))
+        return null;
+    /* The ellipse's larger eigenvalue fixes how far it reaches. */
+    const mid = 0.5 * (a + d);
+    const spread = Math.sqrt(Math.max(0.1, mid * mid - determinant));
+    const radius = Math.min(MAX_RADIUS, REACH * Math.sqrt(Math.max(mid + spread, 0)));
+    return {
+        x: fx * view[0] * invZ + cx,
+        y: fy * view[1] * invZ + cy,
+        a,
+        b,
+        d,
+        depth: z,
+        radius,
+    };
+}
+/** The Gaussians a camera sees, nearest first, and where each lands. */
+export function visibleGaussians(set, camera) {
+    const projected = [];
+    const order = [];
+    for (let at = 0; at < set.count; at += 1) {
+        const one = projectGaussian(set, at, camera);
+        projected.push(one);
+        if (one === null)
+            continue;
+        if (one.x + one.radius < 0 || one.x - one.radius >= camera.width)
+            continue;
+        if (one.y + one.radius < 0 || one.y - one.radius >= camera.height)
+            continue;
+        order.push(at);
+    }
+    /* Nearest first, ties by index so the answer does not depend on the sort's stability. */
+    order.sort((p, q) => {
+        const first = projected[p].depth;
+        const second = projected[q].depth;
+        return first === second ? p - q : first - second;
+    });
+    return { order, projected };
+}

package/dist/gaussians/projectGradient.d.ts

@@ -0,0 +1,15 @@
+/**
+ * The screen-space gradients of one Gaussian carried back to its own parameters.
+ *
+ * **`project.ts` run backwards**, term for term: the perspective Jacobian, the covariance it acts
+ * on, the rotation and scale that covariance is built from, and the quaternion behind the rotation.
+ * Nothing here knows about pixels or compositing — it takes what `gradients.ts` gathered over a
+ * Gaussian's own pixels and answers what that means for where it stands and how it is shaped.
+ */
+import { type GaussianSet, type RasterCamera } from './project.ts';
+import type { GaussianGradients } from './gradients.ts';
+/**
+ * The screen-space gradients carried back to a Gaussian's position, scale and rotation, by the
+ * same chain the projection is built from.
+ */
+export declare function screenToParameters(set: GaussianSet, at: number, camera: RasterCamera, dMeanX: number, dMeanY: number, dA: number, dB: number, dD: number, out: GaussianGradients): void;