@driftengine/capture 4.0.0

package/src/gaussians/parameters.ts

@@ -0,0 +1,186 @@
+/**
+ * What the fit actually steps, and how it becomes what the rasteriser draws.
+ *
+ * **Two of the six families are not the quantity they stand for.** A scale is held as its
+ * logarithm and an opacity as its logit, so that no step can carry either past the end of its own
+ * range — a negative standard deviation is not a covariance, and an opacity above one is light
+ * from a Gaussian that was never there. Everything downstream works in the real quantities, so
+ * this is where the two are made and where the gradients come back through them.
+ *
+ * **The chain back is one multiplication each and is nearly invisible to Adam**, which divides a
+ * step by the size of its own gradients and so cannot see a positive factor applied to every one
+ * of them. That is exactly why it is tested here by its own arithmetic rather than through a fit:
+ * a fit descends about as well with the factor missing, and the gradient is still wrong.
+ */
+import { exactExp, exactLog } from '@driftengine/core';
+import type { SplatSource } from '@driftengine/splats';
+
+import { createFamily, type Family } from './adam.ts';
+import { SCREEN_GRADIENTS, type GaussianGradients } from './gradients.ts';
+import { SH1_COEFFICIENTS } from './harmonics.ts';
+import type { GaussianSet } from './project.ts';
+import type { RefineCloud } from './refine.ts';
+
+/*
+ * The step sizes, as fractions of each family's own scale — which is what Adam makes them. They are
+ * the reference implementation's, and they are *decisions*: a fit that comes out soft wants the
+ * position rate lower and the opacity rate higher, and no test here asserts any of them.
+ *
+ * **The position rate is the one in real units**, so it is a fraction of the scene rather than a
+ * number: the same fit of a room and of a tabletop wants the same *relative* step.
+ */
+const POSITION_RATE = 1.6e-4;
+const SCALE_RATE = 5e-3;
+const ROTATION_RATE = 1e-3;
+const COLOUR_RATE = 2.5e-3;
+/** The band moves at a twentieth of the constant term, as the reference has it. */
+const SH1_RATE = COLOUR_RATE / 20;
+const OPACITY_RATE = 5e-2;
+
+/** Where a Gaussian starts: faint enough that the fit adds light rather than removing it. */
+const INITIAL_OPACITY = 0.1;
+/** A Gaussian wider than this fraction of the scene is split rather than cloned. */
+export const SPLIT_FRACTION = 0.01;
+
+/** The families, and the scratch the refinement rewrites them through. */
+export interface Cloud extends RefineCloud {
+  readonly colors: Family;
+  readonly sh1: Family | null;
+  readonly logits: Family;
+  readonly scales: Float64Array;
+  readonly budget: number;
+}
+
+export function createCloud(budget: number, extent: number, viewDependent: boolean): Cloud {
+  const positions = createFamily(3, POSITION_RATE * extent, budget);
+  const logScales = createFamily(3, SCALE_RATE, budget);
+  const rotations = createFamily(4, ROTATION_RATE, budget);
+  const colors = createFamily(3, COLOUR_RATE, budget);
+  const sh1 = viewDependent ? createFamily(SH1_COEFFICIENTS, SH1_RATE, budget) : null;
+  const logits = createFamily(1, OPACITY_RATE, budget);
+  const families = [positions, logScales, rotations, colors, logits];
+  if (sh1 !== null) families.push(sh1);
+  return {
+    count: 0,
+    families,
+    positions,
+    logScales,
+    rotations,
+    colors,
+    sh1,
+    logits,
+    scales: new Float64Array(budget * 3),
+    opacities: new Float64Array(budget),
+    motion: new Float64Array(budget),
+    sources: new Int32Array(budget),
+    reverse: new Int32Array(budget),
+    fresh: new Uint8Array(budget),
+    scratch: new Float64Array(budget * SH1_COEFFICIENTS),
+    budget,
+  };
+}
+
+/** Every seed as one grey, faint, round Gaussian pointing the way the world does. */
+export function start(cloud: Cloud, seeds: Float64Array, count: number, scale: number): void {
+  if (!(scale > 0)) {
+    throw new RangeError('capture: a fit needs a starting scale, and a single seed has no spacing');
+  }
+  const logScale = exactLog(scale);
+  const logit = exactLog(INITIAL_OPACITY / (1 - INITIAL_OPACITY));
+  for (let at = 0; at < count; at += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      cloud.positions.values[at * 3 + c] = seeds[at * 3 + c] as number;
+      cloud.logScales.values[at * 3 + c] = logScale;
+      cloud.colors.values[at * 3 + c] = 0.5;
+    }
+    cloud.rotations.values[at * 4 + 3] = 1;
+    cloud.logits.values[at] = logit;
+  }
+  cloud.count = count;
+}
+
+/** The transformed parameters as the rasteriser wants them: real scales and real opacities. */
+export function materialise(cloud: Cloud): void {
+  for (let at = 0; at < cloud.count; at += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      cloud.scales[at * 3 + c] = exactExp(cloud.logScales.values[at * 3 + c] as number);
+    }
+    cloud.opacities[at] = 1 / (1 + exactExp(-(cloud.logits.values[at] as number)));
+  }
+}
+
+/** One `GaussianSet` over the cloud's own arrays, whose `count` moves as the cloud does. */
+export function viewOf(cloud: Cloud, budget: number): GaussianSet {
+  const set: GaussianSet = {
+    count: 0,
+    positions: cloud.positions.values,
+    scales: cloud.scales,
+    rotations: cloud.rotations.values,
+    colors: cloud.colors.values,
+    opacities: cloud.opacities,
+  };
+  return cloud.sh1 === null ? set : { ...set, sh1: cloud.sh1.values.subarray(0, budget * 9) };
+}
+
+export function clear(gradients: GaussianGradients, count: number): void {
+  gradients.positions.fill(0, 0, count * 3);
+  gradients.scales.fill(0, 0, count * 3);
+  gradients.rotations.fill(0, 0, count * 4);
+  gradients.colors.fill(0, 0, count * 3);
+  gradients.sh1.fill(0, 0, count * SH1_COEFFICIENTS);
+  gradients.screen.fill(0, 0, count * SCREEN_GRADIENTS);
+  gradients.opacities.fill(0, 0, count);
+}
+
+/**
+ * The render's gradients onto the parameters that are actually stepped.
+ *
+ * Position, rotation, colour and the band are themselves; a scale arrives as a derivative by the
+ * metre and is wanted by the logarithm, which is one factor of the scale; an opacity arrives by the
+ * proportion and is wanted by the logit, which is the logistic's own derivative. **This is also
+ * where each Gaussian's pull is recorded** — how hard the frames are asking it to move, which is
+ * what the refinement grows the busiest of.
+ */
+export function chain(cloud: Cloud, gradients: GaussianGradients): void {
+  for (let at = 0; at < cloud.count; at += 1) {
+    let pull = 0;
+    for (let c = 0; c < 3; c += 1) {
+      const move = gradients.positions[at * 3 + c] as number;
+      cloud.positions.gradient[at * 3 + c] = move;
+      pull += move * move;
+      cloud.logScales.gradient[at * 3 + c] =
+        (gradients.scales[at * 3 + c] as number) * (cloud.scales[at * 3 + c] as number);
+      cloud.colors.gradient[at * 3 + c] = gradients.colors[at * 3 + c] as number;
+    }
+    cloud.motion[at] = (cloud.motion[at] as number) + Math.sqrt(pull);
+    for (let c = 0; c < 4; c += 1) {
+      cloud.rotations.gradient[at * 4 + c] = gradients.rotations[at * 4 + c] as number;
+    }
+    const opacity = cloud.opacities[at] as number;
+    cloud.logits.gradient[at] = (gradients.opacities[at] as number) * opacity * (1 - opacity);
+    if (cloud.sh1 === null) continue;
+    for (let c = 0; c < SH1_COEFFICIENTS; c += 1) {
+      cloud.sh1.gradient[at * SH1_COEFFICIENTS + c] = gradients.sh1[
+        at * SH1_COEFFICIENTS + c
+      ] as number;
+    }
+  }
+}
+
+/** The fitted cloud as a capture: single precision, cut to what it holds. */
+export function harvest(cloud: Cloud): SplatSource {
+  const { count } = cloud;
+  const source: SplatSource = {
+    count,
+    positions: Float32Array.from(cloud.positions.values.subarray(0, count * 3)),
+    scales: Float32Array.from(cloud.scales.subarray(0, count * 3)),
+    rotations: Float32Array.from(cloud.rotations.values.subarray(0, count * 4)),
+    colors: Float32Array.from(cloud.colors.values.subarray(0, count * 3)),
+    opacities: Float32Array.from(cloud.opacities.subarray(0, count)),
+  };
+  if (cloud.sh1 === null) return source;
+  return {
+    ...source,
+    sh1: Float32Array.from(cloud.sh1.values.subarray(0, count * SH1_COEFFICIENTS)),
+  };
+}

package/src/gaussians/project.ts

@@ -0,0 +1,207 @@
+/**
+ * Where a Gaussian lands on screen: the projection `@driftengine/splats`' shader performs, in
+ * JavaScript, and the order a frame draws them in.
+ *
+ * **The same model, in the capture's own frame.** A perspective divide is not linear, so a
+ * three-dimensional Gaussian does not project to a two-dimensional one; the standard treatment,
+ * which that shader uses and this repeats, linearises about the splat's centre — `Σ' = J·R·Σ·Rᵀ·Jᵀ`
+ * — and adds a low-pass of 0.3 to the screen variances so that a splat smaller than a pixel still
+ * covers one. The one difference is handedness: the shader works in the renderer's view space,
+ * where z runs away from the camera negatively, and a capture's cameras look down positive z. The
+ * arithmetic is the same and the frame is the caller's.
+ *
+ * **It exists to be differentiated.** `projectGradient.ts` is this chain run backwards, and
+ * `gradients.test.ts` holds the two together by a central difference of one against the other.
+ */
+export interface GaussianSet {
+  count: number;
+  /** Three per Gaussian, in the capture's world. */
+  readonly positions: Float64Array;
+  /** Three per Gaussian: a standard deviation along each of its own axes, in metres. */
+  readonly scales: Float64Array;
+  /** Four per Gaussian, xyzw, as the engine's quaternions are. */
+  readonly rotations: Float64Array;
+  /** Three per Gaussian, linear, which is a spherical harmonic's constant term. */
+  readonly colors: Float64Array;
+  /** Nine per Gaussian for the degree-1 band, or absent for a cloud with no view dependence. */
+  readonly sh1?: Float64Array;
+  /** One per Gaussian, 0 to 1. */
+  readonly opacities: Float64Array;
+}
+
+/**
+ * A frame this rasteriser writes: **single precision where it stands in for a render target, double
+ * where the caller is differentiating through it.** The fitting loop accumulates its loss in double
+ * and a frame quantised to single is a floor under every difference taken through one — measured at
+ * about a part in five hundred of the slope, against a part in ten million when the frame is double.
+ */
+export type Frame = Float32Array | Float64Array;
+
+export interface RasterCamera {
+  readonly width: number;
+  readonly height: number;
+  /** `fx`, `fy`, `cx`, `cy`. */
+  readonly intrinsics: readonly [number, number, number, number];
+  /** 3 × 4 row-major world-to-camera, as `estimatePoses` answers. */
+  readonly worldToCamera: Float64Array;
+}
+
+/** The low-pass the shader adds to the screen variances, in square pixels. */
+export const LOW_PASS = 0.3;
+/** How many standard deviations of a splat are drawn, and the largest radius in pixels. */
+export const REACH = 3;
+const MAX_RADIUS = 256;
+
+/** One Gaussian as the screen sees it. */
+export interface Projected {
+  /** Where its centre lands, in pixels. */
+  readonly x: number;
+  readonly y: number;
+  /** Its screen covariance, `[a, b, d]` for `[[a, b], [b, d]]`, the low-pass included. */
+  readonly a: number;
+  readonly b: number;
+  readonly d: number;
+  /** Along the camera's own axis, which is what the painter's order is by. */
+  readonly depth: number;
+  /** The half-width of its axis-aligned box, in pixels. */
+  readonly radius: number;
+}
+
+/** The rotation a quaternion stands for, into `out` (3 × 3 row-major). */
+export function rotationFromQuaternion(q: ArrayLike<number>, at: number, out: Float64Array): void {
+  const x = q[at * 4] as number;
+  const y = q[at * 4 + 1] as number;
+  const z = q[at * 4 + 2] as number;
+  const w = q[at * 4 + 3] as number;
+  const length = Math.sqrt(x * x + y * y + z * z + w * w) || 1;
+  const nx = x / length;
+  const ny = y / length;
+  const nz = z / length;
+  const nw = w / length;
+  out[0] = 1 - 2 * (ny * ny + nz * nz);
+  out[1] = 2 * (nx * ny - nz * nw);
+  out[2] = 2 * (nx * nz + ny * nw);
+  out[3] = 2 * (nx * ny + nz * nw);
+  out[4] = 1 - 2 * (nx * nx + nz * nz);
+  out[5] = 2 * (ny * nz - nx * nw);
+  out[6] = 2 * (nx * nz - ny * nw);
+  out[7] = 2 * (ny * nz + nx * nw);
+  out[8] = 1 - 2 * (nx * nx + ny * ny);
+}
+
+/** One Gaussian projected, or null where it is behind the camera or too flat to draw. */
+export function projectGaussian(
+  set: GaussianSet,
+  at: number,
+  camera: RasterCamera,
+): Projected | null {
+  const [fx, fy, cx, cy] = camera.intrinsics;
+  const m = camera.worldToCamera;
+  const view = [0, 0, 0];
+  for (let r = 0; r < 3; r += 1) {
+    view[r] =
+      (m[r * 4] as number) * (set.positions[at * 3] as number) +
+      (m[r * 4 + 1] as number) * (set.positions[at * 3 + 1] as number) +
+      (m[r * 4 + 2] as number) * (set.positions[at * 3 + 2] as number) +
+      (m[r * 4 + 3] as number);
+  }
+  const z = view[2] as number;
+  if (!(z > 1e-3)) return null;
+
+  /* The world covariance: R · diag(s²) · Rᵀ, turned into the camera's frame by the view rotation. */
+  const rotation = new Float64Array(9);
+  rotationFromQuaternion(set.rotations, at, rotation);
+  const scaled = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      const s = set.scales[at * 3 + c] as number;
+      scaled[r * 3 + c] = (rotation[r * 3 + c] as number) * s;
+    }
+  }
+  const world = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      let sum = 0;
+      for (let k = 0; k < 3; k += 1)
+        sum += (scaled[r * 3 + k] as number) * (scaled[c * 3 + k] as number);
+      world[r * 3 + c] = sum;
+    }
+  }
+  const inCamera = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      let sum = 0;
+      for (let k = 0; k < 3; k += 1) {
+        for (let j = 0; j < 3; j += 1) {
+          sum += (m[r * 4 + k] as number) * (world[k * 3 + j] as number) * (m[c * 4 + j] as number);
+        }
+      }
+      inCamera[r * 3 + c] = sum;
+    }
+  }
+
+  /* The projection's Jacobian at this point, in pixels. */
+  const invZ = 1 / z;
+  const invZ2 = invZ * invZ;
+  const j = [
+    [fx * invZ, 0, -fx * (view[0] as number) * invZ2],
+    [0, fy * invZ, -fy * (view[1] as number) * invZ2],
+  ];
+  const screen = [0, 0, 0, 0];
+  for (let r = 0; r < 2; r += 1) {
+    for (let c = 0; c < 2; c += 1) {
+      let sum = 0;
+      for (let k = 0; k < 3; k += 1) {
+        for (let l = 0; l < 3; l += 1) {
+          sum +=
+            ((j[r] as number[])[k] as number) *
+            (inCamera[k * 3 + l] as number) *
+            ((j[c] as number[])[l] as number);
+        }
+      }
+      screen[r * 2 + c] = sum;
+    }
+  }
+  const a = (screen[0] as number) + LOW_PASS;
+  const b = screen[1] as number;
+  const d = (screen[3] as number) + LOW_PASS;
+  const determinant = a * d - b * b;
+  if (!(determinant > 1e-12)) return null;
+  /* The ellipse's larger eigenvalue fixes how far it reaches. */
+  const mid = 0.5 * (a + d);
+  const spread = Math.sqrt(Math.max(0.1, mid * mid - determinant));
+  const radius = Math.min(MAX_RADIUS, REACH * Math.sqrt(Math.max(mid + spread, 0)));
+  return {
+    x: fx * (view[0] as number) * invZ + cx,
+    y: fy * (view[1] as number) * invZ + cy,
+    a,
+    b,
+    d,
+    depth: z,
+    radius,
+  };
+}
+
+/** The Gaussians a camera sees, nearest first, and where each lands. */
+export function visibleGaussians(
+  set: GaussianSet,
+  camera: RasterCamera,
+): { readonly order: number[]; readonly projected: (Projected | null)[] } {
+  const projected: (Projected | null)[] = [];
+  const order: number[] = [];
+  for (let at = 0; at < set.count; at += 1) {
+    const one = projectGaussian(set, at, camera);
+    projected.push(one);
+    if (one === null) continue;
+    if (one.x + one.radius < 0 || one.x - one.radius >= camera.width) continue;
+    if (one.y + one.radius < 0 || one.y - one.radius >= camera.height) continue;
+    order.push(at);
+  }
+  /* Nearest first, ties by index so the answer does not depend on the sort's stability. */
+  order.sort((p, q) => {
+    const first = (projected[p] as Projected).depth;
+    const second = (projected[q] as Projected).depth;
+    return first === second ? p - q : first - second;
+  });
+  return { order, projected };
+}

package/src/gaussians/projectGradient.ts

@@ -0,0 +1,234 @@
+/**
+ * The screen-space gradients of one Gaussian carried back to its own parameters.
+ *
+ * **`project.ts` run backwards**, term for term: the perspective Jacobian, the covariance it acts
+ * on, the rotation and scale that covariance is built from, and the quaternion behind the rotation.
+ * Nothing here knows about pixels or compositing — it takes what `gradients.ts` gathered over a
+ * Gaussian's own pixels and answers what that means for where it stands and how it is shaped.
+ */
+import { rotationFromQuaternion, type GaussianSet, type RasterCamera } from './project.ts';
+
+import type { GaussianGradients } from './gradients.ts';
+
+/**
+ * The screen-space gradients carried back to a Gaussian's position, scale and rotation, by the
+ * same chain the projection is built from.
+ */
+export function screenToParameters(
+  set: GaussianSet,
+  at: number,
+  camera: RasterCamera,
+  dMeanX: number,
+  dMeanY: number,
+  dA: number,
+  dB: number,
+  dD: number,
+  out: GaussianGradients,
+): void {
+  const [fx, fy] = camera.intrinsics;
+  const m = camera.worldToCamera;
+  const view = [0, 0, 0];
+  for (let r = 0; r < 3; r += 1) {
+    view[r] =
+      (m[r * 4] as number) * (set.positions[at * 3] as number) +
+      (m[r * 4 + 1] as number) * (set.positions[at * 3 + 1] as number) +
+      (m[r * 4 + 2] as number) * (set.positions[at * 3 + 2] as number) +
+      (m[r * 4 + 3] as number);
+  }
+  const z = view[2] as number;
+  const invZ = 1 / z;
+  const invZ2 = invZ * invZ;
+  const invZ3 = invZ2 * invZ;
+
+  /* The world covariance in the camera's frame, which the Jacobian acts on. */
+  const rotation = new Float64Array(9);
+  rotationFromQuaternion(set.rotations, at, rotation);
+  const scaled = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      scaled[r * 3 + c] = (rotation[r * 3 + c] as number) * (set.scales[at * 3 + c] as number);
+    }
+  }
+  const world = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      let sum = 0;
+      for (let k = 0; k < 3; k += 1)
+        sum += (scaled[r * 3 + k] as number) * (scaled[c * 3 + k] as number);
+      world[r * 3 + c] = sum;
+    }
+  }
+  const viewRotation = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) viewRotation[r * 3 + c] = m[r * 4 + c] as number;
+  }
+  const inCamera = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      let sum = 0;
+      for (let k = 0; k < 3; k += 1) {
+        for (let j = 0; j < 3; j += 1) {
+          sum +=
+            (viewRotation[r * 3 + k] as number) *
+            (world[k * 3 + j] as number) *
+            (viewRotation[c * 3 + j] as number);
+        }
+      }
+      inCamera[r * 3 + c] = sum;
+    }
+  }
+
+  const j = [
+    [fx * invZ, 0, -fx * (view[0] as number) * invZ2],
+    [0, fy * invZ, -fy * (view[1] as number) * invZ2],
+  ];
+  /* dΣ'/dΣcam, then the same into the world and the Gaussian's own axes. */
+  const dScreen = [
+    [dA, dB / 2],
+    [dB / 2, dD],
+  ];
+  const dCamera = new Float64Array(9);
+  for (let k = 0; k < 3; k += 1) {
+    for (let l = 0; l < 3; l += 1) {
+      let sum = 0;
+      for (let r = 0; r < 2; r += 1) {
+        for (let c = 0; c < 2; c += 1) {
+          sum +=
+            ((dScreen[r] as number[])[c] as number) *
+            ((j[r] as number[])[k] as number) *
+            ((j[c] as number[])[l] as number);
+        }
+      }
+      dCamera[k * 3 + l] = sum;
+    }
+  }
+  const dWorld = new Float64Array(9);
+  for (let k = 0; k < 3; k += 1) {
+    for (let l = 0; l < 3; l += 1) {
+      let sum = 0;
+      for (let r = 0; r < 3; r += 1) {
+        for (let c = 0; c < 3; c += 1) {
+          sum +=
+            (viewRotation[r * 3 + k] as number) *
+            (dCamera[r * 3 + c] as number) *
+            (viewRotation[c * 3 + l] as number);
+        }
+      }
+      dWorld[k * 3 + l] = sum;
+    }
+  }
+
+  /* Σ = M · Mᵀ with M = R · diag(s), so dΣ/dM = 2 · dΣ · M for a symmetric dΣ. */
+  const dM = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      let sum = 0;
+      for (let k = 0; k < 3; k += 1)
+        sum += (dWorld[r * 3 + k] as number) * (scaled[k * 3 + c] as number);
+      dM[r * 3 + c] = 2 * sum;
+    }
+  }
+  for (let c = 0; c < 3; c += 1) {
+    let sum = 0;
+    for (let r = 0; r < 3; r += 1)
+      sum += (dM[r * 3 + c] as number) * (rotation[r * 3 + c] as number);
+    out.scales[at * 3 + c] = (out.scales[at * 3 + c] as number) + sum;
+  }
+  /* And through the rotation to the quaternion, by its own derivatives. */
+  const dRotation = new Float64Array(9);
+  for (let r = 0; r < 3; r += 1) {
+    for (let c = 0; c < 3; c += 1) {
+      dRotation[r * 3 + c] = (dM[r * 3 + c] as number) * (set.scales[at * 3 + c] as number);
+    }
+  }
+  quaternionGradient(set.rotations, at, dRotation, out.rotations);
+
+  /*
+   * The position moves the mean and, through the Jacobian, the covariance. The second term is what
+   * a gradient written from the mean alone misses — it is small at the centre of the frame and is
+   * not small at its edges.
+   */
+  const dView = [
+    dMeanX * fx * invZ,
+    dMeanY * fy * invZ,
+    -dMeanX * fx * (view[0] as number) * invZ2 - dMeanY * fy * (view[1] as number) * invZ2,
+  ];
+  /* dJ/dview, the only part of Σ' that the position enters. */
+  const dJ = [
+    [
+      [0, 0, -fx * invZ2],
+      [0, 0, 0],
+    ],
+    [
+      [0, 0, 0],
+      [0, 0, -fy * invZ2],
+    ],
+    [
+      [-fx * invZ2, 0, 2 * fx * (view[0] as number) * invZ3],
+      [0, -fy * invZ2, 2 * fy * (view[1] as number) * invZ3],
+    ],
+  ];
+  for (let k = 0; k < 3; k += 1) {
+    for (let l = 0; l < 3; l += 1) {
+      for (let axis = 0; axis < 3; axis += 1) {
+        let sum = 0;
+        for (let r = 0; r < 2; r += 1) {
+          for (let c = 0; c < 2; c += 1) {
+            const term =
+              (((dJ[axis] as number[][])[r] as number[])[k] as number) *
+                ((j[c] as number[])[l] as number) +
+              ((j[r] as number[])[k] as number) *
+                (((dJ[axis] as number[][])[c] as number[])[l] as number);
+            sum += ((dScreen[r] as number[])[c] as number) * term * (inCamera[k * 3 + l] as number);
+          }
+        }
+        dView[axis] = (dView[axis] as number) + sum;
+      }
+    }
+  }
+  for (let axis = 0; axis < 3; axis += 1) {
+    let sum = 0;
+    for (let r = 0; r < 3; r += 1) sum += (dView[r] as number) * (m[r * 4 + axis] as number);
+    out.positions[at * 3 + axis] = (out.positions[at * 3 + axis] as number) + sum;
+  }
+}
+
+/** A rotation matrix's gradient carried back to the quaternion it came from. */
+function quaternionGradient(
+  q: Float64Array,
+  at: number,
+  dRotation: Float64Array,
+  out: Float64Array,
+): void {
+  const x = q[at * 4] as number;
+  const y = q[at * 4 + 1] as number;
+  const z = q[at * 4 + 2] as number;
+  const w = q[at * 4 + 3] as number;
+  const length = Math.sqrt(x * x + y * y + z * z + w * w) || 1;
+  const nx = x / length;
+  const ny = y / length;
+  const nz = z / length;
+  const nw = w / length;
+  /* dR/d(normalised quaternion), the nine entries against each of the four. */
+  const d = [
+    [0, 2 * ny, 2 * nz, 2 * ny, -4 * nx, -2 * nw, 2 * nz, 2 * nw, -4 * nx],
+    [-4 * ny, 2 * nx, 2 * nw, 2 * nx, 0, 2 * nz, -2 * nw, 2 * nz, -4 * ny],
+    [-4 * nz, -2 * nw, 2 * nx, 2 * nw, -4 * nz, 2 * ny, 2 * nx, 2 * ny, 0],
+    [0, -2 * nz, 2 * ny, 2 * nz, 0, -2 * nx, -2 * ny, 2 * nx, 0],
+  ];
+  /* Through the normalisation: the part of a change that lengthens the quaternion does nothing. */
+  const raw = [0, 0, 0, 0];
+  for (let p = 0; p < 4; p += 1) {
+    let sum = 0;
+    for (let i = 0; i < 9; i += 1)
+      sum += ((d[p] as number[])[i] as number) * (dRotation[i] as number);
+    raw[p] = sum;
+  }
+  const unit = [nx, ny, nz, nw];
+  let along = 0;
+  for (let p = 0; p < 4; p += 1) along += (raw[p] as number) * (unit[p] as number);
+  for (let p = 0; p < 4; p += 1) {
+    out[at * 4 + p] =
+      (out[at * 4 + p] as number) + ((raw[p] as number) - along * (unit[p] as number)) / length;
+  }
+}