@driftengine/capture 4.0.0

package/src/fusion.ts

@@ -0,0 +1,229 @@
+/**
+ * Many views of a scene fused into one volume: a truncated signed distance, weighted by how
+ * squarely each view saw the surface.
+ *
+ * **Why a volume rather than the depth maps themselves.** Every view has a surface in it and no two
+ * agree exactly — noise, a fit's own error, and the silhouettes where a splat cloud thins out. A
+ * volume is where they are averaged: each view writes its opinion of the distance to the nearest
+ * surface into the samples along its own rays, and what comes out is the surface every view agrees
+ * on. `marchVolume` then reads the crossing.
+ *
+ * **Negative inside**, which is `ObjectSdf`'s convention in `@driftengine/assets` and the one the
+ * rest of this engine reads a distance field by. A sample nearer the camera than the surface is
+ * outside it and positive.
+ *
+ * **Truncated, because a depth map knows nothing about the far side of what it sees.** Past a few
+ * samples behind the surface a view has no information at all — the space there might be solid,
+ * might be another room — so writing a large negative distance would be inventing one. Everything
+ * beyond the truncation on the far side is left alone; everything beyond it on the near side is
+ * free space and is written as such, which is what carves the empty air out.
+ *
+ * **Weighted by the angle, which is what stops a grazing view winning.** A surface seen almost
+ * edge-on is a few pixels wide and every one of them is uncertain in depth by the whole width of
+ * the surface; seen square on, the same surface is measured well. The weight is the cosine between
+ * the view's own ray and the surface's normal, taken from the depth map's own gradient, so a
+ * grazing view contributes what it is worth rather than as much as any other.
+ *
+ * **A surface one view saw is kept, at the weight one view is worth.** Dropping it would leave a
+ * hole where the capture only ever looked once, which is most of a real clip's edges.
+ */
+import type { RasterCamera } from './gaussians/project.ts';
+
+/**
+ * One view's opinion: its depth, how well covered each pixel was, and where it stood.
+ *
+ * **Not `DepthView`, which is the depth *model's* shape** — that one carries a confidence and its
+ * own 3 × 3 intrinsics and does not know the size of its own frame. The two say much the same thing
+ * in two forms, and the pipeline that joins them is the place to convert, not here.
+ */
+export interface SurfaceView {
+  /** Along the camera's own axis, in metres; zero where nothing was seen. */
+  readonly depth: Float32Array;
+  /** Nought to one, as `renderDepth` answers it. */
+  readonly coverage: Float32Array;
+  readonly camera: RasterCamera;
+}
+
+/** A field of samples over a box of the world, `x` fastest, then `y`, then `z`. */
+export interface Volume {
+  readonly dims: readonly [number, number, number];
+  /** Where the first sample sits, in the capture's world. */
+  readonly origin: readonly [number, number, number];
+  /** Metres between samples, the same on every axis. */
+  readonly spacing: number;
+  /** Truncated signed distance in metres, **negative inside**. */
+  readonly distance: Float32Array;
+  /** How much evidence each sample has; zero where no view ever saw it. */
+  readonly weight: Float32Array;
+}
+
+export interface FuseOptions {
+  /**
+   * How far either side of a surface a view is believed, in metres.
+   *
+   * Three samples by default: enough that the crossing is bracketed by samples with real evidence
+   * on both sides, which is what marching needs, and short enough that one view's surface does not
+   * reach through a wall into the next room.
+   */
+  readonly truncation?: number;
+  /** Below this coverage a pixel's depth is an average of too little light to believe. */
+  readonly minimumCoverage?: number;
+}
+
+const DEFAULT_COVERAGE = 0.5;
+
+export function createVolume(
+  dims: readonly [number, number, number],
+  origin: readonly [number, number, number],
+  spacing: number,
+): Volume {
+  const [nx, ny, nz] = dims;
+  return {
+    dims,
+    origin,
+    spacing,
+    distance: new Float32Array(nx * ny * nz),
+    weight: new Float32Array(nx * ny * nz),
+  };
+}
+
+/**
+ * Every view's surface written into `volume`, which accumulates rather than replaces.
+ *
+ * Calling this twice with two halves of the views is the same as calling it once with all of them,
+ * because a weighted mean is what each sample holds — which is what lets a clip be fused as it
+ * arrives rather than all at once.
+ */
+export function fuseDepth(
+  views: readonly SurfaceView[],
+  volume: Volume,
+  options: FuseOptions = {},
+): void {
+  const [nx, ny, nz] = volume.dims;
+  const truncation = options.truncation ?? volume.spacing * 3;
+  const minimumCoverage = options.minimumCoverage ?? DEFAULT_COVERAGE;
+  if (!(truncation > 0)) throw new RangeError('capture: a fusion needs a truncation above zero');
+
+  for (const view of views) {
+    const { width, height, intrinsics } = view.camera;
+    const [fx, fy, cx, cy] = intrinsics;
+    const m = view.camera.worldToCamera;
+    const normals = surfaceNormals(view);
+
+    for (let k = 0; k < nz; k += 1) {
+      for (let j = 0; j < ny; j += 1) {
+        for (let i = 0; i < nx; i += 1) {
+          const world = [
+            (volume.origin[0] as number) + i * volume.spacing,
+            (volume.origin[1] as number) + j * volume.spacing,
+            (volume.origin[2] as number) + k * volume.spacing,
+          ];
+          let viewX = 0;
+          let viewY = 0;
+          let viewZ = 0;
+          for (let r = 0; r < 3; r += 1) {
+            const value =
+              (m[r * 4] as number) * (world[0] as number) +
+              (m[r * 4 + 1] as number) * (world[1] as number) +
+              (m[r * 4 + 2] as number) * (world[2] as number) +
+              (m[r * 4 + 3] as number);
+            if (r === 0) viewX = value;
+            else if (r === 1) viewY = value;
+            else viewZ = value;
+          }
+          if (!(viewZ > 0)) continue;
+          const px = Math.floor((fx * viewX) / viewZ + cx);
+          const py = Math.floor((fy * viewY) / viewZ + cy);
+          if (px < 0 || px >= width || py < 0 || py >= height) continue;
+          const pixel = py * width + px;
+          if ((view.coverage[pixel] as number) < minimumCoverage) continue;
+          const measured = view.depth[pixel] as number;
+          if (!(measured > 0)) continue;
+
+          /*
+           * How squarely this view saw that surface: the ray through the pixel against the
+           * surface's own normal, both in the camera's frame. A view looking straight at it is
+           * worth one and a grazing view almost nothing.
+           */
+          const rayX = (px + 0.5 - cx) / fx;
+          const rayY = (py + 0.5 - cy) / fy;
+          const rayLength = Math.sqrt(rayX * rayX + rayY * rayY + 1);
+          const facing = -(
+            (rayX * (normals[pixel * 3] as number) +
+              rayY * (normals[pixel * 3 + 1] as number) +
+              (normals[pixel * 3 + 2] as number)) /
+            rayLength
+          );
+          const angle = facing > 0 ? facing : 0;
+          const weight = angle * (view.coverage[pixel] as number);
+          if (!(weight > 0)) continue;
+
+          /*
+           * **In front of the surface is outside it, and the distance is measured along the
+           * surface's normal rather than along the camera's axis.** The difference of two depths is
+           * a distance along the view's own ray, which is the perpendicular distance only where the
+           * surface faces the camera squarely; everywhere else it overstates it by the obliquity.
+           * For a plane the correction is exact, and for a curved surface it is the first-order
+           * term — measured on a sphere fused from eight views, it took the median error in the
+           * band from 25 mm to 11 and the ninetieth percentile from 55 mm to 43. **The worst sample
+           * does not move**, because it is one at the very edge of the band whose distance
+           * saturates at the truncation either way.
+           */
+          const signed = (measured - viewZ) * rayLength * angle;
+          if (signed < -truncation) continue;
+          const clamped = signed > truncation ? truncation : signed;
+
+          const at = (k * ny + j) * nx + i;
+          const held = volume.weight[at] as number;
+          const total = held + weight;
+          volume.distance[at] = ((volume.distance[at] as number) * held + clamped * weight) / total;
+          volume.weight[at] = total;
+        }
+      }
+    }
+  }
+}
+
+/**
+ * A normal per pixel, in the camera's frame, from the depth map's own gradient.
+ *
+ * **Central differences, and a pixel whose neighbours are not both surfaces gets the view's own
+ * axis instead.** At a silhouette the difference is across the gap between a near surface and a far
+ * one, which is not a gradient of anything; answering the axis there says "seen square on", which
+ * is generous rather than wrong, and the coverage is what actually discounts those pixels.
+ */
+function surfaceNormals(view: SurfaceView): Float64Array {
+  const { width, height, intrinsics } = view.camera;
+  const [fx, fy, cx, cy] = intrinsics;
+  const out = new Float64Array(width * height * 3);
+  const at = (x: number, y: number): readonly [number, number, number] | null => {
+    if (x < 0 || x >= width || y < 0 || y >= height) return null;
+    const z = view.depth[y * width + x] as number;
+    if (!(z > 0)) return null;
+    return [((x + 0.5 - cx) * z) / fx, ((y + 0.5 - cy) * z) / fy, z];
+  };
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      const pixel = y * width + x;
+      out[pixel * 3 + 2] = -1;
+      const left = at(x - 1, y);
+      const right = at(x + 1, y);
+      const up = at(x, y - 1);
+      const down = at(x, y + 1);
+      if (left === null || right === null || up === null || down === null) continue;
+      const ax = [right[0] - left[0], right[1] - left[1], right[2] - left[2]];
+      const ay = [down[0] - up[0], down[1] - up[1], down[2] - up[2]];
+      const nx = (ay[1] as number) * (ax[2] as number) - (ay[2] as number) * (ax[1] as number);
+      const ny = (ay[2] as number) * (ax[0] as number) - (ay[0] as number) * (ax[2] as number);
+      const nz = (ay[0] as number) * (ax[1] as number) - (ay[1] as number) * (ax[0] as number);
+      const length = Math.sqrt(nx * nx + ny * ny + nz * nz);
+      if (!(length > 0)) continue;
+      /* Towards the camera, which is the negative z half of its own frame. */
+      const sign = nz > 0 ? -1 : 1;
+      out[pixel * 3] = (sign * nx) / length;
+      out[pixel * 3 + 1] = (sign * ny) / length;
+      out[pixel * 3 + 2] = (sign * nz) / length;
+    }
+  }
+  return out;
+}

package/src/gaussians/adam.ts

@@ -0,0 +1,131 @@
+/**
+ * Adam over a cloud's parameters, one moment pair per number.
+ *
+ * **Why Adam and not a plain descent.** The six families being fitted are in wildly different
+ * units — metres, logarithms of metres, a quaternion, a colour on nought to one — and a single
+ * step size that suits one of them moves another by nothing or by everything. Adam divides each
+ * number's step by the size of its own recent gradients, so a rate here is a *fraction of the
+ * parameter's own scale* and the families become comparable. It is also what makes the loss's
+ * absolute size irrelevant: a frame of a million pixels and one of a thousand descend alike.
+ *
+ * **The bias correction is a running product rather than a power.** `β₁ⁿ` would want `**`, which
+ * `packages/capture` may not use — the determinism gate refuses it, because an exponent is one of
+ * the operations a platform may round its own way. Multiplying by β each step is the same number
+ * by the same arithmetic the rest of the fit uses.
+ */
+
+const BETA1 = 0.9;
+const BETA2 = 0.999;
+const EPSILON = 1e-8;
+
+/**
+ * One optimised family: the values, their gradient, and Adam's two moments.
+ *
+ * Every array is allocated at the cloud's budget and used to `count * stride`, because the budget
+ * is the one thing the cloud may not exceed — so densification never reallocates and a plan can
+ * gather into the same arrays it reads.
+ */
+export interface Family {
+  /** Numbers per Gaussian. */
+  readonly stride: number;
+  /** The step size, as a fraction of what this family's own gradients have been. */
+  readonly rate: number;
+  readonly values: Float64Array;
+  readonly gradient: Float64Array;
+  readonly moment: Float64Array;
+  readonly second: Float64Array;
+}
+
+export function createFamily(stride: number, rate: number, budget: number): Family {
+  return {
+    stride,
+    rate,
+    values: new Float64Array(budget * stride),
+    gradient: new Float64Array(budget * stride),
+    moment: new Float64Array(budget * stride),
+    second: new Float64Array(budget * stride),
+  };
+}
+
+/** How far into the decay the fit is: β₁ⁿ and β₂ⁿ, carried rather than raised. */
+export interface AdamState {
+  unbias1: number;
+  unbias2: number;
+}
+
+export function createAdamState(): AdamState {
+  return { unbias1: 1, unbias2: 1 };
+}
+
+/**
+ * One step over every family, on the gradients standing in them.
+ *
+ * **The gradient is read and not owned**: the caller writes every number of it before every step,
+ * so there is nothing here to clear. A caller that accumulated into it instead would need that
+ * clear back, and would be the one to add it.
+ */
+export function adamStep(families: readonly Family[], count: number, state: AdamState): void {
+  state.unbias1 *= BETA1;
+  state.unbias2 *= BETA2;
+  const correction1 = 1 - state.unbias1;
+  const correction2 = 1 - state.unbias2;
+  for (const family of families) {
+    const { values, gradient, moment, second, rate, stride } = family;
+    const end = count * stride;
+    for (let at = 0; at < end; at += 1) {
+      const g = gradient[at] as number;
+      const m = BETA1 * (moment[at] as number) + (1 - BETA1) * g;
+      const v = BETA2 * (second[at] as number) + (1 - BETA2) * g * g;
+      moment[at] = m;
+      second[at] = v;
+      values[at] =
+        (values[at] as number) -
+        (rate * (m / correction1)) / (Math.sqrt(v / correction2) + EPSILON);
+    }
+  }
+}
+
+/**
+ * Rewrite every family by a plan: slot `i` takes its values from `sources[i]`.
+ *
+ * **A slot marked fresh starts Adam over.** A Gaussian that was just split off its parent has none
+ * of that parent's history — its gradients are about to be different ones — and carrying the
+ * moments across would take a step sized for a shape that no longer exists. The reference
+ * implementation zeroes them for the same reason, and it is also what separates two halves of a
+ * split: identical values with identical moments receive identical steps forever.
+ */
+export function gather(
+  families: readonly Family[],
+  sources: Int32Array,
+  fresh: Uint8Array,
+  count: number,
+  scratch: Float64Array,
+): void {
+  for (const family of families) {
+    gatherArray(family.values, family.stride, sources, fresh, count, scratch, false);
+    gatherArray(family.moment, family.stride, sources, fresh, count, scratch, true);
+    gatherArray(family.second, family.stride, sources, fresh, count, scratch, true);
+  }
+}
+
+/** One array gathered through `scratch`, because a slot may take from a slot below itself. */
+function gatherArray(
+  array: Float64Array,
+  stride: number,
+  sources: Int32Array,
+  fresh: Uint8Array,
+  count: number,
+  scratch: Float64Array,
+  blankWhenFresh: boolean,
+): void {
+  for (let slot = 0; slot < count; slot += 1) {
+    const from = (sources[slot] as number) * stride;
+    for (let c = 0; c < stride; c += 1) scratch[slot * stride + c] = array[from + c] as number;
+  }
+  for (let slot = 0; slot < count; slot += 1) {
+    const blank = blankWhenFresh && (fresh[slot] as number) !== 0;
+    for (let c = 0; c < stride; c += 1) {
+      array[slot * stride + c] = blank ? 0 : (scratch[slot * stride + c] as number);
+    }
+  }
+}

package/src/gaussians/depth.ts

@@ -0,0 +1,87 @@
+/**
+ * How far away a cloud of Gaussians is, a pixel at a time.
+ *
+ * **A splat cloud has no surface**, so there is no depth to read off it the way a triangle
+ * rasteriser reads one. What it has is light arriving from a sequence of overlapping blobs, and the
+ * honest answer is where that light came from: each splat's own depth, weighted by how much of the
+ * pixel it actually contributed. A nearly opaque splat in front takes almost all of the weight, so
+ * the answer is its depth; a thin haze in front of a wall averages towards the wall.
+ *
+ * **The coverage is answered beside it, and callers need both.** At a silhouette a cloud thins out
+ * and a pixel may be a tenth covered — a depth averaged out of a tenth of a pixel's light is not a
+ * measurement of anything, and a fusion that treated it as one would carve the silhouette into the
+ * volume. So this answers how much light arrived as well, and the fusion weights by it rather than
+ * choosing a cutoff here on the caller's behalf.
+ *
+ * **Depth along the camera's own axis, not along the ray.** That is what a projection's `z` is and
+ * what `fuseDepth` compares a voxel against; a radial distance would be a different quantity with
+ * the same name, which is the kind of mistake that reads as a scene slightly the wrong shape.
+ */
+import { exactExp } from '@driftengine/core';
+
+import {
+  visibleGaussians,
+  type GaussianSet,
+  type Projected,
+  type RasterCamera,
+} from './project.ts';
+
+/** Below this a Gaussian contributes less than a quarter of a value of an eight-bit channel. */
+const FAINT = 1 / 255 / 4;
+
+/**
+ * The cloud's depth and coverage, one of each per pixel.
+ *
+ * A pixel no light reached is left at zero in both, which is a coverage of nothing rather than a
+ * surface at the camera — a caller reading the depth alone would find a wall in front of its face.
+ */
+export function renderDepth(
+  set: GaussianSet,
+  camera: RasterCamera,
+  depth: Float32Array,
+  coverage: Float32Array,
+): void {
+  const { width, height } = camera;
+  depth.fill(0);
+  coverage.fill(0);
+  const pixels = width * height;
+  const transmittance = new Float64Array(pixels).fill(1);
+  const weighted = new Float64Array(pixels);
+  const { order, projected } = visibleGaussians(set, camera);
+
+  for (const at of order) {
+    const one = projected[at] as Projected;
+    const determinant = one.a * one.d - one.b * one.b;
+    const inverse = [one.d / determinant, -one.b / determinant, one.a / determinant];
+    const left = Math.max(0, Math.floor(one.x - one.radius));
+    const right = Math.min(width - 1, Math.ceil(one.x + one.radius));
+    const top = Math.max(0, Math.floor(one.y - one.radius));
+    const bottom = Math.min(height - 1, Math.ceil(one.y + one.radius));
+    const opacity = set.opacities[at] as number;
+    for (let y = top; y <= bottom; y += 1) {
+      for (let x = left; x <= right; x += 1) {
+        const dx = x + 0.5 - one.x;
+        const dy = y + 0.5 - one.y;
+        const power =
+          -0.5 *
+          ((inverse[0] as number) * dx * dx +
+            2 * (inverse[1] as number) * dx * dy +
+            (inverse[2] as number) * dy * dy);
+        const alpha = opacity * exactExp(power);
+        if (alpha < FAINT) continue;
+        const pixel = y * width + x;
+        const share = alpha * (transmittance[pixel] as number);
+        weighted[pixel] = (weighted[pixel] as number) + share * one.depth;
+        coverage[pixel] = (coverage[pixel] as number) + share;
+        transmittance[pixel] = (transmittance[pixel] as number) * (1 - alpha);
+      }
+    }
+  }
+
+  for (let pixel = 0; pixel < pixels; pixel += 1) {
+    const covered = coverage[pixel] as number;
+    /* Divided by the light that arrived rather than by one: a half-covered pixel is still at the
+       depth of what covered it, and the caller is told how much that was. */
+    if (covered > 0) depth[pixel] = (weighted[pixel] as number) / covered;
+  }
+}