@driftengine/capture 4.0.0

package/src/depth.ts

@@ -0,0 +1,100 @@
+/**
+ * Depth, its confidence and a camera per view, from a clip's frames rather than from tensors.
+ *
+ * **The frames arrive as a host hands them over** — RGBA bytes at whatever size the clip is — and
+ * this prepares them as Depth Anything 3's own processor does, runs the views together so the
+ * model's across-view blocks see each other, and decodes what comes back. A graph bakes its shapes,
+ * so one is built per count of views and prepared size and kept: a clip's frames are all the same
+ * size, so the second estimate over the same clip builds nothing.
+ *
+ * **It answers with its own arrays rather than filling the caller's**, which the engine's hot paths
+ * never do: the prepared size is not known until the frames are prepared, so a caller cannot have
+ * sized anything, and this runs once a clip rather than once a frame.
+ */
+import { graphFromWeights, type NetworkGraph, type WeightSource } from '@driftengine/texture';
+
+import {
+  decodeCamera,
+  decodeDepth,
+  depthAnything3,
+  type DepthAnything3Config,
+} from './models/depthAnything3.ts';
+import { prepareDepthFrame } from './prepare.ts';
+import type { GraphRun } from './run.ts';
+
+/** A frame as a host hands it over: RGBA bytes, and the size they are. */
+export interface RawFrame {
+  readonly pixels: Uint8Array;
+  readonly width: number;
+  readonly height: number;
+}
+
+export interface DepthView {
+  /** `[height, width]` metres, and one confidence each. */
+  readonly depth: Float32Array;
+  readonly confidence: Float32Array;
+  /** 3 × 4 row-major, world to camera. */
+  readonly worldToCamera: Float32Array;
+  /** 3 × 3 row-major. */
+  readonly intrinsics: Float32Array;
+}
+
+export interface DepthEstimate {
+  /** The size the frames were prepared at, which the maps and the cameras are in. */
+  readonly width: number;
+  readonly height: number;
+  readonly views: readonly DepthView[];
+}
+
+export interface DepthEstimator {
+  estimate(frames: readonly RawFrame[]): Promise<DepthEstimate>;
+}
+
+/** An estimator over one model's weights, running its graphs through `run`. */
+export function createDepthEstimator(
+  config: DepthAnything3Config,
+  weights: WeightSource,
+  run: GraphRun,
+  options: { readonly size?: number } = {},
+): DepthEstimator {
+  const size = options.size ?? 504;
+  const graphs = new Map<string, NetworkGraph>();
+  return {
+    async estimate(frames) {
+      if (frames.length === 0) throw new RangeError('a depth estimate needs a frame');
+      const prepared = frames.map((frame) =>
+        prepareDepthFrame(frame.pixels, frame.width, frame.height, size),
+      );
+      const { width, height } = prepared[0] as { width: number; height: number };
+      for (const [at, one] of prepared.entries()) {
+        if (one.width !== width || one.height !== height) {
+          throw new RangeError(
+            `frame ${at} prepares to ${one.width}×${one.height} and the first to ${width}×${height}` +
+              ': one estimate is one clip, at one size',
+          );
+        }
+      }
+      const key = `${frames.length}.${height}x${width}`;
+      let graph = graphs.get(key);
+      if (graph === undefined) {
+        graph = graphFromWeights(weights, depthAnything3(config, frames.length, height, width));
+        graphs.set(key, graph);
+      }
+      const out = await run(
+        graph,
+        new Map(prepared.map((one, view) => [`image${view}`, one.pixels])),
+      );
+      const pose = out.get('pose') as Float32Array;
+      const views = prepared.map((_, view) => {
+        const depth = new Float32Array(width * height);
+        const confidence = new Float32Array(width * height);
+        decodeDepth(out.get(`logits${view}`) as Float32Array, depth, confidence);
+        const worldToCamera = new Float32Array(12);
+        const intrinsics = new Float32Array(9);
+        decodeCamera(pose, view, height, width, worldToCamera, intrinsics);
+        return { depth, confidence, worldToCamera, intrinsics };
+      });
+      return { width, height, views };
+    },
+  };
+}

package/src/features.ts

@@ -0,0 +1,302 @@
+/**
+ * Corners in a frame, a binary description of each, and the matches between two frames.
+ *
+ * **A corner is a place where the image moves in two directions at once**, which is the one thing
+ * that can be found again from another angle: an edge slides along itself and a flat patch could be
+ * anywhere. So the response is Shi–Tomasi's — the smaller eigenvalue of the gradient's structure
+ * tensor, which is how much the weaker direction moves — kept where it is the largest in its own
+ * neighbourhood, and the strongest `budget` of those are the frame's features.
+ *
+ * **Each carries an orientation, and it is a vector rather than an angle.** The intensity centroid
+ * of a patch points away from its darker side; normalised, that is the cosine and sine the
+ * descriptor's pattern is turned by, so no inverse tangent is taken — which matters because this
+ * package is inside the determinism gate and `atan2` is one of the twenty-two functions no two
+ * engines must agree on.
+ *
+ * **The description is 256 comparisons of a blurred patch**, at pairs of places drawn once from a
+ * seed and turned by the feature's own orientation: a bit per pair, so two features compare in a
+ * few instructions and a frame's worth fits in memory. Blurred, because a comparison of two raw
+ * pixels is a comparison of two samples of the noise.
+ *
+ * **A match is mutual and unambiguous.** Each feature's nearest description in the other frame must
+ * also have it as its nearest, and each must be clearly nearer than its own second nearest —
+ * Lowe's ratio, applied from both sides — because a repeated texture answers every query equally
+ * well and a wrong match at this stage is a wrong camera later. **From both sides rather than one**,
+ * so matching the frames the other way round gives the same pairs; tested one-sided, it gave 46
+ * pairs one way and 48 the other, which would make a capture's answer depend on which frame it
+ * called first.
+ */
+import { mulberry32 } from '@driftengine/core';
+
+/** The features of one frame, in the arrays a caller owns. */
+export interface FeatureSet {
+  readonly x: Float32Array;
+  readonly y: Float32Array;
+  /** Shi–Tomasi's smaller eigenvalue: how strong a corner it is. */
+  readonly response: Float32Array;
+  /** The orientation, as a unit vector rather than an angle. */
+  readonly cos: Float32Array;
+  readonly sin: Float32Array;
+  count: number;
+}
+
+/** The descriptions of a frame's features: `BYTES` bytes each, bit by comparison. */
+export interface Descriptors {
+  readonly bits: Uint8Array;
+  readonly count: number;
+}
+
+/** A description is 256 bits, which is 32 bytes. */
+export const DESCRIPTOR_BYTES = 32;
+const PAIRS = DESCRIPTOR_BYTES * 8;
+/** The patch a description is drawn from, and the one an orientation is measured over. */
+const PATCH = 15;
+const SUPPRESS = 3;
+/** A corner must reach this share of the frame's strongest to be one at all. */
+const FLOOR = 0.01;
+/**
+ * And this much movement outright, in luma squared. **A share alone is not enough**: a wall carries
+ * a level or two of sensor noise, and a share of the strongest noise is still noise — measured, an
+ * uncorrelated level of ±1 answers with 265 "corners" of up to 6, and ±2 with 268 of up to 17,
+ * while the test scene's weakest real corner is 180 and its strongest 12,100. **25 sits between
+ * them**, three orders below what a texture scores and above what noise reaches.
+ */
+const LEAST = 25;
+
+export function createFeatureSet(budget: number): FeatureSet {
+  return {
+    x: new Float32Array(budget),
+    y: new Float32Array(budget),
+    response: new Float32Array(budget),
+    cos: new Float32Array(budget),
+    sin: new Float32Array(budget),
+    count: 0,
+  };
+}
+
+/* The pattern, drawn once: pairs of offsets inside the patch, the same on every machine. */
+const PATTERN = ((): Int8Array => {
+  const random = mulberry32(0x0c0ffee1);
+  const out = new Int8Array(PAIRS * 4);
+  const reach = (PATCH - 1) / 2;
+  for (let i = 0; i < PAIRS * 4; i += 1) {
+    /* A rounded draw over the patch, which keeps the pairs inside it once they are turned. */
+    out[i] = Math.round((random() * 2 - 1) * reach * 0.7);
+  }
+  return out;
+})();
+
+/** The frame as luma, one byte a pixel, which everything below reads. */
+function luma(image: Uint8Array, width: number, height: number): Float32Array {
+  const out = new Float32Array(width * height);
+  for (let at = 0; at < width * height; at += 1) {
+    out[at] =
+      0.299 * (image[at * 4] as number) +
+      0.587 * (image[at * 4 + 1] as number) +
+      0.114 * (image[at * 4 + 2] as number);
+  }
+  return out;
+}
+
+/** A 3 × 3 box blur, which is what a description compares rather than raw samples. */
+function blurred(grey: Float32Array, width: number, height: number): Float32Array {
+  const out = new Float32Array(grey.length);
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      let sum = 0;
+      for (let dy = -1; dy <= 1; dy += 1) {
+        const row = Math.min(height - 1, Math.max(0, y + dy)) * width;
+        for (let dx = -1; dx <= 1; dx += 1) {
+          sum += grey[row + Math.min(width - 1, Math.max(0, x + dx))] as number;
+        }
+      }
+      out[y * width + x] = sum / 9;
+    }
+  }
+  return out;
+}
+
+/**
+ * The strongest corners of `image`, RGBA at `width × height`, into `out` — at most `budget` of
+ * them, strongest first.
+ */
+export function detectFeatures(
+  image: Uint8Array,
+  width: number,
+  height: number,
+  out: FeatureSet,
+  budget: number,
+): void {
+  const grey = luma(image, width, height);
+  const response = new Float32Array(width * height);
+  const border = (PATCH - 1) / 2 + 1;
+  let strongest = 0;
+  for (let y = border; y < height - border; y += 1) {
+    for (let x = border; x < width - border; x += 1) {
+      /* The structure tensor over a 3 × 3 window of central differences. */
+      let xx = 0;
+      let yy = 0;
+      let xy = 0;
+      for (let dy = -1; dy <= 1; dy += 1) {
+        for (let dx = -1; dx <= 1; dx += 1) {
+          const at = (y + dy) * width + x + dx;
+          const gx = ((grey[at + 1] as number) - (grey[at - 1] as number)) / 2;
+          const gy = ((grey[at + width] as number) - (grey[at - width] as number)) / 2;
+          xx += gx * gx;
+          yy += gy * gy;
+          xy += gx * gy;
+        }
+      }
+      const half = (xx + yy) / 2;
+      const side = (xx - yy) / 2;
+      /* Shi–Tomasi: the smaller eigenvalue, which is the weaker direction's movement. */
+      const smaller = half - Math.sqrt(side * side + xy * xy);
+      response[y * width + x] = smaller;
+      strongest = Math.max(strongest, smaller);
+    }
+  }
+
+  /* Keep a response that is the largest within its neighbourhood and above the floor. */
+  const floor = Math.max(strongest * FLOOR, LEAST);
+  const kept: number[] = [];
+  for (let y = border; y < height - border; y += 1) {
+    for (let x = border; x < width - border; x += 1) {
+      const here = response[y * width + x] as number;
+      if (here < floor) continue;
+      let best = true;
+      for (let dy = -SUPPRESS; dy <= SUPPRESS && best; dy += 1) {
+        for (let dx = -SUPPRESS; dx <= SUPPRESS; dx += 1) {
+          if (dx === 0 && dy === 0) continue;
+          const other = response[(y + dy) * width + x + dx] as number;
+          /* A tie goes to the earlier pixel, so the answer does not depend on the scan. */
+          if (other > here || (other === here && (dy < 0 || (dy === 0 && dx < 0)))) {
+            best = false;
+            break;
+          }
+        }
+      }
+      if (best) kept.push(y * width + x);
+    }
+  }
+  kept.sort((a, b) => (response[b] as number) - (response[a] as number) || a - b);
+
+  const reach = (PATCH - 1) / 2;
+  out.count = Math.min(budget, out.x.length, kept.length);
+  for (let i = 0; i < out.count; i += 1) {
+    const at = kept[i] as number;
+    const x = at % width;
+    const y = (at - x) / width;
+    out.x[i] = x;
+    out.y[i] = y;
+    out.response[i] = response[at] as number;
+    /* The intensity centroid of the patch, as a direction. */
+    let mx = 0;
+    let my = 0;
+    for (let dy = -reach; dy <= reach; dy += 1) {
+      for (let dx = -reach; dx <= reach; dx += 1) {
+        if (dx * dx + dy * dy > reach * reach) continue;
+        const value = grey[(y + dy) * width + x + dx] as number;
+        mx += dx * value;
+        my += dy * value;
+      }
+    }
+    const length = Math.sqrt(mx * mx + my * my);
+    out.cos[i] = length === 0 ? 1 : mx / length;
+    out.sin[i] = length === 0 ? 0 : my / length;
+  }
+}
+
+/** A description of every feature in `features`, 32 bytes each. */
+export function describeFeatures(
+  image: Uint8Array,
+  width: number,
+  height: number,
+  features: FeatureSet,
+): Descriptors {
+  const smooth = blurred(luma(image, width, height), width, height);
+  const bits = new Uint8Array(features.count * DESCRIPTOR_BYTES);
+  for (let i = 0; i < features.count; i += 1) {
+    const x = features.x[i] as number;
+    const y = features.y[i] as number;
+    const cos = features.cos[i] as number;
+    const sin = features.sin[i] as number;
+    for (let pair = 0; pair < PAIRS; pair += 1) {
+      const sample = (at: number): number => {
+        const dx = PATTERN[pair * 4 + at * 2] as number;
+        const dy = PATTERN[pair * 4 + at * 2 + 1] as number;
+        /* The pattern turns with the feature, which is what makes it survive a rotation. */
+        const rx = Math.round(x + cos * dx - sin * dy);
+        const ry = Math.round(y + sin * dx + cos * dy);
+        const cx = Math.min(width - 1, Math.max(0, rx));
+        const cy = Math.min(height - 1, Math.max(0, ry));
+        return smooth[cy * width + cx] as number;
+      };
+      if (sample(0) < sample(1)) {
+        bits[i * DESCRIPTOR_BYTES + (pair >> 3)] |= 1 << (pair & 7);
+      }
+    }
+  }
+  return { bits, count: features.count };
+}
+
+const ONES = ((): Uint8Array => {
+  const out = new Uint8Array(256);
+  for (let i = 0; i < 256; i += 1) out[i] = (i & 1) + (out[i >> 1] as number);
+  return out;
+})();
+
+function distance(a: Uint8Array, at: number, b: Uint8Array, to: number): number {
+  let sum = 0;
+  for (let byte = 0; byte < DESCRIPTOR_BYTES; byte += 1) {
+    sum += ONES[(a[at + byte] as number) ^ (b[to + byte] as number)] as number;
+  }
+  return sum;
+}
+
+/**
+ * The matches between two frames' descriptions into `out`, two indices each, and how many there
+ * are. A match is mutual and clears Lowe's `ratio`, which defaults to 0.8.
+ */
+export function matchFeatures(
+  a: Descriptors,
+  b: Descriptors,
+  out: Int32Array,
+  options: { readonly ratio?: number } = {},
+): number {
+  const ratio = options.ratio ?? 0.8;
+  const bestOf = (
+    from: Descriptors,
+    to: Descriptors,
+    index: number,
+  ): { at: number; best: number; second: number } => {
+    let at = -1;
+    let best = Infinity;
+    let second = Infinity;
+    for (let j = 0; j < to.count; j += 1) {
+      const off = distance(from.bits, index * DESCRIPTOR_BYTES, to.bits, j * DESCRIPTOR_BYTES);
+      if (off < best) {
+        second = best;
+        best = off;
+        at = j;
+      } else if (off < second) {
+        second = off;
+      }
+    }
+    return { at, best, second };
+  };
+
+  let count = 0;
+  for (let i = 0; i < a.count && count * 2 + 1 < out.length; i += 1) {
+    const forward = bestOf(a, b, i);
+    if (forward.at < 0) continue;
+    /* Unambiguous: the nearest must be clearly nearer than whatever came second. */
+    if (!(forward.best < ratio * forward.second)) continue;
+    /* And mutual, by the same rule from the other side: matching either way gives one answer. */
+    const back = bestOf(b, a, forward.at);
+    if (back.at !== i || !(back.best < ratio * back.second)) continue;
+    out[count * 2] = i;
+    out[count * 2 + 1] = forward.at;
+    count += 1;
+  }
+  return count;
+}

package/src/frames.ts

@@ -0,0 +1,111 @@
+/**
+ * A clip's frames, and which of them a capture keeps.
+ *
+ * **The engine never opens a file.** A host hands it a `FrameSource` — a video element in a
+ * browser, a decoder on a native host, a folder of stills in a test — and the engine reads it one
+ * frame at a time, in order, into a buffer of its own. A minute of 1080p is eight gigabytes of
+ * pixels, so a clip is walked rather than held, and what a capture keeps is chosen as it goes past.
+ *
+ * **A frame may arrive later than it is asked for**, since every browser decoder is asynchronous, so
+ * `frameAt` may answer a promise and the two readers here are `async`. A host with its frames
+ * already in hand answers without one and pays nothing: an awaited value that is not a promise
+ * costs a microtask, and this is a clip's pace rather than a frame's.
+ *
+ * **Frames are chosen by motion, not by number.** What reconstruction needs is baseline: two views
+ * far enough apart to triangulate, and no more of them than the budget allows. So the frames kept
+ * are spaced by equal motion — a slow pan gives up most of its frames, a fast one keeps them — and
+ * a clip that never moves is one frame, which is what it holds. `every` is the other way to choose,
+ * for a caller that knows its clip: every Nth frame, under the same budget.
+ *
+ * **Motion is the mean absolute difference between one frame and the last**, over the colours and
+ * not the alpha, which is a measure of how much the picture changed rather than of how far anything
+ * travelled. **What it gives up** is telling a pan from a light being switched on; what would make
+ * it wrong is a clip whose brightness swings without the camera moving, and the answer then is the
+ * feature matching of Task 9 rather than a cleverer average.
+ */
+
+export interface FrameSize {
+  readonly width: number;
+  readonly height: number;
+}
+
+export interface FrameSource {
+  frameCount(): number;
+  /**
+   * Writes frame `index` as RGBA into `out` and answers the frame's size. A buffer too small for
+   * the frame is left untouched, so a caller may ask with an empty one to learn the size first.
+   */
+  frameAt(index: number, out: Uint8Array): FrameSize | Promise<FrameSize>;
+}
+
+export interface FrameChoice {
+  /** At most this many frames. */
+  readonly budget: number;
+  /** Every Nth frame instead of by motion. */
+  readonly every?: number;
+}
+
+const NOTHING = new Uint8Array(0);
+
+/** Every frame in order, each read over the last into one buffer the visitor must not keep. */
+export async function eachFrame(
+  source: FrameSource,
+  visit: (index: number, pixels: Uint8Array, width: number, height: number) => void,
+): Promise<void> {
+  const count = source.frameCount();
+  if (count <= 0) return;
+  const { width, height } = await source.frameAt(0, NOTHING);
+  const pixels = new Uint8Array(width * height * 4);
+  for (let index = 0; index < count; index += 1) {
+    const size = await source.frameAt(index, pixels);
+    visit(index, pixels, size.width, size.height);
+  }
+}
+
+/** The frames a capture keeps, in order, the first among them. */
+export async function selectFrames(source: FrameSource, choice: FrameChoice): Promise<number[]> {
+  const count = source.frameCount();
+  const budget = Math.max(1, Math.floor(choice.budget));
+  if (count <= 0) return [];
+  if (choice.every !== undefined) {
+    const kept: number[] = [];
+    for (let index = 0; index < count && kept.length < budget; index += choice.every) {
+      kept.push(index);
+    }
+    return kept;
+  }
+  if (count <= budget) return Array.from({ length: count }, (_, index) => index);
+
+  /* One pass for the motion between each frame and the last, keeping a copy of that last frame. */
+  const travelled = new Float64Array(count);
+  let last: Uint8Array | null = null;
+  await eachFrame(source, (index, pixels) => {
+    if (last !== null) {
+      let sum = 0;
+      for (let at = 0; at < pixels.length; at += 4) {
+        sum +=
+          Math.abs((pixels[at] as number) - (last[at] as number)) +
+          Math.abs((pixels[at + 1] as number) - (last[at + 1] as number)) +
+          Math.abs((pixels[at + 2] as number) - (last[at + 2] as number));
+      }
+      travelled[index] = (travelled[index - 1] as number) + sum / ((pixels.length / 4) * 3);
+    } else {
+      last = new Uint8Array(pixels.length);
+    }
+    last.set(pixels);
+  });
+
+  const total = travelled[count - 1] as number;
+  if (total === 0) return [0];
+  /* Spaced so the whole clip's motion divides into the budget's frames, the last one included. */
+  const step = total / (budget - 1);
+  const kept = [0];
+  let since = 0;
+  for (let index = 1; index < count && kept.length < budget; index += 1) {
+    if ((travelled[index] as number) - since >= step * (1 - 1e-9)) {
+      kept.push(index);
+      since = travelled[index] as number;
+    }
+  }
+  return kept;
+}