@driftengine/capture 4.0.0

package/src/prepare.ts

@@ -0,0 +1,306 @@
+/**
+ * A frame as a model takes it: the preparation each upstream does to its own inputs, done here so a
+ * host hands the engine pixels and nothing else.
+ *
+ * **Depth Anything 3**, as its `InputProcessor` prepares a frame: the longest side brought to 504
+ * by an area resize, each side then rounded to the nearest whole patch of 14 by another, the bytes
+ * divided by 255 and normalised by ImageNet's mean and deviation, and laid out channel by row. A
+ * clip's aspect ratio decides the grid, which is why the graph is rebuilt per size.
+ *
+ * **MobileSAM**, as its predictor prepares one: the longest side to 1,024 by Pillow's bilinear,
+ * normalised by the upstream's pixel mean and deviation in 0–255, and padded with zeros after its
+ * last row and column to the square the encoder takes. The prepared frame's own size is answered
+ * too, since a prompt's coordinates are in it and a mask comes back cropped to it.
+ *
+ * **SAM 2.1**, as its processor prepares one: the frame to a 1,024 square whatever its shape —
+ * each axis scaled by its own factor, which is why a prompt's coordinates scale by their own too —
+ * through torchvision's antialiased bilinear, divided by 255 and normalised by ImageNet's.
+ *
+ * **OWLv2**, as its processor prepares one, which is the odd one: the bytes divided by 255 first,
+ * the frame padded with zeros after its last row and column to a square — so a box's coordinates
+ * scale by the longer side — blurred by a gaussian wide enough for the reduction, resized to 960
+ * by a plain bilinear with no antialiasing of its own, and normalised by CLIP's statistics. The
+ * blur is what the antialiasing is here, which is why the resize after it has none.
+ *
+ * **A frame smaller than the model's size is refused rather than guessed at.** The upstream scales
+ * such a frame up with a cubic resize, which this does not have; a wrong preparation is a wrong
+ * depth that looks like a right one, and a refusal names what is missing. What would lift it is
+ * OpenCV's `INTER_CUBIC`, written and held to the same parity as the area resize.
+ */
+import { exactExp } from '@driftengine/core';
+import { resize } from '@driftengine/texture';
+
+import { areaResize, triangleResize, TORCHVISION_PRECISION } from './resample.ts';
+
+export const DEPTH_PIXEL_MEAN = [0.485, 0.456, 0.406] as const;
+export const DEPTH_PIXEL_STD = [0.229, 0.224, 0.225] as const;
+
+export interface PreparedFrame {
+  /** `[3, height, width]`, normalised. */
+  readonly pixels: Float32Array;
+  readonly width: number;
+  readonly height: number;
+}
+
+/** Python's `round`: halves to even, which is what the upstream's sizes are rounded by. */
+function halfToEven(value: number): number {
+  const down = Math.floor(value);
+  const rest = value - down;
+  if (rest > 0.5) return down + 1;
+  if (rest < 0.5) return down;
+  return down % 2 === 0 ? down : down + 1;
+}
+
+/** The nearest multiple of `patch`, the larger one on a tie, as the upstream rounds it. */
+function nearestMultiple(value: number, patch: number): number {
+  const down = Math.floor(value / patch) * patch;
+  const up = down + patch;
+  return up - value <= value - down ? up : down;
+}
+
+/**
+ * `rgba`, `width × height` pixels of four bytes, as Depth Anything 3 takes it at `size`, in whole
+ * patches of `patch`.
+ */
+export function prepareDepthFrame(
+  rgba: Uint8Array,
+  width: number,
+  height: number,
+  size = 504,
+  patch = 14,
+): PreparedFrame {
+  const longest = Math.max(width, height);
+  if (longest < size) {
+    throw new RangeError(
+      `a frame of ${width}×${height} is smaller than the model's ${size} and would have to be ` +
+        'scaled up, which needs the cubic resize this does not have',
+    );
+  }
+  const scale = size / longest;
+  let held = rgba;
+  let at = { width, height };
+  if (longest !== size) {
+    at = {
+      width: Math.max(1, halfToEven(width * scale)),
+      height: Math.max(1, halfToEven(height * scale)),
+    };
+    held = new Uint8Array(at.width * at.height * 4);
+    areaResize(rgba, width, height, 4, held, at.width, at.height);
+  }
+  const whole = {
+    width: Math.max(1, nearestMultiple(at.width, patch)),
+    height: Math.max(1, nearestMultiple(at.height, patch)),
+  };
+  if (whole.width > at.width || whole.height > at.height) {
+    throw new RangeError(
+      `a frame of ${at.width}×${at.height} rounds up to ${whole.width}×${whole.height} whole ` +
+        'patches, which needs the cubic resize this does not have',
+    );
+  }
+  if (whole.width !== at.width || whole.height !== at.height) {
+    const rounded = new Uint8Array(whole.width * whole.height * 4);
+    areaResize(held, at.width, at.height, 4, rounded, whole.width, whole.height);
+    held = rounded;
+    at = whole;
+  }
+  const pixels = new Float32Array(3 * at.width * at.height);
+  const cells = at.width * at.height;
+  for (let channel = 0; channel < 3; channel += 1) {
+    const mean = DEPTH_PIXEL_MEAN[channel] as number;
+    const deviation = DEPTH_PIXEL_STD[channel] as number;
+    for (let cell = 0; cell < cells; cell += 1) {
+      const value = (held[cell * 4 + channel] as number) / 255;
+      pixels[channel * cells + cell] = Math.fround((Math.fround(value) - mean) / deviation);
+    }
+  }
+  return { pixels, width: at.width, height: at.height };
+}
+
+/** The size a frame is resized to, its longest side at `size`, rounded up at a half. */
+export function longestSideSize(
+  width: number,
+  height: number,
+  size: number,
+): { readonly width: number; readonly height: number } {
+  const scale = size / Math.max(width, height);
+  return {
+    width: Math.trunc(width * scale + 0.5),
+    height: Math.trunc(height * scale + 0.5),
+  };
+}
+
+/**
+ * `rgba` as MobileSAM's predictor prepares it: `[3, square, square]`, the frame itself in the top
+ * left at the size this answers, and zeros after it.
+ */
+export function prepareSamFrame(
+  rgba: Uint8Array,
+  width: number,
+  height: number,
+  mean: readonly number[],
+  deviation: readonly number[],
+  square = 1024,
+): PreparedFrame {
+  const at = longestSideSize(width, height, square);
+  let held = rgba;
+  if (at.width !== width || at.height !== height) {
+    held = new Uint8Array(at.width * at.height * 4);
+    triangleResize(rgba, width, height, 4, held, at.width, at.height);
+  }
+  const pixels = new Float32Array(3 * square * square);
+  for (let channel = 0; channel < 3; channel += 1) {
+    const centre = mean[channel] as number;
+    const spread = deviation[channel] as number;
+    for (let y = 0; y < at.height; y += 1) {
+      for (let x = 0; x < at.width; x += 1) {
+        const value = held[(y * at.width + x) * 4 + channel] as number;
+        pixels[channel * square * square + y * square + x] = Math.fround((value - centre) / spread);
+      }
+    }
+  }
+  return { pixels, width: at.width, height: at.height };
+}
+
+/**
+ * `rgba` as SAM 2's processor prepares it: `[3, square, square]`, each axis scaled on its own, and
+ * normalised by `mean` and `deviation` in 0–1.
+ */
+export function prepareSam2Frame(
+  rgba: Uint8Array,
+  width: number,
+  height: number,
+  mean: readonly number[] = DEPTH_PIXEL_MEAN,
+  deviation: readonly number[] = DEPTH_PIXEL_STD,
+  square = 1024,
+): PreparedFrame {
+  let held = rgba;
+  if (width !== square || height !== square) {
+    held = new Uint8Array(square * square * 4);
+    triangleResize(rgba, width, height, 4, held, square, square, TORCHVISION_PRECISION);
+  }
+  const pixels = new Float32Array(3 * square * square);
+  const cells = square * square;
+  for (let channel = 0; channel < 3; channel += 1) {
+    const centre = mean[channel] as number;
+    const spread = deviation[channel] as number;
+    for (let cell = 0; cell < cells; cell += 1) {
+      const value = Math.fround((held[cell * 4 + channel] as number) / 255);
+      pixels[channel * cells + cell] = Math.fround((value - centre) / spread);
+    }
+  }
+  return { pixels, width: square, height: square };
+}
+
+/* torchvision's 1-D gaussian: samples about the centre, normalised to sum to one. */
+function gaussianKernel(size: number, sigma: number): Float32Array {
+  const half = (size - 1) * 0.5;
+  const kernel = new Float32Array(size);
+  let total = 0;
+  for (let i = 0; i < size; i += 1) {
+    /* `linspace` from −half to half, which is how torchvision places the samples. */
+    const at = size === 1 ? 0 : -half + (2 * half * i) / (size - 1);
+    const value = Math.fround(exactExp(-0.5 * (at / sigma) * (at / sigma)));
+    kernel[i] = value;
+    total += value;
+  }
+  for (let i = 0; i < size; i += 1) kernel[i] = Math.fround((kernel[i] as number) / total);
+  return kernel;
+}
+
+/* A separable blur with the edges reflected, as torchvision's `gaussian_blur` pads them. */
+function blur(
+  values: Float32Array,
+  channels: number,
+  height: number,
+  width: number,
+  down: Float32Array,
+  across: Float32Array,
+): Float32Array {
+  const reflect = (at: number, length: number): number => {
+    let index = at;
+    while (index < 0 || index >= length) index = index < 0 ? -index : 2 * length - index - 2;
+    return index;
+  };
+  const middle = new Float32Array(values.length);
+  const half = (across.length - 1) / 2;
+  for (let c = 0; c < channels; c += 1) {
+    for (let y = 0; y < height; y += 1) {
+      for (let x = 0; x < width; x += 1) {
+        let sum = 0;
+        for (let k = 0; k < across.length; k += 1) {
+          sum = Math.fround(
+            sum +
+              Math.fround(
+                (values[(c * height + y) * width + reflect(x + k - half, width)] as number) *
+                  (across[k] as number),
+              ),
+          );
+        }
+        middle[(c * height + y) * width + x] = sum;
+      }
+    }
+  }
+  const out = new Float32Array(values.length);
+  const tall = (down.length - 1) / 2;
+  for (let c = 0; c < channels; c += 1) {
+    for (let y = 0; y < height; y += 1) {
+      for (let x = 0; x < width; x += 1) {
+        let sum = 0;
+        for (let k = 0; k < down.length; k += 1) {
+          sum = Math.fround(
+            sum +
+              Math.fround(
+                (middle[(c * height + reflect(y + k - tall, height)) * width + x] as number) *
+                  (down[k] as number),
+              ),
+          );
+        }
+        out[(c * height + y) * width + x] = sum;
+      }
+    }
+  }
+  return out;
+}
+
+/** `rgba` as OWLv2's processor prepares it: `[3, square, square]` at the model's size. */
+export function prepareOwlv2Frame(
+  rgba: Uint8Array,
+  width: number,
+  height: number,
+  mean: readonly number[],
+  deviation: readonly number[],
+  square = 960,
+): PreparedFrame {
+  const side = Math.max(width, height);
+  const padded = new Float32Array(3 * side * side);
+  for (let channel = 0; channel < 3; channel += 1) {
+    for (let y = 0; y < height; y += 1) {
+      for (let x = 0; x < width; x += 1) {
+        padded[(channel * side + y) * side + x] = Math.fround(
+          (rgba[(y * width + x) * 4 + channel] as number) / 255,
+        );
+      }
+    }
+  }
+  const factor = side / square;
+  const sigma = Math.max(0, (factor - 1) / 2);
+  let blurred: Float32Array = padded;
+  if (sigma > 0) {
+    const size = 2 * Math.ceil(3 * sigma) + 1;
+    const kernel = gaussianKernel(size, sigma);
+    blurred = blur(padded, 3, side, side, kernel, kernel);
+  }
+  const resized = new Float32Array(3 * square * square);
+  resize(resized, blurred, 3, side, side, square, square, 'bilinear', false);
+  const cells = square * square;
+  for (let channel = 0; channel < 3; channel += 1) {
+    const centre = mean[channel] as number;
+    const spread = deviation[channel] as number;
+    for (let cell = 0; cell < cells; cell += 1) {
+      resized[channel * cells + cell] = Math.fround(
+        ((resized[channel * cells + cell] as number) - centre) / spread,
+      );
+    }
+  }
+  return { pixels: resized, width: square, height: square };
+}

package/src/propose.ts

@@ -0,0 +1,96 @@
+/**
+ * Each region as something a scene could hold — a *proposal*, and the word is load-bearing.
+ *
+ * **This is stage four and it is labelled as such.** A capture that guesses what an object is and
+ * acts on the guess is a capture that puts a door where a cupboard was, and the cost is paid by
+ * somebody who cannot see why. So a region arrives with what is known about it and nothing more,
+ * and **a region nobody labelled becomes static scenery** — the safe default in the strict sense:
+ * drawn, solid, and doing nothing. Being wrong about that costs nothing at all.
+ *
+ * The component names are the engine's own, as data. Nothing here imports an entity model, because
+ * a proposal a consumer has not accepted yet should not have built anything.
+ */
+import { exactCos } from '@driftengine/core';
+
+import type { Region } from './segment.ts';
+
+export interface ProposeOptions {
+  /** The steepest surface a character could stand on, in degrees. */
+  readonly slopeDegrees?: number;
+}
+
+const DEFAULT_SLOPE = 45;
+
+/** What a scene could make of a region. */
+export interface EntityProposal {
+  /** Its index in the regions it came from. */
+  readonly region: number;
+  readonly bounds: Float64Array;
+  readonly label: string | null;
+  /**
+   * Components a scene might give it, by the engine's own names.
+   *
+   * A *suggestion*, and the word is load-bearing: nothing here knows what the thing is, only what
+   * shape it is and whether anybody named it.
+   */
+  readonly components: readonly string[];
+  /** Whether a character could stand on it. */
+  readonly walkable: boolean;
+}
+
+/** Drawn and solid and nothing else, which is what an unlabelled region gets. */
+export const SCENERY_COMPONENTS: readonly string[] = ['transform', 'mesh', 'collider'];
+/** The same, and a navigation surface on top of it. */
+export const WALKABLE_COMPONENTS: readonly string[] = [
+  'transform',
+  'mesh',
+  'collider',
+  'navigation',
+];
+/** Something small enough and named: a scene may want to move it. */
+export const MOVABLE_COMPONENTS: readonly string[] = ['transform', 'mesh', 'collider', 'body'];
+
+/**
+ * Each region as a proposal.
+ *
+ * **An unlabelled region is scenery**, whatever its shape: drawn, solid, and doing nothing. A named
+ * region small enough to pick up is offered as movable, and the floor — the largest surface a
+ * character could stand on — is offered with a navigation surface. Nothing here is decided; a
+ * consumer reads `components` as a starting point and overrides whatever it likes.
+ */
+export function proposeEntities(
+  regions: readonly Region[],
+  options: ProposeOptions = {},
+): EntityProposal[] {
+  const slope = exactCos(((options.slopeDegrees ?? DEFAULT_SLOPE) * Math.PI) / 180);
+  let floor = -1;
+  let widest = 0;
+  regions.forEach((region, at) => {
+    if ((region.normal[1] as number) < slope) return;
+    if (region.area <= widest) return;
+    widest = region.area;
+    floor = at;
+  });
+
+  return regions.map((region, at) => {
+    const walkable = at === floor;
+    const span = Math.max(
+      (region.bounds[3] as number) - (region.bounds[0] as number),
+      (region.bounds[4] as number) - (region.bounds[1] as number),
+      (region.bounds[5] as number) - (region.bounds[2] as number),
+    );
+    /* Named and small enough that a person could shift it: the only case that is not scenery. */
+    const movable = region.label !== null && span < 1.2;
+    return {
+      region: at,
+      bounds: region.bounds,
+      label: region.label,
+      components: walkable
+        ? WALKABLE_COMPONENTS
+        : movable
+          ? MOVABLE_COMPONENTS
+          : SCENERY_COMPONENTS,
+      walkable,
+    };
+  });
+}

package/src/resample.ts

@@ -0,0 +1,197 @@
+/**
+ * The resampling a model's own preparation does, as the library its upstream uses does it.
+ *
+ * **Area resizing is OpenCV's `INTER_AREA`**, which Depth Anything 3's `InputProcessor` resizes a
+ * frame down with: an output pixel is the average of the source rectangle it covers, with the
+ * pixels at either end weighted by the fraction of them inside it. It is the right filter for
+ * shrinking a photograph — every source pixel reaches the output exactly once, so nothing aliases
+ * and nothing is dropped — and it is what the model was trained on, which is the reason that
+ * matters here.
+ *
+ * **The antialiased bilinear is the other one**, which MobileSAM's predictor and SAM 2's processor
+ * each resize a frame with: a triangle filter whose support grows with the reduction, so shrinking
+ * averages rather than samples. One axis at a time, through an eight-bit intermediate, with
+ * coefficients normalised to sum to one and held in fixed point — **and how many fraction bits they
+ * keep is the whole difference between the two libraries**: Pillow keeps 22 and torchvision's
+ * eight-bit path keeps 15, and at each the engine answers what that library answers, measured on
+ * the pinned frames at 0 of 3,145,728 values apart. A coefficient rounded differently is a pixel
+ * rounded differently, which is why this is a number rather than an approximation.
+ *
+ * **The area resize is summed the way OpenCV sums it**: one weight per source pixel of the rectangle, the row's
+ * times the column's, accumulated in single precision, and rounded by halves to even as `cvRound`
+ * does. Measured against OpenCV on the two pinned frames, that leaves **24 values of 846,720 a
+ * single level of 255 apart** — a sum landing either side of a half — where summing in double and
+ * rounding halves away from zero leaves 46. What would close the rest is OpenCV's own fixed-point
+ * table, which is worth writing when a measurement says a level of 255 matters.
+ */
+
+const f = Math.fround;
+
+/** `cvRound`: halves to even, which is what OpenCV's saturating cast does. */
+function toByte(value: number): number {
+  const down = Math.floor(value);
+  const rest = value - down;
+  const rounded = rest > 0.5 ? down + 1 : rest < 0.5 ? down : down % 2 === 0 ? down : down + 1;
+  return Math.min(255, Math.max(0, rounded));
+}
+
+/** One output axis's source span: where it starts, and the weight of each source pixel in it. */
+interface Span {
+  readonly first: number;
+  readonly weights: Float64Array;
+}
+
+function spans(from: number, to: number): Span[] {
+  const scale = from / to;
+  const out: Span[] = [];
+  for (let at = 0; at < to; at += 1) {
+    const start = at * scale;
+    const end = start + scale;
+    const first = Math.ceil(start);
+    const last = Math.floor(end);
+    /* The cell is the part of it inside the image, as OpenCV's `resizeArea` measures it. */
+    const cell = Math.min(scale, from - start);
+    const weights: number[] = [];
+    if (first > start) weights.push((first - start) / cell);
+    for (let source = first; source < last; source += 1) weights.push(1 / cell);
+    if (end > last && last < from) weights.push((end - last) / cell);
+    out.push({
+      first: Math.min(first > start ? first - 1 : first, from - 1),
+      weights: Float64Array.from(weights),
+    });
+  }
+  return out;
+}
+
+/**
+ * `src`, `width × height` pixels of `channels` bytes, into `out` at `outWidth × outHeight`. Only
+ * shrinking is defined, which is what a frame's preparation asks for.
+ */
+export function areaResize(
+  src: Uint8Array,
+  width: number,
+  height: number,
+  channels: number,
+  out: Uint8Array,
+  outWidth: number,
+  outHeight: number,
+): void {
+  const across = spans(width, outWidth);
+  const down = spans(height, outHeight);
+  for (let y = 0; y < outHeight; y += 1) {
+    const rows = down[y] as Span;
+    for (let x = 0; x < outWidth; x += 1) {
+      const columns = across[x] as Span;
+      for (let c = 0; c < channels; c += 1) {
+        let sum = 0;
+        for (let dy = 0; dy < rows.weights.length; dy += 1) {
+          const row = (rows.first + dy) * width;
+          for (let dx = 0; dx < columns.weights.length; dx += 1) {
+            const weight = f(f(rows.weights[dy] as number) * f(columns.weights[dx] as number));
+            sum = f(sum + f((src[(row + columns.first + dx) * channels + c] as number) * weight));
+          }
+        }
+        out[(y * outWidth + x) * channels + c] = toByte(sum);
+      }
+    }
+  }
+}
+
+/** Pillow's fraction bits, and torchvision's eight-bit path's. */
+export const PILLOW_PRECISION = 22;
+export const TORCHVISION_PRECISION = 15;
+
+/** One output pixel's source pixels and their fixed-point weights. */
+function triangleCoefficients(
+  from: number,
+  to: number,
+  precision: number,
+): { first: number; weights: Int32Array }[] {
+  const scale = from / to;
+  const filterScale = Math.max(1, scale);
+  const support = filterScale;
+  const out: { first: number; weights: Int32Array }[] = [];
+  for (let at = 0; at < to; at += 1) {
+    const centre = (at + 0.5) * scale;
+    const first = Math.max(0, Math.trunc(centre - support + 0.5));
+    const last = Math.min(from, Math.trunc(centre + support + 0.5));
+    const weights: number[] = [];
+    let total = 0;
+    for (let source = first; source < last; source += 1) {
+      /* The triangle, over a support the reduction widens. */
+      const distance = Math.abs((source - centre + 0.5) / filterScale);
+      const weight = distance < 1 ? 1 - distance : 0;
+      weights.push(weight);
+      total += weight;
+    }
+    const fixed = new Int32Array(weights.length);
+    for (let i = 0; i < weights.length; i += 1) {
+      const share = total === 0 ? 0 : (weights[i] as number) / total;
+      /* A coefficient rounds away from zero at this precision. */
+      fixed[i] =
+        share < 0
+          ? -Math.floor(-share * (1 << precision) + 0.5)
+          : Math.floor(share * (1 << precision) + 0.5);
+    }
+    out.push({ first, weights: fixed });
+  }
+  return out;
+}
+
+/** The clamp of a fixed-point sum back to a byte. */
+function clip8(sum: number, precision: number): number {
+  const value = Math.floor(sum / (1 << precision));
+  return value < 0 ? 0 : value > 255 ? 255 : value;
+}
+
+/**
+ * `src` into `out` by the antialiased bilinear: across first, then down, through an eight-bit
+ * intermediate, which is the order both libraries' passes have. `precision` says whose rounding.
+ */
+export function triangleResize(
+  src: Uint8Array,
+  width: number,
+  height: number,
+  channels: number,
+  out: Uint8Array,
+  outWidth: number,
+  outHeight: number,
+  precision: number = PILLOW_PRECISION,
+): void {
+  const rounding = 1 << (precision - 1);
+  const across = triangleCoefficients(width, outWidth, precision);
+  const middle = width === outWidth ? src : new Uint8Array(outWidth * height * channels);
+  if (middle !== src) {
+    for (let y = 0; y < height; y += 1) {
+      for (let x = 0; x < outWidth; x += 1) {
+        const { first, weights } = across[x] as { first: number; weights: Int32Array };
+        for (let c = 0; c < channels; c += 1) {
+          let sum = rounding;
+          for (let i = 0; i < weights.length; i += 1) {
+            sum += (src[(y * width + first + i) * channels + c] as number) * (weights[i] as number);
+          }
+          middle[(y * outWidth + x) * channels + c] = clip8(sum, precision);
+        }
+      }
+    }
+  }
+  const down = triangleCoefficients(height, outHeight, precision);
+  if (height === outHeight) {
+    out.set(middle.subarray(0, outWidth * outHeight * channels));
+    return;
+  }
+  for (let y = 0; y < outHeight; y += 1) {
+    const { first, weights } = down[y] as { first: number; weights: Int32Array };
+    for (let x = 0; x < outWidth; x += 1) {
+      for (let c = 0; c < channels; c += 1) {
+        let sum = rounding;
+        for (let i = 0; i < weights.length; i += 1) {
+          sum +=
+            (middle[((first + i) * outWidth + x) * channels + c] as number) *
+            (weights[i] as number);
+        }
+        out[(y * outWidth + x) * channels + c] = clip8(sum, precision);
+      }
+    }
+  }
+}

package/src/run.ts

@@ -0,0 +1,7 @@
+/** How a capture runs a graph: the device's runner, or the reference evaluator. */
+import type { NetworkGraph } from '@driftengine/texture';
+
+export type GraphRun = (
+  graph: NetworkGraph,
+  inputs: ReadonlyMap<string, Float32Array>,
+) => Promise<ReadonlyMap<string, Float32Array>>;