@driftengine/capture 4.0.0

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@driftengine/capture",
+  "version": "4.0.0",
+  "description": "A video to a playable scene, on the player's device: the models it runs and the stages that use them",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "drift-source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json",
+    "./*": "./*"
+  },
+  "files": [
+    "dist",
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/*.test.mjs",
+    "!src/**/__snapshots__",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ],
+  "sideEffects": false,
+  "peerDependencies": {
+    "@driftengine/core": "4.0.0",
+    "@driftengine/drft": "4.0.0",
+    "@driftengine/nav": "4.0.0",
+    "@driftengine/physics": "4.0.0",
+    "@driftengine/splats": "4.0.0",
+    "@driftengine/texture": "4.0.0"
+  },
+  "author": "Drift Technologies",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/drftrun/driftengine.git",
+    "directory": "packages/capture"
+  },
+  "homepage": "https://github.com/drftrun/driftengine#readme",
+  "bugs": "https://github.com/drftrun/driftengine/issues",
+  "keywords": [
+    "driftengine",
+    "3d",
+    "typescript",
+    "photogrammetry",
+    "depth-estimation",
+    "gaussian-splatting",
+    "reconstruction"
+  ],
+  "engines": {
+    "node": ">=22.12.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "prepack": "node ../../scripts/build.mjs @driftengine/capture"
+  }
+}

package/src/browserFrames.ts

@@ -0,0 +1,145 @@
+/**
+ * A clip in a browser as a `FrameSource`: a video element decoded into a canvas, a frame at a time.
+ *
+ * **A declared browser module** — a native host writes its own source over its own decoder, which is
+ * the whole point of the seam. Nothing else in this package touches a DOM.
+ *
+ * **Why a video element rather than WebCodecs.** `VideoDecoder` takes encoded chunks and a browser
+ * ships no demuxer to make them from an MP4, so a WebCodecs source means a demuxing dependency in a
+ * package whose bar for one is high. A video element demuxes, decodes and seeks with nothing added,
+ * and `drawImage` into a 2D canvas is how its frame becomes bytes. **What it gives up**: a frame is
+ * addressed by time rather than by its place in the stream — the middle of frame `index` at the
+ * clip's rate — so a clip of variable frame rate answers the frame covering that instant rather
+ * than the `index`-th one. What would make that wrong is a capture that needs a particular frame
+ * rather than a moment; nothing here does, since frames are chosen by motion.
+ *
+ * **The rate is measured, not assumed**, from the first frames the decoder presents, and falls back
+ * to the caller's `frameRate` where `requestVideoFrameCallback` is absent.
+ */
+import type { FrameSize, FrameSource } from './frames.ts';
+
+/** A browser's clip, and how to let it go. */
+export interface BrowserClip extends FrameSource {
+  /** Releases the element and the object URL; the source answers nothing afterwards. */
+  close(): void;
+}
+
+interface FrameCallbackMetadata {
+  readonly mediaTime: number;
+  readonly presentedFrames: number;
+}
+type VideoWithCallback = HTMLVideoElement & {
+  requestVideoFrameCallback?: (
+    callback: (now: number, metadata: FrameCallbackMetadata) => void,
+  ) => number;
+};
+
+const once = (video: HTMLVideoElement, event: string): Promise<void> =>
+  new Promise((resolve, reject) => {
+    const done = (): void => {
+      video.removeEventListener(event, done);
+      video.removeEventListener('error', failed);
+      resolve();
+    };
+    const failed = (): void => {
+      video.removeEventListener(event, done);
+      video.removeEventListener('error', failed);
+      reject(new Error(`the clip could not be read: ${video.error?.message ?? 'no reason given'}`));
+    };
+    video.addEventListener(event, done);
+    video.addEventListener('error', failed);
+  });
+
+/** The rate the decoder presents frames at, from two it presents, or null where it cannot say. */
+async function measuredRate(video: VideoWithCallback): Promise<number | null> {
+  if (typeof video.requestVideoFrameCallback !== 'function') return null;
+  const times: number[] = [];
+  await new Promise<void>((resolve) => {
+    const take = (_now: number, metadata: FrameCallbackMetadata): void => {
+      times.push(metadata.mediaTime);
+      if (times.length >= 3) resolve();
+      else video.requestVideoFrameCallback?.(take);
+    };
+    video.requestVideoFrameCallback?.(take);
+    void video.play().catch(() => resolve());
+    setTimeout(resolve, 2000);
+  });
+  video.pause();
+  const steps: number[] = [];
+  for (let i = 1; i < times.length; i += 1)
+    steps.push((times[i] as number) - (times[i - 1] as number));
+  const shortest = Math.min(...steps.filter((step) => step > 0));
+  return Number.isFinite(shortest) && shortest > 0 ? 1 / shortest : null;
+}
+
+/**
+ * The clip's frames, addressed at the clip's rate. `frameRate` is the fallback where the browser
+ * cannot say what the rate is.
+ */
+export async function browserFrameSource(
+  clip: Blob,
+  options: { readonly frameRate?: number } = {},
+): Promise<BrowserClip> {
+  // platform: browser default — this module is the browser's own `FrameSource`.
+  const url = URL.createObjectURL(clip);
+  const video = document.createElement('video') as VideoWithCallback;
+  video.preload = 'auto';
+  video.muted = true;
+  video.playsInline = true;
+  video.src = url;
+  await once(video, 'loadedmetadata');
+  /*
+   * **A container that parses is not a track that decodes**, and the difference is silent.
+   *
+   * Chrome on Linux plays no HEVC, which is what every recent phone records by default. The file
+   * loads, `loadedmetadata` fires, `duration` comes back correct to the millisecond — and
+   * `videoWidth` is zero, because nothing decoded. Left alone, the first frame draws a zero-wide
+   * image into a zero-wide canvas and dies inside `getImageData` with "the source width is 0",
+   * which names neither the clip nor the codec and sends a reader into the canvas code.
+   *
+   * So it is refused here, by name, with the duration as evidence that the file itself is fine.
+   * **What this cannot do is say which codec it was**: the element reports no track it could not
+   * decode, so the message names the one that always works rather than guessing at the one that
+   * did not.
+   */
+  if (video.videoWidth === 0 || video.videoHeight === 0) {
+    const seconds = Number.isFinite(video.duration) ? video.duration.toFixed(2) : 'an unknown';
+    throw new Error(
+      `this browser cannot decode the clip's video track: it read ${seconds} s of container and ` +
+        'no picture. A codec this browser does not have is the usual cause — HEVC on a build ' +
+        'without it, which is what most phones record. Convert the clip to H.264 and try again.',
+    );
+  }
+  const rate = (await measuredRate(video)) ?? options.frameRate ?? 30;
+  const width = video.videoWidth;
+  const height = video.videoHeight;
+  const count = Math.max(1, Math.floor(video.duration * rate));
+  const canvas = new OffscreenCanvas(width, height);
+  const context = canvas.getContext('2d', { willReadFrequently: true });
+  if (context === null) throw new Error('a clip needs a 2D canvas to become pixels');
+  let open = true;
+
+  return {
+    frameCount: () => (open ? count : 0),
+    async frameAt(index: number, out: Uint8Array): Promise<FrameSize> {
+      if (!open) throw new Error('the clip is closed');
+      if (out.length >= width * height * 4) {
+        /* The middle of the frame, so rounding cannot land on the boundary either side. */
+        const at = Math.min((index + 0.5) / rate, Math.max(0, video.duration - 1 / rate));
+        if (Math.abs(video.currentTime - at) > 1e-6) {
+          video.currentTime = at;
+          await once(video, 'seeked');
+        }
+        context.drawImage(video, 0, 0, width, height);
+        out.set(context.getImageData(0, 0, width, height).data);
+      }
+      return { width, height };
+    },
+    close() {
+      open = false;
+      video.removeAttribute('src');
+      video.load();
+      URL.revokeObjectURL(url);
+    },
+  };
+}

package/src/collide.ts

@@ -0,0 +1,128 @@
+/**
+ * A captured mesh as something to stand on and to put in a file.
+ *
+ * **The mesh a capture draws and the mesh it collides against are the same triangles and not the
+ * same array.** What comes out of `marchVolume` and `decimate` is built for a picture: it carries
+ * normals and colours a collision test never reads, and it carries the occasional triangle with no
+ * area, which is harmless in a render and is not harmless in a contact — a plane needs a normal,
+ * and a triangle with no area has none. So the collision mesh is the drawn one with the arealess
+ * triangles taken out and the vertices nothing uses any more taken with them.
+ *
+ * **A room is a static mesh body and a prop is a set of hulls, and the difference is not size.** A
+ * mesh body is exact and cannot move: every triangle is tested as it stands, which is what a floor
+ * and a wall want. A convex decomposition is approximate and can be thrown: a hull has a centre of
+ * mass, an inertia and a cheap support function, which is what a crate wants. Giving a room hulls
+ * would round off its corners; giving a crate a mesh body would leave it unable to be picked up.
+ *
+ * **The container's caps are the decomposition's brief.** `COLL` holds at most thirty-two hulls of
+ * at most sixty-four points, so a decomposition that ignored them would produce a prop that cannot
+ * be written — found at the writer, after the expensive part. It is asked for inside them instead.
+ */
+import { MAX_COLLIDER_HULLS, type MeshData } from '@driftengine/drft';
+import { decomposeConvex, type DecomposeOptions } from '@driftengine/physics';
+
+/** A mesh as `meshShape` takes it, and what had to go for it to be one. */
+export interface CollisionSource {
+  readonly positions: Float32Array;
+  readonly indices: Uint32Array;
+  /** Triangles dropped for having no area to have a plane from. */
+  readonly dropped: number;
+}
+
+export interface CollisionOptions {
+  /**
+   * The smallest triangle kept, in square metres.
+   *
+   * **Not zero**, because a triangle of a square micrometre has a plane in the arithmetic and none
+   * in any useful sense: its normal is the direction of its own rounding error, and a contact
+   * against it points somewhere arbitrary. A square millimetre is far below anything a capture
+   * resolves and far above where a cross product stops meaning anything.
+   */
+  readonly minimumArea?: number;
+}
+
+const DEFAULT_MINIMUM_AREA = 1e-6;
+
+/** `mesh`'s triangles, less the ones with no area, renumbered onto the vertices that remain. */
+export function collisionMesh(mesh: MeshData, options: CollisionOptions = {}): CollisionSource {
+  const minimumArea = options.minimumArea ?? DEFAULT_MINIMUM_AREA;
+  const triangles = mesh.indices.length / 3;
+  const renumbered = new Map<number, number>();
+  const positions: number[] = [];
+  const indices: number[] = [];
+  let dropped = 0;
+
+  for (let face = 0; face < triangles; face += 1) {
+    const a = mesh.indices[face * 3] as number;
+    const b = mesh.indices[face * 3 + 1] as number;
+    const c = mesh.indices[face * 3 + 2] as number;
+    if (areaOf(mesh.positions, a, b, c) < minimumArea) {
+      dropped += 1;
+      continue;
+    }
+    for (const vertex of [a, b, c]) {
+      let at = renumbered.get(vertex);
+      if (at === undefined) {
+        at = positions.length / 3;
+        renumbered.set(vertex, at);
+        for (let k = 0; k < 3; k += 1) positions.push(mesh.positions[vertex * 3 + k] as number);
+      }
+      indices.push(at);
+    }
+  }
+
+  return {
+    positions: Float32Array.from(positions),
+    indices: Uint32Array.from(indices),
+    dropped,
+  };
+}
+
+/** Twice the area of a triangle, halved: the length of the cross product of two of its edges. */
+function areaOf(positions: Float32Array, a: number, b: number, c: number): number {
+  const ux = (positions[b * 3] as number) - (positions[a * 3] as number);
+  const uy = (positions[b * 3 + 1] as number) - (positions[a * 3 + 1] as number);
+  const uz = (positions[b * 3 + 2] as number) - (positions[a * 3 + 2] as number);
+  const vx = (positions[c * 3] as number) - (positions[a * 3] as number);
+  const vy = (positions[c * 3 + 1] as number) - (positions[a * 3 + 1] as number);
+  const vz = (positions[c * 3 + 2] as number) - (positions[a * 3 + 2] as number);
+  const nx = uy * vz - uz * vy;
+  const ny = uz * vx - ux * vz;
+  const nz = ux * vy - uy * vx;
+  return Math.sqrt(nx * nx + ny * ny + nz * nz) / 2;
+}
+
+/** A prop's hulls, and how much empty space they added. */
+export interface PropHulls {
+  /** xyz-packed points per hull, as `COLL` writes them and `hullShape` takes them. */
+  readonly hulls: readonly Float32Array[];
+  /** `(hullVolume − sourceVolume) / sourceVolume`, from the decomposition itself. */
+  readonly bloat: number;
+}
+
+/**
+ * `mesh` decomposed into convex hulls the container can hold.
+ *
+ * **Asked for inside the caps rather than checked against them afterwards.** A decomposition is the
+ * expensive part of baking a prop, and one that comes back with thirty-three hulls is one that has
+ * to be thrown away and asked for again — so the cap is what it is asked for. A hull that still
+ * arrives with more points than a record can hold is refused by name, because silently dropping its
+ * points changes the shape a game collides against without telling anybody.
+ */
+export function propHulls(
+  mesh: MeshData | CollisionSource,
+  options: DecomposeOptions = {},
+): PropHulls {
+  const decomposition = decomposeConvex(mesh.positions, mesh.indices, {
+    ...options,
+    maxHulls: Math.min(options.maxHulls ?? MAX_COLLIDER_HULLS, MAX_COLLIDER_HULLS),
+  });
+  /*
+   * **The points need no cap of their own**, and it is worth saying why rather than checking for
+   * something that cannot happen. A part carries at most one point per support direction, which is
+   * fifty, and a collider record holds sixty-four — so the decomposition is already inside the
+   * container by construction. `collide.test.ts` asserts that relationship rather than this file
+   * testing each hull against a limit it can never reach.
+   */
+  return { hulls: decomposition.parts.map((part) => part.points), bloat: decomposition.bloat };
+}