@driftengine/capture 4.0.0

package/src/math/dense.ts

@@ -0,0 +1,317 @@
+/**
+ * The dense decompositions structure from motion needs, in double precision: a singular value
+ * decomposition, a symmetric eigendecomposition, a Cholesky factor, and Levenberg–Marquardt over a
+ * least-squares problem.
+ *
+ * **Written here rather than depended on.** Each is a few dozen lines against a dependency this
+ * engine would carry into every game that captures a scene, and each is held to its own identity —
+ * a decomposition that reconstructs what it decomposed, a factor whose square is the matrix — which
+ * is a stronger check than agreeing with another implementation.
+ *
+ * **Both decompositions are Jacobi's**, rotating away one off-diagonal at a time. It is not the
+ * fastest way to a large spectrum and it is the most accurate on the small, nearly rank-deficient
+ * matrices this is for — a homography from four points, an essential matrix, a 3 × 3 covariance —
+ * where the answer's smallest value is the one that matters. **What would make it wrong** is a
+ * matrix of thousands of rows, which wants a bidiagonal reduction instead; nothing here has one.
+ *
+ * **Everything is `Float64Array`.** Geometry accumulates: a pose error of a millionth is visible
+ * once a hundred frames carry it. Only arithmetic and `Math.sqrt` are reached for, which IEEE 754
+ * fixes, so a capture reproduces on any engine — `scripts/determinism.mjs` holds this package to
+ * that.
+ */
+
+/** How far off the diagonal a Jacobi sweep tolerates before it stops. */
+const SETTLED = 1e-15;
+const SWEEPS = 60;
+
+/**
+ * One-sided Jacobi: `m`, `rows × cols` row-major with `rows ≥ cols`, into `u` (`rows × cols`),
+ * `s` (`cols`, falling) and `v` (`cols × cols`), where `m = u · diag(s) · vᵀ`.
+ */
+export function svdN(
+  m: Float64Array,
+  rows: number,
+  cols: number,
+  u: Float64Array,
+  s: Float64Array,
+  v: Float64Array,
+): void {
+  if (rows < cols) {
+    throw new RangeError(`a ${rows}×${cols} matrix has more columns than rows; transpose it first`);
+  }
+  const a = Float64Array.from(m);
+  v.fill(0);
+  for (let i = 0; i < cols; i += 1) v[i * cols + i] = 1;
+
+  for (let sweep = 0; sweep < SWEEPS; sweep += 1) {
+    let off = 0;
+    for (let p = 0; p < cols - 1; p += 1) {
+      for (let q = p + 1; q < cols; q += 1) {
+        let alpha = 0;
+        let beta = 0;
+        let gamma = 0;
+        for (let r = 0; r < rows; r += 1) {
+          const left = a[r * cols + p] as number;
+          const right = a[r * cols + q] as number;
+          alpha += left * left;
+          beta += right * right;
+          gamma += left * right;
+        }
+        if (gamma === 0) continue;
+        const scale = Math.sqrt(alpha * beta);
+        off = Math.max(off, scale === 0 ? 0 : Math.abs(gamma) / scale);
+        if (scale === 0 || Math.abs(gamma) <= SETTLED * scale) continue;
+        /* The rotation that makes the two columns orthogonal, by the stable root. */
+        const zeta = (beta - alpha) / (2 * gamma);
+        const t = (zeta >= 0 ? 1 : -1) / (Math.abs(zeta) + Math.sqrt(1 + zeta * zeta));
+        const c = 1 / Math.sqrt(1 + t * t);
+        const sn = c * t;
+        for (let r = 0; r < rows; r += 1) {
+          const left = a[r * cols + p] as number;
+          const right = a[r * cols + q] as number;
+          a[r * cols + p] = c * left - sn * right;
+          a[r * cols + q] = sn * left + c * right;
+        }
+        for (let r = 0; r < cols; r += 1) {
+          const left = v[r * cols + p] as number;
+          const right = v[r * cols + q] as number;
+          v[r * cols + p] = c * left - sn * right;
+          v[r * cols + q] = sn * left + c * right;
+        }
+      }
+    }
+    if (off <= SETTLED) break;
+  }
+
+  /* What is left of each column is its singular value times a unit vector. */
+  const order = Array.from({ length: cols }, (_, i) => i);
+  const lengths = new Float64Array(cols);
+  for (let c = 0; c < cols; c += 1) {
+    let sum = 0;
+    for (let r = 0; r < rows; r += 1) {
+      const value = a[r * cols + c] as number;
+      sum += value * value;
+    }
+    lengths[c] = Math.sqrt(sum);
+  }
+  order.sort((p, q) => (lengths[q] as number) - (lengths[p] as number));
+  const held = Float64Array.from(v);
+  for (const [at, from] of order.entries()) {
+    const length = lengths[from] as number;
+    s[at] = length;
+    for (let r = 0; r < rows; r += 1) {
+      u[r * cols + at] = length === 0 ? 0 : (a[r * cols + from] as number) / length;
+    }
+    for (let r = 0; r < cols; r += 1) v[r * cols + at] = held[r * cols + from] as number;
+  }
+}
+
+/** The same for the 3 × 3 matrices geometry is full of. */
+export function svd3(m: Float64Array, u: Float64Array, s: Float64Array, v: Float64Array): void {
+  svdN(m, 3, 3, u, s, v);
+}
+
+/**
+ * Jacobi's eigendecomposition of a symmetric `m`, `n × n`: `values` in falling order and `vectors`
+ * by column, so `m · vectors[:, k] = values[k] · vectors[:, k]`.
+ */
+export function symmetricEigen(
+  m: Float64Array,
+  n: number,
+  values: Float64Array,
+  vectors: Float64Array,
+): void {
+  const a = Float64Array.from(m);
+  vectors.fill(0);
+  for (let i = 0; i < n; i += 1) vectors[i * n + i] = 1;
+
+  for (let sweep = 0; sweep < SWEEPS; sweep += 1) {
+    let off = 0;
+    for (let p = 0; p < n - 1; p += 1) {
+      for (let q = p + 1; q < n; q += 1) off = Math.max(off, Math.abs(a[p * n + q] as number));
+    }
+    if (off <= SETTLED) break;
+    for (let p = 0; p < n - 1; p += 1) {
+      for (let q = p + 1; q < n; q += 1) {
+        const pivot = a[p * n + q] as number;
+        if (Math.abs(pivot) <= SETTLED * off) continue;
+        const zeta = ((a[q * n + q] as number) - (a[p * n + p] as number)) / (2 * pivot);
+        const t = (zeta >= 0 ? 1 : -1) / (Math.abs(zeta) + Math.sqrt(1 + zeta * zeta));
+        const c = 1 / Math.sqrt(1 + t * t);
+        const sn = c * t;
+        for (let k = 0; k < n; k += 1) {
+          const left = a[k * n + p] as number;
+          const right = a[k * n + q] as number;
+          a[k * n + p] = c * left - sn * right;
+          a[k * n + q] = sn * left + c * right;
+        }
+        for (let k = 0; k < n; k += 1) {
+          const left = a[p * n + k] as number;
+          const right = a[q * n + k] as number;
+          a[p * n + k] = c * left - sn * right;
+          a[q * n + k] = sn * left + c * right;
+        }
+        for (let k = 0; k < n; k += 1) {
+          const left = vectors[k * n + p] as number;
+          const right = vectors[k * n + q] as number;
+          vectors[k * n + p] = c * left - sn * right;
+          vectors[k * n + q] = sn * left + c * right;
+        }
+      }
+    }
+  }
+
+  const order = Array.from({ length: n }, (_, i) => i);
+  order.sort((p, q) => (a[q * n + q] as number) - (a[p * n + p] as number));
+  const held = Float64Array.from(vectors);
+  for (const [at, from] of order.entries()) {
+    values[at] = a[from * n + from] as number;
+    for (let r = 0; r < n; r += 1) vectors[r * n + at] = held[r * n + from] as number;
+  }
+}
+
+/**
+ * The lower triangle `l` with `l · lᵀ = m`, or false where `m` is not positive definite — which is
+ * how a caller learns that its normal equations are singular, rather than by finding a NaN later.
+ *
+ * **`out` may be `m`**: only the lower triangle is read, each element before it is written, so a
+ * caller with one matrix to spare factors it in place. Zeroing the whole of `out` first would have
+ * erased the matrix in that case, which is a silent wrong answer rather than a refusal.
+ */
+export function cholesky(m: Float64Array, n: number, out: Float64Array): boolean {
+  for (let r = 0; r < n; r += 1) {
+    for (let c = r + 1; c < n; c += 1) out[r * n + c] = 0;
+    for (let c = 0; c <= r; c += 1) {
+      let sum = m[r * n + c] as number;
+      for (let k = 0; k < c; k += 1) sum -= (out[r * n + k] as number) * (out[c * n + k] as number);
+      if (r === c) {
+        if (!(sum > 0)) return false;
+        out[r * n + c] = Math.sqrt(sum);
+      } else {
+        out[r * n + c] = sum / (out[c * n + c] as number);
+      }
+    }
+  }
+  return true;
+}
+
+/** Solves `l · lᵀ · x = b` in place over `x`, given the Cholesky factor. */
+export function choleskySolve(l: Float64Array, n: number, x: Float64Array): void {
+  for (let r = 0; r < n; r += 1) {
+    let sum = x[r] as number;
+    for (let k = 0; k < r; k += 1) sum -= (l[r * n + k] as number) * (x[k] as number);
+    x[r] = sum / (l[r * n + r] as number);
+  }
+  for (let r = n - 1; r >= 0; r -= 1) {
+    let sum = x[r] as number;
+    for (let k = r + 1; k < n; k += 1) sum -= (l[k * n + r] as number) * (x[k] as number);
+    x[r] = sum / (l[r * n + r] as number);
+  }
+}
+
+/** A least-squares problem: how many residuals, how many parameters, and both at a point. */
+export interface LeastSquares {
+  readonly residuals: number;
+  readonly parameters: number;
+  /** The residuals at `x`, and — unless `jacobian` is null — their derivatives, row by residual. */
+  evaluate(x: Float64Array, residuals: Float64Array, jacobian: Float64Array | null): void;
+}
+
+export interface FitOptions {
+  /** The first damping; **zero is plain Gauss–Newton**, steps taken whatever they cost. */
+  readonly damping?: number;
+  readonly maxIterations?: number;
+  /** Stop once a step moves every parameter less than this. */
+  readonly tolerance?: number;
+}
+
+/**
+ * Levenberg–Marquardt over `x`, in place: the Gauss–Newton step with `damping · diag(JᵀJ)` added,
+ * the damping raised where a step would cost more and lowered where it pays, which is the whole
+ * reason it is not Gauss–Newton — **a step that overshoots is rejected rather than taken**, and a
+ * start far from the answer is where that decides between converging and diverging.
+ */
+export function levenbergMarquardt(
+  problem: LeastSquares,
+  x: Float64Array,
+  options: FitOptions = {},
+): { readonly iterations: number; readonly cost: number } {
+  const { residuals: rows, parameters: columns } = problem;
+  const maxIterations = options.maxIterations ?? 100;
+  const tolerance = options.tolerance ?? 1e-14;
+  let damping = options.damping ?? 1e-3;
+  const undamped = damping === 0;
+
+  const residual = new Float64Array(rows);
+  const jacobian = new Float64Array(rows * columns);
+  const normals = new Float64Array(columns * columns);
+  const factor = new Float64Array(columns * columns);
+  const gradient = new Float64Array(columns);
+  const step = new Float64Array(columns);
+  const trial = new Float64Array(columns);
+
+  const costOf = (values: Float64Array): number => {
+    let sum = 0;
+    for (let i = 0; i < values.length; i += 1) sum += (values[i] as number) * (values[i] as number);
+    return sum;
+  };
+
+  problem.evaluate(x, residual, null);
+  let cost = costOf(residual);
+  let iterations = 0;
+  while (iterations < maxIterations) {
+    iterations += 1;
+    problem.evaluate(x, residual, jacobian);
+    normals.fill(0);
+    gradient.fill(0);
+    for (let r = 0; r < rows; r += 1) {
+      for (let p = 0; p < columns; p += 1) {
+        const value = jacobian[r * columns + p] as number;
+        gradient[p] = (gradient[p] as number) - value * (residual[r] as number);
+        for (let q = p; q < columns; q += 1) {
+          normals[p * columns + q] =
+            (normals[p * columns + q] as number) + value * (jacobian[r * columns + q] as number);
+        }
+      }
+    }
+    for (let p = 0; p < columns; p += 1) {
+      for (let q = 0; q < p; q += 1) normals[p * columns + q] = normals[q * columns + p] as number;
+    }
+
+    let taken = false;
+    for (let attempt = 0; attempt < 12 && !taken; attempt += 1) {
+      factor.set(normals);
+      for (let p = 0; p < columns; p += 1) {
+        /* Marquardt's scaling: the damping rides each parameter's own curvature. */
+        const diagonal = normals[p * columns + p] as number;
+        factor[p * columns + p] = diagonal * (1 + damping) + damping;
+      }
+      if (!cholesky(factor, columns, factor)) {
+        if (undamped) return { iterations, cost };
+        damping *= 10;
+        continue;
+      }
+      step.set(gradient);
+      choleskySolve(factor, columns, step);
+      let moved = 0;
+      for (let p = 0; p < columns; p += 1) {
+        trial[p] = (x[p] as number) + (step[p] as number);
+        moved = Math.max(moved, Math.abs(step[p] as number));
+      }
+      problem.evaluate(trial, residual, null);
+      const next = costOf(residual);
+      if (undamped || next < cost) {
+        x.set(trial);
+        cost = next;
+        taken = true;
+        if (!undamped) damping = Math.max(damping / 10, 1e-12);
+        if (moved <= tolerance) return { iterations, cost };
+      } else {
+        damping *= 10;
+        if (damping > 1e12) return { iterations, cost };
+      }
+    }
+    if (!taken) return { iterations, cost };
+  }
+  return { iterations, cost };
+}

package/src/math/schur.ts

@@ -0,0 +1,167 @@
+/**
+ * The Schur complement over points: a bundle adjustment's normal equations solved by eliminating
+ * the points first, which leaves a system the size of the cameras.
+ *
+ * **It is an ordering, not an approximation.** A point appears in the rows of its own observations
+ * and nowhere else, so its three parameters can be solved for in terms of the cameras and
+ * substituted out exactly; what is left is the cameras' own system with a correction per link, and
+ * the points come back by back-substitution. The answer is the answer a dense solve would give —
+ * `schur.test.ts` holds it to one — and the saving is that a clip with a thousand points and twenty
+ * cameras solves a matrix of 120 rows rather than 3,120.
+ *
+ * **The damping is the caller's, applied here**, since it has to sit on the diagonal before the
+ * points are eliminated rather than after: `diagonal · (1 + λ) + λ`, which is Marquardt's scaling
+ * with a floor for a parameter whose curvature is zero.
+ *
+ * **A block that cannot be factored is refused rather than stepped through.** A point nothing
+ * constrains would be a division by nothing, and a reduced system that is singular means the
+ * cameras are not determined by what was seen — both are answers a caller must have, so they come
+ * back as false rather than as a step full of infinities.
+ */
+import { cholesky, choleskySolve } from './dense.ts';
+
+/** The normal equations in blocks: cameras, points of three, and one link per observation. */
+export interface BlockNormals {
+  readonly cameras: number;
+  /** Parameters a camera; the points are three each. */
+  readonly cameraSize: number;
+  readonly points: number;
+  /** `cameras · cameraSize²`, each symmetric and row-major. */
+  readonly cameraBlocks: Float64Array;
+  /** `points · 9`, each symmetric and row-major. */
+  readonly pointBlocks: Float64Array;
+  /** `links · cameraSize · 3`, row-major: a camera's rows against a point's columns. */
+  readonly links: Float64Array;
+  readonly linkCamera: Int32Array;
+  readonly linkPoint: Int32Array;
+  /** The right-hand side: `cameras · cameraSize` and `points · 3`. */
+  readonly cameraGradient: Float64Array;
+  readonly pointGradient: Float64Array;
+}
+
+const POINT = 3;
+
+/**
+ * The step for `damping`, into `cameraStep` (`cameras · cameraSize`) and `pointStep`
+ * (`points · 3`); false where a point's block or the reduced system cannot be factored.
+ */
+export function schurSolve(
+  normals: BlockNormals,
+  damping: number,
+  cameraStep: Float64Array,
+  pointStep: Float64Array,
+): boolean {
+  const { cameras, cameraSize, points, links, linkCamera, linkPoint } = normals;
+  const width = cameras * cameraSize;
+  const linkCount = linkCamera.length;
+
+  /* Each point's own block, damped and factored once: everything below reads it. */
+  const factors = new Float64Array(points * POINT * POINT);
+  const block = new Float64Array(POINT * POINT);
+  for (let p = 0; p < points; p += 1) {
+    for (let r = 0; r < POINT; r += 1) {
+      for (let c = 0; c < POINT; c += 1) {
+        const value = normals.pointBlocks[(p * POINT + r) * POINT + c] as number;
+        block[r * POINT + c] = r === c ? value * (1 + damping) + damping : value;
+      }
+    }
+    if (!cholesky(block, POINT, block)) return false;
+    factors.set(block, p * POINT * POINT);
+  }
+
+  /* The reduced system: the cameras' own blocks, less each link's share of its point. */
+  const reduced = new Float64Array(width * width);
+  const right = new Float64Array(width);
+  for (let c = 0; c < cameras; c += 1) {
+    for (let r = 0; r < cameraSize; r += 1) {
+      right[c * cameraSize + r] = normals.cameraGradient[c * cameraSize + r] as number;
+      for (let k = 0; k < cameraSize; k += 1) {
+        const value = normals.cameraBlocks[(c * cameraSize + r) * cameraSize + k] as number;
+        reduced[(c * cameraSize + r) * width + c * cameraSize + k] =
+          r === k ? value * (1 + damping) + damping : value;
+      }
+    }
+  }
+
+  /* Each link's `w · v⁻¹`, kept for the reduction and for the back-substitution after it. */
+  const solved = new Float64Array(linkCount * cameraSize * POINT);
+  const column = new Float64Array(POINT);
+  for (let link = 0; link < linkCount; link += 1) {
+    const point = linkPoint[link] as number;
+    const factor = factors.subarray(point * POINT * POINT, (point + 1) * POINT * POINT);
+    for (let r = 0; r < cameraSize; r += 1) {
+      for (let k = 0; k < POINT; k += 1) {
+        column[k] = links[(link * cameraSize + r) * POINT + k] as number;
+      }
+      choleskySolve(factor, POINT, column);
+      for (let k = 0; k < POINT; k += 1) {
+        solved[(link * cameraSize + r) * POINT + k] = column[k] as number;
+      }
+    }
+  }
+
+  /* Two links on one point couple their cameras, which is where the reduction's off-diagonals are. */
+  for (let a = 0; a < linkCount; a += 1) {
+    for (let b = 0; b < linkCount; b += 1) {
+      if ((linkPoint[a] as number) !== (linkPoint[b] as number)) continue;
+      const rowCamera = (linkCamera[a] as number) * cameraSize;
+      const colCamera = (linkCamera[b] as number) * cameraSize;
+      for (let r = 0; r < cameraSize; r += 1) {
+        for (let c = 0; c < cameraSize; c += 1) {
+          let sum = 0;
+          for (let k = 0; k < POINT; k += 1) {
+            sum +=
+              (solved[(a * cameraSize + r) * POINT + k] as number) *
+              (links[(b * cameraSize + c) * POINT + k] as number);
+          }
+          reduced[(rowCamera + r) * width + colCamera + c] -= sum;
+        }
+      }
+    }
+  }
+  for (let link = 0; link < linkCount; link += 1) {
+    const camera = (linkCamera[link] as number) * cameraSize;
+    const point = (linkPoint[link] as number) * POINT;
+    for (let r = 0; r < cameraSize; r += 1) {
+      let sum = 0;
+      for (let k = 0; k < POINT; k += 1) {
+        sum +=
+          (solved[(link * cameraSize + r) * POINT + k] as number) *
+          (normals.pointGradient[point + k] as number);
+      }
+      right[camera + r] -= sum;
+    }
+  }
+
+  const factor = new Float64Array(width * width);
+  if (!cholesky(reduced, width, factor)) return false;
+  cameraStep.set(right);
+  choleskySolve(factor, width, cameraStep);
+
+  /* And the points come back: their own gradient, less what the cameras just took. */
+  for (let p = 0; p < points; p += 1) {
+    for (let k = 0; k < POINT; k += 1) {
+      pointStep[p * POINT + k] = normals.pointGradient[p * POINT + k] as number;
+    }
+  }
+  for (let link = 0; link < linkCount; link += 1) {
+    const camera = (linkCamera[link] as number) * cameraSize;
+    const point = (linkPoint[link] as number) * POINT;
+    for (let k = 0; k < POINT; k += 1) {
+      let sum = 0;
+      for (let r = 0; r < cameraSize; r += 1) {
+        sum +=
+          (links[(link * cameraSize + r) * POINT + k] as number) *
+          (cameraStep[camera + r] as number);
+      }
+      pointStep[point + k] = (pointStep[point + k] as number) - sum;
+    }
+  }
+  for (let p = 0; p < points; p += 1) {
+    const factored = factors.subarray(p * POINT * POINT, (p + 1) * POINT * POINT);
+    for (let k = 0; k < POINT; k += 1) column[k] = pointStep[p * POINT + k] as number;
+    choleskySolve(factored, POINT, column);
+    for (let k = 0; k < POINT; k += 1) pointStep[p * POINT + k] = column[k] as number;
+  }
+  return true;
+}

package/src/models/batchNorm.ts

@@ -0,0 +1,57 @@
+/**
+ * A convolution with the batch norm after it folded in, once, where the weights are converted.
+ *
+ * **Folded as the upstream's `Conv2d_BN.fuse` folds it**: the kernel scaled by `γ/√(var + ε)` per
+ * output, and the bias `β − mean·γ/√(var + ε)`. The converted file holds the folded pair under
+ * names of its own and none of the statistics, and the step counter a batch norm keeps is set aside
+ * by name. ε is PyTorch's default, 1e-5.
+ *
+ * What it gives up: the fold is computed in double precision and rounded once, where the upstream
+ * evaluates the norm after the convolution in single precision — the two agree to a rounding.
+ */
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+
+const EPSILON = 1e-5;
+
+/**
+ * `conv2d` of `x` by the convolution and batch norm under `p` — `c.` and `bn.`, the upstream's
+ * names — padded to keep the grid at stride one.
+ */
+export function convBatchNorm(
+  weights: Weights,
+  graph: GraphBuilder,
+  p: string,
+  x: string,
+  cin: number,
+  cout: number,
+  kernel: number,
+  stride: number,
+  groups: number,
+): string {
+  const shape = [cout, cin / groups, kernel, kernel];
+  const scale = (values: (part: string, shape?: readonly number[]) => Float32Array): number[] => {
+    const variance = values(`${p}bn.running_var`, [cout]);
+    return Array.from(
+      values(`${p}bn.weight`, [cout]),
+      (gamma, o) => gamma / Math.sqrt((variance[o] as number) + EPSILON),
+    );
+  };
+  const weight = weights.derive(`${p}folded.weight`, shape, (values) => {
+    const factor = scale(values);
+    const per = (cin / groups) * kernel * kernel;
+    return Float32Array.from(
+      values(`${p}c.weight`, shape),
+      (w, i) => w * (factor[Math.floor(i / per)] as number),
+    );
+  });
+  const bias = weights.derive(`${p}folded.bias`, [cout], (values) => {
+    const factor = scale(values);
+    const mean = values(`${p}bn.running_mean`, [cout]);
+    return Float32Array.from(
+      values(`${p}bn.bias`, [cout]),
+      (beta, o) => beta - (mean[o] as number) * (factor[o] as number),
+    );
+  });
+  weights.ignore(`${p}bn.num_batches_tracked`);
+  return graph.node('conv2d', [x, weight, bias], { stride, padding: (kernel - 1) / 2, groups });
+}

package/src/models/clipTokenizer.ts

@@ -0,0 +1,129 @@
+/**
+ * CLIP's tokenizer — byte-level BPE — as Transformers' `CLIPTokenizer` runs it at the revision the
+ * manifest pins, for OWLv2's text queries.
+ *
+ * **Its whole vocabulary is its merges.** A byte's token is the byte's place in GPT-2's order —
+ * the printable bytes first, then the rest — the same byte ending a word is 256 further on, the
+ * merge of rank r makes token 512 + r, and the start and end of text follow the last merge. So a
+ * converted file carries the merges as pairs of token ids and nothing else, and the conversion
+ * checks the upstream's `vocab.json` says the same.
+ *
+ * **In the upstream's order**: `!` and `<|endoftext|>` are taken out of the raw text as the added
+ * tokens they are — `!` is the padding token, 0, wherever it appears; the rest is composed (NFC)
+ * and lowercased a character at a time; `<|startoftext|>` is then taken out of that; what is left
+ * is split by CLIP's pattern into contractions, runs of letters, single digits and runs of anything
+ * else; each piece's UTF-8 bytes become tokens with the last marked as a word's end, and adjacent
+ * pairs merge lowest rank first, leftmost among equals. A pair listed twice ranks as its last, as
+ * the upstream's map of merges keeps it.
+ *
+ * **What it leaves out, and why that changes nothing**: the upstream also makes each run of
+ * whitespace one space, and splits again by GPT-2's pattern after CLIP's. No piece holds
+ * whitespace, so the first cannot change a token; and neither pattern crosses whitespace while
+ * every piece CLIP's makes is one GPT-2's matches whole, so the second divides nothing. What would
+ * make that wrong is a pattern change on either side, which the hand-run parity would show.
+ */
+
+export interface ClipTokenizer {
+  /** The text's tokens, between the start and end of text. */
+  encode(text: string): number[];
+}
+
+/* CLIP's pieces: its specials, contractions, letter runs, one digit, and runs of anything else. */
+const PIECES =
+  /<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+/gu;
+
+/* Each byte's token: its place in GPT-2's order, printable bytes first. */
+const BYTE_TOKEN = ((): Uint16Array => {
+  const printable = (b: number): boolean =>
+    (b >= 33 && b <= 126) || (b >= 161 && b <= 172) || (b >= 174 && b <= 255);
+  const table = new Uint16Array(256);
+  let next = 0;
+  for (let b = 0; b < 256; b += 1) if (printable(b)) table[b] = next++;
+  for (let b = 0; b < 256; b += 1) if (!printable(b)) table[b] = next++;
+  return table;
+})();
+const WORD_END = 256;
+const MERGED = 512;
+const PADDING = 0;
+
+/** A tokenizer over `merges`, `[count, 2]` token ids in rank order. */
+export function clipTokenizer(merges: Float32Array): ClipTokenizer {
+  const count = merges.length / 2;
+  const ranks = new Map<number, number>();
+  for (let r = 0; r < count; r += 1) {
+    const key = (merges[2 * r] as number) * 65536 + (merges[2 * r + 1] as number);
+    ranks.set(key, r);
+  }
+  const start = MERGED + count;
+  const end = start + 1;
+  const bytes = new TextEncoder();
+
+  const word = (piece: string, out: number[]): void => {
+    const symbols = Array.from(bytes.encode(piece), (b) => BYTE_TOKEN[b] as number);
+    symbols[symbols.length - 1] = (symbols[symbols.length - 1] as number) + WORD_END;
+    for (;;) {
+      let best = -1;
+      let rank = count;
+      for (let i = 0; i + 1 < symbols.length; i += 1) {
+        const r = ranks.get((symbols[i] as number) * 65536 + (symbols[i + 1] as number));
+        if (r !== undefined && r < rank) {
+          rank = r;
+          best = i;
+        }
+      }
+      if (best < 0) break;
+      symbols.splice(best, 2, MERGED + rank);
+    }
+    out.push(...symbols);
+  };
+  const normalised = (text: string, out: number[]): void => {
+    const lower = Array.from(text.normalize('NFC'), (c) => c.toLowerCase()).join('');
+    lower.split('<|startoftext|>').forEach((part, i) => {
+      if (i > 0) out.push(start);
+      for (const [piece] of part.matchAll(PIECES)) word(piece, out);
+    });
+  };
+
+  return {
+    encode(text) {
+      const out = [start];
+      text.split('<|endoftext|>').forEach((part, i) => {
+        if (i > 0) out.push(end);
+        part.split('!').forEach((run, j) => {
+          if (j > 0) out.push(PADDING);
+          normalised(run, out);
+        });
+      });
+      out.push(end);
+      return out;
+    },
+  };
+}
+
+/**
+ * OWLv2's text queries as its text graph takes them: `tokens`, `[queries · positions]`, each query
+ * padded with zeros to `positions`, and `ends`, `[queries]`, the row of each query's end of text in
+ * that list — the upstream pools the row of the largest token, which is that one. A query longer
+ * than `positions` is refused, as the upstream's position table refuses it.
+ */
+export function owlv2Tokens(
+  tokenizer: ClipTokenizer,
+  texts: readonly string[],
+  positions: number,
+): { readonly tokens: Float32Array; readonly ends: Float32Array } {
+  const tokens = new Float32Array(texts.length * positions);
+  const ends = new Float32Array(texts.length);
+  texts.forEach((text, q) => {
+    const ids = tokenizer.encode(text);
+    if (ids.length > positions) {
+      throw new RangeError(`"${text}" is ${ids.length} tokens and the model reads ${positions}`);
+    }
+    tokens.set(ids, q * positions);
+    let largest = 0;
+    ids.forEach((id, i) => {
+      if (id > (ids[largest] as number)) largest = i;
+    });
+    ends[q] = q * positions + largest;
+  });
+  return { tokens, ends };
+}