@driftengine/capture 4.0.0

package/dist/math/schur.js

@@ -0,0 +1,140 @@
+/**
+ * The Schur complement over points: a bundle adjustment's normal equations solved by eliminating
+ * the points first, which leaves a system the size of the cameras.
+ *
+ * **It is an ordering, not an approximation.** A point appears in the rows of its own observations
+ * and nowhere else, so its three parameters can be solved for in terms of the cameras and
+ * substituted out exactly; what is left is the cameras' own system with a correction per link, and
+ * the points come back by back-substitution. The answer is the answer a dense solve would give —
+ * `schur.test.ts` holds it to one — and the saving is that a clip with a thousand points and twenty
+ * cameras solves a matrix of 120 rows rather than 3,120.
+ *
+ * **The damping is the caller's, applied here**, since it has to sit on the diagonal before the
+ * points are eliminated rather than after: `diagonal · (1 + λ) + λ`, which is Marquardt's scaling
+ * with a floor for a parameter whose curvature is zero.
+ *
+ * **A block that cannot be factored is refused rather than stepped through.** A point nothing
+ * constrains would be a division by nothing, and a reduced system that is singular means the
+ * cameras are not determined by what was seen — both are answers a caller must have, so they come
+ * back as false rather than as a step full of infinities.
+ */
+import { cholesky, choleskySolve } from './dense.js';
+const POINT = 3;
+/**
+ * The step for `damping`, into `cameraStep` (`cameras · cameraSize`) and `pointStep`
+ * (`points · 3`); false where a point's block or the reduced system cannot be factored.
+ */
+export function schurSolve(normals, damping, cameraStep, pointStep) {
+    const { cameras, cameraSize, points, links, linkCamera, linkPoint } = normals;
+    const width = cameras * cameraSize;
+    const linkCount = linkCamera.length;
+    /* Each point's own block, damped and factored once: everything below reads it. */
+    const factors = new Float64Array(points * POINT * POINT);
+    const block = new Float64Array(POINT * POINT);
+    for (let p = 0; p < points; p += 1) {
+        for (let r = 0; r < POINT; r += 1) {
+            for (let c = 0; c < POINT; c += 1) {
+                const value = normals.pointBlocks[(p * POINT + r) * POINT + c];
+                block[r * POINT + c] = r === c ? value * (1 + damping) + damping : value;
+            }
+        }
+        if (!cholesky(block, POINT, block))
+            return false;
+        factors.set(block, p * POINT * POINT);
+    }
+    /* The reduced system: the cameras' own blocks, less each link's share of its point. */
+    const reduced = new Float64Array(width * width);
+    const right = new Float64Array(width);
+    for (let c = 0; c < cameras; c += 1) {
+        for (let r = 0; r < cameraSize; r += 1) {
+            right[c * cameraSize + r] = normals.cameraGradient[c * cameraSize + r];
+            for (let k = 0; k < cameraSize; k += 1) {
+                const value = normals.cameraBlocks[(c * cameraSize + r) * cameraSize + k];
+                reduced[(c * cameraSize + r) * width + c * cameraSize + k] =
+                    r === k ? value * (1 + damping) + damping : value;
+            }
+        }
+    }
+    /* Each link's `w · v⁻¹`, kept for the reduction and for the back-substitution after it. */
+    const solved = new Float64Array(linkCount * cameraSize * POINT);
+    const column = new Float64Array(POINT);
+    for (let link = 0; link < linkCount; link += 1) {
+        const point = linkPoint[link];
+        const factor = factors.subarray(point * POINT * POINT, (point + 1) * POINT * POINT);
+        for (let r = 0; r < cameraSize; r += 1) {
+            for (let k = 0; k < POINT; k += 1) {
+                column[k] = links[(link * cameraSize + r) * POINT + k];
+            }
+            choleskySolve(factor, POINT, column);
+            for (let k = 0; k < POINT; k += 1) {
+                solved[(link * cameraSize + r) * POINT + k] = column[k];
+            }
+        }
+    }
+    /* Two links on one point couple their cameras, which is where the reduction's off-diagonals are. */
+    for (let a = 0; a < linkCount; a += 1) {
+        for (let b = 0; b < linkCount; b += 1) {
+            if (linkPoint[a] !== linkPoint[b])
+                continue;
+            const rowCamera = linkCamera[a] * cameraSize;
+            const colCamera = linkCamera[b] * cameraSize;
+            for (let r = 0; r < cameraSize; r += 1) {
+                for (let c = 0; c < cameraSize; c += 1) {
+                    let sum = 0;
+                    for (let k = 0; k < POINT; k += 1) {
+                        sum +=
+                            solved[(a * cameraSize + r) * POINT + k] *
+                                links[(b * cameraSize + c) * POINT + k];
+                    }
+                    reduced[(rowCamera + r) * width + colCamera + c] -= sum;
+                }
+            }
+        }
+    }
+    for (let link = 0; link < linkCount; link += 1) {
+        const camera = linkCamera[link] * cameraSize;
+        const point = linkPoint[link] * POINT;
+        for (let r = 0; r < cameraSize; r += 1) {
+            let sum = 0;
+            for (let k = 0; k < POINT; k += 1) {
+                sum +=
+                    solved[(link * cameraSize + r) * POINT + k] *
+                        normals.pointGradient[point + k];
+            }
+            right[camera + r] -= sum;
+        }
+    }
+    const factor = new Float64Array(width * width);
+    if (!cholesky(reduced, width, factor))
+        return false;
+    cameraStep.set(right);
+    choleskySolve(factor, width, cameraStep);
+    /* And the points come back: their own gradient, less what the cameras just took. */
+    for (let p = 0; p < points; p += 1) {
+        for (let k = 0; k < POINT; k += 1) {
+            pointStep[p * POINT + k] = normals.pointGradient[p * POINT + k];
+        }
+    }
+    for (let link = 0; link < linkCount; link += 1) {
+        const camera = linkCamera[link] * cameraSize;
+        const point = linkPoint[link] * POINT;
+        for (let k = 0; k < POINT; k += 1) {
+            let sum = 0;
+            for (let r = 0; r < cameraSize; r += 1) {
+                sum +=
+                    links[(link * cameraSize + r) * POINT + k] *
+                        cameraStep[camera + r];
+            }
+            pointStep[point + k] = pointStep[point + k] - sum;
+        }
+    }
+    for (let p = 0; p < points; p += 1) {
+        const factored = factors.subarray(p * POINT * POINT, (p + 1) * POINT * POINT);
+        for (let k = 0; k < POINT; k += 1)
+            column[k] = pointStep[p * POINT + k];
+        choleskySolve(factored, POINT, column);
+        for (let k = 0; k < POINT; k += 1)
+            pointStep[p * POINT + k] = column[k];
+    }
+    return true;
+}

package/dist/models/batchNorm.d.ts

@@ -0,0 +1,17 @@
+/**
+ * A convolution with the batch norm after it folded in, once, where the weights are converted.
+ *
+ * **Folded as the upstream's `Conv2d_BN.fuse` folds it**: the kernel scaled by `γ/√(var + ε)` per
+ * output, and the bias `β − mean·γ/√(var + ε)`. The converted file holds the folded pair under
+ * names of its own and none of the statistics, and the step counter a batch norm keeps is set aside
+ * by name. ε is PyTorch's default, 1e-5.
+ *
+ * What it gives up: the fold is computed in double precision and rounded once, where the upstream
+ * evaluates the norm after the convolution in single precision — the two agree to a rounding.
+ */
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+/**
+ * `conv2d` of `x` by the convolution and batch norm under `p` — `c.` and `bn.`, the upstream's
+ * names — padded to keep the grid at stride one.
+ */
+export declare function convBatchNorm(weights: Weights, graph: GraphBuilder, p: string, x: string, cin: number, cout: number, kernel: number, stride: number, groups: number): string;

package/dist/models/batchNorm.js

@@ -0,0 +1,24 @@
+const EPSILON = 1e-5;
+/**
+ * `conv2d` of `x` by the convolution and batch norm under `p` — `c.` and `bn.`, the upstream's
+ * names — padded to keep the grid at stride one.
+ */
+export function convBatchNorm(weights, graph, p, x, cin, cout, kernel, stride, groups) {
+    const shape = [cout, cin / groups, kernel, kernel];
+    const scale = (values) => {
+        const variance = values(`${p}bn.running_var`, [cout]);
+        return Array.from(values(`${p}bn.weight`, [cout]), (gamma, o) => gamma / Math.sqrt(variance[o] + EPSILON));
+    };
+    const weight = weights.derive(`${p}folded.weight`, shape, (values) => {
+        const factor = scale(values);
+        const per = (cin / groups) * kernel * kernel;
+        return Float32Array.from(values(`${p}c.weight`, shape), (w, i) => w * factor[Math.floor(i / per)]);
+    });
+    const bias = weights.derive(`${p}folded.bias`, [cout], (values) => {
+        const factor = scale(values);
+        const mean = values(`${p}bn.running_mean`, [cout]);
+        return Float32Array.from(values(`${p}bn.bias`, [cout]), (beta, o) => beta - mean[o] * factor[o]);
+    });
+    weights.ignore(`${p}bn.num_batches_tracked`);
+    return graph.node('conv2d', [x, weight, bias], { stride, padding: (kernel - 1) / 2, groups });
+}

package/dist/models/clipTokenizer.d.ts

@@ -0,0 +1,40 @@
+/**
+ * CLIP's tokenizer — byte-level BPE — as Transformers' `CLIPTokenizer` runs it at the revision the
+ * manifest pins, for OWLv2's text queries.
+ *
+ * **Its whole vocabulary is its merges.** A byte's token is the byte's place in GPT-2's order —
+ * the printable bytes first, then the rest — the same byte ending a word is 256 further on, the
+ * merge of rank r makes token 512 + r, and the start and end of text follow the last merge. So a
+ * converted file carries the merges as pairs of token ids and nothing else, and the conversion
+ * checks the upstream's `vocab.json` says the same.
+ *
+ * **In the upstream's order**: `!` and `<|endoftext|>` are taken out of the raw text as the added
+ * tokens they are — `!` is the padding token, 0, wherever it appears; the rest is composed (NFC)
+ * and lowercased a character at a time; `<|startoftext|>` is then taken out of that; what is left
+ * is split by CLIP's pattern into contractions, runs of letters, single digits and runs of anything
+ * else; each piece's UTF-8 bytes become tokens with the last marked as a word's end, and adjacent
+ * pairs merge lowest rank first, leftmost among equals. A pair listed twice ranks as its last, as
+ * the upstream's map of merges keeps it.
+ *
+ * **What it leaves out, and why that changes nothing**: the upstream also makes each run of
+ * whitespace one space, and splits again by GPT-2's pattern after CLIP's. No piece holds
+ * whitespace, so the first cannot change a token; and neither pattern crosses whitespace while
+ * every piece CLIP's makes is one GPT-2's matches whole, so the second divides nothing. What would
+ * make that wrong is a pattern change on either side, which the hand-run parity would show.
+ */
+export interface ClipTokenizer {
+    /** The text's tokens, between the start and end of text. */
+    encode(text: string): number[];
+}
+/** A tokenizer over `merges`, `[count, 2]` token ids in rank order. */
+export declare function clipTokenizer(merges: Float32Array): ClipTokenizer;
+/**
+ * OWLv2's text queries as its text graph takes them: `tokens`, `[queries · positions]`, each query
+ * padded with zeros to `positions`, and `ends`, `[queries]`, the row of each query's end of text in
+ * that list — the upstream pools the row of the largest token, which is that one. A query longer
+ * than `positions` is refused, as the upstream's position table refuses it.
+ */
+export declare function owlv2Tokens(tokenizer: ClipTokenizer, texts: readonly string[], positions: number): {
+    readonly tokens: Float32Array;
+    readonly ends: Float32Array;
+};

package/dist/models/clipTokenizer.js

@@ -0,0 +1,122 @@
+/**
+ * CLIP's tokenizer — byte-level BPE — as Transformers' `CLIPTokenizer` runs it at the revision the
+ * manifest pins, for OWLv2's text queries.
+ *
+ * **Its whole vocabulary is its merges.** A byte's token is the byte's place in GPT-2's order —
+ * the printable bytes first, then the rest — the same byte ending a word is 256 further on, the
+ * merge of rank r makes token 512 + r, and the start and end of text follow the last merge. So a
+ * converted file carries the merges as pairs of token ids and nothing else, and the conversion
+ * checks the upstream's `vocab.json` says the same.
+ *
+ * **In the upstream's order**: `!` and `<|endoftext|>` are taken out of the raw text as the added
+ * tokens they are — `!` is the padding token, 0, wherever it appears; the rest is composed (NFC)
+ * and lowercased a character at a time; `<|startoftext|>` is then taken out of that; what is left
+ * is split by CLIP's pattern into contractions, runs of letters, single digits and runs of anything
+ * else; each piece's UTF-8 bytes become tokens with the last marked as a word's end, and adjacent
+ * pairs merge lowest rank first, leftmost among equals. A pair listed twice ranks as its last, as
+ * the upstream's map of merges keeps it.
+ *
+ * **What it leaves out, and why that changes nothing**: the upstream also makes each run of
+ * whitespace one space, and splits again by GPT-2's pattern after CLIP's. No piece holds
+ * whitespace, so the first cannot change a token; and neither pattern crosses whitespace while
+ * every piece CLIP's makes is one GPT-2's matches whole, so the second divides nothing. What would
+ * make that wrong is a pattern change on either side, which the hand-run parity would show.
+ */
+/* CLIP's pieces: its specials, contractions, letter runs, one digit, and runs of anything else. */
+const PIECES = /<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|\p{L}+|\p{N}|[^\s\p{L}\p{N}]+/gu;
+/* Each byte's token: its place in GPT-2's order, printable bytes first. */
+const BYTE_TOKEN = (() => {
+    const printable = (b) => (b >= 33 && b <= 126) || (b >= 161 && b <= 172) || (b >= 174 && b <= 255);
+    const table = new Uint16Array(256);
+    let next = 0;
+    for (let b = 0; b < 256; b += 1)
+        if (printable(b))
+            table[b] = next++;
+    for (let b = 0; b < 256; b += 1)
+        if (!printable(b))
+            table[b] = next++;
+    return table;
+})();
+const WORD_END = 256;
+const MERGED = 512;
+const PADDING = 0;
+/** A tokenizer over `merges`, `[count, 2]` token ids in rank order. */
+export function clipTokenizer(merges) {
+    const count = merges.length / 2;
+    const ranks = new Map();
+    for (let r = 0; r < count; r += 1) {
+        const key = merges[2 * r] * 65536 + merges[2 * r + 1];
+        ranks.set(key, r);
+    }
+    const start = MERGED + count;
+    const end = start + 1;
+    const bytes = new TextEncoder();
+    const word = (piece, out) => {
+        const symbols = Array.from(bytes.encode(piece), (b) => BYTE_TOKEN[b]);
+        symbols[symbols.length - 1] = symbols[symbols.length - 1] + WORD_END;
+        for (;;) {
+            let best = -1;
+            let rank = count;
+            for (let i = 0; i + 1 < symbols.length; i += 1) {
+                const r = ranks.get(symbols[i] * 65536 + symbols[i + 1]);
+                if (r !== undefined && r < rank) {
+                    rank = r;
+                    best = i;
+                }
+            }
+            if (best < 0)
+                break;
+            symbols.splice(best, 2, MERGED + rank);
+        }
+        out.push(...symbols);
+    };
+    const normalised = (text, out) => {
+        const lower = Array.from(text.normalize('NFC'), (c) => c.toLowerCase()).join('');
+        lower.split('<|startoftext|>').forEach((part, i) => {
+            if (i > 0)
+                out.push(start);
+            for (const [piece] of part.matchAll(PIECES))
+                word(piece, out);
+        });
+    };
+    return {
+        encode(text) {
+            const out = [start];
+            text.split('<|endoftext|>').forEach((part, i) => {
+                if (i > 0)
+                    out.push(end);
+                part.split('!').forEach((run, j) => {
+                    if (j > 0)
+                        out.push(PADDING);
+                    normalised(run, out);
+                });
+            });
+            out.push(end);
+            return out;
+        },
+    };
+}
+/**
+ * OWLv2's text queries as its text graph takes them: `tokens`, `[queries · positions]`, each query
+ * padded with zeros to `positions`, and `ends`, `[queries]`, the row of each query's end of text in
+ * that list — the upstream pools the row of the largest token, which is that one. A query longer
+ * than `positions` is refused, as the upstream's position table refuses it.
+ */
+export function owlv2Tokens(tokenizer, texts, positions) {
+    const tokens = new Float32Array(texts.length * positions);
+    const ends = new Float32Array(texts.length);
+    texts.forEach((text, q) => {
+        const ids = tokenizer.encode(text);
+        if (ids.length > positions) {
+            throw new RangeError(`"${text}" is ${ids.length} tokens and the model reads ${positions}`);
+        }
+        tokens.set(ids, q * positions);
+        let largest = 0;
+        ids.forEach((id, i) => {
+            if (id > ids[largest])
+                largest = i;
+        });
+        ends[q] = q * positions + largest;
+    });
+    return { tokens, ends };
+}

package/dist/models/clipTower.d.ts

@@ -0,0 +1,27 @@
+/**
+ * CLIP's encoder tower, which OWLv2's image and text graphs are each one of.
+ *
+ * **Pre-norm layers**: a layer norm, attention over the row's own window, and the projection added
+ * back; then a layer norm, a perceptron, and that added back. The activation is the upstream's
+ * quick GELU, `x·σ(1.702·x)`, which is the logistic and two multiplies rather than the error
+ * function `gelu` computes.
+ *
+ * **A causal tower masks its scores** with −10⁴ above the diagonal, where the upstream uses the
+ * least finite single. Both give a masked key a weight of exactly zero — exp of anything below
+ * −104 is zero in single precision, and scores stay within a few hundred of each other — and −10⁴
+ * is exact at half precision, where the upstream's value would be infinite and a device may assume
+ * no infinities. **What would make it wrong** is scores reaching thousands, which would need a
+ * checkpoint whose norms let them.
+ */
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+export interface ClipTowerConfig {
+    readonly dim: number;
+    readonly heads: number;
+    readonly depth: number;
+    readonly hidden: number;
+}
+/**
+ * CLIP's layers over `x`, `[batch · rows, dim]`, each batch's rows attending among themselves —
+ * causally when `causal` is set.
+ */
+export declare function clipLayers(weights: Weights, graph: GraphBuilder, prefix: string, config: ClipTowerConfig, x: string, batch: number, rows: number, causal: boolean): string;

package/dist/models/clipTower.js

@@ -0,0 +1,45 @@
+const EPSILON = 1e-5;
+const MASKED = -1e4;
+/**
+ * CLIP's layers over `x`, `[batch · rows, dim]`, each batch's rows attending among themselves —
+ * causally when `causal` is set.
+ */
+export function clipLayers(weights, graph, prefix, config, x, batch, rows, causal) {
+    const { dim, heads, hidden } = config;
+    const node = graph.node.bind(graph);
+    const w = (name, shape) => weights.read(`${prefix}${name}`, shape);
+    const linear = (value, p, out, inputs) => node('linear', [value, w(`${p}.weight`, [out, inputs]), w(`${p}.bias`, [out])]);
+    const norm = (value, p) => node('layerNorm', [value, w(`${p}.weight`, [dim]), w(`${p}.bias`, [dim])], {
+        epsilon: EPSILON,
+    });
+    const quick = weights.constant(`quick_gelu.${hidden}`, [hidden], new Float32Array(hidden).fill(1.702));
+    let mask = null;
+    if (causal) {
+        const table = new Float32Array(heads * rows * rows);
+        for (let h = 0; h < heads; h += 1) {
+            for (let q = 0; q < rows; q += 1) {
+                for (let k = q + 1; k < rows; k += 1)
+                    table[(h * rows + q) * rows + k] = MASKED;
+            }
+        }
+        mask = weights.constant(`causal.${heads}x${rows}`, [heads, rows, rows], table);
+    }
+    const batched = (value) => batch > 1 || mask !== null ? node('reshape', [value], { shape: [batch, rows, dim] }) : value;
+    let out = x;
+    for (let i = 0; i < config.depth; i += 1) {
+        const p = `encoder.layers.${i}.`;
+        const h = norm(out, `${p}layer_norm1`);
+        const q = batched(linear(h, `${p}self_attn.q_proj`, dim, dim));
+        const k = batched(linear(h, `${p}self_attn.k_proj`, dim, dim));
+        const v = batched(linear(h, `${p}self_attn.v_proj`, dim, dim));
+        let attended = node('attention', mask === null ? [q, k, v] : [q, k, v, mask], { heads });
+        if (batch > 1 || mask !== null) {
+            attended = node('reshape', [attended], { shape: [batch * rows, dim] });
+        }
+        out = node('add', [out, linear(attended, `${p}self_attn.out_proj`, dim, dim)]);
+        const inner = linear(norm(out, `${p}layer_norm2`), `${p}mlp.fc1`, hidden, dim);
+        const activated = node('mul', [inner, node('sigmoid', [node('mul', [inner, quick])])]);
+        out = node('add', [out, linear(activated, `${p}mlp.fc2`, dim, hidden)]);
+    }
+    return out;
+}

package/dist/models/depthAnything2.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Depth Anything V2 Small, the lighter fallback: plain DINOv2 and a DPT, in the Transformers layout
+ * its Apache-2.0 checkpoint is published in.
+ *
+ * **Plain where Depth Anything 3 is not**: one view, no camera token, no query-key norms, no rotary
+ * embedding, and positions resized to the grid by the ratio of the sizes, as Transformers' DINOv2
+ * resizes them. Four taps — after the third, sixth, ninth and twelfth blocks, each through the
+ * backbone's last norm — are reassembled into a pyramid and fused as Depth Anything 3's head fuses
+ * its own, without the sinusoidal positions that head adds. The output is one channel through a
+ * rectifier: relative inverse depth, larger nearer, with no scale.
+ *
+ * **The checkpoint keeps Transformers' older names** — `attention.attention.query`,
+ * `layer_scale1.lambda1` — which the library renames as it loads; this reads them as stored. Its
+ * mask token is for training and set aside, and so is the first fusion block's lateral unit, which
+ * the upstream builds and never runs because the coarsest level has no level below it.
+ */
+import type { Architecture } from '@driftengine/texture';
+export interface DepthAnything2Config {
+    readonly dim: number;
+    readonly heads: number;
+    readonly depth: number;
+    readonly hidden: number;
+    readonly patch: number;
+    readonly trainedGrid: number;
+    /** The blocks whose outputs are tapped, counted from zero. */
+    readonly taps: readonly [number, number, number, number];
+    readonly neck: readonly [number, number, number, number];
+    readonly fusion: number;
+    readonly headHidden: number;
+}
+/** The checkpoint's configuration at its pinned revision. */
+export declare const DEPTH_ANYTHING_2_SMALL: DepthAnything2Config;
+/**
+ * The graph for one image of `height × width` pixels, each a multiple of the patch, normalised by
+ * ImageNet's mean and deviation: input `image`, `[3, height, width]`; output `depth`,
+ * `[1, height, width]`.
+ */
+export declare function depthAnything2(config: DepthAnything2Config, height: number, width: number): Architecture;

package/dist/models/depthAnything2.js

@@ -0,0 +1,163 @@
+/** The checkpoint's configuration at its pinned revision. */
+export const DEPTH_ANYTHING_2_SMALL = {
+    dim: 384,
+    heads: 6,
+    depth: 12,
+    hidden: 1536,
+    patch: 14,
+    trainedGrid: 37,
+    taps: [2, 5, 8, 11],
+    neck: [48, 96, 192, 384],
+    fusion: 64,
+    headHidden: 32,
+};
+/** Transformers' DINOv2 trains every norm at ε = 1e-6. */
+const EPSILON = 1e-6;
+/**
+ * The graph for one image of `height × width` pixels, each a multiple of the patch, normalised by
+ * ImageNet's mean and deviation: input `image`, `[3, height, width]`; output `depth`,
+ * `[1, height, width]`.
+ */
+export function depthAnything2(config, height, width) {
+    const rows = height / config.patch;
+    const cols = width / config.patch;
+    if (!Number.isInteger(rows) || !Number.isInteger(cols)) {
+        throw new RangeError(`${height} × ${width} is not a whole number of ${config.patch}-pixel patches`);
+    }
+    return (weights, graph) => {
+        const { dim, heads, trainedGrid } = config;
+        const node = graph.node.bind(graph);
+        const w = (name, shape) => weights.read(name, shape);
+        const linear = (x, name, to, from) => node('linear', [x, w(`${name}.weight`, [to, from]), w(`${name}.bias`, [to])]);
+        const conv = (x, name, to, from, kernel, attributes = {}, bias = true) => node('conv2d', bias
+            ? [x, w(`${name}.weight`, [to, from, kernel, kernel]), w(`${name}.bias`, [to])]
+            : [x, w(`${name}.weight`, [to, from, kernel, kernel])], attributes);
+        weights.ignore('backbone.embeddings.mask_token');
+        weights.ignore('neck.fusion_stage.layers.0.residual_layer1.');
+        const e = 'backbone.embeddings.';
+        const patches = rows * cols;
+        const tokens = patches + 1;
+        const table = node('reshape', [w(`${e}position_embeddings`, [1, trainedGrid * trainedGrid + 1, dim])], {
+            shape: [trainedGrid * trainedGrid + 1, dim],
+        });
+        let positions = table;
+        if (rows !== trainedGrid || cols !== trainedGrid) {
+            const grid = node('permute', [
+                node('reshape', [node('slice', [table], { axis: 0, start: 1, end: trainedGrid * trainedGrid + 1 })], {
+                    shape: [trainedGrid, trainedGrid, dim],
+                }),
+            ], { order: [2, 0, 1] });
+            const resized = node('resize', [grid], {
+                height: rows,
+                width: cols,
+                mode: 'bicubic',
+                alignCorners: false,
+            });
+            positions = node('concat', [
+                node('slice', [table], { axis: 0, start: 0, end: 1 }),
+                node('reshape', [node('permute', [resized], { order: [1, 2, 0] })], {
+                    shape: [patches, dim],
+                }),
+            ], { axis: 0 });
+        }
+        const embedded = node('patchEmbed', [
+            'image',
+            w(`${e}patch_embeddings.projection.weight`, [dim, 3, config.patch, config.patch]),
+            w(`${e}patch_embeddings.projection.bias`, [dim]),
+        ], { patch: config.patch });
+        let x = node('add', [
+            node('concat', [node('reshape', [w(`${e}cls_token`, [1, 1, dim])], { shape: [1, dim] }), embedded], { axis: 0 }),
+            positions,
+        ]);
+        const taps = [];
+        for (let i = 0; i < config.depth; i += 1) {
+            const p = `backbone.encoder.layer.${i}.`;
+            const h = node('layerNorm', [x, w(`${p}norm1.weight`, [dim]), w(`${p}norm1.bias`, [dim])], {
+                epsilon: EPSILON,
+            });
+            const attended = node('attention', [
+                linear(h, `${p}attention.attention.query`, dim, dim),
+                linear(h, `${p}attention.attention.key`, dim, dim),
+                linear(h, `${p}attention.attention.value`, dim, dim),
+            ], { heads });
+            const out = linear(attended, `${p}attention.output.dense`, dim, dim);
+            x = node('add', [x, node('mul', [out, w(`${p}layer_scale1.lambda1`, [dim])])]);
+            const h2 = node('layerNorm', [x, w(`${p}norm2.weight`, [dim]), w(`${p}norm2.bias`, [dim])], {
+                epsilon: EPSILON,
+            });
+            const mlp = linear(node('gelu', [linear(h2, `${p}mlp.fc1`, config.hidden, dim)]), `${p}mlp.fc2`, dim, config.hidden);
+            x = node('add', [x, node('mul', [mlp, w(`${p}layer_scale2.lambda1`, [dim])])]);
+            if (config.taps.includes(i)) {
+                taps.push(node('layerNorm', [x, w('backbone.layernorm.weight', [dim]), w('backbone.layernorm.bias', [dim])], { epsilon: EPSILON }));
+            }
+        }
+        const half = (size) => Math.floor((size - 1) / 2) + 1;
+        const sizes = [
+            [4 * rows, 4 * cols],
+            [2 * rows, 2 * cols],
+            [rows, cols],
+            [half(rows), half(cols)],
+        ];
+        const levels = taps.map((tap, k) => {
+            const channels = config.neck[k];
+            const r = `neck.reassemble_stage.layers.${k}.`;
+            const image = node('reshape', [
+                node('permute', [node('slice', [tap], { axis: 0, start: 1, end: tokens })], {
+                    order: [1, 0],
+                }),
+            ], {
+                shape: [dim, rows, cols],
+            });
+            const projected = conv(image, `${r}projection`, channels, dim, 1);
+            const resized = k === 0
+                ? node('convTranspose2d', [
+                    projected,
+                    w(`${r}resize.weight`, [channels, channels, 4, 4]),
+                    w(`${r}resize.bias`, [channels]),
+                ], { stride: 4 })
+                : k === 1
+                    ? node('convTranspose2d', [
+                        projected,
+                        w(`${r}resize.weight`, [channels, channels, 2, 2]),
+                        w(`${r}resize.bias`, [channels]),
+                    ], { stride: 2 })
+                    : k === 2
+                        ? projected
+                        : conv(projected, `${r}resize`, channels, channels, 3, { stride: 2, padding: 1 });
+            return conv(resized, `neck.convs.${k}`, config.fusion, channels, 3, { padding: 1 }, false);
+        });
+        const f = config.fusion;
+        const residual = (value, name) => {
+            const first = conv(node('relu', [value]), `${name}.convolution1`, f, f, 3, { padding: 1 });
+            return node('add', [
+                conv(node('relu', [first]), `${name}.convolution2`, f, f, 3, { padding: 1 }),
+                value,
+            ]);
+        };
+        /* Coarsest first: layer 0 takes the fourth level alone, and each after it adds the next finer. */
+        let fused = '';
+        for (let n = 0; n < 4; n += 1) {
+            const level = levels[3 - n];
+            const name = `neck.fusion_stage.layers.${n}`;
+            const merged = n === 0 ? level : node('add', [fused, residual(level, `${name}.residual_layer1`)]);
+            const refined = residual(merged, `${name}.residual_layer2`);
+            const size = n < 3
+                ? sizes[2 - n]
+                : [2 * sizes[0][0], 2 * sizes[0][1]];
+            const up = node('resize', [refined], {
+                height: size[0],
+                width: size[1],
+                mode: 'bilinear',
+                alignCorners: true,
+            });
+            fused = conv(up, `${name}.projection`, f, f, 1);
+        }
+        const first = conv(fused, 'head.conv1', f / 2, f, 3, { padding: 1 });
+        const up = node('resize', [first], { height, width, mode: 'bilinear', alignCorners: true });
+        const second = node('relu', [
+            conv(up, 'head.conv2', config.headHidden, f / 2, 3, { padding: 1 }),
+        ]);
+        node('relu', [conv(second, 'head.conv3', 1, config.headHidden, 1)], {}, 'depth');
+        return { inputs: [{ name: 'image', shape: [3, height, width] }], outputs: ['depth'] };
+    };
+}

package/dist/models/depthAnything3.d.ts

@@ -0,0 +1,26 @@
+import type { Architecture } from '@driftengine/texture';
+import { type HeadConfig } from './dualDpt.ts';
+import { type VitConfig } from './vit.ts';
+export interface DepthAnything3Config {
+    readonly backbone: VitConfig;
+    readonly head: HeadConfig;
+}
+/** The two accepted sizes, from each checkpoint's own configuration at its pinned revision. */
+export declare const DEPTH_ANYTHING_3: Readonly<Record<'small' | 'base', DepthAnything3Config>>;
+/**
+ * The graph for `views` images of `height × width` pixels, each a multiple of 14, normalised by
+ * ImageNet's mean and deviation. Inputs are `image0` … as `[3, height, width]`; outputs are
+ * `logits0` … as `[2, height, width]` and `pose` as `[views, 9]`. The first view is the reference.
+ */
+export declare function depthAnything3(config: DepthAnything3Config, views: number, height: number, width: number): Architecture;
+/**
+ * Depth and confidence from one view's logits, `[2, height, width]`: `exp` of the first channel,
+ * and one more than `exp` of the second, into the caller's maps.
+ */
+export declare function decodeDepth(logits: Float32Array, depth: Float32Array, confidence: Float32Array): void;
+/**
+ * One view's camera from its pose encoding: `worldToCamera`, 3 × 4 row-major — the inverse of the
+ * camera-to-world rotation and translation the decoder predicts — and `intrinsics`, 3 × 3 row-major,
+ * with focal lengths from the two fields of view and the principal point at the centre.
+ */
+export declare function decodeCamera(pose: Float32Array, at: number, height: number, width: number, worldToCamera: Float32Array, intrinsics: Float32Array): void;