@driftengine/capture 4.0.0

package/dist/models/depthAnything3.js

@@ -0,0 +1,171 @@
+/**
+ * Depth Anything 3, as a definition the runtime builds a graph from at whatever size a clip needs.
+ *
+ * **From one to a handful of views at once, a depth map, its confidence and a camera per view.**
+ * The backbone is `vit.ts`, the head `dualDpt.ts`, and the camera decoder here: two layers over
+ * each view's camera token from the last tap, then a translation, a scalar-last quaternion and two
+ * fields of view. The graph ends at logits and at that pose encoding; `decodeDepth` and
+ * `decodeCamera` turn them into metres of relative depth, a confidence of at least one, a
+ * world-to-camera transform and a pinhole, as the upstream's own code does after its network.
+ *
+ * **What is set aside, by name**: the camera encoder, which conditions on poses a caller already
+ * has and this capture never does; and the head's ray branch, whose output the upstream discards
+ * whenever the camera decoder is present. Converting a checkpoint that holds anything else fails
+ * naming it.
+ *
+ * **What would make this wrong**: a checkpoint trained with another block layout — the upstream's
+ * large and giant models use a different feed-forward — which their configurations would say, and
+ * which are refused for their licences in any case.
+ */
+import { exactCos, exactExp, exactSin } from '@driftengine/core';
+import { dualDptHead } from './dualDpt.js';
+import { vitBackbone } from './vit.js';
+const PATCH = 14;
+/** The two accepted sizes, from each checkpoint's own configuration at its pinned revision. */
+export const DEPTH_ANYTHING_3 = {
+    small: {
+        backbone: {
+            dim: 384,
+            heads: 6,
+            depth: 12,
+            hidden: 1536,
+            patch: PATCH,
+            trainedGrid: 37,
+            taps: [5, 7, 9, 11],
+            altStart: 4,
+            qkNormStart: 4,
+            ropeStart: 4,
+        },
+        head: { dimIn: 768, features: 64, outChannels: [48, 96, 192, 384], outputDim: 2, patch: PATCH },
+    },
+    base: {
+        backbone: {
+            dim: 768,
+            heads: 12,
+            depth: 12,
+            hidden: 3072,
+            patch: PATCH,
+            trainedGrid: 37,
+            taps: [5, 7, 9, 11],
+            altStart: 4,
+            qkNormStart: 4,
+            ropeStart: 4,
+        },
+        head: {
+            dimIn: 1536,
+            features: 128,
+            outChannels: [96, 192, 384, 768],
+            outputDim: 2,
+            patch: PATCH,
+        },
+    },
+};
+/** What the upstream checkpoints hold and no graph here reads. */
+const SET_ASIDE = [
+    'model.cam_enc.',
+    'model.head.scratch.output_conv1_aux.',
+    'model.head.scratch.output_conv2_aux.',
+    'model.head.scratch.refinenet1_aux.',
+    'model.head.scratch.refinenet2_aux.',
+    'model.head.scratch.refinenet3_aux.',
+    'model.head.scratch.refinenet4_aux.',
+];
+/**
+ * The graph for `views` images of `height × width` pixels, each a multiple of 14, normalised by
+ * ImageNet's mean and deviation. Inputs are `image0` … as `[3, height, width]`; outputs are
+ * `logits0` … as `[2, height, width]` and `pose` as `[views, 9]`. The first view is the reference.
+ */
+export function depthAnything3(config, views, height, width) {
+    const rows = height / PATCH;
+    const cols = width / PATCH;
+    if (!Number.isInteger(rows) || !Number.isInteger(cols)) {
+        throw new RangeError(`${height} × ${width} is not a whole number of ${PATCH}-pixel patches`);
+    }
+    return (weights, graph) => {
+        for (const prefix of SET_ASIDE)
+            weights.ignore(prefix);
+        const images = Array.from({ length: views }, (_, v) => `image${v}`);
+        const { features, cameras } = vitBackbone(weights, graph, config.backbone, 'model.backbone.pretrained.', images, rows, cols);
+        const outputs = images.map((_, v) => dualDptHead(weights, graph, config.head, 'model.head.', features.map((tap) => tap[v]), rows, cols, `logits${v}`));
+        const width2 = 2 * config.backbone.dim;
+        const node = graph.node.bind(graph);
+        const layer = (x, name, to) => node('linear', [
+            x,
+            weights.read(`model.cam_dec.${name}.weight`, [to, width2]),
+            weights.read(`model.cam_dec.${name}.bias`, [to]),
+        ]);
+        const trunk = node('relu', [
+            layer(node('relu', [layer(cameras[cameras.length - 1], 'backbone.0', width2)]), 'backbone.2', width2),
+        ]);
+        node('concat', [
+            layer(trunk, 'fc_t', 3),
+            layer(trunk, 'fc_qvec', 4),
+            node('relu', [layer(trunk, 'fc_fov.0', 2)]),
+        ], { axis: 1 }, 'pose');
+        return {
+            inputs: images.map((name) => ({ name, shape: [3, height, width] })),
+            outputs: [...outputs, 'pose'],
+        };
+    };
+}
+/**
+ * Depth and confidence from one view's logits, `[2, height, width]`: `exp` of the first channel,
+ * and one more than `exp` of the second, into the caller's maps.
+ */
+export function decodeDepth(logits, depth, confidence) {
+    const pixels = depth.length;
+    for (let i = 0; i < pixels; i += 1) {
+        depth[i] = exactExp(logits[i]);
+        confidence[i] = exactExp(logits[pixels + i]) + 1;
+    }
+}
+/**
+ * One view's camera from its pose encoding: `worldToCamera`, 3 × 4 row-major — the inverse of the
+ * camera-to-world rotation and translation the decoder predicts — and `intrinsics`, 3 × 3 row-major,
+ * with focal lengths from the two fields of view and the principal point at the centre.
+ */
+export function decodeCamera(pose, at, height, width, worldToCamera, intrinsics) {
+    const p = at * 9;
+    const tx = pose[p];
+    const ty = pose[p + 1];
+    const tz = pose[p + 2];
+    const i = pose[p + 3];
+    const j = pose[p + 4];
+    const k = pose[p + 5];
+    const r = pose[p + 6];
+    const s = 2 / (i * i + j * j + k * k + r * r);
+    /*
+     * The camera-to-world rotation is `quat_to_mat`'s; its inverse is the transpose, so row `n` here
+     * is column `n` there, and the translation is carried back through it.
+     */
+    const r00 = 1 - s * (j * j + k * k);
+    const r01 = s * (i * j - k * r);
+    const r02 = s * (i * k + j * r);
+    const r10 = s * (i * j + k * r);
+    const r11 = 1 - s * (i * i + k * k);
+    const r12 = s * (j * k - i * r);
+    const r20 = s * (i * k - j * r);
+    const r21 = s * (j * k + i * r);
+    const r22 = 1 - s * (i * i + j * j);
+    worldToCamera[0] = r00;
+    worldToCamera[1] = r10;
+    worldToCamera[2] = r20;
+    worldToCamera[3] = -(r00 * tx + r10 * ty + r20 * tz);
+    worldToCamera[4] = r01;
+    worldToCamera[5] = r11;
+    worldToCamera[6] = r21;
+    worldToCamera[7] = -(r01 * tx + r11 * ty + r21 * tz);
+    worldToCamera[8] = r02;
+    worldToCamera[9] = r12;
+    worldToCamera[10] = r22;
+    worldToCamera[11] = -(r02 * tx + r12 * ty + r22 * tz);
+    const tangent = (angle) => exactSin(angle / 2) / exactCos(angle / 2);
+    const fy = height / 2 / Math.max(tangent(pose[p + 7]), 1e-6);
+    const fx = width / 2 / Math.max(tangent(pose[p + 8]), 1e-6);
+    intrinsics.fill(0);
+    intrinsics[0] = fx;
+    intrinsics[2] = width / 2;
+    intrinsics[4] = fy;
+    intrinsics[5] = height / 2;
+    intrinsics[8] = 1;
+}

package/dist/models/dualDpt.d.ts

@@ -0,0 +1,19 @@
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+export interface HeadConfig {
+    readonly dimIn: number;
+    readonly features: number;
+    readonly outChannels: readonly [number, number, number, number];
+    readonly outputDim: number;
+    readonly patch: number;
+}
+/**
+ * `[channels, height, width]`: the upstream's `_add_pos_embed` for a map of this size in an image of
+ * `aspect` width over height — sine and cosine of the horizontal coordinate, then of the vertical,
+ * over `channels / 4` frequencies each, times a tenth.
+ */
+export declare function uvEmbedding(channels: number, height: number, width: number, aspect: number): Float32Array;
+/**
+ * The head over one view's four taps, each `[rows · cols, dimIn]`; the result is the logits,
+ * `[outputDim, rows · patch, cols · patch]`, named `output`. Several views share its constants.
+ */
+export declare function dualDptHead(weights: Weights, graph: GraphBuilder, config: HeadConfig, prefix: string, taps: readonly string[], rows: number, cols: number, output: string): string;

package/dist/models/dualDpt.js

@@ -0,0 +1,138 @@
+/**
+ * Depth Anything 3's head, the DPT that turns four taps of tokens into depth and its confidence.
+ *
+ * **Each tap is normed, laid back out as an image, projected, and resized to its level of a
+ * pyramid** — four times the patch grid, twice, once, and half — then four fusion blocks walk the
+ * pyramid from coarse to fine, each adding the level below through a residual unit, and the result
+ * is brought up to the image by an aligned bilinear resize and read out by two convolutions: a
+ * depth logit and a confidence logit per pixel. Their activations, `exp` and `exp + 1`, are the
+ * caller's, applied to the two channels as they are read back, since the runtime has no `exp` and
+ * one pass over a map on the host costs less than a kernel for it.
+ *
+ * **A sinusoidal position is added three times**, after each projection and before the readout:
+ * a grid spanning the image's aspect normalised by its diagonal, as the upstream's `create_uv_grid`
+ * makes it, embedded at base 100 and scaled by a tenth. It depends only on shapes, so each is a
+ * constant of the graph.
+ *
+ * **The upstream's second head is not built.** It predicts rays for a pose estimate the camera
+ * decoder already makes, and the upstream deletes its output when that decoder is present, as it is
+ * in every accepted checkpoint; its weights are set aside by name.
+ */
+import { exactCos, exactExp, exactSin } from '@driftengine/core';
+import { LOG_BASE } from './rope.js';
+/** The head's own norms keep PyTorch's default ε. */
+const EPSILON = 1e-5;
+const f = Math.fround;
+/**
+ * `[channels, height, width]`: the upstream's `_add_pos_embed` for a map of this size in an image of
+ * `aspect` width over height — sine and cosine of the horizontal coordinate, then of the vertical,
+ * over `channels / 4` frequencies each, times a tenth.
+ */
+export function uvEmbedding(channels, height, width, aspect) {
+    const diagonal = Math.sqrt(aspect * aspect + 1);
+    const spanX = aspect / diagonal;
+    const spanY = 1 / diagonal;
+    const coordinate = (span, count, i) => {
+        const edge = (span * (count - 1)) / count;
+        return count === 1 ? -edge : f(-edge + ((2 * edge) / (count - 1)) * i);
+    };
+    const quarter = channels / 4;
+    const omega = Float32Array.from({ length: quarter }, (_, k) => f(1 / f(exactExp(f(k / quarter) * LOG_BASE))));
+    const out = new Float32Array(channels * height * width);
+    for (let y = 0; y < height; y += 1) {
+        const v = coordinate(spanY, height, y);
+        for (let x = 0; x < width; x += 1) {
+            const u = coordinate(spanX, width, x);
+            for (let k = 0; k < quarter; k += 1) {
+                const au = f(u * omega[k]);
+                const av = f(v * omega[k]);
+                const at = y * width + x;
+                const plane = height * width;
+                out[k * plane + at] = f(f(exactSin(au)) * 0.1);
+                out[(quarter + k) * plane + at] = f(f(exactCos(au)) * 0.1);
+                out[(2 * quarter + k) * plane + at] = f(f(exactSin(av)) * 0.1);
+                out[(3 * quarter + k) * plane + at] = f(f(exactCos(av)) * 0.1);
+            }
+        }
+    }
+    return out;
+}
+/** The size a stride-2 3×3 convolution with padding 1 leaves. */
+const halved = (size) => Math.floor((size - 1) / 2) + 1;
+/**
+ * The head over one view's four taps, each `[rows · cols, dimIn]`; the result is the logits,
+ * `[outputDim, rows · patch, cols · patch]`, named `output`. Several views share its constants.
+ */
+export function dualDptHead(weights, graph, config, prefix, taps, rows, cols, output) {
+    const node = graph.node.bind(graph);
+    const w = (name, shape) => weights.read(`${prefix}${name}`, shape);
+    const height = rows * config.patch;
+    const width = cols * config.patch;
+    const aspect = width / height;
+    const features = config.features;
+    const embed = (x, channels, h, wide) => node('add', [
+        x,
+        weights.constant(`uv.${channels}.${h}.${wide}`, [channels, h, wide], uvEmbedding(channels, h, wide, aspect)),
+    ]);
+    const conv = (x, name, from, to, kernel, attributes = {}, bias = true, named) => node('conv2d', bias
+        ? [x, w(`${name}.weight`, [to, from, kernel, kernel]), w(`${name}.bias`, [to])]
+        : [x, w(`${name}.weight`, [to, from, kernel, kernel])], attributes, named);
+    const sizes = [
+        [4 * rows, 4 * cols],
+        [2 * rows, 2 * cols],
+        [rows, cols],
+        [halved(rows), halved(cols)],
+    ];
+    const levels = taps.map((tap, k) => {
+        const channels = config.outChannels[k];
+        const normed = node('layerNorm', [tap, w('norm.weight', [config.dimIn]), w('norm.bias', [config.dimIn])], { epsilon: EPSILON });
+        const image = node('reshape', [node('permute', [normed], { order: [1, 0] })], {
+            shape: [config.dimIn, rows, cols],
+        });
+        const projected = embed(conv(image, `projects.${k}`, config.dimIn, channels, 1), channels, rows, cols);
+        const resized = k === 0
+            ? node('convTranspose2d', [
+                projected,
+                w('resize_layers.0.weight', [channels, channels, 4, 4]),
+                w('resize_layers.0.bias', [channels]),
+            ], { stride: 4 })
+            : k === 1
+                ? node('convTranspose2d', [
+                    projected,
+                    w('resize_layers.1.weight', [channels, channels, 2, 2]),
+                    w('resize_layers.1.bias', [channels]),
+                ], { stride: 2 })
+                : k === 2
+                    ? projected
+                    : conv(projected, 'resize_layers.3', channels, channels, 3, { stride: 2, padding: 1 });
+        return conv(resized, `scratch.layer${k + 1}_rn`, channels, features, 3, { padding: 1 }, false);
+    });
+    const residual = (x, name) => {
+        const first = conv(node('relu', [x]), `${name}.conv1`, features, features, 3, { padding: 1 });
+        return node('add', [
+            conv(node('relu', [first]), `${name}.conv2`, features, features, 3, { padding: 1 }),
+            x,
+        ]);
+    };
+    const fuse = (level, top, lateral, size) => {
+        const name = `scratch.refinenet${level}`;
+        const merged = lateral === null ? top : node('add', [top, residual(lateral, `${name}.resConfUnit1`)]);
+        const refined = residual(merged, `${name}.resConfUnit2`);
+        const up = node('resize', [refined], {
+            height: size[0],
+            width: size[1],
+            mode: 'bilinear',
+            alignCorners: true,
+        });
+        return conv(up, `${name}.out_conv`, features, features, 1);
+    };
+    const [l1, l2, l3, l4] = levels;
+    let out = fuse(4, l4, null, sizes[2]);
+    out = fuse(3, out, l3, sizes[1]);
+    out = fuse(2, out, l2, sizes[0]);
+    out = fuse(1, out, l1, [2 * sizes[0][0], 2 * sizes[0][1]]);
+    out = conv(out, 'scratch.output_conv1', features, features / 2, 3, { padding: 1 });
+    const up = node('resize', [out], { height, width, mode: 'bilinear', alignCorners: true });
+    const read = conv(embed(up, features / 2, height, width), 'scratch.output_conv2.0', features / 2, 32, 3, { padding: 1 });
+    return conv(node('relu', [read]), 'scratch.output_conv2.2', 32, config.outputDim, 1, {}, true, output);
+}

package/dist/models/hiera.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Hiera and its feature pyramid, as SAM 2 runs them: an image in, three levels of features out.
+ *
+ * **Four stages of transformer blocks over a shrinking grid.** A 7×7 convolution at stride 4 takes
+ * the image to a quarter; each later stage's first block halves the grid by pooling its queries 2×2
+ * — within each window for attention, and over the whole grid for the residual, which a projection
+ * widens — so attention reads every key and writes a quarter as many queries. Blocks attend within
+ * windows, padded with zeros after the block's norm where the window does not divide the grid, and
+ * the blocks the configuration names attend over the whole grid instead. A first block partitions by
+ * the previous stage's window and unpartitions by half of it.
+ *
+ * **The positional embedding is computed in the graph, from its two weights**: a table resized
+ * bicubically to the grid, plus a window's table tiled across it. Laid out at conversion it would be
+ * 12.6 MB of the file at half precision; computed, it costs two permutes a run.
+ *
+ * **The pyramid runs top-down only into the levels the configuration names**: each stage's output is
+ * projected to the pyramid's width, and a named level adds the level above it doubled by nearest
+ * resizing. The two finest levels are then narrowed by the mask decoder's own projections, as the
+ * upstream's `get_image_features` narrows them once rather than on every prompt. Norms take ε 1e-6.
+ */
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+type Four = readonly [number, number, number, number];
+export interface HieraConfig {
+    readonly dims: Four;
+    readonly blocks: Four;
+    readonly heads: Four;
+    readonly windows: Four;
+    /** Block indices, counted across stages, that attend over the whole grid. */
+    readonly global: readonly number[];
+    /** The positional table's trained size, square. */
+    readonly background: number;
+    readonly mlpRatio: number;
+    /** The pyramid's width, and the levels (by stage) that take the level above them. */
+    readonly fpn: number;
+    readonly topDown: readonly number[];
+}
+export interface HieraOutput {
+    /** `[fpn, grid, grid]` at a sixteenth of the image: the features the decoder and memory read. */
+    readonly features: string;
+    /** `[fpn / 4, 2·grid, 2·grid]` and `[fpn / 8, 4·grid, 4·grid]`: the decoder's skips, narrowed. */
+    readonly high1: string;
+    readonly high0: string;
+}
+/**
+ * The encoder over `image`, `[3, size, size]` normalised as the upstream prepares it, writing its
+ * three outputs under the names given.
+ */
+export declare function hiera(weights: Weights, graph: GraphBuilder, config: HieraConfig, image: string, size: number, names: HieraOutput): void;
+export {};

package/dist/models/hiera.js

@@ -0,0 +1,120 @@
+import { hieraAttention } from './hieraAttention.js';
+const EPSILON = 1e-6;
+/**
+ * The encoder over `image`, `[3, size, size]` normalised as the upstream prepares it, writing its
+ * three outputs under the names given.
+ */
+export function hiera(weights, graph, config, image, size, names) {
+    const node = graph.node.bind(graph);
+    const b = 'vision_encoder.backbone.';
+    const read = weights.read.bind(weights);
+    const linear = (x, p, out, inputs) => node('linear', [x, read(`${p}weight`, [out, inputs]), read(`${p}bias`, [out])]);
+    const norm = (x, p, dim) => node('layerNorm', [x, read(`${p}weight`, [dim]), read(`${p}bias`, [dim])], {
+        epsilon: EPSILON,
+    });
+    /* The patch embedding, then the positions, both channel-major. */
+    const [d0] = config.dims;
+    let grid = size / 4;
+    const patches = node('conv2d', [
+        image,
+        read(`${b}patch_embed.projection.weight`, [d0, 3, 7, 7]),
+        read(`${b}patch_embed.projection.bias`, [d0]),
+    ], { stride: 4, padding: 3 });
+    const window0 = config.windows[0];
+    const tiles = grid / window0;
+    const background = node('resize', [
+        node('reshape', [read(`${b}pos_embed`, [1, d0, config.background, config.background])], {
+            shape: [d0, config.background, config.background],
+        }),
+    ], { height: grid, width: grid, mode: 'bicubic', alignCorners: false });
+    /* The window's table added to every tile: tiles to the leading axes, the window to the last. */
+    const byTile = node('reshape', [
+        node('permute', [node('reshape', [background], { shape: [d0, tiles, window0, tiles, window0] })], { order: [1, 3, 0, 2, 4] }),
+    ], { shape: [tiles * tiles, d0 * window0 * window0] });
+    const window = node('reshape', [read(`${b}pos_embed_window`, [1, d0, window0, window0])], {
+        shape: [d0 * window0 * window0],
+    });
+    const positions = node('reshape', [
+        node('permute', [
+            node('reshape', [node('add', [byTile, window])], {
+                shape: [tiles, tiles, d0, window0, window0],
+            }),
+        ], { order: [2, 0, 3, 1, 4] }),
+    ], { shape: [d0, grid, grid] });
+    const embedded = node('add', [patches, positions]);
+    /* Tokens, row by row, `[grid · grid, dim]`, from here to the pyramid. */
+    let x = node('permute', [node('reshape', [embedded], { shape: [d0, grid * grid] })], {
+        order: [1, 0],
+    });
+    const pooled = (tokens, rows, dim) => {
+        const map = node('reshape', [node('permute', [tokens], { order: [1, 0] })], {
+            shape: [dim, rows, rows],
+        });
+        const pool = node('maxPool2d', [map], { kernel: 2 });
+        return node('permute', [node('reshape', [pool], { shape: [dim, (rows / 2) * (rows / 2)] })], {
+            order: [1, 0],
+        });
+    };
+    const outputs = [];
+    let index = 0;
+    for (let stage = 0; stage < 4; stage += 1) {
+        for (let block = 0; block < config.blocks[stage]; block += 1) {
+            const p = `${b}blocks.${index}.`;
+            const first = stage > 0 && block === 0;
+            const dim = first ? config.dims[stage - 1] : config.dims[stage];
+            const out = config.dims[stage];
+            const heads = config.heads[stage];
+            const side = config.global.includes(index)
+                ? 0
+                : first
+                    ? config.windows[stage - 1]
+                    : config.windows[stage];
+            const normed = norm(x, `${p}layer_norm1.`, dim);
+            const residual = first ? pooled(linear(normed, `${p}proj.`, out, dim), grid, out) : x;
+            const attended = hieraAttention(weights, graph, p, normed, grid, dim, out, heads, side, first);
+            if (first)
+                grid /= 2;
+            x = node('add', [residual, attended]);
+            const hidden = node('gelu', [
+                linear(norm(x, `${p}layer_norm2.`, out), `${p}mlp.proj_in.`, out * config.mlpRatio, out),
+            ]);
+            x = node('add', [x, linear(hidden, `${p}mlp.proj_out.`, out, out * config.mlpRatio)]);
+            index += 1;
+        }
+        outputs.push(x);
+    }
+    /* The pyramid, coarsest first: each stage projected, and a named level adding the one above. */
+    const n = 3;
+    const sides = [size / 4, size / 8, size / 16, size / 32];
+    const levels = [];
+    let previous = '';
+    for (let i = n; i >= 0; i -= 1) {
+        const p = `vision_encoder.neck.convs.${n - i}.`;
+        const channels = config.dims[i];
+        const side = sides[i];
+        const map = node('reshape', [node('permute', [outputs[i]], { order: [1, 0] })], {
+            shape: [channels, side, side],
+        });
+        const down = config.topDown.includes(i) && i !== n;
+        const name = i === 2 ? names.features : undefined;
+        let lateral = node('conv2d', [map, read(`${p}weight`, [config.fpn, channels, 1, 1]), read(`${p}bias`, [config.fpn])], {}, down ? undefined : name);
+        if (down) {
+            const above = node('resize', [previous], { height: side, width: side, mode: 'nearest' });
+            lateral = node('add', [lateral, above], {}, name);
+        }
+        previous = lateral;
+        levels[i] = lateral;
+    }
+    const d = 'mask_decoder.';
+    node('conv2d', [
+        levels[0],
+        read(`${d}conv_s0.weight`, [config.fpn / 8, config.fpn, 1, 1]),
+        read(`${d}conv_s0.bias`, [config.fpn / 8]),
+    ], {}, names.high0);
+    node('conv2d', [
+        levels[1],
+        read(`${d}conv_s1.weight`, [config.fpn / 4, config.fpn, 1, 1]),
+        read(`${d}conv_s1.bias`, [config.fpn / 4]),
+    ], {}, names.high1);
+    /* The coarsest level feeds the one below and is not itself an output, as the upstream keeps three. */
+}

package/dist/models/hieraAttention.d.ts

@@ -0,0 +1,16 @@
+/**
+ * One Hiera block's attention: within windows of its grid, or over the whole grid, its queries
+ * pooled 2×2 within each window where the block changes stage.
+ *
+ * **Windows are partitioned after the block's norm**, padded with zeros where the window does not
+ * divide the grid — so a padded token is zero, where TinyViT's is its norm's β — and unpartitioned
+ * by the queries' window, the padding recomputed for the pooled grid, as the upstream's
+ * `window_unpartition` is given it. The projection lays each head's queries, keys and values out
+ * contiguously, as the attention operator takes them.
+ */
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+/**
+ * `[rows² or (rows/2)², out]` from `normed`, `[rows², dim]`: the attention under `p` within windows
+ * of `side` — the whole grid for 0 — pooling its queries when `pool`.
+ */
+export declare function hieraAttention(weights: Weights, graph: GraphBuilder, p: string, normed: string, rows: number, dim: number, out: number, heads: number, side: number, pool: boolean): string;

package/dist/models/hieraAttention.js

@@ -0,0 +1,68 @@
+/**
+ * `[rows² or (rows/2)², out]` from `normed`, `[rows², dim]`: the attention under `p` within windows
+ * of `side` — the whole grid for 0 — pooling its queries when `pool`.
+ */
+export function hieraAttention(weights, graph, p, normed, rows, dim, out, heads, side, pool) {
+    const node = graph.node.bind(graph);
+    const linear = (x, q, outputs, inputs) => node('linear', [
+        x,
+        weights.read(`${q}weight`, [outputs, inputs]),
+        weights.read(`${q}bias`, [outputs]),
+    ]);
+    const span = side === 0 ? rows : side;
+    const pad = (span - (rows % span)) % span;
+    const across = (rows + pad) / span;
+    const count = across * across;
+    let tokens = normed;
+    if (side > 0) {
+        let map = node('reshape', [normed], { shape: [rows, rows, dim] });
+        if (pad > 0)
+            map = node('pad', [map], { after: [pad, pad, 0] });
+        tokens = node('reshape', [
+            node('permute', [node('reshape', [map], { shape: [across, span, across, span, dim] })], {
+                order: [0, 2, 1, 3, 4],
+            }),
+        ], { shape: [count * span * span, dim] });
+    }
+    const qkv = linear(tokens, `${p}attn.qkv.`, 3 * out, dim);
+    const part = (i) => node('slice', [qkv], { axis: 1, start: i * out, end: (i + 1) * out });
+    let queries = node('reshape', [part(0)], { shape: [count, span * span, out] });
+    let qSpan = span;
+    if (pool) {
+        /* Each window's queries as a map, pooled 2×2: windows and channels together as the pool's planes. */
+        const maps = node('reshape', [
+            node('permute', [node('reshape', [part(0)], { shape: [count, span, span, out] })], {
+                order: [0, 3, 1, 2],
+            }),
+        ], { shape: [count * out, span, span] });
+        qSpan = span / 2;
+        queries = node('reshape', [
+            node('permute', [
+                node('reshape', [node('maxPool2d', [maps], { kernel: 2 })], {
+                    shape: [count, out, qSpan, qSpan],
+                }),
+            ], { order: [0, 2, 3, 1] }),
+        ], { shape: [count, qSpan * qSpan, out] });
+    }
+    const keys = node('reshape', [part(1)], { shape: [count, span * span, out] });
+    const values = node('reshape', [part(2)], { shape: [count, span * span, out] });
+    const attended = node('attention', [queries, keys, values], { heads });
+    const projected = linear(node('reshape', [attended], { shape: [count * qSpan * qSpan, out] }), `${p}attn.proj.`, out, out);
+    if (side === 0)
+        return projected;
+    /* Unpartitioned by the queries' window, the padding recomputed for the pooled grid. */
+    const pooledRows = pool ? rows / 2 : rows;
+    const qPad = (qSpan - (pooledRows % qSpan)) % qSpan;
+    const qAcross = (pooledRows + qPad) / qSpan;
+    let map = node('reshape', [
+        node('permute', [node('reshape', [projected], { shape: [qAcross, qAcross, qSpan, qSpan, out] })], { order: [0, 2, 1, 3, 4] }),
+    ], { shape: [pooledRows + qPad, pooledRows + qPad, out] });
+    if (qPad > 0) {
+        map = node('slice', [node('slice', [map], { axis: 0, start: 0, end: pooledRows })], {
+            axis: 1,
+            start: 0,
+            end: pooledRows,
+        });
+    }
+    return node('reshape', [map], { shape: [pooledRows * pooledRows, out] });
+}

package/dist/models/miniature.d.ts

@@ -0,0 +1,36 @@
+import type { GraphTensor } from '@driftengine/texture';
+import type { DepthAnything2Config } from './depthAnything2.ts';
+import type { DepthAnything3Config } from './depthAnything3.ts';
+/**
+ * Four blocks of 32 channels in four heads of 8 — the smallest a two-dimensional rotary embedding
+ * divides — with the camera token, query-key norms and rotary embedding from the third block, so
+ * one within-view and one across-view block each carry all three; a trained grid of 2, so any other
+ * grid is resized; and a head of 16 features.
+ */
+export declare const MINIATURE_DEPTH_ANYTHING_3: DepthAnything3Config;
+/** The seed the tests and the oracle share. */
+export declare const MINIATURE_SEED = 20260919;
+/** One view on a grid its positions are resized to, and two views on the trained grid. */
+export declare const MINIATURE_CASES: {
+    readonly single: {
+        readonly views: 1;
+        readonly height: 28;
+        readonly width: 42;
+    };
+    readonly pair: {
+        readonly views: 2;
+        readonly height: 28;
+        readonly width: 28;
+    };
+};
+/** Every tensor the upstream's backbone, head and camera decoder hold, named as the checkpoints are. */
+export declare function miniatureCheckpoint(config: DepthAnything3Config, seed: number): Map<string, GraphTensor>;
+/**
+ * Depth Anything V2 in miniature: four blocks of 32 channels in four heads, every block tapped, a
+ * trained grid of 2, and a neck and head of 16 and 8.
+ */
+export declare const MINIATURE_DEPTH_ANYTHING_2: DepthAnything2Config;
+/** Every tensor Transformers' Depth Anything holds, in the older names its checkpoints keep. */
+export declare function miniatureCheckpoint2(config: DepthAnything2Config, seed: number): Map<string, GraphTensor>;
+/** Seeded images for `views` views, normalised as the model expects, `[3, height, width]` each. */
+export declare function miniatureImages(views: number, height: number, width: number, seed: number): Float32Array[];