@driftengine/capture 4.0.0

package/dist/models/miniature.js

@@ -0,0 +1,216 @@
+/**
+ * A seeded miniature of Depth Anything 3: the upstream's layout, every tensor name and shape of it,
+ * at a size small enough to evaluate on the CPU in milliseconds.
+ *
+ * **It is what the model's tests and its hand-run oracle share.** The oracle loads this checkpoint
+ * into the upstream's own modules — strictly, so a name or shape that differs from theirs fails
+ * there — and runs their code; the tests run the same checkpoint through the definition here and
+ * compare. So the numbers a test asserts come from an implementation that is not the one under
+ * test, and no weights are downloaded or committed: the checkpoint is a function of a seed.
+ *
+ * **Every tensor the upstream holds is here, those the definition sets aside included** — the ray
+ * branch and, since it would only be loaded to be ignored, not the camera encoder, which the
+ * oracle does not build. Shipped, as `@driftengine/nav`'s test field is, because a consumer
+ * checking a port of their own wants the same fixture.
+ */
+import { mulberry32 } from '@driftengine/core';
+import { draw, nameSeed } from './seeded.js';
+/**
+ * Four blocks of 32 channels in four heads of 8 — the smallest a two-dimensional rotary embedding
+ * divides — with the camera token, query-key norms and rotary embedding from the third block, so
+ * one within-view and one across-view block each carry all three; a trained grid of 2, so any other
+ * grid is resized; and a head of 16 features.
+ */
+export const MINIATURE_DEPTH_ANYTHING_3 = {
+    backbone: {
+        dim: 32,
+        heads: 4,
+        depth: 4,
+        hidden: 128,
+        patch: 14,
+        trainedGrid: 2,
+        taps: [0, 1, 2, 3],
+        altStart: 2,
+        qkNormStart: 2,
+        ropeStart: 2,
+    },
+    head: { dimIn: 64, features: 16, outChannels: [4, 8, 8, 16], outputDim: 2, patch: 14 },
+};
+/** The seed the tests and the oracle share. */
+export const MINIATURE_SEED = 20260919;
+/** One view on a grid its positions are resized to, and two views on the trained grid. */
+export const MINIATURE_CASES = {
+    single: { views: 1, height: 28, width: 42 },
+    pair: { views: 2, height: 28, width: 28 },
+};
+/** Every tensor the upstream's backbone, head and camera decoder hold, named as the checkpoints are. */
+export function miniatureCheckpoint(config, seed) {
+    const out = new Map();
+    const add = (name, shape) => {
+        out.set(name, { shape, data: draw(name, shape, seed) });
+    };
+    const { dim, heads, depth, hidden, patch, trainedGrid } = config.backbone;
+    const b = 'model.backbone.pretrained.';
+    add(`${b}cls_token`, [1, 1, dim]);
+    add(`${b}camera_token`, [1, 2, dim]);
+    add(`${b}pos_embed`, [1, trainedGrid * trainedGrid + 1, dim]);
+    add(`${b}patch_embed.proj.weight`, [dim, 3, patch, patch]);
+    add(`${b}patch_embed.proj.bias`, [dim]);
+    for (let i = 0; i < depth; i += 1) {
+        const p = `${b}blocks.${i}.`;
+        add(`${p}norm1.weight`, [dim]);
+        add(`${p}norm1.bias`, [dim]);
+        add(`${p}attn.qkv.weight`, [3 * dim, dim]);
+        add(`${p}attn.qkv.bias`, [3 * dim]);
+        if (i >= config.backbone.qkNormStart) {
+            for (const norm of ['q_norm', 'k_norm']) {
+                add(`${p}attn.${norm}.weight`, [dim / heads]);
+                add(`${p}attn.${norm}.bias`, [dim / heads]);
+            }
+        }
+        add(`${p}attn.proj.weight`, [dim, dim]);
+        add(`${p}attn.proj.bias`, [dim]);
+        add(`${p}ls1.gamma`, [dim]);
+        add(`${p}norm2.weight`, [dim]);
+        add(`${p}norm2.bias`, [dim]);
+        add(`${p}mlp.fc1.weight`, [hidden, dim]);
+        add(`${p}mlp.fc1.bias`, [hidden]);
+        add(`${p}mlp.fc2.weight`, [dim, hidden]);
+        add(`${p}mlp.fc2.bias`, [dim]);
+        add(`${p}ls2.gamma`, [dim]);
+    }
+    add(`${b}norm.weight`, [dim]);
+    add(`${b}norm.bias`, [dim]);
+    const h = 'model.head.';
+    const { dimIn, features, outChannels, outputDim } = config.head;
+    const conv = (name, to, from, kernel, bias = true) => {
+        add(`${name}.weight`, [to, from, kernel, kernel]);
+        if (bias)
+            add(`${name}.bias`, [to]);
+    };
+    add(`${h}norm.weight`, [dimIn]);
+    add(`${h}norm.bias`, [dimIn]);
+    outChannels.forEach((channels, k) => {
+        conv(`${h}projects.${k}`, channels, dimIn, 1);
+        conv(`${h}scratch.layer${k + 1}_rn`, features, channels, 3, false);
+    });
+    conv(`${h}resize_layers.0`, outChannels[0], outChannels[0], 4);
+    conv(`${h}resize_layers.1`, outChannels[1], outChannels[1], 2);
+    conv(`${h}resize_layers.3`, outChannels[3], outChannels[3], 3);
+    for (const branch of ['', '_aux']) {
+        for (let level = 1; level <= 4; level += 1) {
+            const r = `${h}scratch.refinenet${level}${branch}.`;
+            for (const unit of level === 4 ? ['resConfUnit2'] : ['resConfUnit1', 'resConfUnit2']) {
+                conv(`${r}${unit}.conv1`, features, features, 3);
+                conv(`${r}${unit}.conv2`, features, features, 3);
+            }
+            conv(`${r}out_conv`, features, features, 1);
+        }
+    }
+    conv(`${h}scratch.output_conv1`, features / 2, features, 3);
+    conv(`${h}scratch.output_conv2.0`, 32, features / 2, 3);
+    conv(`${h}scratch.output_conv2.2`, outputDim, 32, 1);
+    for (let level = 0; level < 4; level += 1) {
+        const widths = [features / 2, features, features / 2, features, features / 2];
+        widths.forEach((to, n) => {
+            conv(`${h}scratch.output_conv1_aux.${level}.${n}`, to, n === 0 ? features : widths[n - 1], 3);
+        });
+        conv(`${h}scratch.output_conv2_aux.${level}.0`, 32, features / 2, 3);
+        add(`${h}scratch.output_conv2_aux.${level}.2.weight`, [32]);
+        add(`${h}scratch.output_conv2_aux.${level}.2.bias`, [32]);
+        conv(`${h}scratch.output_conv2_aux.${level}.5`, 7, 32, 1);
+    }
+    const c = 'model.cam_dec.';
+    const wide = 2 * dim;
+    for (const [name, to] of [
+        ['backbone.0', wide],
+        ['backbone.2', wide],
+        ['fc_t', 3],
+        ['fc_qvec', 4],
+        ['fc_fov.0', 2],
+    ]) {
+        add(`${c}${name}.weight`, [to, wide]);
+        add(`${c}${name}.bias`, [to]);
+    }
+    return out;
+}
+/**
+ * Depth Anything V2 in miniature: four blocks of 32 channels in four heads, every block tapped, a
+ * trained grid of 2, and a neck and head of 16 and 8.
+ */
+export const MINIATURE_DEPTH_ANYTHING_2 = {
+    dim: 32,
+    heads: 4,
+    depth: 4,
+    hidden: 128,
+    patch: 14,
+    trainedGrid: 2,
+    taps: [0, 1, 2, 3],
+    neck: [4, 8, 8, 16],
+    fusion: 16,
+    headHidden: 8,
+};
+/** Every tensor Transformers' Depth Anything holds, in the older names its checkpoints keep. */
+export function miniatureCheckpoint2(config, seed) {
+    const out = new Map();
+    const add = (name, shape) => {
+        out.set(name, { shape, data: draw(name, shape, seed) });
+    };
+    const pair = (name, to, from) => {
+        add(`${name}.weight`, [to, from]);
+        add(`${name}.bias`, [to]);
+    };
+    const conv = (name, to, from, kernel, bias = true) => {
+        add(`${name}.weight`, [to, from, kernel, kernel]);
+        if (bias)
+            add(`${name}.bias`, [to]);
+    };
+    const { dim, depth, hidden, patch, trainedGrid, neck, fusion, headHidden } = config;
+    const e = 'backbone.embeddings.';
+    add(`${e}cls_token`, [1, 1, dim]);
+    add(`${e}mask_token`, [1, dim]);
+    add(`${e}position_embeddings`, [1, trainedGrid * trainedGrid + 1, dim]);
+    conv(`${e}patch_embeddings.projection`, dim, 3, patch);
+    for (let i = 0; i < depth; i += 1) {
+        const p = `backbone.encoder.layer.${i}.`;
+        add(`${p}norm1.weight`, [dim]);
+        add(`${p}norm1.bias`, [dim]);
+        for (const part of ['query', 'key', 'value'])
+            pair(`${p}attention.attention.${part}`, dim, dim);
+        pair(`${p}attention.output.dense`, dim, dim);
+        add(`${p}layer_scale1.lambda1`, [dim]);
+        add(`${p}norm2.weight`, [dim]);
+        add(`${p}norm2.bias`, [dim]);
+        pair(`${p}mlp.fc1`, hidden, dim);
+        pair(`${p}mlp.fc2`, dim, hidden);
+        add(`${p}layer_scale2.lambda1`, [dim]);
+    }
+    add('backbone.layernorm.weight', [dim]);
+    add('backbone.layernorm.bias', [dim]);
+    neck.forEach((channels, k) => {
+        const r = `neck.reassemble_stage.layers.${k}.`;
+        conv(`${r}projection`, channels, dim, 1);
+        if (k !== 2)
+            conv(`${r}resize`, channels, channels, k === 0 ? 4 : k === 1 ? 2 : 3);
+        conv(`neck.convs.${k}`, fusion, channels, 3, false);
+    });
+    for (let n = 0; n < 4; n += 1) {
+        const name = `neck.fusion_stage.layers.${n}`;
+        conv(`${name}.projection`, fusion, fusion, 1);
+        for (const unit of ['residual_layer1', 'residual_layer2']) {
+            conv(`${name}.${unit}.convolution1`, fusion, fusion, 3);
+            conv(`${name}.${unit}.convolution2`, fusion, fusion, 3);
+        }
+    }
+    conv('head.conv1', fusion / 2, fusion, 3);
+    conv('head.conv2', headHidden, fusion / 2, 3);
+    conv('head.conv3', 1, headHidden, 1);
+    return out;
+}
+/** Seeded images for `views` views, normalised as the model expects, `[3, height, width]` each. */
+export function miniatureImages(views, height, width, seed) {
+    return Array.from({ length: views }, (_, v) => {
+        const next = mulberry32(nameSeed(`image${v}`, seed));
+        return Float32Array.from({ length: 3 * height * width }, () => (next() * 2 - 1) * 2);
+    });
+}

package/dist/models/mobileSam.d.ts

@@ -0,0 +1,41 @@
+/**
+ * MobileSAM: masks from a point or a box, in two graphs — an encoder run once per image, and a
+ * decoder run once per prompt against the embedding it left.
+ *
+ * **The encoder takes the image at 1024 square**: resized with its longest side to 1024, normalised
+ * by the upstream's pixel mean and deviation, and padded with zeros after its last row and column.
+ * The decoder takes that embedding, a prompt's tokens from `samPromptTokens`, and optionally the
+ * previous mask's logits to refine, and answers four masks at a quarter of the image's size with a
+ * predicted quality each; `samMasksToImage` brings them to the original image as the upstream's
+ * `postprocess_masks` does — up to the encoder's size, cropped to the prepared image, and down to the
+ * original.
+ *
+ * **One decoder graph per prompt size**, since every kernel bakes its shapes: a point and its
+ * padding are two tokens, and a box is two.
+ */
+import type { Architecture } from '@driftengine/texture';
+import { type SamDecoderConfig } from './samDecoder.ts';
+import { type TinyVitConfig } from './tinyVit.ts';
+export interface MobileSamConfig {
+    /** The side the encoder takes an image at. */
+    readonly size: number;
+    readonly encoder: TinyVitConfig;
+    readonly decoder: SamDecoderConfig;
+}
+export declare const MOBILE_SAM: MobileSamConfig;
+/** The pixel mean and deviation the encoder's input is normalised by, in 0–255 RGB. */
+export declare const SAM_PIXEL_MEAN: readonly [123.675, 116.28, 103.53];
+export declare const SAM_PIXEL_STD: readonly [58.395, 57.12, 57.375];
+/** `image`, `[3, size, size]`, to `embedding`, `[neck, grid, grid]`. */
+export declare function mobileSamEncoder(config: MobileSamConfig): Architecture;
+/**
+ * `embedding` and `prompt`, `[tokens, dim]` — absent when `tokens` is zero — and `mask` when
+ * `refine` is set, to `masks`, `[4, 4·grid, 4·grid]`, and `quality`, `[1, 4]`.
+ */
+export declare function mobileSamDecoder(config: MobileSamConfig, tokens: number, refine?: boolean): Architecture;
+/**
+ * `masks`, `[count, 4·grid, 4·grid]`, to `image`, `[count, height, width]`: up to the encoder's
+ * size, cropped to the image as it was prepared, and down to the original — bilinearly, without
+ * aligned corners, as the upstream's `postprocess_masks` does both. Reads no weights.
+ */
+export declare function samMasksToImage(config: MobileSamConfig, count: number, height: number, width: number): Architecture;

package/dist/models/mobileSam.js

@@ -0,0 +1,88 @@
+import { samDecoder } from './samDecoder.js';
+import { samInputSize } from './samPrompt.js';
+import { tinyVit, tinyVitGrid } from './tinyVit.js';
+export const MOBILE_SAM = {
+    size: 1024,
+    encoder: {
+        embedDims: [64, 128, 160, 320],
+        depths: [2, 2, 6, 2],
+        heads: [2, 4, 5, 10],
+        windows: [7, 7, 14, 7],
+        mergeStrides: [2, 2, 1],
+        mlpRatio: 4,
+        expandRatio: 4,
+        neck: 256,
+    },
+    decoder: {
+        dim: 256,
+        heads: 8,
+        mlp: 2048,
+        depth: 2,
+        downsample: 2,
+        masks: 4,
+        iouHidden: 256,
+        iouDepth: 3,
+        maskChannels: 16,
+    },
+};
+/** The pixel mean and deviation the encoder's input is normalised by, in 0–255 RGB. */
+export const SAM_PIXEL_MEAN = [123.675, 116.28, 103.53];
+export const SAM_PIXEL_STD = [58.395, 57.12, 57.375];
+/** `image`, `[3, size, size]`, to `embedding`, `[neck, grid, grid]`. */
+export function mobileSamEncoder(config) {
+    return (weights, graph) => {
+        weights.ignore('prompt_encoder.');
+        weights.ignore('mask_decoder.');
+        tinyVit(weights, graph, config.encoder, 'image_encoder.', 'image', config.size, 'embedding');
+        return {
+            inputs: [{ name: 'image', shape: [3, config.size, config.size] }],
+            outputs: ['embedding'],
+        };
+    };
+}
+/**
+ * `embedding` and `prompt`, `[tokens, dim]` — absent when `tokens` is zero — and `mask` when
+ * `refine` is set, to `masks`, `[4, 4·grid, 4·grid]`, and `quality`, `[1, 4]`.
+ */
+export function mobileSamDecoder(config, tokens, refine = false) {
+    return (weights, graph) => {
+        weights.ignore('image_encoder.');
+        const { dim } = config.decoder;
+        const grid = tinyVitGrid(config.encoder, config.size);
+        const inputs = [
+            { name: 'embedding', shape: [config.encoder.neck, grid, grid] },
+            ...(tokens > 0 ? [{ name: 'prompt', shape: [tokens, dim] }] : []),
+            ...(refine ? [{ name: 'mask', shape: [1, 4 * grid, 4 * grid] }] : []),
+        ];
+        samDecoder(weights, graph, config.decoder, 'embedding', tokens > 0 ? 'prompt' : null, refine ? 'mask' : null, grid, { masks: 'masks', quality: 'quality' });
+        return { inputs, outputs: ['masks', 'quality'] };
+    };
+}
+/**
+ * `masks`, `[count, 4·grid, 4·grid]`, to `image`, `[count, height, width]`: up to the encoder's
+ * size, cropped to the image as it was prepared, and down to the original — bilinearly, without
+ * aligned corners, as the upstream's `postprocess_masks` does both. Reads no weights.
+ */
+export function samMasksToImage(config, count, height, width) {
+    return (_weights, graph) => {
+        const grid = tinyVitGrid(config.encoder, config.size);
+        const [inputHeight, inputWidth] = samInputSize(height, width, config.size);
+        const bilinear = { mode: 'bilinear', alignCorners: false };
+        let x = graph.node('resize', ['masks'], {
+            height: config.size,
+            width: config.size,
+            ...bilinear,
+        });
+        if (inputHeight < config.size) {
+            x = graph.node('slice', [x], { axis: 1, start: 0, end: inputHeight });
+        }
+        if (inputWidth < config.size) {
+            x = graph.node('slice', [x], { axis: 2, start: 0, end: inputWidth });
+        }
+        graph.node('resize', [x], { height, width, ...bilinear }, 'image');
+        return {
+            inputs: [{ name: 'masks', shape: [count, 4 * grid, 4 * grid] }],
+            outputs: ['image'],
+        };
+    };
+}

package/dist/models/owlv2.d.ts

@@ -0,0 +1,49 @@
+/**
+ * OWLv2: boxes for things named in words, in two graphs — the image's, run once a frame, and the
+ * text's, run once a set of queries — which the host joins (`owlv2Detect.ts`).
+ *
+ * **Both are CLIP's transformer**: pre-norm layers of attention and a perceptron whose activation is
+ * `x·σ(1.702·x)`, every norm at PyTorch's ε. The image is cut into patches of 16, a class token put
+ * first, and after the last layer each patch is multiplied by the class token and normed again;
+ * from that come a class embedding, a logit shift and scale, a box and an objectness, each a patch.
+ * A box is the upstream's: a perceptron's four numbers plus the patch's own prior — the logit of its
+ * corner and of the grid's cell size — through the logistic, a centre and a size in the image.
+ *
+ * **The text reads its queries causally** — `clipTower.ts` holds that mask — and each query is
+ * read at the row of its end-of-text token, which is what the upstream pools.
+ *
+ * **The tokenizer travels in the text graph**, as `tokenizer.merges`, which no node reads.
+ */
+import type { Architecture } from '@driftengine/texture';
+import { type ClipTowerConfig } from './clipTower.ts';
+export interface Owlv2Config {
+    /** The side the image graph takes an image at. */
+    readonly size: number;
+    readonly patch: number;
+    readonly vision: ClipTowerConfig;
+    /**
+     * The text tower, and how many tokens a query may be. Its width is also the class embedding's and
+     * the projected query's, which the upstream's class head and projection each take from it.
+     */
+    readonly text: ClipTowerConfig & {
+        readonly positions: number;
+    };
+}
+export declare const OWLV2_BASE: Owlv2Config;
+/** The pixel mean and deviation an image is normalised by, CLIP's, in 0–1 RGB. */
+export declare const OWLV2_PIXEL_MEAN: readonly [0.48145466, 0.4578275, 0.40821073];
+export declare const OWLV2_PIXEL_STD: readonly [0.26862954, 0.26130258, 0.27577711];
+/** `[x, y, w, h]` for every cell, row by row: the logits of its far corner and of the cell's size. */
+export declare function owlv2BoxBias(grid: number): Float32Array;
+/**
+ * `image`, `[3, size, size]`, to five values a patch: `classes`, `[cells, text.dim]`, the class
+ * embedding before it is normalised; `shift` and `scale`, `[cells, 1]`, the logit's shift and its
+ * scale before the ELU; `boxes`, `[cells, 4]`, centre and size in the image's fractions; and
+ * `objectness`, `[cells, 1]`, a logit.
+ */
+export declare function owlv2Image(config: Owlv2Config): Architecture;
+/**
+ * `tokens`, `[queries · positions]`, and `ends`, `[queries]`, as `owlv2Tokens` makes them, to
+ * `queries`, `[queries, text.dim]`: each query's embedding before it is normalised.
+ */
+export declare function owlv2Text(config: Owlv2Config, queries: number): Architecture;

package/dist/models/owlv2.js

@@ -0,0 +1,131 @@
+import { exactLog } from '@driftengine/core';
+import { clipLayers } from './clipTower.js';
+export const OWLV2_BASE = {
+    size: 960,
+    patch: 16,
+    vision: { dim: 768, heads: 12, depth: 12, hidden: 3072 },
+    text: { dim: 512, heads: 8, depth: 12, hidden: 2048, positions: 16 },
+};
+/** The pixel mean and deviation an image is normalised by, CLIP's, in 0–1 RGB. */
+export const OWLV2_PIXEL_MEAN = [0.48145466, 0.4578275, 0.40821073];
+export const OWLV2_PIXEL_STD = [0.26862954, 0.26130258, 0.27577711];
+const EPSILON = 1e-5;
+const f = Math.fround;
+/** `[x, y, w, h]` for every cell, row by row: the logits of its far corner and of the cell's size. */
+export function owlv2BoxBias(grid) {
+    /* The upstream's order of roundings: log(c + 1e-4) − log1p(−c + 1e-4), every step single. */
+    const offset = f(1e-4);
+    const logit = (c) => f(f(exactLog(f(c + offset))) - f(exactLog(1 + f(-c + offset))));
+    const size = logit(f(1 / grid));
+    const out = new Float32Array(grid * grid * 4);
+    for (let row = 0; row < grid; row += 1) {
+        for (let col = 0; col < grid; col += 1) {
+            const at = (row * grid + col) * 4;
+            out[at] = logit(f((col + 1) / grid));
+            out[at + 1] = logit(f((row + 1) / grid));
+            out[at + 2] = size;
+            out[at + 3] = size;
+        }
+    }
+    return out;
+}
+const TEXT = 'owlv2.text_model.';
+const VISION = 'owlv2.vision_model.';
+/**
+ * `image`, `[3, size, size]`, to five values a patch: `classes`, `[cells, text.dim]`, the class
+ * embedding before it is normalised; `shift` and `scale`, `[cells, 1]`, the logit's shift and its
+ * scale before the ELU; `boxes`, `[cells, 4]`, centre and size in the image's fractions; and
+ * `objectness`, `[cells, 1]`, a logit.
+ */
+export function owlv2Image(config) {
+    return (weights, graph) => {
+        for (const aside of [TEXT, 'owlv2.text_projection.weight', 'owlv2.visual_projection.weight']) {
+            weights.ignore(aside);
+        }
+        weights.ignore('owlv2.logit_scale');
+        weights.ignore('tokenizer.merges');
+        const node = graph.node.bind(graph);
+        const { dim } = config.vision;
+        const grid = config.size / config.patch;
+        const cells = grid * grid;
+        const read = weights.read.bind(weights);
+        const norm = (value, p) => node('layerNorm', [value, read(`${p}.weight`, [dim]), read(`${p}.bias`, [dim])], {
+            epsilon: EPSILON,
+        });
+        const linear = (value, p, out, output) => node('linear', [value, read(`${p}.weight`, [out, dim]), read(`${p}.bias`, [out])], {}, output);
+        const patches = node('patchEmbed', [
+            'image',
+            read(`${VISION}embeddings.patch_embedding.weight`, [dim, 3, config.patch, config.patch]),
+        ], { patch: config.patch });
+        const cls = node('reshape', [read(`${VISION}embeddings.class_embedding`, [dim])], {
+            shape: [1, dim],
+        });
+        let x = node('add', [
+            node('concat', [cls, patches], { axis: 0 }),
+            read(`${VISION}embeddings.position_embedding.weight`, [cells + 1, dim]),
+        ]);
+        x = norm(x, `${VISION}pre_layernorm`);
+        x = clipLayers(weights, graph, VISION, config.vision, x, 1, cells + 1, false);
+        x = norm(x, `${VISION}post_layernorm`);
+        const token = node('reshape', [node('slice', [x], { axis: 0, start: 0, end: 1 })], {
+            shape: [dim],
+        });
+        const features = norm(node('mul', [node('slice', [x], { axis: 0, start: 1, end: cells + 1 }), token]), 'layer_norm');
+        linear(features, 'class_head.dense0', config.text.dim, 'classes');
+        linear(features, 'class_head.logit_shift', 1, 'shift');
+        linear(features, 'class_head.logit_scale', 1, 'scale');
+        const head = (p, out, output) => {
+            const inner = node('gelu', [
+                linear(node('gelu', [linear(features, `${p}.dense0`, dim)]), `${p}.dense1`, dim),
+            ]);
+            return linear(inner, `${p}.dense2`, out, output);
+        };
+        const bias = weights.constant(`box_bias.${grid}`, [cells, 4], owlv2BoxBias(grid));
+        graph.node('sigmoid', [node('add', [head('box_head', 4), bias])], {}, 'boxes');
+        head('objectness_head', 1, 'objectness');
+        return {
+            inputs: [{ name: 'image', shape: [3, config.size, config.size] }],
+            outputs: ['classes', 'shift', 'scale', 'boxes', 'objectness'],
+        };
+    };
+}
+/**
+ * `tokens`, `[queries · positions]`, and `ends`, `[queries]`, as `owlv2Tokens` makes them, to
+ * `queries`, `[queries, text.dim]`: each query's embedding before it is normalised.
+ */
+export function owlv2Text(config, queries) {
+    return (weights, graph) => {
+        for (const aside of [VISION, 'owlv2.visual_projection.weight', 'owlv2.logit_scale']) {
+            weights.ignore(aside);
+        }
+        for (const aside of ['class_head.', 'box_head.', 'objectness_head.', 'layer_norm.']) {
+            weights.ignore(aside);
+        }
+        weights.read('tokenizer.merges');
+        const node = graph.node.bind(graph);
+        const { dim, positions } = config.text;
+        const read = weights.read.bind(weights);
+        const embedded = node('gather', [read(`${TEXT}embeddings.token_embedding.weight`), 'tokens']);
+        const placed = node('add', [
+            node('reshape', [embedded], { shape: [queries, positions * dim] }),
+            node('reshape', [read(`${TEXT}embeddings.position_embedding.weight`, [positions, dim])], {
+                shape: [positions * dim],
+            }),
+        ]);
+        let x = node('reshape', [placed], { shape: [queries * positions, dim] });
+        x = clipLayers(weights, graph, TEXT, config.text, x, queries, positions, true);
+        x = node('layerNorm', [
+            node('gather', [x, 'ends']),
+            read(`${TEXT}final_layer_norm.weight`, [dim]),
+            read(`${TEXT}final_layer_norm.bias`, [dim]),
+        ], { epsilon: EPSILON });
+        graph.node('linear', [x, read('owlv2.text_projection.weight', [dim, dim])], {}, 'queries');
+        return {
+            inputs: [
+                { name: 'tokens', shape: [queries * positions] },
+                { name: 'ends', shape: [queries] },
+            ],
+            outputs: ['queries'],
+        };
+    };
+}

package/dist/models/owlv2Detect.d.ts

@@ -0,0 +1,25 @@
+export interface Owlv2Patches {
+    /** `[cells, projection]`. */
+    readonly classes: Float32Array;
+    /** `[cells]`. */
+    readonly shift: Float32Array;
+    /** `[cells]`, before the ELU. */
+    readonly scale: Float32Array;
+}
+export interface Owlv2Detection {
+    /** The query it answers. */
+    readonly label: number;
+    readonly score: number;
+    /** Left, top, right and bottom, in the original image's pixels. */
+    readonly box: readonly [number, number, number, number];
+}
+/**
+ * `out`, `[cells, count]`: every patch's logit for every one of `count` queries, `[count, width]`
+ * as the text graph answers them.
+ */
+export declare function owlv2Logits(patches: Owlv2Patches, queries: Float32Array, count: number, out: Float32Array): void;
+/**
+ * The detections above `threshold` among `logits`, `[cells, count]`, with `boxes`, `[cells, 4]` as
+ * the image graph answers them, for an original image of `height` by `width`.
+ */
+export declare function owlv2Detections(logits: Float32Array, boxes: Float32Array, count: number, threshold: number, height: number, width: number): Owlv2Detection[];

package/dist/models/owlv2Detect.js

@@ -0,0 +1,80 @@
+/**
+ * OWLv2's answer to a set of text queries, on the host: the image graph's patches and the text
+ * graph's queries joined into a logit a patch and query, and those into boxes.
+ *
+ * **The logit is the upstream class head's**: the class embedding and the query each divided by
+ * their length plus 10⁻⁶ — the query already once divided by its length alone, as the upstream's
+ * model normalises it before the head does again — their dot product, plus the patch's shift, times
+ * one plus the ELU of its scale. Every step is rounded to single precision where the upstream's is,
+ * with sums taken in double.
+ *
+ * **A detection is the upstream's post-processing**: each patch's best query, kept when the
+ * logistic of its logit is above the threshold, its box turned from centre and size to corners and
+ * scaled by the image's longer side — the side of the square the image was padded to, since
+ * padding goes after the image's end.
+ */
+import { exactExp } from '@driftengine/core';
+const f = Math.fround;
+const APART = f(1e-6);
+/* Each row of `values` divided by its length (plus `apart`), in place, as the upstream rounds it. */
+function normalise(values, width, apart) {
+    for (let at = 0; at < values.length; at += width) {
+        let sum = 0;
+        for (let i = 0; i < width; i += 1)
+            sum += values[at + i] * values[at + i];
+        const length = f(f(Math.sqrt(sum)) + apart);
+        for (let i = 0; i < width; i += 1)
+            values[at + i] = f(values[at + i] / length);
+    }
+}
+/**
+ * `out`, `[cells, count]`: every patch's logit for every one of `count` queries, `[count, width]`
+ * as the text graph answers them.
+ */
+export function owlv2Logits(patches, queries, count, out) {
+    const width = queries.length / count;
+    const cells = patches.shift.length;
+    const query = Float32Array.from(queries);
+    normalise(query, width, 0);
+    normalise(query, width, APART);
+    const classes = Float32Array.from(patches.classes);
+    normalise(classes, width, APART);
+    for (let cell = 0; cell < cells; cell += 1) {
+        const raw = patches.scale[cell];
+        const scale = f((raw > 0 ? raw : f(exactExp(raw) - 1)) + 1);
+        const shift = patches.shift[cell];
+        for (let q = 0; q < count; q += 1) {
+            let dot = 0;
+            for (let i = 0; i < width; i += 1) {
+                dot += classes[cell * width + i] * query[q * width + i];
+            }
+            out[cell * count + q] = f(f(f(dot) + shift) * scale);
+        }
+    }
+}
+/**
+ * The detections above `threshold` among `logits`, `[cells, count]`, with `boxes`, `[cells, 4]` as
+ * the image graph answers them, for an original image of `height` by `width`.
+ */
+export function owlv2Detections(logits, boxes, count, threshold, height, width) {
+    const side = Math.max(height, width);
+    const found = [];
+    for (let cell = 0; cell < boxes.length / 4; cell += 1) {
+        let label = 0;
+        for (let q = 1; q < count; q += 1) {
+            if (logits[cell * count + q] > logits[cell * count + label])
+                label = q;
+        }
+        const score = f(1 / (1 + exactExp(-logits[cell * count + label])));
+        if (!(score > threshold))
+            continue;
+        const corner = (centre, extent, sign) => f(f(boxes[cell * 4 + centre] +
+            sign * f(0.5 * boxes[cell * 4 + extent])) * side);
+        found.push({
+            label,
+            score,
+            box: [corner(0, 2, -1), corner(1, 3, -1), corner(0, 2, 1), corner(1, 3, 1)],
+        });
+    }
+    return found;
+}

package/dist/models/owlv2Miniature.d.ts

@@ -0,0 +1,27 @@
+/**
+ * A seeded miniature of OWLv2: every tensor name of the upstream's detector, at a size the CPU
+ * evaluates in a moment — what the model's tests and its hand-run oracle share.
+ *
+ * **Small where the upstream lets it be**: a 64-pixel image in patches of 16, so a grid of four;
+ * towers two layers deep of 32 and 24 channels in two heads. **The text keeps
+ * the upstream's 16 positions**, since a query's length is what its causal mask and its pooling are
+ * about, and the vocabulary is three merges' worth — 517 tokens — which is also the tokenizer's
+ * hand-worked table (`clipTokenizer.test.ts`).
+ */
+import type { GraphTensor } from '@driftengine/texture';
+import type { Owlv2Config } from './owlv2.ts';
+export declare const MINIATURE_OWLV2: Owlv2Config;
+/** c+a, ca+t</w> and a+t</w>: tokens 512 to 514, and the start and end of text 515 and 516. */
+export declare const MINIATURE_OWLV2_MERGES: Float32Array<ArrayBuffer>;
+/**
+ * Three queries as token ids: one ending early, one with the padding token inside it — the "!" a
+ * text may hold — and one filling every position.
+ */
+export declare const MINIATURE_OWLV2_QUERIES: readonly (readonly number[])[];
+/** The original image's size the boxes are scaled to: wider than tall, padded below. */
+export declare const MINIATURE_OWLV2_IMAGE: {
+    readonly height: 36;
+    readonly width: 48;
+};
+/** Every tensor the upstream's `Owlv2ForObjectDetection` holds, named as its checkpoints are. */
+export declare function miniatureOwlv2Checkpoint(config: Owlv2Config, seed: number): Map<string, GraphTensor>;