@driftengine/capture 4.0.0

package/dist/models/sam2Positions.d.ts

@@ -0,0 +1,24 @@
+/**
+ * `[2·features, height, width]`: the upstream's normalised sine embedding — a row's and a column's
+ * index from one, over the last and times 2π, each divided by `features` frequencies taken in pairs,
+ * sine then cosine alternating — with the rows' `features` channels before the columns'.
+ */
+export declare function sam2SinePositions(features: number, height: number, width: number): Float32Array;
+/**
+ * `[positions.length, dim]`: the upstream's `get_1d_sine_pe` — each position divided by `dim / 2`
+ * frequencies taken in pairs, its sines then its cosines.
+ */
+export declare function sam2PointerPositions(positions: readonly number[], dim: number): Float32Array;
+/**
+ * The memory attention's axial rotary tables over a `width` by `height` grid, row by row, for a
+ * head of `dim`, at the upstream's base of 10,000 — its configuration's `rope_theta`, the one value
+ * any checkpoint here uses: `[cells, dim]` cosines and sines, the first half of the channels turning by the
+ * column and the second by the row, each frequency on a pair of channels. **The sine carries the
+ * rotation's sign** — negative on a pair's first channel, positive on its second — so a rotation is
+ * `x·cos + swap(x)·sin`, `swap` exchanging each pair, which a graph composes from its shape
+ * operators.
+ */
+export declare function sam2Rotary(dim: number, width: number, height: number): {
+    readonly cos: Float32Array;
+    readonly sin: Float32Array;
+};

package/dist/models/sam2Positions.js

@@ -0,0 +1,94 @@
+/**
+ * SAM 2's position tables, which depend only on shapes: the feature pyramid's and the memory's
+ * two-dimensional sines, the object pointers' one-dimensional sines, and the memory attention's
+ * rotary tables. Each is laid out once per size and used as a constant of a graph or by the host.
+ *
+ * **Every rounding is the upstream's, in single precision**: a coordinate normalised to the grid's
+ * extent, divided by its frequency, and only then taken through `sin` or `cos` — the engine's
+ * reproducible ones, since a capture reproduces. A frequency is `10000^t`, taken as `exp(t·ln 10000)`
+ * rather than `**`, which is not the same bits on every engine.
+ */
+import { exactCos, exactExp, exactSin } from '@driftengine/core';
+const f = Math.fround;
+const TWO_PI = f(2 * Math.PI);
+/** `ln 10000`, the temperature's logarithm. */
+const LOG_TEMPERATURE = 9.210340371976184;
+/* `10000^(2·⌊i/2⌋/count)` in single precision, as the upstream's `temperature ** (...)` makes it. */
+function frequency(i, count) {
+    return f(exactExp(f((2 * Math.floor(i / 2)) / count) * LOG_TEMPERATURE));
+}
+/**
+ * `[2·features, height, width]`: the upstream's normalised sine embedding — a row's and a column's
+ * index from one, over the last and times 2π, each divided by `features` frequencies taken in pairs,
+ * sine then cosine alternating — with the rows' `features` channels before the columns'.
+ */
+export function sam2SinePositions(features, height, width) {
+    const out = new Float32Array(2 * features * height * width);
+    const plane = height * width;
+    const embed = (index, count) => f(f((index + 1) / f(count + 1e-6)) * TWO_PI);
+    for (let i = 0; i < features; i += 1) {
+        const divisor = frequency(i, features);
+        const wave = i % 2 === 0 ? exactSin : exactCos;
+        for (let y = 0; y < height; y += 1) {
+            const row = wave(f(embed(y, height) / divisor));
+            for (let x = 0; x < width; x += 1) {
+                out[i * plane + y * width + x] = row;
+                out[(features + i) * plane + y * width + x] = wave(f(embed(x, width) / divisor));
+            }
+        }
+    }
+    return out;
+}
+/**
+ * `[positions.length, dim]`: the upstream's `get_1d_sine_pe` — each position divided by `dim / 2`
+ * frequencies taken in pairs, its sines then its cosines.
+ */
+export function sam2PointerPositions(positions, dim) {
+    const half = dim / 2;
+    const out = new Float32Array(positions.length * dim);
+    for (let p = 0; p < positions.length; p += 1) {
+        for (let k = 0; k < half; k += 1) {
+            const angle = f(f(positions[p]) / frequency(k, half));
+            out[p * dim + k] = exactSin(angle);
+            out[p * dim + half + k] = exactCos(angle);
+        }
+    }
+    return out;
+}
+/**
+ * The memory attention's axial rotary tables over a `width` by `height` grid, row by row, for a
+ * head of `dim`, at the upstream's base of 10,000 — its configuration's `rope_theta`, the one value
+ * any checkpoint here uses: `[cells, dim]` cosines and sines, the first half of the channels turning by the
+ * column and the second by the row, each frequency on a pair of channels. **The sine carries the
+ * rotation's sign** — negative on a pair's first channel, positive on its second — so a rotation is
+ * `x·cos + swap(x)·sin`, `swap` exchanging each pair, which a graph composes from its shape
+ * operators.
+ */
+export function sam2Rotary(dim, width, height) {
+    const spatial = dim / 2;
+    const count = spatial / 2;
+    const inverse = Float32Array.from({ length: count }, (_, k) => f(1 / f(exactExp(f((2 * k) / spatial) * LOG_TEMPERATURE))));
+    const cells = width * height;
+    const cos = new Float32Array(cells * dim);
+    const sin = new Float32Array(cells * dim);
+    for (let y = 0; y < height; y += 1) {
+        for (let x = 0; x < width; x += 1) {
+            const at = (y * width + x) * dim;
+            for (let k = 0; k < count; k += 1) {
+                for (const [offset, position] of [
+                    [0, x],
+                    [spatial, y],
+                ]) {
+                    const angle = f(position * inverse[k]);
+                    const c = exactCos(angle);
+                    const s = exactSin(angle);
+                    cos[at + offset + 2 * k] = c;
+                    cos[at + offset + 2 * k + 1] = c;
+                    sin[at + offset + 2 * k] = -s;
+                    sin[at + offset + 2 * k + 1] = s;
+                }
+            }
+        }
+    }
+    return { cos, sin };
+}

package/dist/models/sam2Tracker.d.ts

@@ -0,0 +1,69 @@
+/**
+ * SAM 2.1 tracking objects through a video: the upstream's `Sam2VideoModel` loop on the host, its
+ * networks as the engine's graphs, run by whatever the caller gives — the device's runner, or the
+ * CPU's evaluator.
+ *
+ * **A frame is decoded against what the tracker remembers.** A prompted frame, the first time, reads
+ * nothing and is given `no_memory_embedding`; every other frame's features are first conditioned by
+ * the memory attention on the frames and pointers `sam2Memories.ts` chooses, and `sam2Choice.ts`
+ * keeps one of the four masks decoded.
+ *
+ * **Then each object's frame becomes a memory**, all of a frame's objects together: the mask
+ * brought to the frame's size, binarised when any of them was just prompted and taken through a
+ * sigmoid otherwise — the upstream decides it once for the batch — encoded with the frame's
+ * features, the occlusion embedding added where the object is absent, and rounded to bfloat16.
+ *
+ * What it gives up: a mask as a prompt, which the upstream downsamples with an antialiased resize
+ * the runtime does not yet have, and more prompted frames than all of them — the upstream's default
+ * — are the two options not taken.
+ */
+import type { WeightSource } from '@driftengine/texture';
+import type { GraphRun } from '../run.ts';
+import { type Sam21Config } from './sam21.ts';
+import { type SamPrompt } from './samPrompt.ts';
+export type { GraphRun } from '../run.ts';
+/** The weights each graph reads: a converted file's graphs, or a checkpoint for all four. */
+export interface Sam21Weights {
+    readonly encoder: WeightSource;
+    readonly decoder: WeightSource;
+    readonly memoryEncoder: WeightSource;
+    readonly memoryAttention: WeightSource;
+}
+export interface Sam21Video {
+    readonly frames: number;
+    /** The original frame's size, which prompts are given in. */
+    readonly height: number;
+    readonly width: number;
+    /** A frame prepared as the encoder takes it, `[3, size, size]`. */
+    frame(index: number): Float32Array;
+}
+/** A frame's result: every object's mask logits at a quarter of the encoder's size, and its score. */
+export interface Sam21Frame {
+    readonly frame: number;
+    readonly objects: readonly number[];
+    /** `[objects, (size / 4)²]`. */
+    readonly masks: Float32Array;
+    readonly scores: Float32Array;
+}
+export declare class Sam21Tracker {
+    private readonly config;
+    private readonly weights;
+    private readonly run;
+    private readonly video;
+    private readonly objects;
+    private fresh;
+    private readonly graphs;
+    private encoded;
+    private readonly cellPositions;
+    constructor(config: Sam21Config, weights: Sam21Weights, run: GraphRun, video: Sam21Video);
+    /** Prompts on `frame`, replacing each object's earlier ones there; the next step decodes them. */
+    prompt(frame: number, prompts: ReadonlyMap<number, SamPrompt>): void;
+    /** Every object on `frame`, as the upstream's `forward` over its session answers it. */
+    step(frame: number, reverse?: boolean): Promise<Sam21Frame>;
+    private graph;
+    private features;
+    private decode;
+    private condition;
+    private remember;
+    private tensor;
+}

package/dist/models/sam2Tracker.js

@@ -0,0 +1,172 @@
+import { graphFromWeights } from '@driftengine/texture';
+import { memoriesFor, memoryTokens, pointersFor, toBfloat16, } from './sam2Memories.js';
+import { chooseMask } from './sam2Choice.js';
+import { sam2SinePositions } from './sam2Positions.js';
+import { sam21Decoder, sam21Encoder } from './sam21.js';
+import { sam21MemoryAttention, sam21MemoryEncoder, sam21Upscale } from './sam21Video.js';
+import { SAM2_VIDEO_PROMPT, samPromptTokens, samTokenCount } from './samPrompt.js';
+const NO_WEIGHTS = { get: () => undefined, names: () => [] };
+export class Sam21Tracker {
+    config;
+    weights;
+    run;
+    video;
+    objects = new Map();
+    fresh = [];
+    graphs = new Map();
+    encoded = null;
+    cellPositions;
+    constructor(config, weights, run, video) {
+        this.config = config;
+        this.weights = weights;
+        this.run = run;
+        this.video = video;
+        const grid = config.size / 16;
+        const width = config.memory.dim;
+        const sine = sam2SinePositions(width / 2, grid, grid);
+        this.cellPositions = new Float32Array(grid * grid * width);
+        for (let c = 0; c < width; c += 1) {
+            for (let i = 0; i < grid * grid; i += 1)
+                this.cellPositions[i * width + c] = sine[c * grid * grid + i];
+        }
+    }
+    /** Prompts on `frame`, replacing each object's earlier ones there; the next step decodes them. */
+    prompt(frame, prompts) {
+        for (const [object, prompt] of prompts) {
+            let state = this.objects.get(object);
+            if (state === undefined) {
+                state = { prompts: new Map(), prompted: new Map(), tracked: new Map(), seen: new Set() };
+                this.objects.set(object, state);
+            }
+            state.prompts.set(frame, prompt);
+        }
+        this.fresh = [...prompts.keys()];
+    }
+    /** Every object on `frame`, as the upstream's `forward` over its session answers it. */
+    async step(frame, reverse = false) {
+        const objects = [...this.objects.keys()];
+        const side = this.config.size / 4;
+        const cells = side * side;
+        const masks = new Float32Array(objects.length * cells);
+        const scores = new Float32Array(objects.length);
+        const pending = [];
+        let backwards = reverse;
+        for (const [i, object] of objects.entries()) {
+            const state = this.objects.get(object);
+            const isNew = this.fresh.includes(object);
+            let output = state.prompted.get(frame);
+            let first = true;
+            if (isNew || output === undefined) {
+                first = false;
+                let prompt = null;
+                if (isNew) {
+                    first = !state.seen.has(frame);
+                    if (first)
+                        backwards = false;
+                    prompt = state.prompts.get(frame) ?? null;
+                    if (prompt !== null)
+                        this.fresh = this.fresh.filter((id) => id !== object);
+                }
+                output = await this.decode(state, frame, first, prompt, backwards);
+                (first ? state.prompted : state.tracked).set(frame, output);
+                pending.push({ output, fromPrompt: prompt !== null });
+            }
+            masks.set(output.lowRes, i * cells);
+            scores[i] = output.score;
+            if (!first)
+                state.seen.add(frame);
+        }
+        await this.remember(frame, pending);
+        return { frame, objects, masks, scores };
+    }
+    graph(key, source, build) {
+        let graph = this.graphs.get(key);
+        if (graph === undefined) {
+            graph = graphFromWeights(source, build());
+            this.graphs.set(key, graph);
+        }
+        return graph;
+    }
+    async features(frame) {
+        if (this.encoded?.frame !== frame) {
+            const graph = this.graph('encoder', this.weights.encoder, () => sam21Encoder(this.config));
+            const out = await this.run(graph, new Map([['image', this.video.frame(frame)]]));
+            this.encoded = {
+                frame,
+                values: new Map([...out].map(([name, values]) => [name, Float32Array.from(values)])),
+            };
+        }
+        return this.encoded.values;
+    }
+    async decode(state, frame, first, prompt, reverse) {
+        const { config } = this;
+        const { dim } = config.decoder;
+        const features = await this.features(frame);
+        let embedding = features.get('features');
+        if (!first)
+            embedding = await this.condition(state, frame, embedding, reverse);
+        const request = prompt ?? {};
+        const tokens = new Float32Array(samTokenCount(request, SAM2_VIDEO_PROMPT) * dim);
+        samPromptTokens(this.weights.decoder, dim, config.size, request, this.video.height, this.video.width, tokens, SAM2_VIDEO_PROMPT);
+        const n = tokens.length / dim;
+        const decoder = this.graph(`decoder.${n}.${first}`, this.weights.decoder, () => sam21Decoder(config, n, { noMemory: first }));
+        const out = await this.run(decoder, new Map([
+            ['embedding', embedding],
+            ['high1', features.get('high1')],
+            ['high0', features.get('high0')],
+            ['prompt', tokens],
+        ]));
+        const score = out.get('object')[0];
+        const { lowRes, pointer } = chooseMask(out.get('masks'), out.get('quality'), out.get('pointers'), score, (prompt?.points?.length ?? 0) + (prompt?.box === undefined ? 0 : 2), this.tensor(this.weights.decoder, 'no_object_pointer'));
+        return { lowRes, pointer, score, memory: null };
+    }
+    async condition(state, frame, features, reverse) {
+        const { config } = this;
+        const memories = memoriesFor(state, frame, config.memory.frames, reverse);
+        const pointers = pointersFor(state, frame, this.video.frames, config.memory.pointers, reverse);
+        const { memory, positions, pointerTokens } = memoryTokens(this.weights.memoryAttention, memories, pointers, this.cellPositions, config.memory.dim, config.decoder.dim);
+        const graph = this.graph(`attention.${memories.length}.${pointerTokens}`, this.weights.memoryAttention, () => sam21MemoryAttention(config, memories.length, pointerTokens));
+        const out = await this.run(graph, new Map([
+            ['features', features],
+            ['memory', memory],
+            ['positions', positions],
+        ]));
+        return Float32Array.from(out.get('conditioned'));
+    }
+    async remember(frame, pending) {
+        if (pending.length === 0)
+            return;
+        const { config } = this;
+        const binary = pending.some(({ fromPrompt }) => fromPrompt);
+        const features = (await this.features(frame)).get('features');
+        const upscale = this.graph('upscale', NO_WEIGHTS, () => sam21Upscale(config, 1));
+        const encoder = this.graph(`memory.${binary}`, this.weights.memoryEncoder, () => sam21MemoryEncoder(config, binary));
+        const occlusion = this.tensor(this.weights.memoryEncoder, 'occlusion_spatial_embedding_parameter');
+        const width = config.memory.dim;
+        for (const { output } of pending) {
+            const high = Float32Array.from((await this.run(upscale, new Map([['masks', output.lowRes]]))).get('high'));
+            if (binary)
+                for (let i = 0; i < high.length; i += 1)
+                    high[i] = high[i] > 0 ? 1 : 0;
+            const encoded = (await this.run(encoder, new Map([
+                ['features', features],
+                ['mask', high],
+            ]))).get('memory');
+            const cells = encoded.length / width;
+            const memory = new Float32Array(encoded.length);
+            for (let c = 0; c < width; c += 1) {
+                const add = output.score > 0 ? 0 : occlusion[c];
+                for (let i = 0; i < cells; i += 1)
+                    memory[i * width + c] = Math.fround(encoded[c * cells + i] + add);
+            }
+            toBfloat16(memory);
+            output.memory = memory;
+        }
+    }
+    tensor(source, name) {
+        const held = source.get(name);
+        if (held === undefined)
+            throw new Error(`the tracker's weights have no "${name}"`);
+        return held.data;
+    }
+}

package/dist/models/samDecoder.d.ts

@@ -0,0 +1,45 @@
+/**
+ * SAM's mask decoder: an image embedding and a prompt's tokens in, masks and their predicted
+ * quality out — MobileSAM's, and the original SAM's, which share it.
+ *
+ * **A two-way transformer** (`twoWay.ts`, under the original's names) over the prompt's tokens,
+ * behind a learned token for the quality estimate and one per mask, and the image with its dense
+ * prompt added.
+ *
+ * **Masks come from a product, not a convolution**: the image is upsampled four times by two
+ * transposed convolutions, and each mask token, through an MLP of its own, becomes a vector that
+ * is dotted with every upsampled pixel. The first mask is the one to take for an unambiguous
+ * prompt, and the other three for an ambiguous one, as the upstream's `multimask_output` chooses.
+ *
+ * **Every prompt weight is read, whether this graph's nodes use it or not**: the prompt's tokens
+ * are made on the host from the Fourier weights, and the mask input's downscaling is only in the
+ * graph that takes one — but the converted file is built from one graph, and each has to find
+ * every weight the others need in it.
+ */
+import type { GraphBuilder, Weights } from '@driftengine/texture';
+export interface SamDecoderConfig {
+    readonly dim: number;
+    readonly heads: number;
+    readonly mlp: number;
+    readonly depth: number;
+    /** The width attention between tokens and image narrows by. */
+    readonly downsample: number;
+    /** The mask tokens: one for an unambiguous prompt and three for an ambiguous one. */
+    readonly masks: number;
+    readonly iouHidden: number;
+    readonly iouDepth: number;
+    /** The channels a mask input is downscaled through. */
+    readonly maskChannels: number;
+}
+/** What the decoder's two outputs are named. */
+export interface SamDecoderOutput {
+    /** `[masks, 4·grid, 4·grid]`: each mask's logits at four times the embedding's grid. */
+    readonly masks: string;
+    /** `[1, masks]`: each mask's predicted intersection over union. */
+    readonly quality: string;
+}
+/**
+ * The decoder over `embedding`, `[dim, grid, grid]`, and `promptTokens`, `[tokens, dim]` or null for a
+ * prompt of none, with `mask`, `[1, 4·grid, 4·grid]`, the previous logits to refine, or null.
+ */
+export declare function samDecoder(weights: Weights, graph: GraphBuilder, config: SamDecoderConfig, embedding: string, promptTokens: string | null, mask: string | null, grid: number, names: SamDecoderOutput): void;

package/dist/models/samDecoder.js

@@ -0,0 +1,96 @@
+import { samGridPositions } from './samPrompt.js';
+import { SAM_NAMES, twoWay } from './twoWay.js';
+const CHANNEL_EPSILON = 1e-6;
+/**
+ * The decoder over `embedding`, `[dim, grid, grid]`, and `promptTokens`, `[tokens, dim]` or null for a
+ * prompt of none, with `mask`, `[1, 4·grid, 4·grid]`, the previous logits to refine, or null.
+ */
+export function samDecoder(weights, graph, config, embedding, promptTokens, mask, grid, names) {
+    const node = graph.node.bind(graph);
+    const { dim, masks } = config;
+    const cells = grid * grid;
+    const d = 'mask_decoder.';
+    const t = `${d}transformer.`;
+    const pe = 'prompt_encoder.';
+    const read = weights.read.bind(weights);
+    const linear = (x, p, out, inputs, name) => node('linear', [x, read(`${p}weight`, [out, inputs]), read(`${p}bias`, [out])], {}, name);
+    const tokensOf = (map, channels, count) => node('permute', [node('reshape', [map], { shape: [channels, count] })], { order: [1, 0] });
+    const channelNorm = (map, p, channels, rows) => node('reshape', [
+        node('permute', [
+            node('layerNorm', [
+                tokensOf(map, channels, rows * rows),
+                read(`${p}weight`, [channels]),
+                read(`${p}bias`, [channels]),
+            ], { epsilon: CHANNEL_EPSILON }),
+        ], { order: [1, 0] }),
+    ], { shape: [channels, rows, rows] });
+    const perceptron = (x, p, sizes, name) => {
+        let y = x;
+        for (let i = 0; i + 1 < sizes.length; i += 1) {
+            const last = i + 2 === sizes.length;
+            y = linear(y, `${p}layers.${i}.`, sizes[i + 1], sizes[i], last ? name : undefined);
+            if (i + 2 < sizes.length)
+                y = node('relu', [y]);
+        }
+        return y;
+    };
+    /* The prompt encoder's weights, all of them: see the header. */
+    read(`${pe}pe_layer.positional_encoding_gaussian_matrix`, [2, dim / 2]);
+    for (let i = 0; i < 4; i += 1)
+        read(`${pe}point_embeddings.${i}.weight`, [1, dim]);
+    read(`${pe}not_a_point_embed.weight`, [1, dim]);
+    const c = config.maskChannels;
+    const downscaled = (input) => {
+        const p = `${pe}mask_downscaling.`;
+        let x = node('conv2d', [input, read(`${p}0.weight`, [c / 4, 1, 2, 2]), read(`${p}0.bias`, [c / 4])], { stride: 2 });
+        x = node('gelu', [channelNorm(x, `${p}1.`, c / 4, 2 * grid)]);
+        x = node('conv2d', [x, read(`${p}3.weight`, [c, c / 4, 2, 2]), read(`${p}3.bias`, [c])], {
+            stride: 2,
+        });
+        x = node('gelu', [channelNorm(x, `${p}4.`, c, grid)]);
+        x = node('conv2d', [x, read(`${p}6.weight`, [dim, c, 1, 1]), read(`${p}6.bias`, [dim])]);
+        return tokensOf(x, dim, cells);
+    };
+    const noMask = read(`${pe}no_mask_embed.weight`, [1, dim]);
+    if (mask === null)
+        downscaledWeights(read, `${pe}mask_downscaling.`, dim, c);
+    const dense = mask === null ? node('reshape', [noMask], { shape: [dim] }) : downscaled(mask);
+    const positions = weights.derive(`${pe}grid_positions.${grid}x${grid}`, [cells, dim], (values) => samGridPositions(values(`${pe}pe_layer.positional_encoding_gaussian_matrix`, [2, dim / 2]), dim, grid));
+    const tokens = node('concat', [
+        read(`${d}iou_token.weight`, [1, dim]),
+        read(`${d}mask_tokens.weight`, [masks, dim]),
+        ...(promptTokens === null ? [] : [promptTokens]),
+    ], { axis: 0 });
+    const { queries, keys } = twoWay(weights, graph, t, config, SAM_NAMES, tokens, node('add', [tokensOf(embedding, dim, cells), dense]), positions);
+    /* Upsampled four times, and each mask token's vector dotted with every pixel. */
+    const u = `${d}output_upscaling.`;
+    const image = node('reshape', [node('permute', [keys], { order: [1, 0] })], {
+        shape: [dim, grid, grid],
+    });
+    let up = node('convTranspose2d', [image, read(`${u}0.weight`, [dim, dim / 4, 2, 2]), read(`${u}0.bias`, [dim / 4])], { stride: 2 });
+    up = node('gelu', [channelNorm(up, `${u}1.`, dim / 4, 2 * grid)]);
+    up = node('gelu', [
+        node('convTranspose2d', [up, read(`${u}3.weight`, [dim / 4, dim / 8, 2, 2]), read(`${u}3.bias`, [dim / 8])], { stride: 2 }),
+    ]);
+    const vectors = Array.from({ length: masks }, (_, i) => perceptron(node('slice', [queries], { axis: 0, start: 1 + i, end: 2 + i }), `${d}output_hypernetworks_mlps.${i}.`, [dim, dim, dim, dim / 8]));
+    const pixels = 16 * cells;
+    const products = node('linear', [
+        tokensOf(up, dim / 8, pixels),
+        node('concat', vectors, { axis: 0 }),
+    ]);
+    node('reshape', [node('permute', [products], { order: [1, 0] })], { shape: [masks, 4 * grid, 4 * grid] }, names.masks);
+    perceptron(node('slice', [queries], { axis: 0, start: 0, end: 1 }), `${d}iou_prediction_head.`, [dim, ...Array.from({ length: config.iouDepth - 1 }, () => config.iouHidden), masks], names.quality);
+}
+/* The mask input's downscaling weights, read into a graph that takes no mask input. */
+function downscaledWeights(read, p, dim, c) {
+    read(`${p}0.weight`, [c / 4, 1, 2, 2]);
+    read(`${p}0.bias`, [c / 4]);
+    read(`${p}1.weight`, [c / 4]);
+    read(`${p}1.bias`, [c / 4]);
+    read(`${p}3.weight`, [c, c / 4, 2, 2]);
+    read(`${p}3.bias`, [c]);
+    read(`${p}4.weight`, [c]);
+    read(`${p}4.bias`, [c]);
+    read(`${p}6.weight`, [dim, c, 1, 1]);
+    read(`${p}6.bias`, [dim]);
+}

package/dist/models/samMiniature.d.ts

@@ -0,0 +1,43 @@
+/**
+ * A seeded miniature of MobileSAM: the upstream's layout, every tensor name and shape of it, at a
+ * size the CPU evaluates in a moment — what the model's tests and its hand-run oracle share, as the
+ * depth models' miniature is.
+ *
+ * **Small everywhere the upstream lets it be.** A 64-pixel image, four stages of 8, 16, 24 and 320
+ * channels — the last must be 320, since the upstream keeps its last merge's stride at one by that
+ * width alone — windows of 3, 4 and 2 over grids of 8, 4 and 4, so one stage pads its windows, one
+ * is attended whole and one divides evenly, and one or two blocks a stage. The neck's 256 channels
+ * are the upstream's own constant, so the decoder is 256 wide too; its MLP and quality head are
+ * narrow, and its grid is 4.
+ *
+ * **Every tensor the upstream holds is here**, the classification head the definition sets aside
+ * and the batch norms' counters included, so the oracle loads it strictly.
+ */
+import type { GraphTensor } from '@driftengine/texture';
+import type { MobileSamConfig } from './mobileSam.ts';
+export declare const MINIATURE_MOBILE_SAM: MobileSamConfig;
+/** The classes of the miniature's classification head, which nothing runs. */
+export declare const MINIATURE_SAM_CLASSES = 10;
+/**
+ * The original image the prompts are given in: 36 by 48, prepared at 48 by 64 in the encoder's 64
+ * square, so the masks are cropped before they come down to it and coordinates are scaled by 4/3.
+ */
+export declare const MINIATURE_SAM_IMAGE: {
+    readonly height: 36;
+    readonly width: 48;
+};
+/** A point, which a padding point follows; a box; and two points with a box, which none follows. */
+export declare const MINIATURE_SAM_PROMPTS: {
+    readonly point: {
+        readonly points: readonly [readonly [20.5, 14.25, 1]];
+    };
+    readonly box: {
+        readonly box: readonly [6, 5, 40.5, 30];
+    };
+    readonly both: {
+        readonly points: readonly [readonly [20.5, 14.25, 1], readonly [33, 25.75, 0]];
+        readonly box: readonly [6, 5, 40.5, 30];
+    };
+};
+/** Every tensor the upstream's encoder, prompt encoder and mask decoder hold. */
+export declare function miniatureSamCheckpoint(config: MobileSamConfig, seed: number): Map<string, GraphTensor>;