@driftengine/texture 4.0.0

package/dist/tensor/architecture.d.ts

@@ -0,0 +1,53 @@
+/**
+ * A network's definition, written as a function of its weights, built into a graph the runtime runs.
+ *
+ * **One definition, run over two sources.** At conversion it reads a checkpoint; afterwards it reads
+ * the converted file, whenever the runtime needs the graph at a new size — a vision transformer's
+ * patch grid follows the image's aspect ratio, and every device kernel bakes its shapes, so a model
+ * is one graph per size and not one graph. The two runs agree because of three rules:
+ *
+ * - **A weight is read by name, at the shape the definition expects**, and a name the source lacks
+ *   or a shape it disagrees with is refused naming both.
+ * - **A weight derived from others** — a batch norm folded into its convolution — is computed once,
+ *   at conversion, and kept under its own name; a source that already holds that name answers with
+ *   it and the computation never runs, which is why a converted file needs none of the parts.
+ * - **A constant** — a table computed from the shapes, such as rotary angles — carries a name
+ *   beginning `@`, is rebuilt on every run, and is never taken for a weight.
+ *
+ * **And every weight the source holds is read, derived from, or set aside by name.** A forgotten
+ * bias or layer scale is a network that runs and answers wrongly — the one failure nothing later can
+ * see — so a weight left over is refused. What that costs is an explicit `ignore` for every head a
+ * definition does not run, which is the point.
+ */
+import { type GraphTensor, type GraphValue, type NetworkGraph } from './graph.ts';
+import type { Attributes } from './operators.ts';
+/** Where a definition's weights come from: a checkpoint being converted, or a converted file. */
+export interface WeightSource {
+    get(name: string): GraphTensor | undefined;
+    names(): Iterable<string>;
+}
+/** What a constant's name begins with: a table of the shapes, never a weight. */
+export declare const CONSTANT_PREFIX = "@";
+export interface Weights {
+    /** The weight `name` as a value of the graph; refused if absent or held at another shape. */
+    read(name: string, shape?: readonly number[]): string;
+    /**
+     * A weight computed once from others, kept under `name`. `compute` reads its parts through
+     * `values`, which counts each as read; a source already holding `name` answers with it instead.
+     */
+    derive(name: string, shape: readonly number[], compute: (values: (part: string, shape?: readonly number[]) => Float32Array) => Float32Array): string;
+    /** A table computed from the graph's shapes, rebuilt on every run and never a weight. */
+    constant(name: string, shape: readonly number[], data: Float32Array): string;
+    /** Weights the definition deliberately does not read: a name, or a prefix ending in a dot. */
+    ignore(nameOrPrefix: string): void;
+}
+export interface GraphBuilder {
+    /** A node; its output is named `output` when given, and otherwise by the builder. */
+    node(op: string, inputs: readonly string[], attributes?: Attributes, output?: string): string;
+}
+/** A definition: it reads its weights, adds its nodes, and says what goes in and comes out. */
+export type Architecture = (weights: Weights, graph: GraphBuilder) => {
+    readonly inputs: readonly GraphValue[];
+    readonly outputs: readonly string[];
+};
+export declare function graphFromWeights(source: WeightSource, architecture: Architecture): NetworkGraph;

package/dist/tensor/architecture.js

@@ -0,0 +1,96 @@
+/**
+ * A network's definition, written as a function of its weights, built into a graph the runtime runs.
+ *
+ * **One definition, run over two sources.** At conversion it reads a checkpoint; afterwards it reads
+ * the converted file, whenever the runtime needs the graph at a new size — a vision transformer's
+ * patch grid follows the image's aspect ratio, and every device kernel bakes its shapes, so a model
+ * is one graph per size and not one graph. The two runs agree because of three rules:
+ *
+ * - **A weight is read by name, at the shape the definition expects**, and a name the source lacks
+ *   or a shape it disagrees with is refused naming both.
+ * - **A weight derived from others** — a batch norm folded into its convolution — is computed once,
+ *   at conversion, and kept under its own name; a source that already holds that name answers with
+ *   it and the computation never runs, which is why a converted file needs none of the parts.
+ * - **A constant** — a table computed from the shapes, such as rotary angles — carries a name
+ *   beginning `@`, is rebuilt on every run, and is never taken for a weight.
+ *
+ * **And every weight the source holds is read, derived from, or set aside by name.** A forgotten
+ * bias or layer scale is a network that runs and answers wrongly — the one failure nothing later can
+ * see — so a weight left over is refused. What that costs is an explicit `ignore` for every head a
+ * definition does not run, which is the point.
+ */
+import { validateGraph } from './graph.js';
+/** What a constant's name begins with: a table of the shapes, never a weight. */
+export const CONSTANT_PREFIX = '@';
+const shapeText = (shape) => `[${shape.join(', ')}]`;
+export function graphFromWeights(source, architecture) {
+    const tensors = new Map();
+    const read = new Set();
+    const ignored = [];
+    const nodes = [];
+    const take = (name, shape) => {
+        const tensor = source.get(name);
+        if (tensor === undefined) {
+            throw new Error(`the source has no weight "${name}", which the definition reads`);
+        }
+        if (shape !== undefined && shapeText(shape) !== shapeText(tensor.shape)) {
+            throw new Error(`"${name}" is ${shapeText(tensor.shape)} in the source and the definition reads it as ` +
+                shapeText(shape));
+        }
+        read.add(name);
+        return tensor;
+    };
+    const weights = {
+        read(name, shape) {
+            if (!tensors.has(name))
+                tensors.set(name, take(name, shape));
+            else
+                take(name, shape);
+            return name;
+        },
+        derive(name, shape, compute) {
+            if (tensors.has(name))
+                return name;
+            const held = source.get(name);
+            const data = held === undefined
+                ? compute((part, partShape) => take(part, partShape).data)
+                : take(name, shape).data;
+            if (data.length !== shape.reduce((total, d) => total * d, 1)) {
+                throw new Error(`the derived weight "${name}" has ${data.length} values for ${shapeText(shape)}`);
+            }
+            tensors.set(name, { shape, data });
+            return name;
+        },
+        constant(name, shape, data) {
+            const key = `${CONSTANT_PREFIX}${name}`;
+            tensors.set(key, { shape, data });
+            return key;
+        },
+        ignore(nameOrPrefix) {
+            ignored.push(nameOrPrefix);
+        },
+    };
+    let count = 0;
+    const graph = {
+        node(op, inputs, attributes = {}, output = `v${(count += 1)}`) {
+            nodes.push({ op, inputs, output, attributes });
+            return output;
+        },
+    };
+    const { inputs, outputs } = architecture(weights, graph);
+    const aside = (name) => name.startsWith(CONSTANT_PREFIX) ||
+        ignored.some((rule) => (rule.endsWith('.') ? name.startsWith(rule) : name === rule));
+    const unread = [...source.names()].filter((name) => !read.has(name) && !aside(name));
+    if (unread.length > 0) {
+        const named = unread.slice(0, 5).map((name) => `"${name}"`);
+        throw new Error(`the definition never reads ${named.join(', ')}` +
+            (unread.length > 5 ? ` and ${unread.length - 5} more` : '') +
+            ': a weight left behind is a network that runs and answers wrongly, so read it or set it ' +
+            'aside by name');
+    }
+    const network = { inputs, outputs, nodes, tensors };
+    const problem = validateGraph(network);
+    if (problem !== null)
+        throw new Error(`the definition's graph cannot run: ${problem}`);
+    return network;
+}

package/dist/tensor/attention.d.ts

@@ -0,0 +1,5 @@
+/**
+ * `out[batch][queries][channels]` from `q` of that shape and `k` and `v` of
+ * `[batch][keys][channels]`. `scratch` holds at least `queries × keys` values and is overwritten.
+ */
+export declare function attention(out: Float32Array, q: Float32Array, k: Float32Array, v: Float32Array, queries: number, keys: number, channels: number, heads: number, scratch: Float32Array, bias?: Float32Array | null, batch?: number): void;

package/dist/tensor/attention.js

@@ -0,0 +1,62 @@
+/**
+ * Multi-head attention, as a composition of the dense operators.
+ *
+ * **Heads are contiguous runs of channels, in the upstream order**: head `h` of `heads` owns
+ * channels `h·d` to `(h+1)·d − 1`, where `d` is the channel count over the head count. A port that
+ * interleaves them instead evaluates a network nobody trained.
+ *
+ * The queries, keys and values arrive already projected — the projections are matrix multiplies a
+ * graph states on its own — so this is the part that is attention's alone: scores scaled by
+ * `1/√d`, a stable softmax over each query's row, and the weighted sum of values.
+ *
+ * **A bias is added to the scaled scores**, `[heads][queries][keys]`, as a relative-position table
+ * or a mask is upstream; and **a batch is independent windows**, each attending over its own keys
+ * with the one bias. Queries and keys may differ in number, as they do where one set of tokens
+ * reads another.
+ */
+import { softmax } from './linear.js';
+/**
+ * `out[batch][queries][channels]` from `q` of that shape and `k` and `v` of
+ * `[batch][keys][channels]`. `scratch` holds at least `queries × keys` values and is overwritten.
+ */
+export function attention(out, q, k, v, queries, keys, channels, heads, scratch, bias = null, batch = 1) {
+    const d = channels / heads;
+    if (!Number.isInteger(d)) {
+        throw new RangeError(`attention: ${channels} channels do not split into ${heads} heads`);
+    }
+    const scale = 1 / Math.sqrt(d);
+    for (let b = 0; b < batch; b += 1) {
+        const qAt = b * queries * channels;
+        const kAt = b * keys * channels;
+        for (let head = 0; head < heads; head += 1) {
+            const first = head * d;
+            for (let query = 0; query < queries; query += 1) {
+                for (let key = 0; key < keys; key += 1) {
+                    let dot = 0;
+                    for (let c = 0; c < d; c += 1) {
+                        dot +=
+                            q[qAt + query * channels + first + c] *
+                                k[kAt + key * channels + first + c];
+                    }
+                    const at = query * keys + key;
+                    scratch[at] = dot * scale;
+                    if (bias !== null) {
+                        scratch[at] = scratch[at] + bias[head * queries * keys + at];
+                    }
+                }
+            }
+            softmax(scratch, queries, keys, scratch);
+            for (let query = 0; query < queries; query += 1) {
+                for (let c = 0; c < d; c += 1) {
+                    let sum = 0;
+                    for (let key = 0; key < keys; key += 1) {
+                        sum +=
+                            scratch[query * keys + key] *
+                                v[kAt + key * channels + first + c];
+                    }
+                    out[qAt + query * channels + first + c] = sum;
+                }
+            }
+        }
+    }
+}

package/dist/tensor/denseOperators.d.ts

@@ -0,0 +1,2 @@
+import { type Operator } from './operatorKit.ts';
+export declare const DENSE_OPERATORS: readonly (readonly [string, Operator])[];

package/dist/tensor/denseOperators.js

@@ -0,0 +1,136 @@
+/** The dense operators' rows: projections, activations, normalisation, attention. */
+import { attention } from './attention.js';
+import { gelu, layerNorm, sigmoid, softmax } from './linear.js';
+import { elementwise, num, product, same } from './operatorKit.js';
+export const DENSE_OPERATORS = [
+    [
+        'linear',
+        {
+            ranks: [2, 2, 1],
+            arity: [2, 3],
+            shape: ([x, w, b]) => {
+                const [rows, width] = x;
+                const [outs, ins] = w;
+                if (width !== ins)
+                    return `x has ${width} columns and the weight takes ${ins}`;
+                if (b !== undefined && b[0] !== outs)
+                    return `the bias has ${b[0]} and the weight ${outs}`;
+                return [rows, outs];
+            },
+            evaluate: ([x, w, b], [xs, ws], _attributes, out) => {
+                const [rows, ins] = xs;
+                const outs = ws[0];
+                for (let row = 0; row < rows; row += 1) {
+                    for (let o = 0; o < outs; o += 1) {
+                        let dot = 0;
+                        for (let i = 0; i < ins; i += 1) {
+                            dot +=
+                                x[row * ins + i] *
+                                    w[o * ins + i];
+                        }
+                        /* The bias joins the double-precision sum, and the result is rounded once. */
+                        out[row * outs + o] = dot + (b === undefined ? 0 : b[o]);
+                    }
+                }
+            },
+        },
+    ],
+    [
+        'relu',
+        {
+            arity: [1, 1],
+            shape: ([x]) => [...x],
+            evaluate: ([x], _shapes, _attributes, out) => {
+                for (let i = 0; i < out.length; i += 1)
+                    out[i] = Math.max(0, x[i]);
+            },
+        },
+    ],
+    [
+        'gelu',
+        {
+            arity: [1, 1],
+            shape: ([x]) => [...x],
+            evaluate: ([x], _shapes, _attributes, out) => gelu(x, out),
+        },
+    ],
+    [
+        'sigmoid',
+        {
+            arity: [1, 1],
+            shape: ([x]) => [...x],
+            evaluate: ([x], _shapes, _attributes, out) => sigmoid(x, out),
+        },
+    ],
+    ['add', elementwise((a, b) => a + b)],
+    ['mul', elementwise((a, b) => a * b)],
+    [
+        'layerNorm',
+        {
+            ranks: [2, 1, 1],
+            arity: [3, 3],
+            shape: ([x, gamma]) => {
+                const cols = x[1];
+                return gamma?.[0] === cols ? [...x] : `gamma has ${gamma?.[0]} and x ${cols}`;
+            },
+            evaluate: ([x, gamma, beta], [xs], attributes, out) => {
+                const [rows, cols] = xs;
+                const epsilon = num(attributes, 'epsilon', 1e-6);
+                layerNorm(x, rows, cols, gamma, beta, epsilon, out);
+            },
+        },
+    ],
+    [
+        'softmax',
+        {
+            arity: [1, 1],
+            shape: ([x]) => [...x],
+            evaluate: ([x], [xs], _attributes, out) => {
+                const shape = xs;
+                const cols = shape[shape.length - 1];
+                softmax(x, product(shape) / cols, cols, out);
+            },
+        },
+    ],
+    [
+        'attention',
+        {
+            /*
+             * Rank 2, or 3 for a batch of windows; checked here rather than by `ranks`, which states one.
+             * The bias, where there is one, is a score for each head, query and key.
+             */
+            arity: [3, 4],
+            shape: ([q, k, v, b], attributes) => {
+                const qs = q;
+                const ks = k;
+                if (qs.length !== 2 && qs.length !== 3)
+                    return `the queries have rank ${qs.length}, not 2 or 3`;
+                if (ks.length !== qs.length)
+                    return `the keys have rank ${ks.length} and the queries ${qs.length}`;
+                if (!same(ks, v))
+                    return 'the keys and values differ in shape';
+                if (qs.length === 3 && qs[0] !== ks[0]) {
+                    return `a batch of ${qs[0]} queries and of ${ks[0]} keys`;
+                }
+                const [queries, channels] = qs.slice(-2);
+                const [keys, keyChannels] = ks.slice(-2);
+                if (channels !== keyChannels) {
+                    return `the queries have ${channels} channels and the keys ${keyChannels}`;
+                }
+                const heads = num(attributes, 'heads', 1);
+                if (channels % heads !== 0)
+                    return `${channels} channels do not split into ${heads} heads`;
+                if (b !== undefined && !same(b, [heads, queries, keys])) {
+                    return `the bias is [${b.join(', ')}], not [${heads}, ${queries}, ${keys}]`;
+                }
+                return [...qs];
+            },
+            scratch: ([q, k]) => q.at(-2) * k.at(-2),
+            evaluate: ([q, k, v, b], [qs, ks], attributes, out, scratch) => {
+                const shape = qs;
+                const [queries, channels] = shape.slice(-2);
+                attention(out, q, k, v, queries, ks.at(-2), channels, num(attributes, 'heads', 1), scratch, b ?? null, shape.length === 3 ? shape[0] : 1);
+            },
+        },
+    ],
+];

package/dist/tensor/graph.d.ts

@@ -0,0 +1,83 @@
+import { type Attributes } from './operators.ts';
+export interface GraphValue {
+    readonly name: string;
+    readonly shape: readonly number[];
+}
+export interface GraphNode {
+    readonly op: string;
+    readonly inputs: readonly string[];
+    readonly output: string;
+    readonly attributes: Attributes;
+}
+export interface GraphTensor {
+    readonly shape: readonly number[];
+    readonly data: Float32Array;
+}
+export interface NetworkGraph {
+    readonly inputs: readonly GraphValue[];
+    readonly outputs: readonly string[];
+    readonly nodes: readonly GraphNode[];
+    readonly tensors: ReadonlyMap<string, GraphTensor>;
+}
+/**
+ * A graph as a file stores it: its tensors in either precision, half as the bits a device uploads.
+ * `@driftengine/drft`'s `DrftGraph` is one, structurally, which is how the two packages meet
+ * without either importing the other.
+ */
+export interface StoredGraph {
+    readonly inputs: readonly GraphValue[];
+    readonly outputs: readonly string[];
+    readonly nodes: readonly GraphNode[];
+    readonly tensors: readonly {
+        readonly name: string;
+        readonly shape: readonly number[];
+        readonly data: Float32Array | Uint16Array;
+    }[];
+}
+/**
+ * A stored graph as one the runtime runs, half-precision tensors decoded to single; validated, and
+ * refused naming what is wrong — an operator the runtime lacks is a fact about the model, and this
+ * is when a caller learns it.
+ */
+export declare function graphFromStored(stored: StoredGraph): NetworkGraph;
+/** Every value's shape — inputs, tensors and intermediates — refused, with the reason, if invalid. */
+export declare function graphShapes(graph: NetworkGraph): ReadonlyMap<string, readonly number[]>;
+/**
+ * A graph as `@driftengine/core`'s runner takes it: its tensors listed by name, and every value's
+ * shape as inferred here — **nothing evaluated and nothing sized**, where asking an evaluator for its
+ * shapes would allocate every intermediate value first. Structural, because core does not import
+ * this package; the shapes are this module's, so there is one copy of every operator's rule.
+ */
+export declare function graphForDevice(graph: NetworkGraph): {
+    readonly inputs: readonly GraphValue[];
+    readonly outputs: readonly string[];
+    readonly nodes: readonly GraphNode[];
+    readonly tensors: readonly {
+        readonly name: string;
+        readonly shape: readonly number[];
+        readonly data: Float32Array;
+    }[];
+    readonly shapes: ReadonlyMap<string, readonly number[]>;
+};
+/** Null when the graph can run; otherwise the reason, naming what it is about. */
+export declare function validateGraph(graph: NetworkGraph): string | null;
+export interface GraphEvaluator {
+    /** Every value's shape, inputs, tensors and intermediates alike. */
+    readonly shapes: ReadonlyMap<string, readonly number[]>;
+    /**
+     * Evaluate with these inputs. The returned arrays are the evaluator's own and are overwritten by
+     * the next run.
+     */
+    run(inputs: ReadonlyMap<string, Float32Array>): Map<string, Float32Array>;
+}
+/**
+ * Validate, size every value once, and return something that evaluates without allocating.
+ *
+ * **Values that cannot be alive at once share a buffer**, by `planReuse` — the rule the device's
+ * runner plans by too. Weights and inputs stay in the caller's own arrays and are planned by
+ * neither. What it buys is the difference between holding every value of a graph and holding the
+ * few alive at any moment: OWLv2's image graph at 960² is 3,647 MB of values and 196 MB in ten buffers,
+ * which is the difference between a reference that runs on an ordinary machine and one that is
+ * killed for its memory.
+ */
+export declare function createGraphEvaluator(graph: NetworkGraph): GraphEvaluator;

package/dist/tensor/graph.js

@@ -0,0 +1,175 @@
+/**
+ * A network as a graph of the runtime's operators, checked before it runs and evaluated on the CPU.
+ *
+ * **Validation says what is wrong in words, and names what it is about**: an operator the runtime
+ * lacks, a value read before anything writes it, a shape that does not fit the node that reads it,
+ * an output nothing produces. A graph is checked when it is loaded — a model is refused then, with
+ * the reason, rather than at its first frame with a picture of zeros.
+ *
+ * **The evaluator is the reference**, which the device runner is held to: it sizes every value and
+ * its scratch once, when it is created, and a run writes into those buffers and allocates nothing.
+ * What it gives up is memory: every intermediate value keeps its own buffer, which is the plainest
+ * thing to compare against and the device runner's job to improve on.
+ */
+import { fromHalfBits } from '../half.js';
+import { OPERATORS } from './operators.js';
+import { planReuse } from '@driftengine/core';
+const size = (shape) => shape.reduce((total, d) => total * d, 1);
+/** Every value's shape, or the first reason the graph cannot run. */
+function inferShapes(graph) {
+    const shapes = new Map();
+    for (const input of graph.inputs)
+        shapes.set(input.name, input.shape);
+    for (const [name, tensor] of graph.tensors) {
+        if (tensor.data.length !== size(tensor.shape)) {
+            return `tensor "${name}" holds ${tensor.data.length} values and its shape [${tensor.shape.join(', ')}] needs ${size(tensor.shape)}`;
+        }
+        shapes.set(name, tensor.shape);
+    }
+    for (const node of graph.nodes) {
+        const operator = OPERATORS.get(node.op);
+        if (operator === undefined) {
+            return `the runtime has no operator "${node.op}" (writing "${node.output}")`;
+        }
+        const [fewest, most] = operator.arity;
+        if (node.inputs.length < fewest || node.inputs.length > most) {
+            return `${node.op} writing "${node.output}" takes ${fewest} to ${most} inputs and was given ${node.inputs.length}`;
+        }
+        const inputShapes = [];
+        for (const input of node.inputs) {
+            const shape = shapes.get(input);
+            if (shape === undefined)
+                return `${node.op} writing "${node.output}" reads "${input}", which nothing writes`;
+            inputShapes.push(shape);
+        }
+        if (shapes.has(node.output))
+            return `"${node.output}" is written twice`;
+        const ranks = operator.ranks ?? [];
+        for (let i = 0; i < inputShapes.length && i < ranks.length; i += 1) {
+            const rank = inputShapes[i].length;
+            if (rank !== ranks[i]) {
+                return `${node.op} writing "${node.output}": "${node.inputs[i]}" has rank ${rank} and this input takes rank ${ranks[i]}`;
+            }
+        }
+        const shape = operator.shape(inputShapes, node.attributes);
+        if (typeof shape === 'string')
+            return `${node.op} writing "${node.output}": ${shape}`;
+        shapes.set(node.output, shape);
+    }
+    for (const output of graph.outputs) {
+        if (!shapes.has(output))
+            return `the output "${output}" is written by nothing`;
+    }
+    return shapes;
+}
+/**
+ * A stored graph as one the runtime runs, half-precision tensors decoded to single; validated, and
+ * refused naming what is wrong — an operator the runtime lacks is a fact about the model, and this
+ * is when a caller learns it.
+ */
+export function graphFromStored(stored) {
+    const tensors = new Map();
+    for (const tensor of stored.tensors) {
+        const data = tensor.data instanceof Uint16Array
+            ? Float32Array.from(tensor.data, (bits) => fromHalfBits(bits))
+            : tensor.data;
+        tensors.set(tensor.name, { shape: tensor.shape, data });
+    }
+    const graph = {
+        inputs: stored.inputs,
+        outputs: stored.outputs,
+        nodes: stored.nodes,
+        tensors,
+    };
+    const problem = validateGraph(graph);
+    if (problem !== null)
+        throw new Error(`network graph: ${problem}`);
+    return graph;
+}
+/** Every value's shape — inputs, tensors and intermediates — refused, with the reason, if invalid. */
+export function graphShapes(graph) {
+    const shapes = inferShapes(graph);
+    if (typeof shapes === 'string')
+        throw new Error(`network graph: ${shapes}`);
+    return shapes;
+}
+/**
+ * A graph as `@driftengine/core`'s runner takes it: its tensors listed by name, and every value's
+ * shape as inferred here — **nothing evaluated and nothing sized**, where asking an evaluator for its
+ * shapes would allocate every intermediate value first. Structural, because core does not import
+ * this package; the shapes are this module's, so there is one copy of every operator's rule.
+ */
+export function graphForDevice(graph) {
+    return {
+        inputs: graph.inputs,
+        outputs: graph.outputs,
+        nodes: graph.nodes,
+        tensors: [...graph.tensors].map(([name, tensor]) => ({ name, ...tensor })),
+        shapes: graphShapes(graph),
+    };
+}
+/** Null when the graph can run; otherwise the reason, naming what it is about. */
+export function validateGraph(graph) {
+    const shapes = inferShapes(graph);
+    return typeof shapes === 'string' ? shapes : null;
+}
+/**
+ * Validate, size every value once, and return something that evaluates without allocating.
+ *
+ * **Values that cannot be alive at once share a buffer**, by `planReuse` — the rule the device's
+ * runner plans by too. Weights and inputs stay in the caller's own arrays and are planned by
+ * neither. What it buys is the difference between holding every value of a graph and holding the
+ * few alive at any moment: OWLv2's image graph at 960² is 3,647 MB of values and 196 MB in ten buffers,
+ * which is the difference between a reference that runs on an ordinary machine and one that is
+ * killed for its memory.
+ */
+export function createGraphEvaluator(graph) {
+    const shapes = inferShapes(graph);
+    if (typeof shapes === 'string')
+        throw new Error(`network graph: ${shapes}`);
+    const buffers = new Map();
+    for (const [name, tensor] of graph.tensors)
+        buffers.set(name, tensor.data);
+    const plan = planReuse({ nodes: graph.nodes, held: [], outputs: graph.outputs }, (name) => size(shapes.get(name)));
+    const slots = plan.sizes.map((values) => new Float32Array(values));
+    /* A buffer is as large as the largest value it holds, so each value takes a view of its own. */
+    for (const [name, slot] of plan.slotOf) {
+        const room = size(shapes.get(name));
+        const held = slots[slot];
+        buffers.set(name, held.length === room ? held : held.subarray(0, room));
+    }
+    let scratchSize = 0;
+    for (const node of graph.nodes) {
+        const operator = OPERATORS.get(node.op);
+        const inputShapes = node.inputs.map((input) => shapes.get(input));
+        scratchSize = Math.max(scratchSize, operator?.scratch?.(inputShapes, node.attributes) ?? 0);
+    }
+    const scratch = new Float32Array(scratchSize);
+    const outputs = new Map();
+    return {
+        shapes,
+        run(inputs) {
+            for (const input of graph.inputs) {
+                const data = inputs.get(input.name);
+                if (data === undefined)
+                    throw new Error(`network graph: no value for the input "${input.name}"`);
+                if (data.length !== size(input.shape)) {
+                    throw new Error(`network graph: "${input.name}" holds ${data.length} values and needs ${size(input.shape)}`);
+                }
+                buffers.set(input.name, data);
+            }
+            for (const node of graph.nodes) {
+                const operator = OPERATORS.get(node.op);
+                if (operator === undefined)
+                    continue;
+                const values = node.inputs.map((input) => buffers.get(input));
+                const inputShapes = node.inputs.map((input) => shapes.get(input));
+                operator.evaluate(values, inputShapes, node.attributes, buffers.get(node.output), scratch);
+            }
+            outputs.clear();
+            for (const output of graph.outputs)
+                outputs.set(output, buffers.get(output));
+            return outputs;
+        },
+    };
+}

package/dist/tensor/linear.d.ts

@@ -0,0 +1,49 @@
+/**
+ * The dense operators a transformer is built from: matrix multiply, bias, layer norm, GELU, softmax.
+ *
+ * **The references the device kernels are held to**, which is why they are written for exactness
+ * rather than speed: every sum accumulates in double precision and is rounded once, on the way into
+ * the caller's single-precision output. A kernel that disagrees with one of these is wrong, not
+ * this.
+ *
+ * **Contiguous row-major arrays with their dimensions, and nothing that allocates.** A strided view
+ * that permutes without copying would make every operator handle strides, and every device kernel
+ * would materialise the permutation anyway; so a permutation is a copy into a buffer the caller
+ * owns, and the operators stay one loop each.
+ *
+ * **The exact GELU, `x·Φ(x)`, and never the tanh approximation.** The two differ by 4e-4 at 3, and
+ * a port of a network trained with one and evaluated with the other is a different network. `erf`
+ * is here for that reason, accurate to double precision.
+ */
+/**
+ * `out[m×n] = a[m×k] · b`, where `b` is `k×n`, or `n×k` read as its transpose when `transposeB`.
+ * The offsets are where each matrix starts in its array, which is how one head of many is taken.
+ */
+export declare function matmul(out: Float32Array, a: Float32Array, b: Float32Array, m: number, k: number, n: number, transposeB?: boolean, outAt?: number, aAt?: number, bAt?: number): void;
+/** Add `bias[cols]` to every one of `rows` rows of `x`, in place. */
+export declare function addBias(x: Float32Array, rows: number, cols: number, bias: Float32Array): void;
+/**
+ * Each row normalised to zero mean and unit variance, then scaled by `gamma` and shifted by `beta`.
+ *
+ * **The population variance, divided by the row's length and not one less**, which is what the
+ * upstream frameworks compute and therefore what their trained weights expect.
+ */
+export declare function layerNorm(x: Float32Array, rows: number, cols: number, gamma: Float32Array, beta: Float32Array, epsilon: number, out: Float32Array): void;
+/**
+ * The error function, to double precision.
+ *
+ * A Maclaurin series below 3, where its alternating terms peak near 170 at `x = 3` and cost about
+ * three digits to cancellation — still 1e-13; a continued fraction for the complement above, which
+ * converges fastest exactly where the series is worst; and ±1 past 6, where the complement is below
+ * a double's last place.
+ */
+export declare function erf(x: number): number;
+/** `x·Φ(x)`, the exact GELU, from `x` into `out`. */
+export declare function gelu(x: Float32Array, out: Float32Array): void;
+/** The logistic, `1/(1 + e^−x)`, from `x` into `out`: 1 far to the right, and 0 far to the left. */
+export declare function sigmoid(x: Float32Array, out: Float32Array): void;
+/**
+ * Each row of `x` to a distribution, stably: the row's largest value is subtracted before the
+ * exponential, so logits near a thousand give their logistic rather than `Infinity / Infinity`.
+ */
+export declare function softmax(x: Float32Array, rows: number, cols: number, out: Float32Array): void;