@seanhogg/builderforce-memory-engine 2026.6.18

package/dist/kernels/weight_update.js

@@ -0,0 +1,119 @@
+// Weight Update WGSL Kernel (AdamW Optimizer)
+// Implements fused AdamW parameter update on the GPU.
+//
+// AdamW update rule:
+//   m_t = beta1 * m_{t-1} + (1 - beta1) * g_t
+//   v_t = beta2 * v_{t-1} + (1 - beta2) * g_t^2
+//   m_hat = m_t / (1 - beta1^t)
+//   v_hat = v_t / (1 - beta2^t)
+//   theta_t = theta_{t-1} * (1 - lr * weight_decay) - lr * m_hat / (sqrt(v_hat) + eps)
+export const WEIGHT_UPDATE_WGSL = /* wgsl */ `
+
+struct AdamParams {
+    num_elements   : u32,
+    lr             : f32,   // learning rate
+    beta1          : f32,   // default 0.9
+    beta2          : f32,   // default 0.999
+    eps            : f32,   // default 1e-8
+    weight_decay   : f32,   // default 0.01
+    beta1_t        : f32,   // beta1^t  (precomputed bias correction term)
+    beta2_t        : f32,   // beta2^t
+};
+
+@group(0) @binding(0) var<uniform>             adam     : AdamParams;
+// param (N,)   – weight tensor (read-write: updated in-place)
+@group(0) @binding(1) var<storage, read_write> param    : array<f32>;
+// grad  (N,)   – gradient
+@group(0) @binding(2) var<storage, read>       grad     : array<f32>;
+// m     (N,)   – first moment
+@group(0) @binding(3) var<storage, read_write> m_state  : array<f32>;
+// v     (N,)   – second moment
+@group(0) @binding(4) var<storage, read_write> v_state  : array<f32>;
+
+// Dispatch: (ceil(N / 256), 1, 1)
+@compute @workgroup_size(256, 1, 1)
+fn adamw_update(
+    @builtin(global_invocation_id) gid : vec3<u32>,
+) {
+    let i = gid.x;
+    if (i >= adam.num_elements) { return; }
+
+    let g = grad[i];
+    let p = param[i];
+
+    // Moment updates
+    let m_new = adam.beta1 * m_state[i] + (1.0 - adam.beta1) * g;
+    let v_new = adam.beta2 * v_state[i] + (1.0 - adam.beta2) * g * g;
+    m_state[i] = m_new;
+    v_state[i] = v_new;
+
+    // Bias-corrected estimates
+    let m_hat = m_new / (1.0 - adam.beta1_t);
+    let v_hat = v_new / (1.0 - adam.beta2_t);
+
+    // Weight decay (decoupled) + gradient step
+    param[i] = p * (1.0 - adam.lr * adam.weight_decay) -
+               adam.lr * m_hat / (sqrt(v_hat) + adam.eps);
+}
+`;
+// Gradient clipping kernel – clips global gradient norm to max_norm.
+// Run before weight updates.  Two-pass: first compute squared norm, then scale.
+export const GRAD_CLIP_WGSL = /* wgsl */ `
+
+struct ClipParams {
+    num_elements : u32,
+    max_norm_sq  : f32,   // max_norm^2
+};
+
+@group(0) @binding(0) var<uniform>             clip_p  : ClipParams;
+@group(0) @binding(1) var<storage, read_write> grad    : array<f32>;
+@group(0) @binding(2) var<storage, read_write> norm_sq : array<f32>;  // size 1, atomic accumulator
+
+var<workgroup> local_sq : array<f32, 256>;
+
+// Pass 1: reduce sum of squares into norm_sq[0]
+@compute @workgroup_size(256, 1, 1)
+fn grad_norm_reduce(
+    @builtin(global_invocation_id)   gid : vec3<u32>,
+    @builtin(local_invocation_index) lid : u32,
+) {
+    let i = gid.x;
+    local_sq[lid] = 0.0;
+    if (i < clip_p.num_elements) {
+        local_sq[lid] = grad[i] * grad[i];
+    }
+    workgroupBarrier();
+
+    // Parallel reduction within workgroup
+    var s: u32 = 128u;
+    loop {
+        if (s == 0u) { break; }
+        if (lid < s) {
+            local_sq[lid] = local_sq[lid] + local_sq[lid + s];
+        }
+        workgroupBarrier();
+        s = s >> 1u;
+    }
+
+    if (lid == 0u) {
+        // Non-atomic accumulation (single workgroup assumption for small models)
+        norm_sq[0] = norm_sq[0] + local_sq[0];
+    }
+}
+
+// Pass 2: scale gradients if norm exceeds max_norm
+@compute @workgroup_size(256, 1, 1)
+fn grad_clip_scale(
+    @builtin(global_invocation_id) gid : vec3<u32>,
+) {
+    let i = gid.x;
+    if (i >= clip_p.num_elements) { return; }
+
+    let ns = norm_sq[0];
+    if (ns > clip_p.max_norm_sq) {
+        let scale = sqrt(clip_p.max_norm_sq / ns);
+        grad[i] = grad[i] * scale;
+    }
+}
+`;
+//# sourceMappingURL=weight_update.js.map

package/dist/kernels/weight_update.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"weight_update.js","sourceRoot":"","sources":["../../src/kernels/weight_update.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAC9C,sDAAsD;AACtD,EAAE;AACF,qBAAqB;AACrB,8CAA8C;AAC9C,gDAAgD;AAChD,gCAAgC;AAChC,gCAAgC;AAChC,uFAAuF;AAEvF,MAAM,CAAC,MAAM,kBAAkB,GAAW,UAAU,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgDnD,CAAC;AAEF,qEAAqE;AACrE,gFAAgF;AAChF,MAAM,CAAC,MAAM,cAAc,GAAW,UAAU,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyD/C,CAAC"}

package/dist/model/attention_block.d.ts

@@ -0,0 +1,48 @@
+/**
+ * attention_block.ts – Causal Multi-Head Self-Attention Block.
+ *
+ * Intentionally simple for WebGPU — naive O(L²) tiled attention,
+ * no Flash-Attention dependency. Suitable for hybrid (Jamba/Zamba) schedules
+ * where a few attention layers interleave with SSM layers.
+ *
+ * Data flow:
+ *   Input (B, L, D_model)
+ *     └─ RMSNorm
+ *     └─ wQKV → Q (B,L,H,dh), K (B,L,H,dh), V (B,L,H,dh)
+ *     └─ causal attention scores = Q·Kᵀ / √dh  (masked)
+ *     └─ softmax
+ *     └─ weighted V sum
+ *     └─ concat heads → wO → D_model
+ *     └─ + residual
+ *   [optional FFN sublayer]
+ *
+ * Implements SequenceLayer.
+ */
+import type { SequenceLayer, LayerForwardResult, LayerParam } from './sequence_layer.js';
+export interface AttentionBlockConfig {
+    dModel: number;
+    nHeads: number;
+    dHead?: number;
+    hasFfn?: boolean;
+    ffnMult?: number;
+}
+export interface AttentionCache {
+    scores: GPUBuffer;
+}
+export declare class AttentionBlock implements SequenceLayer {
+    readonly layerType: "attention";
+    device: GPUDevice;
+    config: Required<AttentionBlockConfig>;
+    dHead: number;
+    gpuWeights: Record<string, GPUBuffer>;
+    pipelines: Record<string, GPUComputePipeline>;
+    constructor(device: GPUDevice, config: AttentionBlockConfig);
+    private _initWeights;
+    private _buildPipelines;
+    forward(xBuf: GPUBuffer, batch: number, seqLen: number): LayerForwardResult;
+    parameters(): LayerParam[];
+    getTrainableParams(): LayerParam[];
+    setWSLAMode(_enabled: boolean): void;
+    destroy(): void;
+}
+//# sourceMappingURL=attention_block.d.ts.map

package/dist/model/attention_block.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"attention_block.d.ts","sourceRoot":"","sources":["../../src/model/attention_block.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAoBH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,oBAAoB;IACjC,MAAM,EAAI,MAAM,CAAC;IACjB,MAAM,EAAI,MAAM,CAAC;IACjB,KAAK,CAAC,EAAI,MAAM,CAAC;IACjB,MAAM,CAAC,EAAG,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC3B,MAAM,EAAE,SAAS,CAAC;CACrB;AA6BD,qBAAa,cAAe,YAAW,aAAa;IAChD,QAAQ,CAAC,SAAS,EAAG,WAAW,CAAU;IAE1C,MAAM,EAAG,SAAS,CAAC;IACnB,MAAM,EAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IACxC,KAAK,EAAI,MAAM,CAAC;IAEhB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACtC,SAAS,EAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;gBAEnC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,oBAAoB;IAyB3D,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,eAAe;IAgBvB,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,kBAAkB;IAsJ3E,UAAU,IAAI,UAAU,EAAE;IAuB1B,kBAAkB,IAAI,UAAU,EAAE;IAKlC,WAAW,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAIpC,OAAO,IAAI,IAAI;CAIlB"}

package/dist/model/attention_block.js

@@ -0,0 +1,262 @@
+/**
+ * attention_block.ts – Causal Multi-Head Self-Attention Block.
+ *
+ * Intentionally simple for WebGPU — naive O(L²) tiled attention,
+ * no Flash-Attention dependency. Suitable for hybrid (Jamba/Zamba) schedules
+ * where a few attention layers interleave with SSM layers.
+ *
+ * Data flow:
+ *   Input (B, L, D_model)
+ *     └─ RMSNorm
+ *     └─ wQKV → Q (B,L,H,dh), K (B,L,H,dh), V (B,L,H,dh)
+ *     └─ causal attention scores = Q·Kᵀ / √dh  (masked)
+ *     └─ softmax
+ *     └─ weighted V sum
+ *     └─ concat heads → wO → D_model
+ *     └─ + residual
+ *   [optional FFN sublayer]
+ *
+ * Implements SequenceLayer.
+ */
+import { createComputePipeline, createBindGroup, createStorageBuffer, createEmptyStorageBuffer, createUniformBuffer, dispatchKernel, cdiv, } from '../utils/gpu_utils.js';
+import { ATTENTION_FORWARD_WGSL, SOFTMAX_WGSL, } from '../kernels/attention.js';
+import { LINEAR_FORWARD_WGSL } from '../kernels/linear_projection.js';
+import { gaussianArray } from '../utils/rng.js';
+import { ACTIVATIONS_WGSL } from '../kernels/activations.js';
+const ADD_SHADER = /* wgsl */ `
+@group(0) @binding(0) var<storage, read>       a : array<f32>;
+@group(0) @binding(1) var<storage, read>       b : array<f32>;
+@group(0) @binding(2) var<storage, read_write> c : array<f32>;
+@group(0) @binding(3) var<uniform>             n : u32;
+@compute @workgroup_size(256)
+fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
+    let i = gid.x;
+    if (i < n) { c[i] = a[i] + b[i]; }
+}
+`;
+// SiLU for FFN
+const SILU_SHADER = /* wgsl */ `
+struct ActParams { num_elements: u32; };
+@group(0) @binding(0) var<uniform>             p : ActParams;
+@group(0) @binding(1) var<storage, read>       x : array<f32>;
+@group(0) @binding(2) var<storage, read_write> y : array<f32>;
+@compute @workgroup_size(256, 1, 1)
+fn silu_forward(@builtin(global_invocation_id) gid: vec3<u32>) {
+    let i = gid.x;
+    if (i >= p.num_elements) { return; }
+    let v = x[i];
+    y[i] = v / (1.0 + exp(-v));
+}
+`;
+export class AttentionBlock {
+    layerType = 'attention';
+    device;
+    config;
+    dHead;
+    gpuWeights;
+    pipelines;
+    constructor(device, config) {
+        this.device = device;
+        if (config.dModel % config.nHeads !== 0) {
+            throw new Error(`AttentionBlock: dModel (${config.dModel}) must be divisible by nHeads (${config.nHeads}).`);
+        }
+        this.config = {
+            dHead: config.dModel / config.nHeads,
+            hasFfn: false,
+            ffnMult: 4,
+            ...config,
+        };
+        this.dHead = this.config.dHead;
+        this.gpuWeights = {};
+        this.pipelines = {};
+        this._initWeights();
+        this._buildPipelines();
+    }
+    _initWeights() {
+        const { dModel, nHeads, hasFfn, ffnMult } = this.config;
+        const randn = (n, std = 0.02) => gaussianArray(n, std);
+        const zeros = (n) => new Float32Array(n);
+        const ones = (n) => new Float32Array(n).fill(1.0);
+        const mk = (arr) => createStorageBuffer(this.device, arr, true);
+        this.gpuWeights = {
+            wQKV: mk(randn(3 * dModel * dModel)),
+            bQKV: mk(zeros(3 * dModel)),
+            wO: mk(randn(dModel * dModel)),
+            bO: mk(zeros(dModel)),
+            normWeight: mk(ones(dModel)),
+        };
+        if (hasFfn) {
+            const ffnDim = dModel * ffnMult;
+            this.gpuWeights['wFfn1'] = mk(randn(ffnDim * dModel));
+            this.gpuWeights['bFfn1'] = mk(zeros(ffnDim));
+            this.gpuWeights['wFfn2'] = mk(randn(dModel * ffnDim));
+            this.gpuWeights['bFfn2'] = mk(zeros(dModel));
+        }
+    }
+    _buildPipelines() {
+        const d = this.device;
+        this.pipelines = {
+            linear: createComputePipeline(d, LINEAR_FORWARD_WGSL, 'linear_forward'),
+            rmsnorm: createComputePipeline(d, ACTIVATIONS_WGSL, 'rmsnorm_forward'),
+            attn_fwd: createComputePipeline(d, ATTENTION_FORWARD_WGSL, 'attention_forward'),
+            attn_val: createComputePipeline(d, ATTENTION_FORWARD_WGSL, 'attention_value'),
+            softmax: createComputePipeline(d, SOFTMAX_WGSL, 'softmax_forward'),
+            elAdd: createComputePipeline(d, ADD_SHADER, 'main'),
+        };
+        if (this.config.hasFfn) {
+            this.pipelines['silu'] = createComputePipeline(d, SILU_SHADER, 'silu_forward');
+        }
+    }
+    forward(xBuf, batch, seqLen) {
+        const d = this.device;
+        const { dModel, nHeads, hasFfn } = this.config;
+        const dh = this.dHead;
+        const B = batch;
+        const L = seqLen;
+        const M = B * L;
+        const H = nHeads;
+        // 1. Pre-block RMSNorm
+        const normOut = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        const normInv = createEmptyStorageBuffer(d, M * 4, true);
+        {
+            const params = new ArrayBuffer(16);
+            new Uint32Array(params, 0, 2).set([M, dModel]);
+            new Float32Array(params, 8, 1).set([1e-6]);
+            const pBuf = createUniformBuffer(d, params);
+            const bg = createBindGroup(d, this.pipelines['rmsnorm'], [pBuf, xBuf, this.gpuWeights['normWeight'], normOut, normInv]);
+            dispatchKernel(d, this.pipelines['rmsnorm'], bg, [cdiv(M, 64), 1, 1]);
+        }
+        normInv.destroy();
+        // 2. QKV projection: [B, L, 3*D]
+        const qkvOut = createEmptyStorageBuffer(d, M * 3 * dModel * 4, true);
+        {
+            const params = new Uint32Array([M, dModel, 3 * dModel]).buffer;
+            const pBuf = createUniformBuffer(d, params);
+            const bg = createBindGroup(d, this.pipelines['linear'], [pBuf, normOut, this.gpuWeights['wQKV'], this.gpuWeights['bQKV'], qkvOut]);
+            dispatchKernel(d, this.pipelines['linear'], bg, [cdiv(M, 16), cdiv(3 * dModel, 16), 1]);
+        }
+        normOut.destroy();
+        // Split QKV into Q, K, V: each [B, L, H, dh] = [B, L, D]
+        const QBuf = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        const KBuf = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        const VBuf = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        {
+            const enc = d.createCommandEncoder();
+            enc.copyBufferToBuffer(qkvOut, 0, QBuf, 0, M * dModel * 4);
+            enc.copyBufferToBuffer(qkvOut, M * dModel * 4, KBuf, 0, M * dModel * 4);
+            enc.copyBufferToBuffer(qkvOut, 2 * M * dModel * 4, VBuf, 0, M * dModel * 4);
+            d.queue.submit([enc.finish()]);
+        }
+        qkvOut.destroy();
+        // 3. Attention scores: [B, H, L, L]
+        const scores = createEmptyStorageBuffer(d, B * H * L * L * 4, true);
+        {
+            const attnParams = new Uint32Array([B, L, dModel, H, dh]).buffer;
+            const pBuf = createUniformBuffer(d, attnParams);
+            const bg = createBindGroup(d, this.pipelines['attn_fwd'], [pBuf, QBuf, KBuf, VBuf, scores,
+                createEmptyStorageBuffer(d, M * dModel * 4, true)]); // out_buf placeholder
+            dispatchKernel(d, this.pipelines['attn_fwd'], bg, [cdiv(L, 16), H, B]);
+        }
+        // 4. Softmax (causal) per row: dispatch (L, H, B)
+        {
+            const smParams = new Uint32Array([L, L, 1]).buffer;
+            const pBuf = createUniformBuffer(d, smParams);
+            const bg = createBindGroup(d, this.pipelines['softmax'], [pBuf, scores]);
+            dispatchKernel(d, this.pipelines['softmax'], bg, [L, H, B]);
+        }
+        // 5. Weighted V sum → attn output [B, L, H, dh]
+        const attnOut = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        {
+            const attnParams = new Uint32Array([B, L, dModel, H, dh]).buffer;
+            const pBuf = createUniformBuffer(d, attnParams);
+            const bg = createBindGroup(d, this.pipelines['attn_val'], [pBuf, QBuf, KBuf, VBuf, scores, attnOut]);
+            dispatchKernel(d, this.pipelines['attn_val'], bg, [cdiv(L, 16), H, B]);
+        }
+        QBuf.destroy();
+        KBuf.destroy();
+        VBuf.destroy();
+        // 6. Output projection: [B, L, D] → [B, L, D]
+        const outProjOut = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        {
+            const params = new Uint32Array([M, dModel, dModel]).buffer;
+            const pBuf = createUniformBuffer(d, params);
+            const bg = createBindGroup(d, this.pipelines['linear'], [pBuf, attnOut, this.gpuWeights['wO'], this.gpuWeights['bO'], outProjOut]);
+            dispatchKernel(d, this.pipelines['linear'], bg, [cdiv(M, 16), cdiv(dModel, 16), 1]);
+        }
+        attnOut.destroy();
+        // 7. Residual add
+        let current = createEmptyStorageBuffer(d, M * dModel * 4, true);
+        {
+            const nBuf = createUniformBuffer(d, new Uint32Array([M * dModel]).buffer);
+            const bg = createBindGroup(d, this.pipelines['elAdd'], [outProjOut, xBuf, current, nBuf]);
+            dispatchKernel(d, this.pipelines['elAdd'], bg, [cdiv(M * dModel, 256), 1, 1]);
+        }
+        outProjOut.destroy();
+        // 8. Optional FFN sublayer
+        if (hasFfn) {
+            const { ffnMult } = this.config;
+            const ffnDim = dModel * ffnMult;
+            const ffn1Out = createEmptyStorageBuffer(d, M * ffnDim * 4, true);
+            {
+                const params = new Uint32Array([M, dModel, ffnDim]).buffer;
+                const pBuf = createUniformBuffer(d, params);
+                const bg = createBindGroup(d, this.pipelines['linear'], [pBuf, current, this.gpuWeights['wFfn1'], this.gpuWeights['bFfn1'], ffn1Out]);
+                dispatchKernel(d, this.pipelines['linear'], bg, [cdiv(M, 16), cdiv(ffnDim, 16), 1]);
+            }
+            const siluOut = createEmptyStorageBuffer(d, M * ffnDim * 4, true);
+            {
+                const nBuf = createUniformBuffer(d, new Uint32Array([M * ffnDim]).buffer);
+                const bg = createBindGroup(d, this.pipelines['silu'], [nBuf, ffn1Out, siluOut]);
+                dispatchKernel(d, this.pipelines['silu'], bg, [cdiv(M * ffnDim, 256), 1, 1]);
+            }
+            ffn1Out.destroy();
+            const ffn2Out = createEmptyStorageBuffer(d, M * dModel * 4, true);
+            {
+                const params = new Uint32Array([M, ffnDim, dModel]).buffer;
+                const pBuf = createUniformBuffer(d, params);
+                const bg = createBindGroup(d, this.pipelines['linear'], [pBuf, siluOut, this.gpuWeights['wFfn2'], this.gpuWeights['bFfn2'], ffn2Out]);
+                dispatchKernel(d, this.pipelines['linear'], bg, [cdiv(M, 16), cdiv(dModel, 16), 1]);
+            }
+            siluOut.destroy();
+            const residual2 = createEmptyStorageBuffer(d, M * dModel * 4, true);
+            {
+                const nBuf = createUniformBuffer(d, new Uint32Array([M * dModel]).buffer);
+                const bg = createBindGroup(d, this.pipelines['elAdd'], [ffn2Out, current, residual2, nBuf]);
+                dispatchKernel(d, this.pipelines['elAdd'], bg, [cdiv(M * dModel, 256), 1, 1]);
+            }
+            ffn2Out.destroy();
+            current.destroy();
+            current = residual2;
+        }
+        const cache = { scores };
+        return { output: current, cache };
+    }
+    parameters() {
+        const { dModel, hasFfn, ffnMult } = this.config;
+        const params = [
+            { buf: this.gpuWeights['wQKV'], numel: 3 * dModel * dModel, name: 'wQKV' },
+            { buf: this.gpuWeights['bQKV'], numel: 3 * dModel, name: 'bQKV' },
+            { buf: this.gpuWeights['wO'], numel: dModel * dModel, name: 'wO' },
+            { buf: this.gpuWeights['bO'], numel: dModel, name: 'bO' },
+            { buf: this.gpuWeights['normWeight'], numel: dModel, name: 'normWeight' },
+        ];
+        if (hasFfn) {
+            const ffnDim = dModel * ffnMult;
+            params.push({ buf: this.gpuWeights['wFfn1'], numel: ffnDim * dModel, name: 'wFfn1' }, { buf: this.gpuWeights['bFfn1'], numel: ffnDim, name: 'bFfn1' }, { buf: this.gpuWeights['wFfn2'], numel: dModel * ffnDim, name: 'wFfn2' }, { buf: this.gpuWeights['bFfn2'], numel: dModel, name: 'bFfn2' });
+        }
+        return params;
+    }
+    getTrainableParams() {
+        // Attention layers are always fully trained — no WSLA subset
+        return this.parameters();
+    }
+    setWSLAMode(_enabled) {
+        // No-op for attention: WSLA does not apply
+    }
+    destroy() {
+        for (const buf of Object.values(this.gpuWeights))
+            buf.destroy();
+        this.gpuWeights = {};
+    }
+}
+//# sourceMappingURL=attention_block.js.map

package/dist/model/attention_block.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"attention_block.js","sourceRoot":"","sources":["../../src/model/attention_block.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EACH,qBAAqB,EACrB,eAAe,EACf,mBAAmB,EACnB,wBAAwB,EACxB,mBAAmB,EACnB,cAAc,EACd,IAAI,GACP,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACH,sBAAsB,EACtB,YAAY,GACf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAS,2BAA2B,CAAC;AAgBhE,MAAM,UAAU,GAAG,UAAU,CAAA;;;;;;;;;;CAU5B,CAAC;AAEF,eAAe;AACf,MAAM,WAAW,GAAG,UAAU,CAAA;;;;;;;;;;;;CAY7B,CAAC;AAEF,MAAM,OAAO,cAAc;IACd,SAAS,GAAG,WAAoB,CAAC;IAE1C,MAAM,CAAa;IACnB,MAAM,CAAkC;IACxC,KAAK,CAAW;IAEhB,UAAU,CAA4B;IACtC,SAAS,CAAsC;IAE/C,YAAY,MAAiB,EAAE,MAA4B;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACX,2BAA2B,MAAM,CAAC,MAAM,kCAAkC,MAAM,CAAC,MAAM,IAAI,CAC9F,CAAC;QACN,CAAC;QAED,IAAI,CAAC,MAAM,GAAG;YACV,KAAK,EAAI,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YACtC,MAAM,EAAG,KAAK;YACd,OAAO,EAAE,CAAC;YACV,GAAG,MAAM;SACsB,CAAC;QAEpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAE/B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,SAAS,GAAI,EAAE,CAAC;QAErB,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,EAAE,CAAC;IAC3B,CAAC;IAEO,YAAY;QAChB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAExD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,GAAG,GAAG,IAAI,EAAgB,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAE7E,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,IAAI,GAAI,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,EAAE,GAAM,CAAC,GAAiB,EAAE,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAEjF,IAAI,CAAC,UAAU,GAAG;YACd,IAAI,EAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;YAC1C,IAAI,EAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;YACjC,EAAE,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;YACtC,EAAE,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC7B,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SAC/B,CAAC;QAEF,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YAChC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7C,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACjD,CAAC;IACL,CAAC;IAEO,eAAe;QACnB,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG;YACb,MAAM,EAAI,qBAAqB,CAAC,CAAC,EAAE,mBAAmB,EAAM,gBAAgB,CAAC;YAC7E,OAAO,EAAG,qBAAqB,CAAC,CAAC,EAAE,gBAAgB,EAAS,iBAAiB,CAAC;YAC9E,QAAQ,EAAE,qBAAqB,CAAC,CAAC,EAAE,sBAAsB,EAAG,mBAAmB,CAAC;YAChF,QAAQ,EAAE,qBAAqB,CAAC,CAAC,EAAE,sBAAsB,EAAG,iBAAiB,CAAC;YAC9E,OAAO,EAAG,qBAAqB,CAAC,CAAC,EAAE,YAAY,EAAa,iBAAiB,CAAC;YAC9E,KAAK,EAAK,qBAAqB,CAAC,CAAC,EAAE,UAAU,EAAe,MAAM,CAAC;SACtE,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,qBAAqB,CAAC,CAAC,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAED,OAAO,CAAC,IAAe,EAAE,KAAa,EAAE,MAAc;QAClD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACtB,MAAM,CAAC,GAAI,KAAK,CAAC;QACjB,MAAM,CAAC,GAAI,MAAM,CAAC;QAClB,MAAM,CAAC,GAAI,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,CAAC,GAAI,MAAM,CAAC;QAElB,uBAAuB;QACvB,MAAM,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACzD,CAAC;YACG,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;YAC/C,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC5C,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAE,EACpD,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YACpE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,OAAO,EAAE,CAAC;QAElB,iCAAiC;QACjC,MAAM,MAAM,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACrE,CAAC;YACG,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAC/D,MAAM,IAAI,GAAK,mBAAmB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EACnD,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAE,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAE,EAAE,MAAM,CAAC,CAAC,CAAC;YACjF,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7F,CAAC;QACD,OAAO,CAAC,OAAO,EAAE,CAAC;QAElB,yDAAyD;QACzD,MAAM,IAAI,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;YACG,MAAM,GAAG,GAAG,CAAC,CAAC,oBAAoB,EAAE,CAAC;YACrC,GAAG,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,EAAgB,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;YACzE,GAAG,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAI,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;YAC1E,GAAG,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;YAC5E,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,CAAC,OAAO,EAAE,CAAC;QAEjB,oCAAoC;QACpC,MAAM,MAAM,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACpE,CAAC;YACG,MAAM,UAAU,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACjE,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YAChD,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAE,EACrD,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM;gBAC9B,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAE,sBAAsB;YACjF,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,kDAAkD;QAClD,CAAC;YACG,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACnD,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YAC9C,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAE,EACpD,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YACpB,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,gDAAgD;QAChD,MAAM,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;YACG,MAAM,UAAU,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;YACjE,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YAChD,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAE,EACrD,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;YAC/C,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,OAAO,EAAE,CAAC;QAEf,8CAA8C;QAC9C,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACrE,CAAC;YACG,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3D,MAAM,IAAI,GAAK,mBAAmB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC9C,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EACnD,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAE,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAE,EAAE,UAAU,CAAC,CAAC,CAAC;YACjF,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACzF,CAAC;QACD,OAAO,CAAC,OAAO,EAAE,CAAC;QAElB,kBAAkB;QAClB,IAAI,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QAChE,CAAC;YACG,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC1E,MAAM,EAAE,GAAK,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAE,EACpD,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YACvC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACnF,CAAC;QACD,UAAU,CAAC,OAAO,EAAE,CAAC;QAErB,2BAA2B;QAC3B,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YAEhC,MAAM,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YAClE,CAAC;gBACG,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC3D,MAAM,IAAI,GAAK,mBAAmB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC9C,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EACnD,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,OAAO,CAAC,CAAC,CAAC;gBACpF,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACzF,CAAC;YAED,MAAM,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YAClE,CAAC;gBACG,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBAC1E,MAAM,EAAE,GAAK,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAE,EACnD,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC9B,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAClF,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,CAAC;YAElB,MAAM,OAAO,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YAClE,CAAC;gBACG,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC3D,MAAM,IAAI,GAAK,mBAAmB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC9C,MAAM,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EACnD,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,OAAO,CAAC,CAAC,CAAC;gBACpF,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACzF,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,CAAC;YAElB,MAAM,SAAS,GAAG,wBAAwB,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;YACpE,CAAC;gBACG,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,EAAE,IAAI,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBAC1E,MAAM,EAAE,GAAK,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAE,EACpD,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;gBACzC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,GAAG,SAAS,CAAC;QACxB,CAAC;QAED,MAAM,KAAK,GAAmB,EAAE,MAAM,EAAE,CAAC;QACzC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACtC,CAAC;IAED,UAAU;QACN,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAChD,MAAM,MAAM,GAAiB;YACzB,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAE,EAAO,KAAK,EAAE,CAAC,GAAG,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,MAAM,EAAO;YACrF,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAE,EAAO,KAAK,EAAE,CAAC,GAAG,MAAM,EAAW,IAAI,EAAE,MAAM,EAAO;YACrF,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAE,EAAU,KAAK,EAAE,MAAM,GAAG,MAAM,EAAM,IAAI,EAAE,IAAI,EAAS;YACtF,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAE,EAAU,KAAK,EAAE,MAAM,EAAe,IAAI,EAAE,IAAI,EAAS;YACtF,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,YAAY,CAAE,EAAE,KAAK,EAAE,MAAM,EAAe,IAAI,EAAE,YAAY,EAAC;SACzF,CAAC;QAEF,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;YAChC,MAAM,CAAC,IAAI,CACP,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EACzE,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,KAAK,EAAE,MAAM,EAAW,IAAI,EAAE,OAAO,EAAE,EACzE,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EACzE,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAE,EAAE,KAAK,EAAE,MAAM,EAAW,IAAI,EAAE,OAAO,EAAE,CAC5E,CAAC;QACN,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,kBAAkB;QACd,6DAA6D;QAC7D,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;IAED,WAAW,CAAC,QAAiB;QACzB,2CAA2C;IAC/C,CAAC;IAED,OAAO;QACH,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YAAE,GAAG,CAAC,OAAO,EAAE,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;IACzB,CAAC;CACJ"}

package/dist/model/mamba1_block.d.ts

@@ -0,0 +1,70 @@
+/**
+ * mamba1_block.ts – Mamba-1 Mixer Block (S6 selective scan).
+ *
+ * Renamed from mamba_block.ts; MambaBlock is kept as a deprecated alias.
+ * Implements SequenceLayer so HybridMambaModel can iterate blocks generically.
+ */
+import type { SequenceLayer, LayerForwardResult, LayerParam } from './sequence_layer.js';
+export interface Mamba1BlockConfig {
+    dModel: number;
+    dState?: number;
+    dConv?: number;
+    expand?: number;
+    dtRank?: number;
+    biasConv?: boolean;
+}
+/** @deprecated Use LayerParam */
+export type BlockParam = LayerParam;
+export interface BlockCache {
+    normInv: GPUBuffer;
+    normIn: GPUBuffer;
+    normOut: GPUBuffer;
+    zBuf: GPUBuffer;
+    xConvIn: GPUBuffer;
+    convOut: GPUBuffer;
+    siluOut: GPUBuffer;
+    deltaFull: GPUBuffer;
+    B_raw: GPUBuffer;
+    C_raw: GPUBuffer;
+    hCache: GPUBuffer;
+}
+export interface BlockForwardResult extends LayerForwardResult {
+    output: GPUBuffer;
+    cache: BlockCache;
+}
+export declare class Mamba1Block implements SequenceLayer {
+    readonly layerType: "mamba1";
+    device: GPUDevice;
+    config: Required<Mamba1BlockConfig>;
+    dInner: number;
+    dtRank: number;
+    wInProj: Float32Array;
+    bInProj: Float32Array;
+    wConv: Float32Array;
+    bConv: Float32Array;
+    wXProj: Float32Array;
+    bXProj: Float32Array;
+    wDtProj: Float32Array;
+    bDtProj: Float32Array;
+    A_log: Float32Array;
+    D_vec: Float32Array;
+    wOutProj: Float32Array;
+    bOutProj: Float32Array;
+    normWeight: Float32Array;
+    gpuWeights: Record<string, GPUBuffer>;
+    pipelines: Record<string, GPUComputePipeline>;
+    private _wslaMode;
+    constructor(device: GPUDevice, config: Mamba1BlockConfig);
+    private _initWeights;
+    private _uploadWeightsToGPU;
+    private _buildPipelines;
+    forward(xBuf: GPUBuffer, batch: number, seqLen: number): BlockForwardResult;
+    parameters(): LayerParam[];
+    getTrainableParams(): LayerParam[];
+    setWSLAMode(enabled: boolean): void;
+    destroy(): void;
+}
+export { Mamba1Block as MambaBlock };
+/** @deprecated Use Mamba1BlockConfig */
+export type MambaBlockConfig = Mamba1BlockConfig;
+//# sourceMappingURL=mamba1_block.d.ts.map

package/dist/model/mamba1_block.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mamba1_block.d.ts","sourceRoot":"","sources":["../../src/model/mamba1_block.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAkBH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAK,MAAM,CAAC;IAClB,MAAM,CAAC,EAAI,MAAM,CAAC;IAClB,KAAK,CAAC,EAAK,MAAM,CAAC;IAClB,MAAM,CAAC,EAAI,MAAM,CAAC;IAClB,MAAM,CAAC,EAAI,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,iCAAiC;AACjC,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC;AAEpC,MAAM,WAAW,UAAU;IACvB,OAAO,EAAK,SAAS,CAAC;IACtB,MAAM,EAAM,SAAS,CAAC;IACtB,OAAO,EAAK,SAAS,CAAC;IACtB,IAAI,EAAQ,SAAS,CAAC;IACtB,OAAO,EAAK,SAAS,CAAC;IACtB,OAAO,EAAK,SAAS,CAAC;IACtB,OAAO,EAAK,SAAS,CAAC;IACtB,SAAS,EAAG,SAAS,CAAC;IACtB,KAAK,EAAO,SAAS,CAAC;IACtB,KAAK,EAAO,SAAS,CAAC;IACtB,MAAM,EAAM,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,kBAAmB,SAAQ,kBAAkB;IAC1D,MAAM,EAAG,SAAS,CAAC;IACnB,KAAK,EAAI,UAAU,CAAC;CACvB;AA8BD,qBAAa,WAAY,YAAW,aAAa;IAC7C,QAAQ,CAAC,SAAS,EAAG,QAAQ,CAAU;IAEvC,MAAM,EAAI,SAAS,CAAC;IACpB,MAAM,EAAI,QAAQ,CAAC,iBAAiB,CAAC,CAAC;IACtC,MAAM,EAAI,MAAM,CAAC;IACjB,MAAM,EAAI,MAAM,CAAC;IAEjB,OAAO,EAAK,YAAY,CAAC;IACzB,OAAO,EAAK,YAAY,CAAC;IACzB,KAAK,EAAO,YAAY,CAAC;IACzB,KAAK,EAAO,YAAY,CAAC;IACzB,MAAM,EAAM,YAAY,CAAC;IACzB,MAAM,EAAM,YAAY,CAAC;IACzB,OAAO,EAAK,YAAY,CAAC;IACzB,OAAO,EAAK,YAAY,CAAC;IACzB,KAAK,EAAO,YAAY,CAAC;IACzB,KAAK,EAAO,YAAY,CAAC;IACzB,QAAQ,EAAI,YAAY,CAAC;IACzB,QAAQ,EAAI,YAAY,CAAC;IACzB,UAAU,EAAE,YAAY,CAAC;IAEzB,UAAU,EAAG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACvC,SAAS,EAAI,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAEhD,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAiB;IAmCxD,OAAO,CAAC,YAAY;IAoCpB,OAAO,CAAC,mBAAmB;IAqB3B,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,kBAAkB;IAyK3E,UAAU,IAAI,UAAU,EAAE;IAwB1B,kBAAkB,IAAI,UAAU,EAAE;IAUlC,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAInC,OAAO,IAAI,IAAI;CAMlB;AAGD,OAAO,EAAE,WAAW,IAAI,UAAU,EAAE,CAAC;AAErC,wCAAwC;AACxC,MAAM,MAAM,gBAAgB,GAAG,iBAAiB,CAAC"}