@simulatte/doppler 0.1.5 → 0.1.7

package/src/inference/pipelines/text/ops.js

@@ -175,103 +175,103 @@ export async function doConv(
   }
 
   // Use the first 2x hidden projection channels as a gated conv-state projection.
-  const inProj = await doMatmul(
-    inputTensor,
-    convInProj,
-    numTokens,
-    hiddenSize * 2,
-    hiddenSize,
-    {
-      transposeB: 'auto',
-      label: `${label}.in_proj`,
-      layerIdx,
-      kernelPath,
-      role: 'conv_in_proj',
-    },
-    recorder
-  );
-  const activated = await doSiLURowSplit(inProj, {
-    numTokens,
-    dim: hiddenSize,
-    activation: 'silu',
-    swigluLimit: options.swigluLimit ?? null,
-    label: `${label}.activation`,
-    layerIdx,
-  }, recorder);
-
-  if (recorder) {
-    recorder.trackTemporaryBuffer(inProj.buffer);
-  } else {
-    releaseBuffer(inProj.buffer);
-  }
-
-  // Optional generic conv2d stage when explicit shape metadata is provided.
-  // LFM2 depthwise conv kernels use model-specific packing, so this path is best-effort only.
-  let convInput = activated;
-  if (convKernel && options.conv2d && options.conv2d.enabled === true) {
-    const convTensorInput = createTensor(activated.buffer, activated.dtype, [
-      options.conv2d.inChannels,
-      options.conv2d.height,
-      options.conv2d.width,
-    ], `${label}.conv_input`);
-    const convOptions = {
-      inChannels: options.conv2d.inChannels,
-      outChannels: options.conv2d.outChannels,
-      height: options.conv2d.height,
-      width: options.conv2d.width,
-      kernelH: options.conv2d.kernelH,
-      kernelW: options.conv2d.kernelW,
-      stride: options.conv2d.stride ?? 1,
-      pad: options.conv2d.pad ?? 0,
-    };
-    const convResult = recorder
-      ? await recordConv2D(recorder, convTensorInput, convKernel, null, convOptions)
-      : await runConv2D(convTensorInput, convKernel, null, convOptions);
-    convInput = createTensor(
-      convResult.buffer,
-      convResult.dtype,
-      [numTokens, hiddenSize],
-      `${label}.conv_output`
+  let inProj = null;
+  let activated = null;
+  let convInput = null;
+  let outProj = null;
+  try {
+    inProj = await doMatmul(
+      inputTensor,
+      convInProj,
+      numTokens,
+      hiddenSize * 2,
+      hiddenSize,
+      {
+        transposeB: 'auto',
+        label: `${label}.in_proj`,
+        layerIdx,
+        kernelPath,
+        role: 'conv_in_proj',
+      },
+      recorder
     );
-    if (recorder) {
-      recorder.trackTemporaryBuffer(activated.buffer);
-    } else {
-      releaseBuffer(activated.buffer);
+    activated = await doSiLURowSplit(inProj, {
+      numTokens,
+      dim: hiddenSize,
+      activation: 'silu',
+      swigluLimit: options.swigluLimit ?? null,
+      label: `${label}.activation`,
+      layerIdx,
+    }, recorder);
+
+    releaseOrTrack(recorder, inProj.buffer);
+    inProj = null;
+
+    convInput = activated;
+    if (convKernel && options.conv2d && options.conv2d.enabled === true) {
+      const convTensorInput = createTensor(activated.buffer, activated.dtype, [
+        options.conv2d.inChannels,
+        options.conv2d.height,
+        options.conv2d.width,
+      ], `${label}.conv_input`);
+      const convOptions = {
+        inChannels: options.conv2d.inChannels,
+        outChannels: options.conv2d.outChannels,
+        height: options.conv2d.height,
+        width: options.conv2d.width,
+        kernelH: options.conv2d.kernelH,
+        kernelW: options.conv2d.kernelW,
+        stride: options.conv2d.stride ?? 1,
+        pad: options.conv2d.pad ?? 0,
+      };
+      const convResult = recorder
+        ? await recordConv2D(recorder, convTensorInput, convKernel, null, convOptions)
+        : await runConv2D(convTensorInput, convKernel, null, convOptions);
+      convInput = createTensor(
+        convResult.buffer,
+        convResult.dtype,
+        [numTokens, hiddenSize],
+        `${label}.conv_output`
+      );
+      releaseOrTrack(recorder, activated.buffer);
+      activated = null;
     }
-  }
 
-  const outProj = await doMatmul(
-    convInput,
-    convOutProj,
-    numTokens,
-    hiddenSize,
-    hiddenSize,
-    {
-      transposeB: 'auto',
-      label: `${label}.out_proj`,
-      layerIdx,
-      kernelPath,
-      role: 'conv_out_proj',
-    },
-    recorder
-  );
+    outProj = await doMatmul(
+      convInput,
+      convOutProj,
+      numTokens,
+      hiddenSize,
+      hiddenSize,
+      {
+        transposeB: 'auto',
+        label: `${label}.out_proj`,
+        layerIdx,
+        kernelPath,
+        role: 'conv_out_proj',
+      },
+      recorder
+    );
 
-  if (convInput.buffer !== activated.buffer) {
-    if (recorder) {
-      recorder.trackTemporaryBuffer(convInput.buffer);
-    } else {
-      releaseBuffer(convInput.buffer);
+    if (convInput && (!activated || convInput.buffer !== activated.buffer)) {
+      releaseOrTrack(recorder, convInput.buffer);
+      convInput = null;
+    } else if (activated) {
+      releaseOrTrack(recorder, activated.buffer);
+      activated = null;
     }
-  } else if (recorder) {
-    recorder.trackTemporaryBuffer(activated.buffer);
-  } else {
-    releaseBuffer(activated.buffer);
-  }
 
-  if (kernelTrace.enabled && !recorder) {
-    await traceStep('conv', label, layerIdx, outProj.buffer, [numTokens, hiddenSize]);
+    if (kernelTrace.enabled && !recorder) {
+      await traceStep('conv', label, layerIdx, outProj.buffer, [numTokens, hiddenSize]);
+    }
+    return outProj;
+  } catch (error) {
+    if (outProj) releaseOrTrack(recorder, outProj.buffer);
+    if (convInput && (!activated || convInput.buffer !== activated.buffer)) releaseOrTrack(recorder, convInput.buffer);
+    if (activated) releaseOrTrack(recorder, activated.buffer);
+    if (inProj) releaseOrTrack(recorder, inProj.buffer);
+    throw error;
   }
-  return outProj;
 }
 
 export async function doCast(input, toDtype, recorder) {

package/src/inference/pipelines/text/probes.js

@@ -4,6 +4,7 @@ import { trace } from '../../../debug/index.js';
 import { getDevice } from '../../../gpu/device.js';
 import { allowReadback } from '../../../gpu/perf-guards.js';
 import { f16ToF32 } from '../../../loader/dtype-utils.js';
+import { readBufferSlice } from '../../../memory/buffer-pool.js';
 
 
 const STAGE_DEFAULT_CATEGORY = {
@@ -11,6 +12,11 @@ const STAGE_DEFAULT_CATEGORY = {
   // Attention stages (per-layer)
   attn_input: 'attn',
   attn_normed: 'attn',
+  linear_qkv_proj: 'attn',
+  linear_z_proj: 'attn',
+  linear_a_proj: 'attn',
+  linear_b_proj: 'attn',
+  linear_core_out: 'attn',
   q_proj: 'attn',
   k_proj: 'attn',
   v_proj: 'attn',
@@ -139,22 +145,16 @@ export async function runProbes(stage, buffer, options) {
         const alignedOffset = Math.floor(byteOffset / 4) * 4;
         const offsetWithinRead = byteOffset - alignedOffset;
         const readSize = 4; // Always read 4 bytes (aligned)
-        const staging =  (device).createBuffer({ size: readSize, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ });
-        const enc =  (device).createCommandEncoder();
-        enc.copyBufferToBuffer( (buffer), alignedOffset, staging, 0, readSize);
-         (device).queue.submit([enc.finish()]);
-        await staging.mapAsync(GPUMapMode.READ);
+        const readback = await readBufferSlice(buffer, alignedOffset, readSize);
         let value;
         if (dtype === 'f16') {
           // offsetWithinRead is 0 or 2 for F16 - extract correct u16
-          const u16Array = new Uint16Array(staging.getMappedRange().slice(0));
+          const u16Array = new Uint16Array(readback);
           const u16Index = offsetWithinRead / 2;
           value = f16ToF32(u16Array[u16Index]);
         } else {
-          value = new Float32Array(staging.getMappedRange().slice(0))[0];
+          value = new Float32Array(readback)[0];
         }
-        staging.unmap();
-        staging.destroy();
         values.push(`${dimIdx}=${value.toFixed(4)}`);
       }
 

package/src/inference/pipelines/text/weights.js

@@ -1,7 +1,7 @@
 
 
 import { getDevice } from '../../../gpu/device.js';
-import { acquireBuffer } from '../../../memory/buffer-pool.js';
+import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
 import { log } from '../../../debug/index.js';
 import { isWeightBuffer, isCpuWeightBuffer, tagBufferDtype } from '../../../gpu/weight-buffer.js';
 
@@ -53,9 +53,14 @@ export function getWeightBuffer(weight, label) {
   }
 
   const buf = acquireBuffer(data.byteLength, undefined, label);
-  device.queue.writeBuffer(buf, 0,  ( (data)));
-  tagBufferDtype(buf, bufferDtype);
-  return buf;
+  try {
+    device.queue.writeBuffer(buf, 0,  ( (data)));
+    tagBufferDtype(buf, bufferDtype);
+    return buf;
+  } catch (error) {
+    releaseBuffer(buf);
+    throw error;
+  }
 }
 
 
@@ -92,9 +97,14 @@ export function getNormWeightBuffer(weight, label, config, debugFlags) {
   }
 
   const buf = acquireBuffer(data.byteLength, undefined, label);
-  device.queue.writeBuffer(buf, 0,  ( (data)));
-  tagBufferDtype(buf, 'f32');
-  return buf;
+  try {
+    device.queue.writeBuffer(buf, 0,  ( (data)));
+    tagBufferDtype(buf, 'f32');
+    return buf;
+  } catch (error) {
+    releaseBuffer(buf);
+    throw error;
+  }
 }
 
 

package/src/inference/pipelines/text.js

@@ -6,7 +6,7 @@ import { configurePerfGuards } from '../../gpu/perf-guards.js';
 import { MoERouter } from '../moe-router.js';
 import { DecodeBufferManager } from '../decode-buffers.js';
 import { DecodeRing } from '../decode-ring.js';
-import { applyPipelineContexts } from './context.js';
+import { applyPipelineContexts, restorePipelineContexts } from './context.js';
 import { createInitializedPipeline } from './factory.js';
 
 // Pipeline sub-modules
@@ -44,6 +44,11 @@ import { getDopplerLoader } from '../../loader/doppler-loader.js';
 import { registerPipeline, getPipelineFactory } from './registry.js';
 import { selectRuleValue } from '../../rules/rule-registry.js';
 
+function destroyMoERouter(router) {
+  if (router && typeof router.destroy === 'function') {
+    router.destroy();
+  }
+}
 
 
 // ============================================================================
@@ -102,6 +107,8 @@ export class InferencePipeline extends PipelineState {
     this.manifest = manifest;
     this.decodeRing?.release();
     this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
+    destroyMoERouter(this.moeRouter);
+    this.moeRouter = null;
 
     const executionV0Runtime = applyExecutionV0RuntimeConfig({
       runtimeConfig: this.runtimeConfig,
@@ -299,9 +306,13 @@ export class InferencePipeline extends PipelineState {
     const maxSeqLen = config.maxSeqLen;
     const ropeBuffers = await initRoPEFrequencies({
       headDim: config.headDim,
+      rotaryDim: config.ropeRotaryDim,
       maxSeqLen,
       ropeTheta: config.ropeTheta,
       ropeLocalTheta: config.ropeLocalTheta,
+      mropeInterleaved: config.ropeInterleaved,
+      mropeSection: config.mropeSection,
+      partialRotaryFactor: config.partialRotaryFactor,
       ropeScale: config.ropeScale,
       ropeLocalScale: config.ropeLocalScale,
       ropeScalingType: config.ropeScalingType,
@@ -486,12 +497,15 @@ export class InferencePipeline extends PipelineState {
     this.expertWeights.clear();
     this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
     this.lora = null;
+    destroyMoERouter(this.moeRouter);
+    this.moeRouter = null;
     if (this.finitenessBuffer) {
       this.finitenessBuffer.destroy();
       this.finitenessBuffer = null;
     }
     this.isLoaded = false;
     this.currentSeqLen = 0;
+    restorePipelineContexts(this);
     log.info('Pipeline', 'Unloaded');
   }
 
@@ -529,6 +543,8 @@ export class InferencePipeline extends PipelineState {
   releaseGPUResources() {
     this.decodeBuffers?.release();
     this.decodeRing?.release();
+    destroyMoERouter(this.moeRouter);
+    this.moeRouter = null;
     if (this.finitenessBuffer) {
       this.finitenessBuffer.destroy();
       this.finitenessBuffer = null;

package/src/inference/speculative.d.ts

@@ -66,8 +66,8 @@ export interface SpeculativeConfig {
   enableTreeDraft: boolean;
   /** Temperature for draft sampling */
   temperature: number;
-  /** Optional deterministic seed for speculative sampling */
-  randomSeed?: number | null;
+  /** Deterministic seed for speculative sampling */
+  randomSeed: number;
 }
 
 /**

package/src/inference/speculative.js

@@ -10,22 +10,6 @@ function createRng(seed) {
   };
 }
 
-function createUnseededRng() {
-  let fallbackState = ((Date.now() >>> 0) ^ 0xa341316c) >>> 0;
-  return () => {
-    const cryptoApi = typeof globalThis !== 'undefined' ? globalThis.crypto : null;
-    if (cryptoApi && typeof cryptoApi.getRandomValues === 'function') {
-      const random = new Uint32Array(1);
-      cryptoApi.getRandomValues(random);
-      return random[0] / 4294967296;
-    }
-    fallbackState = (fallbackState + 0x6d2b79f5) | 0;
-    let t = Math.imul(fallbackState ^ (fallbackState >>> 15), 1 | fallbackState);
-    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
-    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
-  };
-}
-
 function coerceLogitsVector(value, label) {
   if (value instanceof Float32Array) {
     if (value.length === 0) {
@@ -110,6 +94,9 @@ export class SpeculativeDecoder {
     if (config.temperature == null) {
       throw new Error('SpeculativeDecoder requires temperature.');
     }
+    if (!Number.isFinite(config.randomSeed)) {
+      throw new Error('SpeculativeDecoder requires randomSeed.');
+    }
 
     assertTemperature(config.temperature, 'temperature');
     this.numDraftTokens = config.numDraftTokens;
@@ -117,8 +104,7 @@ export class SpeculativeDecoder {
     this.enableTreeDraft = config.enableTreeDraft;
     this.temperature = config.temperature;
 
-    const seed = Number.isFinite(config.randomSeed) ? Math.floor(config.randomSeed) : null;
-    this.random = seed === null ? createUnseededRng() : createRng(seed);
+    this.random = createRng(Math.floor(config.randomSeed));
   }
 
   setDraftModel(model) {

package/src/inference/test-harness.d.ts

@@ -74,7 +74,7 @@ export interface InitializeResult {
 /**
  * Discover available models from the catalog.json endpoint.
  *
- * @param fallbackModels - Models to return if catalog fails
+ * @param fallbackModels - Explicit fallback models to use when catalog fetch is unavailable
  * @returns Array of model info objects
  */
 export declare function discoverModels(

package/src/inference/test-harness.js

@@ -25,7 +25,7 @@ import {
 
 
 export async function discoverModels(
-  fallbackModels = ['gemma3-1b-q4', 'mistral-7b-q4', 'llama3-8b-q4']
+  fallbackModels
 ) {
   try {
     const resp = await fetch('/models/catalog.json');
@@ -40,10 +40,13 @@ export async function discoverModels(
         }));
       }
     }
-  } catch (e) {
-    // Catalog not available, use fallback
+  } catch (e) {}
+
+  if (Array.isArray(fallbackModels) && fallbackModels.length > 0) {
+    return fallbackModels.map((id) => ({ id, name: id }));
   }
-  return fallbackModels.map((id) => ({ id, name: id }));
+
+  throw new Error('discoverModels: failed to fetch /models/catalog.json and no explicit fallback model list was provided.');
 }
 
 // ============================================================================
@@ -238,7 +241,13 @@ export async function initializeInference(modelUrl, options = {}) {
       onProgress('hotswap', 0.05, 'Loading hot-swap manifest...');
       log(`Hot-swap: loading manifest ${hotSwapConfig.manifestUrl}`);
       const hotSwapManifest = await fetchHotSwapManifest(hotSwapConfig.manifestUrl);
-      const verification = await verifyHotSwapManifest(hotSwapManifest, hotSwapConfig);
+      const verification = await verifyHotSwapManifest(hotSwapManifest, hotSwapConfig, {
+        source: {
+          kind: 'remote',
+          isLocal: false,
+          url: hotSwapConfig.manifestUrl,
+        },
+      });
       if (!verification.ok) {
         throw new Error(`Hot-swap manifest rejected: ${verification.reason}`);
       }
@@ -309,6 +318,7 @@ export async function initializeInference(modelUrl, options = {}) {
   const pipeline = await createPipeline( ( (manifest)), {
     storage: { loadShard },
     gpu: { device },
+    runtime,
     baseUrl: modelUrl,
     onProgress: ( progress) => {
       const pct = 0.2 + progress.percent * 0.8;

package/src/inference/tokenizer.d.ts

@@ -46,11 +46,6 @@ export declare class Tokenizer {
    */
   initialize(manifest: ModelManifest, options?: TokenizerInitOptions): Promise<void>;
 
-  /**
-   * Infer HuggingFace model ID from manifest architecture
-   */
-  private _inferHuggingFaceModel(manifest: ModelManifest): string | null;
-
   /**
    * Encode text to token IDs
    */

package/src/inference/tokenizer.js

@@ -130,14 +130,12 @@ export class Tokenizer {
       );
     }
 
-    let hfModel = tokenizerConfig.hfModel;
+    let hfModel = tokenizerConfig.hfModel ?? tokenizerConfig.modelId ?? null;
     const allowArchFallback = tokenizerConfig.allowArchFallback === true;
     if (allowArchFallback && !hfModel) {
-      const inferred = this._inferHuggingFaceModel(manifest);
-      if (inferred) {
-        hfModel = inferred;
-        log.warn('Tokenizer', `Using inferred HuggingFace model: ${inferred}`);
-      }
+      throw new Error(
+        `[Tokenizer] tokenizer.allowArchFallback requires explicit tokenizer.hfModel or tokenizer.modelId for model "${modelId}".`
+      );
     }
 
     if (hfModel) {
@@ -212,23 +210,6 @@ export class Tokenizer {
 
     this.config = tokenizerConfig;
   }
-
-  
-  _inferHuggingFaceModel(manifest) {
-    const tokenizer = manifest?.tokenizer ?? {};
-    if (typeof tokenizer.modelId === 'string' && tokenizer.modelId.length > 0) {
-      return tokenizer.modelId;
-    }
-    if (typeof tokenizer.hfModel === 'string' && tokenizer.hfModel.length > 0) {
-      return tokenizer.hfModel;
-    }
-    if (typeof manifest?.modelId === 'string' && manifest.modelId.length > 0) {
-      return manifest.modelId;
-    }
-    return null;
-  }
-
-  
   encode(text) {
     if (!this.backend) {
       throw new Error('Tokenizer not initialized');

package/src/inference/tokenizers/bpe.js

@@ -21,8 +21,17 @@ export class BPETokenizer extends BaseTokenizer {
     });
   }
 
+  #resetState() {
+    this.#vocab.clear();
+    this.#reverseVocab.clear();
+    this.#merges = [];
+    this.#mergeRanks.clear();
+    this.vocabSize = 0;
+  }
+
   
   load(vocab, merges) {
+    this.#resetState();
     // Build vocab maps
     for (const [token, id] of Object.entries(vocab)) {
       this.#vocab.set(token, id);