@simulatte/doppler 0.1.9 → 0.2.0

package/src/inference/decode-buffers.d.ts

@@ -1,108 +0,0 @@
-/**
- * Decode Buffer Manager
- *
- * Pre-allocates and reuses GPU buffers for the decode phase.
- * During decode (M=1), the same buffer sizes are needed every step.
- * Instead of acquiring from pool each time, we keep dedicated buffers.
- *
- * WebLLM-inspired optimization: decode uses fixed-size buffers that
- * can be reused across tokens without pool overhead.
- */
-
-/**
- * Pre-allocated buffers for decode operations
- */
-export interface DecodeBuffers {
-  /** Hidden state buffer (1 x hiddenSize) */
-  hidden: GPUBuffer;
-  /** Attention output buffer (1 x hiddenSize) */
-  attnOutput: GPUBuffer;
-  /** FFN intermediate buffer (1 x intermediateSize) */
-  ffnIntermediate: GPUBuffer;
-  /** Alternate hidden buffer for ping-pong (optional, for 2C) */
-  hiddenAlt?: GPUBuffer;
-}
-
-/**
- * Configuration for decode buffer sizes
- */
-export interface DecodeBufferConfig {
-  hiddenSize: number;
-  intermediateSize: number;
-  /** Enable ping-pong buffers (alternating between two hidden buffers) */
-  enablePingPong?: boolean;
-  /** Activation dtype for hidden buffers - 'f16' uses 2 bytes, 'f32' uses 4 bytes (default) */
-  activationDtype?: 'f16' | 'f32';
-}
-
-/**
- * Manages pre-allocated buffers for efficient decode operations.
- *
- * Usage:
- * 1. Call ensureBuffers() after model config is known
- * 2. Use getHiddenBuffer() to get decode hidden state buffer
- * 3. Call release() when done with generation
- */
-export declare class DecodeBufferManager {
-  private buffers;
-  private config;
-  private pingPongIndex;
-
-  /**
-   * Ensure buffers are allocated for the given config.
-   * No-op if already allocated with matching config.
-   */
-  ensureBuffers(config: DecodeBufferConfig): DecodeBuffers;
-
-  /**
-   * Get the current hidden state buffer.
-   * If ping-pong is enabled, returns the current input buffer.
-   */
-  getHiddenBuffer(): GPUBuffer | null;
-
-  /**
-   * Get the output hidden state buffer for next layer.
-   * If ping-pong is enabled, returns the alternate buffer.
-   */
-  getOutputHiddenBuffer(): GPUBuffer | null;
-
-  /**
-   * Swap ping-pong buffers (call after each layer).
-   */
-  swapPingPong(): void;
-
-  /**
-   * Reset ping-pong state (call at start of each decode step).
-   */
-  resetPingPong(): void;
-
-  /**
-   * Get attention output buffer.
-   */
-  getAttnOutputBuffer(): GPUBuffer | null;
-
-  /**
-   * Get FFN intermediate buffer.
-   */
-  getFFNIntermediateBuffer(): GPUBuffer | null;
-
-  /**
-   * Check if buffers are allocated.
-   */
-  hasBuffers(): boolean;
-
-  /**
-   * Get buffer sizes for debugging.
-   */
-  getStats(): { hiddenBytes: number; intermediateBytes: number; totalBytes: number; activationDtype: 'f16' | 'f32' } | null;
-
-  /**
-   * Check whether a buffer is managed by this decode buffer manager.
-   */
-  ownsBuffer(buffer: GPUBuffer): boolean;
-
-  /**
-   * Release all buffers.
-   */
-  release(): void;
-}

package/src/inference/decode-buffers.js

@@ -1,181 +0,0 @@
-
-
-import { getDevice } from '../gpu/device.js';
-import { selectRuleValue } from '../rules/rule-registry.js';
-import { padToQ4KBlock } from '../config/schema/index.js';
-
-
-
-
-export class DecodeBufferManager {
-  
-  buffers = null;
-
-  
-  config = null;
-
-  
-  pingPongIndex = 0;
-
-  
-  ensureBuffers(config) {
-    if (config.enablePingPong == null) {
-      throw new Error('DecodeBufferManager requires enablePingPong in config.');
-    }
-    if (config.activationDtype == null) {
-      throw new Error('DecodeBufferManager requires activationDtype in config.');
-    }
-    const normalizedConfig = { ...config };
-
-    // Check if we already have matching buffers
-    if (this.buffers && this.config &&
-        this.config.hiddenSize === normalizedConfig.hiddenSize &&
-        this.config.intermediateSize === normalizedConfig.intermediateSize &&
-        this.config.activationDtype === normalizedConfig.activationDtype &&
-        this.config.enablePingPong === normalizedConfig.enablePingPong) {
-      return this.buffers;
-    }
-
-    // Release old buffers if config changed
-    if (this.buffers) {
-      this.release();
-    }
-
-    const device = getDevice();
-    if (!device) {
-      throw new Error('GPU device not initialized');
-    }
-
-    // Allocate buffers
-    // For decode, we process 1 token at a time (M=1)
-    // F16 activations use 2 bytes per element, F32 uses 4 bytes
-    const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
-      dtype: normalizedConfig.activationDtype,
-    });
-
-    // Pad dimensions to Q4K super-block alignment for fused kernels.
-    // Q4K kernels process 256 elements per block and read out of bounds
-    // without padding. Extra padding elements remain zero (from WebGPU
-    // buffer init) since all kernels write exactly hiddenSize/intermediateSize.
-    const paddedHiddenSize = padToQ4KBlock(normalizedConfig.hiddenSize);
-    const paddedIntermediateSize = padToQ4KBlock(normalizedConfig.intermediateSize);
-
-    const hiddenBytes = paddedHiddenSize * bytesPerElement;
-    const intermediateBytes = paddedIntermediateSize * bytesPerElement;
-
-    const hidden = device.createBuffer({
-      label: 'decode_hidden',
-      size: hiddenBytes,
-      usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC,
-    });
-
-    const attnOutput = device.createBuffer({
-      label: 'decode_attn_output',
-      size: hiddenBytes,
-      usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC,
-    });
-
-    const ffnIntermediate = device.createBuffer({
-      label: 'decode_ffn_intermediate',
-      size: intermediateBytes,
-      usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC,
-    });
-
-    this.buffers = { hidden, attnOutput, ffnIntermediate };
-    this.config = normalizedConfig;
-
-    // Allocate alternate hidden buffer for ping-pong if enabled
-    if (normalizedConfig.enablePingPong) {
-      this.buffers.hiddenAlt = device.createBuffer({
-        label: 'decode_hidden_alt',
-        size: hiddenBytes,
-        usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC,
-      });
-    }
-
-    this.pingPongIndex = 0;
-
-    return this.buffers;
-  }
-
-  
-  getHiddenBuffer() {
-    if (!this.buffers) return null;
-    if (this.buffers.hiddenAlt && this.pingPongIndex === 1) {
-      return this.buffers.hiddenAlt;
-    }
-    return this.buffers.hidden;
-  }
-
-  
-  getOutputHiddenBuffer() {
-    if (!this.buffers) return null;
-    if (this.buffers.hiddenAlt) {
-      // Return the other buffer
-      return this.pingPongIndex === 0 ? this.buffers.hiddenAlt : this.buffers.hidden;
-    }
-    return this.buffers.hidden;
-  }
-
-  
-  swapPingPong() {
-    if (this.buffers?.hiddenAlt) {
-      this.pingPongIndex = 1 - this.pingPongIndex;
-    }
-  }
-
-  
-  resetPingPong() {
-    this.pingPongIndex = 0;
-  }
-
-  
-  getAttnOutputBuffer() {
-    return this.buffers?.attnOutput ?? null;
-  }
-
-  
-  getFFNIntermediateBuffer() {
-    return this.buffers?.ffnIntermediate ?? null;
-  }
-
-  
-  hasBuffers() {
-    return this.buffers !== null;
-  }
-
-  
-  getStats() {
-    if (!this.config) return null;
-    const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', {
-      dtype: this.config.activationDtype,
-    });
-    const hiddenBytes = this.config.hiddenSize * bytesPerElement;
-    const intermediateBytes = this.config.intermediateSize * bytesPerElement;
-    const bufferCount = this.buffers?.hiddenAlt ? 4 : 3;
-    const totalBytes = hiddenBytes * (bufferCount - 1) + intermediateBytes;
-    return { hiddenBytes, intermediateBytes, totalBytes, activationDtype: this.config.activationDtype };
-  }
-
-  
-  ownsBuffer(buffer) {
-    if (!this.buffers) return false;
-    return buffer === this.buffers.hidden
-      || buffer === this.buffers.hiddenAlt
-      || buffer === this.buffers.attnOutput
-      || buffer === this.buffers.ffnIntermediate;
-  }
-
-  
-  release() {
-    if (this.buffers) {
-      this.buffers.hidden.destroy();
-      this.buffers.attnOutput.destroy();
-      this.buffers.ffnIntermediate.destroy();
-      this.buffers.hiddenAlt?.destroy();
-      this.buffers = null;
-    }
-    this.config = null;
-    this.pingPongIndex = 0;
-  }
-}

package/src/inference/decode-ring.d.ts

@@ -1,52 +0,0 @@
-/**
- * Decode ring buffer pool for GPU batch decode.
- *
- * Preallocates fixed GPUBuffer rings for token outputs, stop flags, and staging readback.
- * Ring sizing derives from runtime batching config (batchSize × readbackInterval).
- */
-
-export interface DecodeRingConfig {
-  batchSize: number;
-  tokensPerInterval: number;
-  stopCheckMode: 'batch' | 'per-token';
-  ringTokens: number | null;
-  ringStop: number | null;
-  ringStaging: number | null;
-}
-
-export interface DecodeRingSlot {
-  index: number;
-  tokens: GPUBuffer | null;
-  stop: GPUBuffer | null;
-  stagingTokens: GPUBuffer | null;
-  stagingStop: GPUBuffer | null;
-  tokensPerInterval: number;
-  zeroStopData: Uint32Array | null;
-}
-
-export interface DecodeRingSlotStats {
-  allocated: number;
-  uses: number;
-  reuses: number;
-}
-
-export interface DecodeRingStats {
-  tokens: DecodeRingSlotStats;
-  stop: DecodeRingSlotStats;
-  stagingTokens: DecodeRingSlotStats;
-  stagingStop: DecodeRingSlotStats;
-  acquires: number;
-  advances: number;
-  resets: number;
-  ringSize: number;
-  tokensPerInterval: number;
-}
-
-export declare class DecodeRing {
-  ensure(config: DecodeRingConfig): void;
-  acquire(): DecodeRingSlot | null;
-  advance(): void;
-  reset(): void;
-  getStats(): DecodeRingStats | null;
-  release(): void;
-}

package/src/inference/decode-ring.js

@@ -1,273 +0,0 @@
-import { getDevice, getDeviceLimits } from '../gpu/device.js';
-
-const TOKEN_BYTES = 4;
-
-function clampRingSize(size) {
-  if (size == null) return 0;
-  if (!Number.isFinite(size) || size <= 0) {
-    throw new Error('DecodeRing requires positive ring sizes or null.');
-  }
-  return Math.floor(size);
-}
-
-function createSlotStats(allocated) {
-  return { allocated, uses: 0, reuses: 0 };
-}
-
-function resetSlotStats(stats) {
-  stats.uses = 0;
-  stats.reuses = 0;
-}
-
-function trackSlotUse(stats) {
-  stats.uses += 1;
-  if (stats.uses > stats.allocated) {
-    stats.reuses += 1;
-  }
-}
-
-function createRingStats(buffers, config, ringSize) {
-  return {
-    tokens: createSlotStats(buffers.tokens?.length ?? 0),
-    stop: createSlotStats(buffers.stop?.length ?? 0),
-    stagingTokens: createSlotStats(buffers.stagingTokens?.length ?? 0),
-    stagingStop: createSlotStats(buffers.stagingStop?.length ?? 0),
-    acquires: 0,
-    advances: 0,
-    resets: 0,
-    ringSize,
-    tokensPerInterval: config.tokensPerInterval,
-  };
-}
-
-function resetRingUsage(stats) {
-  if (!stats) return;
-  stats.acquires = 0;
-  stats.advances = 0;
-  stats.resets += 1;
-  resetSlotStats(stats.tokens);
-  resetSlotStats(stats.stop);
-  resetSlotStats(stats.stagingTokens);
-  resetSlotStats(stats.stagingStop);
-}
-
-function assertBufferFits(label, size, isStorage, limits) {
-  if (!limits) return;
-  const maxBufferSize = limits.maxBufferSize ?? Infinity;
-  if (size > maxBufferSize) {
-    throw new Error(`DecodeRing ${label} size ${size} exceeds maxBufferSize (${maxBufferSize}).`);
-  }
-  if (isStorage) {
-    const maxStorageSize = limits.maxStorageBufferBindingSize ?? Infinity;
-    if (size > maxStorageSize) {
-      throw new Error(
-        `DecodeRing ${label} size ${size} exceeds maxStorageBufferBindingSize (${maxStorageSize}).`
-      );
-    }
-  }
-}
-
-function sameConfig(a, b) {
-  if (!a || !b) return false;
-  return a.batchSize === b.batchSize
-    && a.tokensPerInterval === b.tokensPerInterval
-    && a.stopCheckMode === b.stopCheckMode
-    && a.ringTokens === b.ringTokens
-    && a.ringStop === b.ringStop
-    && a.ringStaging === b.ringStaging;
-}
-
-export class DecodeRing {
-  buffers = null;
-  config = null;
-  index = 0;
-  ringSize = 0;
-  zeroStopData = null;
-  stats = null;
-
-  ensure(config) {
-    if (!config) {
-      throw new Error('DecodeRing requires config.');
-    }
-    if (!Number.isFinite(config.batchSize) || config.batchSize <= 0) {
-      throw new Error('DecodeRing requires positive batchSize.');
-    }
-    if (!Number.isFinite(config.tokensPerInterval) || config.tokensPerInterval <= 0) {
-      throw new Error('DecodeRing requires positive tokensPerInterval.');
-    }
-    if (!config.stopCheckMode) {
-      throw new Error('DecodeRing requires stopCheckMode.');
-    }
-
-    const normalized = {
-      batchSize: Math.floor(config.batchSize),
-      tokensPerInterval: Math.floor(config.tokensPerInterval),
-      stopCheckMode: config.stopCheckMode,
-      ringTokens: clampRingSize(config.ringTokens),
-      ringStop: clampRingSize(config.ringStop),
-      ringStaging: clampRingSize(config.ringStaging),
-    };
-
-    if (this.buffers && sameConfig(this.config, normalized)) {
-      return;
-    }
-
-    this.release();
-
-    const device = getDevice();
-    if (!device) {
-      throw new Error('GPU device not initialized');
-    }
-    const limits = getDeviceLimits();
-
-    const tokensBytes = (normalized.tokensPerInterval + 1) * TOKEN_BYTES;
-    const stopBytes = (normalized.tokensPerInterval + 1) * TOKEN_BYTES;
-    const stagingBytes = normalized.tokensPerInterval * TOKEN_BYTES;
-
-    assertBufferFits('tokens', tokensBytes, true, limits);
-    assertBufferFits('stagingTokens', stagingBytes, false, limits);
-    if (normalized.stopCheckMode === 'per-token') {
-      assertBufferFits('stop', stopBytes, true, limits);
-      assertBufferFits('stagingStop', stagingBytes, false, limits);
-    }
-
-    const buffers = {
-      tokens: null,
-      stop: null,
-      stagingTokens: null,
-      stagingStop: null,
-    };
-
-    if (normalized.ringTokens > 0) {
-      buffers.tokens = Array.from({ length: normalized.ringTokens }, (_, i) => (
-        device.createBuffer({
-          label: `decode_ring_tokens_${i}`,
-          size: tokensBytes,
-          usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST,
-        })
-      ));
-    }
-
-    if (normalized.ringStaging > 0) {
-      buffers.stagingTokens = Array.from({ length: normalized.ringStaging }, (_, i) => (
-        device.createBuffer({
-          label: `decode_ring_staging_tokens_${i}`,
-          size: stagingBytes,
-          usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
-        })
-      ));
-    }
-
-    if (normalized.stopCheckMode === 'per-token' && normalized.ringStop > 0) {
-      buffers.stop = Array.from({ length: normalized.ringStop }, (_, i) => (
-        device.createBuffer({
-          label: `decode_ring_stop_${i}`,
-          size: stopBytes,
-          usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST,
-        })
-      ));
-    }
-
-    if (normalized.stopCheckMode === 'per-token' && normalized.ringStaging > 0) {
-      buffers.stagingStop = Array.from({ length: normalized.ringStaging }, (_, i) => (
-        device.createBuffer({
-          label: `decode_ring_staging_stop_${i}`,
-          size: stagingBytes,
-          usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
-        })
-      ));
-    }
-
-    this.buffers = buffers;
-    this.config = normalized;
-    this.index = 0;
-    this.ringSize = Math.max(
-      1,
-      normalized.ringTokens,
-      normalized.ringStop,
-      normalized.ringStaging
-    );
-    this.zeroStopData = normalized.stopCheckMode === 'per-token'
-      ? new Uint32Array(normalized.tokensPerInterval + 1)
-      : null;
-    this.stats = createRingStats(buffers, normalized, this.ringSize);
-  }
-
-  acquire() {
-    if (!this.buffers || !this.config) return null;
-    const idx = this.index;
-    const tokens = this.buffers.tokens
-      ? this.buffers.tokens[idx % this.buffers.tokens.length]
-      : null;
-    const stop = this.buffers.stop
-      ? this.buffers.stop[idx % this.buffers.stop.length]
-      : null;
-    const stagingTokens = this.buffers.stagingTokens
-      ? this.buffers.stagingTokens[idx % this.buffers.stagingTokens.length]
-      : null;
-    const stagingStop = this.buffers.stagingStop
-      ? this.buffers.stagingStop[idx % this.buffers.stagingStop.length]
-      : null;
-
-    if (this.stats) {
-      this.stats.acquires += 1;
-      if (tokens) trackSlotUse(this.stats.tokens);
-      if (stop) trackSlotUse(this.stats.stop);
-      if (stagingTokens) trackSlotUse(this.stats.stagingTokens);
-      if (stagingStop) trackSlotUse(this.stats.stagingStop);
-    }
-
-    return {
-      index: idx,
-      tokens,
-      stop,
-      stagingTokens,
-      stagingStop,
-      tokensPerInterval: this.config.tokensPerInterval,
-      zeroStopData: this.zeroStopData,
-    };
-  }
-
-  advance() {
-    if (!this.buffers) return;
-    this.index = (this.index + 1) % this.ringSize;
-    if (this.stats) {
-      this.stats.advances += 1;
-    }
-  }
-
-  reset() {
-    this.index = 0;
-    resetRingUsage(this.stats);
-  }
-
-  getStats() {
-    if (!this.stats) return null;
-    return {
-      tokens: { ...this.stats.tokens },
-      stop: { ...this.stats.stop },
-      stagingTokens: { ...this.stats.stagingTokens },
-      stagingStop: { ...this.stats.stagingStop },
-      acquires: this.stats.acquires,
-      advances: this.stats.advances,
-      resets: this.stats.resets,
-      ringSize: this.stats.ringSize,
-      tokensPerInterval: this.stats.tokensPerInterval,
-    };
-  }
-
-  release() {
-    if (this.buffers) {
-      this.buffers.tokens?.forEach((buffer) => buffer.destroy());
-      this.buffers.stop?.forEach((buffer) => buffer.destroy());
-      this.buffers.stagingTokens?.forEach((buffer) => buffer.destroy());
-      this.buffers.stagingStop?.forEach((buffer) => buffer.destroy());
-    }
-    this.buffers = null;
-    this.config = null;
-    this.index = 0;
-    this.ringSize = 0;
-    this.zeroStopData = null;
-    this.stats = null;
-  }
-}

package/src/inference/expert-router.d.ts

@@ -1,27 +0,0 @@
-/**
- * Expert router for multi-model execution.
- *
- * @module inference/expert-router
- */
-
-export interface ExpertProfile {
-  id: string;
-  embedding?: number[];
-  metadata?: Record<string, unknown>;
-}
-
-export declare class ExpertRouter {
-  private experts;
-
-  constructor();
-
-  registerExpert(profile: ExpertProfile): void;
-
-  removeExpert(id: string): void;
-
-  listExperts(): ExpertProfile[];
-
-  selectByEmbedding(embedding: number[], topK?: number): ExpertProfile[];
-
-  private cosineSimilarity(a: number[], b: number[]): number;
-}

package/src/inference/expert-router.js

@@ -1,55 +0,0 @@
-
-
-
-
-export class ExpertRouter {
-  
-  experts;
-
-  constructor() {
-    this.experts = new Map();
-  }
-
-  
-  registerExpert(profile) {
-    this.experts.set(profile.id, profile);
-  }
-
-  
-  removeExpert(id) {
-    this.experts.delete(id);
-  }
-
-  
-  listExperts() {
-    return Array.from(this.experts.values());
-  }
-
-  
-  selectByEmbedding(embedding, topK = 1) {
-    
-    const scored = [];
-    for (const expert of this.experts.values()) {
-      if (!expert.embedding) continue;
-      const score = this.cosineSimilarity(embedding, expert.embedding);
-      scored.push({ expert, score });
-    }
-    scored.sort((a, b) => b.score - a.score);
-    return scored.slice(0, topK).map((item) => item.expert);
-  }
-
-  
-  cosineSimilarity(a, b) {
-    if (!a || !b || a.length !== b.length) return 0;
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-      dot += a[i] * b[i];
-      normA += a[i] * a[i];
-      normB += b[i] * b[i];
-    }
-    const denom = Math.sqrt(normA) * Math.sqrt(normB);
-    return denom === 0 ? 0 : dot / denom;
-  }
-}

package/src/inference/functiongemma.d.ts

@@ -1,15 +0,0 @@
-/**
- * FunctionGemma Primitives
- *
- * Doppler provides execution primitives only. Orchestration lives in the host app.
- *
- * @module inference/functiongemma
- * @see ../../docs/architecture.md#engine-vs-orchestrator-boundary
- */
-export {
-  MultiModelNetwork,
-  type ExpertNode,
-  type CombinerConfig,
-  type ExpertTask,
-  type TopologyRouter,
-} from './multi-model-network.js';