@simulatte/doppler 0.1.8 → 0.2.0

package/src/loader/experts/expert-cache.js

@@ -1,382 +0,0 @@
-
-
-import { releaseBuffer } from '../../memory/buffer-pool.js';
-import { log, trace } from '../../debug/index.js';
-import { getRuntimeConfig } from '../../config/runtime.js';
-import { isWeightBuffer } from '../../gpu/weight-buffer.js';
-
-function isGpuBufferInstance(value) {
-  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
-}
-
-
-
-
-export class ExpertCache {
-  
-  #cache = new Map();
-
-  
-  #maxBytes;
-
-  
-  #currentBytes = 0;
-
-  
-  #accessCounter = 0;
-
-  
-  #config;
-
-  #layerResidency = new Map();
-
-  // Statistics
-  
-  #hits = 0;
-
-  
-  #misses = 0;
-
-  
-  #evictions = 0;
-
-  
-  #inUse = new Set();
-
-  
-  #pinned = new Set();
-
-  
-  constructor(maxBytes, config) {
-    this.#config = config ?? getRuntimeConfig().loading.expertCache;
-    this.#maxBytes = maxBytes ?? this.#config.defaultSizeBytes;
-  }
-
-  
-  configure(config, maxBytes) {
-    this.#config = config;
-    this.#maxBytes = maxBytes ?? config.defaultSizeBytes;
-  }
-
-  
-  async autoTune() {
-    const { defaultSizeBytes, maxBufferPercentage } = this.#config;
-    const defaultSizeMB = (defaultSizeBytes / 1024 / 1024).toFixed(0);
-
-    if (typeof navigator === 'undefined' || !('gpu' in navigator)) {
-      log.info('ExpertCache', `WebGPU not available, using default ${defaultSizeMB}MB`);
-      return;
-    }
-
-    try {
-      const adapter = await  (navigator).gpu.requestAdapter();
-      if (!adapter) {
-        log.info('ExpertCache', `No GPU adapter, using default ${defaultSizeMB}MB`);
-        return;
-      }
-
-      const limits = adapter.limits;
-      const maxBufferSize = limits?.maxBufferSize ?? this.#config.maxBufferFallbackBytes;
-
-      // Heuristic: Use up to default size or configured percentage of max buffer size, whichever is smaller
-      // This leaves room for model weights, KV cache, and activations
-      const autoSize = Math.min(
-        defaultSizeBytes,
-        Math.floor(maxBufferSize * maxBufferPercentage)
-      );
-
-      this.#maxBytes = autoSize;
-      log.info('ExpertCache', `Auto-tuned to ${(this.#maxBytes / 1024 / 1024).toFixed(0)}MB (maxBuffer: ${(maxBufferSize / 1024 / 1024).toFixed(0)}MB)`);
-    } catch (e) {
-      log.warn('ExpertCache', `Auto-tune failed, using default ${defaultSizeMB}MB:`, e);
-    }
-  }
-
-  
-  #getKey(layerIdx, expertIdx) {
-    return `${layerIdx}_${expertIdx}`;
-  }
-
-  
-  get(layerIdx, expertIdx) {
-    const key = this.#getKey(layerIdx, expertIdx);
-    const entry = this.#cache.get(key);
-
-    if (entry) {
-      // Update access time for LRU tracking
-      entry.lastAccess = ++this.#accessCounter;
-      this.#hits++;
-      return entry.weights;
-    }
-
-    this.#misses++;
-    return null;
-  }
-
-  
-  put(layerIdx, expertIdx, weights, sizeBytes) {
-    const key = this.#getKey(layerIdx, expertIdx);
-    const existing = this.#cache.get(key);
-    let existingSize = existing?.sizeBytes ?? 0;
-
-    const highWatermarkRatio = this.#config.evictionHighWatermark ?? 0.9;
-    const highWatermarkBytes = Math.floor(this.#maxBytes * highWatermarkRatio);
-    const trimRatio = this.#config.emergencyTrimToRatio ?? 0.75;
-    const trimTargetBytes = Math.floor(this.#maxBytes * trimRatio);
-
-    // If already in cache, update it
-    let projectedBytes = this.#currentBytes - existingSize + sizeBytes;
-    if (projectedBytes > highWatermarkBytes && this.#cache.size > 0) {
-      this.#evictUntil(Math.max(trimTargetBytes, highWatermarkBytes), key);
-      projectedBytes = this.#currentBytes - existingSize + sizeBytes;
-    }
-    while (projectedBytes > this.#maxBytes && this.#cache.size > 0) {
-      const evicted = this.evictLRU();
-      if (!evicted) {
-        log.warn('ExpertCache', `Cache full; unable to evict for ${key}. Skipping cache insert.`);
-        return;
-      }
-      if (!this.#cache.has(key)) {
-        existingSize = 0;
-      }
-      projectedBytes = this.#currentBytes - existingSize + sizeBytes;
-    }
-
-    // Add to cache
-    this.#cache.set(key, {
-      weights,
-      lastAccess: ++this.#accessCounter,
-      sizeBytes,
-      layerIdx,
-      expertIdx,
-    });
-    const deltaBytes = sizeBytes - existingSize;
-    this.#currentBytes += deltaBytes;
-    this.#addLayerResidency(layerIdx, deltaBytes);
-  }
-
-  
-  has(layerIdx, expertIdx) {
-    return this.#cache.has(this.#getKey(layerIdx, expertIdx));
-  }
-
-  
-  evictLRU(protectedKey = null) {
-    if (this.#cache.size === 0) return false;
-
-    
-    let lruKey = null;
-    let lruTime = Infinity;
-
-    for (const [key, entry] of this.#cache) {
-      if (protectedKey && key === protectedKey) continue;
-      // Skip in-use experts (currently being used in inference)
-      if (this.#inUse.has(key)) continue;
-      // Skip pinned experts (shared experts that should never be evicted)
-      if (this.#pinned.has(key)) continue;
-
-      if (entry.lastAccess < lruTime) {
-        lruTime = entry.lastAccess;
-        lruKey = key;
-      }
-    }
-
-    if (lruKey) {
-      this.#evict(lruKey);
-      return true;
-    }
-
-    // All experts are either in-use or pinned
-    return false;
-  }
-
-  
-  markInUse(layerIdx, expertIdx) {
-    this.#inUse.add(this.#getKey(layerIdx, expertIdx));
-  }
-
-  
-  markNotInUse(layerIdx, expertIdx) {
-    this.#inUse.delete(this.#getKey(layerIdx, expertIdx));
-  }
-
-  
-  clearInUse() {
-    this.#inUse.clear();
-  }
-
-  
-  pinExpert(layerIdx, expertIdx) {
-    this.#pinned.add(this.#getKey(layerIdx, expertIdx));
-  }
-
-  
-  unpinExpert(layerIdx, expertIdx) {
-    this.#pinned.delete(this.#getKey(layerIdx, expertIdx));
-  }
-
-  
-  pinSharedExperts(sharedExpertIndices, numLayers) {
-    for (let layer = 0; layer < numLayers; layer++) {
-      for (const expertIdx of sharedExpertIndices) {
-        this.pinExpert(layer, expertIdx);
-      }
-    }
-    log.info('ExpertCache', `Pinned ${sharedExpertIndices.length} shared experts across ${numLayers} layers`);
-  }
-
-  
-  isPinned(layerIdx, expertIdx) {
-    return this.#pinned.has(this.#getKey(layerIdx, expertIdx));
-  }
-
-  
-  #evict(key) {
-    const entry = this.#cache.get(key);
-    if (!entry) return;
-
-    // Release GPU buffers
-    this.#releaseExpertBuffers(entry.weights);
-
-    this.#currentBytes -= entry.sizeBytes;
-    this.#addLayerResidency(entry.layerIdx, -entry.sizeBytes);
-    this.#cache.delete(key);
-    this.#evictions++;
-
-    trace.loader(`Evicted expert ${key}, freed ${(entry.sizeBytes / 1024 / 1024).toFixed(1)}MB`);
-  }
-
-  
-  #releaseExpertBuffers(weights) {
-    const buffers = [
-      weights.gate,
-      weights.up,
-      weights.down,
-      weights.gateUpBlocks,
-      weights.gateUpScales,
-      weights.gateUpBias,
-      weights.downBlocks,
-      weights.downScales,
-      weights.downBias,
-    ];
-
-    for (const buf of buffers) {
-      const gpuBuffer = isWeightBuffer(buf)
-        ? buf.buffer
-        : (isGpuBufferInstance(buf) ? buf : null);
-      if (!gpuBuffer) continue;
-      try {
-        releaseBuffer(gpuBuffer);
-      } catch (e) {
-        // Buffer may already be released
-      }
-    }
-  }
-
-  
-  getMemoryUsage() {
-    return this.#currentBytes;
-  }
-
-  
-  getStats() {
-    const total = this.#hits + this.#misses;
-    const layerResidency = Array.from(this.#layerResidency.entries())
-      .map(([layerIdx, bytes]) => ({ layerIdx, bytes }))
-      .sort((a, b) => b.bytes - a.bytes);
-    return {
-      hits: this.#hits,
-      misses: this.#misses,
-      evictions: this.#evictions,
-      currentSize: this.#currentBytes,
-      maxSize: this.#maxBytes,
-      expertCount: this.#cache.size,
-      hitRate: total > 0 ? this.#hits / total : 0,
-      inUseCount: this.#inUse.size,
-      pinnedCount: this.#pinned.size,
-      layerResidency,
-    };
-  }
-
-  
-  clear() {
-    for (const [, entry] of this.#cache) {
-      this.#releaseExpertBuffers(entry.weights);
-    }
-    this.#cache.clear();
-    this.#layerResidency.clear();
-    this.#currentBytes = 0;
-    this.#inUse.clear();
-    // Note: pinned is NOT cleared - shared experts stay pinned
-    log.info('ExpertCache', 'Cache cleared');
-  }
-
-  
-  setMaxSize(maxBytes) {
-    this.#maxBytes = maxBytes;
-
-    // Evict if over new limit
-    while (this.#currentBytes > this.#maxBytes && this.#cache.size > 0) {
-      this.evictLRU();
-    }
-  }
-
-  
-  prefetch(_layerIdx, _expertIndices) {
-    // Prefetch hint - the loader should implement actual prefetch logic
-  }
-
-  
-  getCachedExperts() {
-    
-    const result = [];
-    for (const key of this.#cache.keys()) {
-      const [layer, expert] = key.split('_').map(Number);
-      result.push({ layerIdx: layer, expertIdx: expert });
-    }
-    return result;
-  }
-
-  #addLayerResidency(layerIdx, deltaBytes) {
-    if (!Number.isFinite(layerIdx)) return;
-    const prev = this.#layerResidency.get(layerIdx) ?? 0;
-    const next = prev + deltaBytes;
-    if (next <= 0) {
-      this.#layerResidency.delete(layerIdx);
-      return;
-    }
-    this.#layerResidency.set(layerIdx, next);
-  }
-
-  #evictUntil(targetBytes, protectedKey = null) {
-    while (this.#currentBytes > targetBytes && this.#cache.size > 0) {
-      const evicted = this.evictLRU(protectedKey);
-      if (!evicted) {
-        break;
-      }
-    }
-  }
-}
-
-
-let globalCache = null;
-
-
-export function getExpertCache(config) {
-  if (!globalCache) {
-    const resolvedConfig = config ?? getRuntimeConfig().loading.expertCache;
-    globalCache = new ExpertCache(undefined, resolvedConfig);
-  } else {
-    const resolvedConfig = config ?? getRuntimeConfig().loading.expertCache;
-    globalCache.configure(resolvedConfig);
-  }
-  return globalCache;
-}
-
-
-export function createExpertCache(maxBytes, config) {
-  const resolvedConfig = config ?? getRuntimeConfig().loading.expertCache;
-  return new ExpertCache(maxBytes, resolvedConfig);
-}
-
-export default ExpertCache;

package/src/loader/experts/expert-loader.d.ts

@@ -1,108 +0,0 @@
-/**
- * Expert Loader - MoE expert weight loading.
- *
- * Handles lazy loading of expert weights for Mixture-of-Experts models.
- * Supports both Mixtral-style (separate tensors) and GPT-OSS-style
- * (packed blocks) expert formats.
- *
- * @module loader/expert-loader
- */
-
-import type { RDRRManifest } from '../../formats/rdrr/index.js';
-import type { ExpertCache } from './expert-cache.js';
-import type { ExpertWeights } from '../weights.js';
-import type { WeightBuffer } from '../../gpu/weight-buffer.js';
-import type { ShardLoadOptions } from '../loader-types.js';
-
-/** Tensor loading function signature */
-export type TensorLoader = (
-  name: string,
-  toGPU?: boolean,
-  silent?: boolean
-) => Promise<GPUBuffer | WeightBuffer | Float32Array | Uint8Array | null>;
-
-/** Shard loading function signature */
-export type ShardLoader = (
-  shardIndex: number,
-  options?: ShardLoadOptions
-) => Promise<ArrayBuffer>;
-
-/** Shard cache interface */
-export interface ShardCacheInterface {
-  has(shardIndex: number): boolean;
-}
-
-/**
- * Context required for expert loading operations.
- */
-export interface ExpertLoaderContext {
-  /** Model manifest */
-  manifest: RDRRManifest | null;
-  /** Load a tensor by name */
-  loadTensor: TensorLoader;
-  /** Load a shard by index */
-  loadShard: ShardLoader;
-  /** Shard cache for checking loaded shards */
-  shardCache: ShardCacheInterface;
-  /** Expert LRU cache */
-  expertCache: ExpertCache | null;
-  /** Simple map for GPT-OSS packed experts */
-  experts: Map<string, ExpertWeights>;
-  /** GPU buffers to track for cleanup */
-  gpuBuffers: Set<GPUBuffer>;
-  /** Keep F32 weights (skip downcast) */
-  keepF32Weights: boolean;
-}
-
-/**
- * Pre-load specific shards for an expert (lazy loading support).
- *
- * @param ctx - Expert loader context
- * @param layerIdx - Layer index
- * @param expertIdx - Expert index
- */
-export declare function preloadShardsForExpert(
-  ctx: ExpertLoaderContext,
-  layerIdx: number,
-  expertIdx: number,
-  options?: ShardLoadOptions
-): Promise<void>;
-
-/**
- * Prefetch experts for next layer (overlap loading with compute).
- * Call this after router selects experts for current layer.
- *
- * @param ctx - Expert loader context
- * @param nextLayerIdx - Layer index to prefetch for
- * @param expertIndices - Expert indices likely to be used
- * @param isMoE - Whether model is MoE
- */
-export declare function prefetchExperts(
-  ctx: ExpertLoaderContext,
-  nextLayerIdx: number,
-  expertIndices: number[],
-  isMoE: boolean
-): void;
-
-/**
- * Get likely experts for next layer based on current layer's routing.
- * Simple heuristic: same experts tend to be selected across layers.
- *
- * @param currentExperts - Experts selected in current layer
- * @returns Predicted experts for next layer
- */
-export declare function predictNextLayerExperts(currentExperts: number[]): number[];
-
-/**
- * Load expert weights on demand (lazy loading from OPFS).
- *
- * @param ctx - Expert loader context
- * @param layerIdx - Layer index
- * @param expertIdx - Expert index
- * @returns Loaded expert weights
- */
-export declare function loadExpert(
-  ctx: ExpertLoaderContext,
-  layerIdx: number,
-  expertIdx: number
-): Promise<ExpertWeights>;