@simulatte/doppler 0.1.9 → 0.2.0

package/src/loader/dtype-utils.d.ts

@@ -1,40 +0,0 @@
-/**
- * Dtype Utilities
- *
- * Data type conversion utilities for tensor loading.
- *
- * @module loader/dtype-utils
- */
-
-import type { TensorLocation } from './loader-types.js';
-
-/**
- * Convert F16 (half precision) to F32 (single precision)
- */
-export declare function f16ToF32(h: number): number;
-
-/**
- * Convert BF16 buffer to F32 on GPU
- */
-export declare function convertBF16ToF32GPU(
-  srcBuffer: GPUBuffer,
-  numElements: number,
-  name: string
-): Promise<GPUBuffer>;
-
-/**
- * Decide whether a quantized tensor should be dequantized directly to f16.
- * Uses manifest tensor roles (matmul, embedding, lm_head, router).
- */
-export declare function shouldDequantizeToF16(location: TensorLocation): boolean;
-
-/**
- * Apply layout metadata to a GPU buffer if the tensor has column-major storage.
- * Note: Layout is now tracked via WeightBuffer for matmul weights.
- * This function is kept for API compatibility but is a no-op for non-matmul weights (norms).
- */
-export declare function applyBufferLayout(
-  buffer: GPUBuffer,
-  _location: TensorLocation,
-  outputDtype?: string | null
-): GPUBuffer;

package/src/loader/dtype-utils.js

@@ -1,61 +0,0 @@
-
-import { selectRuleValue } from '../rules/rule-registry.js';
-import { tagBufferDtype } from '../gpu/weight-buffer.js';
-
-
-export function f16ToF32(h) {
-  const sign = (h >> 15) & 0x1;
-  const exp = (h >> 10) & 0x1f;
-  const mant = h & 0x3ff;
-
-  if (exp === 0) {
-    if (mant === 0) return sign ? -0 : 0;
-    const f = mant / 1024 * Math.pow(2, -14);
-    return sign ? -f : f;
-  }
-  if (exp === 31) {
-    return mant ? NaN : (sign ? -Infinity : Infinity);
-  }
-
-  const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
-  return sign ? -f : f;
-}
-
-
-export async function convertBF16ToF32GPU(srcBuffer, numElements, name) {
-  const { runBF16ToF32 } = await import('../gpu/kernels/cast.js');
-  const resultTensor = await runBF16ToF32(srcBuffer, [numElements], name);
-  return resultTensor.buffer;
-}
-
-
-export function shouldDequantizeToF16(location) {
-  const role = location?.role;
-  if (!role) {
-    throw new Error('Tensor role is required to determine dequantization target.');
-  }
-  return selectRuleValue('loader', 'weights', 'dequantizeToF16', { role }) === true;
-}
-
-
-function normalizeBufferDtype(locationDtype, outputDtype) {
-  const explicit = typeof outputDtype === 'string' ? outputDtype.toLowerCase() : null;
-  if (explicit) {
-    return explicit;
-  }
-  const location = typeof locationDtype === 'string' ? locationDtype.toUpperCase() : null;
-  if (!location) {
-    return null;
-  }
-  return selectRuleValue('loader', 'weights', 'floatLocationDtype', { locationDtype: location });
-}
-
-export function applyBufferLayout(buffer, location, outputDtype = null) {
-  // Layout tracking is carried by WeightBuffer. For raw GPUBuffer paths (norms),
-  // we still tag runtime dtype so kernels can choose correct weight interpretation.
-  const dtype = normalizeBufferDtype(location?.dtype ?? null, outputDtype);
-  if (dtype) {
-    tagBufferDtype(buffer, dtype);
-  }
-  return buffer;
-}

package/src/loader/embedding-loader.d.ts

@@ -1,56 +0,0 @@
-/**
- * Embedding Loader - Load embedding weights.
- *
- * Handles loading of token embedding weights with support for:
- * - GPU and CPU paths
- * - Large weight streaming
- * - F32 to F16 downcast
- * - WeightBuffer wrapping
- *
- * @module loader/embedding-loader
- */
-
-import type {
-  WeightBuffer,
-  WeightLayout,
-  CpuWeightBuffer,
-} from '../gpu/weight-buffer.js';
-import type { TensorLocation } from './loader-types.js';
-
-/** Tensor loading function signature */
-export type TensorLoader = (
-  name: string,
-  toGPU?: boolean,
-  silent?: boolean
-) => Promise<GPUBuffer | WeightBuffer | Float32Array | Uint8Array | null>;
-
-/**
- * Context required for embedding loading.
- */
-export interface EmbeddingLoaderContext {
-  /** Tensor locations map */
-  tensorLocations: Map<string, TensorLocation>;
-  /** Load a tensor by name */
-  loadTensor: TensorLoader;
-  /** Check if large weight should stream to CPU */
-  shouldStreamLargeWeight: (name: string, loc: TensorLocation, label: string) => boolean;
-  /** Resolve weight layout from location */
-  resolveWeightLayout: (loc: TensorLocation) => WeightLayout;
-  /** GPU buffers to track for cleanup */
-  gpuBuffers: Set<GPUBuffer>;
-  /** Keep F32 weights (skip downcast) */
-  keepF32Weights: boolean;
-  /** Preserve F32 embeddings when manifest quantization requires F32 embedding weights */
-  preserveF32Embeddings?: boolean;
-}
-
-/** Result of embedding loading */
-export type EmbeddingResult = GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
-
-/**
- * Load embedding weights.
- *
- * @param ctx - Embedding loader context
- * @returns Loaded embeddings or null if not found
- */
-export declare function loadEmbeddings(ctx: EmbeddingLoaderContext): Promise<EmbeddingResult>;

package/src/loader/embedding-loader.js

@@ -1,211 +0,0 @@
-
-
-import {
-  createWeightBuffer,
-  createCpuWeightBuffer,
-  isWeightBuffer,
-  getWeightDtype,
-  getLayout,
-} from '../gpu/weight-buffer.js';
-import { maybeDowncastToF16 } from './weight-downcast.js';
-import { getTensorNamesByRole } from './tensors/tensor-role.js';
-import { log } from '../debug/index.js';
-import { selectRuleValue } from '../rules/rule-registry.js';
-import { createTensor } from '../gpu/tensor.js';
-import { castF16ToF32 } from '../gpu/kernel-selector.js';
-import { releaseBuffer } from '../memory/buffer-pool.js';
-
-// ============================================================================
-// Constants
-// ============================================================================
-
-
-const EMBEDDING_ROLE = 'embedding';
-const EMBEDDING_GROUP = 'embed';
-
-function isGpuBufferInstance(value) {
-  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
-}
-
-// ============================================================================
-// Main Function
-// ============================================================================
-
-
-export async function loadEmbeddings(ctx) {
-  const embeddingNames = getTensorNamesByRole(ctx.tensorLocations, EMBEDDING_ROLE, EMBEDDING_GROUP);
-  const candidates = embeddingNames.length > 0
-    ? embeddingNames
-    : getTensorNamesByRole(ctx.tensorLocations, EMBEDDING_ROLE);
-
-  if (candidates.length === 0) {
-    throw new Error(
-      `[Loader] Embeddings not found. Expected tensor with role="${EMBEDDING_ROLE}"` +
-      ` and group="${EMBEDDING_GROUP}". Re-convert the model with tensor roles.`
-    );
-  }
-
-  for (const name of candidates) {
-    const loc = ctx.tensorLocations.get(name);
-    const shouldStream = loc ? ctx.shouldStreamLargeWeight(name, loc, 'Embedding') : false;
-
-    // Load tensor (to CPU if streaming, to GPU otherwise)
-    const tensor = await ctx.loadTensor(name, !shouldStream, true);
-
-    // Skip if not found
-    if (!tensor) continue;
-
-    // Handle streaming path (CPU)
-    if (shouldStream && !(tensor instanceof Float32Array)) {
-      throw new Error(
-        `[Loader] Embedding "${name}" too large for GPU and cannot be loaded on CPU (dtype=${loc?.dtype ?? 'unknown'}).`
-      );
-    }
-
-    // Handle valid tensor types
-    if (isGpuBufferInstance(tensor) || isWeightBuffer(tensor) || tensor instanceof Float32Array) {
-      const result = await processEmbeddingTensor(ctx, tensor, name, loc, shouldStream);
-      if (result) {
-        return result;
-      }
-    }
-  }
-
-  throw new Error(
-    `[Loader] Embeddings not found. Tried: ${candidates.join(', ')}`
-  );
-}
-
-// ============================================================================
-// Internal Helpers
-// ============================================================================
-
-
-async function processEmbeddingTensor(ctx, tensor, name, loc, shouldStream) {
-  log.info(
-    'Loader',
-    `Embeddings tensor loaded: name=${name}, hasShape=${!!loc?.shape}, ` +
-    `shape=${loc?.shape ? `[${loc.shape.join(',')}]` : 'none'}, isWeightBuffer=${isWeightBuffer(tensor)}`
-  );
-
-  // Preserve F32 embedding path when required by manifest/kernels.
-  // This also repairs legacy manifests where embeddings were stored as F16
-  // but loaded with an F32 gather kernel.
-  const promoted = await maybePromoteEmbeddingsToF32(ctx, tensor, name, loc);
-
-  // WeightBuffer already has layout set correctly from _loadTensor
-  if (isWeightBuffer(promoted)) {
-    return maybeDowncastEmbeddings(ctx, promoted, name, loc);
-  }
-
-  // Float32Array streaming path
-  if (promoted instanceof Float32Array && loc?.shape && shouldStream) {
-    const layout = ctx.resolveWeightLayout(loc);
-    
-    const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
-      locationDtype: loc.dtype,
-    });
-    const result = createCpuWeightBuffer(promoted, dtype, layout, loc.shape, name);
-    log.warn('Loader', `Embeddings stored on CPU for chunked gather (layout=${layout})`);
-    return result;
-  }
-
-  // Raw GPUBuffer - wrap with dtype/layout metadata
-  if (isGpuBufferInstance(promoted) && loc?.shape && loc.shape.length === 2) {
-    const layout = ctx.resolveWeightLayout(loc);
-    
-    const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
-      locationDtype: loc.dtype,
-    });
-    const wrapped = createWeightBuffer(promoted, dtype, layout, loc.shape, name);
-    log.info('Loader', `Wrapped embeddings as WeightBuffer (layout=${layout}, dtype=${dtype})`);
-    return maybeDowncastEmbeddings(ctx, wrapped, name, loc);
-  }
-
-  // Fall back to raw tensor
-  return maybeDowncastEmbeddings(ctx, promoted, name, loc);
-}
-
-async function maybePromoteEmbeddingsToF32(ctx, current, name, loc) {
-  if (!ctx.preserveF32Embeddings) return current;
-  if (current instanceof Float32Array) return current;
-
-  if (isWeightBuffer(current)) {
-    const dtype = getWeightDtype(current);
-    if (dtype !== 'f16') return current;
-    const elems = Math.floor(current.buffer.size / 2);
-    const inputTensor = createTensor(current.buffer, 'f16', [elems], `${name}_f16`);
-    const promoted = await castF16ToF32(inputTensor);
-    const shape = Array.from(current.shape);
-    const layout = getLayout(current) || 'row';
-    const wrapped = createWeightBuffer(promoted.buffer, 'f32', layout, shape, name);
-    ctx.gpuBuffers.add(promoted.buffer);
-    releaseBuffer(current.buffer);
-    return wrapped;
-  }
-
-  if (!isGpuBufferInstance(current)) return current;
-
-  const sourceDtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
-    locationDtype: loc?.dtype,
-  });
-  if (sourceDtype !== 'f16') return current;
-
-  const elems = Math.floor(current.size / 2);
-  const inputTensor = createTensor(current, 'f16', [elems], `${name}_f16`);
-  const promoted = await castF16ToF32(inputTensor);
-  ctx.gpuBuffers.add(promoted.buffer);
-  releaseBuffer(current);
-
-  if (loc?.shape && loc.shape.length === 2) {
-    const layout = ctx.resolveWeightLayout(loc);
-    return createWeightBuffer(promoted.buffer, 'f32', layout, loc.shape, name);
-  }
-
-  return promoted.buffer;
-}
-
-
-async function maybeDowncastEmbeddings(ctx, current, name, loc) {
-  // Can't downcast Float32Array or CpuWeightBuffer
-  if (current instanceof Float32Array) {
-    return current;
-  }
-
-  // Get current dtype
-  const dtype = isWeightBuffer(current)
-    ? current.dtype
-    : selectRuleValue('loader', 'weights', 'floatLocationDtype', {
-      locationDtype: loc?.dtype,
-    });
-
-  // Skip if not F32
-  if (dtype !== 'f32') {
-    return current;
-  }
-
-  // Get buffer for downcast
-  const buffer = isWeightBuffer(current) ? current.buffer : current;
-  const elems = buffer.size / 4;
-
-  // Attempt downcast
-  const keepF32 = ctx.keepF32Weights || ctx.preserveF32Embeddings === true;
-  const result = await maybeDowncastToF16(current, {
-    label: name,
-    keepF32,
-    dtype,
-    shape: isWeightBuffer(current)
-      ? Array.from(current.shape)
-      : (loc?.shape ?? [elems]),
-    layout: isWeightBuffer(current)
-      ? current.layout
-      : (loc ? ctx.resolveWeightLayout(loc) : 'row'),
-  });
-
-  if (result?.wasDowncast && result.newBuffer) {
-    ctx.gpuBuffers.add(result.newBuffer);
-    return result.buffer;
-  }
-
-  return current;
-}

package/src/loader/experts/expert-cache.d.ts

@@ -1,156 +0,0 @@
-/**
- * Expert LRU Cache for MoE Models
- *
- * Tracks expert residency in VRAM and implements LRU eviction
- * to manage memory pressure during inference.
- *
- * @module loader/expert-cache
- */
-
-import type { ExpertWeights } from '../weights.js';
-import type { ExpertCacheConfigSchema } from '../../config/schema/loading.schema.js';
-
-/**
- * Cache statistics
- */
-export interface CacheStats {
-  hits: number;
-  misses: number;
-  evictions: number;
-  currentSize: number;
-  maxSize: number;
-  expertCount: number;
-  hitRate: number;
-  inUseCount: number;
-  pinnedCount: number;
-  layerResidency: Array<{ layerIdx: number; bytes: number }>;
-}
-
-/**
- * Expert LRU Cache
- *
- * Manages expert weight residency in VRAM with LRU eviction policy.
- */
-export declare class ExpertCache {
-  /**
-   * Create expert cache
-   * @param maxBytes Maximum cache size in bytes (uses config default if not specified)
-   * @param config Expert cache configuration
-   */
-  constructor(maxBytes?: number, config?: ExpertCacheConfigSchema);
-
-  /**
-   * Update cache configuration at runtime.
-   */
-  configure(config: ExpertCacheConfigSchema, maxBytes?: number): void;
-
-  /**
-   * Auto-tune cache size based on available VRAM
-   * Call this after WebGPU is initialized
-   */
-  autoTune(): Promise<void>;
-
-  /**
-   * Get expert from cache
-   * @returns Expert weights or null if not in cache
-   */
-  get(layerIdx: number, expertIdx: number): ExpertWeights | null;
-
-  /**
-   * Put expert into cache
-   * @param weights Expert weights to cache
-   * @param sizeBytes Size of expert in bytes (for memory tracking)
-   */
-  put(layerIdx: number, expertIdx: number, weights: ExpertWeights, sizeBytes: number): void;
-
-  /**
-   * Check if expert is in cache
-   */
-  has(layerIdx: number, expertIdx: number): boolean;
-
-  /**
-   * Evict least recently used expert
-   * Skips experts that are in-use or pinned
-   * @returns true if an expert was evicted, false if all experts are protected
-   */
-  evictLRU(protectedKey?: string | null): boolean;
-
-  /**
-   * Mark expert as in-use (prevents eviction during inference)
-   */
-  markInUse(layerIdx: number, expertIdx: number): void;
-
-  /**
-   * Mark expert as no longer in use (allows eviction)
-   */
-  markNotInUse(layerIdx: number, expertIdx: number): void;
-
-  /**
-   * Clear all in-use markers (call after inference completes)
-   */
-  clearInUse(): void;
-
-  /**
-   * Pin expert (prevents eviction, for shared experts)
-   */
-  pinExpert(layerIdx: number, expertIdx: number): void;
-
-  /**
-   * Unpin expert (allows eviction)
-   */
-  unpinExpert(layerIdx: number, expertIdx: number): void;
-
-  /**
-   * Pin all shared experts for a model
-   */
-  pinSharedExperts(sharedExpertIndices: number[], numLayers: number): void;
-
-  /**
-   * Check if expert is pinned
-   */
-  isPinned(layerIdx: number, expertIdx: number): boolean;
-
-  /**
-   * Get current memory usage in bytes
-   */
-  getMemoryUsage(): number;
-
-  /**
-   * Get cache statistics
-   */
-  getStats(): CacheStats;
-
-  /**
-   * Clear all cached experts
-   */
-  clear(): void;
-
-  /**
-   * Set maximum cache size
-   * @param maxBytes New maximum size in bytes
-   */
-  setMaxSize(maxBytes: number): void;
-
-  /**
-   * Prefetch experts (hint for future access)
-   * This is a no-op in the cache - actual prefetch happens in the loader
-   */
-  prefetch(_layerIdx: number, _expertIndices: number[]): void;
-
-  /**
-   * Get all cached expert keys
-   */
-  getCachedExperts(): Array<{ layerIdx: number; expertIdx: number }>;
-}
-
-/**
- * Get global expert cache instance
- */
-export declare function getExpertCache(config?: ExpertCacheConfigSchema): ExpertCache;
-
-/**
- * Create new expert cache with custom size
- */
-export declare function createExpertCache(maxBytes?: number, config?: ExpertCacheConfigSchema): ExpertCache;
-
-export default ExpertCache;