@simulatte/doppler 0.1.9 → 0.2.0

package/src/inference/kv-cache/types.d.ts

@@ -1,188 +0,0 @@
-/**
- * KV Cache Types - Shared interfaces and utilities
- *
- * @module inference/kv-cache/types
- */
-
-// ============================================================================
-// Configuration
-// ============================================================================
-
-/**
- * KV Cache Configuration
- */
-export interface KVCacheConfig {
-  numLayers: number;
-  numHeads: number;
-  headDim: number;
-  maxSeqLen: number;
-  useGPU: boolean;
-  layout: 'contiguous' | 'paged' | 'tiered' | 'bdpa' | 'bdpa_paged';
-  pageSize: number;
-  kvDtype: 'f16' | 'f32';
-  /** Window size for sliding window cache */
-  windowSize?: number;
-  /** Tiered KV cache settings (required when layout = 'tiered') */
-  tiering?: {
-    mode: 'off' | 'fp16' | 'int8' | 'int4';
-    hotWindow: number;
-    coldPageSize: number;
-    coldDtype: 'f16' | 'f32';
-    compression: { mode: 'none' | 'int8' | 'int4'; blockSize: number };
-    gating: { mode: 'auto' | 'force_on' | 'force_off'; minAluBwRatio: number };
-  };
-}
-
-// ============================================================================
-// Layer Cache Types
-// ============================================================================
-
-/**
- * Cache entry for a single layer (contiguous layout)
- */
-export interface ContiguousLayerCache {
-  keys: Float32Array;
-  values: Float32Array;
-  keysGPU: GPUBuffer | null;
-  valuesGPU: GPUBuffer | null;
-  seqLen: number;
-}
-
-/**
- * Cache entry for a single layer (paged layout)
- */
-export interface PagedLayerCache {
-  keyPages: (Float32Array | null)[];
-  valuePages: (Float32Array | null)[];
-  keysGPU?: GPUBuffer | null;
-  valuesGPU?: GPUBuffer | null;
-  pageTable?: Uint32Array | null;
-  pageTableGPU?: GPUBuffer | null;
-  allocatedPages: number;
-  seqLen: number;
-}
-
-/**
- * Union type for layer cache entries
- */
-export type LayerCache = ContiguousLayerCache | PagedLayerCache;
-
-// ============================================================================
-// Result Types
-// ============================================================================
-
-/**
- * Page location information
- */
-export interface PageLocation {
-  pageIdx: number;
-  offset: number;
-}
-
-/**
- * KV cache get result
- */
-export interface KVGetResult {
-  keys: Float32Array;
-  values: Float32Array;
-}
-
-/**
- * GPU buffers result (Contiguous or Paged)
- */
-export interface StandardGPUBuffersResult {
-  keysGPU: GPUBuffer;
-  valuesGPU: GPUBuffer;
-  seqLen: number;
-  pageTableGPU?: GPUBuffer;
-  pageSize?: number;
-}
-
-export interface TieredGPUBuffersResult {
-  layout: 'tiered';
-  seqLen: number;
-  hotKeysGPU: GPUBuffer;
-  hotValuesGPU: GPUBuffer;
-  hotSeqLen: number;
-  hotStart: number;
-  hotWindow: number;
-  coldKeysGPU: GPUBuffer;
-  coldValuesGPU: GPUBuffer;
-  coldScalesKGPU?: GPUBuffer;
-  coldScalesVGPU?: GPUBuffer;
-  coldSeqLen: number;
-  coldPageTableGPU?: GPUBuffer;
-  coldPageSize?: number;
-  coldPackedStride?: number;
-  coldQuantMode?: 'none' | 'int8' | 'int4';
-}
-
-export interface BDPAGPUBuffersResult {
-  layout: 'bdpa';
-  seqLen: number;
-  basisGPU: { k: GPUBuffer; v: GPUBuffer };
-  pagedGPU: { k: GPUBuffer; v: GPUBuffer };
-  indexGPU: GPUBuffer;
-  numBasisVectors: number;
-  pageSize: number;
-}
-
-export type GPUBuffersResult = StandardGPUBuffersResult | TieredGPUBuffersResult | BDPAGPUBuffersResult;
-
-/**
- * Memory statistics
- */
-export interface MemoryStats {
-  theoretical: number;
-  allocated: number;
-  used: number;
-  efficiency: number;
-  seqLen: number;
-  maxSeqLen: number;
-  layout: 'contiguous' | 'paged' | 'tiered' | 'bdpa_paged';
-}
-
-/**
- * GPU context for cache migration
- */
-export interface GPUContext {
-  device: GPUDevice;
-}
-
-// ============================================================================
-// Type Guards
-// ============================================================================
-
-/**
- * Type guard to check if layer is contiguous
- */
-export function isContiguousLayer(layer: LayerCache): layer is ContiguousLayerCache;
-
-/**
- * Type guard to check if layer is paged
- */
-export function isPagedLayer(layer: LayerCache): layer is PagedLayerCache;
-
-// ============================================================================
-// F16 Conversion Utilities
-// ============================================================================
-
-/**
- * Convert a single F32 value to F16 bits
- */
-export function f32ToF16Bits(value: number): number;
-
-/**
- * Convert F16 bits to F32 value
- */
-export function f16ToF32Bits(h: number): number;
-
-/**
- * Convert F32 array to F16 (Uint16Array)
- */
-export function f32ToF16Array(input: Float32Array): Uint16Array;
-
-/**
- * Convert F16 array to F32
- */
-export function f16ToF32Array(input: Uint16Array): Float32Array;

package/src/inference/kv-cache/types.js

@@ -1,80 +0,0 @@
-
-
-// ============================================================================
-// Type Guards
-// ============================================================================
-
-
-export function isContiguousLayer(layer) {
-  return 'keys' in layer && 'values' in layer;
-}
-
-
-export function isPagedLayer(layer) {
-  return 'keyPages' in layer && 'valuePages' in layer;
-}
-
-// ============================================================================
-// F16 Conversion Utilities
-// ============================================================================
-
-const f32View = new Float32Array(1);
-const u32View = new Uint32Array(f32View.buffer);
-
-
-export function f32ToF16Bits(value) {
-  f32View[0] = value;
-  const x = u32View[0];
-  const sign = (x >> 16) & 0x8000;
-  let exp = ((x >> 23) & 0xff) - 127 + 15;
-  let mant = x & 0x7fffff;
-
-  if (exp <= 0) {
-    if (exp < -10) return sign;
-    mant = (mant | 0x800000) >> (1 - exp);
-    return sign | ((mant + 0x1000) >> 13);
-  }
-
-  if (exp >= 31) {
-    return sign | 0x7c00 | (mant ? 0x200 : 0);
-  }
-
-  return sign | (exp << 10) | ((mant + 0x1000) >> 13);
-}
-
-
-export function f16ToF32Bits(h) {
-  const sign = (h >> 15) & 0x1;
-  const exp = (h >> 10) & 0x1f;
-  const mant = h & 0x3ff;
-
-  if (exp === 0) {
-    if (mant === 0) return sign ? -0 : 0;
-    const f = mant / 1024 * Math.pow(2, -14);
-    return sign ? -f : f;
-  }
-  if (exp === 31) {
-    return mant ? NaN : (sign ? -Infinity : Infinity);
-  }
-
-  const f = (1 + mant / 1024) * Math.pow(2, exp - 15);
-  return sign ? -f : f;
-}
-
-
-export function f32ToF16Array(input) {
-  const out = new Uint16Array(input.length);
-  for (let i = 0; i < input.length; i++) {
-    out[i] = f32ToF16Bits(input[i]);
-  }
-  return out;
-}
-
-
-export function f16ToF32Array(input) {
-  const out = new Float32Array(input.length);
-  for (let i = 0; i < input.length; i++) {
-    out[i] = f16ToF32Bits(input[i]);
-  }
-  return out;
-}

package/src/inference/kv-cache.d.ts

@@ -1,36 +0,0 @@
-/**
- * KV Cache Module - Re-export facade
- *
- * This file re-exports from the kv-cache/ directory for backward compatibility.
- * New code should import directly from 'inference/kv-cache/index.js'.
- *
- * @module inference/kv-cache
- */
-
-export {
-  // Types
-  type KVCacheConfig,
-  type ContiguousLayerCache,
-  type PagedLayerCache,
-  type LayerCache,
-  type PageLocation,
-  type KVGetResult,
-  type GPUBuffersResult,
-  type TieredGPUBuffersResult,
-  type BDPAGPUBuffersResult,
-  type MemoryStats,
-  type GPUContext,
-  isContiguousLayer,
-  isPagedLayer,
-  f32ToF16Bits,
-  f16ToF32Bits,
-  f32ToF16Array,
-  f16ToF32Array,
-  // Classes
-  KVCache,
-  SlidingWindowKVCache,
-  TieredKVCache,
-  BasisDecomposedPagedCache,
-  // Default
-  KVCache as default,
-} from './kv-cache/index.js';

package/src/inference/kv-cache.js

@@ -1,18 +0,0 @@
-
-
-export {
-  // Type guards and utility functions
-  isContiguousLayer,
-  isPagedLayer,
-  f32ToF16Bits,
-  f16ToF32Bits,
-  f32ToF16Array,
-  f16ToF32Array,
-  // Classes
-  KVCache,
-  SlidingWindowKVCache,
-  TieredKVCache,
-  BasisDecomposedPagedCache,
-  // Default
-  KVCache as default,
-} from './kv-cache/index.js';

package/src/inference/moe-router.d.ts

@@ -1,212 +0,0 @@
-/**
- * moe-router.ts - Mixture of Experts Router
- *
- * Implements top-k expert selection for Mixtral-style MoE models.
- * Handles gating network computation and expert selection.
- *
- * @module inference/moe-router
- */
-
-/**
- * MoE Router Configuration (extended)
- */
-export interface MoEConfig {
-  /** Total number of experts (e.g., 8 for Mixtral) */
-  numExperts: number;
-  /** Number of experts to select per token (e.g., 2) */
-  topK: number;
-  /** Hidden dimension size */
-  hiddenSize: number;
-  /** Whether to renormalize weights after top-k */
-  normalizeWeights?: boolean;
-}
-
-/**
- * Expert Selection Result for a single token
- */
-export interface ExpertSelection {
-  /** Selected expert indices */
-  indices: number[];
-  /** Corresponding weights for each selected expert */
-  weights: Float32Array;
-  /** Raw router logits (for auxiliary loss) */
-  routerLogits: Float32Array;
-}
-
-/**
- * GPU context for computations
- */
-interface GpuContext {
-  device: GPUDevice;
-}
-
-interface RouterLogitsOptions {
-  inputDtype?: 'f16' | 'f32';
-  outputDtype?: 'f16' | 'f32';
-}
-
-/**
- * Load balancing statistics
- */
-interface LoadBalanceStats {
-  expertCounts: Uint32Array;
-  totalTokens: number;
-}
-
-/**
- * Expert utilization stats
- */
-export interface UtilizationStats {
-  experts: Array<{
-    index: number;
-    count: number;
-    percentage: number;
-  }>;
-  totalTokens: number;
-  loadBalanceLoss: number;
-}
-
-export declare class MoERouter {
-  numExperts: number;
-  topK: number;
-  hiddenSize: number;
-  normalizeWeights: boolean;
-
-  // Router gate weights (linear projection: hidden_size -> num_experts)
-  // Will be loaded from model weights
-  gateWeight: Float32Array | GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer | null;
-  // Router bias (optional, used by GPT-OSS)
-  gateBias: Float32Array | GPUBuffer | null;
-
-  // Track active experts for the current batch
-  activeExperts: Set<number>;
-
-  // Auxiliary load balancing stats
-  loadBalanceStats: LoadBalanceStats;
-  lastLogitsDtype: 'f16' | 'f32';
-
-  constructor(config: MoEConfig);
-
-  /**
-   * Load router gate weights from model
-   * @param weights - Gate weight matrix [hidden_size, num_experts]
-   * @param bias - Optional gate bias vector [num_experts]
-   */
-  loadWeights(weights: Float32Array | GPUBuffer, bias?: Float32Array | GPUBuffer | null): void;
-
-  /**
-   * Compute router logits from hidden states (CPU fallback)
-   * @param hiddenStates - Input tensor [batchSize * seqLen, hiddenSize]
-   * @param numTokens - Number of tokens
-   * @returns Router logits [numTokens, numExperts]
-   */
-  computeRouterLogitsCPU(hiddenStates: Float32Array, numTokens: number): Float32Array;
-
-  /**
-   * Compute router logits using GPU (when available)
-   * @param hiddenStates - Input tensor on GPU [numTokens, hiddenSize]
-   * @param numTokens - Number of tokens
-   * @param gpuContext - GPU context (optional, uses global device if not provided)
-   * @returns Router logits on GPU [numTokens, numExperts]
-   */
-  computeRouterLogitsGPU(
-    hiddenStates: GPUBuffer,
-    numTokens: number,
-    gpuContext?: GpuContext | null,
-    options?: RouterLogitsOptions
-  ): Promise<GPUBuffer>;
-
-  /**
-   * Route tokens using GPU and read back results
-   * @param hiddenStates - Hidden states on GPU
-   * @param numTokens - Number of tokens
-   * @returns Expert selections for each token
-   */
-  routeGPU(hiddenStates: GPUBuffer, numTokens: number): Promise<ExpertSelection[]>;
-
-  /**
-   * Apply softmax to logits
-   * @param logits - Input logits
-   * @param size - Size of softmax dimension
-   * @returns Softmax probabilities
-   */
-  softmax(logits: Float32Array, size: number): Float32Array;
-
-  /**
-   * Select top-k experts for a single token
-   * @param logits - Router logits for one token [numExperts]
-   * @returns Selected experts with weights
-   */
-  selectExpertsForToken(logits: Float32Array): ExpertSelection;
-
-  /**
-   * Route a batch of tokens to experts
-   * @param hiddenStates - Input hidden states [numTokens, hiddenSize]
-   * @param numTokens - Number of tokens
-   * @returns Expert selections for each token
-   */
-  route(hiddenStates: Float32Array, numTokens: number): ExpertSelection[];
-
-  /**
-   * Get currently active expert indices
-   * @returns Array of active expert indices
-   */
-  getActiveExperts(): number[];
-
-  /**
-   * Compute auxiliary load balancing loss
-   * Used during training to encourage balanced expert utilization.
-   * @returns Load balancing loss value
-   */
-  computeLoadBalanceLoss(): number;
-
-  /**
-   * Reset load balancing statistics
-   */
-  resetStats(): void;
-
-  /**
-   * Get expert utilization statistics
-   * @returns Utilization stats per expert
-   */
-  getUtilizationStats(): UtilizationStats;
-}
-
-/**
- * Expert execution plan entry
- */
-interface ExpertExecutionPlanEntry {
-  tokenIndices: number[];
-  weights: Float32Array;
-}
-
-/**
- * Create a grouped expert execution plan
- * Groups tokens by their selected experts for efficient batched computation
- *
- * @param selections - Expert selections for all tokens
- * @param numExperts - Total number of experts
- * @returns Map of expert index to token indices and weights
- */
-export declare function createExpertExecutionPlan(
-  selections: ExpertSelection[],
-  numExperts: number
-): Map<number, ExpertExecutionPlanEntry>;
-
-/**
- * Combine expert outputs with routing weights
- *
- * @param expertOutputs - Output from each expert [numTokens, hiddenSize]
- * @param selections - Original routing decisions
- * @param numTokens - Number of tokens
- * @param hiddenSize - Hidden dimension
- * @returns Combined output [numTokens, hiddenSize]
- */
-export declare function combineExpertOutputs(
-  expertOutputs: Map<number, Float32Array>,
-  selections: ExpertSelection[],
-  numTokens: number,
-  hiddenSize: number
-): Float32Array;
-
-export default MoERouter;