@simulatte/doppler 0.1.8 → 0.2.0

package/src/inference/pipelines/text/sampling.js

@@ -1,249 +0,0 @@
-
-
-import { log, trace, isTraceEnabled } from '../../../debug/index.js';
-import { getRuntimeConfig } from '../../../config/runtime.js';
-
-function seededRandom(seed) {
-  const x = Math.sin(seed) * 10000;
-  return x - Math.floor(x);
-}
-
-let fallbackRandomState = (Date.now() >>> 0) || 0x6d2b79f5;
-
-function unseededRandom() {
-  if (typeof crypto !== 'undefined' && typeof crypto.getRandomValues === 'function') {
-    const values = new Uint32Array(1);
-    crypto.getRandomValues(values);
-    return values[0] / 4294967296;
-  }
-  fallbackRandomState = (fallbackRandomState + 0x6d2b79f5) >>> 0;
-  return fallbackRandomState / 4294967296;
-}
-
-
-export function applyRepetitionPenalty(logits, previousTokens, penalty) {
-  if (penalty === 1.0) return;
-
-  const windowSize = getRuntimeConfig().inference.sampling.repetitionPenaltyWindow;
-  const seen = new Set(previousTokens.slice(-windowSize));
-  for (const token of seen) {
-    if (token < logits.length) {
-      logits[token] = logits[token] > 0
-        ? logits[token] / penalty
-        : logits[token] * penalty;
-    }
-  }
-}
-
-
-export function softmax(logits) {
-  const n = logits.length;
-  let max = -Infinity;
-  for (let i = 0; i < n; i++) {
-    if (logits[i] > max) max = logits[i];
-  }
-
-  const exps = new Float32Array(n);
-  let sum = 0;
-  for (let i = 0; i < n; i++) {
-    const e = Math.exp(logits[i] - max);
-    exps[i] = e;
-    sum += e;
-  }
-
-  const invSum = sum > 0 ? 1 / sum : 0;
-  for (let i = 0; i < n; i++) {
-    exps[i] *= invSum;
-  }
-  return exps;
-}
-
-function countFiniteCandidates(logits, padTokenId) {
-  let finiteCandidateCount = 0;
-  for (let i = 0; i < logits.length; i++) {
-    if (padTokenId != null && i === padTokenId) {
-      continue;
-    }
-    if (Number.isFinite(logits[i])) {
-      finiteCandidateCount += 1;
-    }
-  }
-  return finiteCandidateCount;
-}
-
-function assertFiniteSamplingCandidates(logits, padTokenId, label) {
-  const finiteCandidateCount = countFiniteCandidates(logits, padTokenId);
-  if (finiteCandidateCount > 0) {
-    return;
-  }
-  throw new Error(
-    `[Sampling] ${label} has no finite candidate logits after masking the pad token. ` +
-    'Upstream decode likely produced NaN/Inf or an all-masked distribution.'
-  );
-}
-
-
-export function sample(logits, opts) {
-  const { temperature, topP, topK, decode, debug = false, padTokenId, seed } = opts;
-
-  if (padTokenId !== undefined && padTokenId >= 0 && padTokenId < logits.length) {
-    logits[padTokenId] = -Infinity;
-  }
-
-  assertFiniteSamplingCandidates(logits, padTokenId, 'Logits');
-
-  // Greedy (argmax) when temperature = 0
-  if (temperature === 0) {
-    let maxIdx = -1;
-    let maxVal = -Infinity;
-    for (let i = 0; i < logits.length; i++) {
-      const value = logits[i];
-      if (!Number.isFinite(value)) {
-        continue;
-      }
-      if (value > maxVal) {
-        maxVal = value;
-        maxIdx = i;
-      }
-    }
-    if (maxIdx < 0) {
-      throw new Error(
-        '[Sampling] Greedy sampling could not find a finite candidate logit. ' +
-        'Upstream decode likely produced NaN/Inf.'
-      );
-    }
-    if (debug) {
-      const text = decode?.([maxIdx]) ?? '?';
-      trace.sample(`Greedy: id=${maxIdx} "${text}" logit=${maxVal.toFixed(4)}`);
-    }
-    return maxIdx;
-  }
-
-  // Apply temperature
-  if (temperature !== 1.0) {
-    for (let i = 0; i < logits.length; i++) {
-      logits[i] /= temperature;
-    }
-  }
-
-  const probs = softmax(logits);
-
-  // Build candidate list
-
-  let candidates = [];
-  for (let i = 0; i < probs.length; i++) {
-    const probability = probs[i];
-    if (!Number.isFinite(probability) || probability <= 0) {
-      continue;
-    }
-    candidates.push({ token: i, prob: probability });
-  }
-  if (candidates.length === 0) {
-    throw new Error(
-      '[Sampling] Softmax produced no finite candidate probabilities. ' +
-      'Upstream decode likely produced NaN/Inf logits.'
-    );
-  }
-  candidates.sort((a, b) => b.prob - a.prob);
-
-  // Top-k filtering
-  if (topK > 0) {
-    candidates = candidates.slice(0, topK);
-  }
-
-  // Top-p (nucleus) filtering
-  if (topP < 1.0) {
-    let cumProb = 0;
-
-    const filtered = [];
-    for (const c of candidates) {
-      filtered.push(c);
-      cumProb += c.prob;
-      if (cumProb >= topP) break;
-    }
-    candidates = filtered;
-  }
-
-  // Renormalize with guard against zero sum
-  const probSum = candidates.reduce((s, c) => s + c.prob, 0);
-  if (probSum > 0) {
-    for (const c of candidates) {
-      c.prob /= probSum;
-    }
-  } else {
-    // If all probabilities are zero, fall back to uniform distribution
-    const uniformProb = 1.0 / candidates.length;
-    for (const c of candidates) {
-      c.prob = uniformProb;
-    }
-  }
-
-  if (debug) {
-    const top5 = candidates.slice(0, 5).map(c => {
-      const text = decode?.([c.token]) ?? '?';
-      return `"${text}"(${(c.prob * 100).toFixed(1)}%)`;
-    });
-    trace.sample(`Top-5 (temp=${temperature}, topK=${topK}, topP=${topP}): ${top5.join(', ')}`);
-  }
-
-  // Sample from distribution
-  const r = seed !== undefined ? seededRandom(seed) : unseededRandom();
-  let cumProb = 0;
-  for (const c of candidates) {
-    cumProb += c.prob;
-    if (r < cumProb) return c.token;
-  }
-
-  return candidates[candidates.length - 1].token;
-}
-
-
-export function getTopK(logits, k = 5, decode) {
-  const probs = softmax(new Float32Array(logits));
-
-
-  const indexed = [];
-  for (let i = 0; i < logits.length; i++) {
-    indexed.push({ token: i, logit: logits[i], prob: probs[i] });
-  }
-  indexed.sort((a, b) => b.logit - a.logit);
-
-  return indexed.slice(0, k).map(t => ({
-    token: t.token,
-    logit: t.logit,
-    prob: t.prob,
-    text: decode?.([t.token]) ?? `[${t.token}]`,
-  }));
-}
-
-
-export function logitsSanity(logits, label, decode) {
-  let min = Infinity;
-  let max = -Infinity;
-  let nanCount = 0;
-  let infCount = 0;
-
-  for (let i = 0; i < logits.length; i++) {
-    const v = logits[i];
-    if (Number.isNaN(v)) {
-      nanCount++;
-    } else if (!Number.isFinite(v)) {
-      infCount++;
-    } else {
-      if (v < min) min = v;
-      if (v > max) max = v;
-    }
-  }
-
-  const top5 = getTopK(logits, 5, decode);
-  if (isTraceEnabled('sample')) {
-    const top5Str = top5.map(t => `"${t.text}"(${(t.prob * 100).toFixed(1)}%)`).join(', ');
-    trace.sample(`${label} logits: min=${min.toFixed(2)}, max=${max.toFixed(2)} | top-5: ${top5Str}`);
-  }
-
-  if (nanCount > 0 || infCount > 0) {
-    log.warn('Sampling', `${label} logits have ${nanCount} NaN, ${infCount} Inf values`);
-  }
-
-  return { min, max, nanCount, infCount, top5 };
-}

package/src/inference/pipelines/text/state.d.ts

@@ -1,112 +0,0 @@
-/**
- * Pipeline State
- *
- * Holds the state of the inference pipeline:
- * - Model configuration and weights
- * - Runtime state (tokenizer, KV cache, etc.)
- * - Statistics
- *
- * @module inference/pipelines/text/state
- */
-
-import { Tokenizer } from '../../tokenizer.js';
-import { KVCache, SlidingWindowKVCache, TieredKVCache, BasisDecomposedPagedCache } from '../../kv-cache.js';
-import { MoERouter } from '../../moe-router.js';
-import { SpeculativeDecoder } from '../../speculative.js';
-import { DecodeBufferManager } from '../../decode-buffers.js';
-import { DecodeRing } from '../../decode-ring.js';
-import type { Manifest, ParsedModelConfig } from './config.js';
-import type { LayerWeights, ExpertWeights, RouterWeights, PipelineStats, BatchingStats } from './types.js';
-import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
-import type { DopplerLoader } from '../../../loader/doppler-loader.js';
-import type { CompiledLayerPipeline } from './layer-plan.js';
-import type { ExecutionPlanState } from './execution-plan.js';
-import type { CompiledExecutionV0RuntimeState } from './execution-v0.js';
-import type { LoRAAdapter } from './lora.js';
-import type { RuntimeConfigSchema, KernelPathSchema } from '../../../config/schema/index.js';
-import type { WeightDebugFlags } from './weights.js';
-import type { LogitsDebugFlags } from './logits/index.js';
-import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
-import type { EmulationContext } from '/proto/simulator/index.js';
-import type { PipelineStorageContext } from './init.js';
-import type { LinearAttentionRuntime } from './linear-attention.js';
-
-export class PipelineState {
-  // Components
-  tokenizer: Tokenizer | null;
-  kvCache: KVCache | SlidingWindowKVCache | TieredKVCache | BasisDecomposedPagedCache | null;
-  linearAttentionRuntime: LinearAttentionRuntime | null;
-  moeRouter: MoERouter | null;
-  speculativeDecoder: SpeculativeDecoder | null;
-  decodeBuffers: DecodeBufferManager | null;
-  decodeRing: DecodeRing | null;
-
-  // Emulation context (null when emulation is disabled)
-  emulation: EmulationContext | null;
-
-  // Debug flags (combined for both layer and logits)
-  debugFlags: WeightDebugFlags & LogitsDebugFlags;
-  decodeStepCount: number;
-  resolvedKernelPath: KernelPathSchema | null;
-  kernelPathSource: KernelPathSource;
-  executionPlanState: ExecutionPlanState | null;
-  executionV0State: CompiledExecutionV0RuntimeState | null;
-  disableRecordedLogits: boolean;
-  disableFusedDecode: boolean;
-
-  // Model state
-  manifest: Manifest | null;
-  modelConfig: ParsedModelConfig | null;
-  weights: Map<string, LayerWeights | GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null>;
-  expertWeights: Map<string, ExpertWeights>;
-
-  // Runtime state
-  isLoaded: boolean;
-  isGenerating: boolean;
-  currentSeqLen: number;
-  currentTokenIds: number[] | null;
-  runtimeConfig: RuntimeConfigSchema;
-
-  // DopplerLoader instance
-  dopplerLoader: DopplerLoader | null;
-
-  // GPU context
-  gpuContext: { device?: GPUDevice } | null;
-  useGPU: boolean;
-
-  // Memory and storage contexts
-  memoryContext: Record<string, unknown> | null;
-  storageContext: PipelineStorageContext | null;
-
-  // Stats
-  stats: PipelineStats;
-  batchingStats: BatchingStats;
-
-  // Base URL for loading assets
-  baseUrl: string | null;
-
-  // RoPE frequency buffers (global for full_attention layers)
-  ropeFreqsCos: Float32Array | GPUBuffer | null;
-  ropeFreqsSin: Float32Array | GPUBuffer | null;
-  // Local RoPE frequencies for sliding_attention layers (Gemma 3: 10K theta vs 1M global)
-  ropeLocalCos: Float32Array | GPUBuffer | null;
-  ropeLocalSin: Float32Array | GPUBuffer | null;
-
-  // Debug
-  debug: boolean;
-  // Optional layer pipeline plan (JSON-configured)
-  layerPipelinePlan: CompiledLayerPipeline | null;
-
-  // Tied embeddings
-  useTiedEmbeddings: boolean;
-  embeddingVocabSize: number | null;
-  embeddingTranspose: boolean;
-
-  // MoE router weights per layer
-  layerRouterWeights: Map<number, RouterWeights> | null;
-
-  // LoRA adapter (optional)
-  lora: LoRAAdapter | null;
-
-  constructor();
-}

package/src/inference/pipelines/text/state.js

@@ -1,152 +0,0 @@
-
-
-import { getRuntimeConfig } from '../../../config/runtime.js';
-
-export class PipelineState {
-  constructor() {
-    // Components
-
-    this.tokenizer = null;
-
-    this.kvCache = null;
-
-    this.linearAttentionRuntime = {
-      schemaVersion: 1,
-      layers: new Map(),
-    };
-
-    this.moeRouter = null;
-
-    this.speculativeDecoder = null;
-
-    this.decodeBuffers = null;
-
-    this.decodeRing = null;
-
-    this.finitenessBuffer = null;
-
-    // Emulation context (null when emulation is disabled).
-
-    this.emulation = null;
-
-    // Debug flags (combined for both layer and logits)
-
-    this.debugFlags = {};
-
-    this.decodeStepCount = 0;
-
-    this.resolvedKernelPath = null;
-
-    this.kernelPathSource = 'none';
-
-    this.executionPlanState = null;
-    this.executionV0State = null;
-
-    this.disableRecordedLogits = false;
-
-    this.disableFusedDecode = false;
-
-    // Model state
-
-    this.manifest = null;
-
-    this.modelConfig = null;
-
-    this.weights = new Map();
-
-    this.expertWeights = new Map();
-
-    // Runtime state
-
-    this.isLoaded = false;
-
-    this.isGenerating = false;
-
-    this.currentSeqLen = 0;
-
-    this.currentTokenIds = null;
-
-    this.runtimeConfig = getRuntimeConfig();
-
-    // DopplerLoader instance
-
-    this.dopplerLoader = null;
-
-    // GPU context
-
-    this.gpuContext = null;
-
-    this.useGPU = false;
-
-    // Memory and storage contexts
-
-    this.memoryContext = null;
-
-    this.storageContext = null;
-
-    // Stats
-
-    this.stats = {
-      prefillTimeMs: 0,
-      decodeTimeMs: 0,
-      ttftMs: 0,
-      prefillTokens: 0,
-      decodeTokens: 0,
-      memoryUsageBytes: 0,
-      tokensGenerated: 0,
-      totalTimeMs: 0,
-      decodeRecordMs: 0,
-      decodeSubmitWaitMs: 0,
-      decodeReadbackWaitMs: 0,
-      decodeProfileSteps: [],
-      attentionInputs: [],
-    };
-
-
-    this.batchingStats = {
-      batchedForwardCalls: 0,
-      unbatchedForwardCalls: 0,
-      totalBatchedTimeMs: 0,
-      totalUnbatchedTimeMs: 0,
-      gpuSubmissions: 0,
-    };
-
-    // Base URL for loading assets
-
-    this.baseUrl = null;
-
-    // RoPE frequency buffers (global for full_attention layers)
-
-    this.ropeFreqsCos = null;
-
-    this.ropeFreqsSin = null;
-    // Local RoPE frequencies for sliding_attention layers (different theta than global)
-
-    this.ropeLocalCos = null;
-
-    this.ropeLocalSin = null;
-
-    // Debug
-
-    this.debug = false;
-    // Optional layer pipeline plan (JSON-configured)
-
-    this.layerPipelinePlan = null;
-
-    // Tied embeddings
-
-    this.useTiedEmbeddings = false;
-
-    this.embeddingVocabSize = null;
-
-    this.embeddingTranspose = false;
-
-    // MoE router weights per layer
-
-    this.layerRouterWeights = null;
-
-    // LoRA adapter (optional)
-
-    this.lora = null;
-  }
-}