@simulatte/doppler 0.1.0

package/src/config/schema/inference-defaults.schema.d.ts

@@ -0,0 +1,276 @@
+/**
+ * Inference Defaults Config Schema
+ *
+ * Default values for inference pipeline: batching, sampling, generation.
+ * These defaults are used when no model-specific or user overrides are provided.
+ *
+ * Note: SamplingDefaultsSchema provides defaults for fields from SamplingSchema
+ * (in inference.schema.js), plus greedyThreshold and repetitionPenaltyWindow
+ * which are unique to defaults.
+ *
+ * @module config/schema/inference-defaults
+ */
+
+import type {
+  ChatTemplateSchema,
+  InferenceConfigSchema,
+  LayerPipelineSchema,
+  SamplingSchema,
+  TokenizerConfigSchema,
+} from './inference.schema.js';
+import type {
+  ExecutionV0PatchSchema,
+  ExecutionV0SessionDefaultsSchema,
+} from './execution-v0.schema.js';
+import type { KVCacheConfigSchema } from './kvcache.schema.js';
+import type { MoERuntimeConfigSchema } from './moe.schema.js';
+import type { KernelPathRef } from './kernel-path.schema.js';
+import type { ManifestInferenceSchema } from './manifest.schema.js';
+import type { SpeculativeConfigSchema } from './speculative.schema.js';
+import type { DiffusionConfigSchema } from './diffusion.schema.js';
+import type { EnergyConfigSchema } from './energy.schema.js';
+
+// Re-export for convenience
+export type { ManifestInferenceSchema };
+
+/**
+ * Deep partial type for runtime overrides of model-specific inference config.
+ * Allows overriding any nested field in ManifestInferenceSchema.
+ */
+export type ModelInferenceOverrides = {
+  [P in keyof ManifestInferenceSchema]?: ManifestInferenceSchema[P] extends object
+    ? { [K in keyof ManifestInferenceSchema[P]]?: ManifestInferenceSchema[P][K] }
+    : ManifestInferenceSchema[P];
+};
+
+/**
+ * Default batching configuration for inference.
+ *
+ * Controls how tokens are batched during generation.
+ */
+export interface BatchingDefaultsSchema {
+  /** Number of sequences to process in parallel (default: 1) */
+  batchSize: number;
+
+  /** Maximum tokens to generate per sequence (default: 512) */
+  maxTokens: number;
+
+  /** When to check for stop conditions: per-token or per-batch (default: 'batch') */
+  stopCheckMode: 'per-token' | 'batch';
+
+  /**
+   * Number of GPU decode batches to run before readback.
+   * null means read back after every batch (default: 1).
+   */
+  readbackInterval: number | null;
+
+  /** Ring size for token output buffers (null = disable ring) */
+  ringTokens: number | null;
+
+  /** Ring size for per-token stop buffers (null = disable ring) */
+  ringStop: number | null;
+
+  /** Ring size for staging readback buffers (null = disable ring) */
+  ringStaging: number | null;
+}
+
+/** Default batching configuration */
+export declare const DEFAULT_BATCHING_DEFAULTS: BatchingDefaultsSchema;
+
+/**
+ * Default compute precision configuration.
+ *
+ * Controls dtype for intermediate activations and compute operations.
+ * F16 reduces memory bandwidth by 2x but may have precision implications.
+ */
+export interface ComputeDefaultsSchema {
+  /** Dtype for hidden state activations (default: 'f32', experimental: 'f16') */
+  activationDtype: 'f16' | 'f32';
+
+  /** Parameter count threshold for "large model" classification (default: 4e9 = 4B params) */
+  largeModelParamThreshold: number;
+
+  /** Multiplier for estimating model params from hidden^2 × layers (default: 12) */
+  paramEstimationMultiplier: number;
+
+  /** Keep weights in F32 (skip downcast even when F16 is available) */
+  keepF32Weights: boolean;
+
+  /**
+   * Range-aware selective widening policy used by finiteness guard.
+   * When enabled, decode checks for abs(value) > absThreshold and widens
+   * dangerous tokens to F32 fallback before KV corruption.
+   */
+  rangeAwareSelectiveWidening: RangeAwareSelectiveWideningDefaultsSchema;
+
+  /**
+   * Number of decode tokens to keep in widened (F32) mode after an overflow
+   * hit before restoring the original dtype path.
+   */
+  deferredRoundingWindowTokens: number;
+}
+
+export interface RangeAwareSelectiveWideningDefaultsSchema {
+  /** Enable range-aware widening guard for F16 activations. */
+  enabled: boolean;
+  /** Absolute-value threshold used for early widening checks. */
+  absThreshold: number;
+  /** Whether non-finite values should always trigger widening. */
+  includeNonFinite: boolean;
+}
+
+/** Default compute configuration */
+export declare const DEFAULT_COMPUTE_DEFAULTS: ComputeDefaultsSchema;
+
+/**
+ * Configuration for oversized weights (embeddings, LM head).
+ *
+ * When weights exceed device binding limits, DOPPLER can keep them on CPU
+ * and stream chunks to the GPU for matmul or gather operations.
+ */
+export interface LargeWeightConfigSchema {
+  /** Enable CPU-backed chunking for oversized weights */
+  enabled: boolean;
+  /** Safety ratio applied to GPU binding limits (0..1). Default: 0.9 */
+  safetyRatio: number;
+  /** Prefer uploading F16 chunks when supported (reduces chunk size) */
+  preferF16: boolean;
+  /** Optional override for LM head chunk rows (null = auto) */
+  lmHeadChunkRows?: number | null;
+}
+
+/** Default large-weight configuration */
+export declare const DEFAULT_LARGE_WEIGHT_CONFIG: LargeWeightConfigSchema;
+
+/**
+ * Default sampling configuration for token selection.
+ *
+ * Extends Required<SamplingSchema> with greedyThreshold and repetitionPenaltyWindow
+ * for runtime decisions.
+ * SamplingSchema (in inference.schema.js) uses optional fields for partial overrides;
+ * this schema provides concrete defaults for all sampling parameters.
+ */
+export interface SamplingDefaultsSchema extends Required<SamplingSchema> {
+  /** Temperature below this uses greedy decoding (default: 0.01) */
+  greedyThreshold: number;
+
+  /** Number of recent tokens to consider for repetition penalty (default: 100) */
+  repetitionPenaltyWindow: number;
+}
+
+/** Default sampling configuration */
+export declare const DEFAULT_SAMPLING_DEFAULTS: SamplingDefaultsSchema;
+
+/** Default generation behavior flags used by generator runtime paths. */
+export interface GenerationDefaultsSchema {
+  useSpeculative: boolean;
+  profile: boolean;
+  benchmark: boolean;
+  disableCommandBatching: boolean;
+  disableMultiTokenDecode: boolean;
+  embeddingMode: 'last' | 'mean';
+}
+
+/** Default generation behavior flags */
+export declare const DEFAULT_GENERATION_CONFIG: GenerationDefaultsSchema;
+
+/**
+ * Default tokenizer configuration.
+ *
+ * Provides defaults for common tokenizer behavior. Actual token strings
+ * come from the model's tokenizer config; these control runtime behavior.
+ */
+export interface TokenizerDefaultsSchema {
+  /** Add BOS token to input (default: true for most models) */
+  addBosToken: boolean;
+
+  /** Add EOS token to output (default: false, model decides) */
+  addEosToken: boolean;
+
+  /** Defer token validation until token metadata is fully loaded */
+  deferSpecialTokens: boolean;
+}
+
+/** Default tokenizer configuration */
+export declare const DEFAULT_TOKENIZER_DEFAULTS: TokenizerDefaultsSchema;
+
+/**
+ * Complete inference defaults configuration schema.
+ *
+ * Combines batching, sampling, compute, and tokenizer defaults for the inference pipeline.
+ */
+export interface InferenceDefaultsConfigSchema {
+  batching: BatchingDefaultsSchema;
+  sampling: SamplingDefaultsSchema;
+  compute: ComputeDefaultsSchema;
+  tokenizer: TokenizerDefaultsSchema;
+  /** Handling for oversized embeddings/LM head */
+  largeWeights: LargeWeightConfigSchema;
+  /** KV cache configuration */
+  kvcache: KVCacheConfigSchema;
+  /** MoE routing and cache configuration */
+  moe: MoERuntimeConfigSchema;
+  /** Speculative decoding configuration */
+  speculative: SpeculativeConfigSchema;
+  /** Generation behavior defaults */
+  generation: GenerationDefaultsSchema;
+  /** Diffusion pipeline configuration */
+  diffusion: DiffusionConfigSchema;
+  /** Energy pipeline configuration */
+  energy: EnergyConfigSchema;
+  /** Optional default prompt text for test harnesses */
+  prompt?: string | null;
+  /** Log token ids in test harness output */
+  debugTokens?: boolean;
+  pipeline?: LayerPipelineSchema | null;
+  /** Session overrides for execution v0 (session-only runtime knobs). */
+  session?: ExecutionV0SessionDefaultsSchema;
+  /** Atomic execution graph patch for execution v0. */
+  executionPatch?: ExecutionV0PatchSchema;
+  /**
+   * Kernel path for explicit kernel dispatch ordering.
+   * Specifies exactly which kernels run, in what order, with what configs.
+   * Can be a preset ID (e.g., 'gemma2-q4k-fused-f32a') or inline KernelPathSchema.
+   */
+  kernelPath?: KernelPathRef;
+  /**
+   * Internal kernel-path source hint used by execution-v0 runtime patching.
+   * Not intended for direct user configuration.
+   */
+  kernelPathSource?: 'runtime' | 'config' | 'model' | 'manifest' | 'execution-v0' | 'none';
+  /**
+   * Kernel-path auto-selection policy.
+   * locked: preserve configured kernelPath as-is.
+   * capability-aware: allow rule-driven remap for selected sources.
+   */
+  kernelPathPolicy?: KernelPathPolicySchema;
+  /**
+   * Chat template override for runtime config.
+   * When set, overrides the model preset's chatTemplate.enabled setting.
+   */
+  chatTemplate?: ChatTemplateSchema;
+  /**
+   * Model-specific inference overrides.
+   * Allows runtime override of any manifest inference field (attention, normalization, rope, etc.).
+   * These are merged with manifest.inference via mergeConfig().
+   */
+  modelOverrides?: ModelInferenceOverrides;
+}
+
+export interface KernelPathPolicySchema {
+  mode: 'locked' | 'capability-aware';
+  sourceScope: Array<'model' | 'manifest' | 'config' | 'runtime' | 'execution-v0'>;
+  /**
+   * Backward-compatible alias for sourceScope.
+   * Prefer sourceScope in new configs.
+   */
+  allowSources?: Array<'model' | 'manifest' | 'config' | 'runtime' | 'execution-v0'>;
+  onIncompatible: 'error' | 'remap';
+}
+
+/** Default inference configuration */
+export declare const DEFAULT_INFERENCE_DEFAULTS_CONFIG: InferenceDefaultsConfigSchema;
+export declare const DEFAULT_KERNEL_PATH_POLICY: KernelPathPolicySchema;
+
+/** Default inference configuration for model presets */
+export declare const DEFAULT_PRESET_INFERENCE_CONFIG: InferenceConfigSchema;

package/src/config/schema/inference-defaults.schema.js

@@ -0,0 +1,185 @@
+import { DEFAULT_KVCACHE_CONFIG } from './kvcache.schema.js';
+import { DEFAULT_MOE_RUNTIME_CONFIG } from './moe.schema.js';
+import { DEFAULT_SPECULATIVE_CONFIG } from './speculative.schema.js';
+import { DEFAULT_RMS_NORM_EPS } from './manifest.schema.js';
+import { DEFAULT_DIFFUSION_CONFIG } from './diffusion.schema.js';
+import { DEFAULT_ENERGY_CONFIG } from './energy.schema.js';
+import {
+  DEFAULT_EXECUTION_V0_PATCH,
+  DEFAULT_EXECUTION_V0_SESSION_DEFAULTS,
+} from './execution-v0.schema.js';
+
+// =============================================================================
+// Generation Defaults (generator.js runtime options)
+// =============================================================================
+
+export const DEFAULT_GENERATION_CONFIG = {
+  useSpeculative: false,
+  profile: false,
+  benchmark: false,
+  disableCommandBatching: false,
+  disableMultiTokenDecode: false,
+  embeddingMode: 'last',
+};
+
+// =============================================================================
+// Batching Defaults
+// =============================================================================
+
+export const DEFAULT_BATCHING_DEFAULTS = {
+  batchSize: 4,  // GPU run-ahead window: record N tokens, submit once, readback once
+  maxTokens: 256,
+  stopCheckMode: 'batch',
+  readbackInterval: 1,
+  ringTokens: 1,
+  ringStop: 1,
+  ringStaging: 1,
+};
+
+// =============================================================================
+// Compute Defaults
+// =============================================================================
+
+export const DEFAULT_COMPUTE_DEFAULTS = {
+  activationDtype: 'f16',  // Default to F16 for web inference; fallback to F32 when unsupported
+  largeModelParamThreshold: 4e9,  // 4B parameters
+  paramEstimationMultiplier: 12,  // Rough approximation: 12 * hidden^2 * layers
+  keepF32Weights: false,  // Skip weight downcast (debug/compat)
+  rangeAwareSelectiveWidening: {
+    enabled: true,
+    absThreshold: 65500,
+    includeNonFinite: true,
+  },
+  deferredRoundingWindowTokens: 1,
+};
+
+// =============================================================================
+// Large Weight Handling
+// =============================================================================
+
+export const DEFAULT_LARGE_WEIGHT_CONFIG = {
+  enabled: true,
+  safetyRatio: 0.9,
+  preferF16: true,
+  lmHeadChunkRows: null,
+};
+
+// =============================================================================
+// Sampling Defaults
+// =============================================================================
+
+export const DEFAULT_SAMPLING_DEFAULTS = {
+  temperature: 1.0,
+  topP: 0.95,
+  topK: 50,
+  repetitionPenalty: 1.1,
+  greedyThreshold: 0.01,
+  repetitionPenaltyWindow: 100,
+};
+
+// =============================================================================
+// Tokenizer Defaults
+// =============================================================================
+
+export const DEFAULT_TOKENIZER_DEFAULTS = {
+  addBosToken: true,
+  addEosToken: false,
+  deferSpecialTokens: true,
+  addSpacePrefix: null,  // null = auto-detect from tokenizer.json
+};
+
+// =============================================================================
+// Chat Template Defaults
+// =============================================================================
+
+export const DEFAULT_CHAT_TEMPLATE_CONFIG = {
+  enabled: false,
+};
+
+export const DEFAULT_KERNEL_PATH_POLICY = {
+  mode: 'locked',
+  sourceScope: ['model', 'manifest'],
+  onIncompatible: 'error',
+};
+
+// =============================================================================
+// Complete Inference Defaults Config
+// =============================================================================
+
+export const DEFAULT_INFERENCE_DEFAULTS_CONFIG = {
+  batching: DEFAULT_BATCHING_DEFAULTS,
+  sampling: DEFAULT_SAMPLING_DEFAULTS,
+  compute: DEFAULT_COMPUTE_DEFAULTS,
+  tokenizer: DEFAULT_TOKENIZER_DEFAULTS,
+  largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
+  kvcache: DEFAULT_KVCACHE_CONFIG,
+  moe: DEFAULT_MOE_RUNTIME_CONFIG,
+  speculative: DEFAULT_SPECULATIVE_CONFIG,
+  generation: DEFAULT_GENERATION_CONFIG,
+  chatTemplate: DEFAULT_CHAT_TEMPLATE_CONFIG,
+  diffusion: DEFAULT_DIFFUSION_CONFIG,
+  energy: DEFAULT_ENERGY_CONFIG,
+  pipeline: null,
+  kernelPath: undefined,
+  kernelPathSource: undefined,
+  kernelPathPolicy: DEFAULT_KERNEL_PATH_POLICY,
+  session: DEFAULT_EXECUTION_V0_SESSION_DEFAULTS,
+  executionPatch: DEFAULT_EXECUTION_V0_PATCH,
+};
+
+// =============================================================================
+// Preset Inference Defaults
+// =============================================================================
+
+export const DEFAULT_PRESET_INFERENCE_CONFIG = {
+  attention: {
+    slidingWindow: null,
+    attnLogitSoftcapping: null,
+    queryKeyNorm: false,
+    attentionOutputGate: false,
+    causal: true,
+    ropeScalingType: null,
+    ropeScalingFactor: 1.0,
+  },
+  normalization: {
+    rmsNormWeightOffset: false,
+    rmsNormEps: DEFAULT_RMS_NORM_EPS,
+    postAttentionNorm: false,
+    preFeedforwardNorm: false,
+    postFeedforwardNorm: false,
+  },
+  ffn: {
+    activation: 'silu',
+    gatedActivation: true,
+  },
+  output: {
+    finalLogitSoftcapping: null,
+    tieWordEmbeddings: false,
+    scaleEmbeddings: false,
+    embeddingTranspose: false,
+    embeddingVocabSize: null,
+  },
+  layerPattern: {
+    type: 'all_attention',
+  },
+  rope: {
+    ropeTheta: 10000,
+    ropeLocalTheta: null,
+    ropeScalingType: null,
+    ropeScalingFactor: 1.0,
+    ropeLocalScalingType: null,
+    ropeLocalScalingFactor: 1.0,
+    yarnBetaFast: 32,
+    yarnBetaSlow: 1,
+    yarnOriginalMaxPos: 4096,
+    ropeLocalYarnBetaFast: null,
+    ropeLocalYarnBetaSlow: null,
+    ropeLocalYarnOriginalMaxPos: null,
+  },
+  moe: null,
+  pipeline: null,
+  chatTemplate: {
+    type: null,
+  },
+  kernelPath: undefined,
+};