@simulatte/doppler 0.1.6 → 0.1.8

package/src/inference/pipelines/text/attention/run.js

@@ -28,10 +28,12 @@ import { runProbes } from '../probes.js';
 import { SlidingWindowKVCache } from '../../../kv-cache.js';
 import {
   recordAttentionInputs,
+  shouldForceF32AttentionProjectionForRoPE,
   resolveAttentionProjectionOutputDtype,
   projectAttentionQKV,
   applyAttentionQKNorm,
 } from './projections.js';
+import { prepareAttentionProjectionInput } from './output-projection.js';
 
 import {
   shouldDebugLayer,
@@ -97,9 +99,20 @@ export async function runLayerAttentionGPU(
   const allowF16Attention = wantsF16Output && kvCacheDtype === 'f16';
   let attentionInput = input;
   let attentionInputTemp = false;
+  let normed = attentionInput;
+  let qTensor = null;
+  let qGateTensor = null;
+  let kTensor = null;
+  let vTensor = null;
+  let attnOutput = null;
+  let attnForProjection = null;
+  let output = null;
+  let finalOutput = null;
+  let oProjInputTemp = null;
   if (wantsF16Output && !allowF16Attention) {
     attentionInput = await castF16ToF32(input);
     attentionInputTemp = true;
+    normed = attentionInput;
   }
 
   // Debug: attention input for configured layers
@@ -123,7 +136,7 @@ export async function runLayerAttentionGPU(
 
   // 1. Input norm
   
-  let normed = attentionInput;
+  try {
   if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
     const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
 
@@ -182,8 +195,16 @@ export async function runLayerAttentionGPU(
   }
 
   // 2. Q/K/V projections
-  const matmulOutputDtype = resolveAttentionProjectionOutputDtype(desiredOutputDtype);
-  let { qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
+  const matmulOutputDtype = resolveAttentionProjectionOutputDtype(desiredOutputDtype, {
+    forceF32: shouldForceF32AttentionProjectionForRoPE({
+      attentionInputDtype: desiredOutputDtype,
+      headDim,
+      rotaryDim: config.ropeRotaryDim,
+      interleaved: config.ropeInterleaved,
+    }),
+  });
+  let usedFusedQKV = false;
+  ({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV } = await projectAttentionQKV({
     recorder: null,
     normed,
     layerWeights,
@@ -204,7 +225,7 @@ export async function runLayerAttentionGPU(
         trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
       }
       : null,
-  });
+  }));
 
   // Trace Q/K/V projections
   if (kernelTrace.enabled) {
@@ -212,6 +233,27 @@ export async function runLayerAttentionGPU(
     await traceStep('matmul', `L${layerIdx}.k_proj`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
     await traceStep('matmul', `L${layerIdx}.v_proj`, layerIdx, vTensor.buffer, [numTokens, numKVHeads * headDim]);
   }
+  await runProbes('q_proj', qTensor.buffer, {
+    layerIdx,
+    numTokens,
+    hiddenSize: numHeads * headDim,
+    probes: state.debugProbes,
+    dtype: qTensor.dtype,
+  });
+  await runProbes('k_proj', kTensor.buffer, {
+    layerIdx,
+    numTokens,
+    hiddenSize: numKVHeads * headDim,
+    probes: state.debugProbes,
+    dtype: kTensor.dtype,
+  });
+  await runProbes('v_proj', vTensor.buffer, {
+    layerIdx,
+    numTokens,
+    hiddenSize: numKVHeads * headDim,
+    probes: state.debugProbes,
+    dtype: vTensor.dtype,
+  });
 
   // Kernel step debug: Q/K/V projections
   if (isKernelDebugEnabled(layerIdx)) {
@@ -319,6 +361,20 @@ export async function runLayerAttentionGPU(
       await traceStep('rope', `L${layerIdx}.k_rope`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
     }
   }
+  await runProbes('q_rope', qTensor.buffer, {
+    layerIdx,
+    numTokens,
+    hiddenSize: numHeads * headDim,
+    probes: state.debugProbes,
+    dtype: qTensor.dtype,
+  });
+  await runProbes('k_rope', kTensor.buffer, {
+    layerIdx,
+    numTokens,
+    hiddenSize: numKVHeads * headDim,
+    probes: state.debugProbes,
+    dtype: kTensor.dtype,
+  });
   if (isKernelDebugEnabled(layerIdx)) {
     logKernelStep('rope', { layerIdx, label: `startPos=${currentSeqLen}` });
     await dumpTokenVector(qTensor.buffer, 'Q_rope', {
@@ -669,7 +725,7 @@ export async function runLayerAttentionGPU(
     throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
   }
 
-  const attnOutput = await runAttentionKernel();
+  attnOutput = await runAttentionKernel();
 
   // Trace attention output
   if (kernelTrace.enabled) {
@@ -692,7 +748,7 @@ export async function runLayerAttentionGPU(
     await debugCheckBuffer(attnOutput.buffer, `L${layerIdx} attention output (before o_proj, GPU)`, numTokens, numHeads * headDim);
   }
 
-  let attnForProjection = attnOutput;
+  attnForProjection = attnOutput;
   if (qGateTensor) {
     attnForProjection = await runSiLU(attnOutput, {
       size: numTokens * numHeads * headDim,
@@ -706,19 +762,19 @@ export async function runLayerAttentionGPU(
 
   // 6. Output projection (with optional fused residual for decode)
   
-  let output;
+  output = null;
   let residualFused = false;
   let oProjInput = attnForProjection;
-  let oProjInputTemp = null;
+  oProjInputTemp = null;
   if (layerWeights.oProj && getWeightBuffer) {
+    ({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
+      attnForProjection,
+      matmulOutputDtype,
+      castF32ToF16
+    ));
     const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
     const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
 
-    if (matmulOutputDtype === 'f16' && attnOutput.dtype !== 'f16') {
-      oProjInput = await castF32ToF16(attnOutput);
-      oProjInputTemp = oProjInput;
-    }
-
     // Use fused o_proj + residual for decode when possible
     // Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
     const oProjDtype = getWeightDtype(oProjBuf);
@@ -807,7 +863,7 @@ export async function runLayerAttentionGPU(
     await debugCheckBuffer(output.buffer, `L${layerIdx} attention output (after o_proj, GPU)`, numTokens, hiddenSize);
   }
 
-  let finalOutput = output;
+  finalOutput = output;
   
   const buffersToRelease = [];
   if (output.buffer !== attnForProjection.buffer) {
@@ -832,4 +888,46 @@ export async function runLayerAttentionGPU(
   }
 
   return { output: finalOutput, residualFused };
+  } catch (error) {
+    const released = new Set();
+    const releaseOnce = (buffer) => {
+      if (!buffer || released.has(buffer)) return;
+      released.add(buffer);
+      releaseBuffer(buffer);
+    };
+    if (finalOutput?.buffer && finalOutput.buffer !== output?.buffer) {
+      releaseOnce(finalOutput.buffer);
+    }
+    if (output?.buffer && output.buffer !== attnForProjection?.buffer) {
+      releaseOnce(output.buffer);
+    }
+    if (oProjInputTemp?.buffer) {
+      releaseOnce(oProjInputTemp.buffer);
+    }
+    if (attnForProjection?.buffer && attnForProjection.buffer !== attnOutput?.buffer) {
+      releaseOnce(attnForProjection.buffer);
+    }
+    if (attnOutput?.buffer) {
+      releaseOnce(attnOutput.buffer);
+    }
+    if (qGateTensor?.buffer) {
+      releaseOnce(qGateTensor.buffer);
+    }
+    if (qTensor?.buffer) {
+      releaseOnce(qTensor.buffer);
+    }
+    if (kTensor?.buffer) {
+      releaseOnce(kTensor.buffer);
+    }
+    if (vTensor?.buffer) {
+      releaseOnce(vTensor.buffer);
+    }
+    if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {
+      releaseOnce(normed.buffer);
+    }
+    if (attentionInputTemp && attentionInput?.buffer) {
+      releaseOnce(attentionInput.buffer);
+    }
+    throw error;
+  }
 }

package/src/inference/pipelines/text/config.js

@@ -134,11 +134,10 @@ function resolveIntermediateSizeForRuntime(manifest, inf, arch, modelId) {
   if (inferred == null || inferred === fromArch) {
     return fromArch;
   }
-  log.warn(
-    'Config',
-    `Manifest "${modelId}" has intermediateSize=${fromArch}, inferred ${inferred} from FFN tensor shapes; using inferred value.`
+  throw new Error(
+    `Manifest "${modelId}" has intermediateSize=${fromArch}, but FFN tensors imply ${inferred}. ` +
+    'Re-convert the model so manifest architecture matches the weights.'
   );
-  return inferred;
 }
 
 // =============================================================================
@@ -483,6 +482,20 @@ export function toParsedConfigFromMerged(merged, manifest) {
   const queryPreAttnScalar = inf.attention.queryPreAttnScalar;
   const causalAttention = inf.attention.causal;
 
+  // Cross-field sanity: queryPreAttnScalar should typically equal headDim.
+  // A value of sqrt(headDim) indicates a known converter bug that produces
+  // attnScale = 1/sqrt(sqrt(headDim)) instead of the correct 1/sqrt(headDim).
+  if (queryPreAttnScalar != null && headDim != null
+      && queryPreAttnScalar !== headDim
+      && Math.abs(queryPreAttnScalar - Math.sqrt(headDim)) < 0.01) {
+    throw new Error(
+      `Model "${merged.modelId}": queryPreAttnScalar (${queryPreAttnScalar}) ` +
+      `equals sqrt(headDim) instead of headDim (${headDim}). ` +
+      `This is a known converter bug — the manifest must be regenerated ` +
+      `with the corrected converter.`
+    );
+  }
+
   // Get stop token IDs (cast to Manifest for compatibility)
   const stopTokenIds = getStopTokenIds(manifest);
 

package/src/inference/pipelines/text/embed.js

@@ -319,14 +319,8 @@ export async function embed(tokenIds, embedBuffer, config) {
     const firstTokenId = tokenIdArray[0];
     const bytesPerElement = useF16 ? 2 : 4;
     const sampleSize = Math.min(32 * bytesPerElement, hiddenSize * bytesPerElement);
-    const staging = device.createBuffer({ size: sampleSize, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ });
-    const enc = device.createCommandEncoder();
-    enc.copyBufferToBuffer(gatherOutput.buffer, 0, staging, 0, sampleSize);
-    device.queue.submit([enc.finish()]);
-    await staging.mapAsync(GPUMapMode.READ);
-    const data = decodeReadback(staging.getMappedRange().slice(0), gatherOptions.outputDtype);
-    staging.unmap();
-    staging.destroy();
+    const readback = await readBuffer(gatherOutput.buffer, sampleSize);
+    const data = decodeReadback(readback, gatherOptions.outputDtype);
 
     // Compute statistics
     let sum = 0, sumSq = 0;

package/src/inference/pipelines/text/execution-plan.js

@@ -1,4 +1,3 @@
-import { log } from '../../../debug/index.js';
 import { resolveKernelPath } from '../../../config/kernel-path-loader.js';
 import { selectRuleValue } from '../../../rules/rule-registry.js';
 import {
@@ -9,19 +8,36 @@ import {
 export const PRIMARY_EXECUTION_PLAN_ID = 'primary';
 export const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
 
-function normalizePositiveInt(value, fallback, label) {
-  if (!Number.isFinite(value)) return fallback;
-  const normalized = Math.floor(value);
-  if (normalized >= 1) return normalized;
-  log.warn('Pipeline', `[ExecutionPlan] ${label}=${value} is invalid; using ${fallback}.`);
-  return fallback;
+function assertOptionalBoolean(value, label) {
+  if (value === undefined) {
+    return undefined;
+  }
+  if (typeof value !== 'boolean') {
+    throw new Error(`[ExecutionPlan] ${label} must be boolean when provided; got ${JSON.stringify(value)}.`);
+  }
+  return value;
 }
 
-function normalizeStopCheckMode(value, fallback) {
-  if (value === 'batch' || value === 'per-token') {
-    return value;
+function assertOptionalPositiveInt(value, label) {
+  if (value === undefined) {
+    return undefined;
   }
-  return fallback;
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`[ExecutionPlan] ${label} must be a positive integer when provided; got ${JSON.stringify(value)}.`);
+  }
+  return value;
+}
+
+function assertOptionalStopCheckMode(value) {
+  if (value === undefined) {
+    return undefined;
+  }
+  if (value !== 'batch' && value !== 'per-token') {
+    throw new Error(
+      `[ExecutionPlan] stopCheckMode must be "batch" or "per-token" when provided; got ${JSON.stringify(value)}.`
+    );
+  }
+  return value;
 }
 
 function resolveFallbackActivationDtype(primaryActivationDtype) {
@@ -42,10 +58,11 @@ function resolveFallbackActivationDtype(primaryActivationDtype) {
 function resolveFallbackKernelPath(primaryKernelPath) {
   const primaryKernelPathId = primaryKernelPath?.id ?? null;
   if (!primaryKernelPathId) {
-    throw new Error(
-      '[ExecutionPlan] F16 finiteness fallback requires a primary kernel path with a stable id. ' +
-      'Add a registered kernelPath id and a finiteness fallback rule.'
-    );
+    return {
+      kernelPath: null,
+      kernelPathId: null,
+      kernelPathSource: 'none',
+    };
   }
 
   const explicitFallbackKernelPathId = typeof primaryKernelPath?.finitenessFallbackKernelPathId === 'string'
@@ -244,11 +261,17 @@ export function activateFallbackExecutionPlan(container) {
 
 function resolveExecutionOverrides(options = {}) {
   return {
-    disableCommandBatching: options.disableCommandBatching,
-    disableMultiTokenDecode: options.disableMultiTokenDecode,
-    batchSize: options.batchSize,
-    stopCheckMode: options.stopCheckMode,
-    maxTokens: options.maxTokens,
+    disableCommandBatching: assertOptionalBoolean(
+      options.disableCommandBatching,
+      'disableCommandBatching'
+    ),
+    disableMultiTokenDecode: assertOptionalBoolean(
+      options.disableMultiTokenDecode,
+      'disableMultiTokenDecode'
+    ),
+    batchSize: assertOptionalPositiveInt(options.batchSize, 'batchSize'),
+    stopCheckMode: assertOptionalStopCheckMode(options.stopCheckMode),
+    maxTokens: assertOptionalPositiveInt(options.maxTokens, 'maxTokens'),
   };
 }
 
@@ -268,9 +291,9 @@ export function resolveExecutionSessionPlan(container, options = {}) {
     deferredRoundingWindowTokens: activePlan.deferredRoundingWindowTokens,
     disableCommandBatching: overrides.disableCommandBatching ?? activePlan.defaultDisableCommandBatching,
     disableMultiTokenDecode: overrides.disableMultiTokenDecode ?? activePlan.defaultDisableMultiTokenDecode,
-    batchSize: normalizePositiveInt(overrides.batchSize, activePlan.defaultBatchSize, 'batchSize'),
-    stopCheckMode: normalizeStopCheckMode(overrides.stopCheckMode, activePlan.defaultStopCheckMode),
-    maxTokens: normalizePositiveInt(overrides.maxTokens, activePlan.defaultMaxTokens, 'maxTokens'),
+    batchSize: overrides.batchSize ?? activePlan.defaultBatchSize,
+    stopCheckMode: overrides.stopCheckMode ?? activePlan.defaultStopCheckMode,
+    maxTokens: overrides.maxTokens ?? activePlan.defaultMaxTokens,
     readbackInterval: activePlan.readbackInterval,
     ringTokens: activePlan.ringTokens,
     ringStop: activePlan.ringStop,

package/src/inference/pipelines/text/execution-v0-contract-helpers.d.ts

@@ -0,0 +1,59 @@
+export declare function cloneJson<T>(value: T): T;
+export declare function validateManifestSessionDefaultsContract(manifestInference: Record<string, unknown> | null): void;
+export declare function isPhaseMatch(phase: string, targetPhase: string): boolean;
+export declare function stepHasLayer(step: Record<string, unknown>, layerIdx: number): boolean;
+export declare function normalizePhase(value: unknown, label: string): string;
+export declare function normalizeSection(value: unknown, label: string): string;
+export declare function normalizeSlot(value: unknown, label: string): string;
+export declare function createSourceTrace(): { session: Record<string, unknown>; steps: Record<string, unknown> };
+export declare function setSourceTrace(trace: Record<string, unknown>, path: string, source: string): void;
+export declare function collectLeafPaths(value: unknown, prefix?: string[], out?: string[][]): string[][];
+export declare function hasDefinedPath(root: unknown, pathSegments: string[]): boolean;
+export declare function validateStepShape(step: Record<string, unknown>, index: number): void;
+export declare function assertExecutionRuntimeOverlay(runtimeInference: Record<string, unknown> | null | undefined): void;
+export declare function validateUniqueStepIds(steps: Array<Record<string, unknown>>): void;
+export declare function hasExecutionV0(manifestInference: Record<string, unknown> | null | undefined): boolean;
+export declare function assertExecutionV0Schema(manifestInference: Record<string, unknown> | null | undefined): void;
+export declare function applyExecutionPatchAtomic(
+  baseSteps: Array<Record<string, unknown>>,
+  patch: Record<string, unknown> | null | undefined
+): Array<Record<string, unknown>>;
+export declare function indexRuntimePatchMeta(
+  patch: Record<string, unknown> | null | undefined
+): {
+  addedSteps: Set<string>;
+  precisionFieldsByStep: Map<string, Set<string>>;
+  kvIOFieldsByStep: Set<string>;
+};
+export declare function requireSessionActivationDtype(
+  sessionDefaults: Record<string, unknown> | null | undefined,
+  label?: string
+): string;
+export declare function createInitialSlotDtypes(sessionDefaults: Record<string, unknown>): Map<string, string>;
+export declare function resolvePhaseSteps(
+  phase: string,
+  steps: Array<Record<string, unknown>>,
+  sessionDefaults: Record<string, unknown>,
+  profileIndex: Map<string, unknown>,
+  policies: Record<string, unknown>,
+  options?: Record<string, unknown>
+): {
+  steps: Array<Record<string, unknown>>;
+  finalSlotDtypes: Map<string, string>;
+};
+export declare function normalizeRuntimeSessionForExecutionV0(
+  runtimeSession: Record<string, unknown> | null | undefined,
+  manifestInference: Record<string, unknown> | null | undefined,
+  defaultComputeDefaults: Record<string, unknown>
+): Record<string, unknown> | null | undefined;
+export declare function validatePhaseBoundaryCompatibility(options: Record<string, unknown>): void;
+export declare function assertKVLayoutExecutionCompatibility(
+  steps: Array<Record<string, unknown>>,
+  sessionDefaults: Record<string, unknown>
+): void;
+export declare const buildKernelProfileKey: (
+  kernelRef: Record<string, unknown> | null | undefined,
+  step?: Record<string, unknown> | null | undefined
+) => string;
+export declare const indexKernelProfiles: (sessionDefaults: Record<string, unknown>) => Map<string, unknown>;
+export declare const normalizeDtype: (value: unknown, label: string) => string;