@simulatte/doppler 0.1.7 → 0.1.9

package/src/inference/pipelines/text/generator-runtime.js

@@ -139,6 +139,12 @@ export function resolveStepOptions(state, options = {}) {
   const executionPlan = resolveExecutionSessionPlan(state, options);
 
   return {
+    seed: resolveConfiguredValue(
+      options.seed,
+      undefined,
+      'options.seed',
+      (value) => Number.isFinite(value) && value >= 0
+    ),
     temperature: resolveConfiguredValue(options.temperature, samplingDefaults.temperature, 'options.temperature'),
     topP: resolveConfiguredValue(options.topP, samplingDefaults.topP, 'options.topP'),
     topK: resolveConfiguredValue(options.topK, samplingDefaults.topK, 'options.topK'),
@@ -165,6 +171,12 @@ export function resolveGenerateOptions(state, options = {}) {
   const executionPlan = resolveExecutionSessionPlan(state, options);
 
   return {
+    seed: resolveConfiguredValue(
+      options.seed,
+      undefined,
+      'options.seed',
+      (value) => Number.isFinite(value) && value >= 0
+    ),
     maxTokens: executionPlan.maxTokens,
     temperature: resolveConfiguredValue(options.temperature, samplingDefaults.temperature, 'options.temperature'),
     topP: resolveConfiguredValue(options.topP, samplingDefaults.topP, 'options.topP'),
@@ -191,6 +203,7 @@ export function resolveGenerateOptions(state, options = {}) {
     batchSize: executionPlan.batchSize,
     stopCheckMode: executionPlan.stopCheckMode,
     executionPlan,
+    images: options.images ?? null,
   };
 }
 
@@ -205,6 +218,7 @@ export function resolvePrefillOptions(state, options = {}) {
     disableCommandBatching: executionPlan.disableCommandBatching,
     disableMultiTokenDecode: executionPlan.disableMultiTokenDecode,
     executionPlan,
+    images: options.images ?? null,
   };
 }
 
@@ -213,6 +227,10 @@ export function resolvePrefillEmbeddingOptions(state, options = {}) {
     ? state.manifest.modelType.toLowerCase()
     : '';
   const generationDefaults = state.runtimeConfig.inference.generation;
+  // Embedding models default to 'mean' pooling — this is a model-category behavior,
+  // not a model-family identity check. Ideally embedding model presets would set
+  // generation.embeddingMode='mean' in their runtime config; the modelType fallback
+  // provides this default for manifests that predate runtime-preset embedding mode.
   const defaultEmbeddingMode = modelType === 'embedding'
     ? 'mean'
     : generationDefaults.embeddingMode;
@@ -226,6 +244,7 @@ export function resolveAdvanceEmbeddingMode(state, options = {}) {
   const modelType = typeof state.manifest?.modelType === 'string'
     ? state.manifest.modelType.toLowerCase()
     : '';
+  // See resolvePrefillEmbeddingOptions for embedding-model pooling rationale.
   const configuredMode = state.runtimeConfig.inference.generation.embeddingMode;
   return resolveConfiguredValue(
     options.embeddingMode,

package/src/inference/pipelines/text/generator-steps.d.ts

@@ -12,6 +12,15 @@ export interface BatchDecodeSelectionConfig {
 
 export declare function shouldUseBatchDecode(config: BatchDecodeSelectionConfig): boolean;
 
+export interface FusedDecodeSamplingConfig {
+  recorderEnabled: boolean;
+  gpuSamplingEnabled: boolean;
+  fusedDecodeDisabled: boolean;
+  layerTypes?: string[] | null;
+}
+
+export declare function shouldUseFusedDecodeSampling(config: FusedDecodeSamplingConfig): boolean;
+
 export declare function resolveBatchStop(
   tokens: number[],
   stopFlags: Uint32Array | null,
@@ -19,6 +28,12 @@ export declare function resolveBatchStop(
   eosTokenId: number | undefined | null
 ): number;
 
+export declare function findInvalidGeneratedToken(
+  tokens: number[],
+  vocabSize: number,
+  padTokenId?: number | null
+): { index: number; tokenId: number } | null;
+
 export interface SampledTokenStagingBuffer {
   mapAsync(mode: number): Promise<void>;
   getMappedRange(): ArrayBufferLike;

package/src/inference/pipelines/text/generator-steps.js

@@ -1,6 +1,6 @@
 import { getDevice, setTrackSubmits } from '../../../gpu/device.js';
 import { releaseBuffer, readBuffer } from '../../../memory/buffer-pool.js';
-import { runArgmax, runGPUSample, recordArgmax, recordGPUSample, isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
+import { recordArgmax, recordGPUSample, isGPUSamplingAvailable } from '../../../gpu/kernels/sample.js';
 import { recordCheckStop } from '../../../gpu/kernels/check-stop.js';
 import { resetSubmitStats, logSubmitStats } from '../../../gpu/submit-tracker.js';
 import { createCommandRecorder, createProfilingRecorder, CommandRecorder } from '../../../gpu/command-recorder.js';
@@ -20,6 +20,7 @@ import { decodeReadback } from './debug-utils/index.js';
 import { getFinalNormWeights, extractEmbeddingFromHidden } from './generator-runtime.js';
 import { parseFinitenessStatusWords } from './finiteness-guard-status.js';
 import { hasLinearAttentionLayers } from './linear-attention.js';
+import { hasConvLayers } from './layer.js';
 
 const UNKNOWN_TOKEN_TEXT = '<unknown>';
 
@@ -91,6 +92,13 @@ export function shouldUseBatchDecode(config) {
   return isBatchDecodeEnabled(config);
 }
 
+export function shouldUseFusedDecodeSampling(config) {
+  return config.recorderEnabled === true
+    && config.gpuSamplingEnabled === true
+    && config.fusedDecodeDisabled !== true
+    && !hasConvLayers(config.layerTypes ?? []);
+}
+
 export function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
   let actualCount = tokens.length;
   if (stopFlags) {
@@ -113,6 +121,20 @@ export function resolveBatchStop(tokens, stopFlags, stopTokenIds, eosTokenId) {
   return actualCount;
 }
 
+export function findInvalidGeneratedToken(tokens, vocabSize, padTokenId = null) {
+  for (let i = 0; i < tokens.length; i++) {
+    const tokenId = tokens[i];
+    const isInvalid = !Number.isFinite(tokenId)
+      || tokenId < 0
+      || tokenId >= vocabSize
+      || (padTokenId != null ? tokenId === padTokenId : tokenId === 0);
+    if (isInvalid) {
+      return { index: i, tokenId };
+    }
+  }
+  return null;
+}
+
 export async function readSampledTokenFromStagingBuffer(stagingBuffer, options = {}) {
   const ownsStagingBuffer = options.ownsStagingBuffer === true;
   const hasFinitenessBuffer = options.hasFinitenessBuffer === true;
@@ -240,11 +262,9 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
     throw new Error('Embed buffer not found or not a supported buffer type');
   }
   const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
-  const embedDtype = isWeightBuffer(embedBufferRaw)
-    ? getWeightDtype(embedBufferRaw)
-    : isCpuWeightBuffer(embedBufferRaw)
-      ? embedBufferRaw.dtype
-      : null;
+  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
+    ? embedBufferRaw.dtype
+    : getWeightDtype(embedBufferRaw);
   const activationDtype = getEffectiveActivationDtype(state, opts);
 
   const embedTensor = await embed([tokenId], embedBuffer, {
@@ -326,11 +346,9 @@ export async function decodeStep(state, currentIds, opts, helpers) {
     throw new Error('Embed buffer not found or not a supported buffer type');
   }
   const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
-  const embedDtype = isWeightBuffer(embedBufferRaw)
-    ? getWeightDtype(embedBufferRaw)
-    : isCpuWeightBuffer(embedBufferRaw)
-      ? embedBufferRaw.dtype
-      : null;
+  const embedDtype = isCpuWeightBuffer(embedBufferRaw)
+    ? embedBufferRaw.dtype
+    : getWeightDtype(embedBufferRaw);
   const activationDtype = getEffectiveActivationDtype(state, opts);
   const activationBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: activationDtype });
 
@@ -393,7 +411,12 @@ export async function decodeStep(state, currentIds, opts, helpers) {
   const padTokenId = state.tokenizer?.getSpecialTokens?.()?.pad ?? null;
   const lmHeadIsCpu = isCpuWeightBuffer(state.weights.get('lm_head'));
   const useGPUSampling = state.useGPU && isGPUSamplingAvailable() && !lmHeadIsCpu;
-  const useFusedDecode = recorder && useGPUSampling && !state.disableFusedDecode;
+  const useFusedDecode = shouldUseFusedDecodeSampling({
+    recorderEnabled: Boolean(recorder),
+    gpuSamplingEnabled: useGPUSampling,
+    fusedDecodeDisabled: state.disableFusedDecode,
+    layerTypes: config.layerTypes,
+  });
 
   if (useFusedDecode) {
     const ring = state.decodeRing;
@@ -621,21 +644,30 @@ export async function decodeStep(state, currentIds, opts, helpers) {
     );
     if (logitsResult) {
       const { logitsBuffer, vocabSize, logitsDtype } = logitsResult;
+      const logitsBytes = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype: logitsDtype });
+      const logitsData = await readBuffer(logitsBuffer, numTokens * vocabSize * logitsBytes);
+      releaseBuffer(logitsBuffer);
 
-      const nextToken = opts.temperature < samplingDefaults.greedyThreshold
-        ? await runArgmax(logitsBuffer, vocabSize, { padTokenId, logitSoftcap, logitsDtype, outputIndex: 0 })
-        : await runGPUSample(logitsBuffer, vocabSize, {
-          temperature: opts.temperature,
-          topK: opts.topK,
-          padTokenId,
-          logitSoftcap,
-          logitsDtype,
-          outputIndex: 0,
-          greedyThreshold: samplingDefaults.greedyThreshold,
-          randomSeed: opts.seed,
-        });
+      const rawLogits = decodeReadback(logitsData, logitsDtype);
+      const finalizedLogits = await finalizeLogits(
+        rawLogits,
+        numTokens,
+        vocabSize,
+        config.vocabSize,
+        config,
+        state.runtimeConfig.shared.debug.probes
+      );
+      const sampledLogits = extractLastPositionLogits(finalizedLogits, numTokens, config.vocabSize);
+
+      applyRepetitionPenalty(sampledLogits, currentIds, opts.repetitionPenalty);
+      const nextToken = sample(sampledLogits, {
+        temperature: opts.temperature,
+        topP: opts.topP,
+        topK: opts.topK,
+        padTokenId,
+        seed: opts.seed,
+      });
 
-      releaseBuffer(logitsBuffer);
       if (!context.decodeBuffers?.ownsBuffer(hiddenStates)) {
         releaseBuffer(hiddenStates);
       }
@@ -867,6 +899,11 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
       '[Pipeline] Batch decode path is disabled for linear_attention models; use single-token decode.'
     );
   }
+  if (hasConvLayers(config.layerTypes)) {
+    throw new Error(
+      '[Pipeline] Batch decode path is disabled for conv models; use single-token decode.'
+    );
+  }
   const samplingDefaults = state.runtimeConfig.inference.sampling;
   const executionPlan = opts.executionPlan;
   const batchSize = executionPlan?.batchSize ?? opts.batchSize ?? state.runtimeConfig.inference.batching.batchSize;
@@ -981,7 +1018,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
       throw new Error('Embed buffer not found or not a GPUBuffer/WeightBuffer');
     }
     const embedBuffer = isWeightBuffer(embedBufferRaw) ? embedBufferRaw.buffer : embedBufferRaw;
-    const embedDtype = isWeightBuffer(embedBufferRaw) ? getWeightDtype(embedBufferRaw) : null;
+    const embedDtype = getWeightDtype(embedBufferRaw);
     const activationDtype = getEffectiveActivationDtype(state, opts);
 
     for (let i = 0; i < N; i++) {
@@ -1125,10 +1162,18 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
 
     const actualCount = resolveBatchStop(tokens, stopFlags, stopTokenIds, eosToken);
     const generatedTokens = tokens.slice(0, actualCount);
+    const invalidToken = findInvalidGeneratedToken(generatedTokens, config.vocabSize, padTokenId);
 
     if (isInfinite) {
       throw new FinitenessError(`F16 bounds exceeded during batch generation${metadata}`);
     }
+    if (invalidToken) {
+      state.disableFusedDecode = true;
+      throw new Error(
+        `[Pipeline] Batch decode produced invalid token ${invalidToken.tokenId} ` +
+        `at batch index ${invalidToken.index} (vocabSize=${config.vocabSize}, padTokenId=${padTokenId ?? 'none'}).`
+      );
+    }
 
     if (opts.profile && recorder.isProfilingEnabled()) {
       const timings = await recorder.resolveProfileTimings();

package/src/inference/pipelines/text/generator.d.ts

@@ -27,6 +27,11 @@ export declare class PipelineGenerator {
    * Batching and readback cadence are controlled by runtime.inference.batching.
    */
   generate(prompt: PromptInput, options?: GenerateOptions): AsyncGenerator<string, void, void>;
+  generateTokens(prompt: PromptInput, options?: GenerateOptions): AsyncGenerator<number, void, void>;
+  generateTokenIds(
+    prompt: PromptInput,
+    options?: GenerateOptions
+  ): Promise<{ tokenIds: number[]; stats: import('./types.js').PipelineStats }>;
   prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
   prefillWithEmbedding(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillEmbeddingResult>;
   prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;