doppler-gpu 0.4.5 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # doppler-gpu
2
2
 
3
+ ![Doppler WebGPU inference runtime](./assets/doppler-readme-hero.svg)
4
+
3
5
  WebGPU inference runtime for browser, Node, Bun, CLI, and local server use.
4
6
  Doppler runs JavaScript orchestration over WGSL kernels and loads models from
5
7
  RDRR manifests.
@@ -42,6 +44,8 @@ completion gate is `src/config/goal-completion-matrix.json`, checked by
42
44
 
43
45
  ## How it works
44
46
 
47
+ ![Doppler runtime contract map](./assets/doppler-runtime-map.svg)
48
+
45
49
  1. A registry ID or model URL resolves to an RDRR manifest and weight shards.
46
50
  2. The manifest owns model parameters, tokenizer metadata, session policy, and
47
51
  execution graph.
@@ -0,0 +1,84 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" width="1200" height="360" viewBox="0 0 1200 360" role="img" aria-labelledby="title desc">
2
+ <title id="title">Doppler WebGPU inference runtime</title>
3
+ <desc id="desc">A product hero for Doppler showing browser, manifest, WebGPU kernels, and receipt-backed evidence lanes.</desc>
4
+ <defs>
5
+ <linearGradient id="hero-bg" x1="40" y1="20" x2="1160" y2="330" gradientUnits="userSpaceOnUse">
6
+ <stop offset="0" stop-color="#fff7ed"/>
7
+ <stop offset="0.36" stop-color="#edf7ff"/>
8
+ <stop offset="0.72" stop-color="#f1fff6"/>
9
+ <stop offset="1" stop-color="#fff0f4"/>
10
+ </linearGradient>
11
+ <linearGradient id="lane-a" x1="0" y1="0" x2="310" y2="0" gradientUnits="userSpaceOnUse">
12
+ <stop offset="0" stop-color="#ff6b7a"/>
13
+ <stop offset="0.48" stop-color="#00a88f"/>
14
+ <stop offset="1" stop-color="#4f7cff"/>
15
+ </linearGradient>
16
+ <linearGradient id="lane-b" x1="0" y1="0" x2="190" y2="140" gradientUnits="userSpaceOnUse">
17
+ <stop offset="0" stop-color="#111318"/>
18
+ <stop offset="1" stop-color="#353946"/>
19
+ </linearGradient>
20
+ <filter id="soft-shadow" x="-20%" y="-25%" width="140%" height="150%">
21
+ <feDropShadow dx="0" dy="10" stdDeviation="12" flood-color="#111318" flood-opacity="0.14"/>
22
+ </filter>
23
+ <style>
24
+ text { font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; fill: #111318; }
25
+ .mono { font-family: "SFMono-Regular", "Cascadia Code", "Liberation Mono", Menlo, monospace; }
26
+ .title { font-size: 72px; font-weight: 900; letter-spacing: 2px; }
27
+ .subtitle { font-size: 25px; font-weight: 700; fill: #3b414c; }
28
+ .small { font-size: 17px; font-weight: 800; letter-spacing: 1px; }
29
+ .tiny { font-size: 14px; font-weight: 800; fill: #4c5360; }
30
+ .white { fill: #ffffff; }
31
+ </style>
32
+ </defs>
33
+
34
+ <rect width="1200" height="360" rx="42" fill="#ffffff"/>
35
+ <rect x="18" y="18" width="1164" height="324" rx="34" fill="url(#hero-bg)"/>
36
+ <rect x="18" y="18" width="1164" height="324" rx="34" fill="none" stroke="#111318" stroke-width="2"/>
37
+
38
+ <g transform="translate(72 68)" filter="url(#soft-shadow)">
39
+ <rect x="0" y="0" width="228" height="228" rx="52" fill="#111318"/>
40
+ <path d="M66 65h48c61 0 99 37 99 86s-38 86-99 86H66z" transform="translate(-27 -18)" fill="none" stroke="#ffffff" stroke-width="30" stroke-linecap="round" stroke-linejoin="round"/>
41
+ <path d="M102 58c46 0 76 31 76 70s-30 70-76 70" fill="none" stroke="url(#lane-a)" stroke-width="17" stroke-linecap="round"/>
42
+ <path d="M32 158h48M32 180h76" stroke="#ffffff" stroke-opacity="0.92" stroke-width="10" stroke-linecap="round"/>
43
+ <circle cx="180" cy="58" r="11" fill="#ff6b7a"/>
44
+ <circle cx="188" cy="198" r="11" fill="#4f7cff"/>
45
+ </g>
46
+
47
+ <g transform="translate(350 78)">
48
+ <text x="0" y="58" class="title">Doppler</text>
49
+ <text x="4" y="101" class="subtitle">Browser-native WebGPU inference with explicit runtime contracts.</text>
50
+
51
+ <g transform="translate(4 137)">
52
+ <rect x="0" y="0" width="188" height="42" rx="21" fill="#111318"/>
53
+ <text x="94" y="28" text-anchor="middle" class="small white">RDRR manifests</text>
54
+ <rect x="204" y="0" width="136" height="42" rx="21" fill="#ffffff" stroke="#111318" stroke-width="2"/>
55
+ <text x="272" y="28" text-anchor="middle" class="small">JS orchestration</text>
56
+ <rect x="356" y="0" width="134" height="42" rx="21" fill="#ffffff" stroke="#111318" stroke-width="2"/>
57
+ <text x="423" y="28" text-anchor="middle" class="small">WGSL kernels</text>
58
+ <rect x="506" y="0" width="152" height="42" rx="21" fill="#ffffff" stroke="#111318" stroke-width="2"/>
59
+ <text x="582" y="28" text-anchor="middle" class="small">evidence gates</text>
60
+ </g>
61
+
62
+ <g transform="translate(8 218)">
63
+ <path d="M0 0h310" stroke="url(#lane-a)" stroke-width="9" stroke-linecap="round"/>
64
+ <circle cx="0" cy="0" r="12" fill="#ff6b7a" stroke="#111318" stroke-width="2"/>
65
+ <circle cx="104" cy="0" r="12" fill="#f5b942" stroke="#111318" stroke-width="2"/>
66
+ <circle cx="208" cy="0" r="12" fill="#00a88f" stroke="#111318" stroke-width="2"/>
67
+ <circle cx="310" cy="0" r="12" fill="#4f7cff" stroke="#111318" stroke-width="2"/>
68
+ <text x="0" y="36" text-anchor="middle" class="tiny">model</text>
69
+ <text x="104" y="36" text-anchor="middle" class="tiny">cache</text>
70
+ <text x="208" y="36" text-anchor="middle" class="tiny">decode</text>
71
+ <text x="310" y="36" text-anchor="middle" class="tiny">receipt</text>
72
+ </g>
73
+ </g>
74
+
75
+ <g transform="translate(922 88)" filter="url(#soft-shadow)">
76
+ <rect x="0" y="0" width="190" height="180" rx="34" fill="url(#lane-b)"/>
77
+ <path d="M40 126c33-74 77-88 114-18" fill="none" stroke="#ffffff" stroke-width="10" stroke-linecap="round" opacity="0.9"/>
78
+ <path d="M40 126c33-74 77-88 114-18" fill="none" stroke="url(#lane-a)" stroke-width="4" stroke-linecap="round"/>
79
+ <circle cx="40" cy="126" r="13" fill="#ff6b7a" stroke="#ffffff" stroke-width="2"/>
80
+ <circle cx="93" cy="49" r="13" fill="#f5b942" stroke="#ffffff" stroke-width="2"/>
81
+ <circle cx="154" cy="108" r="13" fill="#4f7cff" stroke="#ffffff" stroke-width="2"/>
82
+ <text x="95" y="153" text-anchor="middle" class="small white">Metal / Vulkan</text>
83
+ </g>
84
+ </svg>
@@ -0,0 +1,84 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420" role="img" aria-labelledby="title desc">
2
+ <title id="title">Doppler runtime contract map</title>
3
+ <desc id="desc">A diagram of the Doppler path from manifest and shards through loader, JavaScript orchestration, WGSL kernels, output, and evidence.</desc>
4
+ <defs>
5
+ <linearGradient id="map-bg" x1="40" y1="30" x2="1060" y2="390" gradientUnits="userSpaceOnUse">
6
+ <stop offset="0" stop-color="#ffffff"/>
7
+ <stop offset="0.42" stop-color="#eef8f4"/>
8
+ <stop offset="0.74" stop-color="#edf3ff"/>
9
+ <stop offset="1" stop-color="#fff5ec"/>
10
+ </linearGradient>
11
+ <linearGradient id="arrow" x1="0" y1="0" x2="900" y2="0" gradientUnits="userSpaceOnUse">
12
+ <stop offset="0" stop-color="#ff6b7a"/>
13
+ <stop offset="0.32" stop-color="#f5b942"/>
14
+ <stop offset="0.63" stop-color="#00a88f"/>
15
+ <stop offset="1" stop-color="#4f7cff"/>
16
+ </linearGradient>
17
+ <filter id="card-shadow" x="-10%" y="-20%" width="120%" height="145%">
18
+ <feDropShadow dx="0" dy="8" stdDeviation="9" flood-color="#111318" flood-opacity="0.11"/>
19
+ </filter>
20
+ <style>
21
+ text { font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; fill: #111318; }
22
+ .label { font-size: 24px; font-weight: 900; }
23
+ .body { font-size: 15px; font-weight: 700; fill: #4a515d; }
24
+ .chip { font-size: 13px; font-weight: 850; letter-spacing: 1px; }
25
+ .mono { font-family: "SFMono-Regular", "Cascadia Code", "Liberation Mono", Menlo, monospace; font-size: 14px; font-weight: 800; fill: #303640; }
26
+ </style>
27
+ </defs>
28
+
29
+ <rect width="1100" height="420" rx="34" fill="#ffffff"/>
30
+ <rect x="18" y="18" width="1064" height="384" rx="28" fill="url(#map-bg)"/>
31
+ <rect x="18" y="18" width="1064" height="384" rx="28" fill="none" stroke="#111318" stroke-width="2"/>
32
+
33
+ <path d="M132 212h835" stroke="url(#arrow)" stroke-width="10" stroke-linecap="round"/>
34
+ <path d="M965 212l-27-19v38z" fill="#4f7cff"/>
35
+
36
+ <g transform="translate(58 92)" filter="url(#card-shadow)">
37
+ <rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
38
+ <rect x="22" y="22" width="134" height="34" rx="17" fill="#111318"/>
39
+ <text x="89" y="45" text-anchor="middle" class="chip" fill="#ffffff" style="fill:#ffffff">JSON PLANE</text>
40
+ <text x="28" y="89" class="label">Manifest</text>
41
+ <text x="28" y="120" class="body">model config</text>
42
+ <text x="28" y="144" class="body">tokenizer identity</text>
43
+ </g>
44
+
45
+ <g transform="translate(272 92)" filter="url(#card-shadow)">
46
+ <rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
47
+ <rect x="22" y="22" width="134" height="34" rx="17" fill="#ffffff" stroke="#111318" stroke-width="2"/>
48
+ <text x="89" y="45" text-anchor="middle" class="chip">LOADER</text>
49
+ <text x="28" y="89" class="label">Shards</text>
50
+ <text x="28" y="120" class="body">OPFS / disk cache</text>
51
+ <text x="28" y="144" class="body">hash checks</text>
52
+ </g>
53
+
54
+ <g transform="translate(486 92)" filter="url(#card-shadow)">
55
+ <rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
56
+ <rect x="22" y="22" width="134" height="34" rx="17" fill="#111318"/>
57
+ <text x="89" y="45" text-anchor="middle" class="chip" fill="#ffffff" style="fill:#ffffff">JS PLANE</text>
58
+ <text x="28" y="89" class="label">Runtime</text>
59
+ <text x="28" y="120" class="body">prefill / decode</text>
60
+ <text x="28" y="144" class="body">KV cache policy</text>
61
+ </g>
62
+
63
+ <g transform="translate(700 92)" filter="url(#card-shadow)">
64
+ <rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
65
+ <rect x="22" y="22" width="134" height="34" rx="17" fill="#ffffff" stroke="#111318" stroke-width="2"/>
66
+ <text x="89" y="45" text-anchor="middle" class="chip">WGSL PLANE</text>
67
+ <text x="28" y="89" class="label">Kernels</text>
68
+ <text x="28" y="120" class="body">matmul / attention</text>
69
+ <text x="28" y="144" class="body">dtype lanes</text>
70
+ </g>
71
+
72
+ <g transform="translate(914 92)" filter="url(#card-shadow)">
73
+ <rect width="128" height="176" rx="24" fill="#111318"/>
74
+ <text x="64" y="48" text-anchor="middle" class="chip" fill="#ffffff" style="fill:#ffffff">OUTPUT</text>
75
+ <path d="M36 84h56M36 110h42M36 136h60" stroke="#ffffff" stroke-width="8" stroke-linecap="round"/>
76
+ <circle cx="88" cy="110" r="12" fill="#f5b942" stroke="#ffffff" stroke-width="2"/>
77
+ </g>
78
+
79
+ <g transform="translate(156 314)">
80
+ <rect x="0" y="0" width="788" height="54" rx="27" fill="#ffffff" stroke="#111318" stroke-width="2"/>
81
+ <text x="32" y="35" class="mono">kernel paths, dtype policy, benchmark contracts, and release evidence stay explicit</text>
82
+ <circle cx="740" cy="27" r="13" fill="#00a88f" stroke="#111318" stroke-width="2"/>
83
+ </g>
84
+ </svg>
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "doppler-gpu",
3
- "version": "0.4.5",
3
+ "version": "0.4.6",
4
4
  "description": "Browser-native WebGPU inference engine for local intent and inference loops",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -22,6 +22,7 @@ export interface DopplerModelHandle {
22
22
  generateText(prompt: string, options?: DopplerGenerateOptions): Promise<string>;
23
23
  chat(messages: ChatMessage[], options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
24
24
  chatText(messages: ChatMessage[], options?: DopplerGenerateOptions): Promise<DopplerChatResponse>;
25
+ resetGenerationState(): void;
25
26
  loadLoRA(adapter: LoRAManifest | RDRRManifest | string, loadOptions?: LoRALoadOptions): Promise<void>;
26
27
  activateLoRAFromTrainingOutput(
27
28
  trainingOutput:
@@ -115,6 +115,15 @@ export function createModelHandle(pipeline, resolved) {
115
115
  async unloadLoRA() {
116
116
  return unloadLoRAAdapterForPipeline(pipeline);
117
117
  },
118
+ resetGenerationState() {
119
+ if (typeof pipeline.resetGenerationState === 'function') {
120
+ return pipeline.resetGenerationState();
121
+ }
122
+ if (typeof pipeline.resetToSeqLen === 'function') {
123
+ return pipeline.resetToSeqLen(0);
124
+ }
125
+ throw new Error('Loaded Doppler pipeline does not expose generation-state reset');
126
+ },
118
127
  async unload() {
119
128
  await pipeline.unload();
120
129
  },
package/src/gpu/device.js CHANGED
@@ -369,6 +369,59 @@ function buildFeatureRequests(available) {
369
369
  return requested;
370
370
  }
371
371
 
372
+ function addFeatureRequestCandidate(candidates, seen, features) {
373
+ const unique = features.filter((feature, index) => features.indexOf(feature) === index);
374
+ const key = unique.join('|');
375
+ if (seen.has(key)) return;
376
+ seen.add(key);
377
+ candidates.push(unique);
378
+ }
379
+
380
+ function buildFeatureRequestCandidates(requestedFeatures) {
381
+ const requested = Array.isArray(requestedFeatures) ? requestedFeatures : [];
382
+ const candidates = [];
383
+ const seen = new Set();
384
+ addFeatureRequestCandidate(candidates, seen, requested);
385
+ addFeatureRequestCandidate(
386
+ candidates,
387
+ seen,
388
+ requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY)
389
+ );
390
+ addFeatureRequestCandidate(
391
+ candidates,
392
+ seen,
393
+ requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SUBGROUPS)
394
+ );
395
+ addFeatureRequestCandidate(
396
+ candidates,
397
+ seen,
398
+ requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SHADER_F16)
399
+ );
400
+ addFeatureRequestCandidate(candidates, seen, []);
401
+ return candidates;
402
+ }
403
+
404
+ async function requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits) {
405
+ const candidates = buildFeatureRequestCandidates(requestedFeatures);
406
+ let lastError = null;
407
+ for (const features of candidates) {
408
+ try {
409
+ if (features.length === 0) {
410
+ return await adapter.requestDevice();
411
+ }
412
+ return await adapter.requestDevice({
413
+ requiredFeatures: features,
414
+ requiredLimits: limits,
415
+ });
416
+ } catch (error) {
417
+ lastError = error;
418
+ const label = features.length > 0 ? features.join(', ') : 'none';
419
+ log.warn('GPU', 'Failed to request device with features [' + label + ']: ' + error.message);
420
+ }
421
+ }
422
+ throw lastError || new Error('Failed to request WebGPU device');
423
+ }
424
+
372
425
 
373
426
  function buildLimits(adapter) {
374
427
  const adapterLimits = adapter.limits;
@@ -457,17 +510,7 @@ export async function initDevice() {
457
510
  // Get adapter info (adapter.info is synchronous in modern WebGPU)
458
511
  const adapterInfo = adapter.info || { vendor: 'unknown', architecture: 'unknown', device: 'unknown', description: '' };
459
512
 
460
- try {
461
- gpuDevice = await adapter.requestDevice({
462
- requiredFeatures: requestedFeatures,
463
- requiredLimits: limits,
464
- });
465
- } catch (e) {
466
- // Fallback: request device without optional features
467
- const lostFeatures = requestedFeatures.length > 0 ? requestedFeatures.join(', ') : 'none';
468
- log.warn('GPU', 'Failed to request device with features [' + lostFeatures + '], trying minimal config: ' + (e).message);
469
- gpuDevice = await adapter.requestDevice();
470
- }
513
+ gpuDevice = await requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits);
471
514
 
472
515
  if (!gpuDevice) {
473
516
  throw createDopplerError(ERROR_CODES.GPU_DEVICE_FAILED, 'Failed to create WebGPU device');
@@ -334,7 +334,11 @@ function resolveMatmulOverride(
334
334
  if (!weightDtypeOk) {
335
335
  const overridePolicy = selectKernelRuleValue(
336
336
  'matmul', 'weightDtypeMismatchPolicy',
337
- { requiredWeightDtype, actualWeightDtype: bDtype }
337
+ {
338
+ requiredWeightDtype,
339
+ actualWeightDtype: bDtype,
340
+ hasSubgroups: capabilities?.hasSubgroups === true,
341
+ }
338
342
  );
339
343
  if (overridePolicy === 'fallthrough') {
340
344
  return null;
@@ -212,7 +212,7 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
212
212
  return entries;
213
213
  }
214
214
 
215
- function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
215
+ function resolvePreferredWeightDtype(variant, hasQ4KMaterialization, capabilities) {
216
216
  if (typeof variant !== 'string' || variant.length === 0) {
217
217
  return null;
218
218
  }
@@ -232,9 +232,34 @@ function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
232
232
  return selectKernelRuleValue('matmul', 'preferredWeightDtype', {
233
233
  variantWeightDtype,
234
234
  hasQ4KMaterialization,
235
+ hasSubgroups: capabilities?.hasSubgroups === true,
235
236
  });
236
237
  }
237
238
 
239
+ function buildF16CapabilityErrorDetail({
240
+ role,
241
+ layerIdx,
242
+ pathVariant,
243
+ preferredWeightDtype,
244
+ weightDtype,
245
+ weightLabel,
246
+ weightLayout,
247
+ weightShape,
248
+ hasQ4KMaterialization,
249
+ }) {
250
+ const parts = [];
251
+ if (role) parts.push(`role=${role}`);
252
+ if (Number.isFinite(layerIdx)) parts.push(`layer=${layerIdx}`);
253
+ if (pathVariant) parts.push(`variant=${pathVariant}`);
254
+ if (preferredWeightDtype) parts.push(`preferredWeightDtype=${preferredWeightDtype}`);
255
+ if (weightDtype) parts.push(`weightDtype=${weightDtype}`);
256
+ if (weightLabel) parts.push(`label=${weightLabel}`);
257
+ if (weightLayout) parts.push(`layout=${weightLayout}`);
258
+ if (weightShape) parts.push(`shape=${weightShape}`);
259
+ if (hasQ4KMaterialization) parts.push('q4kMaterialization=true');
260
+ return parts.length > 0 ? ` (${parts.join(', ')})` : '';
261
+ }
262
+
238
263
  async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
239
264
  const isRecord = Boolean(recorder);
240
265
  const mode = isRecord ? 'record' : 'run';
@@ -254,7 +279,7 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
254
279
  const phase = resolveMatmulPhase(M, options.phaseOverride ?? null);
255
280
  const pathVariant = getKernelPathMatmulVariant(options.role, phase, options.layerIdx, options.kernelPath);
256
281
  const hasQ4KMat = isWeightBuffer(B) && B.materializations?.q4k?.buffer != null;
257
- const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat);
282
+ const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat, capabilities);
258
283
  const resolvedWeight = resolveWeightBufferMaterialization(B, preferredWeightDtype);
259
284
  const bBuffer = getBuffer(resolvedWeight);
260
285
  const weightDtype = getWeightDtype(resolvedWeight);
@@ -282,7 +307,18 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
282
307
  const requestedOutputDtype = options.outputDtype || A.dtype;
283
308
 
284
309
  if (bDtype === 'f16' && capabilities?.hasF16 !== true) {
285
- throw new Error(`[${opLabel}] f16 weights require shader-f16 support.`);
310
+ const detail = buildF16CapabilityErrorDetail({
311
+ role: options.role,
312
+ layerIdx: options.layerIdx,
313
+ pathVariant,
314
+ preferredWeightDtype,
315
+ weightDtype,
316
+ weightLabel,
317
+ weightLayout,
318
+ weightShape,
319
+ hasQ4KMaterialization: hasQ4KMat,
320
+ });
321
+ throw new Error(`[${opLabel}] f16 weights require shader-f16 support.${detail}`);
286
322
  }
287
323
  if (requestedOutputDtype === 'f16' && capabilities?.hasF16 !== true) {
288
324
  throw new Error(`[${opLabel}] f16 output requires shader-f16 support.`);
@@ -62,6 +62,7 @@ export declare class PipelineGenerator {
62
62
  prompt: PromptInput,
63
63
  options?: GenerateOptions
64
64
  ): Promise<{ tokenIds: number[]; stats: import('./types.js').PipelineStats }>;
65
+ resetGenerationState(): void;
65
66
  resetToSeqLen(seqLen: number): void;
66
67
  prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
67
68
  computeDiffusionGemmaCanvasLogits(
@@ -430,6 +430,14 @@ export class PipelineGenerator {
430
430
  this.#state.currentSeqLen = 0;
431
431
  }
432
432
 
433
+ resetGenerationState() {
434
+ if (this.#state.isGenerating) {
435
+ throw new Error('InferencePipeline.resetGenerationState: cannot reset while generation is in progress');
436
+ }
437
+ this._resetReplayPrefillRuntimeState();
438
+ this._resetDecodeRuntimeState();
439
+ }
440
+
433
441
  async _replayPrefillDecodeLogits(currentIds, opts) {
434
442
  // Guard: cap replay-prefill sequence length to the config-owned maxSeqLen.
435
443
  // Without KV cache creation, this bound is not enforced elsewhere.
@@ -58,24 +58,65 @@ export function softmax(logits) {
58
58
  return exps;
59
59
  }
60
60
 
61
- function countFiniteCandidates(logits) {
62
- let finiteCandidateCount = 0;
61
+ function summarizeLogitHealth(logits) {
62
+ const health = {
63
+ length: logits.length,
64
+ finite: 0,
65
+ nan: 0,
66
+ posInf: 0,
67
+ negInf: 0,
68
+ min: Infinity,
69
+ max: -Infinity,
70
+ };
63
71
  for (let i = 0; i < logits.length; i++) {
64
- if (Number.isFinite(logits[i])) {
65
- finiteCandidateCount += 1;
72
+ const value = logits[i];
73
+ if (Number.isNaN(value)) {
74
+ health.nan += 1;
75
+ continue;
76
+ }
77
+ if (value === Infinity) {
78
+ health.posInf += 1;
79
+ continue;
80
+ }
81
+ if (value === -Infinity) {
82
+ health.negInf += 1;
83
+ continue;
66
84
  }
85
+ if (Number.isFinite(value)) {
86
+ health.finite += 1;
87
+ if (value < health.min) health.min = value;
88
+ if (value > health.max) health.max = value;
89
+ }
90
+ }
91
+ if (health.finite === 0) {
92
+ health.min = null;
93
+ health.max = null;
67
94
  }
68
- return finiteCandidateCount;
95
+ return health;
96
+ }
97
+
98
+ function formatLogitHealth(health) {
99
+ return [
100
+ `len=${health.length}`,
101
+ `finite=${health.finite}`,
102
+ `nan=${health.nan}`,
103
+ `posInf=${health.posInf}`,
104
+ `negInf=${health.negInf}`,
105
+ `min=${health.min === null ? 'n/a' : Number(health.min).toPrecision(6)}`,
106
+ `max=${health.max === null ? 'n/a' : Number(health.max).toPrecision(6)}`,
107
+ ].join(',');
69
108
  }
70
109
 
71
- function assertFiniteSamplingCandidates(logits, label) {
72
- const finiteCandidateCount = countFiniteCandidates(logits);
73
- if (finiteCandidateCount > 0) {
110
+ function assertFiniteSamplingCandidates(logits, label, beforeMaskHealth) {
111
+ const afterMaskHealth = summarizeLogitHealth(logits);
112
+ if (afterMaskHealth.finite > 0) {
74
113
  return;
75
114
  }
76
115
  throw new Error(
77
116
  `[Sampling] ${label} has no finite candidate logits after masking suppressed tokens. ` +
78
- 'Upstream decode likely produced NaN/Inf or an all-masked distribution.'
117
+ 'Upstream decode likely produced NaN/Inf or an all-masked distribution. ' +
118
+ `beforeMask={${formatLogitHealth(beforeMaskHealth || afterMaskHealth)}} ` +
119
+ `afterMask={${formatLogitHealth(afterMaskHealth)}}`
79
120
  );
80
121
  }
81
122
 
@@ -215,6 +256,7 @@ function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug,
215
256
 
216
257
  export function sample(logits, opts) {
217
258
  const { temperature, topP, topK, decode, debug = false, padTokenId, seed, suppressTokenIds } = opts;
259
+ const beforeMaskHealth = summarizeLogitHealth(logits);
218
260
 
219
261
  if (padTokenId !== undefined && padTokenId >= 0 && padTokenId < logits.length) {
220
262
  logits[padTokenId] = -Infinity;
@@ -227,7 +269,7 @@ export function sample(logits, opts) {
227
269
  }
228
270
  }
229
271
 
230
- assertFiniteSamplingCandidates(logits, 'Logits');
272
+ assertFiniteSamplingCandidates(logits, 'Logits', beforeMaskHealth);
231
273
 
232
274
  // Greedy (argmax) when temperature = 0
233
275
  if (temperature === 0) {
@@ -1019,14 +1019,48 @@ export async function loadFloat(shardData, location, name, config) {
1019
1019
  writeBufferAligned(device, buffer, shardData);
1020
1020
  }
1021
1021
 
1022
- const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
1023
- locationDtype: location.dtype,
1024
- });
1025
1022
  const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
1026
1023
  layout: location.layout ?? null,
1027
1024
  useColumnWise: false,
1028
1025
  });
1029
1026
  const isMatmulWeight = shouldDequantizeToF16(location);
1027
+ const caps = config.gpuCapabilities || getKernelCapabilities();
1028
+ const dtype = isMatmulWeight
1029
+ ? selectRuleValue('loader', 'weights', 'matmulWeightDtype', {
1030
+ locationDtype: location.dtype,
1031
+ hasF16: Boolean(caps?.hasF16),
1032
+ isMatmulWeight,
1033
+ keepF32Weights: Boolean(config.keepF32Weights),
1034
+ })
1035
+ : selectRuleValue('loader', 'weights', 'floatLocationDtype', {
1036
+ locationDtype: location.dtype,
1037
+ });
1038
+
1039
+ if (location.dtype === 'F16' && dtype === 'f32') {
1040
+ if (isGpuBufferInstance(shardData)) {
1041
+ throw new Error(
1042
+ `Tensor "${name}" requires CPU F16->F32 materialization on this runtime, ` +
1043
+ 'but the source was already uploaded to a GPU buffer.'
1044
+ );
1045
+ }
1046
+ const f32Values = convertF16ToF32CPU(toUint16View(shardData, `F16 tensor load for ${name}`));
1047
+ const f32Buffer = acquireAlignedBuffer(f32Values.byteLength, `${name}_f32`);
1048
+ resultBuffer = f32Buffer;
1049
+ writeBufferAligned(device, f32Buffer, new Uint8Array(f32Values.buffer, f32Values.byteOffset, f32Values.byteLength));
1050
+ releaseOwnedGpuBuffer(buffer, ownsBuffer);
1051
+ ownsBuffer = false;
1052
+ resultBuffer = null;
1053
+ if (isMatmulWeight) {
1054
+ return {
1055
+ data: createWeightBuffer(f32Buffer, 'f32', layout, location.shape, name),
1056
+ allocatedBuffers: [f32Buffer],
1057
+ };
1058
+ }
1059
+ return {
1060
+ data: applyBufferLayout(f32Buffer, location, 'f32'),
1061
+ allocatedBuffers: [f32Buffer],
1062
+ };
1063
+ }
1030
1064
 
1031
1065
  if (isMatmulWeight) {
1032
1066
  ownsBuffer = false;
@@ -10,6 +10,7 @@
10
10
  { "match": {}, "value": "f32" }
11
11
  ],
12
12
  "matmulWeightDtype": [
13
+ { "match": { "locationDtype": "F16", "hasF16": false, "isMatmulWeight": true }, "value": "f32" },
13
14
  { "match": { "locationDtype": "F16" }, "value": "f16" },
14
15
  { "match": { "locationDtype": "BF16", "hasF16": true, "isMatmulWeight": true }, "value": "f16" },
15
16
  { "match": { "locationDtype": "BF16" }, "value": "f32" },
package/src/version.js CHANGED
@@ -1,2 +1,2 @@
1
- export const DOPPLER_VERSION = '0.4.5';
1
+ export const DOPPLER_VERSION = '0.4.6';
2
2
  export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;