doppler-gpu 0.4.5 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/assets/doppler-readme-hero.svg +84 -0
- package/assets/doppler-runtime-map.svg +84 -0
- package/package.json +1 -1
- package/src/client/runtime/model-session.d.ts +1 -0
- package/src/client/runtime/model-session.js +9 -0
- package/src/gpu/device.js +54 -11
- package/src/gpu/kernels/matmul-selection.js +5 -1
- package/src/gpu/kernels/matmul.js +39 -3
- package/src/inference/pipelines/text/generator.d.ts +1 -0
- package/src/inference/pipelines/text/generator.js +8 -0
- package/src/inference/pipelines/text/sampling.js +52 -10
- package/src/loader/tensors/tensor-loader.js +37 -3
- package/src/rules/loader/weights.rules.json +1 -0
- package/src/version.js +1 -1
package/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# doppler-gpu
|
|
2
2
|
|
|
3
|
+

|
|
4
|
+
|
|
3
5
|
WebGPU inference runtime for browser, Node, Bun, CLI, and local server use.
|
|
4
6
|
Doppler runs JavaScript orchestration over WGSL kernels and loads models from
|
|
5
7
|
RDRR manifests.
|
|
@@ -42,6 +44,8 @@ completion gate is `src/config/goal-completion-matrix.json`, checked by
|
|
|
42
44
|
|
|
43
45
|
## How it works
|
|
44
46
|
|
|
47
|
+

|
|
48
|
+
|
|
45
49
|
1. A registry ID or model URL resolves to an RDRR manifest and weight shards.
|
|
46
50
|
2. The manifest owns model parameters, tokenizer metadata, session policy, and
|
|
47
51
|
execution graph.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="360" viewBox="0 0 1200 360" role="img" aria-labelledby="title desc">
|
|
2
|
+
<title id="title">Doppler WebGPU inference runtime</title>
|
|
3
|
+
<desc id="desc">A product hero for Doppler showing browser, manifest, WebGPU kernels, and receipt-backed evidence lanes.</desc>
|
|
4
|
+
<defs>
|
|
5
|
+
<linearGradient id="hero-bg" x1="40" y1="20" x2="1160" y2="330" gradientUnits="userSpaceOnUse">
|
|
6
|
+
<stop offset="0" stop-color="#fff7ed"/>
|
|
7
|
+
<stop offset="0.36" stop-color="#edf7ff"/>
|
|
8
|
+
<stop offset="0.72" stop-color="#f1fff6"/>
|
|
9
|
+
<stop offset="1" stop-color="#fff0f4"/>
|
|
10
|
+
</linearGradient>
|
|
11
|
+
<linearGradient id="lane-a" x1="0" y1="0" x2="310" y2="0" gradientUnits="userSpaceOnUse">
|
|
12
|
+
<stop offset="0" stop-color="#ff6b7a"/>
|
|
13
|
+
<stop offset="0.48" stop-color="#00a88f"/>
|
|
14
|
+
<stop offset="1" stop-color="#4f7cff"/>
|
|
15
|
+
</linearGradient>
|
|
16
|
+
<linearGradient id="lane-b" x1="0" y1="0" x2="190" y2="140" gradientUnits="userSpaceOnUse">
|
|
17
|
+
<stop offset="0" stop-color="#111318"/>
|
|
18
|
+
<stop offset="1" stop-color="#353946"/>
|
|
19
|
+
</linearGradient>
|
|
20
|
+
<filter id="soft-shadow" x="-20%" y="-25%" width="140%" height="150%">
|
|
21
|
+
<feDropShadow dx="0" dy="10" stdDeviation="12" flood-color="#111318" flood-opacity="0.14"/>
|
|
22
|
+
</filter>
|
|
23
|
+
<style>
|
|
24
|
+
text { font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; fill: #111318; }
|
|
25
|
+
.mono { font-family: "SFMono-Regular", "Cascadia Code", "Liberation Mono", Menlo, monospace; }
|
|
26
|
+
.title { font-size: 72px; font-weight: 900; letter-spacing: 2px; }
|
|
27
|
+
.subtitle { font-size: 25px; font-weight: 700; fill: #3b414c; }
|
|
28
|
+
.small { font-size: 17px; font-weight: 800; letter-spacing: 1px; }
|
|
29
|
+
.tiny { font-size: 14px; font-weight: 800; fill: #4c5360; }
|
|
30
|
+
.white { fill: #ffffff; }
|
|
31
|
+
</style>
|
|
32
|
+
</defs>
|
|
33
|
+
|
|
34
|
+
<rect width="1200" height="360" rx="42" fill="#ffffff"/>
|
|
35
|
+
<rect x="18" y="18" width="1164" height="324" rx="34" fill="url(#hero-bg)"/>
|
|
36
|
+
<rect x="18" y="18" width="1164" height="324" rx="34" fill="none" stroke="#111318" stroke-width="2"/>
|
|
37
|
+
|
|
38
|
+
<g transform="translate(72 68)" filter="url(#soft-shadow)">
|
|
39
|
+
<rect x="0" y="0" width="228" height="228" rx="52" fill="#111318"/>
|
|
40
|
+
<path d="M66 65h48c61 0 99 37 99 86s-38 86-99 86H66z" transform="translate(-27 -18)" fill="none" stroke="#ffffff" stroke-width="30" stroke-linecap="round" stroke-linejoin="round"/>
|
|
41
|
+
<path d="M102 58c46 0 76 31 76 70s-30 70-76 70" fill="none" stroke="url(#lane-a)" stroke-width="17" stroke-linecap="round"/>
|
|
42
|
+
<path d="M32 158h48M32 180h76" stroke="#ffffff" stroke-opacity="0.92" stroke-width="10" stroke-linecap="round"/>
|
|
43
|
+
<circle cx="180" cy="58" r="11" fill="#ff6b7a"/>
|
|
44
|
+
<circle cx="188" cy="198" r="11" fill="#4f7cff"/>
|
|
45
|
+
</g>
|
|
46
|
+
|
|
47
|
+
<g transform="translate(350 78)">
|
|
48
|
+
<text x="0" y="58" class="title">Doppler</text>
|
|
49
|
+
<text x="4" y="101" class="subtitle">Browser-native WebGPU inference with explicit runtime contracts.</text>
|
|
50
|
+
|
|
51
|
+
<g transform="translate(4 137)">
|
|
52
|
+
<rect x="0" y="0" width="188" height="42" rx="21" fill="#111318"/>
|
|
53
|
+
<text x="94" y="28" text-anchor="middle" class="small white">RDRR manifests</text>
|
|
54
|
+
<rect x="204" y="0" width="136" height="42" rx="21" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
55
|
+
<text x="272" y="28" text-anchor="middle" class="small">JS orchestration</text>
|
|
56
|
+
<rect x="356" y="0" width="134" height="42" rx="21" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
57
|
+
<text x="423" y="28" text-anchor="middle" class="small">WGSL kernels</text>
|
|
58
|
+
<rect x="506" y="0" width="152" height="42" rx="21" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
59
|
+
<text x="582" y="28" text-anchor="middle" class="small">evidence gates</text>
|
|
60
|
+
</g>
|
|
61
|
+
|
|
62
|
+
<g transform="translate(8 218)">
|
|
63
|
+
<path d="M0 0h310" stroke="url(#lane-a)" stroke-width="9" stroke-linecap="round"/>
|
|
64
|
+
<circle cx="0" cy="0" r="12" fill="#ff6b7a" stroke="#111318" stroke-width="2"/>
|
|
65
|
+
<circle cx="104" cy="0" r="12" fill="#f5b942" stroke="#111318" stroke-width="2"/>
|
|
66
|
+
<circle cx="208" cy="0" r="12" fill="#00a88f" stroke="#111318" stroke-width="2"/>
|
|
67
|
+
<circle cx="310" cy="0" r="12" fill="#4f7cff" stroke="#111318" stroke-width="2"/>
|
|
68
|
+
<text x="0" y="36" text-anchor="middle" class="tiny">model</text>
|
|
69
|
+
<text x="104" y="36" text-anchor="middle" class="tiny">cache</text>
|
|
70
|
+
<text x="208" y="36" text-anchor="middle" class="tiny">decode</text>
|
|
71
|
+
<text x="310" y="36" text-anchor="middle" class="tiny">receipt</text>
|
|
72
|
+
</g>
|
|
73
|
+
</g>
|
|
74
|
+
|
|
75
|
+
<g transform="translate(922 88)" filter="url(#soft-shadow)">
|
|
76
|
+
<rect x="0" y="0" width="190" height="180" rx="34" fill="url(#lane-b)"/>
|
|
77
|
+
<path d="M40 126c33-74 77-88 114-18" fill="none" stroke="#ffffff" stroke-width="10" stroke-linecap="round" opacity="0.9"/>
|
|
78
|
+
<path d="M40 126c33-74 77-88 114-18" fill="none" stroke="url(#lane-a)" stroke-width="4" stroke-linecap="round"/>
|
|
79
|
+
<circle cx="40" cy="126" r="13" fill="#ff6b7a" stroke="#ffffff" stroke-width="2"/>
|
|
80
|
+
<circle cx="93" cy="49" r="13" fill="#f5b942" stroke="#ffffff" stroke-width="2"/>
|
|
81
|
+
<circle cx="154" cy="108" r="13" fill="#4f7cff" stroke="#ffffff" stroke-width="2"/>
|
|
82
|
+
<text x="95" y="153" text-anchor="middle" class="small white">Metal / Vulkan</text>
|
|
83
|
+
</g>
|
|
84
|
+
</svg>
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420" role="img" aria-labelledby="title desc">
|
|
2
|
+
<title id="title">Doppler runtime contract map</title>
|
|
3
|
+
<desc id="desc">A diagram of the Doppler path from manifest and shards through loader, JavaScript orchestration, WGSL kernels, output, and evidence.</desc>
|
|
4
|
+
<defs>
|
|
5
|
+
<linearGradient id="map-bg" x1="40" y1="30" x2="1060" y2="390" gradientUnits="userSpaceOnUse">
|
|
6
|
+
<stop offset="0" stop-color="#ffffff"/>
|
|
7
|
+
<stop offset="0.42" stop-color="#eef8f4"/>
|
|
8
|
+
<stop offset="0.74" stop-color="#edf3ff"/>
|
|
9
|
+
<stop offset="1" stop-color="#fff5ec"/>
|
|
10
|
+
</linearGradient>
|
|
11
|
+
<linearGradient id="arrow" x1="0" y1="0" x2="900" y2="0" gradientUnits="userSpaceOnUse">
|
|
12
|
+
<stop offset="0" stop-color="#ff6b7a"/>
|
|
13
|
+
<stop offset="0.32" stop-color="#f5b942"/>
|
|
14
|
+
<stop offset="0.63" stop-color="#00a88f"/>
|
|
15
|
+
<stop offset="1" stop-color="#4f7cff"/>
|
|
16
|
+
</linearGradient>
|
|
17
|
+
<filter id="card-shadow" x="-10%" y="-20%" width="120%" height="145%">
|
|
18
|
+
<feDropShadow dx="0" dy="8" stdDeviation="9" flood-color="#111318" flood-opacity="0.11"/>
|
|
19
|
+
</filter>
|
|
20
|
+
<style>
|
|
21
|
+
text { font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; fill: #111318; }
|
|
22
|
+
.label { font-size: 24px; font-weight: 900; }
|
|
23
|
+
.body { font-size: 15px; font-weight: 700; fill: #4a515d; }
|
|
24
|
+
.chip { font-size: 13px; font-weight: 850; letter-spacing: 1px; }
|
|
25
|
+
.mono { font-family: "SFMono-Regular", "Cascadia Code", "Liberation Mono", Menlo, monospace; font-size: 14px; font-weight: 800; fill: #303640; }
|
|
26
|
+
</style>
|
|
27
|
+
</defs>
|
|
28
|
+
|
|
29
|
+
<rect width="1100" height="420" rx="34" fill="#ffffff"/>
|
|
30
|
+
<rect x="18" y="18" width="1064" height="384" rx="28" fill="url(#map-bg)"/>
|
|
31
|
+
<rect x="18" y="18" width="1064" height="384" rx="28" fill="none" stroke="#111318" stroke-width="2"/>
|
|
32
|
+
|
|
33
|
+
<path d="M132 212h835" stroke="url(#arrow)" stroke-width="10" stroke-linecap="round"/>
|
|
34
|
+
<path d="M965 212l-27-19v38z" fill="#4f7cff"/>
|
|
35
|
+
|
|
36
|
+
<g transform="translate(58 92)" filter="url(#card-shadow)">
|
|
37
|
+
<rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
38
|
+
<rect x="22" y="22" width="134" height="34" rx="17" fill="#111318"/>
|
|
39
|
+
<text x="89" y="45" text-anchor="middle" class="chip" fill="#ffffff" style="fill:#ffffff">JSON PLANE</text>
|
|
40
|
+
<text x="28" y="89" class="label">Manifest</text>
|
|
41
|
+
<text x="28" y="120" class="body">model config</text>
|
|
42
|
+
<text x="28" y="144" class="body">tokenizer identity</text>
|
|
43
|
+
</g>
|
|
44
|
+
|
|
45
|
+
<g transform="translate(272 92)" filter="url(#card-shadow)">
|
|
46
|
+
<rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
47
|
+
<rect x="22" y="22" width="134" height="34" rx="17" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
48
|
+
<text x="89" y="45" text-anchor="middle" class="chip">LOADER</text>
|
|
49
|
+
<text x="28" y="89" class="label">Shards</text>
|
|
50
|
+
<text x="28" y="120" class="body">OPFS / disk cache</text>
|
|
51
|
+
<text x="28" y="144" class="body">hash checks</text>
|
|
52
|
+
</g>
|
|
53
|
+
|
|
54
|
+
<g transform="translate(486 92)" filter="url(#card-shadow)">
|
|
55
|
+
<rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
56
|
+
<rect x="22" y="22" width="134" height="34" rx="17" fill="#111318"/>
|
|
57
|
+
<text x="89" y="45" text-anchor="middle" class="chip" fill="#ffffff" style="fill:#ffffff">JS PLANE</text>
|
|
58
|
+
<text x="28" y="89" class="label">Runtime</text>
|
|
59
|
+
<text x="28" y="120" class="body">prefill / decode</text>
|
|
60
|
+
<text x="28" y="144" class="body">KV cache policy</text>
|
|
61
|
+
</g>
|
|
62
|
+
|
|
63
|
+
<g transform="translate(700 92)" filter="url(#card-shadow)">
|
|
64
|
+
<rect width="178" height="176" rx="24" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
65
|
+
<rect x="22" y="22" width="134" height="34" rx="17" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
66
|
+
<text x="89" y="45" text-anchor="middle" class="chip">WGSL PLANE</text>
|
|
67
|
+
<text x="28" y="89" class="label">Kernels</text>
|
|
68
|
+
<text x="28" y="120" class="body">matmul / attention</text>
|
|
69
|
+
<text x="28" y="144" class="body">dtype lanes</text>
|
|
70
|
+
</g>
|
|
71
|
+
|
|
72
|
+
<g transform="translate(914 92)" filter="url(#card-shadow)">
|
|
73
|
+
<rect width="128" height="176" rx="24" fill="#111318"/>
|
|
74
|
+
<text x="64" y="48" text-anchor="middle" class="chip" fill="#ffffff" style="fill:#ffffff">OUTPUT</text>
|
|
75
|
+
<path d="M36 84h56M36 110h42M36 136h60" stroke="#ffffff" stroke-width="8" stroke-linecap="round"/>
|
|
76
|
+
<circle cx="88" cy="110" r="12" fill="#f5b942" stroke="#ffffff" stroke-width="2"/>
|
|
77
|
+
</g>
|
|
78
|
+
|
|
79
|
+
<g transform="translate(156 314)">
|
|
80
|
+
<rect x="0" y="0" width="788" height="54" rx="27" fill="#ffffff" stroke="#111318" stroke-width="2"/>
|
|
81
|
+
<text x="32" y="35" class="mono">kernel paths, dtype policy, benchmark contracts, and release evidence stay explicit</text>
|
|
82
|
+
<circle cx="740" cy="27" r="13" fill="#00a88f" stroke="#111318" stroke-width="2"/>
|
|
83
|
+
</g>
|
|
84
|
+
</svg>
|
package/package.json
CHANGED
|
@@ -22,6 +22,7 @@ export interface DopplerModelHandle {
|
|
|
22
22
|
generateText(prompt: string, options?: DopplerGenerateOptions): Promise<string>;
|
|
23
23
|
chat(messages: ChatMessage[], options?: DopplerGenerateOptions): AsyncGenerator<string, void, void>;
|
|
24
24
|
chatText(messages: ChatMessage[], options?: DopplerGenerateOptions): Promise<DopplerChatResponse>;
|
|
25
|
+
resetGenerationState(): void;
|
|
25
26
|
loadLoRA(adapter: LoRAManifest | RDRRManifest | string, loadOptions?: LoRALoadOptions): Promise<void>;
|
|
26
27
|
activateLoRAFromTrainingOutput(
|
|
27
28
|
trainingOutput:
|
|
@@ -115,6 +115,15 @@ export function createModelHandle(pipeline, resolved) {
|
|
|
115
115
|
async unloadLoRA() {
|
|
116
116
|
return unloadLoRAAdapterForPipeline(pipeline);
|
|
117
117
|
},
|
|
118
|
+
resetGenerationState() {
|
|
119
|
+
if (typeof pipeline.resetGenerationState === 'function') {
|
|
120
|
+
return pipeline.resetGenerationState();
|
|
121
|
+
}
|
|
122
|
+
if (typeof pipeline.resetToSeqLen === 'function') {
|
|
123
|
+
return pipeline.resetToSeqLen(0);
|
|
124
|
+
}
|
|
125
|
+
throw new Error('Loaded Doppler pipeline does not expose generation-state reset');
|
|
126
|
+
},
|
|
118
127
|
async unload() {
|
|
119
128
|
await pipeline.unload();
|
|
120
129
|
},
|
package/src/gpu/device.js
CHANGED
|
@@ -369,6 +369,59 @@ function buildFeatureRequests(available) {
|
|
|
369
369
|
return requested;
|
|
370
370
|
}
|
|
371
371
|
|
|
372
|
+
function addFeatureRequestCandidate(candidates, seen, features) {
|
|
373
|
+
const unique = features.filter((feature, index) => features.indexOf(feature) === index);
|
|
374
|
+
const key = unique.join('|');
|
|
375
|
+
if (seen.has(key)) return;
|
|
376
|
+
seen.add(key);
|
|
377
|
+
candidates.push(unique);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function buildFeatureRequestCandidates(requestedFeatures) {
|
|
381
|
+
const requested = Array.isArray(requestedFeatures) ? requestedFeatures : [];
|
|
382
|
+
const candidates = [];
|
|
383
|
+
const seen = new Set();
|
|
384
|
+
addFeatureRequestCandidate(candidates, seen, requested);
|
|
385
|
+
addFeatureRequestCandidate(
|
|
386
|
+
candidates,
|
|
387
|
+
seen,
|
|
388
|
+
requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY)
|
|
389
|
+
);
|
|
390
|
+
addFeatureRequestCandidate(
|
|
391
|
+
candidates,
|
|
392
|
+
seen,
|
|
393
|
+
requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SUBGROUPS)
|
|
394
|
+
);
|
|
395
|
+
addFeatureRequestCandidate(
|
|
396
|
+
candidates,
|
|
397
|
+
seen,
|
|
398
|
+
requested.filter((feature) => feature !== FEATURES.TIMESTAMP_QUERY && feature !== FEATURES.SHADER_F16)
|
|
399
|
+
);
|
|
400
|
+
addFeatureRequestCandidate(candidates, seen, []);
|
|
401
|
+
return candidates;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
async function requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits) {
|
|
405
|
+
const candidates = buildFeatureRequestCandidates(requestedFeatures);
|
|
406
|
+
let lastError = null;
|
|
407
|
+
for (const features of candidates) {
|
|
408
|
+
try {
|
|
409
|
+
if (features.length === 0) {
|
|
410
|
+
return await adapter.requestDevice();
|
|
411
|
+
}
|
|
412
|
+
return await adapter.requestDevice({
|
|
413
|
+
requiredFeatures: features,
|
|
414
|
+
requiredLimits: limits,
|
|
415
|
+
});
|
|
416
|
+
} catch (error) {
|
|
417
|
+
lastError = error;
|
|
418
|
+
const label = features.length > 0 ? features.join(', ') : 'none';
|
|
419
|
+
log.warn('GPU', 'Failed to request device with features [' + label + ']: ' + error.message);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
throw lastError || new Error('Failed to request WebGPU device');
|
|
423
|
+
}
|
|
424
|
+
|
|
372
425
|
|
|
373
426
|
function buildLimits(adapter) {
|
|
374
427
|
const adapterLimits = adapter.limits;
|
|
@@ -457,17 +510,7 @@ export async function initDevice() {
|
|
|
457
510
|
// Get adapter info (adapter.info is synchronous in modern WebGPU)
|
|
458
511
|
const adapterInfo = adapter.info || { vendor: 'unknown', architecture: 'unknown', device: 'unknown', description: '' };
|
|
459
512
|
|
|
460
|
-
|
|
461
|
-
gpuDevice = await adapter.requestDevice({
|
|
462
|
-
requiredFeatures: requestedFeatures,
|
|
463
|
-
requiredLimits: limits,
|
|
464
|
-
});
|
|
465
|
-
} catch (e) {
|
|
466
|
-
// Fallback: request device without optional features
|
|
467
|
-
const lostFeatures = requestedFeatures.length > 0 ? requestedFeatures.join(', ') : 'none';
|
|
468
|
-
log.warn('GPU', 'Failed to request device with features [' + lostFeatures + '], trying minimal config: ' + (e).message);
|
|
469
|
-
gpuDevice = await adapter.requestDevice();
|
|
470
|
-
}
|
|
513
|
+
gpuDevice = await requestDeviceWithFeatureFallback(adapter, requestedFeatures, limits);
|
|
471
514
|
|
|
472
515
|
if (!gpuDevice) {
|
|
473
516
|
throw createDopplerError(ERROR_CODES.GPU_DEVICE_FAILED, 'Failed to create WebGPU device');
|
|
@@ -334,7 +334,11 @@ function resolveMatmulOverride(
|
|
|
334
334
|
if (!weightDtypeOk) {
|
|
335
335
|
const overridePolicy = selectKernelRuleValue(
|
|
336
336
|
'matmul', 'weightDtypeMismatchPolicy',
|
|
337
|
-
{
|
|
337
|
+
{
|
|
338
|
+
requiredWeightDtype,
|
|
339
|
+
actualWeightDtype: bDtype,
|
|
340
|
+
hasSubgroups: capabilities?.hasSubgroups === true,
|
|
341
|
+
}
|
|
338
342
|
);
|
|
339
343
|
if (overridePolicy === 'fallthrough') {
|
|
340
344
|
return null;
|
|
@@ -212,7 +212,7 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
212
212
|
return entries;
|
|
213
213
|
}
|
|
214
214
|
|
|
215
|
-
function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
|
|
215
|
+
function resolvePreferredWeightDtype(variant, hasQ4KMaterialization, capabilities) {
|
|
216
216
|
if (typeof variant !== 'string' || variant.length === 0) {
|
|
217
217
|
return null;
|
|
218
218
|
}
|
|
@@ -232,9 +232,34 @@ function resolvePreferredWeightDtype(variant, hasQ4KMaterialization) {
|
|
|
232
232
|
return selectKernelRuleValue('matmul', 'preferredWeightDtype', {
|
|
233
233
|
variantWeightDtype,
|
|
234
234
|
hasQ4KMaterialization,
|
|
235
|
+
hasSubgroups: capabilities?.hasSubgroups === true,
|
|
235
236
|
});
|
|
236
237
|
}
|
|
237
238
|
|
|
239
|
+
function buildF16CapabilityErrorDetail({
|
|
240
|
+
role,
|
|
241
|
+
layerIdx,
|
|
242
|
+
pathVariant,
|
|
243
|
+
preferredWeightDtype,
|
|
244
|
+
weightDtype,
|
|
245
|
+
weightLabel,
|
|
246
|
+
weightLayout,
|
|
247
|
+
weightShape,
|
|
248
|
+
hasQ4KMaterialization,
|
|
249
|
+
}) {
|
|
250
|
+
const parts = [];
|
|
251
|
+
if (role) parts.push(`role=${role}`);
|
|
252
|
+
if (Number.isFinite(layerIdx)) parts.push(`layer=${layerIdx}`);
|
|
253
|
+
if (pathVariant) parts.push(`variant=${pathVariant}`);
|
|
254
|
+
if (preferredWeightDtype) parts.push(`preferredWeightDtype=${preferredWeightDtype}`);
|
|
255
|
+
if (weightDtype) parts.push(`weightDtype=${weightDtype}`);
|
|
256
|
+
if (weightLabel) parts.push(`label=${weightLabel}`);
|
|
257
|
+
if (weightLayout) parts.push(`layout=${weightLayout}`);
|
|
258
|
+
if (weightShape) parts.push(`shape=${weightShape}`);
|
|
259
|
+
if (hasQ4KMaterialization) parts.push('q4kMaterialization=true');
|
|
260
|
+
return parts.length > 0 ? ` (${parts.join(', ')})` : '';
|
|
261
|
+
}
|
|
262
|
+
|
|
238
263
|
async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
239
264
|
const isRecord = Boolean(recorder);
|
|
240
265
|
const mode = isRecord ? 'record' : 'run';
|
|
@@ -254,7 +279,7 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
254
279
|
const phase = resolveMatmulPhase(M, options.phaseOverride ?? null);
|
|
255
280
|
const pathVariant = getKernelPathMatmulVariant(options.role, phase, options.layerIdx, options.kernelPath);
|
|
256
281
|
const hasQ4KMat = isWeightBuffer(B) && B.materializations?.q4k?.buffer != null;
|
|
257
|
-
const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat);
|
|
282
|
+
const preferredWeightDtype = resolvePreferredWeightDtype(pathVariant, hasQ4KMat, capabilities);
|
|
258
283
|
const resolvedWeight = resolveWeightBufferMaterialization(B, preferredWeightDtype);
|
|
259
284
|
const bBuffer = getBuffer(resolvedWeight);
|
|
260
285
|
const weightDtype = getWeightDtype(resolvedWeight);
|
|
@@ -282,7 +307,18 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
282
307
|
const requestedOutputDtype = options.outputDtype || A.dtype;
|
|
283
308
|
|
|
284
309
|
if (bDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
285
|
-
|
|
310
|
+
const detail = buildF16CapabilityErrorDetail({
|
|
311
|
+
role: options.role,
|
|
312
|
+
layerIdx: options.layerIdx,
|
|
313
|
+
pathVariant,
|
|
314
|
+
preferredWeightDtype,
|
|
315
|
+
weightDtype,
|
|
316
|
+
weightLabel,
|
|
317
|
+
weightLayout,
|
|
318
|
+
weightShape,
|
|
319
|
+
hasQ4KMaterialization: hasQ4KMat,
|
|
320
|
+
});
|
|
321
|
+
throw new Error(`[${opLabel}] f16 weights require shader-f16 support.${detail}`);
|
|
286
322
|
}
|
|
287
323
|
if (requestedOutputDtype === 'f16' && capabilities?.hasF16 !== true) {
|
|
288
324
|
throw new Error(`[${opLabel}] f16 output requires shader-f16 support.`);
|
|
@@ -62,6 +62,7 @@ export declare class PipelineGenerator {
|
|
|
62
62
|
prompt: PromptInput,
|
|
63
63
|
options?: GenerateOptions
|
|
64
64
|
): Promise<{ tokenIds: number[]; stats: import('./types.js').PipelineStats }>;
|
|
65
|
+
resetGenerationState(): void;
|
|
65
66
|
resetToSeqLen(seqLen: number): void;
|
|
66
67
|
prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
|
|
67
68
|
computeDiffusionGemmaCanvasLogits(
|
|
@@ -430,6 +430,14 @@ export class PipelineGenerator {
|
|
|
430
430
|
this.#state.currentSeqLen = 0;
|
|
431
431
|
}
|
|
432
432
|
|
|
433
|
+
resetGenerationState() {
|
|
434
|
+
if (this.#state.isGenerating) {
|
|
435
|
+
throw new Error('InferencePipeline.resetGenerationState: cannot reset while generation is in progress');
|
|
436
|
+
}
|
|
437
|
+
this._resetReplayPrefillRuntimeState();
|
|
438
|
+
this._resetDecodeRuntimeState();
|
|
439
|
+
}
|
|
440
|
+
|
|
433
441
|
async _replayPrefillDecodeLogits(currentIds, opts) {
|
|
434
442
|
// Guard: cap replay-prefill sequence length to the config-owned maxSeqLen.
|
|
435
443
|
// Without KV cache creation, this bound is not enforced elsewhere.
|
|
@@ -58,24 +58,65 @@ export function softmax(logits) {
|
|
|
58
58
|
return exps;
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
function
|
|
62
|
-
|
|
61
|
+
function summarizeLogitHealth(logits) {
|
|
62
|
+
const health = {
|
|
63
|
+
length: logits.length,
|
|
64
|
+
finite: 0,
|
|
65
|
+
nan: 0,
|
|
66
|
+
posInf: 0,
|
|
67
|
+
negInf: 0,
|
|
68
|
+
min: Infinity,
|
|
69
|
+
max: -Infinity,
|
|
70
|
+
};
|
|
63
71
|
for (let i = 0; i < logits.length; i++) {
|
|
64
|
-
|
|
65
|
-
|
|
72
|
+
const value = logits[i];
|
|
73
|
+
if (Number.isNaN(value)) {
|
|
74
|
+
health.nan += 1;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (value === Infinity) {
|
|
78
|
+
health.posInf += 1;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (value === -Infinity) {
|
|
82
|
+
health.negInf += 1;
|
|
83
|
+
continue;
|
|
66
84
|
}
|
|
85
|
+
if (Number.isFinite(value)) {
|
|
86
|
+
health.finite += 1;
|
|
87
|
+
if (value < health.min) health.min = value;
|
|
88
|
+
if (value > health.max) health.max = value;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (health.finite === 0) {
|
|
92
|
+
health.min = null;
|
|
93
|
+
health.max = null;
|
|
67
94
|
}
|
|
68
|
-
return
|
|
95
|
+
return health;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function formatLogitHealth(health) {
|
|
99
|
+
return [
|
|
100
|
+
`len=${health.length}`,
|
|
101
|
+
`finite=${health.finite}`,
|
|
102
|
+
`nan=${health.nan}`,
|
|
103
|
+
`posInf=${health.posInf}`,
|
|
104
|
+
`negInf=${health.negInf}`,
|
|
105
|
+
`min=${health.min === null ? 'n/a' : Number(health.min).toPrecision(6)}`,
|
|
106
|
+
`max=${health.max === null ? 'n/a' : Number(health.max).toPrecision(6)}`,
|
|
107
|
+
].join(',');
|
|
69
108
|
}
|
|
70
109
|
|
|
71
|
-
function assertFiniteSamplingCandidates(logits, label) {
|
|
72
|
-
const
|
|
73
|
-
if (
|
|
110
|
+
function assertFiniteSamplingCandidates(logits, label, beforeMaskHealth) {
|
|
111
|
+
const afterMaskHealth = summarizeLogitHealth(logits);
|
|
112
|
+
if (afterMaskHealth.finite > 0) {
|
|
74
113
|
return;
|
|
75
114
|
}
|
|
76
115
|
throw new Error(
|
|
77
116
|
`[Sampling] ${label} has no finite candidate logits after masking suppressed tokens. ` +
|
|
78
|
-
'Upstream decode likely produced NaN/Inf or an all-masked distribution.'
|
|
117
|
+
'Upstream decode likely produced NaN/Inf or an all-masked distribution. ' +
|
|
118
|
+
`beforeMask={${formatLogitHealth(beforeMaskHealth || afterMaskHealth)}} ` +
|
|
119
|
+
`afterMask={${formatLogitHealth(afterMaskHealth)}}`
|
|
79
120
|
);
|
|
80
121
|
}
|
|
81
122
|
|
|
@@ -215,6 +256,7 @@ function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug,
|
|
|
215
256
|
|
|
216
257
|
export function sample(logits, opts) {
|
|
217
258
|
const { temperature, topP, topK, decode, debug = false, padTokenId, seed, suppressTokenIds } = opts;
|
|
259
|
+
const beforeMaskHealth = summarizeLogitHealth(logits);
|
|
218
260
|
|
|
219
261
|
if (padTokenId !== undefined && padTokenId >= 0 && padTokenId < logits.length) {
|
|
220
262
|
logits[padTokenId] = -Infinity;
|
|
@@ -227,7 +269,7 @@ export function sample(logits, opts) {
|
|
|
227
269
|
}
|
|
228
270
|
}
|
|
229
271
|
|
|
230
|
-
assertFiniteSamplingCandidates(logits, 'Logits');
|
|
272
|
+
assertFiniteSamplingCandidates(logits, 'Logits', beforeMaskHealth);
|
|
231
273
|
|
|
232
274
|
// Greedy (argmax) when temperature = 0
|
|
233
275
|
if (temperature === 0) {
|
|
@@ -1019,14 +1019,48 @@ export async function loadFloat(shardData, location, name, config) {
|
|
|
1019
1019
|
writeBufferAligned(device, buffer, shardData);
|
|
1020
1020
|
}
|
|
1021
1021
|
|
|
1022
|
-
const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
1023
|
-
locationDtype: location.dtype,
|
|
1024
|
-
});
|
|
1025
1022
|
const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
|
|
1026
1023
|
layout: location.layout ?? null,
|
|
1027
1024
|
useColumnWise: false,
|
|
1028
1025
|
});
|
|
1029
1026
|
const isMatmulWeight = shouldDequantizeToF16(location);
|
|
1027
|
+
const caps = config.gpuCapabilities || getKernelCapabilities();
|
|
1028
|
+
const dtype = isMatmulWeight
|
|
1029
|
+
? selectRuleValue('loader', 'weights', 'matmulWeightDtype', {
|
|
1030
|
+
locationDtype: location.dtype,
|
|
1031
|
+
hasF16: Boolean(caps?.hasF16),
|
|
1032
|
+
isMatmulWeight,
|
|
1033
|
+
keepF32Weights: Boolean(config.keepF32Weights),
|
|
1034
|
+
})
|
|
1035
|
+
: selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
1036
|
+
locationDtype: location.dtype,
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
if (location.dtype === 'F16' && dtype === 'f32') {
|
|
1040
|
+
if (isGpuBufferInstance(shardData)) {
|
|
1041
|
+
throw new Error(
|
|
1042
|
+
`Tensor "${name}" requires CPU F16->F32 materialization on this runtime, ` +
|
|
1043
|
+
'but the source was already uploaded to a GPU buffer.'
|
|
1044
|
+
);
|
|
1045
|
+
}
|
|
1046
|
+
const f32Values = convertF16ToF32CPU(toUint16View(shardData, `F16 tensor load for ${name}`));
|
|
1047
|
+
const f32Buffer = acquireAlignedBuffer(f32Values.byteLength, `${name}_f32`);
|
|
1048
|
+
resultBuffer = f32Buffer;
|
|
1049
|
+
writeBufferAligned(device, f32Buffer, new Uint8Array(f32Values.buffer, f32Values.byteOffset, f32Values.byteLength));
|
|
1050
|
+
releaseOwnedGpuBuffer(buffer, ownsBuffer);
|
|
1051
|
+
ownsBuffer = false;
|
|
1052
|
+
resultBuffer = null;
|
|
1053
|
+
if (isMatmulWeight) {
|
|
1054
|
+
return {
|
|
1055
|
+
data: createWeightBuffer(f32Buffer, 'f32', layout, location.shape, name),
|
|
1056
|
+
allocatedBuffers: [f32Buffer],
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
return {
|
|
1060
|
+
data: applyBufferLayout(f32Buffer, location, 'f32'),
|
|
1061
|
+
allocatedBuffers: [f32Buffer],
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1030
1064
|
|
|
1031
1065
|
if (isMatmulWeight) {
|
|
1032
1066
|
ownsBuffer = false;
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
{ "match": {}, "value": "f32" }
|
|
11
11
|
],
|
|
12
12
|
"matmulWeightDtype": [
|
|
13
|
+
{ "match": { "locationDtype": "F16", "hasF16": false, "isMatmulWeight": true }, "value": "f32" },
|
|
13
14
|
{ "match": { "locationDtype": "F16" }, "value": "f16" },
|
|
14
15
|
{ "match": { "locationDtype": "BF16", "hasF16": true, "isMatmulWeight": true }, "value": "f16" },
|
|
15
16
|
{ "match": { "locationDtype": "BF16" }, "value": "f32" },
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.4.
|
|
1
|
+
export const DOPPLER_VERSION = '0.4.6';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|