doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -1,389 +1,32 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
1
|
import { log, trace, isTraceEnabled } from '../../../debug/index.js';
|
|
4
2
|
import { getRuntimeConfig } from '../../../config/runtime.js';
|
|
3
|
+
import {
|
|
4
|
+
applyRepetitionPenalty as applyScalarRepetitionPenalty,
|
|
5
|
+
applyPresencePenalty as applyScalarPresencePenalty,
|
|
6
|
+
sample as sampleTokens, softmax,
|
|
7
|
+
} from '../../token-sampling.js';
|
|
5
8
|
|
|
6
|
-
|
|
7
|
-
const x = Math.sin(seed) * 10000;
|
|
8
|
-
return x - Math.floor(x);
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
let fallbackRandomState = (Date.now() >>> 0) || 0x6d2b79f5;
|
|
12
|
-
|
|
13
|
-
function unseededRandom() {
|
|
14
|
-
if (typeof crypto !== 'undefined' && typeof crypto.getRandomValues === 'function') {
|
|
15
|
-
const values = new Uint32Array(1);
|
|
16
|
-
crypto.getRandomValues(values);
|
|
17
|
-
return values[0] / 4294967296;
|
|
18
|
-
}
|
|
19
|
-
fallbackRandomState = (fallbackRandomState + 0x6d2b79f5) >>> 0;
|
|
20
|
-
return fallbackRandomState / 4294967296;
|
|
21
|
-
}
|
|
22
|
-
|
|
9
|
+
export { softmax };
|
|
23
10
|
|
|
24
|
-
export function applyRepetitionPenalty(logits, previousTokens, penalty) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const windowSize = getRuntimeConfig().inference.sampling.repetitionPenaltyWindow;
|
|
28
|
-
const seen = new Set(previousTokens.slice(-windowSize));
|
|
29
|
-
for (const token of seen) {
|
|
30
|
-
if (token < logits.length) {
|
|
31
|
-
logits[token] = logits[token] > 0
|
|
32
|
-
? logits[token] / penalty
|
|
33
|
-
: logits[token] * penalty;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
export function softmax(logits) {
|
|
40
|
-
const n = logits.length;
|
|
41
|
-
let max = -Infinity;
|
|
42
|
-
for (let i = 0; i < n; i++) {
|
|
43
|
-
if (logits[i] > max) max = logits[i];
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
const exps = new Float32Array(n);
|
|
47
|
-
let sum = 0;
|
|
48
|
-
for (let i = 0; i < n; i++) {
|
|
49
|
-
const e = Math.exp(logits[i] - max);
|
|
50
|
-
exps[i] = e;
|
|
51
|
-
sum += e;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
const invSum = sum > 0 ? 1 / sum : 0;
|
|
55
|
-
for (let i = 0; i < n; i++) {
|
|
56
|
-
exps[i] *= invSum;
|
|
57
|
-
}
|
|
58
|
-
return exps;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function summarizeLogitHealth(logits) {
|
|
62
|
-
const health = {
|
|
63
|
-
length: logits.length,
|
|
64
|
-
finite: 0,
|
|
65
|
-
nan: 0,
|
|
66
|
-
posInf: 0,
|
|
67
|
-
negInf: 0,
|
|
68
|
-
min: Infinity,
|
|
69
|
-
max: -Infinity,
|
|
70
|
-
};
|
|
71
|
-
for (let i = 0; i < logits.length; i++) {
|
|
72
|
-
const value = logits[i];
|
|
73
|
-
if (Number.isNaN(value)) {
|
|
74
|
-
health.nan += 1;
|
|
75
|
-
continue;
|
|
76
|
-
}
|
|
77
|
-
if (value === Infinity) {
|
|
78
|
-
health.posInf += 1;
|
|
79
|
-
continue;
|
|
80
|
-
}
|
|
81
|
-
if (value === -Infinity) {
|
|
82
|
-
health.negInf += 1;
|
|
83
|
-
continue;
|
|
84
|
-
}
|
|
85
|
-
if (Number.isFinite(value)) {
|
|
86
|
-
health.finite += 1;
|
|
87
|
-
if (value < health.min) health.min = value;
|
|
88
|
-
if (value > health.max) health.max = value;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
if (health.finite === 0) {
|
|
92
|
-
health.min = null;
|
|
93
|
-
health.max = null;
|
|
94
|
-
}
|
|
95
|
-
return health;
|
|
11
|
+
export function applyRepetitionPenalty(logits, previousTokens, penalty, windowSize = getRuntimeConfig().inference.sampling.repetitionPenaltyWindow) {
|
|
12
|
+
applyScalarRepetitionPenalty(logits, previousTokens, penalty, windowSize);
|
|
96
13
|
}
|
|
97
14
|
|
|
98
|
-
function
|
|
99
|
-
|
|
100
|
-
`len=${health.length}`,
|
|
101
|
-
`finite=${health.finite}`,
|
|
102
|
-
`nan=${health.nan}`,
|
|
103
|
-
`posInf=${health.posInf}`,
|
|
104
|
-
`negInf=${health.negInf}`,
|
|
105
|
-
`min=${health.min === null ? 'n/a' : Number(health.min).toPrecision(6)}`,
|
|
106
|
-
`max=${health.max === null ? 'n/a' : Number(health.max).toPrecision(6)}`,
|
|
107
|
-
].join(',');
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
function assertFiniteSamplingCandidates(logits, label, beforeMaskHealth) {
|
|
111
|
-
const afterMaskHealth = summarizeLogitHealth(logits);
|
|
112
|
-
if (afterMaskHealth.finite > 0) {
|
|
113
|
-
return;
|
|
114
|
-
}
|
|
115
|
-
throw new Error(
|
|
116
|
-
`[Sampling] ${label} has no finite candidate logits after masking suppressed tokens. ` +
|
|
117
|
-
'Upstream decode likely produced NaN/Inf or an all-masked distribution. ' +
|
|
118
|
-
`beforeMask={${formatLogitHealth(beforeMaskHealth || afterMaskHealth)}} ` +
|
|
119
|
-
`afterMask={${formatLogitHealth(afterMaskHealth)}}`
|
|
120
|
-
);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function isHigherPriorityCandidate(candidate, current) {
|
|
124
|
-
return candidate.logit > current.logit
|
|
125
|
-
|| (candidate.logit === current.logit && candidate.token < current.token);
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
function isLowerPriorityCandidate(candidate, current) {
|
|
129
|
-
return candidate.logit < current.logit
|
|
130
|
-
|| (candidate.logit === current.logit && candidate.token > current.token);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function heapSwap(heap, a, b) {
|
|
134
|
-
const tmp = heap[a];
|
|
135
|
-
heap[a] = heap[b];
|
|
136
|
-
heap[b] = tmp;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function heapPushLowestFirst(heap, candidate) {
|
|
140
|
-
heap.push(candidate);
|
|
141
|
-
let index = heap.length - 1;
|
|
142
|
-
while (index > 0) {
|
|
143
|
-
const parent = Math.floor((index - 1) / 2);
|
|
144
|
-
if (!isLowerPriorityCandidate(heap[index], heap[parent])) {
|
|
145
|
-
break;
|
|
146
|
-
}
|
|
147
|
-
heapSwap(heap, index, parent);
|
|
148
|
-
index = parent;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
function heapifyLowestFirst(heap, index) {
|
|
153
|
-
while (true) {
|
|
154
|
-
const left = index * 2 + 1;
|
|
155
|
-
const right = left + 1;
|
|
156
|
-
let smallest = index;
|
|
157
|
-
if (left < heap.length && isLowerPriorityCandidate(heap[left], heap[smallest])) {
|
|
158
|
-
smallest = left;
|
|
159
|
-
}
|
|
160
|
-
if (right < heap.length && isLowerPriorityCandidate(heap[right], heap[smallest])) {
|
|
161
|
-
smallest = right;
|
|
162
|
-
}
|
|
163
|
-
if (smallest === index) {
|
|
164
|
-
return;
|
|
165
|
-
}
|
|
166
|
-
heapSwap(heap, index, smallest);
|
|
167
|
-
index = smallest;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
function selectTopKLogitCandidates(logits, topK) {
|
|
172
|
-
const limit = Math.max(1, Math.floor(topK));
|
|
173
|
-
if (!Number.isFinite(limit) || limit >= logits.length) {
|
|
174
|
-
return null;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
const heap = [];
|
|
178
|
-
for (let token = 0; token < logits.length; token++) {
|
|
179
|
-
const logit = logits[token];
|
|
180
|
-
if (!Number.isFinite(logit)) {
|
|
181
|
-
continue;
|
|
182
|
-
}
|
|
183
|
-
const candidate = { token, logit };
|
|
184
|
-
if (heap.length < limit) {
|
|
185
|
-
heapPushLowestFirst(heap, candidate);
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
if (isHigherPriorityCandidate(candidate, heap[0])) {
|
|
189
|
-
heap[0] = candidate;
|
|
190
|
-
heapifyLowestFirst(heap, 0);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
if (heap.length === 0) {
|
|
194
|
-
return [];
|
|
195
|
-
}
|
|
196
|
-
return heap.sort((a, b) => b.logit - a.logit || a.token - b.token);
|
|
15
|
+
export function applyPresencePenalty(logits, previousTokens, penalty, windowSize = getRuntimeConfig().inference.sampling.repetitionPenaltyWindow) {
|
|
16
|
+
applyScalarPresencePenalty(logits, previousTokens, penalty, windowSize);
|
|
197
17
|
}
|
|
198
18
|
|
|
199
|
-
function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug, topK, topP) {
|
|
200
|
-
if (candidates.length === 0) {
|
|
201
|
-
throw new Error(
|
|
202
|
-
'[Sampling] Top-k filtering produced no finite candidates. ' +
|
|
203
|
-
'Upstream decode likely produced NaN/Inf logits.'
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
if (candidates.length === 1) {
|
|
207
|
-
return candidates[0].token;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
const invTemperature = 1 / temperature;
|
|
211
|
-
let maxScaled = -Infinity;
|
|
212
|
-
for (const candidate of candidates) {
|
|
213
|
-
const scaled = candidate.logit * invTemperature;
|
|
214
|
-
candidate.scaled = scaled;
|
|
215
|
-
if (scaled > maxScaled) {
|
|
216
|
-
maxScaled = scaled;
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
let sum = 0;
|
|
221
|
-
for (const candidate of candidates) {
|
|
222
|
-
const weight = Math.exp(candidate.scaled - maxScaled);
|
|
223
|
-
candidate.prob = weight;
|
|
224
|
-
sum += weight;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
if (sum > 0) {
|
|
228
|
-
const invSum = 1 / sum;
|
|
229
|
-
for (const candidate of candidates) {
|
|
230
|
-
candidate.prob *= invSum;
|
|
231
|
-
}
|
|
232
|
-
} else {
|
|
233
|
-
const uniformProb = 1.0 / candidates.length;
|
|
234
|
-
for (const candidate of candidates) {
|
|
235
|
-
candidate.prob = uniformProb;
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
if (debug) {
|
|
240
|
-
const top5 = candidates.slice(0, 5).map(c => {
|
|
241
|
-
const text = decode?.([c.token]) ?? '?';
|
|
242
|
-
return `"${text}"(${(c.prob * 100).toFixed(1)}%)`;
|
|
243
|
-
});
|
|
244
|
-
trace.sample(`Top-5 (temp=${temperature}, topK=${topK}, topP=${topP}): ${top5.join(', ')}`);
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
const r = seed !== undefined ? seededRandom(seed) : unseededRandom();
|
|
248
|
-
let cumProb = 0;
|
|
249
|
-
for (const candidate of candidates) {
|
|
250
|
-
cumProb += candidate.prob;
|
|
251
|
-
if (r < cumProb) return candidate.token;
|
|
252
|
-
}
|
|
253
|
-
return candidates[candidates.length - 1].token;
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
|
|
257
19
|
export function sample(logits, opts) {
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
if (Number.isInteger(tokenId) && tokenId >= 0 && tokenId < logits.length) {
|
|
267
|
-
logits[tokenId] = -Infinity;
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
assertFiniteSamplingCandidates(logits, 'Logits', beforeMaskHealth);
|
|
273
|
-
|
|
274
|
-
// Greedy (argmax) when temperature = 0
|
|
275
|
-
if (temperature === 0) {
|
|
276
|
-
let maxIdx = -1;
|
|
277
|
-
let maxVal = -Infinity;
|
|
278
|
-
for (let i = 0; i < logits.length; i++) {
|
|
279
|
-
const value = logits[i];
|
|
280
|
-
if (!Number.isFinite(value)) {
|
|
281
|
-
continue;
|
|
282
|
-
}
|
|
283
|
-
if (value > maxVal) {
|
|
284
|
-
maxVal = value;
|
|
285
|
-
maxIdx = i;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
if (maxIdx < 0) {
|
|
289
|
-
throw new Error(
|
|
290
|
-
'[Sampling] Greedy sampling could not find a finite candidate logit. ' +
|
|
291
|
-
'Upstream decode likely produced NaN/Inf.'
|
|
292
|
-
);
|
|
293
|
-
}
|
|
294
|
-
if (debug) {
|
|
295
|
-
const text = decode?.([maxIdx]) ?? '?';
|
|
296
|
-
trace.sample(`Greedy: id=${maxIdx} "${text}" logit=${maxVal.toFixed(4)}`);
|
|
297
|
-
}
|
|
298
|
-
return maxIdx;
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
if (topP >= 1.0 && Number.isFinite(topK) && topK > 0) {
|
|
302
|
-
const candidates = selectTopKLogitCandidates(logits, topK);
|
|
303
|
-
if (candidates) {
|
|
304
|
-
return sampleFromLogitCandidates(candidates, temperature, seed, decode, debug, topK, topP);
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
// Apply temperature
|
|
309
|
-
if (temperature !== 1.0) {
|
|
310
|
-
for (let i = 0; i < logits.length; i++) {
|
|
311
|
-
logits[i] /= temperature;
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
const probs = softmax(logits);
|
|
316
|
-
|
|
317
|
-
// Build candidate list
|
|
318
|
-
|
|
319
|
-
let candidates = [];
|
|
320
|
-
for (let i = 0; i < probs.length; i++) {
|
|
321
|
-
const probability = probs[i];
|
|
322
|
-
if (!Number.isFinite(probability) || probability <= 0) {
|
|
323
|
-
continue;
|
|
324
|
-
}
|
|
325
|
-
candidates.push({ token: i, prob: probability });
|
|
326
|
-
}
|
|
327
|
-
if (candidates.length === 0) {
|
|
328
|
-
throw new Error(
|
|
329
|
-
'[Sampling] Softmax produced no finite candidate probabilities. ' +
|
|
330
|
-
'Upstream decode likely produced NaN/Inf logits.'
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
candidates.sort((a, b) => b.prob - a.prob);
|
|
334
|
-
|
|
335
|
-
// Top-k filtering
|
|
336
|
-
if (topK > 0) {
|
|
337
|
-
candidates = candidates.slice(0, topK);
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
// Top-p (nucleus) filtering
|
|
341
|
-
if (topP < 1.0) {
|
|
342
|
-
let cumProb = 0;
|
|
343
|
-
|
|
344
|
-
const filtered = [];
|
|
345
|
-
for (const c of candidates) {
|
|
346
|
-
filtered.push(c);
|
|
347
|
-
cumProb += c.prob;
|
|
348
|
-
if (cumProb >= topP) break;
|
|
349
|
-
}
|
|
350
|
-
candidates = filtered;
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
// Renormalize with guard against zero sum
|
|
354
|
-
const probSum = candidates.reduce((s, c) => s + c.prob, 0);
|
|
355
|
-
if (probSum > 0) {
|
|
356
|
-
for (const c of candidates) {
|
|
357
|
-
c.prob /= probSum;
|
|
358
|
-
}
|
|
359
|
-
} else {
|
|
360
|
-
// If all probabilities are zero, fall back to uniform distribution
|
|
361
|
-
const uniformProb = 1.0 / candidates.length;
|
|
362
|
-
for (const c of candidates) {
|
|
363
|
-
c.prob = uniformProb;
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
if (debug) {
|
|
368
|
-
const top5 = candidates.slice(0, 5).map(c => {
|
|
369
|
-
const text = decode?.([c.token]) ?? '?';
|
|
370
|
-
return `"${text}"(${(c.prob * 100).toFixed(1)}%)`;
|
|
371
|
-
});
|
|
372
|
-
trace.sample(`Top-5 (temp=${temperature}, topK=${topK}, topP=${topP}): ${top5.join(', ')}`);
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
// Sample from distribution
|
|
376
|
-
const r = seed !== undefined ? seededRandom(seed) : unseededRandom();
|
|
377
|
-
let cumProb = 0;
|
|
378
|
-
for (const c of candidates) {
|
|
379
|
-
cumProb += c.prob;
|
|
380
|
-
if (r < cumProb) return c.token;
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
return candidates[candidates.length - 1].token;
|
|
20
|
+
return sampleTokens(logits, {
|
|
21
|
+
...opts,
|
|
22
|
+
onCandidates: opts.debug ? candidates => {
|
|
23
|
+
const top = candidates.slice(0, 5).map(candidate =>
|
|
24
|
+
`"${opts.decode?.([candidate.token]) ?? '?'}"(${(candidate.prob * 100).toFixed(1)}%)`);
|
|
25
|
+
trace.sample(`Top-5 (temp=${opts.temperature}, topK=${opts.topK}, topP=${opts.topP}): ${top.join(', ')}`);
|
|
26
|
+
} : undefined,
|
|
27
|
+
});
|
|
384
28
|
}
|
|
385
29
|
|
|
386
|
-
|
|
387
30
|
export function getTopK(logits, k = 5, decode) {
|
|
388
31
|
const probs = softmax(new Float32Array(logits));
|
|
389
32
|
|
|
@@ -344,10 +344,10 @@ export interface GenerateOptions {
|
|
|
344
344
|
|
|
345
345
|
/** @category hybrid Apply chat template (auto-detected for Gemma) */
|
|
346
346
|
useChatTemplate?: boolean;
|
|
347
|
-
|
|
348
|
-
/** Callback for each generated token */
|
|
349
347
|
onToken?: ((tokenId: number, text: string) => void) | null;
|
|
350
|
-
|
|
348
|
+
/** Synchronous constraint; errors abort, and generatedIds excludes prompt tokens. */
|
|
349
|
+
logitMaskFn?: ((logits: Float32Array, context: { generatedIds: number[]; tokenizer: unknown; vocabSize: number }) => void) | null;
|
|
350
|
+
logitMaskIdentity?: { id: string; contentDigest: string } | null;
|
|
351
351
|
/** @internal Callback with finalized per-step logits before sampling returns the token. */
|
|
352
352
|
onLogits?: ((logits: Float32Array, context: {
|
|
353
353
|
tokenId: number;
|