doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -68,6 +68,7 @@ function shouldUseGreedyLmHeadArgmaxFusion(state, opts, samplingDefaults, repeti
|
|
|
68
68
|
return state.runtimeConfig?.inference?.session?.useGreedyLmHeadArgmaxFusion === true && (state.modelConfig?.logitOutputScale ?? 1) === 1
|
|
69
69
|
&& opts.temperature < samplingDefaults.greedyThreshold
|
|
70
70
|
&& repetitionPenalty === 1.0
|
|
71
|
+
&& opts.presencePenalty === 0
|
|
71
72
|
&& state.operatorDiagnostics == null
|
|
72
73
|
&& !(Array.isArray(probes) && probes.length > 0);
|
|
73
74
|
}
|
|
@@ -561,19 +562,20 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
561
562
|
const embedMetadata = getWeightMetadata(embedBufferRaw);
|
|
562
563
|
const activationDtype = getEffectiveActivationDtype(state, opts);
|
|
563
564
|
|
|
564
|
-
//
|
|
565
|
+
// Preserve order and duplicates so the window can advance within the batch.
|
|
565
566
|
const repetitionPenalty = opts.repetitionPenalty ?? samplingDefaults.repetitionPenalty;
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
567
|
+
const presencePenalty = opts.presencePenalty ?? samplingDefaults.presencePenalty;
|
|
568
|
+
const repPenaltyWindow = opts.repetitionPenaltyWindow ?? samplingDefaults.repetitionPenaltyWindow;
|
|
569
|
+
if (repetitionPenalty !== 1.0 || presencePenalty !== 0) {
|
|
570
|
+
const historyTokens = currentIds.slice(-repPenaltyWindow);
|
|
571
|
+
repHistoryCount = historyTokens.length;
|
|
572
|
+
const historyData = new Uint32Array(historyTokens);
|
|
571
573
|
repHistoryBuffer = device.createBuffer({
|
|
572
574
|
size: Math.max(4, historyData.byteLength),
|
|
573
575
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
574
576
|
label: 'rep_penalty_history',
|
|
575
577
|
});
|
|
576
|
-
device.queue.writeBuffer(repHistoryBuffer, 0, historyData);
|
|
578
|
+
if (historyData.byteLength) device.queue.writeBuffer(repHistoryBuffer, 0, historyData);
|
|
577
579
|
}
|
|
578
580
|
|
|
579
581
|
// Hoist loop-invariant values to avoid repeated rule lookups and allocations.
|
|
@@ -694,11 +696,13 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
694
696
|
logitsBuffer = logits.logitsBuffer;
|
|
695
697
|
|
|
696
698
|
// Apply GPU-side repetition penalty before sampling
|
|
697
|
-
if (repHistoryBuffer
|
|
699
|
+
if (repHistoryBuffer) {
|
|
698
700
|
await recordRepPenalty(recorder, logitsBuffer, repHistoryBuffer, tokensBuffer, {
|
|
699
701
|
vocabSize,
|
|
700
702
|
historyCount: repHistoryCount,
|
|
701
703
|
penalty: repetitionPenalty,
|
|
704
|
+
presencePenalty,
|
|
705
|
+
repetitionPenaltyWindow: repPenaltyWindow,
|
|
702
706
|
batchCount: i,
|
|
703
707
|
batchOffset: 1,
|
|
704
708
|
logitsDtype,
|
|
@@ -717,12 +721,14 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
717
721
|
await recordGPUSample(recorder, logitsBuffer, vocabSize, {
|
|
718
722
|
temperature: opts.temperature,
|
|
719
723
|
topK: opts.topK,
|
|
724
|
+
topP: opts.topP,
|
|
720
725
|
padTokenId,
|
|
721
726
|
logitSoftcap,
|
|
722
727
|
logitsDtype,
|
|
723
728
|
outputBuffer: tokensBuffer,
|
|
724
729
|
outputIndex,
|
|
725
730
|
greedyThreshold: samplingDefaults.greedyThreshold,
|
|
731
|
+
randomSeed: opts.seed,
|
|
726
732
|
});
|
|
727
733
|
}
|
|
728
734
|
}
|
|
@@ -839,6 +845,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
839
845
|
getUniformCache().flushPendingDestruction();
|
|
840
846
|
|
|
841
847
|
const tokens = readback.tokens;
|
|
848
|
+
if (tokens.includes(0xFFFFFFFF)) throw new Error('[Sampling] No finite candidate logits.');
|
|
842
849
|
const stopFlags = readback.stopFlags;
|
|
843
850
|
|
|
844
851
|
if (stopFlags) {
|
|
@@ -28,7 +28,7 @@ import { validateCallTimeOptions } from '../../../config/param-validator.js';
|
|
|
28
28
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
29
29
|
|
|
30
30
|
// Pipeline sub-modules
|
|
31
|
-
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
31
|
+
import { sample, applyRepetitionPenalty, applyPresencePenalty, logitsSanity, getTopK } from './sampling.js';
|
|
32
32
|
import { createKVCache, isStopToken } from './init.js';
|
|
33
33
|
import { embed } from './embed.js';
|
|
34
34
|
import { processLayer } from './layer.js';
|
|
@@ -359,20 +359,26 @@ export class PipelineGenerator {
|
|
|
359
359
|
|
|
360
360
|
_sampleNextTokenFromLogits(logits, generatedIds, opts) {
|
|
361
361
|
const sampledLogits = Float32Array.from(logits);
|
|
362
|
-
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty);
|
|
362
|
+
applyRepetitionPenalty(sampledLogits, generatedIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
363
|
+
if (opts.presencePenalty) {
|
|
364
|
+
applyPresencePenalty(sampledLogits, generatedIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
365
|
+
}
|
|
363
366
|
// Optional pre-sample logit mask. Callers pass `opts.logitMaskFn` to
|
|
364
367
|
// implement grammar/schema-constrained decoding. The hook receives the
|
|
365
368
|
// mutable logit buffer (after repetition penalty) plus the running token
|
|
366
369
|
// sequence so it can track parse state across decode steps.
|
|
367
370
|
if (typeof opts?.logitMaskFn === "function") {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
371
|
+
if (!Number.isSafeInteger(opts.promptTokenCount) || opts.promptTokenCount < 0 || opts.promptTokenCount > generatedIds.length) {
|
|
372
|
+
throw new Error('[Pipeline] logitMaskFn requires an explicit prompt token boundary.');
|
|
373
|
+
}
|
|
374
|
+
const result = opts.logitMaskFn(sampledLogits, {
|
|
375
|
+
generatedIds: generatedIds.slice(opts.promptTokenCount),
|
|
376
|
+
tokenizer: this._state.tokenizer ?? null,
|
|
377
|
+
vocabSize: this._state.modelConfig?.vocabSize ?? sampledLogits.length,
|
|
378
|
+
});
|
|
379
|
+
if (result && typeof result.then === 'function') {
|
|
380
|
+
Promise.resolve(result).catch(() => {});
|
|
381
|
+
throw new Error('[Pipeline] logitMaskFn must be synchronous.');
|
|
376
382
|
}
|
|
377
383
|
}
|
|
378
384
|
const padTokenId = this._state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
@@ -135,6 +135,27 @@ function clearDynamicLayerState(layerState) {
|
|
|
135
135
|
}
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
+
async function observeLinearAttentionState(layerState, options) {
|
|
139
|
+
for (const [name, field] of [
|
|
140
|
+
['conv_weight', 'convWeight'],
|
|
141
|
+
['dt_bias', 'dtBias'],
|
|
142
|
+
['a_log', 'aLog'],
|
|
143
|
+
['norm_weight', 'normWeight'],
|
|
144
|
+
['conv_input', 'convState'],
|
|
145
|
+
['recurrent_input', 'recurrentState'],
|
|
146
|
+
]) {
|
|
147
|
+
await runProbes(`linear_state_${name}`, layerState[`${field}GPU`], {
|
|
148
|
+
layerIdx: options.layerIdx,
|
|
149
|
+
numTokens: 1,
|
|
150
|
+
hiddenSize: layerState[field].length,
|
|
151
|
+
probes: options.debugProbes,
|
|
152
|
+
recorder: options.recorder,
|
|
153
|
+
operatorDiagnostics: options.operatorDiagnostics,
|
|
154
|
+
dtype: 'f32',
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
138
159
|
async function syncLayerRuntimeStateFromGPU(layerState) {
|
|
139
160
|
if (isGpuBuffer(layerState.convStateGPU)) {
|
|
140
161
|
const rawConvState = await readBuffer(
|
|
@@ -579,6 +600,9 @@ export async function runLinearAttentionLayer(inputTensor, layerWeights, options
|
|
|
579
600
|
let coreBTensor = bTensor;
|
|
580
601
|
|
|
581
602
|
try {
|
|
603
|
+
if (options.debugProbes?.some((probe) => probe.stage.startsWith('linear_state_'))) {
|
|
604
|
+
await observeLinearAttentionState(layerState, options);
|
|
605
|
+
}
|
|
582
606
|
await runProbes('linear_qkv_proj', qkvTensor.buffer, {
|
|
583
607
|
layerIdx,
|
|
584
608
|
numTokens,
|
|
@@ -40,8 +40,15 @@ export async function finalizeLogitOutputTensor(tensor, config, options) {
|
|
|
40
40
|
});
|
|
41
41
|
if (recorder) recorder.trackTemporaryBuffer(tensor.buffer);
|
|
42
42
|
else releaseBuffer(tensor.buffer);
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
try {
|
|
44
|
+
await runProbes('logits', output.buffer, {
|
|
45
|
+
numTokens, hiddenSize: targetVocabSize, recorder, operatorDiagnostics, dtype: output.dtype,
|
|
46
|
+
probes: config.debugProbes,
|
|
47
|
+
});
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (recorder) recorder.trackTemporaryBuffer(output.buffer);
|
|
50
|
+
else releaseBuffer(output.buffer);
|
|
51
|
+
throw error;
|
|
52
|
+
}
|
|
46
53
|
return output;
|
|
47
54
|
}
|
|
@@ -1,118 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
'temperature',
|
|
3
|
-
'topP',
|
|
4
|
-
'topK',
|
|
5
|
-
'repetitionPenalty',
|
|
6
|
-
'repetitionPenaltyWindow',
|
|
7
|
-
'greedyThreshold',
|
|
8
|
-
'suppressSpecialTokens',
|
|
9
|
-
'suppressSpecialLikeTokens',
|
|
10
|
-
'suppressTokenIds',
|
|
11
|
-
];
|
|
1
|
+
import { GENERATION_CONTRACT, validateGenerationField } from '../../../config/generation-contract.js';
|
|
12
2
|
|
|
13
3
|
export function resolveSamplingConfig(opts, runtimeConfig) {
|
|
14
|
-
const
|
|
15
|
-
if (!
|
|
4
|
+
const defaults = runtimeConfig?.inference?.sampling;
|
|
5
|
+
if (!defaults || typeof defaults !== 'object' || Array.isArray(defaults)) {
|
|
16
6
|
throw new Error('[Sampling] runtimeConfig.inference.sampling is required.');
|
|
17
7
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
topK: resolveSamplingInteger(
|
|
38
|
-
'topK',
|
|
39
|
-
opts?.topK,
|
|
40
|
-
samplingDefaults.topK,
|
|
41
|
-
(value) => value >= 1
|
|
42
|
-
),
|
|
43
|
-
repetitionPenalty: resolveSamplingNumber(
|
|
44
|
-
'repetitionPenalty',
|
|
45
|
-
opts?.repetitionPenalty,
|
|
46
|
-
samplingDefaults.repetitionPenalty,
|
|
47
|
-
(value) => value > 0
|
|
48
|
-
),
|
|
49
|
-
repetitionPenaltyWindow: resolveSamplingInteger(
|
|
50
|
-
'repetitionPenaltyWindow',
|
|
51
|
-
undefined,
|
|
52
|
-
samplingDefaults.repetitionPenaltyWindow,
|
|
53
|
-
(value) => value >= 0
|
|
54
|
-
),
|
|
55
|
-
greedyThreshold: resolveSamplingNumber(
|
|
56
|
-
'greedyThreshold',
|
|
57
|
-
undefined,
|
|
58
|
-
samplingDefaults.greedyThreshold,
|
|
59
|
-
(value) => value >= 0
|
|
60
|
-
),
|
|
61
|
-
suppressSpecialTokens: resolveSamplingBoolean(
|
|
62
|
-
'suppressSpecialTokens',
|
|
63
|
-
samplingDefaults.suppressSpecialTokens
|
|
64
|
-
),
|
|
65
|
-
suppressSpecialLikeTokens: resolveSamplingBoolean(
|
|
66
|
-
'suppressSpecialLikeTokens',
|
|
67
|
-
samplingDefaults.suppressSpecialLikeTokens
|
|
68
|
-
),
|
|
69
|
-
suppressTokenIds: resolveSamplingTokenIdList(
|
|
70
|
-
'suppressTokenIds',
|
|
71
|
-
samplingDefaults.suppressTokenIds
|
|
72
|
-
),
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function resolveSamplingBoolean(name, runtimeValue) {
|
|
77
|
-
if (runtimeValue === null) {
|
|
78
|
-
throw new Error(`[Sampling] ${name} cannot be null.`);
|
|
79
|
-
}
|
|
80
|
-
if (typeof runtimeValue !== 'boolean') {
|
|
81
|
-
throw new Error(`[Sampling] ${name} must be a boolean; got ${typeof runtimeValue}.`);
|
|
82
|
-
}
|
|
83
|
-
return runtimeValue;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function resolveSamplingTokenIdList(name, runtimeValue) {
|
|
87
|
-
if (!Array.isArray(runtimeValue)) {
|
|
88
|
-
throw new Error(`[Sampling] ${name} must be an array of token IDs.`);
|
|
89
|
-
}
|
|
90
|
-
return runtimeValue.map((tokenId, index) => {
|
|
91
|
-
if (!Number.isInteger(tokenId) || tokenId < 0) {
|
|
92
|
-
throw new Error(`[Sampling] ${name}[${index}] must be a non-negative integer token ID; got ${tokenId}.`);
|
|
93
|
-
}
|
|
94
|
-
return tokenId;
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
function resolveSamplingNumber(name, callValue, runtimeValue, validate) {
|
|
99
|
-
const value = callValue === undefined ? runtimeValue : callValue;
|
|
100
|
-
if (value === null) {
|
|
101
|
-
throw new Error(`[Sampling] ${name} cannot be null.`);
|
|
102
|
-
}
|
|
103
|
-
if (!Number.isFinite(value)) {
|
|
104
|
-
throw new Error(`[Sampling] ${name} must be a finite number; got ${value}.`);
|
|
105
|
-
}
|
|
106
|
-
if (!validate(value)) {
|
|
107
|
-
throw new Error(`[Sampling] ${name} is outside the configured range; got ${value}.`);
|
|
108
|
-
}
|
|
109
|
-
return value;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
function resolveSamplingInteger(name, callValue, runtimeValue, validate) {
|
|
113
|
-
const value = resolveSamplingNumber(name, callValue, runtimeValue, validate);
|
|
114
|
-
if (Math.floor(value) !== value) {
|
|
115
|
-
throw new Error(`[Sampling] ${name} must be an integer; got ${value}.`);
|
|
116
|
-
}
|
|
117
|
-
return value;
|
|
8
|
+
const resolved = {};
|
|
9
|
+
for (const [name, rule] of Object.entries(GENERATION_CONTRACT.options)) {
|
|
10
|
+
if (!rule.runtimeSampling) continue;
|
|
11
|
+
if (defaults[name] === undefined) throw new Error(`[Sampling] runtimeConfig.inference.sampling.${name} is required.`);
|
|
12
|
+
const value = opts?.[name] === undefined ? defaults[name] : opts[name];
|
|
13
|
+
validateGenerationField(name, value, rule);
|
|
14
|
+
resolved[name] = Array.isArray(value) ? [...value] : value;
|
|
15
|
+
}
|
|
16
|
+
const greedyThreshold = defaults.greedyThreshold;
|
|
17
|
+
if (!Number.isFinite(greedyThreshold) || greedyThreshold < 0) {
|
|
18
|
+
throw new Error('[Sampling] greedyThreshold must be a non-negative finite number.');
|
|
19
|
+
}
|
|
20
|
+
resolved.greedyThreshold = greedyThreshold;
|
|
21
|
+
for (const name of ['suppressSpecialTokens', 'suppressSpecialLikeTokens']) {
|
|
22
|
+
const value = opts?.[name] === undefined ? defaults[name] : opts[name];
|
|
23
|
+
if (typeof value !== 'boolean') throw new Error(`[Sampling] ${name} must be a boolean and cannot be null.`);
|
|
24
|
+
resolved[name] = value;
|
|
25
|
+
}
|
|
26
|
+
return resolved;
|
|
118
27
|
}
|
|
@@ -29,7 +29,15 @@ export interface LogitStats {
|
|
|
29
29
|
export function applyRepetitionPenalty(
|
|
30
30
|
logits: Float32Array,
|
|
31
31
|
previousTokens: number[],
|
|
32
|
-
penalty: number
|
|
32
|
+
penalty: number,
|
|
33
|
+
windowSize?: number
|
|
34
|
+
): void;
|
|
35
|
+
|
|
36
|
+
export function applyPresencePenalty(
|
|
37
|
+
logits: Float32Array,
|
|
38
|
+
previousTokens: number[],
|
|
39
|
+
penalty: number,
|
|
40
|
+
windowSize?: number
|
|
33
41
|
): void;
|
|
34
42
|
|
|
35
43
|
export function softmax(logits: Float32Array): Float32Array;
|