doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -1,137 +1,134 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// const mask = createJsonGrammarMask({ tokenizer });
|
|
5
|
-
// for await (const text of pipeline.generate(prompt, { logitMaskFn: mask })) { ... }
|
|
6
|
-
//
|
|
7
|
-
// The mask classifies each candidate token against the current parse state and
|
|
8
|
-
// zeroes logits for tokens that would break a valid JSON object output. It
|
|
9
|
-
// deliberately accepts any token that *looks like* it may extend valid JSON —
|
|
10
|
-
// this is a soft constraint; parsing the final output is still required to
|
|
11
|
-
// validate the full payload.
|
|
12
|
-
//
|
|
13
|
-
// Tokenizer assumption: `tokenizer.decode([id])` returns the literal string
|
|
14
|
-
// piece for that token. That's true for BPE tokenizers used by Qwen, Gemma,
|
|
15
|
-
// Llama. The mask is deliberately token-piece based, not character level,
|
|
16
|
-
// because re-encoding the stream to characters per step is prohibitively slow
|
|
17
|
-
// in-browser.
|
|
18
|
-
//
|
|
19
|
-
// What the mask does enforce:
|
|
20
|
-
// - Until the first "{" is emitted, only tokens whose piece contains "{" or
|
|
21
|
-
// whitespace are allowed.
|
|
22
|
-
// - Once inside the object, matched counts of "{" / "}" / "[" / "]" / "\""
|
|
23
|
-
// determine whether we are inside a string; inside a string, any token is
|
|
24
|
-
// allowed except one that closes more brackets than are open.
|
|
25
|
-
// - After a balanced closing "}" is emitted at depth 0, we forbid any
|
|
26
|
-
// further non-whitespace token (forces a clean stop).
|
|
27
|
-
//
|
|
28
|
-
// What the mask does NOT enforce (intentional, to keep the mask cheap):
|
|
29
|
-
// - Key quoting, trailing-comma avoidance, matching " pairs at token level.
|
|
30
|
-
// Post-decode `parseJSONObjectEnvelope` handles these via recovery
|
|
31
|
-
// transformations.
|
|
32
|
-
// - Schema/key whitelists. Callers can layer a stricter mask on top.
|
|
1
|
+
// Incremental JSON object syntax constraint. Schema validation remains a
|
|
2
|
+
// separate caller-owned boundary; this mask does not enforce application keys.
|
|
3
|
+
// Tokenizers must decode individual pieces without trimming whitespace.
|
|
33
4
|
|
|
34
|
-
function
|
|
35
|
-
|
|
36
|
-
try {
|
|
37
|
-
// decode(ids, skipSpecial, skipBos)
|
|
38
|
-
return String(tokenizer.decode([tokenId], true, false) || "");
|
|
39
|
-
} catch {
|
|
40
|
-
return "";
|
|
41
|
-
}
|
|
5
|
+
function initialState() {
|
|
6
|
+
return { stack: [{ kind: 'root', expect: 'object' }], scalar: null };
|
|
42
7
|
}
|
|
43
8
|
|
|
44
|
-
function
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if (piece[i] === char) count += 1;
|
|
48
|
-
}
|
|
49
|
-
return count;
|
|
9
|
+
function completeValue(state) {
|
|
10
|
+
const parent = state.stack[state.stack.length - 1];
|
|
11
|
+
parent.expect = parent.kind === 'root' ? 'done' : 'comma-or-end';
|
|
50
12
|
}
|
|
51
13
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
return
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
}
|
|
14
|
+
function beginValue(state, char) {
|
|
15
|
+
if (char === '{' || char === '[') {
|
|
16
|
+
state.stack.push({ kind: char === '{' ? 'object' : 'array', expect: char === '{' ? 'key-or-end' : 'value-or-end' });
|
|
17
|
+
} else if (char === '"') state.scalar = { kind: 'string', role: 'value', escaped: false, unicode: 0 };
|
|
18
|
+
else if (char === 't' || char === 'f' || char === 'n') state.scalar = { kind: 'literal', remaining: { t: 'rue', f: 'alse', n: 'ull' }[char] };
|
|
19
|
+
else if (char === '-' || /[0-9]/.test(char)) state.scalar = { kind: 'number', phase: char === '-' ? 'sign' : char === '0' ? 'zero' : 'integer' };
|
|
20
|
+
else return false;
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function advanceNumber(number, char) {
|
|
25
|
+
const digit = /[0-9]/.test(char);
|
|
26
|
+
if (number.phase === 'sign') {
|
|
27
|
+
if (!digit) return 'invalid';
|
|
28
|
+
number.phase = char === '0' ? 'zero' : 'integer';
|
|
29
|
+
} else if (number.phase === 'dot') {
|
|
30
|
+
if (!digit) return 'invalid';
|
|
31
|
+
number.phase = 'fraction';
|
|
32
|
+
} else if (number.phase === 'exponent' || number.phase === 'exponent-sign') {
|
|
33
|
+
if (number.phase === 'exponent' && (char === '+' || char === '-')) number.phase = 'exponent-sign';
|
|
34
|
+
else if (digit) number.phase = 'exponent-digits';
|
|
35
|
+
else return 'invalid';
|
|
36
|
+
} else if (digit) {
|
|
37
|
+
if (number.phase === 'zero') return 'invalid';
|
|
38
|
+
} else if (char === '.' && ['zero', 'integer'].includes(number.phase)) number.phase = 'dot';
|
|
39
|
+
else if ((char === 'e' || char === 'E') && ['zero', 'integer', 'fraction'].includes(number.phase)) number.phase = 'exponent';
|
|
40
|
+
else return /[\x20\t\r\n,\]}]/.test(char) ? 'complete' : 'invalid';
|
|
41
|
+
return 'continue';
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function advance(state, text) {
|
|
45
|
+
for (const char of text) {
|
|
46
|
+
const scalar = state.scalar;
|
|
47
|
+
if (scalar?.kind === 'string') {
|
|
48
|
+
if (scalar.unicode > 0) {
|
|
49
|
+
if (!/[a-fA-F0-9]/.test(char)) return false;
|
|
50
|
+
scalar.unicode -= 1;
|
|
51
|
+
} else if (scalar.escaped) {
|
|
52
|
+
scalar.escaped = false;
|
|
53
|
+
if (char === 'u') scalar.unicode = 4;
|
|
54
|
+
else if (!'"\\/bfnrt'.includes(char)) return false;
|
|
55
|
+
} else if (char === '\\') scalar.escaped = true;
|
|
56
|
+
else if (char === '"') {
|
|
57
|
+
state.scalar = null;
|
|
58
|
+
if (scalar.role === 'key') state.stack[state.stack.length - 1].expect = 'colon';
|
|
59
|
+
else completeValue(state);
|
|
60
|
+
} else if (char.codePointAt(0) < 32) return false;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (scalar?.kind === 'literal') {
|
|
64
|
+
if (char !== scalar.remaining[0]) return false;
|
|
65
|
+
scalar.remaining = scalar.remaining.slice(1);
|
|
66
|
+
if (!scalar.remaining) { state.scalar = null; completeValue(state); }
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (scalar?.kind === 'number') {
|
|
70
|
+
const result = advanceNumber(scalar, char);
|
|
71
|
+
if (result === 'invalid') return false;
|
|
72
|
+
if (result === 'continue') continue;
|
|
73
|
+
state.scalar = null;
|
|
74
|
+
completeValue(state);
|
|
75
|
+
}
|
|
76
|
+
if (/[\x20\t\r\n]/.test(char)) continue;
|
|
77
|
+
const current = state.stack[state.stack.length - 1];
|
|
78
|
+
if (current.expect === 'done') return false;
|
|
79
|
+
if (current.expect === 'object') {
|
|
80
|
+
if (char !== '{') return false;
|
|
81
|
+
if (!beginValue(state, char)) return false;
|
|
82
|
+
} else if (current.expect === 'colon') {
|
|
83
|
+
if (char !== ':') return false;
|
|
84
|
+
current.expect = 'value';
|
|
85
|
+
} else if (current.expect === 'key' || current.expect === 'key-or-end') {
|
|
86
|
+
if (char === '}' && current.expect === 'key-or-end') { state.stack.pop(); completeValue(state); }
|
|
87
|
+
else if (char === '"') state.scalar = { kind: 'string', role: 'key', escaped: false, unicode: 0 };
|
|
88
|
+
else return false;
|
|
89
|
+
} else if (current.expect === 'comma-or-end') {
|
|
90
|
+
if (char === ',') current.expect = current.kind === 'object' ? 'key' : 'value';
|
|
91
|
+
else if (char === (current.kind === 'object' ? '}' : ']')) { state.stack.pop(); completeValue(state); }
|
|
92
|
+
else return false;
|
|
93
|
+
} else if (current.expect === 'value-or-end' && char === ']') {
|
|
94
|
+
state.stack.pop(); completeValue(state);
|
|
95
|
+
} else if (!beginValue(state, char)) return false;
|
|
96
|
+
}
|
|
97
|
+
return true;
|
|
68
98
|
}
|
|
69
99
|
|
|
70
100
|
export function createJsonGrammarMask(opts = {}) {
|
|
71
101
|
const pieceCache = new Map();
|
|
72
102
|
const cacheBudget = Math.max(1024, Math.floor(Number(opts.cacheBudget) || 32768));
|
|
103
|
+
let cachedTokenizer = null;
|
|
73
104
|
|
|
74
|
-
function
|
|
75
|
-
if (pieceCache.has(
|
|
105
|
+
function pieceAt(tokenizer, id) {
|
|
106
|
+
if (pieceCache.has(id)) return pieceCache.get(id);
|
|
76
107
|
if (pieceCache.size >= cacheBudget) pieceCache.clear();
|
|
77
|
-
const piece =
|
|
78
|
-
|
|
108
|
+
const piece = tokenizer.decode([id], true, false);
|
|
109
|
+
if (typeof piece !== 'string') throw new Error('JSON constraint requires synchronous string token decoding');
|
|
110
|
+
pieceCache.set(id, piece);
|
|
79
111
|
return piece;
|
|
80
112
|
}
|
|
81
113
|
|
|
82
|
-
let runningOpenBraces = 0;
|
|
83
|
-
let runningOpenBrackets = 0;
|
|
84
|
-
let runningQuotes = 0;
|
|
85
|
-
let sawFirstBrace = false;
|
|
86
|
-
let balancedClose = false;
|
|
87
|
-
let lastGeneratedLen = 0;
|
|
88
|
-
|
|
89
|
-
function reflectEmittedTokens(tokenizer, generatedIds) {
|
|
90
|
-
// Catch up to any tokens committed since we were last called.
|
|
91
|
-
for (let i = lastGeneratedLen; i < generatedIds.length; i += 1) {
|
|
92
|
-
const piece = cachedPiece(tokenizer, generatedIds[i]);
|
|
93
|
-
const shape = classifyPiece(piece);
|
|
94
|
-
runningOpenBraces += shape.openBraces - shape.closeBraces;
|
|
95
|
-
runningOpenBrackets += shape.openBrackets - shape.closeBrackets;
|
|
96
|
-
runningQuotes += shape.quotes;
|
|
97
|
-
if (!sawFirstBrace && shape.hasOpenBrace) sawFirstBrace = true;
|
|
98
|
-
if (sawFirstBrace && runningOpenBraces <= 0 && runningOpenBrackets <= 0) {
|
|
99
|
-
balancedClose = true;
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
lastGeneratedLen = generatedIds.length;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
114
|
return function logitMask(logits, context) {
|
|
106
|
-
const tokenizer =
|
|
107
|
-
if (!tokenizer)
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
} else if (insideString) {
|
|
124
|
-
// Inside a string literal, accept any piece — even ones with braces or
|
|
125
|
-
// brackets — the parser handles them as content.
|
|
126
|
-
ok = true;
|
|
127
|
-
} else {
|
|
128
|
-
// Outside strings: disallow pieces that close MORE brackets than are open.
|
|
129
|
-
if (shape.closeBraces > runningOpenBraces + shape.openBraces) ok = false;
|
|
130
|
-
if (shape.closeBrackets > runningOpenBrackets + shape.openBrackets) ok = false;
|
|
131
|
-
}
|
|
132
|
-
if (!ok) {
|
|
133
|
-
logits[id] = -Infinity;
|
|
134
|
-
}
|
|
115
|
+
const tokenizer = opts.tokenizer ?? context?.tokenizer;
|
|
116
|
+
if (!tokenizer || typeof tokenizer.decode !== 'function') throw new Error('JSON constraint requires a tokenizer');
|
|
117
|
+
if (!Array.isArray(context?.generatedIds)) throw new Error('JSON constraint requires generated token IDs');
|
|
118
|
+
if (cachedTokenizer !== tokenizer) { pieceCache.clear(); cachedTokenizer = tokenizer; }
|
|
119
|
+
const state = initialState();
|
|
120
|
+
const prefix = tokenizer.decode(context.generatedIds, true, false);
|
|
121
|
+
if (typeof prefix !== 'string' || !advance(state, prefix)) throw new Error('JSON constraint received an invalid generated prefix');
|
|
122
|
+
const complete = state.stack.length === 1 && state.stack[0].expect === 'done';
|
|
123
|
+
const eos = tokenizer.getSpecialTokens?.().eos;
|
|
124
|
+
const stopIds = new Set(opts.stopTokenIds ?? (Array.isArray(eos) ? eos : Number.isInteger(eos) ? [eos] : []));
|
|
125
|
+
if (![...stopIds].every(id => Number.isInteger(id) && id >= 0 && id < logits.length)) throw new Error('JSON constraint has invalid stop token IDs');
|
|
126
|
+
for (let id = 0; id < logits.length; id += 1) {
|
|
127
|
+
if (logits[id] === -Infinity) continue;
|
|
128
|
+
if (stopIds.has(id)) { if (!complete) logits[id] = -Infinity; continue; }
|
|
129
|
+
const piece = pieceAt(tokenizer, id);
|
|
130
|
+
const candidate = { stack: state.stack.map(row => ({ ...row })), scalar: state.scalar ? { ...state.scalar } : null };
|
|
131
|
+
if (!piece || !advance(candidate, piece)) logits[id] = -Infinity;
|
|
135
132
|
}
|
|
136
133
|
};
|
|
137
134
|
}
|
|
@@ -319,6 +319,8 @@ export async function runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, co
|
|
|
319
319
|
layerIdx
|
|
320
320
|
);
|
|
321
321
|
const activated = await dispatchActivation(hiddenActivation, upOutput, {
|
|
322
|
+
kernelPath,
|
|
323
|
+
phase,
|
|
322
324
|
size: numTokens * intermediateSize,
|
|
323
325
|
gate: null,
|
|
324
326
|
label: `L${layerIdx}.ffn_activation`,
|
|
@@ -659,6 +661,9 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
659
661
|
);
|
|
660
662
|
|
|
661
663
|
activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
664
|
+
kernelPath,
|
|
665
|
+
phase,
|
|
666
|
+
layerIdx,
|
|
662
667
|
size: numTokens * intermediateSize,
|
|
663
668
|
gate: gateOutput,
|
|
664
669
|
inputActivation: 'identity',
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
1
|
import {
|
|
4
2
|
doBiasAdd, doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
|
|
5
3
|
releaseOrTrack
|
|
@@ -819,6 +817,8 @@ export async function runDenseFFNGPU(
|
|
|
819
817
|
enqueueRecordedDenseHealth(context, layerIdx, 'ffn_up', upOutput, numTokens * intermediateSize);
|
|
820
818
|
|
|
821
819
|
const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
820
|
+
kernelPath,
|
|
821
|
+
phase,
|
|
822
822
|
size: numTokens * intermediateSize,
|
|
823
823
|
gate: gateOutput,
|
|
824
824
|
inputActivation: 'identity',
|
|
@@ -91,7 +91,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
|
|
|
91
91
|
if (this._state.isGenerating) throw new Error('Generation already in progress');
|
|
92
92
|
|
|
93
93
|
validateCallTimeOptions(options);
|
|
94
|
-
|
|
94
|
+
this._resetReplayPrefillRuntimeState();
|
|
95
95
|
this._state.isGenerating = true;
|
|
96
96
|
this._resetDecodeRuntimeState();
|
|
97
97
|
this._state.stats.gpuTimePrefillMs = undefined;
|
|
@@ -131,13 +131,9 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
|
|
|
131
131
|
|
|
132
132
|
const opts = resolveGenerateOptions(this._state, options);
|
|
133
133
|
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
134
|
-
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
135
134
|
// Validate and normalize sampling parameters through single source of truth
|
|
136
135
|
const samplingConfig = resolveSamplingConfig(options, this._state.runtimeConfig);
|
|
137
|
-
opts
|
|
138
|
-
opts.topP = samplingConfig.topP;
|
|
139
|
-
opts.topK = samplingConfig.topK;
|
|
140
|
-
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
136
|
+
Object.assign(opts, samplingConfig);
|
|
141
137
|
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this._state, samplingConfig);
|
|
142
138
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
143
139
|
|| this._state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
@@ -195,7 +191,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
|
|
|
195
191
|
this._state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
196
192
|
this._assertTokenIdsInRange(inputIds, 'generate.prefillTokens');
|
|
197
193
|
const generatedIds = [...inputIds];
|
|
198
|
-
this._state.stats.prefillTokens = inputIds.length;
|
|
194
|
+
this._state.stats.prefillTokens = opts.promptTokenCount = inputIds.length;
|
|
199
195
|
|
|
200
196
|
if (opts.debug) {
|
|
201
197
|
log.debug('Pipeline', `Input: ${inputIds.length} tokens`);
|
|
@@ -330,7 +326,7 @@ export async function generateTokenIds(prompt, options = {}) {
|
|
|
330
326
|
if (this._state.isGenerating) throw new Error('Generation already in progress');
|
|
331
327
|
|
|
332
328
|
validateCallTimeOptions(options);
|
|
333
|
-
|
|
329
|
+
this._resetReplayPrefillRuntimeState();
|
|
334
330
|
this._state.isGenerating = true;
|
|
335
331
|
this._resetDecodeRuntimeState();
|
|
336
332
|
this._state.stats.gpuTimePrefillMs = undefined;
|
|
@@ -371,10 +367,7 @@ export async function generateTokenIds(prompt, options = {}) {
|
|
|
371
367
|
opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
|
|
372
368
|
// Validate and normalize sampling parameters through single source of truth
|
|
373
369
|
const samplingConfig = resolveSamplingConfig(options, this._state.runtimeConfig);
|
|
374
|
-
opts
|
|
375
|
-
opts.topP = samplingConfig.topP;
|
|
376
|
-
opts.topK = samplingConfig.topK;
|
|
377
|
-
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
370
|
+
Object.assign(opts, samplingConfig);
|
|
378
371
|
opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this._state, samplingConfig);
|
|
379
372
|
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
380
373
|
|| this._state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
@@ -407,7 +400,7 @@ export async function generateTokenIds(prompt, options = {}) {
|
|
|
407
400
|
this._state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
408
401
|
this._assertTokenIdsInRange(inputIds, 'generateTokenIds.prefillTokens');
|
|
409
402
|
const generatedIds = [...inputIds];
|
|
410
|
-
this._state.stats.prefillTokens = inputIds.length;
|
|
403
|
+
this._state.stats.prefillTokens = opts.promptTokenCount = inputIds.length;
|
|
411
404
|
|
|
412
405
|
let firstToken;
|
|
413
406
|
try {
|
|
@@ -542,7 +535,7 @@ export async function* _runDecodeLoop(generatedIds, opts, options, runtime) {
|
|
|
542
535
|
batchSize: executionPlan.batchSize,
|
|
543
536
|
useGPU: this._state.useGPU,
|
|
544
537
|
gpuSamplingAvailable,
|
|
545
|
-
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
|
|
538
|
+
disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens || typeof opts.logitMaskFn === 'function',
|
|
546
539
|
disableCommandBatching: executionPlan.disableCommandBatching,
|
|
547
540
|
isBdpaPagedLayout: this._state.kvCache?.layout === 'bdpa_paged',
|
|
548
541
|
finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
|
|
@@ -558,6 +551,7 @@ export async function* _runDecodeLoop(generatedIds, opts, options, runtime) {
|
|
|
558
551
|
else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
|
|
559
552
|
else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
|
|
560
553
|
else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
|
|
554
|
+
else if (typeof opts.logitMaskFn === 'function') reason = 'token_constraint_requires_per_token_logits';
|
|
561
555
|
else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
|
|
562
556
|
else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
|
|
563
557
|
else if (this._state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { getDevice, setTrackSubmits } from '../../../../gpu/device.js';
|
|
2
2
|
import { releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
3
3
|
import { recordArgmax, recordGPUSample, isGPUSamplingAvailable } from '../../../../gpu/kernels/sample.js';
|
|
4
|
-
import {
|
|
4
|
+
import { recordHistoryPenalties } from '../../../../gpu/kernels/rep-penalty.js';
|
|
5
5
|
import { recordCheckStop } from '../../../../gpu/kernels/check-stop.js';
|
|
6
6
|
import { recordCheckHotVocabStop } from '../../../../gpu/kernels/check-hot-vocab-stop.js';
|
|
7
7
|
import { resetSubmitStats, logSubmitStats } from '../../../../gpu/submit-tracker.js';
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
isDecodeRecorderEnabled,
|
|
16
16
|
isProfileDecodeRecorderEnabled,
|
|
17
17
|
} from '../execution-plan.js';
|
|
18
|
-
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from '../sampling.js';
|
|
18
|
+
import { sample, applyRepetitionPenalty, applyPresencePenalty, logitsSanity, getTopK } from '../sampling.js';
|
|
19
19
|
import { isStopToken } from '../init.js';
|
|
20
20
|
import { embed } from '../embed.js';
|
|
21
21
|
import { resolvePerLayerInputsSession } from './session-context.js';
|
|
@@ -198,6 +198,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
198
198
|
cleanupCompleted = true;
|
|
199
199
|
const copyStart = performance.now();
|
|
200
200
|
const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
|
|
201
|
+
if (mappedWords[0] === 0xFFFFFFFF) throw new Error('[Sampling] No finite candidate logits.');
|
|
201
202
|
const result = {
|
|
202
203
|
nextToken: mappedWords[0],
|
|
203
204
|
finitenessStatus: hasFinitenessBuffer
|
|
@@ -532,7 +533,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
532
533
|
const useFusedDecode = shouldUseFusedDecodeSampling({
|
|
533
534
|
recorderEnabled: Boolean(recorder),
|
|
534
535
|
gpuSamplingEnabled: useGPUSampling,
|
|
535
|
-
fusedDecodeDisabled: state.disableFusedDecode,
|
|
536
|
+
fusedDecodeDisabled: state.disableFusedDecode || opts.suppressTokenIds.length > 0 || typeof opts.logitMaskFn === 'function',
|
|
536
537
|
layerTypes: config.layerTypes,
|
|
537
538
|
});
|
|
538
539
|
|
|
@@ -561,6 +562,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
561
562
|
);
|
|
562
563
|
|
|
563
564
|
const ringTokensBuffer = ringSlot?.tokens ?? null;
|
|
565
|
+
await recordHistoryPenalties(recorder, logitsBuffer, currentIds, { ...opts, vocabSize, logitsDtype });
|
|
564
566
|
const sampleOutputBuffer = opts.temperature < samplingDefaults.greedyThreshold
|
|
565
567
|
? await recordArgmax(recorder, logitsBuffer, vocabSize, {
|
|
566
568
|
padTokenId,
|
|
@@ -572,6 +574,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
572
574
|
: await recordGPUSample(recorder, logitsBuffer, vocabSize, {
|
|
573
575
|
temperature: opts.temperature,
|
|
574
576
|
topK: opts.topK,
|
|
577
|
+
topP: opts.topP,
|
|
575
578
|
padTokenId,
|
|
576
579
|
logitSoftcap,
|
|
577
580
|
logitsDtype,
|
|
@@ -723,7 +726,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
723
726
|
null,
|
|
724
727
|
state.operatorDiagnostics
|
|
725
728
|
);
|
|
726
|
-
applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty);
|
|
729
|
+
applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
730
|
+
if (opts.presencePenalty) {
|
|
731
|
+
applyPresencePenalty(fallbackLogits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
732
|
+
}
|
|
727
733
|
const fallbackToken = sample(fallbackLogits, {
|
|
728
734
|
temperature: opts.temperature,
|
|
729
735
|
topP: opts.topP,
|
|
@@ -800,7 +806,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
800
806
|
const rawLogits = decodeReadback(logitsData, logitsDtype);
|
|
801
807
|
const sampledLogits = extractLastPositionLogits(rawLogits, numTokens, config.vocabSize);
|
|
802
808
|
|
|
803
|
-
applyRepetitionPenalty(sampledLogits, currentIds, opts.repetitionPenalty);
|
|
809
|
+
applyRepetitionPenalty(sampledLogits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
810
|
+
if (opts.presencePenalty) {
|
|
811
|
+
applyPresencePenalty(sampledLogits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
812
|
+
}
|
|
804
813
|
const nextToken = sample(sampledLogits, {
|
|
805
814
|
temperature: opts.temperature,
|
|
806
815
|
topP: opts.topP,
|
|
@@ -854,7 +863,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
854
863
|
logitsSanity(logits, `Decode[${state.decodeStepCount}]`, opts.decode);
|
|
855
864
|
}
|
|
856
865
|
|
|
857
|
-
applyRepetitionPenalty(logits, currentIds, opts.repetitionPenalty);
|
|
866
|
+
applyRepetitionPenalty(logits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
867
|
+
if (opts.presencePenalty) {
|
|
868
|
+
applyPresencePenalty(logits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
869
|
+
}
|
|
858
870
|
const nextToken = sample(logits, {
|
|
859
871
|
temperature: opts.temperature,
|
|
860
872
|
topP: opts.topP,
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
validateCaptureConfig,
|
|
9
9
|
} from '../../../../debug/index.js';
|
|
10
10
|
import { validateCallTimeOptions } from '../../../../config/param-validator.js';
|
|
11
|
-
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from '../sampling.js';
|
|
11
|
+
import { sample, applyRepetitionPenalty, applyPresencePenalty, logitsSanity, getTopK } from '../sampling.js';
|
|
12
12
|
import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from '../logits/index.js';
|
|
13
13
|
import { OperatorEventEmitter } from '../operator-events.js';
|
|
14
14
|
import {
|
|
@@ -705,7 +705,8 @@ export async function* generateWithPrefixKV(prefix, prompt, options = {}) {
|
|
|
705
705
|
const prefillLogits = await this._prefill(inputIds, opts);
|
|
706
706
|
this._state.stats.prefillTimeMs = performance.now() - prefillStart;
|
|
707
707
|
|
|
708
|
-
applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty);
|
|
708
|
+
applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
709
|
+
applyPresencePenalty(prefillLogits, generatedIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
709
710
|
const padTokenId = this._state.tokenizer?.getSpecialTokens?.()?.pad;
|
|
710
711
|
const firstToken = sample(prefillLogits, {
|
|
711
712
|
temperature: opts.temperature,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
2
2
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
3
3
|
import { runFinalizeLogitsTensor } from '../../../gpu/kernels/logit-finalize.js';
|
|
4
|
-
import { applyRepetitionPenalty } from './sampling.js';
|
|
4
|
+
import { applyRepetitionPenalty, applyPresencePenalty } from './sampling.js';
|
|
5
5
|
|
|
6
6
|
export function emitObservedLogits(onLogits, logits, tokenId, currentIds) {
|
|
7
7
|
if (typeof onLogits !== 'function') return false;
|
|
@@ -40,6 +40,9 @@ export async function captureObservedFusedDecodeLogits(
|
|
|
40
40
|
);
|
|
41
41
|
releaseBuffer(finalized.buffer);
|
|
42
42
|
const observedLogits = new Float32Array(logitsData);
|
|
43
|
-
applyRepetitionPenalty(observedLogits, currentIds, opts.repetitionPenalty);
|
|
43
|
+
applyRepetitionPenalty(observedLogits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
|
|
44
|
+
if (opts.presencePenalty) {
|
|
45
|
+
applyPresencePenalty(observedLogits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
|
|
46
|
+
}
|
|
44
47
|
return emitObservedLogits(opts.onLogits, observedLogits, tokenId, currentIds);
|
|
45
48
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readBuffer } from '../../../memory/buffer-pool.js';
|
|
2
|
+
import { resolveSamplingConfig } from './sampling-config.js';
|
|
2
3
|
import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
|
|
3
4
|
import { decodeReadback } from './debug-utils/index.js';
|
|
4
5
|
import { resolveExecutionSessionPlan } from './execution-plan.js';
|
|
@@ -160,24 +161,18 @@ function resolveChatTemplateEnabled(state, options) {
|
|
|
160
161
|
|
|
161
162
|
export function resolveStepOptions(state, options = {}) {
|
|
162
163
|
const runtimeDefaults = state.runtimeConfig.inference;
|
|
163
|
-
const samplingDefaults = runtimeDefaults.sampling;
|
|
164
164
|
const executionPlan = resolveExecutionSessionPlan(state, options);
|
|
165
165
|
|
|
166
166
|
return {
|
|
167
|
+
logitMaskFn: resolveLogitMask(options),
|
|
168
|
+
promptTokenCount: 0,
|
|
167
169
|
seed: resolveConfiguredValue(
|
|
168
170
|
options.seed,
|
|
169
171
|
undefined,
|
|
170
172
|
'options.seed',
|
|
171
173
|
(value) => Number.isFinite(value) && value >= 0
|
|
172
174
|
),
|
|
173
|
-
|
|
174
|
-
topP: resolveConfiguredValue(options.topP, samplingDefaults.topP, 'options.topP'),
|
|
175
|
-
topK: resolveConfiguredValue(options.topK, samplingDefaults.topK, 'options.topK'),
|
|
176
|
-
repetitionPenalty: resolveConfiguredValue(
|
|
177
|
-
options.repetitionPenalty,
|
|
178
|
-
samplingDefaults.repetitionPenalty,
|
|
179
|
-
'options.repetitionPenalty'
|
|
180
|
-
),
|
|
175
|
+
...resolveSamplingConfig(options, state.runtimeConfig),
|
|
181
176
|
debug: resolveConfiguredValue(options.debug, state.debug, 'options.debug', (value) => typeof value === 'boolean'),
|
|
182
177
|
debugLayers: options.debugLayers,
|
|
183
178
|
profile: resolveConfiguredValue(options.profile, runtimeDefaults.generation.profile, 'options.profile'),
|
|
@@ -195,11 +190,16 @@ export function resolveStepOptions(state, options = {}) {
|
|
|
195
190
|
|
|
196
191
|
export function resolveGenerateOptions(state, options = {}) {
|
|
197
192
|
const runtimeDefaults = state.runtimeConfig.inference;
|
|
198
|
-
const samplingDefaults = runtimeDefaults.sampling;
|
|
199
193
|
const generationDefaults = runtimeDefaults.generation;
|
|
200
194
|
const executionPlan = resolveExecutionSessionPlan(state, options);
|
|
195
|
+
const logitMaskFn = resolveLogitMask(options);
|
|
196
|
+
const speculation = resolveSpeculationConfig(state, options);
|
|
197
|
+
if (logitMaskFn && (speculation || (options.useSpeculative ?? generationDefaults.useSpeculative))) {
|
|
198
|
+
throw new Error('[Pipeline] logitMaskFn is incompatible with speculative decoding.');
|
|
199
|
+
}
|
|
201
200
|
|
|
202
201
|
return {
|
|
202
|
+
logitMaskFn,
|
|
203
203
|
seed: resolveConfiguredValue(
|
|
204
204
|
options.seed,
|
|
205
205
|
undefined,
|
|
@@ -207,14 +207,7 @@ export function resolveGenerateOptions(state, options = {}) {
|
|
|
207
207
|
(value) => Number.isFinite(value) && value >= 0
|
|
208
208
|
),
|
|
209
209
|
maxTokens: executionPlan.maxTokens,
|
|
210
|
-
|
|
211
|
-
topP: resolveConfiguredValue(options.topP, samplingDefaults.topP, 'options.topP'),
|
|
212
|
-
topK: resolveConfiguredValue(options.topK, samplingDefaults.topK, 'options.topK'),
|
|
213
|
-
repetitionPenalty: resolveConfiguredValue(
|
|
214
|
-
options.repetitionPenalty,
|
|
215
|
-
samplingDefaults.repetitionPenalty,
|
|
216
|
-
'options.repetitionPenalty'
|
|
217
|
-
),
|
|
210
|
+
...resolveSamplingConfig(options, state.runtimeConfig),
|
|
218
211
|
stopSequences: resolveConfiguredValue(options.stopSequences, [], 'options.stopSequences', Array.isArray),
|
|
219
212
|
useSpeculative: resolveConfiguredValue(
|
|
220
213
|
options.useSpeculative,
|
|
@@ -233,7 +226,7 @@ export function resolveGenerateOptions(state, options = {}) {
|
|
|
233
226
|
stopCheckMode: executionPlan.stopCheckMode,
|
|
234
227
|
executionPlan,
|
|
235
228
|
images: options.images ?? null,
|
|
236
|
-
speculation
|
|
229
|
+
speculation,
|
|
237
230
|
inputIds: resolveExplicitInputIds(options.inputIds, 'options.inputIds'),
|
|
238
231
|
embeddingOverrides: options.embeddingOverrides ?? null,
|
|
239
232
|
embeddingInputSpan: options.__internalEmbeddingInputSpan ?? null,
|
|
@@ -241,6 +234,12 @@ export function resolveGenerateOptions(state, options = {}) {
|
|
|
241
234
|
};
|
|
242
235
|
}
|
|
243
236
|
|
|
237
|
+
function resolveLogitMask(options) {
|
|
238
|
+
if (options.logitMaskFn == null) return null;
|
|
239
|
+
if (typeof options.logitMaskFn !== 'function') throw new Error('[Pipeline] logitMaskFn must be a synchronous function or null.');
|
|
240
|
+
return options.logitMaskFn;
|
|
241
|
+
}
|
|
242
|
+
|
|
244
243
|
function resolveSpeculationConfig(state, options) {
|
|
245
244
|
const sessionSpeculation = state.runtimeConfig?.inference?.session?.speculation;
|
|
246
245
|
const callSpeculation = options.speculation ?? null;
|