doppler-gpu 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +8 -2
  2. package/src/capsule-runtime.d.ts +2 -0
  3. package/src/capsule-runtime.js +1 -0
  4. package/src/client/capsule-host.browser.js +1 -0
  5. package/src/client/capsule-host.d.ts +2 -0
  6. package/src/client/capsule-host.js +1 -0
  7. package/src/client/doppler-api.d.ts +2 -0
  8. package/src/client/doppler-api.js +1 -0
  9. package/src/client/model-host/model-session.d.ts +1 -0
  10. package/src/client/model-host/model-session.js +12 -0
  11. package/src/client/runtime/capsule-operation-adapters.js +15 -14
  12. package/src/client/runtime/capsule-operation-executor.js +3 -2
  13. package/src/client/runtime/capsule-program-adapter.js +2 -0
  14. package/src/client/runtime/capsule-session-execution.js +9 -5
  15. package/src/client/runtime/composition-root.d.ts +6 -3
  16. package/src/client/runtime/composition-root.js +13 -5
  17. package/src/client/runtime/session-controller.d.ts +6 -17
  18. package/src/client/runtime/session-controller.js +42 -120
  19. package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
  20. package/src/config/capsule-loading.d.ts +1 -1
  21. package/src/config/capsule-loading.schema.json +3 -2
  22. package/src/config/capsule-operation.js +6 -1
  23. package/src/config/capsule-operations.json +1 -2
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  26. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  28. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
  31. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
  32. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
  33. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
  37. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
  39. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
  42. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
  43. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  44. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  45. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  46. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  47. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  48. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
  49. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  50. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
  53. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
  55. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
  56. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
  57. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
  58. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
  59. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
  60. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
  61. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
  62. package/src/config/generation-contract.d.ts +39 -0
  63. package/src/config/generation-contract.js +66 -0
  64. package/src/config/generation-contract.json +32 -0
  65. package/src/config/kernel-path-loader.d.ts +7 -0
  66. package/src/config/kernel-path-loader.js +19 -0
  67. package/src/config/kernels/kernel-ref-digests.js +16 -16
  68. package/src/config/kernels/registry.json +35 -10
  69. package/src/config/schema/doppler.schema.js +1 -0
  70. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  71. package/src/converter/forge-qualification.js +2 -1
  72. package/src/converter/forge-stages.js +6 -4
  73. package/src/converter/quantizer.js +18 -4
  74. package/src/gpu/device-state.js +8 -0
  75. package/src/gpu/device.js +6 -1
  76. package/src/gpu/kernels/gelu.d.ts +4 -0
  77. package/src/gpu/kernels/gelu.js +11 -1
  78. package/src/gpu/kernels/gelu.wgsl +16 -0
  79. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  80. package/src/gpu/kernels/rep-penalty.js +30 -5
  81. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  82. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  83. package/src/gpu/kernels/sample.d.ts +1 -0
  84. package/src/gpu/kernels/sample.js +31 -16
  85. package/src/gpu/kernels/sample.wgsl +112 -157
  86. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  87. package/src/index.d.ts +2 -0
  88. package/src/index.js +1 -0
  89. package/src/inference/pipelines/context.js +6 -2
  90. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  91. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
  92. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
  93. package/src/inference/pipelines/text/ffn/dense.js +2 -2
  94. package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
  95. package/src/inference/pipelines/text/generator/decode.js +18 -6
  96. package/src/inference/pipelines/text/generator/training.js +3 -2
  97. package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
  98. package/src/inference/pipelines/text/generator-runtime.js +18 -19
  99. package/src/inference/pipelines/text/generator-steps.js +15 -8
  100. package/src/inference/pipelines/text/generator.js +16 -10
  101. package/src/inference/pipelines/text/linear-attention.js +24 -0
  102. package/src/inference/pipelines/text/logits/output-transform.js +10 -3
  103. package/src/inference/pipelines/text/sampling-config.js +22 -113
  104. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  105. package/src/inference/pipelines/text/sampling.js +18 -375
  106. package/src/inference/pipelines/text/types.d.ts +3 -3
  107. package/src/inference/token-sampling.js +319 -0
  108. package/src/storage/backends/opfs-store.d.ts +41 -0
  109. package/src/tooling/node-browser/static-server.js +1 -0
  110. package/src/tooling-exports/storage.d.ts +2 -0
  111. package/src/tooling-exports/storage.js +1 -0
  112. package/src/version.js +1 -1
  113. package/tests/kernels/browser/test-page.js +1 -0
@@ -1,137 +1,134 @@
1
- // Minimal JSON-grammar sampling constraint.
2
- //
3
- // Usage:
4
- // const mask = createJsonGrammarMask({ tokenizer });
5
- // for await (const text of pipeline.generate(prompt, { logitMaskFn: mask })) { ... }
6
- //
7
- // The mask classifies each candidate token against the current parse state and
8
- // zeroes logits for tokens that would break a valid JSON object output. It
9
- // deliberately accepts any token that *looks like* it may extend valid JSON —
10
- // this is a soft constraint; parsing the final output is still required to
11
- // validate the full payload.
12
- //
13
- // Tokenizer assumption: `tokenizer.decode([id])` returns the literal string
14
- // piece for that token. That's true for BPE tokenizers used by Qwen, Gemma,
15
- // Llama. The mask is deliberately token-piece based, not character level,
16
- // because re-encoding the stream to characters per step is prohibitively slow
17
- // in-browser.
18
- //
19
- // What the mask does enforce:
20
- // - Until the first "{" is emitted, only tokens whose piece contains "{" or
21
- // whitespace are allowed.
22
- // - Once inside the object, matched counts of "{" / "}" / "[" / "]" / "\""
23
- // determine whether we are inside a string; inside a string, any token is
24
- // allowed except one that closes more brackets than are open.
25
- // - After a balanced closing "}" is emitted at depth 0, we forbid any
26
- // further non-whitespace token (forces a clean stop).
27
- //
28
- // What the mask does NOT enforce (intentional, to keep the mask cheap):
29
- // - Key quoting, trailing-comma avoidance, matching " pairs at token level.
30
- // Post-decode `parseJSONObjectEnvelope` handles these via recovery
31
- // transformations.
32
- // - Schema/key whitelists. Callers can layer a stricter mask on top.
1
+ // Incremental JSON object syntax constraint. Schema validation remains a
2
+ // separate caller-owned boundary; this mask does not enforce application keys.
3
+ // Tokenizers must decode individual pieces without trimming whitespace.
33
4
 
34
- function pieceAt(tokenizer, tokenId) {
35
- if (!tokenizer || typeof tokenizer.decode !== "function") return "";
36
- try {
37
- // decode(ids, skipSpecial, skipBos)
38
- return String(tokenizer.decode([tokenId], true, false) || "");
39
- } catch {
40
- return "";
41
- }
5
+ function initialState() {
6
+ return { stack: [{ kind: 'root', expect: 'object' }], scalar: null };
42
7
  }
43
8
 
44
- function countRunes(piece, char) {
45
- let count = 0;
46
- for (let i = 0; i < piece.length; i += 1) {
47
- if (piece[i] === char) count += 1;
48
- }
49
- return count;
9
+ function completeValue(state) {
10
+ const parent = state.stack[state.stack.length - 1];
11
+ parent.expect = parent.kind === 'root' ? 'done' : 'comma-or-end';
50
12
  }
51
13
 
52
- // Piece-level flags used by the mask.
53
- function classifyPiece(piece) {
54
- const openBraces = countRunes(piece, "{");
55
- const closeBraces = countRunes(piece, "}");
56
- const openBrackets = countRunes(piece, "[");
57
- const closeBrackets = countRunes(piece, "]");
58
- const quotes = countRunes(piece, "\"");
59
- return {
60
- openBraces,
61
- closeBraces,
62
- openBrackets,
63
- closeBrackets,
64
- quotes,
65
- isOnlyWhitespace: /^\s*$/.test(piece),
66
- hasOpenBrace: openBraces > 0,
67
- };
14
+ function beginValue(state, char) {
15
+ if (char === '{' || char === '[') {
16
+ state.stack.push({ kind: char === '{' ? 'object' : 'array', expect: char === '{' ? 'key-or-end' : 'value-or-end' });
17
+ } else if (char === '"') state.scalar = { kind: 'string', role: 'value', escaped: false, unicode: 0 };
18
+ else if (char === 't' || char === 'f' || char === 'n') state.scalar = { kind: 'literal', remaining: { t: 'rue', f: 'alse', n: 'ull' }[char] };
19
+ else if (char === '-' || /[0-9]/.test(char)) state.scalar = { kind: 'number', phase: char === '-' ? 'sign' : char === '0' ? 'zero' : 'integer' };
20
+ else return false;
21
+ return true;
22
+ }
23
+
24
+ function advanceNumber(number, char) {
25
+ const digit = /[0-9]/.test(char);
26
+ if (number.phase === 'sign') {
27
+ if (!digit) return 'invalid';
28
+ number.phase = char === '0' ? 'zero' : 'integer';
29
+ } else if (number.phase === 'dot') {
30
+ if (!digit) return 'invalid';
31
+ number.phase = 'fraction';
32
+ } else if (number.phase === 'exponent' || number.phase === 'exponent-sign') {
33
+ if (number.phase === 'exponent' && (char === '+' || char === '-')) number.phase = 'exponent-sign';
34
+ else if (digit) number.phase = 'exponent-digits';
35
+ else return 'invalid';
36
+ } else if (digit) {
37
+ if (number.phase === 'zero') return 'invalid';
38
+ } else if (char === '.' && ['zero', 'integer'].includes(number.phase)) number.phase = 'dot';
39
+ else if ((char === 'e' || char === 'E') && ['zero', 'integer', 'fraction'].includes(number.phase)) number.phase = 'exponent';
40
+ else return /[\x20\t\r\n,\]}]/.test(char) ? 'complete' : 'invalid';
41
+ return 'continue';
42
+ }
43
+
44
+ function advance(state, text) {
45
+ for (const char of text) {
46
+ const scalar = state.scalar;
47
+ if (scalar?.kind === 'string') {
48
+ if (scalar.unicode > 0) {
49
+ if (!/[a-fA-F0-9]/.test(char)) return false;
50
+ scalar.unicode -= 1;
51
+ } else if (scalar.escaped) {
52
+ scalar.escaped = false;
53
+ if (char === 'u') scalar.unicode = 4;
54
+ else if (!'"\\/bfnrt'.includes(char)) return false;
55
+ } else if (char === '\\') scalar.escaped = true;
56
+ else if (char === '"') {
57
+ state.scalar = null;
58
+ if (scalar.role === 'key') state.stack[state.stack.length - 1].expect = 'colon';
59
+ else completeValue(state);
60
+ } else if (char.codePointAt(0) < 32) return false;
61
+ continue;
62
+ }
63
+ if (scalar?.kind === 'literal') {
64
+ if (char !== scalar.remaining[0]) return false;
65
+ scalar.remaining = scalar.remaining.slice(1);
66
+ if (!scalar.remaining) { state.scalar = null; completeValue(state); }
67
+ continue;
68
+ }
69
+ if (scalar?.kind === 'number') {
70
+ const result = advanceNumber(scalar, char);
71
+ if (result === 'invalid') return false;
72
+ if (result === 'continue') continue;
73
+ state.scalar = null;
74
+ completeValue(state);
75
+ }
76
+ if (/[\x20\t\r\n]/.test(char)) continue;
77
+ const current = state.stack[state.stack.length - 1];
78
+ if (current.expect === 'done') return false;
79
+ if (current.expect === 'object') {
80
+ if (char !== '{') return false;
81
+ if (!beginValue(state, char)) return false;
82
+ } else if (current.expect === 'colon') {
83
+ if (char !== ':') return false;
84
+ current.expect = 'value';
85
+ } else if (current.expect === 'key' || current.expect === 'key-or-end') {
86
+ if (char === '}' && current.expect === 'key-or-end') { state.stack.pop(); completeValue(state); }
87
+ else if (char === '"') state.scalar = { kind: 'string', role: 'key', escaped: false, unicode: 0 };
88
+ else return false;
89
+ } else if (current.expect === 'comma-or-end') {
90
+ if (char === ',') current.expect = current.kind === 'object' ? 'key' : 'value';
91
+ else if (char === (current.kind === 'object' ? '}' : ']')) { state.stack.pop(); completeValue(state); }
92
+ else return false;
93
+ } else if (current.expect === 'value-or-end' && char === ']') {
94
+ state.stack.pop(); completeValue(state);
95
+ } else if (!beginValue(state, char)) return false;
96
+ }
97
+ return true;
68
98
  }
69
99
 
70
100
  export function createJsonGrammarMask(opts = {}) {
71
101
  const pieceCache = new Map();
72
102
  const cacheBudget = Math.max(1024, Math.floor(Number(opts.cacheBudget) || 32768));
103
+ let cachedTokenizer = null;
73
104
 
74
- function cachedPiece(tokenizer, tokenId) {
75
- if (pieceCache.has(tokenId)) return pieceCache.get(tokenId);
105
+ function pieceAt(tokenizer, id) {
106
+ if (pieceCache.has(id)) return pieceCache.get(id);
76
107
  if (pieceCache.size >= cacheBudget) pieceCache.clear();
77
- const piece = pieceAt(tokenizer, tokenId);
78
- pieceCache.set(tokenId, piece);
108
+ const piece = tokenizer.decode([id], true, false);
109
+ if (typeof piece !== 'string') throw new Error('JSON constraint requires synchronous string token decoding');
110
+ pieceCache.set(id, piece);
79
111
  return piece;
80
112
  }
81
113
 
82
- let runningOpenBraces = 0;
83
- let runningOpenBrackets = 0;
84
- let runningQuotes = 0;
85
- let sawFirstBrace = false;
86
- let balancedClose = false;
87
- let lastGeneratedLen = 0;
88
-
89
- function reflectEmittedTokens(tokenizer, generatedIds) {
90
- // Catch up to any tokens committed since we were last called.
91
- for (let i = lastGeneratedLen; i < generatedIds.length; i += 1) {
92
- const piece = cachedPiece(tokenizer, generatedIds[i]);
93
- const shape = classifyPiece(piece);
94
- runningOpenBraces += shape.openBraces - shape.closeBraces;
95
- runningOpenBrackets += shape.openBrackets - shape.closeBrackets;
96
- runningQuotes += shape.quotes;
97
- if (!sawFirstBrace && shape.hasOpenBrace) sawFirstBrace = true;
98
- if (sawFirstBrace && runningOpenBraces <= 0 && runningOpenBrackets <= 0) {
99
- balancedClose = true;
100
- }
101
- }
102
- lastGeneratedLen = generatedIds.length;
103
- }
104
-
105
114
  return function logitMask(logits, context) {
106
- const tokenizer = (opts.tokenizer ?? context?.tokenizer) || null;
107
- if (!tokenizer) return;
108
- reflectEmittedTokens(tokenizer, Array.isArray(context?.generatedIds) ? context.generatedIds : []);
109
- const insideString = runningQuotes % 2 === 1;
110
- const vocabSize = Math.min(logits.length, Number(context?.vocabSize || logits.length));
111
- for (let id = 0; id < vocabSize; id += 1) {
112
- if (logits[id] <= -Infinity) continue;
113
- const piece = cachedPiece(tokenizer, id);
114
- if (!piece) continue;
115
- const shape = classifyPiece(piece);
116
- let ok = true;
117
- if (!sawFirstBrace) {
118
- // Before the first "{", only allow whitespace or a piece that opens a brace.
119
- ok = shape.isOnlyWhitespace || shape.hasOpenBrace;
120
- } else if (balancedClose) {
121
- // After the balanced close, only whitespace is allowed (model should stop).
122
- ok = shape.isOnlyWhitespace;
123
- } else if (insideString) {
124
- // Inside a string literal, accept any piece — even ones with braces or
125
- // brackets — the parser handles them as content.
126
- ok = true;
127
- } else {
128
- // Outside strings: disallow pieces that close MORE brackets than are open.
129
- if (shape.closeBraces > runningOpenBraces + shape.openBraces) ok = false;
130
- if (shape.closeBrackets > runningOpenBrackets + shape.openBrackets) ok = false;
131
- }
132
- if (!ok) {
133
- logits[id] = -Infinity;
134
- }
115
+ const tokenizer = opts.tokenizer ?? context?.tokenizer;
116
+ if (!tokenizer || typeof tokenizer.decode !== 'function') throw new Error('JSON constraint requires a tokenizer');
117
+ if (!Array.isArray(context?.generatedIds)) throw new Error('JSON constraint requires generated token IDs');
118
+ if (cachedTokenizer !== tokenizer) { pieceCache.clear(); cachedTokenizer = tokenizer; }
119
+ const state = initialState();
120
+ const prefix = tokenizer.decode(context.generatedIds, true, false);
121
+ if (typeof prefix !== 'string' || !advance(state, prefix)) throw new Error('JSON constraint received an invalid generated prefix');
122
+ const complete = state.stack.length === 1 && state.stack[0].expect === 'done';
123
+ const eos = tokenizer.getSpecialTokens?.().eos;
124
+ const stopIds = new Set(opts.stopTokenIds ?? (Array.isArray(eos) ? eos : Number.isInteger(eos) ? [eos] : []));
125
+ if (![...stopIds].every(id => Number.isInteger(id) && id >= 0 && id < logits.length)) throw new Error('JSON constraint has invalid stop token IDs');
126
+ for (let id = 0; id < logits.length; id += 1) {
127
+ if (logits[id] === -Infinity) continue;
128
+ if (stopIds.has(id)) { if (!complete) logits[id] = -Infinity; continue; }
129
+ const piece = pieceAt(tokenizer, id);
130
+ const candidate = { stack: state.stack.map(row => ({ ...row })), scalar: state.scalar ? { ...state.scalar } : null };
131
+ if (!piece || !advance(candidate, piece)) logits[id] = -Infinity;
135
132
  }
136
133
  };
137
134
  }
@@ -319,6 +319,8 @@ export async function runDenseUngatedFFNGPU(layerIdx, inputTensor, numTokens, co
319
319
  layerIdx
320
320
  );
321
321
  const activated = await dispatchActivation(hiddenActivation, upOutput, {
322
+ kernelPath,
323
+ phase,
322
324
  size: numTokens * intermediateSize,
323
325
  gate: null,
324
326
  label: `L${layerIdx}.ffn_activation`,
@@ -659,6 +661,9 @@ export async function runDenseFFNWithFusedPostNormGPU(
659
661
  );
660
662
 
661
663
  activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
664
+ kernelPath,
665
+ phase,
666
+ layerIdx,
662
667
  size: numTokens * intermediateSize,
663
668
  gate: gateOutput,
664
669
  inputActivation: 'identity',
@@ -1,5 +1,3 @@
1
-
2
-
3
1
  import {
4
2
  doBiasAdd, doMatmul, doSiLU, doGeLU, doSiLURowSplit, doMatmulRMSNormFused,
5
3
  releaseOrTrack
@@ -819,6 +817,8 @@ export async function runDenseFFNGPU(
819
817
  enqueueRecordedDenseHealth(context, layerIdx, 'ffn_up', upOutput, numTokens * intermediateSize);
820
818
 
821
819
  const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
820
+ kernelPath,
821
+ phase,
822
822
  size: numTokens * intermediateSize,
823
823
  gate: gateOutput,
824
824
  inputActivation: 'identity',
@@ -91,7 +91,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
91
91
  if (this._state.isGenerating) throw new Error('Generation already in progress');
92
92
 
93
93
  validateCallTimeOptions(options);
94
-
94
+ this._resetReplayPrefillRuntimeState();
95
95
  this._state.isGenerating = true;
96
96
  this._resetDecodeRuntimeState();
97
97
  this._state.stats.gpuTimePrefillMs = undefined;
@@ -131,13 +131,9 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
131
131
 
132
132
  const opts = resolveGenerateOptions(this._state, options);
133
133
  opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
134
- opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
135
134
  // Validate and normalize sampling parameters through single source of truth
136
135
  const samplingConfig = resolveSamplingConfig(options, this._state.runtimeConfig);
137
- opts.temperature = samplingConfig.temperature;
138
- opts.topP = samplingConfig.topP;
139
- opts.topK = samplingConfig.topK;
140
- opts.repetitionPenalty = samplingConfig.repetitionPenalty;
136
+ Object.assign(opts, samplingConfig);
141
137
  opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this._state, samplingConfig);
142
138
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
143
139
  || this._state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
@@ -195,7 +191,7 @@ export async function* _generateTokensInternal(prompt, options = {}, mode = 'tex
195
191
  this._state.stats.prefillTimeMs = performance.now() - prefillStart;
196
192
  this._assertTokenIdsInRange(inputIds, 'generate.prefillTokens');
197
193
  const generatedIds = [...inputIds];
198
- this._state.stats.prefillTokens = inputIds.length;
194
+ this._state.stats.prefillTokens = opts.promptTokenCount = inputIds.length;
199
195
 
200
196
  if (opts.debug) {
201
197
  log.debug('Pipeline', `Input: ${inputIds.length} tokens`);
@@ -330,7 +326,7 @@ export async function generateTokenIds(prompt, options = {}) {
330
326
  if (this._state.isGenerating) throw new Error('Generation already in progress');
331
327
 
332
328
  validateCallTimeOptions(options);
333
-
329
+ this._resetReplayPrefillRuntimeState();
334
330
  this._state.isGenerating = true;
335
331
  this._resetDecodeRuntimeState();
336
332
  this._state.stats.gpuTimePrefillMs = undefined;
@@ -371,10 +367,7 @@ export async function generateTokenIds(prompt, options = {}) {
371
367
  opts.onLogits = typeof options.onLogits === 'function' ? options.onLogits : null;
372
368
  // Validate and normalize sampling parameters through single source of truth
373
369
  const samplingConfig = resolveSamplingConfig(options, this._state.runtimeConfig);
374
- opts.temperature = samplingConfig.temperature;
375
- opts.topP = samplingConfig.topP;
376
- opts.topK = samplingConfig.topK;
377
- opts.repetitionPenalty = samplingConfig.repetitionPenalty;
370
+ Object.assign(opts, samplingConfig);
378
371
  opts.suppressTokenIds = resolveSuppressedSamplingTokenIds(this._state, samplingConfig);
379
372
  const diagnosticsEnabled = options?.diagnostics?.enabled === true
380
373
  || this._state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
@@ -407,7 +400,7 @@ export async function generateTokenIds(prompt, options = {}) {
407
400
  this._state.stats.prefillTimeMs = performance.now() - prefillStart;
408
401
  this._assertTokenIdsInRange(inputIds, 'generateTokenIds.prefillTokens');
409
402
  const generatedIds = [...inputIds];
410
- this._state.stats.prefillTokens = inputIds.length;
403
+ this._state.stats.prefillTokens = opts.promptTokenCount = inputIds.length;
411
404
 
412
405
  let firstToken;
413
406
  try {
@@ -542,7 +535,7 @@ export async function* _runDecodeLoop(generatedIds, opts, options, runtime) {
542
535
  batchSize: executionPlan.batchSize,
543
536
  useGPU: this._state.useGPU,
544
537
  gpuSamplingAvailable,
545
- disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens,
538
+ disableMultiTokenDecode: executionPlan.disableMultiTokenDecode || hasSuppressedSamplingTokens || typeof opts.logitMaskFn === 'function',
546
539
  disableCommandBatching: executionPlan.disableCommandBatching,
547
540
  isBdpaPagedLayout: this._state.kvCache?.layout === 'bdpa_paged',
548
541
  finitenessFallbackWindowOpen: this._hasFinitenessFallbackWindow(),
@@ -558,6 +551,7 @@ export async function* _runDecodeLoop(generatedIds, opts, options, runtime) {
558
551
  else if (!gpuSamplingAvailable) reason = 'no_gpu_sampling';
559
552
  else if (executionPlan.disableCommandBatching) reason = 'command_batching_disabled';
560
553
  else if (hasSuppressedSamplingTokens) reason = 'sampling_suppression_requires_cpu_logits';
554
+ else if (typeof opts.logitMaskFn === 'function') reason = 'token_constraint_requires_per_token_logits';
561
555
  else if (executionPlan.disableMultiTokenDecode) reason = 'multi_token_decode_disabled';
562
556
  else if (executionPlan.batchSize <= 1) reason = 'batch_size_1';
563
557
  else if (this._state.kvCache?.layout === 'bdpa_paged') reason = 'bdpa_paged_layout';
@@ -1,7 +1,7 @@
1
1
  import { getDevice, setTrackSubmits } from '../../../../gpu/device.js';
2
2
  import { releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
3
3
  import { recordArgmax, recordGPUSample, isGPUSamplingAvailable } from '../../../../gpu/kernels/sample.js';
4
- import { recordRepPenalty } from '../../../../gpu/kernels/rep-penalty.js';
4
+ import { recordHistoryPenalties } from '../../../../gpu/kernels/rep-penalty.js';
5
5
  import { recordCheckStop } from '../../../../gpu/kernels/check-stop.js';
6
6
  import { recordCheckHotVocabStop } from '../../../../gpu/kernels/check-hot-vocab-stop.js';
7
7
  import { resetSubmitStats, logSubmitStats } from '../../../../gpu/submit-tracker.js';
@@ -15,7 +15,7 @@ import {
15
15
  isDecodeRecorderEnabled,
16
16
  isProfileDecodeRecorderEnabled,
17
17
  } from '../execution-plan.js';
18
- import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from '../sampling.js';
18
+ import { sample, applyRepetitionPenalty, applyPresencePenalty, logitsSanity, getTopK } from '../sampling.js';
19
19
  import { isStopToken } from '../init.js';
20
20
  import { embed } from '../embed.js';
21
21
  import { resolvePerLayerInputsSession } from './session-context.js';
@@ -198,6 +198,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
198
198
  cleanupCompleted = true;
199
199
  const copyStart = performance.now();
200
200
  const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
201
+ if (mappedWords[0] === 0xFFFFFFFF) throw new Error('[Sampling] No finite candidate logits.');
201
202
  const result = {
202
203
  nextToken: mappedWords[0],
203
204
  finitenessStatus: hasFinitenessBuffer
@@ -532,7 +533,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
532
533
  const useFusedDecode = shouldUseFusedDecodeSampling({
533
534
  recorderEnabled: Boolean(recorder),
534
535
  gpuSamplingEnabled: useGPUSampling,
535
- fusedDecodeDisabled: state.disableFusedDecode,
536
+ fusedDecodeDisabled: state.disableFusedDecode || opts.suppressTokenIds.length > 0 || typeof opts.logitMaskFn === 'function',
536
537
  layerTypes: config.layerTypes,
537
538
  });
538
539
 
@@ -561,6 +562,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
561
562
  );
562
563
 
563
564
  const ringTokensBuffer = ringSlot?.tokens ?? null;
565
+ await recordHistoryPenalties(recorder, logitsBuffer, currentIds, { ...opts, vocabSize, logitsDtype });
564
566
  const sampleOutputBuffer = opts.temperature < samplingDefaults.greedyThreshold
565
567
  ? await recordArgmax(recorder, logitsBuffer, vocabSize, {
566
568
  padTokenId,
@@ -572,6 +574,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
572
574
  : await recordGPUSample(recorder, logitsBuffer, vocabSize, {
573
575
  temperature: opts.temperature,
574
576
  topK: opts.topK,
577
+ topP: opts.topP,
575
578
  padTokenId,
576
579
  logitSoftcap,
577
580
  logitsDtype,
@@ -723,7 +726,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
723
726
  null,
724
727
  state.operatorDiagnostics
725
728
  );
726
- applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty);
729
+ applyRepetitionPenalty(fallbackLogits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
730
+ if (opts.presencePenalty) {
731
+ applyPresencePenalty(fallbackLogits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
732
+ }
727
733
  const fallbackToken = sample(fallbackLogits, {
728
734
  temperature: opts.temperature,
729
735
  topP: opts.topP,
@@ -800,7 +806,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
800
806
  const rawLogits = decodeReadback(logitsData, logitsDtype);
801
807
  const sampledLogits = extractLastPositionLogits(rawLogits, numTokens, config.vocabSize);
802
808
 
803
- applyRepetitionPenalty(sampledLogits, currentIds, opts.repetitionPenalty);
809
+ applyRepetitionPenalty(sampledLogits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
810
+ if (opts.presencePenalty) {
811
+ applyPresencePenalty(sampledLogits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
812
+ }
804
813
  const nextToken = sample(sampledLogits, {
805
814
  temperature: opts.temperature,
806
815
  topP: opts.topP,
@@ -854,7 +863,10 @@ export async function decodeStep(state, currentIds, opts, helpers) {
854
863
  logitsSanity(logits, `Decode[${state.decodeStepCount}]`, opts.decode);
855
864
  }
856
865
 
857
- applyRepetitionPenalty(logits, currentIds, opts.repetitionPenalty);
866
+ applyRepetitionPenalty(logits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
867
+ if (opts.presencePenalty) {
868
+ applyPresencePenalty(logits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
869
+ }
858
870
  const nextToken = sample(logits, {
859
871
  temperature: opts.temperature,
860
872
  topP: opts.topP,
@@ -8,7 +8,7 @@ import {
8
8
  validateCaptureConfig,
9
9
  } from '../../../../debug/index.js';
10
10
  import { validateCallTimeOptions } from '../../../../config/param-validator.js';
11
- import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from '../sampling.js';
11
+ import { sample, applyRepetitionPenalty, applyPresencePenalty, logitsSanity, getTopK } from '../sampling.js';
12
12
  import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, applySoftcapping } from '../logits/index.js';
13
13
  import { OperatorEventEmitter } from '../operator-events.js';
14
14
  import {
@@ -705,7 +705,8 @@ export async function* generateWithPrefixKV(prefix, prompt, options = {}) {
705
705
  const prefillLogits = await this._prefill(inputIds, opts);
706
706
  this._state.stats.prefillTimeMs = performance.now() - prefillStart;
707
707
 
708
- applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty);
708
+ applyRepetitionPenalty(prefillLogits, generatedIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
709
+ applyPresencePenalty(prefillLogits, generatedIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
709
710
  const padTokenId = this._state.tokenizer?.getSpecialTokens?.()?.pad;
710
711
  const firstToken = sample(prefillLogits, {
711
712
  temperature: opts.temperature,
@@ -1,7 +1,7 @@
1
1
  import { readBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
2
2
  import { createTensor } from '../../../gpu/tensor.js';
3
3
  import { runFinalizeLogitsTensor } from '../../../gpu/kernels/logit-finalize.js';
4
- import { applyRepetitionPenalty } from './sampling.js';
4
+ import { applyRepetitionPenalty, applyPresencePenalty } from './sampling.js';
5
5
 
6
6
  export function emitObservedLogits(onLogits, logits, tokenId, currentIds) {
7
7
  if (typeof onLogits !== 'function') return false;
@@ -40,6 +40,9 @@ export async function captureObservedFusedDecodeLogits(
40
40
  );
41
41
  releaseBuffer(finalized.buffer);
42
42
  const observedLogits = new Float32Array(logitsData);
43
- applyRepetitionPenalty(observedLogits, currentIds, opts.repetitionPenalty);
43
+ applyRepetitionPenalty(observedLogits, currentIds, opts.repetitionPenalty, opts.repetitionPenaltyWindow);
44
+ if (opts.presencePenalty) {
45
+ applyPresencePenalty(observedLogits, currentIds, opts.presencePenalty, opts.repetitionPenaltyWindow);
46
+ }
44
47
  return emitObservedLogits(opts.onLogits, observedLogits, tokenId, currentIds);
45
48
  }
@@ -1,4 +1,5 @@
1
1
  import { readBuffer } from '../../../memory/buffer-pool.js';
2
+ import { resolveSamplingConfig } from './sampling-config.js';
2
3
  import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
3
4
  import { decodeReadback } from './debug-utils/index.js';
4
5
  import { resolveExecutionSessionPlan } from './execution-plan.js';
@@ -160,24 +161,18 @@ function resolveChatTemplateEnabled(state, options) {
160
161
 
161
162
  export function resolveStepOptions(state, options = {}) {
162
163
  const runtimeDefaults = state.runtimeConfig.inference;
163
- const samplingDefaults = runtimeDefaults.sampling;
164
164
  const executionPlan = resolveExecutionSessionPlan(state, options);
165
165
 
166
166
  return {
167
+ logitMaskFn: resolveLogitMask(options),
168
+ promptTokenCount: 0,
167
169
  seed: resolveConfiguredValue(
168
170
  options.seed,
169
171
  undefined,
170
172
  'options.seed',
171
173
  (value) => Number.isFinite(value) && value >= 0
172
174
  ),
173
- temperature: resolveConfiguredValue(options.temperature, samplingDefaults.temperature, 'options.temperature'),
174
- topP: resolveConfiguredValue(options.topP, samplingDefaults.topP, 'options.topP'),
175
- topK: resolveConfiguredValue(options.topK, samplingDefaults.topK, 'options.topK'),
176
- repetitionPenalty: resolveConfiguredValue(
177
- options.repetitionPenalty,
178
- samplingDefaults.repetitionPenalty,
179
- 'options.repetitionPenalty'
180
- ),
175
+ ...resolveSamplingConfig(options, state.runtimeConfig),
181
176
  debug: resolveConfiguredValue(options.debug, state.debug, 'options.debug', (value) => typeof value === 'boolean'),
182
177
  debugLayers: options.debugLayers,
183
178
  profile: resolveConfiguredValue(options.profile, runtimeDefaults.generation.profile, 'options.profile'),
@@ -195,11 +190,16 @@ export function resolveStepOptions(state, options = {}) {
195
190
 
196
191
  export function resolveGenerateOptions(state, options = {}) {
197
192
  const runtimeDefaults = state.runtimeConfig.inference;
198
- const samplingDefaults = runtimeDefaults.sampling;
199
193
  const generationDefaults = runtimeDefaults.generation;
200
194
  const executionPlan = resolveExecutionSessionPlan(state, options);
195
+ const logitMaskFn = resolveLogitMask(options);
196
+ const speculation = resolveSpeculationConfig(state, options);
197
+ if (logitMaskFn && (speculation || (options.useSpeculative ?? generationDefaults.useSpeculative))) {
198
+ throw new Error('[Pipeline] logitMaskFn is incompatible with speculative decoding.');
199
+ }
201
200
 
202
201
  return {
202
+ logitMaskFn,
203
203
  seed: resolveConfiguredValue(
204
204
  options.seed,
205
205
  undefined,
@@ -207,14 +207,7 @@ export function resolveGenerateOptions(state, options = {}) {
207
207
  (value) => Number.isFinite(value) && value >= 0
208
208
  ),
209
209
  maxTokens: executionPlan.maxTokens,
210
- temperature: resolveConfiguredValue(options.temperature, samplingDefaults.temperature, 'options.temperature'),
211
- topP: resolveConfiguredValue(options.topP, samplingDefaults.topP, 'options.topP'),
212
- topK: resolveConfiguredValue(options.topK, samplingDefaults.topK, 'options.topK'),
213
- repetitionPenalty: resolveConfiguredValue(
214
- options.repetitionPenalty,
215
- samplingDefaults.repetitionPenalty,
216
- 'options.repetitionPenalty'
217
- ),
210
+ ...resolveSamplingConfig(options, state.runtimeConfig),
218
211
  stopSequences: resolveConfiguredValue(options.stopSequences, [], 'options.stopSequences', Array.isArray),
219
212
  useSpeculative: resolveConfiguredValue(
220
213
  options.useSpeculative,
@@ -233,7 +226,7 @@ export function resolveGenerateOptions(state, options = {}) {
233
226
  stopCheckMode: executionPlan.stopCheckMode,
234
227
  executionPlan,
235
228
  images: options.images ?? null,
236
- speculation: resolveSpeculationConfig(state, options),
229
+ speculation,
237
230
  inputIds: resolveExplicitInputIds(options.inputIds, 'options.inputIds'),
238
231
  embeddingOverrides: options.embeddingOverrides ?? null,
239
232
  embeddingInputSpan: options.__internalEmbeddingInputSpan ?? null,
@@ -241,6 +234,12 @@ export function resolveGenerateOptions(state, options = {}) {
241
234
  };
242
235
  }
243
236
 
237
+ function resolveLogitMask(options) {
238
+ if (options.logitMaskFn == null) return null;
239
+ if (typeof options.logitMaskFn !== 'function') throw new Error('[Pipeline] logitMaskFn must be a synchronous function or null.');
240
+ return options.logitMaskFn;
241
+ }
242
+
244
243
  function resolveSpeculationConfig(state, options) {
245
244
  const sessionSpeculation = state.runtimeConfig?.inference?.session?.speculation;
246
245
  const callSpeculation = options.speculation ?? null;