doppler-gpu 0.4.9 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/README.md +89 -12
  3. package/models/catalog.json +612 -7
  4. package/package.json +30 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/gpu/device.js +100 -32
  35. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  36. package/src/inference/pipelines/text/attention/projections.js +161 -24
  37. package/src/inference/pipelines/text/attention/record.js +67 -9
  38. package/src/inference/pipelines/text/attention/run.js +60 -7
  39. package/src/inference/pipelines/text/attention/types.js +18 -9
  40. package/src/inference/pipelines/text/config.d.ts +5 -0
  41. package/src/inference/pipelines/text/config.js +91 -0
  42. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  43. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  44. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  45. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  46. package/src/inference/pipelines/text/generator-steps.js +3 -1
  47. package/src/inference/pipelines/text/generator.js +90 -3
  48. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  49. package/src/inference/pipelines/text/init.d.ts +2 -0
  50. package/src/inference/pipelines/text/init.js +2 -0
  51. package/src/inference/pipelines/text/layer.js +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  53. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  54. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  55. package/src/inference/pipelines/text/logits/index.js +85 -11
  56. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  57. package/src/inference/pipelines/text/ops.js +32 -0
  58. package/src/inference/pipelines/text/types.d.ts +38 -0
  59. package/src/inference/pipelines/text/weights.d.ts +10 -0
  60. package/src/inference/pipelines/text/weights.js +33 -1
  61. package/src/inference/pipelines/text.d.ts +4 -2
  62. package/src/inference/pipelines/text.js +112 -0
  63. package/src/inference/tokenizers/behavior-flags.js +5 -1
  64. package/src/inference/tokenizers/bundled.js +99 -8
  65. package/src/loader/doppler-loader.d.ts +2 -0
  66. package/src/loader/doppler-loader.js +9 -0
  67. package/src/loader/final-weights-loader.d.ts +8 -0
  68. package/src/loader/final-weights-loader.js +42 -0
  69. package/src/loader/layer-loader.d.ts +6 -0
  70. package/src/loader/layer-loader.js +102 -3
  71. package/src/loader/loader-types.d.ts +15 -2
  72. package/src/tooling/node-converter.js +33 -1
  73. package/src/tooling/node-webgpu.js +85 -2
  74. package/src/tooling/runtime-optimization.d.ts +109 -0
  75. package/src/tooling/runtime-optimization.js +872 -0
  76. package/src/tooling-exports.shared.d.ts +19 -0
  77. package/src/tooling-exports.shared.js +11 -0
  78. package/src/version.js +1 -1
@@ -390,6 +390,12 @@ export interface GenerateOptions {
390
390
  offset: number;
391
391
  length: number;
392
392
  } | null;
393
+
394
+ /** @internal Return final-normalized embeddings for every input token. */
395
+ __returnTokenEmbeddings?: boolean;
396
+
397
+ /** @internal Return full per-token vocabulary logits. */
398
+ __returnSequenceLogits?: boolean;
393
399
  }
394
400
 
395
401
  /**
@@ -438,6 +444,30 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
438
444
  /** Pooling mode used to construct embedding */
439
445
  embeddingMode: 'last' | 'mean';
440
446
 
447
+ /** Final-normalized token matrix when explicitly requested. */
448
+ tokenEmbeddings?: Float32Array | null;
449
+
450
+ /** Full token-by-vocabulary logits when explicitly requested. */
451
+ logits?: Float32Array | null;
452
+
453
+ phase?: WorkloadPhaseTiming | null;
454
+ }
455
+
456
+ export interface SequenceEncodeOptions extends GenerateOptions {
457
+ includeTokenEmbeddings?: boolean;
458
+ includeLogits?: boolean;
459
+ }
460
+
461
+ export interface SequenceEncodeResult {
462
+ alphabet: 'amino_acid' | 'nucleotide';
463
+ tokens: number[];
464
+ tokenMask: Uint8Array;
465
+ includedTokenCount: number;
466
+ tokenEmbeddings: Float32Array | null;
467
+ pooledEmbedding: Float32Array | null;
468
+ logits: Float32Array | null;
469
+ embeddingDim: number;
470
+ vocabSize: number;
441
471
  phase?: WorkloadPhaseTiming | null;
442
472
  }
443
473
 
@@ -495,16 +525,23 @@ export type LayerWeightBuffer = GPUBuffer | WeightBuffer | Float32Array | CpuWei
495
525
  export interface LayerWeights {
496
526
  // Attention
497
527
  inputNorm: GPUBuffer | Float32Array;
528
+ inputNormBias?: GPUBuffer | Float32Array | null;
498
529
  qProj: LayerWeightBuffer;
530
+ qProjBias?: GPUBuffer | Float32Array | null;
499
531
  kProj: LayerWeightBuffer;
532
+ kProjBias?: GPUBuffer | Float32Array | null;
500
533
  vProj?: LayerWeightBuffer;
534
+ vProjBias?: GPUBuffer | Float32Array | null;
501
535
  oProj: LayerWeightBuffer;
536
+ oProjBias?: GPUBuffer | Float32Array | null;
502
537
  qGateProj?: LayerWeightBuffer | null;
503
538
  convInProj?: LayerWeightBuffer;
504
539
  convKernel?: LayerWeightBuffer;
505
540
  convOutProj?: LayerWeightBuffer;
506
541
  /** Fused Q/K/V projection (runtime-generated for 3->1 matmul optimization) */
507
542
  qkvProj?: GPUBuffer | WeightBuffer | null;
543
+ /** Concatenated Q/K/V projection bias matching qkvSizes. */
544
+ qkvProjBias?: GPUBuffer | Float32Array | null;
508
545
  /** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
509
546
  qkvSizes?: [number, number, number];
510
547
  /** Data type of fused QKV weights (f16, f32, or q4k) */
@@ -516,6 +553,7 @@ export interface LayerWeights {
516
553
 
517
554
  // FFN (dense layers)
518
555
  postAttentionNorm?: GPUBuffer | Float32Array;
556
+ postAttentionNormBias?: GPUBuffer | Float32Array | null;
519
557
  postAttnNorm?: GPUBuffer | Float32Array; // LLaMA-style pre-FFN norm
520
558
  gate?: LayerWeightBuffer;
521
559
  up?: LayerWeightBuffer;
@@ -12,6 +12,7 @@
12
12
 
13
13
  import type { LayerWeights } from './types.js';
14
14
  import type { WeightBuffer, CpuWeightBuffer } from '../../../gpu/weight-buffer.js';
15
+ import type { Tensor } from '../../../gpu/tensor.js';
15
16
 
16
17
  // ============================================================================
17
18
  // Types
@@ -60,3 +61,12 @@ export function getNormWeightBuffer(
60
61
  debugFlags?: WeightDebugFlags,
61
62
  deviceOverride?: GPUDevice | null
62
63
  ): GPUBuffer;
64
+
65
+ export function getVectorTensor(
66
+ weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | { buffer: ArrayBuffer; byteOffset: number; byteLength: number } | CpuWeightBuffer,
67
+ label: string,
68
+ length: number,
69
+ config: WeightBufferConfig,
70
+ debugFlags?: WeightDebugFlags,
71
+ deviceOverride?: GPUDevice | null
72
+ ): { tensor: Tensor; owned: boolean };
@@ -3,7 +3,14 @@
3
3
  import { getDevice } from '../../../gpu/device.js';
4
4
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
5
5
  import { log } from '../../../debug/index.js';
6
- import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, tagBufferDtype } from '../../../gpu/weight-buffer.js';
6
+ import {
7
+ getBufferDtype,
8
+ isWeightBuffer,
9
+ isCpuWeightBuffer,
10
+ isGpuBufferInstance,
11
+ tagBufferDtype,
12
+ } from '../../../gpu/weight-buffer.js';
13
+ import { createTensor } from '../../../gpu/tensor.js';
7
14
 
8
15
  // ============================================================================
9
16
  // Type Guards
@@ -116,6 +123,31 @@ export function getNormWeightBuffer(weight, label, config, debugFlags, deviceOve
116
123
  }
117
124
  }
118
125
 
126
+ export function getVectorTensor(weight, label, length, config, debugFlags, deviceOverride = null) {
127
+ if (!Number.isInteger(length) || length <= 0) {
128
+ throw new Error(`Vector tensor "${label}" requires a positive integer length.`);
129
+ }
130
+ const buffer = getNormWeightBuffer(weight, label, config, debugFlags, deviceOverride);
131
+ let dtype = getBufferDtype(buffer);
132
+ if (dtype == null) {
133
+ const bytesPerElement = buffer.size / length;
134
+ if (bytesPerElement === 2) dtype = 'f16';
135
+ else if (bytesPerElement === 4) dtype = 'f32';
136
+ else {
137
+ throw new Error(
138
+ `Vector tensor "${label}" cannot infer dtype from ${buffer.size} bytes for ${length} elements.`
139
+ );
140
+ }
141
+ }
142
+ if (dtype !== 'f16' && dtype !== 'f32') {
143
+ throw new Error(`Vector tensor "${label}" has unsupported dtype "${String(dtype)}".`);
144
+ }
145
+ return {
146
+ tensor: createTensor(buffer, dtype, [length], label),
147
+ owned: !isGpuBufferInstance(weight) && !isWeightBuffer(weight),
148
+ };
149
+ }
150
+
119
151
 
120
152
  function getGPUWeightBuffer(weight, label) {
121
153
  // Handle WeightBuffer by extracting underlying GPUBuffer
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
15
15
  import { PipelineGenerator } from './text/generator.js';
16
16
  import type { Manifest } from './text/config.js';
17
17
  import type { WeightLoadResult, PipelineContexts } from './text/init.js';
18
- import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
18
+ import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
19
19
  import type { ChatMessage } from './text/chat-format.js';
20
20
  import type { LoRAAdapter } from './text/lora.js';
21
21
  import type { DiffusionPipeline } from './diffusion/pipeline.js';
@@ -31,7 +31,7 @@ import type {
31
31
  } from './text/generator.js';
32
32
 
33
33
  // Re-export types for external use
34
- export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
34
+ export type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, SequenceEncodeOptions, SequenceEncodeResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats };
35
35
  export type { PipelineContexts };
36
36
 
37
37
  export interface ChatRequestInput {
@@ -130,6 +130,8 @@ export declare class InferencePipeline extends PipelineState {
130
130
  embeddingMode: string;
131
131
  }>>;
132
132
 
133
+ encodeSequence(sequence: string, options?: SequenceEncodeOptions): Promise<SequenceEncodeResult>;
134
+
133
135
  prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
134
136
 
135
137
  prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
@@ -74,6 +74,47 @@ function assertNotAborted(signal) {
74
74
  throw new AbortError(typeof signal.reason === 'string' ? signal.reason : 'Doppler: aborted');
75
75
  }
76
76
  }
77
+
78
+ function poolSequenceTokenEmbeddings(tokenEmbeddings, tokenIds, hiddenSize, pooling) {
79
+ if (!(tokenEmbeddings instanceof Float32Array)) {
80
+ throw new Error('Sequence token embeddings are unavailable.');
81
+ }
82
+ if (tokenEmbeddings.length !== tokenIds.length * hiddenSize) {
83
+ throw new Error(
84
+ `Sequence embedding shape mismatch: expected ${tokenIds.length * hiddenSize}, got ${tokenEmbeddings.length}.`
85
+ );
86
+ }
87
+ const excluded = new Set(pooling.excludeTokenIds);
88
+ const includedIndices = [];
89
+ for (let index = 0; index < tokenIds.length; index += 1) {
90
+ if (!excluded.has(tokenIds[index])) includedIndices.push(index);
91
+ }
92
+ if (includedIndices.length === 0) {
93
+ throw new Error('Sequence pooling excluded every input token.');
94
+ }
95
+ const pooled = new Float32Array(hiddenSize);
96
+ if (pooling.mode === 'last') {
97
+ const tokenIndex = includedIndices[includedIndices.length - 1];
98
+ const offset = tokenIndex * hiddenSize;
99
+ pooled.set(tokenEmbeddings.subarray(offset, offset + hiddenSize));
100
+ } else {
101
+ for (const tokenIndex of includedIndices) {
102
+ const offset = tokenIndex * hiddenSize;
103
+ for (let column = 0; column < hiddenSize; column += 1) {
104
+ pooled[column] += tokenEmbeddings[offset + column];
105
+ }
106
+ }
107
+ const scale = 1 / includedIndices.length;
108
+ for (let column = 0; column < hiddenSize; column += 1) {
109
+ pooled[column] *= scale;
110
+ }
111
+ }
112
+ return {
113
+ pooled,
114
+ tokenMask: Uint8Array.from(tokenIds, (tokenId) => excluded.has(tokenId) ? 0 : 1),
115
+ includedTokenCount: includedIndices.length,
116
+ };
117
+ }
77
118
  import { initConvLayerState } from './text/ops.js';
78
119
  import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
79
120
 
@@ -672,7 +713,21 @@ export class InferencePipeline extends PipelineState {
672
713
  result.layerWeights.forEach((w, k) => this.weights.set(k, w));
673
714
  this.weights.set('embed', result.embeddings);
674
715
  this.weights.set('lm_head', result.lmHead);
716
+ this.weights.set('lm_head_bias', result.lmHeadBias);
717
+ if (result.lmHeadBias) {
718
+ // Recorded/fused logits currently keep their result on GPU, while the
719
+ // manifest-owned decoder bias is loaded as an exact CPU tensor. Route
720
+ // biased heads through computeLogits until the execution graph declares
721
+ // and records an explicit GPU bias-add step.
722
+ this.disableRecordedLogits = true;
723
+ this.disableFusedDecode = true;
724
+ }
675
725
  this.weights.set('final_norm', result.finalNorm);
726
+ this.weights.set('final_norm_bias', result.finalNormBias);
727
+ if (result.finalNormBias) {
728
+ this.disableRecordedLogits = true;
729
+ this.disableFusedDecode = true;
730
+ }
676
731
  this.weights.set('diffusion_gemma_self_conditioning', result.diffusionGemmaSelfConditioning);
677
732
  this.weights.set('per_layer_inputs', result.perLayerInputWeights);
678
733
  this.embeddingPostprocessor = result.embeddingPostprocessor;
@@ -1477,6 +1532,7 @@ export class InferencePipeline extends PipelineState {
1477
1532
  get capabilities() {
1478
1533
  const caps = ['generation'];
1479
1534
  if (typeof this.prefillWithEmbedding === 'function') caps.push('embedding');
1535
+ if (this.manifest?.inference?.supportsSequence === true) caps.push('sequence');
1480
1536
  if (this.visionCapable) caps.push('multimodal');
1481
1537
  if (this.audioCapable) caps.push('audio');
1482
1538
  if (this.visionCapable) caps.push('video');
@@ -1619,6 +1675,62 @@ export class InferencePipeline extends PipelineState {
1619
1675
  return outputs;
1620
1676
  }
1621
1677
 
1678
+ async encodeSequence(sequence, options = {}) {
1679
+ assertNotAborted(options?.signal);
1680
+ const contract = this.manifest?.inference?.sequence ?? null;
1681
+ if (this.manifest?.inference?.supportsSequence !== true || !contract) {
1682
+ throw new Error('Model manifest does not declare sequence encoding support.');
1683
+ }
1684
+ if (typeof sequence !== 'string' || sequence.length === 0) {
1685
+ throw new Error('encodeSequence expects a non-empty sequence string.');
1686
+ }
1687
+ const includeTokenEmbeddings = options.includeTokenEmbeddings
1688
+ ?? contract.tokenEmbeddings;
1689
+ const includeLogits = options.includeLogits === true;
1690
+ if (includeTokenEmbeddings && contract.tokenEmbeddings !== true) {
1691
+ throw new Error('Model manifest does not permit token-level sequence embeddings.');
1692
+ }
1693
+ if (includeLogits && contract.logits !== true) {
1694
+ throw new Error('Model manifest does not permit sequence logits.');
1695
+ }
1696
+ const needsTokenEmbeddings = includeTokenEmbeddings || contract.pooledEmbedding !== null;
1697
+
1698
+ this.resetForBatch();
1699
+ try {
1700
+ const result = await this.prefillWithEmbedding(sequence, {
1701
+ ...options,
1702
+ embeddingMode: contract.pooledEmbedding?.mode ?? 'last',
1703
+ __skipStateSnapshot: true,
1704
+ __returnTokenEmbeddings: needsTokenEmbeddings,
1705
+ __returnSequenceLogits: includeLogits,
1706
+ });
1707
+ assertNotAborted(options?.signal);
1708
+ const hiddenSize = this.modelConfig.hiddenSize;
1709
+ const pooledResult = contract.pooledEmbedding
1710
+ ? poolSequenceTokenEmbeddings(
1711
+ result.tokenEmbeddings,
1712
+ result.tokens,
1713
+ hiddenSize,
1714
+ contract.pooledEmbedding
1715
+ )
1716
+ : null;
1717
+ return {
1718
+ alphabet: contract.alphabet,
1719
+ tokens: result.tokens,
1720
+ tokenMask: pooledResult?.tokenMask ?? new Uint8Array(result.tokens.length),
1721
+ includedTokenCount: pooledResult?.includedTokenCount ?? 0,
1722
+ tokenEmbeddings: includeTokenEmbeddings ? result.tokenEmbeddings : null,
1723
+ pooledEmbedding: pooledResult?.pooled ?? null,
1724
+ logits: includeLogits ? result.logits : null,
1725
+ embeddingDim: hiddenSize,
1726
+ vocabSize: this.modelConfig.vocabSize,
1727
+ phase: result.phase ?? null,
1728
+ };
1729
+ } finally {
1730
+ this.resetForBatch();
1731
+ }
1732
+ }
1733
+
1622
1734
  // Run the vision encoder over a single image and return a mean-pooled
1623
1735
  // embedding in the model's text-hidden-size space. Decoder responsibility
1624
1736
  // (jpeg/png -> RGBA pixels) belongs to the caller; this method takes
@@ -7,7 +7,11 @@ function normalizeSpecialTokenLabel(value) {
7
7
 
8
8
  function isBosLikeLabel(value) {
9
9
  const normalized = normalizeSpecialTokenLabel(value);
10
- return normalized === '<bos>' || normalized === '<s>' || normalized.includes('bos');
10
+ return normalized === '<bos>'
11
+ || normalized === '<s>'
12
+ || normalized === '<cls>'
13
+ || normalized === 'cls'
14
+ || normalized.includes('bos');
11
15
  }
12
16
 
13
17
  function isEosLikeLabel(value) {
@@ -25,7 +25,7 @@ function resolveTokenId(value, vocab, label) {
25
25
  return null;
26
26
  }
27
27
 
28
- function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab) {
28
+ function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab, options = {}) {
29
29
  const padCandidate = pickCandidate(
30
30
  specialTokensRaw?.pad,
31
31
  specialTokensRaw?.pad_token,
@@ -58,7 +58,7 @@ function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab) {
58
58
  unk: resolveTokenId(unkCandidate, vocab, 'unk'),
59
59
  };
60
60
 
61
- if (resolved.eos == null) {
61
+ if (resolved.eos == null && options.allowMissingEos !== true) {
62
62
  throw new Error('[Tokenizer] Missing EOS token in tokenizer.json and runtime config.');
63
63
  }
64
64
 
@@ -156,6 +156,17 @@ function resolveIsolatedSplitPretokenizer(preTokenizer) {
156
156
  return { source: normalized.source, flags };
157
157
  }
158
158
 
159
+ function resolvesCharacterSplitPretokenizer(preTokenizer) {
160
+ if (!preTokenizer || typeof preTokenizer !== 'object') return false;
161
+ if (preTokenizer.type === 'Sequence' && Array.isArray(preTokenizer.pretokenizers)) {
162
+ return preTokenizer.pretokenizers.some(resolvesCharacterSplitPretokenizer);
163
+ }
164
+ return preTokenizer.type === 'Split'
165
+ && preTokenizer.behavior === 'Removed'
166
+ && preTokenizer.invert !== true
167
+ && preTokenizer.pattern?.String === '';
168
+ }
169
+
159
170
  function hexNibble(code) {
160
171
  if (code >= 48 && code <= 57) return code - 48;
161
172
  if (code >= 65 && code <= 70) return code - 55;
@@ -333,7 +344,10 @@ function registerAddedTokens(addedTokens, vocab, reverseVocab, patterns, special
333
344
  if (token.special) {
334
345
  specialTokenIds.add(id);
335
346
  if (derivedSpecialTokens) {
336
- if (derivedSpecialTokens.bos == null && (content === '<bos>' || content === '<s>' || content.includes('bos'))) {
347
+ if (
348
+ derivedSpecialTokens.bos == null
349
+ && (content === '<bos>' || content === '<s>' || content === '<cls>' || content.includes('bos'))
350
+ ) {
337
351
  derivedSpecialTokens.bos = id;
338
352
  } else if (derivedSpecialTokens.eos == null && (content === '<eos>' || content === '</s>' || content.includes('eos'))) {
339
353
  derivedSpecialTokens.eos = id;
@@ -451,6 +465,12 @@ export class BundledTokenizer extends BaseTokenizer {
451
465
 
452
466
  #splitPretokenizer = null;
453
467
 
468
+ #splitEveryCharacter = false;
469
+
470
+ #wordPiecePrefix = '##';
471
+
472
+ #wordPieceMaxChars = 100;
473
+
454
474
 
455
475
  constructor(config = {}) {
456
476
  // BundledTokenizer gets vocabSize from load(), so defer validation
@@ -476,6 +496,9 @@ export class BundledTokenizer extends BaseTokenizer {
476
496
  this.#byteEncoder = null;
477
497
  this.#useByteLevelEncoding = false;
478
498
  this.#splitPretokenizer = null;
499
+ this.#splitEveryCharacter = false;
500
+ this.#wordPiecePrefix = '##';
501
+ this.#wordPieceMaxChars = 100;
479
502
  this.vocabSize = 0;
480
503
  }
481
504
 
@@ -550,7 +573,7 @@ export class BundledTokenizer extends BaseTokenizer {
550
573
  throw new Error('[Tokenizer] Missing model.type in HuggingFace tokenizer JSON.');
551
574
  }
552
575
  this.#type = model.type.toLowerCase();
553
- if (this.#type !== 'bpe' && this.#type !== 'unigram') {
576
+ if (this.#type !== 'bpe' && this.#type !== 'unigram' && this.#type !== 'wordpiece') {
554
577
  throw new Error(`[Tokenizer] Unsupported tokenizer type: ${model.type}`);
555
578
  }
556
579
  log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
@@ -566,8 +589,12 @@ export class BundledTokenizer extends BaseTokenizer {
566
589
  this.#scores.push(score);
567
590
  if (i > maxId) maxId = i;
568
591
  }
569
- } else if (this.#type === 'bpe' && model.vocab && typeof model.vocab === 'object') {
570
- // BPE format: { token: id }
592
+ } else if (
593
+ (this.#type === 'bpe' || this.#type === 'wordpiece')
594
+ && model.vocab
595
+ && typeof model.vocab === 'object'
596
+ ) {
597
+ // BPE and WordPiece format: { token: id }
571
598
  maxId = loadObjectVocab(model.vocab, this.#vocab, this.#reverseVocab, this.#byteTokens);
572
599
  } else {
573
600
  throw new Error(`[Tokenizer] Missing vocab for tokenizer type: ${model.type}`);
@@ -610,7 +637,9 @@ export class BundledTokenizer extends BaseTokenizer {
610
637
  eos: model.eos_id ?? this.specialTokens.eos ?? derivedSpecialTokens.eos,
611
638
  unk: model.unk_id ?? this.specialTokens.unk ?? derivedSpecialTokens.unk,
612
639
  };
613
- this.specialTokens = resolveSpecialTokens(specialTokensRaw, fallbackTokens, this.#vocab);
640
+ this.specialTokens = resolveSpecialTokens(specialTokensRaw, fallbackTokens, this.#vocab, {
641
+ allowMissingEos: hf.add_eos_token === false || hf.addEosToken === false,
642
+ });
614
643
  this.#specialTokenIds = specialTokenIds;
615
644
  this.#specialTokenPatterns = specialTokenPatterns;
616
645
  const builtinSpecials = [
@@ -639,6 +668,16 @@ export class BundledTokenizer extends BaseTokenizer {
639
668
  const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
640
669
  const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
641
670
  this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(hf.pre_tokenizer);
671
+ this.#splitEveryCharacter = resolvesCharacterSplitPretokenizer(hf.pre_tokenizer);
672
+ if (this.#type === 'wordpiece') {
673
+ this.#wordPiecePrefix = typeof model.continuing_subword_prefix === 'string'
674
+ ? model.continuing_subword_prefix
675
+ : '##';
676
+ this.#wordPieceMaxChars = Number.isInteger(model.max_input_chars_per_word)
677
+ && model.max_input_chars_per_word > 0
678
+ ? model.max_input_chars_per_word
679
+ : 100;
680
+ }
642
681
  const configuredAddBosToken = this.addBosToken;
643
682
  const configuredAddEosToken = this.addEosToken;
644
683
  const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
@@ -783,7 +822,10 @@ export class BundledTokenizer extends BaseTokenizer {
783
822
  ...derivedSpecialTokens,
784
823
  ...this.specialTokens,
785
824
  },
786
- this.#vocab
825
+ this.#vocab,
826
+ {
827
+ allowMissingEos: tokenizerJson.add_eos_token === false || tokenizerJson.addEosToken === false,
828
+ }
787
829
  );
788
830
  log.debug('Tokenizer', `Special tokens: BOS=${this.specialTokens.bos}, EOS=${this.specialTokens.eos}`);
789
831
  this.#specialTokenIds = specialTokenIds;
@@ -885,6 +927,8 @@ export class BundledTokenizer extends BaseTokenizer {
885
927
  } else if (seg.text && seg.text.length > 0) {
886
928
  if (this.#type === 'unigram') {
887
929
  ids.push(...this.#encodeUnigram(seg.text));
930
+ } else if (this.#type === 'wordpiece') {
931
+ ids.push(...this.#encodeWordPiece(seg.text));
888
932
  } else {
889
933
  ids.push(...this.#encodeBPE(seg.text));
890
934
  }
@@ -1053,6 +1097,42 @@ export class BundledTokenizer extends BaseTokenizer {
1053
1097
  return ids;
1054
1098
  }
1055
1099
 
1100
+ #encodeWordPiece(text) {
1101
+ if (text.length === 0) return [];
1102
+ const words = this.#splitEveryCharacter
1103
+ ? Array.from(text)
1104
+ : text.match(/[\p{L}\p{M}\p{N}]+|[^\s\p{L}\p{M}\p{N}]/gu) ?? [];
1105
+ const ids = [];
1106
+ for (const word of words) {
1107
+ if (/^\s+$/u.test(word)) continue;
1108
+ if (word.length > this.#wordPieceMaxChars) {
1109
+ ids.push(this.#getUnkTokenId());
1110
+ continue;
1111
+ }
1112
+ let start = 0;
1113
+ const wordIds = [];
1114
+ let failed = false;
1115
+ while (start < word.length) {
1116
+ let end = word.length;
1117
+ let tokenId;
1118
+ while (start < end) {
1119
+ const piece = `${start > 0 ? this.#wordPiecePrefix : ''}${word.slice(start, end)}`;
1120
+ tokenId = this.#vocab.get(piece);
1121
+ if (tokenId !== undefined) break;
1122
+ end -= 1;
1123
+ }
1124
+ if (tokenId === undefined) {
1125
+ failed = true;
1126
+ break;
1127
+ }
1128
+ wordIds.push(tokenId);
1129
+ start = end;
1130
+ }
1131
+ ids.push(...(failed ? [this.#getUnkTokenId()] : wordIds));
1132
+ }
1133
+ return ids;
1134
+ }
1135
+
1056
1136
  #splitBpePretokens(text) {
1057
1137
  if (!this.#useByteLevelEncoding || !this.#splitPretokenizer) return [text];
1058
1138
  const regex = new RegExp(this.#splitPretokenizer.source, this.#splitPretokenizer.flags);
@@ -1196,6 +1276,17 @@ export class BundledTokenizer extends BaseTokenizer {
1196
1276
  result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
1197
1277
  // SentencePiece-style markers can still appear in some mixed vocabularies.
1198
1278
  result = result.replace(/▁/g, ' ');
1279
+ } else if (this.#type === 'wordpiece') {
1280
+ if (this.#splitEveryCharacter) {
1281
+ result = tokens.join('');
1282
+ } else {
1283
+ result = tokens.reduce((text, token) => {
1284
+ if (token.startsWith(this.#wordPiecePrefix)) {
1285
+ return `${text}${token.slice(this.#wordPiecePrefix.length)}`;
1286
+ }
1287
+ return text ? `${text} ${token}` : token;
1288
+ }, '');
1289
+ }
1199
1290
  } else {
1200
1291
  // Join and convert ▁ back to spaces, handle GPT-style markers
1201
1292
  result = decodeByteFallbackTokens(tokens)
@@ -75,7 +75,9 @@ export declare class DopplerLoader {
75
75
  layers: Map<number, LayerWeights>;
76
76
  experts: Map<string, ExpertWeights>;
77
77
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
78
+ lmHeadBias: Float32Array | null;
78
79
  finalNorm: GPUBuffer | Float32Array | null;
80
+ finalNormBias: Float32Array | null;
79
81
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
80
82
  diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
81
83
  perLayerInputWeights: PerLayerInputWeights | null;
@@ -224,6 +224,9 @@ export class DopplerLoader {
224
224
  experts = new Map();
225
225
 
226
226
  lmHead = null;
227
+
228
+ lmHeadBias = null;
229
+ finalNormBias = null;
227
230
 
228
231
  finalNorm = null;
229
232
 
@@ -1398,6 +1401,8 @@ export class DopplerLoader {
1398
1401
  resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
1399
1402
  embeddings: this.embeddings,
1400
1403
  embeddingPostprocessor: this.manifest?.inference?.output?.embeddingPostprocessor ?? null,
1404
+ finalNormBiasTensor: this.manifest?.inference?.normalization?.finalNormBiasTensor ?? null,
1405
+ lmHeadBiasTensor: this.manifest?.inference?.output?.lmHeadBiasTensor ?? null,
1401
1406
  diffusionGemmaSelfConditioning: this.manifest?.inference?.diffusionGemma?.selfConditioning === true,
1402
1407
  modelType: this.manifest?.modelType ?? null,
1403
1408
  tieWordEmbeddings,
@@ -1408,7 +1413,9 @@ export class DopplerLoader {
1408
1413
 
1409
1414
  const result = await loadFinalWeights(ctx);
1410
1415
  this.finalNorm = result.finalNorm;
1416
+ this.finalNormBias = result.finalNormBias;
1411
1417
  this.lmHead = result.lmHead;
1418
+ this.lmHeadBias = result.lmHeadBias;
1412
1419
  this.embeddingPostprocessor = result.embeddingPostprocessor;
1413
1420
  this.diffusionGemmaSelfConditioning = result.diffusionGemmaSelfConditioning;
1414
1421
  this.#normOffsetDebugLogged = result.normOffsetDebugLogged;
@@ -1525,6 +1532,8 @@ export class DopplerLoader {
1525
1532
  this.layers.clear();
1526
1533
  this.experts.clear();
1527
1534
  this.lmHead = null;
1535
+ this.lmHeadBias = null;
1536
+ this.finalNormBias = null;
1528
1537
  this.finalNorm = null;
1529
1538
  this.embeddingPostprocessor = null;
1530
1539
  this.perLayerInputWeights = null;
@@ -56,6 +56,10 @@ export interface FinalWeightsContext {
56
56
  embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
57
57
  /** Optional embedding postprocessor declared by manifest */
58
58
  embeddingPostprocessor?: ManifestEmbeddingPostprocessorSchema | null;
59
+ /** Optional exact tensor name for an additive LM-head bias. */
60
+ lmHeadBiasTensor?: string | null;
61
+ /** Optional exact tensor name for a final LayerNorm bias. */
62
+ finalNormBiasTensor?: string | null;
59
63
  /** Manifest-owned DiffusionGemma self-conditioning enablement */
60
64
  diffusionGemmaSelfConditioning?: boolean;
61
65
  /** Manifest model type */
@@ -74,8 +78,12 @@ export interface FinalWeightsContext {
74
78
  export interface FinalWeightsResult {
75
79
  /** Final layer norm tensor */
76
80
  finalNorm: GPUBuffer | Float32Array | null;
81
+ /** Optional final LayerNorm bias, held on CPU for embedding extraction. */
82
+ finalNormBias: Float32Array | null;
77
83
  /** LM head tensor */
78
84
  lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
85
+ /** Optional additive LM-head bias, held on CPU for exact post-projection application. */
86
+ lmHeadBias: Float32Array | null;
79
87
  /** Optional embedding-only postprocessor weights */
80
88
  embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
81
89
  /** Optional DiffusionGemma decoder self-conditioning weights */
@@ -245,9 +245,11 @@ export async function loadFinalWeights(ctx) {
245
245
  if (normDebugLogged) {
246
246
  normOffsetDebugLogged = true;
247
247
  }
248
+ const finalNormBias = await loadFinalNormBias(ctx);
248
249
 
249
250
  // Load LM head
250
251
  const lmHead = await loadLmHead(ctx);
252
+ const lmHeadBias = await loadLmHeadBias(ctx);
251
253
  const embeddingPostprocessor = await loadEmbeddingPostprocessor(ctx);
252
254
  const diffusionGemmaSelfConditioning = shouldLoadDiffusionGemmaSelfConditioning(ctx)
253
255
  ? await loadDiffusionGemmaSelfConditioning(ctx)
@@ -255,13 +257,34 @@ export async function loadFinalWeights(ctx) {
255
257
 
256
258
  return {
257
259
  finalNorm,
260
+ finalNormBias,
258
261
  lmHead,
262
+ lmHeadBias,
259
263
  embeddingPostprocessor,
260
264
  diffusionGemmaSelfConditioning,
261
265
  normOffsetDebugLogged,
262
266
  };
263
267
  }
264
268
 
269
+ async function loadFinalNormBias(ctx) {
270
+ const tensorName = typeof ctx.finalNormBiasTensor === 'string'
271
+ ? ctx.finalNormBiasTensor.trim()
272
+ : '';
273
+ if (!tensorName) {
274
+ return null;
275
+ }
276
+ const location = ctx.tensorLocations.get(tensorName);
277
+ if (!location) {
278
+ throw new Error(`[Loader] Final norm bias tensor "${tensorName}" is missing from the manifest.`);
279
+ }
280
+ if (!Array.isArray(location.shape) || location.shape.length !== 1) {
281
+ throw new Error(
282
+ `[Loader] Final norm bias tensor "${tensorName}" must be rank 1, got ${JSON.stringify(location.shape)}.`
283
+ );
284
+ }
285
+ return loadCpuFloatTensor(ctx, tensorName, `final norm bias "${tensorName}"`);
286
+ }
287
+
265
288
  // ============================================================================
266
289
  // Final Norm Loading
267
290
  // ============================================================================
@@ -398,6 +421,25 @@ async function loadLmHead(ctx) {
398
421
  return lmHead;
399
422
  }
400
423
 
424
+ async function loadLmHeadBias(ctx) {
425
+ const tensorName = typeof ctx.lmHeadBiasTensor === 'string'
426
+ ? ctx.lmHeadBiasTensor.trim()
427
+ : '';
428
+ if (!tensorName) {
429
+ return null;
430
+ }
431
+ const location = ctx.tensorLocations.get(tensorName);
432
+ if (!location) {
433
+ throw new Error(`[Loader] LM head bias tensor "${tensorName}" is missing from the manifest.`);
434
+ }
435
+ if (!Array.isArray(location.shape) || location.shape.length !== 1) {
436
+ throw new Error(
437
+ `[Loader] LM head bias tensor "${tensorName}" must be rank 1, got ${JSON.stringify(location.shape)}.`
438
+ );
439
+ }
440
+ return loadCpuFloatTensor(ctx, tensorName, `LM head bias "${tensorName}"`);
441
+ }
442
+
401
443
  async function loadEmbeddingPostprocessor(ctx) {
402
444
  const config = ctx.embeddingPostprocessor;
403
445
  if (config == null) {