doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
|
@@ -74,6 +74,47 @@ function assertNotAborted(signal) {
|
|
|
74
74
|
throw new AbortError(typeof signal.reason === 'string' ? signal.reason : 'Doppler: aborted');
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
|
+
|
|
78
|
+
function poolSequenceTokenEmbeddings(tokenEmbeddings, tokenIds, hiddenSize, pooling) {
|
|
79
|
+
if (!(tokenEmbeddings instanceof Float32Array)) {
|
|
80
|
+
throw new Error('Sequence token embeddings are unavailable.');
|
|
81
|
+
}
|
|
82
|
+
if (tokenEmbeddings.length !== tokenIds.length * hiddenSize) {
|
|
83
|
+
throw new Error(
|
|
84
|
+
`Sequence embedding shape mismatch: expected ${tokenIds.length * hiddenSize}, got ${tokenEmbeddings.length}.`
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
const excluded = new Set(pooling.excludeTokenIds);
|
|
88
|
+
const includedIndices = [];
|
|
89
|
+
for (let index = 0; index < tokenIds.length; index += 1) {
|
|
90
|
+
if (!excluded.has(tokenIds[index])) includedIndices.push(index);
|
|
91
|
+
}
|
|
92
|
+
if (includedIndices.length === 0) {
|
|
93
|
+
throw new Error('Sequence pooling excluded every input token.');
|
|
94
|
+
}
|
|
95
|
+
const pooled = new Float32Array(hiddenSize);
|
|
96
|
+
if (pooling.mode === 'last') {
|
|
97
|
+
const tokenIndex = includedIndices[includedIndices.length - 1];
|
|
98
|
+
const offset = tokenIndex * hiddenSize;
|
|
99
|
+
pooled.set(tokenEmbeddings.subarray(offset, offset + hiddenSize));
|
|
100
|
+
} else {
|
|
101
|
+
for (const tokenIndex of includedIndices) {
|
|
102
|
+
const offset = tokenIndex * hiddenSize;
|
|
103
|
+
for (let column = 0; column < hiddenSize; column += 1) {
|
|
104
|
+
pooled[column] += tokenEmbeddings[offset + column];
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const scale = 1 / includedIndices.length;
|
|
108
|
+
for (let column = 0; column < hiddenSize; column += 1) {
|
|
109
|
+
pooled[column] *= scale;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
pooled,
|
|
114
|
+
tokenMask: Uint8Array.from(tokenIds, (tokenId) => excluded.has(tokenId) ? 0 : 1),
|
|
115
|
+
includedTokenCount: includedIndices.length,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
77
118
|
import { initConvLayerState } from './text/ops.js';
|
|
78
119
|
import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
|
|
79
120
|
|
|
@@ -672,7 +713,21 @@ export class InferencePipeline extends PipelineState {
|
|
|
672
713
|
result.layerWeights.forEach((w, k) => this.weights.set(k, w));
|
|
673
714
|
this.weights.set('embed', result.embeddings);
|
|
674
715
|
this.weights.set('lm_head', result.lmHead);
|
|
716
|
+
this.weights.set('lm_head_bias', result.lmHeadBias);
|
|
717
|
+
if (result.lmHeadBias) {
|
|
718
|
+
// Recorded/fused logits currently keep their result on GPU, while the
|
|
719
|
+
// manifest-owned decoder bias is loaded as an exact CPU tensor. Route
|
|
720
|
+
// biased heads through computeLogits until the execution graph declares
|
|
721
|
+
// and records an explicit GPU bias-add step.
|
|
722
|
+
this.disableRecordedLogits = true;
|
|
723
|
+
this.disableFusedDecode = true;
|
|
724
|
+
}
|
|
675
725
|
this.weights.set('final_norm', result.finalNorm);
|
|
726
|
+
this.weights.set('final_norm_bias', result.finalNormBias);
|
|
727
|
+
if (result.finalNormBias) {
|
|
728
|
+
this.disableRecordedLogits = true;
|
|
729
|
+
this.disableFusedDecode = true;
|
|
730
|
+
}
|
|
676
731
|
this.weights.set('diffusion_gemma_self_conditioning', result.diffusionGemmaSelfConditioning);
|
|
677
732
|
this.weights.set('per_layer_inputs', result.perLayerInputWeights);
|
|
678
733
|
this.embeddingPostprocessor = result.embeddingPostprocessor;
|
|
@@ -1477,6 +1532,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1477
1532
|
get capabilities() {
|
|
1478
1533
|
const caps = ['generation'];
|
|
1479
1534
|
if (typeof this.prefillWithEmbedding === 'function') caps.push('embedding');
|
|
1535
|
+
if (this.manifest?.inference?.supportsSequence === true) caps.push('sequence');
|
|
1480
1536
|
if (this.visionCapable) caps.push('multimodal');
|
|
1481
1537
|
if (this.audioCapable) caps.push('audio');
|
|
1482
1538
|
if (this.visionCapable) caps.push('video');
|
|
@@ -1619,6 +1675,62 @@ export class InferencePipeline extends PipelineState {
|
|
|
1619
1675
|
return outputs;
|
|
1620
1676
|
}
|
|
1621
1677
|
|
|
1678
|
+
async encodeSequence(sequence, options = {}) {
|
|
1679
|
+
assertNotAborted(options?.signal);
|
|
1680
|
+
const contract = this.manifest?.inference?.sequence ?? null;
|
|
1681
|
+
if (this.manifest?.inference?.supportsSequence !== true || !contract) {
|
|
1682
|
+
throw new Error('Model manifest does not declare sequence encoding support.');
|
|
1683
|
+
}
|
|
1684
|
+
if (typeof sequence !== 'string' || sequence.length === 0) {
|
|
1685
|
+
throw new Error('encodeSequence expects a non-empty sequence string.');
|
|
1686
|
+
}
|
|
1687
|
+
const includeTokenEmbeddings = options.includeTokenEmbeddings
|
|
1688
|
+
?? contract.tokenEmbeddings;
|
|
1689
|
+
const includeLogits = options.includeLogits === true;
|
|
1690
|
+
if (includeTokenEmbeddings && contract.tokenEmbeddings !== true) {
|
|
1691
|
+
throw new Error('Model manifest does not permit token-level sequence embeddings.');
|
|
1692
|
+
}
|
|
1693
|
+
if (includeLogits && contract.logits !== true) {
|
|
1694
|
+
throw new Error('Model manifest does not permit sequence logits.');
|
|
1695
|
+
}
|
|
1696
|
+
const needsTokenEmbeddings = includeTokenEmbeddings || contract.pooledEmbedding !== null;
|
|
1697
|
+
|
|
1698
|
+
this.resetForBatch();
|
|
1699
|
+
try {
|
|
1700
|
+
const result = await this.prefillWithEmbedding(sequence, {
|
|
1701
|
+
...options,
|
|
1702
|
+
embeddingMode: contract.pooledEmbedding?.mode ?? 'last',
|
|
1703
|
+
__skipStateSnapshot: true,
|
|
1704
|
+
__returnTokenEmbeddings: needsTokenEmbeddings,
|
|
1705
|
+
__returnSequenceLogits: includeLogits,
|
|
1706
|
+
});
|
|
1707
|
+
assertNotAborted(options?.signal);
|
|
1708
|
+
const hiddenSize = this.modelConfig.hiddenSize;
|
|
1709
|
+
const pooledResult = contract.pooledEmbedding
|
|
1710
|
+
? poolSequenceTokenEmbeddings(
|
|
1711
|
+
result.tokenEmbeddings,
|
|
1712
|
+
result.tokens,
|
|
1713
|
+
hiddenSize,
|
|
1714
|
+
contract.pooledEmbedding
|
|
1715
|
+
)
|
|
1716
|
+
: null;
|
|
1717
|
+
return {
|
|
1718
|
+
alphabet: contract.alphabet,
|
|
1719
|
+
tokens: result.tokens,
|
|
1720
|
+
tokenMask: pooledResult?.tokenMask ?? new Uint8Array(result.tokens.length),
|
|
1721
|
+
includedTokenCount: pooledResult?.includedTokenCount ?? 0,
|
|
1722
|
+
tokenEmbeddings: includeTokenEmbeddings ? result.tokenEmbeddings : null,
|
|
1723
|
+
pooledEmbedding: pooledResult?.pooled ?? null,
|
|
1724
|
+
logits: includeLogits ? result.logits : null,
|
|
1725
|
+
embeddingDim: hiddenSize,
|
|
1726
|
+
vocabSize: this.modelConfig.vocabSize,
|
|
1727
|
+
phase: result.phase ?? null,
|
|
1728
|
+
};
|
|
1729
|
+
} finally {
|
|
1730
|
+
this.resetForBatch();
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1622
1734
|
// Run the vision encoder over a single image and return a mean-pooled
|
|
1623
1735
|
// embedding in the model's text-hidden-size space. Decoder responsibility
|
|
1624
1736
|
// (jpeg/png -> RGBA pixels) belongs to the caller; this method takes
|
|
@@ -7,7 +7,11 @@ function normalizeSpecialTokenLabel(value) {
|
|
|
7
7
|
|
|
8
8
|
function isBosLikeLabel(value) {
|
|
9
9
|
const normalized = normalizeSpecialTokenLabel(value);
|
|
10
|
-
return normalized === '<bos>'
|
|
10
|
+
return normalized === '<bos>'
|
|
11
|
+
|| normalized === '<s>'
|
|
12
|
+
|| normalized === '<cls>'
|
|
13
|
+
|| normalized === 'cls'
|
|
14
|
+
|| normalized.includes('bos');
|
|
11
15
|
}
|
|
12
16
|
|
|
13
17
|
function isEosLikeLabel(value) {
|
|
@@ -25,7 +25,7 @@ function resolveTokenId(value, vocab, label) {
|
|
|
25
25
|
return null;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab) {
|
|
28
|
+
function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab, options = {}) {
|
|
29
29
|
const padCandidate = pickCandidate(
|
|
30
30
|
specialTokensRaw?.pad,
|
|
31
31
|
specialTokensRaw?.pad_token,
|
|
@@ -58,7 +58,7 @@ function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab) {
|
|
|
58
58
|
unk: resolveTokenId(unkCandidate, vocab, 'unk'),
|
|
59
59
|
};
|
|
60
60
|
|
|
61
|
-
if (resolved.eos == null) {
|
|
61
|
+
if (resolved.eos == null && options.allowMissingEos !== true) {
|
|
62
62
|
throw new Error('[Tokenizer] Missing EOS token in tokenizer.json and runtime config.');
|
|
63
63
|
}
|
|
64
64
|
|
|
@@ -156,6 +156,17 @@ function resolveIsolatedSplitPretokenizer(preTokenizer) {
|
|
|
156
156
|
return { source: normalized.source, flags };
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
function resolvesCharacterSplitPretokenizer(preTokenizer) {
|
|
160
|
+
if (!preTokenizer || typeof preTokenizer !== 'object') return false;
|
|
161
|
+
if (preTokenizer.type === 'Sequence' && Array.isArray(preTokenizer.pretokenizers)) {
|
|
162
|
+
return preTokenizer.pretokenizers.some(resolvesCharacterSplitPretokenizer);
|
|
163
|
+
}
|
|
164
|
+
return preTokenizer.type === 'Split'
|
|
165
|
+
&& preTokenizer.behavior === 'Removed'
|
|
166
|
+
&& preTokenizer.invert !== true
|
|
167
|
+
&& preTokenizer.pattern?.String === '';
|
|
168
|
+
}
|
|
169
|
+
|
|
159
170
|
function hexNibble(code) {
|
|
160
171
|
if (code >= 48 && code <= 57) return code - 48;
|
|
161
172
|
if (code >= 65 && code <= 70) return code - 55;
|
|
@@ -333,7 +344,10 @@ function registerAddedTokens(addedTokens, vocab, reverseVocab, patterns, special
|
|
|
333
344
|
if (token.special) {
|
|
334
345
|
specialTokenIds.add(id);
|
|
335
346
|
if (derivedSpecialTokens) {
|
|
336
|
-
if (
|
|
347
|
+
if (
|
|
348
|
+
derivedSpecialTokens.bos == null
|
|
349
|
+
&& (content === '<bos>' || content === '<s>' || content === '<cls>' || content.includes('bos'))
|
|
350
|
+
) {
|
|
337
351
|
derivedSpecialTokens.bos = id;
|
|
338
352
|
} else if (derivedSpecialTokens.eos == null && (content === '<eos>' || content === '</s>' || content.includes('eos'))) {
|
|
339
353
|
derivedSpecialTokens.eos = id;
|
|
@@ -451,6 +465,12 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
451
465
|
|
|
452
466
|
#splitPretokenizer = null;
|
|
453
467
|
|
|
468
|
+
#splitEveryCharacter = false;
|
|
469
|
+
|
|
470
|
+
#wordPiecePrefix = '##';
|
|
471
|
+
|
|
472
|
+
#wordPieceMaxChars = 100;
|
|
473
|
+
|
|
454
474
|
|
|
455
475
|
constructor(config = {}) {
|
|
456
476
|
// BundledTokenizer gets vocabSize from load(), so defer validation
|
|
@@ -476,6 +496,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
476
496
|
this.#byteEncoder = null;
|
|
477
497
|
this.#useByteLevelEncoding = false;
|
|
478
498
|
this.#splitPretokenizer = null;
|
|
499
|
+
this.#splitEveryCharacter = false;
|
|
500
|
+
this.#wordPiecePrefix = '##';
|
|
501
|
+
this.#wordPieceMaxChars = 100;
|
|
479
502
|
this.vocabSize = 0;
|
|
480
503
|
}
|
|
481
504
|
|
|
@@ -550,7 +573,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
550
573
|
throw new Error('[Tokenizer] Missing model.type in HuggingFace tokenizer JSON.');
|
|
551
574
|
}
|
|
552
575
|
this.#type = model.type.toLowerCase();
|
|
553
|
-
if (this.#type !== 'bpe' && this.#type !== 'unigram') {
|
|
576
|
+
if (this.#type !== 'bpe' && this.#type !== 'unigram' && this.#type !== 'wordpiece') {
|
|
554
577
|
throw new Error(`[Tokenizer] Unsupported tokenizer type: ${model.type}`);
|
|
555
578
|
}
|
|
556
579
|
log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
|
|
@@ -566,8 +589,12 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
566
589
|
this.#scores.push(score);
|
|
567
590
|
if (i > maxId) maxId = i;
|
|
568
591
|
}
|
|
569
|
-
} else if (
|
|
570
|
-
|
|
592
|
+
} else if (
|
|
593
|
+
(this.#type === 'bpe' || this.#type === 'wordpiece')
|
|
594
|
+
&& model.vocab
|
|
595
|
+
&& typeof model.vocab === 'object'
|
|
596
|
+
) {
|
|
597
|
+
// BPE and WordPiece format: { token: id }
|
|
571
598
|
maxId = loadObjectVocab(model.vocab, this.#vocab, this.#reverseVocab, this.#byteTokens);
|
|
572
599
|
} else {
|
|
573
600
|
throw new Error(`[Tokenizer] Missing vocab for tokenizer type: ${model.type}`);
|
|
@@ -610,7 +637,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
610
637
|
eos: model.eos_id ?? this.specialTokens.eos ?? derivedSpecialTokens.eos,
|
|
611
638
|
unk: model.unk_id ?? this.specialTokens.unk ?? derivedSpecialTokens.unk,
|
|
612
639
|
};
|
|
613
|
-
this.specialTokens = resolveSpecialTokens(specialTokensRaw, fallbackTokens, this.#vocab
|
|
640
|
+
this.specialTokens = resolveSpecialTokens(specialTokensRaw, fallbackTokens, this.#vocab, {
|
|
641
|
+
allowMissingEos: hf.add_eos_token === false || hf.addEosToken === false,
|
|
642
|
+
});
|
|
614
643
|
this.#specialTokenIds = specialTokenIds;
|
|
615
644
|
this.#specialTokenPatterns = specialTokenPatterns;
|
|
616
645
|
const builtinSpecials = [
|
|
@@ -639,6 +668,16 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
639
668
|
const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
|
|
640
669
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
|
|
641
670
|
this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(hf.pre_tokenizer);
|
|
671
|
+
this.#splitEveryCharacter = resolvesCharacterSplitPretokenizer(hf.pre_tokenizer);
|
|
672
|
+
if (this.#type === 'wordpiece') {
|
|
673
|
+
this.#wordPiecePrefix = typeof model.continuing_subword_prefix === 'string'
|
|
674
|
+
? model.continuing_subword_prefix
|
|
675
|
+
: '##';
|
|
676
|
+
this.#wordPieceMaxChars = Number.isInteger(model.max_input_chars_per_word)
|
|
677
|
+
&& model.max_input_chars_per_word > 0
|
|
678
|
+
? model.max_input_chars_per_word
|
|
679
|
+
: 100;
|
|
680
|
+
}
|
|
642
681
|
const configuredAddBosToken = this.addBosToken;
|
|
643
682
|
const configuredAddEosToken = this.addEosToken;
|
|
644
683
|
const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
|
|
@@ -783,7 +822,10 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
783
822
|
...derivedSpecialTokens,
|
|
784
823
|
...this.specialTokens,
|
|
785
824
|
},
|
|
786
|
-
this.#vocab
|
|
825
|
+
this.#vocab,
|
|
826
|
+
{
|
|
827
|
+
allowMissingEos: tokenizerJson.add_eos_token === false || tokenizerJson.addEosToken === false,
|
|
828
|
+
}
|
|
787
829
|
);
|
|
788
830
|
log.debug('Tokenizer', `Special tokens: BOS=${this.specialTokens.bos}, EOS=${this.specialTokens.eos}`);
|
|
789
831
|
this.#specialTokenIds = specialTokenIds;
|
|
@@ -885,6 +927,8 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
885
927
|
} else if (seg.text && seg.text.length > 0) {
|
|
886
928
|
if (this.#type === 'unigram') {
|
|
887
929
|
ids.push(...this.#encodeUnigram(seg.text));
|
|
930
|
+
} else if (this.#type === 'wordpiece') {
|
|
931
|
+
ids.push(...this.#encodeWordPiece(seg.text));
|
|
888
932
|
} else {
|
|
889
933
|
ids.push(...this.#encodeBPE(seg.text));
|
|
890
934
|
}
|
|
@@ -1053,6 +1097,42 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1053
1097
|
return ids;
|
|
1054
1098
|
}
|
|
1055
1099
|
|
|
1100
|
+
#encodeWordPiece(text) {
|
|
1101
|
+
if (text.length === 0) return [];
|
|
1102
|
+
const words = this.#splitEveryCharacter
|
|
1103
|
+
? Array.from(text)
|
|
1104
|
+
: text.match(/[\p{L}\p{M}\p{N}]+|[^\s\p{L}\p{M}\p{N}]/gu) ?? [];
|
|
1105
|
+
const ids = [];
|
|
1106
|
+
for (const word of words) {
|
|
1107
|
+
if (/^\s+$/u.test(word)) continue;
|
|
1108
|
+
if (word.length > this.#wordPieceMaxChars) {
|
|
1109
|
+
ids.push(this.#getUnkTokenId());
|
|
1110
|
+
continue;
|
|
1111
|
+
}
|
|
1112
|
+
let start = 0;
|
|
1113
|
+
const wordIds = [];
|
|
1114
|
+
let failed = false;
|
|
1115
|
+
while (start < word.length) {
|
|
1116
|
+
let end = word.length;
|
|
1117
|
+
let tokenId;
|
|
1118
|
+
while (start < end) {
|
|
1119
|
+
const piece = `${start > 0 ? this.#wordPiecePrefix : ''}${word.slice(start, end)}`;
|
|
1120
|
+
tokenId = this.#vocab.get(piece);
|
|
1121
|
+
if (tokenId !== undefined) break;
|
|
1122
|
+
end -= 1;
|
|
1123
|
+
}
|
|
1124
|
+
if (tokenId === undefined) {
|
|
1125
|
+
failed = true;
|
|
1126
|
+
break;
|
|
1127
|
+
}
|
|
1128
|
+
wordIds.push(tokenId);
|
|
1129
|
+
start = end;
|
|
1130
|
+
}
|
|
1131
|
+
ids.push(...(failed ? [this.#getUnkTokenId()] : wordIds));
|
|
1132
|
+
}
|
|
1133
|
+
return ids;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1056
1136
|
#splitBpePretokens(text) {
|
|
1057
1137
|
if (!this.#useByteLevelEncoding || !this.#splitPretokenizer) return [text];
|
|
1058
1138
|
const regex = new RegExp(this.#splitPretokenizer.source, this.#splitPretokenizer.flags);
|
|
@@ -1196,6 +1276,17 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1196
1276
|
result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
|
|
1197
1277
|
// SentencePiece-style markers can still appear in some mixed vocabularies.
|
|
1198
1278
|
result = result.replace(/▁/g, ' ');
|
|
1279
|
+
} else if (this.#type === 'wordpiece') {
|
|
1280
|
+
if (this.#splitEveryCharacter) {
|
|
1281
|
+
result = tokens.join('');
|
|
1282
|
+
} else {
|
|
1283
|
+
result = tokens.reduce((text, token) => {
|
|
1284
|
+
if (token.startsWith(this.#wordPiecePrefix)) {
|
|
1285
|
+
return `${text}${token.slice(this.#wordPiecePrefix.length)}`;
|
|
1286
|
+
}
|
|
1287
|
+
return text ? `${text} ${token}` : token;
|
|
1288
|
+
}, '');
|
|
1289
|
+
}
|
|
1199
1290
|
} else {
|
|
1200
1291
|
// Join and convert ▁ back to spaces, handle GPT-style markers
|
|
1201
1292
|
result = decodeByteFallbackTokens(tokens)
|
|
@@ -75,7 +75,9 @@ export declare class DopplerLoader {
|
|
|
75
75
|
layers: Map<number, LayerWeights>;
|
|
76
76
|
experts: Map<string, ExpertWeights>;
|
|
77
77
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
78
|
+
lmHeadBias: Float32Array | null;
|
|
78
79
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
80
|
+
finalNormBias: Float32Array | null;
|
|
79
81
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
80
82
|
diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
|
|
81
83
|
perLayerInputWeights: PerLayerInputWeights | null;
|
|
@@ -224,6 +224,9 @@ export class DopplerLoader {
|
|
|
224
224
|
experts = new Map();
|
|
225
225
|
|
|
226
226
|
lmHead = null;
|
|
227
|
+
|
|
228
|
+
lmHeadBias = null;
|
|
229
|
+
finalNormBias = null;
|
|
227
230
|
|
|
228
231
|
finalNorm = null;
|
|
229
232
|
|
|
@@ -1398,6 +1401,8 @@ export class DopplerLoader {
|
|
|
1398
1401
|
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
1399
1402
|
embeddings: this.embeddings,
|
|
1400
1403
|
embeddingPostprocessor: this.manifest?.inference?.output?.embeddingPostprocessor ?? null,
|
|
1404
|
+
finalNormBiasTensor: this.manifest?.inference?.normalization?.finalNormBiasTensor ?? null,
|
|
1405
|
+
lmHeadBiasTensor: this.manifest?.inference?.output?.lmHeadBiasTensor ?? null,
|
|
1401
1406
|
diffusionGemmaSelfConditioning: this.manifest?.inference?.diffusionGemma?.selfConditioning === true,
|
|
1402
1407
|
modelType: this.manifest?.modelType ?? null,
|
|
1403
1408
|
tieWordEmbeddings,
|
|
@@ -1408,7 +1413,9 @@ export class DopplerLoader {
|
|
|
1408
1413
|
|
|
1409
1414
|
const result = await loadFinalWeights(ctx);
|
|
1410
1415
|
this.finalNorm = result.finalNorm;
|
|
1416
|
+
this.finalNormBias = result.finalNormBias;
|
|
1411
1417
|
this.lmHead = result.lmHead;
|
|
1418
|
+
this.lmHeadBias = result.lmHeadBias;
|
|
1412
1419
|
this.embeddingPostprocessor = result.embeddingPostprocessor;
|
|
1413
1420
|
this.diffusionGemmaSelfConditioning = result.diffusionGemmaSelfConditioning;
|
|
1414
1421
|
this.#normOffsetDebugLogged = result.normOffsetDebugLogged;
|
|
@@ -1525,6 +1532,8 @@ export class DopplerLoader {
|
|
|
1525
1532
|
this.layers.clear();
|
|
1526
1533
|
this.experts.clear();
|
|
1527
1534
|
this.lmHead = null;
|
|
1535
|
+
this.lmHeadBias = null;
|
|
1536
|
+
this.finalNormBias = null;
|
|
1528
1537
|
this.finalNorm = null;
|
|
1529
1538
|
this.embeddingPostprocessor = null;
|
|
1530
1539
|
this.perLayerInputWeights = null;
|
|
@@ -56,6 +56,10 @@ export interface FinalWeightsContext {
|
|
|
56
56
|
embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
57
57
|
/** Optional embedding postprocessor declared by manifest */
|
|
58
58
|
embeddingPostprocessor?: ManifestEmbeddingPostprocessorSchema | null;
|
|
59
|
+
/** Optional exact tensor name for an additive LM-head bias. */
|
|
60
|
+
lmHeadBiasTensor?: string | null;
|
|
61
|
+
/** Optional exact tensor name for a final LayerNorm bias. */
|
|
62
|
+
finalNormBiasTensor?: string | null;
|
|
59
63
|
/** Manifest-owned DiffusionGemma self-conditioning enablement */
|
|
60
64
|
diffusionGemmaSelfConditioning?: boolean;
|
|
61
65
|
/** Manifest model type */
|
|
@@ -74,8 +78,12 @@ export interface FinalWeightsContext {
|
|
|
74
78
|
export interface FinalWeightsResult {
|
|
75
79
|
/** Final layer norm tensor */
|
|
76
80
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
81
|
+
/** Optional final LayerNorm bias, held on CPU for embedding extraction. */
|
|
82
|
+
finalNormBias: Float32Array | null;
|
|
77
83
|
/** LM head tensor */
|
|
78
84
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
85
|
+
/** Optional additive LM-head bias, held on CPU for exact post-projection application. */
|
|
86
|
+
lmHeadBias: Float32Array | null;
|
|
79
87
|
/** Optional embedding-only postprocessor weights */
|
|
80
88
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
81
89
|
/** Optional DiffusionGemma decoder self-conditioning weights */
|
|
@@ -245,9 +245,11 @@ export async function loadFinalWeights(ctx) {
|
|
|
245
245
|
if (normDebugLogged) {
|
|
246
246
|
normOffsetDebugLogged = true;
|
|
247
247
|
}
|
|
248
|
+
const finalNormBias = await loadFinalNormBias(ctx);
|
|
248
249
|
|
|
249
250
|
// Load LM head
|
|
250
251
|
const lmHead = await loadLmHead(ctx);
|
|
252
|
+
const lmHeadBias = await loadLmHeadBias(ctx);
|
|
251
253
|
const embeddingPostprocessor = await loadEmbeddingPostprocessor(ctx);
|
|
252
254
|
const diffusionGemmaSelfConditioning = shouldLoadDiffusionGemmaSelfConditioning(ctx)
|
|
253
255
|
? await loadDiffusionGemmaSelfConditioning(ctx)
|
|
@@ -255,13 +257,34 @@ export async function loadFinalWeights(ctx) {
|
|
|
255
257
|
|
|
256
258
|
return {
|
|
257
259
|
finalNorm,
|
|
260
|
+
finalNormBias,
|
|
258
261
|
lmHead,
|
|
262
|
+
lmHeadBias,
|
|
259
263
|
embeddingPostprocessor,
|
|
260
264
|
diffusionGemmaSelfConditioning,
|
|
261
265
|
normOffsetDebugLogged,
|
|
262
266
|
};
|
|
263
267
|
}
|
|
264
268
|
|
|
269
|
+
async function loadFinalNormBias(ctx) {
|
|
270
|
+
const tensorName = typeof ctx.finalNormBiasTensor === 'string'
|
|
271
|
+
? ctx.finalNormBiasTensor.trim()
|
|
272
|
+
: '';
|
|
273
|
+
if (!tensorName) {
|
|
274
|
+
return null;
|
|
275
|
+
}
|
|
276
|
+
const location = ctx.tensorLocations.get(tensorName);
|
|
277
|
+
if (!location) {
|
|
278
|
+
throw new Error(`[Loader] Final norm bias tensor "${tensorName}" is missing from the manifest.`);
|
|
279
|
+
}
|
|
280
|
+
if (!Array.isArray(location.shape) || location.shape.length !== 1) {
|
|
281
|
+
throw new Error(
|
|
282
|
+
`[Loader] Final norm bias tensor "${tensorName}" must be rank 1, got ${JSON.stringify(location.shape)}.`
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
return loadCpuFloatTensor(ctx, tensorName, `final norm bias "${tensorName}"`);
|
|
286
|
+
}
|
|
287
|
+
|
|
265
288
|
// ============================================================================
|
|
266
289
|
// Final Norm Loading
|
|
267
290
|
// ============================================================================
|
|
@@ -398,6 +421,25 @@ async function loadLmHead(ctx) {
|
|
|
398
421
|
return lmHead;
|
|
399
422
|
}
|
|
400
423
|
|
|
424
|
+
async function loadLmHeadBias(ctx) {
|
|
425
|
+
const tensorName = typeof ctx.lmHeadBiasTensor === 'string'
|
|
426
|
+
? ctx.lmHeadBiasTensor.trim()
|
|
427
|
+
: '';
|
|
428
|
+
if (!tensorName) {
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
const location = ctx.tensorLocations.get(tensorName);
|
|
432
|
+
if (!location) {
|
|
433
|
+
throw new Error(`[Loader] LM head bias tensor "${tensorName}" is missing from the manifest.`);
|
|
434
|
+
}
|
|
435
|
+
if (!Array.isArray(location.shape) || location.shape.length !== 1) {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`[Loader] LM head bias tensor "${tensorName}" must be rank 1, got ${JSON.stringify(location.shape)}.`
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
return loadCpuFloatTensor(ctx, tensorName, `LM head bias "${tensorName}"`);
|
|
441
|
+
}
|
|
442
|
+
|
|
401
443
|
async function loadEmbeddingPostprocessor(ctx) {
|
|
402
444
|
const config = ctx.embeddingPostprocessor;
|
|
403
445
|
if (config == null) {
|
|
@@ -52,6 +52,12 @@ export interface LayerLoaderContext {
|
|
|
52
52
|
linearValueHeadDim?: number;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
export declare function describeQKNormTensor(
|
|
56
|
+
tensor: GPUBuffer | WeightBuffer | Float32Array | Uint8Array | null,
|
|
57
|
+
location: TensorLocation | null | undefined,
|
|
58
|
+
name: string
|
|
59
|
+
): WeightBuffer | Float32Array | null;
|
|
60
|
+
|
|
55
61
|
/**
|
|
56
62
|
* Load all weights for a single transformer layer.
|
|
57
63
|
*
|