doppler-gpu 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +27 -0
- package/models/catalog.json +612 -7
- package/package.json +13 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/version.js +1 -1
|
@@ -35,13 +35,19 @@ const LAYER_PREFIXES = (layerIdx) => [
|
|
|
35
35
|
|
|
36
36
|
const ATTN_SUFFIXES = {
|
|
37
37
|
inputNorm: ['input_layernorm.weight', 'attn_norm.weight', 'operator_norm.weight'],
|
|
38
|
+
inputNormBias: ['input_layernorm.bias', 'attn_norm.bias', 'operator_norm.bias'],
|
|
38
39
|
qProj: ['self_attn.q_proj.weight', 'attention.wq.weight', 'attn_q.weight'],
|
|
40
|
+
qProjBias: ['self_attn.q_proj.bias', 'attention.wq.bias', 'attn_q.bias'],
|
|
39
41
|
kProj: ['self_attn.k_proj.weight', 'attention.wk.weight', 'attn_k.weight'],
|
|
42
|
+
kProjBias: ['self_attn.k_proj.bias', 'attention.wk.bias', 'attn_k.bias'],
|
|
40
43
|
vProj: ['self_attn.v_proj.weight', 'attention.wv.weight', 'attn_v.weight'],
|
|
44
|
+
vProjBias: ['self_attn.v_proj.bias', 'attention.wv.bias', 'attn_v.bias'],
|
|
41
45
|
oProj: ['self_attn.o_proj.weight', 'self_attn.out_proj.weight', 'attention.wo.weight', 'attn_output.weight'],
|
|
46
|
+
oProjBias: ['self_attn.o_proj.bias', 'self_attn.out_proj.bias', 'attention.wo.bias', 'attn_output.bias'],
|
|
42
47
|
qNorm: ['self_attn.q_norm.weight', 'self_attn.q_layernorm.weight', 'attn_q_norm.weight'],
|
|
43
48
|
kNorm: ['self_attn.k_norm.weight', 'self_attn.k_layernorm.weight', 'attn_k_norm.weight'],
|
|
44
49
|
postAttentionNorm: ['post_attention_layernorm.weight', 'post_attention_norm.weight', 'ffn_norm.weight'],
|
|
50
|
+
postAttentionNormBias: ['post_attention_layernorm.bias', 'post_attention_norm.bias', 'ffn_norm.bias'],
|
|
45
51
|
preFeedforwardNorm: ['pre_feedforward_layernorm.weight'],
|
|
46
52
|
preFeedforwardNorm2: ['pre_feedforward_layernorm_2.weight'],
|
|
47
53
|
postFeedforwardNorm: ['post_feedforward_layernorm.weight', 'post_ffw_norm.weight'],
|
|
@@ -73,8 +79,11 @@ const CONV_SUFFIXES = {
|
|
|
73
79
|
const FFN_SUFFIXES = {
|
|
74
80
|
ffnGateUp: ['mlp.gate_up_proj.weight', 'ffn_gate_up.weight', 'feed_forward.w1_w3.weight'],
|
|
75
81
|
ffnGate: ['mlp.gate_proj.weight', 'feed_forward.w1.weight', 'ffn_gate.weight'],
|
|
82
|
+
ffnGateBias: ['mlp.gate_proj.bias', 'feed_forward.w1.bias', 'ffn_gate.bias'],
|
|
76
83
|
ffnUp: ['mlp.up_proj.weight', 'feed_forward.w3.weight', 'ffn_up.weight'],
|
|
84
|
+
ffnUpBias: ['mlp.up_proj.bias', 'feed_forward.w3.bias', 'ffn_up.bias'],
|
|
77
85
|
ffnDown: ['mlp.down_proj.weight', 'feed_forward.w2.weight', 'ffn_down.weight'],
|
|
86
|
+
ffnDownBias: ['mlp.down_proj.bias', 'feed_forward.w2.bias', 'ffn_down.bias'],
|
|
78
87
|
perLayerInputGate: ['per_layer_input_gate.weight'],
|
|
79
88
|
perLayerProjection: ['per_layer_projection.weight'],
|
|
80
89
|
};
|
|
@@ -334,10 +343,15 @@ export async function loadLayer(ctx, layerIdx) {
|
|
|
334
343
|
|
|
335
344
|
const weights = {
|
|
336
345
|
inputNorm: null,
|
|
346
|
+
inputNormBias: null,
|
|
337
347
|
qProj: null,
|
|
348
|
+
qProjBias: null,
|
|
338
349
|
kProj: null,
|
|
350
|
+
kProjBias: null,
|
|
339
351
|
vProj: null,
|
|
352
|
+
vProjBias: null,
|
|
340
353
|
oProj: null,
|
|
354
|
+
oProjBias: null,
|
|
341
355
|
qkvProj: null,
|
|
342
356
|
qkvSizes: null,
|
|
343
357
|
qkvDtype: null,
|
|
@@ -351,6 +365,7 @@ export async function loadLayer(ctx, layerIdx) {
|
|
|
351
365
|
qNorm: null,
|
|
352
366
|
kNorm: null,
|
|
353
367
|
postAttentionNorm: null,
|
|
368
|
+
postAttentionNormBias: null,
|
|
354
369
|
preFeedforwardNorm: null,
|
|
355
370
|
preFeedforwardNorm2: null,
|
|
356
371
|
postFeedforwardNorm: null,
|
|
@@ -362,8 +377,11 @@ export async function loadLayer(ctx, layerIdx) {
|
|
|
362
377
|
convKernel: null,
|
|
363
378
|
convOutProj: null,
|
|
364
379
|
ffnGate: null,
|
|
380
|
+
ffnGateBias: null,
|
|
365
381
|
ffnUp: null,
|
|
382
|
+
ffnUpBias: null,
|
|
366
383
|
ffnDown: null,
|
|
384
|
+
ffnDownBias: null,
|
|
367
385
|
ffnGateUp: null,
|
|
368
386
|
perLayerInputGate: null,
|
|
369
387
|
perLayerProjection: null,
|
|
@@ -430,12 +448,58 @@ function createTryLoadNorm(ctx, prefixes, tryLoad) {
|
|
|
430
448
|
};
|
|
431
449
|
}
|
|
432
450
|
|
|
451
|
+
export function describeQKNormTensor(tensor, location, name) {
|
|
452
|
+
if (isWeightBuffer(tensor) || tensor instanceof Float32Array) {
|
|
453
|
+
return tensor;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const isGpuBuffer = typeof GPUBuffer !== 'undefined' && tensor instanceof GPUBuffer;
|
|
457
|
+
const isGpuBufferLike = tensor && typeof tensor.size === 'number' && typeof tensor.destroy === 'function';
|
|
458
|
+
if (!isGpuBuffer && !isGpuBufferLike) return null;
|
|
459
|
+
|
|
460
|
+
if (!Array.isArray(location?.shape) || location.shape.length === 0) {
|
|
461
|
+
throw new Error(`[LayerLoader] Q/K norm tensor "${name}" is missing its logical shape.`);
|
|
462
|
+
}
|
|
463
|
+
const dtype = getWeightDtype(tensor);
|
|
464
|
+
if (dtype !== 'f16' && dtype !== 'f32') {
|
|
465
|
+
throw new Error(
|
|
466
|
+
`[LayerLoader] Q/K norm tensor "${name}" has unsupported runtime dtype "${String(dtype)}".`
|
|
467
|
+
);
|
|
468
|
+
}
|
|
469
|
+
return createWeightBuffer(
|
|
470
|
+
tensor,
|
|
471
|
+
dtype,
|
|
472
|
+
location.layout ?? 'row',
|
|
473
|
+
location.shape,
|
|
474
|
+
name
|
|
475
|
+
);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function createTryLoadQKNorm(ctx, prefixes) {
|
|
479
|
+
return async (suffixes) => {
|
|
480
|
+
for (const prefix of prefixes) {
|
|
481
|
+
for (const suffix of suffixes) {
|
|
482
|
+
const name = `${prefix}.${suffix}`;
|
|
483
|
+
const location = ctx.tensorLocations.get(name) ?? null;
|
|
484
|
+
if (!location) continue;
|
|
485
|
+
|
|
486
|
+
const tensor = await ctx.loadTensor(name, true, true);
|
|
487
|
+
if (!tensor) continue;
|
|
488
|
+
const described = describeQKNormTensor(tensor, location, name);
|
|
489
|
+
if (described) return described;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
return null;
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
|
|
433
496
|
// ============================================================================
|
|
434
497
|
// Weight Loading Functions
|
|
435
498
|
// ============================================================================
|
|
436
499
|
|
|
437
500
|
|
|
438
501
|
async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm) {
|
|
502
|
+
const tryLoadQKNorm = createTryLoadQKNorm(ctx, LAYER_PREFIXES(layerIdx));
|
|
439
503
|
const tryLoadCpu = async (suffixes) => {
|
|
440
504
|
for (const prefix of LAYER_PREFIXES(layerIdx)) {
|
|
441
505
|
for (const suffix of suffixes) {
|
|
@@ -450,13 +514,19 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
450
514
|
|
|
451
515
|
const [
|
|
452
516
|
inputNorm,
|
|
517
|
+
inputNormBias,
|
|
453
518
|
qProj,
|
|
519
|
+
qProjBias,
|
|
454
520
|
kProj,
|
|
521
|
+
kProjBias,
|
|
455
522
|
vProj,
|
|
523
|
+
vProjBias,
|
|
456
524
|
oProj,
|
|
525
|
+
oProjBias,
|
|
457
526
|
qNorm,
|
|
458
527
|
kNorm,
|
|
459
528
|
postAttentionNorm,
|
|
529
|
+
postAttentionNormBias,
|
|
460
530
|
preFeedforwardNorm,
|
|
461
531
|
preFeedforwardNorm2,
|
|
462
532
|
postFeedforwardNorm,
|
|
@@ -478,14 +548,20 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
478
548
|
linearNorm,
|
|
479
549
|
] = await Promise.all([
|
|
480
550
|
tryLoadNorm(ATTN_SUFFIXES.inputNorm),
|
|
551
|
+
tryLoadNorm(ATTN_SUFFIXES.inputNormBias),
|
|
481
552
|
tryLoad(ATTN_SUFFIXES.qProj),
|
|
553
|
+
tryLoadNorm(ATTN_SUFFIXES.qProjBias),
|
|
482
554
|
tryLoad(ATTN_SUFFIXES.kProj),
|
|
555
|
+
tryLoadNorm(ATTN_SUFFIXES.kProjBias),
|
|
483
556
|
tryLoad(ATTN_SUFFIXES.vProj),
|
|
557
|
+
tryLoadNorm(ATTN_SUFFIXES.vProjBias),
|
|
484
558
|
tryLoad(ATTN_SUFFIXES.oProj),
|
|
559
|
+
tryLoadNorm(ATTN_SUFFIXES.oProjBias),
|
|
485
560
|
// Gemma 3: q_norm and k_norm use Gemma3RMSNorm with (1+weight) formula
|
|
486
|
-
|
|
487
|
-
|
|
561
|
+
tryLoadQKNorm(ATTN_SUFFIXES.qNorm),
|
|
562
|
+
tryLoadQKNorm(ATTN_SUFFIXES.kNorm),
|
|
488
563
|
tryLoadNorm(ATTN_SUFFIXES.postAttentionNorm),
|
|
564
|
+
tryLoadNorm(ATTN_SUFFIXES.postAttentionNormBias),
|
|
489
565
|
tryLoadNorm(ATTN_SUFFIXES.preFeedforwardNorm),
|
|
490
566
|
tryLoadNorm(ATTN_SUFFIXES.preFeedforwardNorm2),
|
|
491
567
|
tryLoadNorm(ATTN_SUFFIXES.postFeedforwardNorm),
|
|
@@ -508,10 +584,15 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
508
584
|
]);
|
|
509
585
|
|
|
510
586
|
weights.inputNorm = inputNorm;
|
|
587
|
+
weights.inputNormBias = inputNormBias;
|
|
511
588
|
weights.qProj = qProj;
|
|
589
|
+
weights.qProjBias = qProjBias;
|
|
512
590
|
weights.kProj = kProj;
|
|
591
|
+
weights.kProjBias = kProjBias;
|
|
513
592
|
weights.vProj = vProj;
|
|
593
|
+
weights.vProjBias = vProjBias;
|
|
514
594
|
weights.oProj = oProj;
|
|
595
|
+
weights.oProjBias = oProjBias;
|
|
515
596
|
weights.qNorm = qNorm;
|
|
516
597
|
weights.kNorm = kNorm;
|
|
517
598
|
|
|
@@ -525,6 +606,7 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
525
606
|
}
|
|
526
607
|
|
|
527
608
|
weights.postAttentionNorm = postAttentionNorm;
|
|
609
|
+
weights.postAttentionNormBias = postAttentionNormBias;
|
|
528
610
|
weights.preFeedforwardNorm = preFeedforwardNorm;
|
|
529
611
|
weights.preFeedforwardNorm2 = preFeedforwardNorm2;
|
|
530
612
|
weights.postFeedforwardNorm = postFeedforwardNorm;
|
|
@@ -570,11 +652,22 @@ async function loadAttentionWeights(ctx, weights, layerIdx, tryLoad, tryLoadNorm
|
|
|
570
652
|
async function loadFfnWeights(ctx, weights, layerIdx, tryLoad, prefixes) {
|
|
571
653
|
const perLayerProjection = await loadStablePerLayerProjection(ctx, layerIdx, prefixes, tryLoad);
|
|
572
654
|
const stablePerLayerInputGate = await loadStablePerLayerInputGate(ctx, layerIdx, prefixes, tryLoad);
|
|
573
|
-
const [
|
|
655
|
+
const [
|
|
656
|
+
ffnGateUp,
|
|
657
|
+
ffnGate,
|
|
658
|
+
ffnGateBias,
|
|
659
|
+
ffnUp,
|
|
660
|
+
ffnUpBias,
|
|
661
|
+
ffnDown,
|
|
662
|
+
ffnDownBias,
|
|
663
|
+
] = await Promise.all([
|
|
574
664
|
tryLoad(FFN_SUFFIXES.ffnGateUp),
|
|
575
665
|
tryLoad(FFN_SUFFIXES.ffnGate),
|
|
666
|
+
tryLoad(FFN_SUFFIXES.ffnGateBias),
|
|
576
667
|
tryLoad(FFN_SUFFIXES.ffnUp),
|
|
668
|
+
tryLoad(FFN_SUFFIXES.ffnUpBias),
|
|
577
669
|
tryLoad(FFN_SUFFIXES.ffnDown),
|
|
670
|
+
tryLoad(FFN_SUFFIXES.ffnDownBias),
|
|
578
671
|
]);
|
|
579
672
|
|
|
580
673
|
if (ffnGateUp) {
|
|
@@ -584,16 +677,22 @@ async function loadFfnWeights(ctx, weights, layerIdx, tryLoad, prefixes) {
|
|
|
584
677
|
debugTrace.loader(`Layer ${layerIdx}: Using fused gate_up_proj for 2-pass FFN`);
|
|
585
678
|
} else {
|
|
586
679
|
weights.ffnGate = ffnGate;
|
|
680
|
+
weights.ffnGateBias = ffnGateBias;
|
|
587
681
|
weights.ffnUp = ffnUp;
|
|
682
|
+
weights.ffnUpBias = ffnUpBias;
|
|
588
683
|
}
|
|
589
684
|
|
|
590
685
|
weights.ffnDown = ffnDown;
|
|
686
|
+
weights.ffnDownBias = ffnDownBias;
|
|
591
687
|
weights.perLayerInputGate = stablePerLayerInputGate;
|
|
592
688
|
weights.perLayerProjection = perLayerProjection;
|
|
593
689
|
|
|
594
690
|
weights.gate = weights.ffnGate;
|
|
691
|
+
weights.gateBias = weights.ffnGateBias;
|
|
595
692
|
weights.up = weights.ffnUp;
|
|
693
|
+
weights.upBias = weights.ffnUpBias;
|
|
596
694
|
weights.down = weights.ffnDown;
|
|
695
|
+
weights.downBias = weights.ffnDownBias;
|
|
597
696
|
weights.gateUp = weights.ffnGateUp;
|
|
598
697
|
}
|
|
599
698
|
|
|
@@ -38,11 +38,17 @@ export interface TensorLocation {
|
|
|
38
38
|
*/
|
|
39
39
|
export interface LayerWeights {
|
|
40
40
|
inputNorm: GPUBuffer | Float32Array | null;
|
|
41
|
+
inputNormBias?: GPUBuffer | Float32Array | null;
|
|
41
42
|
qProj: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
43
|
+
qProjBias?: GPUBuffer | Float32Array | null;
|
|
42
44
|
kProj: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
45
|
+
kProjBias?: GPUBuffer | Float32Array | null;
|
|
43
46
|
vProj: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
47
|
+
vProjBias?: GPUBuffer | Float32Array | null;
|
|
44
48
|
oProj: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
49
|
+
oProjBias?: GPUBuffer | Float32Array | null;
|
|
45
50
|
qkvProj?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
51
|
+
qkvProjBias?: GPUBuffer | Float32Array | null;
|
|
46
52
|
qkvSizes?: [number, number, number] | null;
|
|
47
53
|
qkvDtype?: 'f16' | 'f32' | null;
|
|
48
54
|
linearABProj?: WeightBuffer | null;
|
|
@@ -54,9 +60,10 @@ export interface LayerWeights {
|
|
|
54
60
|
linearDtBias?: GPUBuffer | Float32Array | null;
|
|
55
61
|
linearALog?: GPUBuffer | Float32Array | null;
|
|
56
62
|
linearNorm?: GPUBuffer | Float32Array | null;
|
|
57
|
-
qNorm: GPUBuffer | Float32Array | null;
|
|
58
|
-
kNorm: GPUBuffer | Float32Array | null;
|
|
63
|
+
qNorm: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
64
|
+
kNorm: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
59
65
|
postAttentionNorm: GPUBuffer | Float32Array | null;
|
|
66
|
+
postAttentionNormBias?: GPUBuffer | Float32Array | null;
|
|
60
67
|
preFeedforwardNorm: GPUBuffer | Float32Array | null;
|
|
61
68
|
preFeedforwardNorm2?: GPUBuffer | Float32Array | null;
|
|
62
69
|
postFeedforwardNorm: GPUBuffer | Float32Array | null;
|
|
@@ -68,14 +75,20 @@ export interface LayerWeights {
|
|
|
68
75
|
convKernel?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
69
76
|
convOutProj?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
70
77
|
ffnGate: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
78
|
+
ffnGateBias?: GPUBuffer | Float32Array | null;
|
|
71
79
|
ffnUp: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
80
|
+
ffnUpBias?: GPUBuffer | Float32Array | null;
|
|
72
81
|
ffnDown: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
82
|
+
ffnDownBias?: GPUBuffer | Float32Array | null;
|
|
73
83
|
/** Fused gate+up projection [intermediateSize*2, hiddenSize] for 2-pass FFN */
|
|
74
84
|
ffnGateUp?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
75
85
|
// Aliases for pipeline compatibility
|
|
76
86
|
gate?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
87
|
+
gateBias?: GPUBuffer | Float32Array | null;
|
|
77
88
|
up?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
89
|
+
upBias?: GPUBuffer | Float32Array | null;
|
|
78
90
|
down?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
91
|
+
downBias?: GPUBuffer | Float32Array | null;
|
|
79
92
|
/** Fused gate+up for pipeline compatibility */
|
|
80
93
|
gateUp?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
81
94
|
routerWeight?: GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer | Float32Array | null;
|
|
@@ -7,6 +7,11 @@ import { installNodeFileFetchShim } from './node-file-fetch.js';
|
|
|
7
7
|
import { NodeConvertWorkerPool } from './node-convert-worker-pool.js';
|
|
8
8
|
import { bootstrapNodeWebGPU } from './node-webgpu.js';
|
|
9
9
|
import { buildManifestIntegrityFromModelDir } from './rdrr-integrity-refresh.js';
|
|
10
|
+
import { applySourceTensorRules } from '../converter/source-tensor-rules.js';
|
|
11
|
+
import {
|
|
12
|
+
buildSourceTokenizerJson,
|
|
13
|
+
validateSourceTokenizerPolicy,
|
|
14
|
+
} from '../converter/source-tokenizer.js';
|
|
10
15
|
import { isPlainObject } from '../utils/plain-object.js';
|
|
11
16
|
import { selectRuleValue } from '../rules/rule-registry.js';
|
|
12
17
|
import { log, trace } from '../debug/index.js';
|
|
@@ -1345,13 +1350,40 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1345
1350
|
architectureHint = parsedTransformer.architectureHint;
|
|
1346
1351
|
embeddingPostprocessor = parsedTransformer.embeddingPostprocessor ?? null;
|
|
1347
1352
|
rerankScoring = parsedTransformer.rerankScoring ?? null;
|
|
1348
|
-
|
|
1353
|
+
tensors = applySourceTensorRules(tensors, converterConfig.sourceTensors);
|
|
1354
|
+
architecture = converterConfig.architecture ?? extractArchitecture(config, null);
|
|
1349
1355
|
const tokenizerJsonPath = path.join(inputDir, 'tokenizer.json');
|
|
1350
1356
|
tokenizerModelPath = path.join(inputDir, 'tokenizer.model');
|
|
1351
1357
|
const tokenizerConfigPath = path.join(inputDir, 'tokenizer_config.json');
|
|
1352
1358
|
tokenizerJson = await readOptionalJson(tokenizerJsonPath);
|
|
1353
1359
|
tokenizerConfig = await readOptionalJson(tokenizerConfigPath);
|
|
1354
1360
|
hasTokenizerModel = await fileExists(tokenizerModelPath);
|
|
1361
|
+
const sourceTokenizerPolicy = validateSourceTokenizerPolicy(converterConfig.sourceTokenizer);
|
|
1362
|
+
if (sourceTokenizerPolicy) {
|
|
1363
|
+
if (tokenizerJson) {
|
|
1364
|
+
throw new Error(
|
|
1365
|
+
'node convert: sourceTokenizer cannot be combined with a source tokenizer.json. ' +
|
|
1366
|
+
'Remove sourceTokenizer or make the source tokenizer identity explicit in one place.'
|
|
1367
|
+
);
|
|
1368
|
+
}
|
|
1369
|
+
if (hasTokenizerModel) {
|
|
1370
|
+
throw new Error(
|
|
1371
|
+
'node convert: sourceTokenizer cannot be combined with a source tokenizer.model.'
|
|
1372
|
+
);
|
|
1373
|
+
}
|
|
1374
|
+
const vocabPath = path.join(inputDir, sourceTokenizerPolicy.vocabFile);
|
|
1375
|
+
let vocabText;
|
|
1376
|
+
try {
|
|
1377
|
+
vocabText = await fs.readFile(vocabPath, 'utf8');
|
|
1378
|
+
} catch (error) {
|
|
1379
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1380
|
+
throw new Error(
|
|
1381
|
+
`node convert: unable to read sourceTokenizer.vocabFile ` +
|
|
1382
|
+
`"${sourceTokenizerPolicy.vocabFile}": ${message}`
|
|
1383
|
+
);
|
|
1384
|
+
}
|
|
1385
|
+
tokenizerJson = buildSourceTokenizerJson(sourceTokenizerPolicy, vocabText);
|
|
1386
|
+
}
|
|
1355
1387
|
}
|
|
1356
1388
|
parseTimer.stop(`${modelKind} tensors=${tensors.length}`);
|
|
1357
1389
|
|
|
@@ -13,6 +13,8 @@ const ADAPTER_PROBE_OPTIONS = Object.freeze([
|
|
|
13
13
|
null,
|
|
14
14
|
]);
|
|
15
15
|
|
|
16
|
+
let ownedProviderInstallation = null;
|
|
17
|
+
|
|
16
18
|
function hasNavigatorGpu() {
|
|
17
19
|
return typeof globalThis.navigator !== 'undefined'
|
|
18
20
|
&& !!globalThis.navigator?.gpu
|
|
@@ -158,6 +160,11 @@ function resolveExportsPath(exportsField, rootPath) {
|
|
|
158
160
|
function installNavigatorGpu(gpu, options = {}) {
|
|
159
161
|
if (!gpu || typeof gpu.requestAdapter !== 'function') return false;
|
|
160
162
|
const force = options.force === true;
|
|
163
|
+
const provider = options.provider ?? null;
|
|
164
|
+
const hadNavigator = typeof globalThis.navigator !== 'undefined';
|
|
165
|
+
const navigatorGpuDescriptor = hadNavigator
|
|
166
|
+
? Object.getOwnPropertyDescriptor(globalThis.navigator, 'gpu') ?? null
|
|
167
|
+
: null;
|
|
161
168
|
if (typeof globalThis.navigator === 'undefined') {
|
|
162
169
|
Object.defineProperty(globalThis, 'navigator', {
|
|
163
170
|
value: { gpu },
|
|
@@ -165,11 +172,34 @@ function installNavigatorGpu(gpu, options = {}) {
|
|
|
165
172
|
configurable: true,
|
|
166
173
|
enumerable: false,
|
|
167
174
|
});
|
|
175
|
+
ownedProviderInstallation = {
|
|
176
|
+
gpu,
|
|
177
|
+
provider,
|
|
178
|
+
hadNavigator: false,
|
|
179
|
+
navigatorGpuDescriptor: null,
|
|
180
|
+
};
|
|
168
181
|
return true;
|
|
169
182
|
}
|
|
170
183
|
|
|
171
184
|
if (force && globalThis.navigator.gpu) {
|
|
172
|
-
|
|
185
|
+
const previousOwnedInstallation = ownedProviderInstallation;
|
|
186
|
+
const replacesOwnedProvider = previousOwnedInstallation?.gpu === globalThis.navigator.gpu;
|
|
187
|
+
Object.defineProperty(globalThis.navigator, 'gpu', {
|
|
188
|
+
value: gpu,
|
|
189
|
+
writable: true,
|
|
190
|
+
configurable: true,
|
|
191
|
+
enumerable: navigatorGpuDescriptor?.enumerable === true,
|
|
192
|
+
});
|
|
193
|
+
ownedProviderInstallation = {
|
|
194
|
+
gpu,
|
|
195
|
+
provider,
|
|
196
|
+
hadNavigator: replacesOwnedProvider
|
|
197
|
+
? previousOwnedInstallation.hadNavigator
|
|
198
|
+
: hadNavigator,
|
|
199
|
+
navigatorGpuDescriptor: replacesOwnedProvider
|
|
200
|
+
? previousOwnedInstallation.navigatorGpuDescriptor
|
|
201
|
+
: navigatorGpuDescriptor,
|
|
202
|
+
};
|
|
173
203
|
return true;
|
|
174
204
|
}
|
|
175
205
|
|
|
@@ -180,10 +210,60 @@ function installNavigatorGpu(gpu, options = {}) {
|
|
|
180
210
|
configurable: true,
|
|
181
211
|
enumerable: false,
|
|
182
212
|
});
|
|
213
|
+
ownedProviderInstallation = {
|
|
214
|
+
gpu,
|
|
215
|
+
provider,
|
|
216
|
+
hadNavigator,
|
|
217
|
+
navigatorGpuDescriptor,
|
|
218
|
+
};
|
|
183
219
|
}
|
|
184
220
|
return true;
|
|
185
221
|
}
|
|
186
222
|
|
|
223
|
+
/**
|
|
224
|
+
* Release a Node WebGPU provider installed by this module.
|
|
225
|
+
*
|
|
226
|
+
* Dawn's Node binding keeps its callback pump alive while the GPU object is
|
|
227
|
+
* reachable. Device destruction is necessary but insufficient: command-line
|
|
228
|
+
* callers must also release Doppler's global provider reference after all GPU
|
|
229
|
+
* work and devices have been torn down. Pre-installed browser or host-owned
|
|
230
|
+
* providers are never removed.
|
|
231
|
+
*/
|
|
232
|
+
export function releaseNodeWebGPU() {
|
|
233
|
+
const installation = ownedProviderInstallation;
|
|
234
|
+
ownedProviderInstallation = null;
|
|
235
|
+
if (!installation) {
|
|
236
|
+
return { released: false, provider: null, reason: 'not-owned' };
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const navigatorObject = globalThis.navigator;
|
|
240
|
+
if (!navigatorObject || navigatorObject.gpu !== installation.gpu) {
|
|
241
|
+
return {
|
|
242
|
+
released: false,
|
|
243
|
+
provider: installation.provider,
|
|
244
|
+
reason: 'provider-replaced',
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
if (!installation.hadNavigator) {
|
|
249
|
+
delete globalThis.navigator;
|
|
250
|
+
} else if (installation.navigatorGpuDescriptor) {
|
|
251
|
+
Object.defineProperty(
|
|
252
|
+
navigatorObject,
|
|
253
|
+
'gpu',
|
|
254
|
+
installation.navigatorGpuDescriptor
|
|
255
|
+
);
|
|
256
|
+
} else {
|
|
257
|
+
delete navigatorObject.gpu;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
return {
|
|
261
|
+
released: true,
|
|
262
|
+
provider: installation.provider,
|
|
263
|
+
reason: null,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
187
267
|
function resolveGpuFromModule(mod) {
|
|
188
268
|
if (!mod) return null;
|
|
189
269
|
|
|
@@ -317,7 +397,10 @@ async function tryInstallAndProbeProvider(providerSpecifier, options = {}) {
|
|
|
317
397
|
};
|
|
318
398
|
}
|
|
319
399
|
|
|
320
|
-
if (!installWebgpuFromModule(mod, {
|
|
400
|
+
if (!installWebgpuFromModule(mod, {
|
|
401
|
+
force: options.force === true,
|
|
402
|
+
provider: providerSpecifier,
|
|
403
|
+
})) {
|
|
321
404
|
return {
|
|
322
405
|
ok: false,
|
|
323
406
|
provider: providerSpecifier,
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.4.
|
|
1
|
+
export const DOPPLER_VERSION = '0.4.11';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|