doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -172,8 +172,8 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
|
|
|
172
172
|
"gather_vec4_f16_out.wgsl#gather_vec4_f16_out": "dd631155d4df58d29a1c444f521a8ebbf28532df3eb37d1b27f8015d3a42d9e6",
|
|
173
173
|
"gather_vec4.wgsl#gather_vec4": "e8d90b826350a9feb96476fc4a5e568aebfc29948cf07a2d7db0dee4a4216b6a",
|
|
174
174
|
"gather.wgsl#main": "4b12653c53247b32ebde7f6cf6a989d6248977e3816c761540b990b5f9818cb6",
|
|
175
|
-
"gelu_f16.wgsl#main": "
|
|
176
|
-
"gelu.wgsl#main": "
|
|
175
|
+
"gelu_f16.wgsl#main": "1f713a30bf6777270e9386792bde2a13824d37284abd0ece1abb1de7e2807fd2",
|
|
176
|
+
"gelu.wgsl#main": "db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
177
177
|
"gemma4_route_q4_matmul_f16a.wgsl#route_matmul_f16a": "7f42e4bc7d7f398cf826df6c708b3da760557a713f5d208aefcd71c6a2512380",
|
|
178
178
|
"gptoss_mxfp4_expert_fused.wgsl#main_expert": "3159e8cd81da13f909cf905e6d35307fefe1dcbbdf1b2b8e8ff0ce923bd71180",
|
|
179
179
|
"gptoss_router_topk.wgsl#softmax_topk": "86e4ea709c0c0084d09c6a4cd07710dc14f380e03f91b8ed9ec871b310be49f1",
|
|
@@ -251,8 +251,8 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
|
|
|
251
251
|
"probe_shader_f16.wgsl#main": "4a44a579136492fdc579997e7d2bd9c249e117039c9f19d44b1f7ab68c668325",
|
|
252
252
|
"relu_f16.wgsl#main": "baac3a33693e5a72e60c7ec9b4a4dbcec10a83ca1fac2972988b07a75780f14a",
|
|
253
253
|
"relu.wgsl#main": "52ffa66f9898a4f291758ae93055000cb1eb488c22a993e6b0568a29d9d3ad28",
|
|
254
|
-
"rep_penalty_f16.wgsl#main": "
|
|
255
|
-
"rep_penalty.wgsl#main": "
|
|
254
|
+
"rep_penalty_f16.wgsl#main": "2efd3f8dc87f715548bb131e67dd64942e7873a1bd9302a270bdeacc0ea8ecfb",
|
|
255
|
+
"rep_penalty.wgsl#main": "833d412b5db7c6da33ab980c4125717bbb8186f20c6e2a36a76e97e41fd99b4a",
|
|
256
256
|
"repeat_channels_f16.wgsl#main": "6eee5f9f74d8c17a71f6f422b503bbfb872350a9486ec0065fd1f67718311883",
|
|
257
257
|
"repeat_channels.wgsl#main": "bce0cb33beed50ce388b2d069961ae0fbe361dd4930b9344cb066b3390475a51",
|
|
258
258
|
"residual_f16_vec4.wgsl#add_vec4": "7735198b679f375c0a980f109ea05d4b95af13a00438347ace7a27d36009aa76",
|
|
@@ -287,18 +287,18 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
|
|
|
287
287
|
"rope.wgsl#rope_ntk_scaled": "01ab3773e2558b8a541c3c27574cb277923ecff72941c9b590acfa3eeb45c6d4",
|
|
288
288
|
"rope.wgsl#rope_qk": "7ba1041b73527c419f22c4f7f08345d411ff23317e9d9107907b3f69637e5064",
|
|
289
289
|
"rope.wgsl#rope_yarn": "ed18b619bc67605525dddcb324e5728652ab63ece52b3327d4403649e0abac0c",
|
|
290
|
-
"sample_f16.wgsl#argmax": "
|
|
291
|
-
"sample_f16.wgsl#argmax_reduce": "
|
|
292
|
-
"sample_f16.wgsl#find_topk_phase1": "
|
|
293
|
-
"sample_f16.wgsl#find_topk_phase2": "
|
|
294
|
-
"sample_f16.wgsl#sample_single_pass": "
|
|
295
|
-
"sample_f16.wgsl#softmax_and_sample": "
|
|
296
|
-
"sample.wgsl#argmax": "
|
|
297
|
-
"sample.wgsl#argmax_reduce": "
|
|
298
|
-
"sample.wgsl#find_topk_phase1": "
|
|
299
|
-
"sample.wgsl#find_topk_phase2": "
|
|
300
|
-
"sample.wgsl#sample_single_pass": "
|
|
301
|
-
"sample.wgsl#softmax_and_sample": "
|
|
290
|
+
"sample_f16.wgsl#argmax": "f43383f5596546e38ebdcf318463c24105c0de03f6350125d1724b0f4a8bdf03",
|
|
291
|
+
"sample_f16.wgsl#argmax_reduce": "5d77c0c647eeb9ff039d8e8f9866e439f4de4bae4c84bebb6cb1b6c890ea661d",
|
|
292
|
+
"sample_f16.wgsl#find_topk_phase1": "bb24e6ef6381193e653ac3aff2eeb14a0b8b376185951c0722428c3a2b466415",
|
|
293
|
+
"sample_f16.wgsl#find_topk_phase2": "5f3ede884117303a27f4b146bdc5c1274fe525684c6ca606268ece2fb093839e",
|
|
294
|
+
"sample_f16.wgsl#sample_single_pass": "7ce553d40f31f3fbe3a0d035e885ab363d9863a006c1478e0f7fe4886bb0972f",
|
|
295
|
+
"sample_f16.wgsl#softmax_and_sample": "418f2661255a9db6b83cc8d111897c427985e5602fd59dae6b326591fce9d8c9",
|
|
296
|
+
"sample.wgsl#argmax": "50af13644f36245b6944468a7c0048ffcfdf5a0e4bf842cac7efcf521b9b3148",
|
|
297
|
+
"sample.wgsl#argmax_reduce": "c197677307ebe8c01e717f6d5fa9e37aaf1dfa38e2b4928257983d558ca5dbea",
|
|
298
|
+
"sample.wgsl#find_topk_phase1": "c6e7d0795682926bc3ffa9eeba1eddfb54667efaf4b58c7214d8ee8214780879",
|
|
299
|
+
"sample.wgsl#find_topk_phase2": "a5eb7235fb97b2208aa921455563ac46bf84faf7fc40e389cc8ee74d0d0f351b",
|
|
300
|
+
"sample.wgsl#sample_single_pass": "85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b",
|
|
301
|
+
"sample.wgsl#softmax_and_sample": "5ec8436e2064d003b7e2e59956758e2f64f2b89cc6cdff49a8c0bf759110bc84",
|
|
302
302
|
"scale_f16.wgsl#main": "4b97fc580eb3f192d89a09986cabf73345a7162957f027024e07589efbb20cd1",
|
|
303
303
|
"scale_f16.wgsl#main_inplace": "9f49a1964d0b8b35b9fe08ea2fcf2307b6dd56ea48339b1762b643159c71ff0a",
|
|
304
304
|
"scale.wgsl#main": "8070894b1488c76bdf6685b99243e0fadd79a2511b52b22b91ba1ead9990831c",
|
|
@@ -8642,11 +8642,11 @@
|
|
|
8642
8642
|
"type": "u32",
|
|
8643
8643
|
"offset": 16
|
|
8644
8644
|
},
|
|
8645
|
-
"
|
|
8646
|
-
"type": "
|
|
8645
|
+
"presence_penalty": {
|
|
8646
|
+
"type": "f32",
|
|
8647
8647
|
"offset": 20
|
|
8648
8648
|
},
|
|
8649
|
-
"
|
|
8649
|
+
"repetition_penalty_window": {
|
|
8650
8650
|
"type": "u32",
|
|
8651
8651
|
"offset": 24
|
|
8652
8652
|
},
|
|
@@ -8683,7 +8683,7 @@
|
|
|
8683
8683
|
}
|
|
8684
8684
|
},
|
|
8685
8685
|
"sample": {
|
|
8686
|
-
"description": "GPU-
|
|
8686
|
+
"description": "Exact GPU top-k/top-p sampling; F16 logits use F32 candidate and probability accumulation",
|
|
8687
8687
|
"baseBindings": [
|
|
8688
8688
|
{
|
|
8689
8689
|
"index": 0,
|
|
@@ -8699,6 +8699,16 @@
|
|
|
8699
8699
|
"index": 2,
|
|
8700
8700
|
"name": "output",
|
|
8701
8701
|
"type": "storage"
|
|
8702
|
+
},
|
|
8703
|
+
{
|
|
8704
|
+
"index": 3,
|
|
8705
|
+
"name": "topk_indices",
|
|
8706
|
+
"type": "storage"
|
|
8707
|
+
},
|
|
8708
|
+
{
|
|
8709
|
+
"index": 4,
|
|
8710
|
+
"name": "topk_logits",
|
|
8711
|
+
"type": "storage"
|
|
8702
8712
|
}
|
|
8703
8713
|
],
|
|
8704
8714
|
"baseUniforms": {
|
|
@@ -8710,24 +8720,39 @@
|
|
|
8710
8720
|
"offset": 0
|
|
8711
8721
|
},
|
|
8712
8722
|
{
|
|
8713
|
-
"name": "
|
|
8714
|
-
"type": "
|
|
8723
|
+
"name": "top_k",
|
|
8724
|
+
"type": "u32",
|
|
8715
8725
|
"offset": 4
|
|
8716
8726
|
},
|
|
8717
8727
|
{
|
|
8718
|
-
"name": "
|
|
8719
|
-
"type": "
|
|
8728
|
+
"name": "temperature",
|
|
8729
|
+
"type": "f32",
|
|
8720
8730
|
"offset": 8
|
|
8721
8731
|
},
|
|
8722
8732
|
{
|
|
8723
|
-
"name": "
|
|
8733
|
+
"name": "random_value",
|
|
8724
8734
|
"type": "f32",
|
|
8725
8735
|
"offset": 12
|
|
8726
8736
|
},
|
|
8727
8737
|
{
|
|
8728
|
-
"name": "
|
|
8738
|
+
"name": "pad_token_id",
|
|
8729
8739
|
"type": "u32",
|
|
8730
8740
|
"offset": 16
|
|
8741
|
+
},
|
|
8742
|
+
{
|
|
8743
|
+
"name": "logit_softcap",
|
|
8744
|
+
"type": "f32",
|
|
8745
|
+
"offset": 20
|
|
8746
|
+
},
|
|
8747
|
+
{
|
|
8748
|
+
"name": "output_index",
|
|
8749
|
+
"type": "u32",
|
|
8750
|
+
"offset": 24
|
|
8751
|
+
},
|
|
8752
|
+
{
|
|
8753
|
+
"name": "top_p",
|
|
8754
|
+
"type": "f32",
|
|
8755
|
+
"offset": 28
|
|
8731
8756
|
}
|
|
8732
8757
|
]
|
|
8733
8758
|
},
|
|
@@ -310,7 +310,7 @@
|
|
|
310
310
|
"gelu": {
|
|
311
311
|
"kernel": "gelu.wgsl",
|
|
312
312
|
"entry": "main",
|
|
313
|
-
"digest": "sha256:
|
|
313
|
+
"digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
|
|
314
314
|
"constants": {
|
|
315
315
|
"HAS_GATE": true
|
|
316
316
|
}
|
|
@@ -370,7 +370,7 @@
|
|
|
370
370
|
"sample": {
|
|
371
371
|
"kernel": "sample.wgsl",
|
|
372
372
|
"entry": "sample_single_pass",
|
|
373
|
-
"digest": "sha256:
|
|
373
|
+
"digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
|
|
374
374
|
}
|
|
375
375
|
},
|
|
376
376
|
"preLayer": [
|
|
@@ -83,7 +83,8 @@ export function buildQualificationRecords(lowered) {
|
|
|
83
83
|
transcriptHash: hashStable(transcript),
|
|
84
84
|
}, ...normalized.qualificationEvidence.map(({ artifact, ...record }) => record)];
|
|
85
85
|
}
|
|
86
|
-
if (normalized.manifest?.modelType === 'embedding'
|
|
86
|
+
if (normalized.manifest?.modelType === 'embedding'
|
|
87
|
+
&& normalized.manifest?.inference?.supportsSequence !== true) {
|
|
87
88
|
throw new Error('Forge requires text embedding qualification for an embedding model; other operation evidence is insufficient.');
|
|
88
89
|
}
|
|
89
90
|
if (transcript?.operation === 'rerank') {
|
|
@@ -337,6 +337,7 @@ export function stageAnalyze(normalized) {
|
|
|
337
337
|
assertModelTopologyRepresentable(manifest);
|
|
338
338
|
const architecture = requireObject(manifest.architecture, 'manifest.architecture');
|
|
339
339
|
const inference = requireObject(manifest.inference, 'manifest.inference');
|
|
340
|
+
const isSequence = inference.supportsSequence === true;
|
|
340
341
|
const attention = requireObject(inference.attention, 'manifest.inference.attention');
|
|
341
342
|
const normalization = requireObject(inference.normalization, 'manifest.inference.normalization');
|
|
342
343
|
const ffn = requireObject(inference.ffn, 'manifest.inference.ffn');
|
|
@@ -399,16 +400,17 @@ export function stageAnalyze(normalized) {
|
|
|
399
400
|
intermediateSize: requirePositiveInteger(architecture.intermediateSize, 'manifest.architecture.intermediateSize'),
|
|
400
401
|
},
|
|
401
402
|
outputTopology: {
|
|
402
|
-
headType:
|
|
403
|
-
:
|
|
403
|
+
headType: isSequence ? 'sequence-encoder'
|
|
404
|
+
: manifest.modelType === 'embedding' ? 'text-embedding' : 'causal-lm',
|
|
404
405
|
tieWeights: output.tieWordEmbeddings === true,
|
|
405
406
|
...(inference.supportsRerank === true
|
|
406
407
|
? { rerank: structuredClone(requireObject(inference.rerank, 'manifest.inference.rerank')) }
|
|
407
408
|
: {}),
|
|
408
|
-
...(
|
|
409
|
+
...(isSequence
|
|
409
410
|
? { sequence: structuredClone(requireObject(inference.sequence, 'manifest.inference.sequence')) }
|
|
410
411
|
: {}),
|
|
411
|
-
...(
|
|
412
|
+
...(!isSequence
|
|
413
|
+
&& (manifest.modelType === 'embedding' || inference.supportsEmbedding === true)
|
|
412
414
|
? { embedding: resolveCapsuleEmbeddingContract(manifest) }
|
|
413
415
|
: {}),
|
|
414
416
|
},
|
|
@@ -18,7 +18,7 @@ export function float32ToFloat16(value) {
|
|
|
18
18
|
|
|
19
19
|
const sign = (f >> 31) & 0x1;
|
|
20
20
|
let exp = (f >> 23) & 0xff;
|
|
21
|
-
|
|
21
|
+
const frac = f & 0x7fffff;
|
|
22
22
|
|
|
23
23
|
if (exp === 0xff) {
|
|
24
24
|
return (sign << 15) | 0x7c00 | (frac ? 0x200 : 0);
|
|
@@ -38,11 +38,25 @@ export function float32ToFloat16(value) {
|
|
|
38
38
|
if (exp < -10) {
|
|
39
39
|
return sign << 15;
|
|
40
40
|
}
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
const significand = frac | 0x800000;
|
|
42
|
+
const shift = 14 - exp;
|
|
43
|
+
let rounded = significand >>> shift;
|
|
44
|
+
const remainder = significand & ((1 << shift) - 1);
|
|
45
|
+
const halfway = 1 << (shift - 1);
|
|
46
|
+
if (remainder > halfway || (remainder === halfway && (rounded & 1))) {
|
|
47
|
+
rounded += 1;
|
|
48
|
+
}
|
|
49
|
+
return (sign << 15) | rounded;
|
|
43
50
|
}
|
|
44
51
|
|
|
45
|
-
|
|
52
|
+
// Round to nearest, ties to even. Carry may advance the exponent or overflow
|
|
53
|
+
// the largest finite half to infinity; truncation biases converted weights.
|
|
54
|
+
let rounded = (exp << 10) | (frac >>> 13);
|
|
55
|
+
const remainder = frac & 0x1fff;
|
|
56
|
+
if (remainder > 0x1000 || (remainder === 0x1000 && (rounded & 1))) {
|
|
57
|
+
rounded += 1;
|
|
58
|
+
}
|
|
59
|
+
return (sign << 15) | rounded;
|
|
46
60
|
}
|
|
47
61
|
|
|
48
62
|
export function float16ToFloat32(h) {
|
package/src/gpu/device-state.js
CHANGED
|
@@ -6,6 +6,9 @@ export function getSharedDeviceState() {
|
|
|
6
6
|
if (!(existing.bufferOwners instanceof WeakMap)) {
|
|
7
7
|
existing.bufferOwners = new WeakMap();
|
|
8
8
|
}
|
|
9
|
+
if (!(existing.lostDevices instanceof WeakSet)) {
|
|
10
|
+
existing.lostDevices = new WeakSet();
|
|
11
|
+
}
|
|
9
12
|
if (!('deviceInitPromise' in existing)) {
|
|
10
13
|
existing.deviceInitPromise = null;
|
|
11
14
|
}
|
|
@@ -19,6 +22,7 @@ export function getSharedDeviceState() {
|
|
|
19
22
|
platformInitialized: false,
|
|
20
23
|
deviceEpoch: 0,
|
|
21
24
|
bufferOwners: new WeakMap(),
|
|
25
|
+
lostDevices: new WeakSet(),
|
|
22
26
|
deviceInitPromise: null,
|
|
23
27
|
};
|
|
24
28
|
Object.defineProperty(globalThis, SHARED_DEVICE_STATE_KEY, {
|
|
@@ -34,3 +38,7 @@ export function getSharedDeviceEpoch() {
|
|
|
34
38
|
const epoch = getSharedDeviceState().deviceEpoch;
|
|
35
39
|
return Number.isInteger(epoch) ? epoch : 0;
|
|
36
40
|
}
|
|
41
|
+
|
|
42
|
+
export function isDeviceLost(device) {
|
|
43
|
+
return device != null && getSharedDeviceState().lostDevices.has(device);
|
|
44
|
+
}
|
package/src/gpu/device.js
CHANGED
|
@@ -5,7 +5,7 @@ import { probeSubmitLatency } from './submit-probe.js';
|
|
|
5
5
|
import { log } from '../debug/index.js';
|
|
6
6
|
import { createDopplerError, ERROR_CODES } from '../errors/doppler-error.js';
|
|
7
7
|
import { GB } from '../config/schema/index.js';
|
|
8
|
-
import { getSharedDeviceState } from './device-state.js';
|
|
8
|
+
import { getSharedDeviceState, isDeviceLost } from './device-state.js';
|
|
9
9
|
|
|
10
10
|
// Re-export submit tracker for convenience
|
|
11
11
|
export { setTrackSubmits };
|
|
@@ -143,6 +143,7 @@ function isValidGPUBuffer(value) {
|
|
|
143
143
|
function isUsableGPUDevice(device) {
|
|
144
144
|
return !!(
|
|
145
145
|
device
|
|
146
|
+
&& !isDeviceLost(device)
|
|
146
147
|
&& typeof device.createBuffer === 'function'
|
|
147
148
|
&& typeof device.createBindGroup === 'function'
|
|
148
149
|
&& typeof device.createCommandEncoder === 'function'
|
|
@@ -255,6 +256,7 @@ function registerDeviceLostHandler(device) {
|
|
|
255
256
|
if (device.lost && typeof device.lost.then === 'function') {
|
|
256
257
|
const trackedDevice = device;
|
|
257
258
|
device.lost.then((info) => {
|
|
259
|
+
sharedDeviceState.lostDevices.add(trackedDevice);
|
|
258
260
|
if (sharedDeviceState.gpuDevice !== trackedDevice) {
|
|
259
261
|
return;
|
|
260
262
|
}
|
|
@@ -270,6 +272,7 @@ function registerDeviceLostHandler(device) {
|
|
|
270
272
|
clearActiveDeviceState();
|
|
271
273
|
advanceDeviceEpoch();
|
|
272
274
|
}).catch((error) => {
|
|
275
|
+
sharedDeviceState.lostDevices.add(trackedDevice);
|
|
273
276
|
if (sharedDeviceState.gpuDevice !== trackedDevice) {
|
|
274
277
|
return;
|
|
275
278
|
}
|
|
@@ -600,6 +603,7 @@ export async function initDevice() {
|
|
|
600
603
|
|
|
601
604
|
export function setDevice(device, options = {}) {
|
|
602
605
|
hydrateDeviceState();
|
|
606
|
+
if (isDeviceLost(device)) throw new Error('Cannot bind a lost GPU device. Explicitly initialize a replacement device.');
|
|
603
607
|
if (!device) {
|
|
604
608
|
clearActiveDeviceState();
|
|
605
609
|
advanceDeviceEpoch();
|
|
@@ -694,6 +698,7 @@ export function resetDeviceState() {
|
|
|
694
698
|
export function destroyDevice() {
|
|
695
699
|
hydrateDeviceState();
|
|
696
700
|
if (gpuDevice) {
|
|
701
|
+
sharedDeviceState.lostDevices.add(gpuDevice);
|
|
697
702
|
gpuDevice.destroy();
|
|
698
703
|
clearActiveDeviceState();
|
|
699
704
|
advanceDeviceEpoch();
|
|
@@ -7,11 +7,15 @@
|
|
|
7
7
|
import type { Tensor } from '../tensor.js';
|
|
8
8
|
import type { CommandRecorder } from '../command-recorder.js';
|
|
9
9
|
import type { OutputBufferOptions } from './types.js';
|
|
10
|
+
import type { KernelPathSchema } from '../../config/schema/kernel-path.schema.js';
|
|
10
11
|
|
|
11
12
|
/** GeLU kernel options */
|
|
12
13
|
export interface GeLUOptions extends OutputBufferOptions {
|
|
13
14
|
size?: number | null;
|
|
14
15
|
gate?: Tensor | null;
|
|
16
|
+
kernelPath?: KernelPathSchema | null;
|
|
17
|
+
phase?: 'prefill' | 'decode';
|
|
18
|
+
layerIdx?: number;
|
|
15
19
|
}
|
|
16
20
|
|
|
17
21
|
/**
|
package/src/gpu/kernels/gelu.js
CHANGED
|
@@ -4,6 +4,7 @@ import { createTensor, dtypeBytes } from '../tensor.js';
|
|
|
4
4
|
import { WORKGROUP_SIZES } from './constants.js';
|
|
5
5
|
import { unifiedKernelWrapper } from './kernel-execution.js';
|
|
6
6
|
import { selectRuleValue } from './rule-registry.js';
|
|
7
|
+
import { getKernelPathActivationSpec } from '../../config/kernel-path-loader.js';
|
|
7
8
|
|
|
8
9
|
function selectGeluVariant(isF16) {
|
|
9
10
|
return selectRuleValue('gelu', 'variant', { isF16 });
|
|
@@ -20,7 +21,16 @@ async function _gelu(target, input, options = {}) {
|
|
|
20
21
|
const isF16 = input.dtype === 'f16';
|
|
21
22
|
const bytesPerElement = dtypeBytes(input.dtype);
|
|
22
23
|
const variant = selectGeluVariant(isF16);
|
|
23
|
-
const
|
|
24
|
+
const declared = getKernelPathActivationSpec('gelu', options.phase, options.layerIdx, options.kernelPath);
|
|
25
|
+
if (declared && declared.variant !== variant) {
|
|
26
|
+
throw new Error(`GeLU input ${input.dtype} does not match declared kernel ${declared.variant}.`);
|
|
27
|
+
}
|
|
28
|
+
for (const [key, actual] of Object.entries({ HAS_GATE: Boolean(gate), USE_ROWSPLIT: false })) {
|
|
29
|
+
if (declared?.constants?.[key] != null && Number(declared.constants[key]) !== Number(actual)) {
|
|
30
|
+
throw new Error(`GeLU ${key} disagrees with the declared execution step.`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const overrides = { ...resolveOverrides({ hasGate: Boolean(gate), useRowsplit: false }), ...declared?.constants };
|
|
24
34
|
|
|
25
35
|
const inferredSize = size || (input.buffer.size / bytesPerElement);
|
|
26
36
|
const outputSize = inferredSize * bytesPerElement;
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
9
9
|
override HAS_GATE: bool = false;
|
|
10
10
|
override USE_ROWSPLIT: bool = false;
|
|
11
|
+
override GELU_ERF: bool = false;
|
|
11
12
|
|
|
12
13
|
struct Uniforms {
|
|
13
14
|
size: u32, // Total output elements
|
|
@@ -21,7 +22,22 @@ struct Uniforms {
|
|
|
21
22
|
@group(0) @binding(2) var<storage, read_write> output: array<f32>;
|
|
22
23
|
@group(0) @binding(3) var<storage, read> gate: array<f32>;
|
|
23
24
|
|
|
25
|
+
// Source erf GELU, evaluated in f32 even for f16 storage. The polynomial
|
|
26
|
+
// approximates erf rather than substituting the distinct tanh GELU function.
|
|
27
|
+
fn erf_gelu(x: f32) -> f32 {
|
|
28
|
+
let z = abs(x) * 0.7071067811865476;
|
|
29
|
+
let t = 1.0 / (1.0 + 0.3275911 * z);
|
|
30
|
+
let polynomial = (((((1.061405429 * t - 1.453152027) * t)
|
|
31
|
+
+ 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
|
|
32
|
+
let complement = polynomial * exp(-z * z);
|
|
33
|
+
let cdf_twice = select(complement, 2.0 - complement, x >= 0.0);
|
|
34
|
+
return 0.5 * x * cdf_twice;
|
|
35
|
+
}
|
|
36
|
+
|
|
24
37
|
fn gelu(x: f32) -> f32 {
|
|
38
|
+
if (GELU_ERF) {
|
|
39
|
+
return erf_gelu(x);
|
|
40
|
+
}
|
|
25
41
|
let sqrt_2_over_pi: f32 = 0.7978845608;
|
|
26
42
|
let c: f32 = 0.044715;
|
|
27
43
|
let inner = sqrt_2_over_pi * (x + c * x * x * x);
|
|
@@ -9,6 +9,7 @@ enable f16;
|
|
|
9
9
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
10
10
|
override HAS_GATE: bool = false;
|
|
11
11
|
override USE_ROWSPLIT: bool = false;
|
|
12
|
+
override GELU_ERF: bool = false;
|
|
12
13
|
|
|
13
14
|
struct Uniforms {
|
|
14
15
|
size: u32, // Total output elements
|
|
@@ -22,7 +23,22 @@ struct Uniforms {
|
|
|
22
23
|
@group(0) @binding(2) var<storage, read_write> output: array<f16>;
|
|
23
24
|
@group(0) @binding(3) var<storage, read> gate: array<f16>;
|
|
24
25
|
|
|
26
|
+
// Source erf GELU, evaluated in f32 even for f16 storage. The polynomial
|
|
27
|
+
// approximates erf rather than substituting the distinct tanh GELU function.
|
|
28
|
+
fn erf_gelu(x: f32) -> f32 {
|
|
29
|
+
let z = abs(x) * 0.7071067811865476;
|
|
30
|
+
let t = 1.0 / (1.0 + 0.3275911 * z);
|
|
31
|
+
let polynomial = (((((1.061405429 * t - 1.453152027) * t)
|
|
32
|
+
+ 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
|
|
33
|
+
let complement = polynomial * exp(-z * z);
|
|
34
|
+
let cdf_twice = select(complement, 2.0 - complement, x >= 0.0);
|
|
35
|
+
return 0.5 * x * cdf_twice;
|
|
36
|
+
}
|
|
37
|
+
|
|
25
38
|
fn gelu(x: f16) -> f16 {
|
|
39
|
+
if (GELU_ERF) {
|
|
40
|
+
return f16(erf_gelu(f32(x)));
|
|
41
|
+
}
|
|
26
42
|
let sqrt_2_over_pi = f16(0.7978845608);
|
|
27
43
|
let c = f16(0.044715);
|
|
28
44
|
let inner = sqrt_2_over_pi * (x + c * x * x * x);
|
|
@@ -3,6 +3,24 @@ import { createPipeline, getOrCreateBindGroupLayout } from './pipeline-cache.js'
|
|
|
3
3
|
import { createUniformBufferWithView } from './uniform-utils.js';
|
|
4
4
|
import { WORKGROUP_SIZES } from './constants.js';
|
|
5
5
|
import { selectRuleValue } from './rule-registry.js';
|
|
6
|
+
import { validateGenerationField } from '../../config/generation-contract.js';
|
|
7
|
+
import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
8
|
+
|
|
9
|
+
export async function recordHistoryPenalties(recorder, logitsBuffer, tokenIds, options) {
|
|
10
|
+
if (options.repetitionPenalty === 1 && options.presencePenalty === 0) return;
|
|
11
|
+
const history = Uint32Array.from(tokenIds.slice(-options.repetitionPenaltyWindow));
|
|
12
|
+
const buffer = acquireBuffer(Math.max(4, history.byteLength), undefined, 'sampling_penalty_history');
|
|
13
|
+
let transferred = false;
|
|
14
|
+
try {
|
|
15
|
+
recorder.trackTemporaryBuffer(buffer);
|
|
16
|
+
transferred = true;
|
|
17
|
+
if (history.byteLength) recorder.device.queue.writeBuffer(buffer, 0, history);
|
|
18
|
+
await recordRepPenalty(recorder, logitsBuffer, buffer, buffer, {
|
|
19
|
+
...options, penalty: options.repetitionPenalty, historyCount: history.length,
|
|
20
|
+
batchCount: 0, batchOffset: 0,
|
|
21
|
+
});
|
|
22
|
+
} finally { if (!transferred) releaseBuffer(buffer); }
|
|
23
|
+
}
|
|
6
24
|
|
|
7
25
|
|
|
8
26
|
function getRepPenaltyBindGroupLayout(device) {
|
|
@@ -32,8 +50,8 @@ function createRepPenaltyUniformBuffer(device, recorder, options) {
|
|
|
32
50
|
view.setFloat32(8, options.penalty, true);
|
|
33
51
|
view.setUint32(12, options.batchCount, true);
|
|
34
52
|
view.setUint32(16, options.batchOffset, true);
|
|
35
|
-
view.
|
|
36
|
-
view.setUint32(24,
|
|
53
|
+
view.setFloat32(20, options.presencePenalty, true);
|
|
54
|
+
view.setUint32(24, options.repetitionPenaltyWindow, true);
|
|
37
55
|
view.setUint32(28, 0, true);
|
|
38
56
|
},
|
|
39
57
|
recorder,
|
|
@@ -52,22 +70,27 @@ export async function recordRepPenalty(
|
|
|
52
70
|
vocabSize,
|
|
53
71
|
historyCount,
|
|
54
72
|
penalty,
|
|
73
|
+
presencePenalty,
|
|
74
|
+
repetitionPenaltyWindow,
|
|
55
75
|
batchCount,
|
|
56
76
|
batchOffset,
|
|
57
77
|
logitsDtype,
|
|
58
78
|
} = options;
|
|
59
79
|
|
|
60
|
-
|
|
80
|
+
validateGenerationField('repetitionPenalty', penalty);
|
|
81
|
+
validateGenerationField('presencePenalty', presencePenalty);
|
|
82
|
+
validateGenerationField('repetitionPenaltyWindow', repetitionPenaltyWindow);
|
|
83
|
+
if (!Number.isSafeInteger(batchCount) || batchCount < 0) {
|
|
61
84
|
throw new Error('[RepPenalty] batchCount is required and must be non-negative.');
|
|
62
85
|
}
|
|
63
|
-
if (!Number.
|
|
86
|
+
if (!Number.isSafeInteger(batchOffset) || batchOffset < 0) {
|
|
64
87
|
throw new Error('[RepPenalty] batchOffset is required and must be non-negative.');
|
|
65
88
|
}
|
|
66
89
|
if (logitsDtype !== 'f16' && logitsDtype !== 'f32') {
|
|
67
90
|
throw new Error('[RepPenalty] logitsDtype must be "f16" or "f32".');
|
|
68
91
|
}
|
|
69
92
|
|
|
70
|
-
if (penalty === 1.0 || (historyCount === 0 && batchCount === 0)) {
|
|
93
|
+
if ((penalty === 1.0 && presencePenalty === 0) || (historyCount === 0 && batchCount === 0)) {
|
|
71
94
|
return;
|
|
72
95
|
}
|
|
73
96
|
|
|
@@ -85,6 +108,8 @@ export async function recordRepPenalty(
|
|
|
85
108
|
vocabSize,
|
|
86
109
|
historyCount,
|
|
87
110
|
penalty,
|
|
111
|
+
presencePenalty,
|
|
112
|
+
repetitionPenaltyWindow,
|
|
88
113
|
batchCount,
|
|
89
114
|
batchOffset,
|
|
90
115
|
});
|
|
@@ -1,14 +1,5 @@
|
|
|
1
|
-
// Repetition
|
|
2
|
-
//
|
|
3
|
-
// Applies repetition penalty to logits in-place before sampling.
|
|
4
|
-
// Reads unique history token IDs (pre-deduplicated on CPU) and
|
|
5
|
-
// current-batch tokens from separate buffers.
|
|
6
|
-
//
|
|
7
|
-
// For each token ID: logit > 0 ? logit / penalty : logit * penalty
|
|
8
|
-
// Matches CPU applyRepetitionPenalty() behavior.
|
|
9
|
-
//
|
|
10
|
-
// One thread per token in (history + batch). Benign data race when the same
|
|
11
|
-
// token appears in both buffers: both threads compute identical results.
|
|
1
|
+
// Repetition followed by presence penalty over the ordered recent context.
|
|
2
|
+
// Only the first occurrence inside the window writes a token's logit.
|
|
12
3
|
|
|
13
4
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
14
5
|
|
|
@@ -17,9 +8,9 @@ struct Uniforms {
|
|
|
17
8
|
history_count: u32,
|
|
18
9
|
penalty: f32,
|
|
19
10
|
batch_count: u32,
|
|
20
|
-
batch_offset: u32,
|
|
21
|
-
|
|
22
|
-
|
|
11
|
+
batch_offset: u32,
|
|
12
|
+
presence_penalty: f32,
|
|
13
|
+
repetition_penalty_window: u32,
|
|
23
14
|
_pad2: u32,
|
|
24
15
|
}
|
|
25
16
|
|
|
@@ -28,30 +19,34 @@ struct Uniforms {
|
|
|
28
19
|
@group(0) @binding(2) var<storage, read> history: array<u32>;
|
|
29
20
|
@group(0) @binding(3) var<storage, read> batch_tokens: array<u32>;
|
|
30
21
|
|
|
22
|
+
fn context_token(idx: u32) -> u32 {
|
|
23
|
+
if (idx < u.history_count) {
|
|
24
|
+
return history[idx];
|
|
25
|
+
}
|
|
26
|
+
return batch_tokens[u.batch_offset + idx - u.history_count];
|
|
27
|
+
}
|
|
28
|
+
|
|
31
29
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
32
30
|
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
33
31
|
let idx = gid.x;
|
|
34
32
|
let total = u.history_count + u.batch_count;
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
var start = 0u;
|
|
34
|
+
if (u.repetition_penalty_window > 0u) {
|
|
35
|
+
start = total - min(total, u.repetition_penalty_window);
|
|
37
36
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
if (idx < u.history_count) {
|
|
41
|
-
token_id = history[idx];
|
|
42
|
-
} else {
|
|
43
|
-
token_id = batch_tokens[u.batch_offset + (idx - u.history_count)];
|
|
37
|
+
if (idx < start || idx >= total) {
|
|
38
|
+
return;
|
|
44
39
|
}
|
|
45
|
-
|
|
40
|
+
let token_id = context_token(idx);
|
|
46
41
|
if (token_id >= u.vocab_size) {
|
|
47
42
|
return;
|
|
48
43
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
logits[token_id] = logit / penalty;
|
|
54
|
-
} else {
|
|
55
|
-
logits[token_id] = logit * penalty;
|
|
44
|
+
for (var prior = start; prior < idx; prior++) {
|
|
45
|
+
if (context_token(prior) == token_id) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
56
48
|
}
|
|
49
|
+
let logit = f32(logits[token_id]);
|
|
50
|
+
let repeated = select(logit * u.penalty, logit / u.penalty, logit > 0.0);
|
|
51
|
+
logits[token_id] = f32(repeated - u.presence_penalty);
|
|
57
52
|
}
|
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
// Repetition
|
|
2
|
-
//
|
|
3
|
-
// F16 variant for models with f16 activations. Reads/writes f16 logits,
|
|
4
|
-
// computes penalty arithmetic in f32 for precision.
|
|
5
|
-
//
|
|
6
|
-
// For each token ID: logit > 0 ? logit / penalty : logit * penalty
|
|
7
|
-
|
|
1
|
+
// Repetition followed by presence penalty over the ordered recent context.
|
|
2
|
+
// Only the first occurrence inside the window writes a token's logit.
|
|
8
3
|
enable f16;
|
|
9
4
|
|
|
10
5
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
@@ -15,8 +10,8 @@ struct Uniforms {
|
|
|
15
10
|
penalty: f32,
|
|
16
11
|
batch_count: u32,
|
|
17
12
|
batch_offset: u32,
|
|
18
|
-
|
|
19
|
-
|
|
13
|
+
presence_penalty: f32,
|
|
14
|
+
repetition_penalty_window: u32,
|
|
20
15
|
_pad2: u32,
|
|
21
16
|
}
|
|
22
17
|
|
|
@@ -25,30 +20,34 @@ struct Uniforms {
|
|
|
25
20
|
@group(0) @binding(2) var<storage, read> history: array<u32>;
|
|
26
21
|
@group(0) @binding(3) var<storage, read> batch_tokens: array<u32>;
|
|
27
22
|
|
|
23
|
+
fn context_token(idx: u32) -> u32 {
|
|
24
|
+
if (idx < u.history_count) {
|
|
25
|
+
return history[idx];
|
|
26
|
+
}
|
|
27
|
+
return batch_tokens[u.batch_offset + idx - u.history_count];
|
|
28
|
+
}
|
|
29
|
+
|
|
28
30
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
29
31
|
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
30
32
|
let idx = gid.x;
|
|
31
33
|
let total = u.history_count + u.batch_count;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
+
var start = 0u;
|
|
35
|
+
if (u.repetition_penalty_window > 0u) {
|
|
36
|
+
start = total - min(total, u.repetition_penalty_window);
|
|
34
37
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
if (idx < u.history_count) {
|
|
38
|
-
token_id = history[idx];
|
|
39
|
-
} else {
|
|
40
|
-
token_id = batch_tokens[u.batch_offset + (idx - u.history_count)];
|
|
38
|
+
if (idx < start || idx >= total) {
|
|
39
|
+
return;
|
|
41
40
|
}
|
|
42
|
-
|
|
41
|
+
let token_id = context_token(idx);
|
|
43
42
|
if (token_id >= u.vocab_size) {
|
|
44
43
|
return;
|
|
45
44
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
logits[token_id] = f16(logit / penalty);
|
|
51
|
-
} else {
|
|
52
|
-
logits[token_id] = f16(logit * penalty);
|
|
45
|
+
for (var prior = start; prior < idx; prior++) {
|
|
46
|
+
if (context_token(prior) == token_id) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
53
49
|
}
|
|
50
|
+
let logit = f32(logits[token_id]);
|
|
51
|
+
let repeated = select(logit * u.penalty, logit / u.penalty, logit > 0.0);
|
|
52
|
+
logits[token_id] = f16(repeated - u.presence_penalty);
|
|
54
53
|
}
|