doppler-gpu 0.4.15 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +48 -1
- package/src/client/runtime/model-session.js +149 -30
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -1,1278 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
recordLayerNorm,
|
|
10
|
-
recordBiasAdd,
|
|
11
|
-
recordRoPE,
|
|
12
|
-
canUseRoPEQK,
|
|
13
|
-
recordRoPEQK,
|
|
14
|
-
recordAttention,
|
|
15
|
-
recordAttentionTiered,
|
|
16
|
-
recordAttentionTieredQuant,
|
|
17
|
-
recordAttentionContiguousQuant,
|
|
18
|
-
recordAttentionBDPA,
|
|
19
|
-
recordSiLU,
|
|
20
|
-
recordCastF16ToF32,
|
|
21
|
-
recordCastF32ToF16,
|
|
22
|
-
recordMatmulResidualFused,
|
|
23
|
-
shouldUseFusedMatmulResidual,
|
|
24
|
-
} from '../../../../gpu/kernel-selector.js';
|
|
25
|
-
import { createTensor } from '../../../../gpu/tensor.js';
|
|
26
|
-
import { applyLoRA } from '../lora-apply.js';
|
|
27
|
-
import { getLoRAModule } from '../lora.js';
|
|
28
|
-
import { log, trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
29
|
-
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
30
|
-
import {
|
|
31
|
-
recordAttentionInputs,
|
|
32
|
-
shouldForceF32AttentionProjectionForRoPE,
|
|
33
|
-
resolveAttentionProjectionOutputDtype,
|
|
34
|
-
projectAttentionQKV,
|
|
35
|
-
applyAttentionQKNorm,
|
|
36
|
-
applyAttentionValueNorm,
|
|
37
|
-
hasAttentionProjectionDiagnostics,
|
|
38
|
-
hasAttentionStageDiagnostics,
|
|
39
|
-
resolveAttentionQKNormState,
|
|
40
|
-
} from './projections.js';
|
|
41
|
-
import { prepareAttentionProjectionInput } from './output-projection.js';
|
|
42
|
-
import { runProbes } from '../probes.js';
|
|
43
|
-
import { decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
|
|
44
|
-
|
|
45
|
-
import { releaseOrTrack, shouldDebugLayer } from './types.js';
|
|
46
|
-
import {
|
|
47
|
-
getKernelPathMatmulPrecision,
|
|
48
|
-
getKernelPathMatmulVariant,
|
|
49
|
-
} from '../../../../config/kernel-path-loader.js';
|
|
50
|
-
import {
|
|
51
|
-
resolveKVCacheState,
|
|
52
|
-
createDiffusionGemmaDecoderKVState,
|
|
53
|
-
buildAttentionDispatchParams,
|
|
54
|
-
buildAttentionInputsData,
|
|
55
|
-
} from './dispatch-params.js';
|
|
56
|
-
import {
|
|
57
|
-
buildTieredQuantAttentionOptions,
|
|
58
|
-
buildContiguousQuantAttentionOptions,
|
|
59
|
-
} from './quant-options.js';
|
|
60
|
-
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
61
|
-
import { getRuntimeConfig } from '../../../../config/runtime.js';
|
|
62
|
-
import {
|
|
63
|
-
resolveAttentionPrecisionContract,
|
|
64
|
-
isAttentionKvDtypeExplicit,
|
|
65
|
-
} from './precision-contract.js';
|
|
66
|
-
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
67
|
-
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
68
|
-
import { getVectorTensor } from '../weights.js';
|
|
69
|
-
|
|
70
|
-
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
71
|
-
|
|
72
|
-
function isWideTileQ4KPhaseEnabled(session, phase) {
|
|
73
|
-
return phase === 'decode'
|
|
74
|
-
? session?.useWideTileQ4KDecode === true
|
|
75
|
-
: session?.useWideTileQ4KPrefill === true;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
function canUseQ4KWideTileResidualFusion(options = {}) {
|
|
79
|
-
return options.hasResidual === true
|
|
80
|
-
&& options.oProjDtype === 'q4k'
|
|
81
|
-
&& options.inputDtype === 'f32'
|
|
82
|
-
&& options.outputDtype === 'f32'
|
|
83
|
-
&& options.residualMatches === true
|
|
84
|
-
&& options.hasLoRA !== true
|
|
85
|
-
&& getKernelCapabilities().hasF16 === true
|
|
86
|
-
&& getKernelCapabilities().hasSubgroups === true
|
|
87
|
-
&& options.session?.useWideTileResidualFusion === true
|
|
88
|
-
&& options.session?.retainQ4KMaterialization === true
|
|
89
|
-
&& isWideTileQ4KPhaseEnabled(options.session, options.phase);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
function shouldTraceRecordedHealth(layerIdx, debugFlags) {
|
|
93
|
-
const debugLayers = debugFlags?.debugLayers;
|
|
94
|
-
return isTraceEnabled('logits')
|
|
95
|
-
&& Array.isArray(debugLayers)
|
|
96
|
-
&& shouldDebugLayer(layerIdx, debugLayers);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
function enqueueRecordedTensorHealth(recorder, label, tensor, dtype, elementCount) {
|
|
100
|
-
if (!recorder || !tensor?.buffer || !Number.isFinite(elementCount) || elementCount <= 0) {
|
|
101
|
-
return;
|
|
102
|
-
}
|
|
103
|
-
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
104
|
-
recorder.enqueueCompletionTask(async () => {
|
|
105
|
-
const data = await readBuffer(tensor.buffer, elementCount * bytesPerElement);
|
|
106
|
-
trace.logits(label, getLogitsHealth(decodeReadback(data, dtype)));
|
|
107
|
-
});
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
function resolveDirectF16KVCacheWrite(options) {
|
|
111
|
-
const {
|
|
112
|
-
state,
|
|
113
|
-
layerIdx,
|
|
114
|
-
currentSeqLen,
|
|
115
|
-
numTokens,
|
|
116
|
-
numKVHeads,
|
|
117
|
-
headDim,
|
|
118
|
-
reusesSharedKV,
|
|
119
|
-
storeSharedKV,
|
|
120
|
-
diffusionGemmaDecoder,
|
|
121
|
-
valueNorm,
|
|
122
|
-
diagnosticsEnabled,
|
|
123
|
-
disableRoPE,
|
|
124
|
-
} = options;
|
|
125
|
-
if (
|
|
126
|
-
numTokens !== 1
|
|
127
|
-
|| reusesSharedKV === true
|
|
128
|
-
|| storeSharedKV === true
|
|
129
|
-
|| diffusionGemmaDecoder === true
|
|
130
|
-
|| valueNorm === true
|
|
131
|
-
|| diagnosticsEnabled === true
|
|
132
|
-
|| disableRoPE === true
|
|
133
|
-
) {
|
|
134
|
-
return null;
|
|
135
|
-
}
|
|
136
|
-
const cache = state?.kvCache;
|
|
137
|
-
if (
|
|
138
|
-
!cache?.hasGPUCache?.()
|
|
139
|
-
|| cache.kvDtype !== 'f16'
|
|
140
|
-
|| cache.layout !== 'contiguous'
|
|
141
|
-
|| cache.windowSize != null
|
|
142
|
-
|| cache.layerSpecs != null
|
|
143
|
-
|| typeof cache.recordF16UpdateAlreadyWrittenFromGPU !== 'function'
|
|
144
|
-
) {
|
|
145
|
-
return null;
|
|
146
|
-
}
|
|
147
|
-
const gpuBuffers = cache.getGPUBuffers(layerIdx);
|
|
148
|
-
const layout = gpuBuffers?.layout ?? cache.layout;
|
|
149
|
-
if (layout !== 'contiguous' || !gpuBuffers?.keysGPU || !gpuBuffers?.valuesGPU) {
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
152
|
-
return {
|
|
153
|
-
keysBuffer: gpuBuffers.keysGPU,
|
|
154
|
-
valuesBuffer: gpuBuffers.valuesGPU,
|
|
155
|
-
dstOffset: currentSeqLen * numKVHeads * headDim,
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
|
|
160
|
-
assertImplicitDtypeTransitionAllowed({
|
|
161
|
-
executionPolicies: state?.executionPolicies ?? null,
|
|
162
|
-
fromDtype,
|
|
163
|
-
toDtype,
|
|
164
|
-
op: 'attention',
|
|
165
|
-
detail,
|
|
166
|
-
transitionDeclaredBy,
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
export async function recordLayerAttentionGPU(
|
|
172
|
-
recorder,
|
|
173
|
-
input,
|
|
174
|
-
layerWeights,
|
|
175
|
-
config,
|
|
176
|
-
state,
|
|
177
|
-
debug = false,
|
|
178
|
-
debugFlags = {},
|
|
179
|
-
getWeightBuffer,
|
|
180
|
-
getNormWeightBuffer,
|
|
181
|
-
debugCheckBuffer,
|
|
182
|
-
lora
|
|
183
|
-
) {
|
|
184
|
-
const {
|
|
185
|
-
layerIdx,
|
|
186
|
-
numTokens,
|
|
187
|
-
isPrefill,
|
|
188
|
-
numHeads,
|
|
189
|
-
numKVHeads,
|
|
190
|
-
headDim,
|
|
191
|
-
hiddenSize,
|
|
192
|
-
rmsNormEps,
|
|
193
|
-
currentSeqLen,
|
|
194
|
-
slidingWindow,
|
|
195
|
-
layerType,
|
|
196
|
-
residualTensor,
|
|
197
|
-
attnSoftcap,
|
|
198
|
-
queryPreAttnScalar,
|
|
199
|
-
skipInputNorm = false,
|
|
200
|
-
tokenIds = null,
|
|
201
|
-
kernelPath = null,
|
|
202
|
-
disableRoPE = false,
|
|
203
|
-
multimodalBidirectionalSpan = null,
|
|
204
|
-
sharedKVSourceLayerIdx = null,
|
|
205
|
-
storeSharedKV = false,
|
|
206
|
-
diffusionGemmaDecoder = false,
|
|
207
|
-
precomputedInputNorm = null,
|
|
208
|
-
} = config;
|
|
209
|
-
|
|
210
|
-
const phase = isPrefill ? 'prefill' : 'decode';
|
|
211
|
-
const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
|
|
212
|
-
const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
|
|
213
|
-
dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
|
|
214
|
-
});
|
|
215
|
-
const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
|
|
216
|
-
const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
|
|
217
|
-
dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
|
|
218
|
-
});
|
|
219
|
-
const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
|
|
220
|
-
dtype: oProjPrecision?.outputDtype
|
|
221
|
-
?? attentionPrecisionContract.resolvedOutputDtype
|
|
222
|
-
?? config.activationDtype,
|
|
223
|
-
});
|
|
224
|
-
const wantsF16Output = oProjOutputDtype === 'f16';
|
|
225
|
-
const useF16Activations = attentionActivationDtype === 'f16';
|
|
226
|
-
const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
|
|
227
|
-
const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
|
|
228
|
-
const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
|
|
229
|
-
const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
|
|
230
|
-
? 'step_precision'
|
|
231
|
-
: null;
|
|
232
|
-
let attentionInput = input;
|
|
233
|
-
let attentionInputTemp = false;
|
|
234
|
-
let normed = attentionInput;
|
|
235
|
-
let qTensor = null;
|
|
236
|
-
let qGateTensor = null;
|
|
237
|
-
let kTensor = null;
|
|
238
|
-
let vTensor = null;
|
|
239
|
-
let attnOutput = null;
|
|
240
|
-
let attnForProjection = null;
|
|
241
|
-
let output = null;
|
|
242
|
-
let finalOutput = null;
|
|
243
|
-
let oProjInputTemp = null;
|
|
244
|
-
let retainSharedKvBuffers = false;
|
|
245
|
-
let valueAliasesKey = false;
|
|
246
|
-
let decoderKVState = null;
|
|
247
|
-
let decoderKTemp = null;
|
|
248
|
-
let decoderVTemp = null;
|
|
249
|
-
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
250
|
-
assertAttentionDtypeTransitionAllowed(
|
|
251
|
-
state,
|
|
252
|
-
input.dtype,
|
|
253
|
-
attentionActivationDtype,
|
|
254
|
-
'The attention kernel selection would widen the input implicitly.',
|
|
255
|
-
attentionInputTransitionDeclaredBy
|
|
256
|
-
);
|
|
257
|
-
attentionInput = attentionActivationDtype === 'f16'
|
|
258
|
-
? await recordCastF32ToF16(recorder, input)
|
|
259
|
-
: await recordCastF16ToF32(recorder, input);
|
|
260
|
-
attentionInputTemp = true;
|
|
261
|
-
normed = attentionInput;
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
if (!layerWeights) {
|
|
265
|
-
const bytesPerElement = wantsF16Output ? 2 : 4;
|
|
266
|
-
const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
|
|
267
|
-
const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
|
|
268
|
-
return { output, residualFused: false };
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
const qSize = numTokens * numHeads * headDim;
|
|
272
|
-
const kvSize = numTokens * numKVHeads * headDim;
|
|
273
|
-
|
|
274
|
-
// 1. Input norm
|
|
275
|
-
// Opt-in fusion: when useFusedRmsnormWideTile is set, defer the standalone
|
|
276
|
-
// rmsnorm into each q/k/v_proj matmul (fused kernel runs norm internally).
|
|
277
|
-
let fusedNormWeightRec = null;
|
|
278
|
-
let fusedNormEpsRec = null;
|
|
279
|
-
let fusedNormOffsetRec = false;
|
|
280
|
-
let fusedNormOwnedRec = false;
|
|
281
|
-
// state.runtimeConfig is a stale snapshot whose inference.session is an empty
|
|
282
|
-
// object; the live module-level runtime config carries the merged
|
|
283
|
-
// profile/override session. Read session-level flags from getRuntimeConfig().
|
|
284
|
-
const rmsNormFusionFlagRec = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
|
|
285
|
-
const canSelectFusedRmsNormProjectionRec = canUseRmsNormWideTileProjectionFusion(
|
|
286
|
-
layerWeights,
|
|
287
|
-
sharedKVSourceLayerIdx != null
|
|
288
|
-
);
|
|
289
|
-
const canFuseInputNormProjRec = rmsNormFusionFlagRec
|
|
290
|
-
&& config.normalizationType === 'rmsnorm'
|
|
291
|
-
&& canSelectFusedRmsNormProjectionRec
|
|
292
|
-
&& !skipInputNorm
|
|
293
|
-
&& layerWeights.inputNorm
|
|
294
|
-
&& getNormWeightBuffer
|
|
295
|
-
&& isPrefill
|
|
296
|
-
&& numTokens > 1
|
|
297
|
-
&& attentionInput.dtype === 'f32';
|
|
298
|
-
|
|
299
|
-
try {
|
|
300
|
-
if (precomputedInputNorm) {
|
|
301
|
-
if (
|
|
302
|
-
isPrefill
|
|
303
|
-
|| numTokens !== 1
|
|
304
|
-
|| skipInputNorm
|
|
305
|
-
|| !layerWeights.inputNorm
|
|
306
|
-
|| !getNormWeightBuffer
|
|
307
|
-
|| precomputedInputNorm.dtype !== attentionInput.dtype
|
|
308
|
-
|| precomputedInputNorm.dtype !== 'f32'
|
|
309
|
-
) {
|
|
310
|
-
throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
|
|
311
|
-
}
|
|
312
|
-
normed = precomputedInputNorm;
|
|
313
|
-
await runProbes('post_input_norm', normed.buffer, {
|
|
314
|
-
layerIdx, numTokens, hiddenSize,
|
|
315
|
-
probes: state.debugProbes, recorder,
|
|
316
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
|
|
317
|
-
});
|
|
318
|
-
} else if (canFuseInputNormProjRec) {
|
|
319
|
-
fusedNormWeightRec = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
320
|
-
fusedNormEpsRec = rmsNormEps;
|
|
321
|
-
fusedNormOffsetRec = config.rmsNormWeightOffset === true;
|
|
322
|
-
fusedNormOwnedRec = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
|
|
323
|
-
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs norm internally.
|
|
324
|
-
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
325
|
-
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
326
|
-
let normBiasTensor = null;
|
|
327
|
-
let normBiasOwned = false;
|
|
328
|
-
if (config.normalizationType === 'layernorm') {
|
|
329
|
-
if (!layerWeights.inputNormBias) {
|
|
330
|
-
throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
|
|
331
|
-
}
|
|
332
|
-
({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
333
|
-
layerWeights.inputNormBias,
|
|
334
|
-
`L${layerIdx}.input_norm_bias`,
|
|
335
|
-
hiddenSize,
|
|
336
|
-
{},
|
|
337
|
-
debugFlags
|
|
338
|
-
));
|
|
339
|
-
normed = await recordLayerNorm(
|
|
340
|
-
recorder,
|
|
341
|
-
attentionInput,
|
|
342
|
-
normWeightBuf,
|
|
343
|
-
normBiasTensor.buffer,
|
|
344
|
-
rmsNormEps,
|
|
345
|
-
{ batchSize: numTokens, hiddenSize, label: 'input_norm' }
|
|
346
|
-
);
|
|
347
|
-
} else {
|
|
348
|
-
normed = await recordRMSNorm(recorder, attentionInput, normWeightBuf, rmsNormEps, {
|
|
349
|
-
batchSize: numTokens,
|
|
350
|
-
hiddenSize,
|
|
351
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
352
|
-
label: 'input_norm',
|
|
353
|
-
});
|
|
354
|
-
}
|
|
355
|
-
if (normBiasOwned && normBiasTensor) releaseOrTrack(recorder, normBiasTensor.buffer);
|
|
356
|
-
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseOrTrack(recorder, normWeightBuf);
|
|
357
|
-
await runProbes('post_input_norm', normed.buffer, {
|
|
358
|
-
layerIdx, numTokens, hiddenSize,
|
|
359
|
-
probes: state.debugProbes, recorder,
|
|
360
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: normed.dtype,
|
|
361
|
-
});
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
const debugLayers = debugFlags.debugLayers;
|
|
365
|
-
const shouldLogLayer = debugLayers === null ? layerIdx === 0 : shouldDebugLayer(layerIdx, debugLayers);
|
|
366
|
-
if (shouldLogLayer) {
|
|
367
|
-
const phase = selectRuleValue('kernels', 'attention', 'phase', { isDecode: !isPrefill });
|
|
368
|
-
const logKey = `L${layerIdx}_${phase}_dtypes`;
|
|
369
|
-
if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
|
|
370
|
-
ATTENTION_DTYPE_LOGGED.add(logKey);
|
|
371
|
-
trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
// 2. Q/K/V projections
|
|
376
|
-
const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
|
|
377
|
-
const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
|
|
378
|
-
const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
|
|
379
|
-
forceF32: shouldForceF32AttentionProjectionForRoPE({
|
|
380
|
-
attentionInputDtype: attentionActivationDtype,
|
|
381
|
-
headDim,
|
|
382
|
-
rotaryDim: config.ropeRotaryDim,
|
|
383
|
-
interleaved: config.ropeInterleaved,
|
|
384
|
-
kernelPathIsF16,
|
|
385
|
-
}),
|
|
386
|
-
});
|
|
387
|
-
let usedFusedQKV = false;
|
|
388
|
-
const sharedKVEntry = sharedKVSourceLayerIdx == null
|
|
389
|
-
? null
|
|
390
|
-
: (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
|
|
391
|
-
if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
|
|
392
|
-
throw new Error(
|
|
393
|
-
`Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
|
|
394
|
-
'but no shared K/V state was stored for that source layer.'
|
|
395
|
-
);
|
|
396
|
-
}
|
|
397
|
-
if (sharedKVEntry && (
|
|
398
|
-
sharedKVEntry.headDim !== headDim
|
|
399
|
-
|| sharedKVEntry.numKVHeads !== numKVHeads
|
|
400
|
-
)) {
|
|
401
|
-
throw new Error(
|
|
402
|
-
`Layer ${layerIdx} shared K/V geometry mismatch. ` +
|
|
403
|
-
`Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
|
|
404
|
-
`got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
|
|
405
|
-
);
|
|
406
|
-
}
|
|
407
|
-
const reusesSharedKV = sharedKVEntry != null;
|
|
408
|
-
retainSharedKvBuffers = reusesSharedKV || storeSharedKV;
|
|
409
|
-
let qkNormApplied = false;
|
|
410
|
-
let ropeApplied = false;
|
|
411
|
-
const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
|
|
412
|
-
const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
|
|
413
|
-
|| shouldTraceRecordedHealth(layerIdx, debugFlags);
|
|
414
|
-
const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
|
|
415
|
-
state,
|
|
416
|
-
['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
|
|
417
|
-
)
|
|
418
|
-
|| shouldTraceRecordedHealth(layerIdx, debugFlags);
|
|
419
|
-
const runtimeSession = getRuntimeConfig()?.inference?.session;
|
|
420
|
-
const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
|
|
421
|
-
const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
|
|
422
|
-
const skipKVCacheWrites = state.skipKVCacheWrites === true;
|
|
423
|
-
const directF16KVCacheWrite = skipKVCacheWrites
|
|
424
|
-
? null
|
|
425
|
-
: resolveDirectF16KVCacheWrite({
|
|
426
|
-
state,
|
|
427
|
-
layerIdx,
|
|
428
|
-
currentSeqLen,
|
|
429
|
-
numTokens,
|
|
430
|
-
numKVHeads,
|
|
431
|
-
headDim,
|
|
432
|
-
reusesSharedKV,
|
|
433
|
-
storeSharedKV,
|
|
434
|
-
diffusionGemmaDecoder,
|
|
435
|
-
valueNorm: config.valueNorm === true,
|
|
436
|
-
diagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
437
|
-
disableRoPE,
|
|
438
|
-
});
|
|
439
|
-
let kvCacheWriteFused = false;
|
|
440
|
-
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied, kvCacheWriteFused } = await projectAttentionQKV({
|
|
441
|
-
recorder,
|
|
442
|
-
normed,
|
|
443
|
-
layerWeights,
|
|
444
|
-
numTokens,
|
|
445
|
-
numHeads,
|
|
446
|
-
numKVHeads,
|
|
447
|
-
headDim,
|
|
448
|
-
hiddenSize,
|
|
449
|
-
layerIdx,
|
|
450
|
-
kernelPath,
|
|
451
|
-
matmulOutputDtype,
|
|
452
|
-
getWeightBuffer,
|
|
453
|
-
lora,
|
|
454
|
-
matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
|
|
455
|
-
attentionOutputGate: config.attentionOutputGate === true,
|
|
456
|
-
sharedKTensor: sharedKVEntry?.kTensor ?? null,
|
|
457
|
-
sharedVTensor: sharedKVEntry?.vTensor ?? null,
|
|
458
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
459
|
-
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
460
|
-
onFusedQKV: layerIdx === 0 && isPrefill
|
|
461
|
-
? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
|
|
462
|
-
trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
|
|
463
|
-
}
|
|
464
|
-
: null,
|
|
465
|
-
fusedNormWeight: fusedNormWeightRec,
|
|
466
|
-
fusedNormEps: fusedNormEpsRec,
|
|
467
|
-
fusedNormOffset: fusedNormOffsetRec,
|
|
468
|
-
qkNormFusion: {
|
|
469
|
-
enabled: qkNormFusionFlag
|
|
470
|
-
&& qkNormState.wantsQKNorm
|
|
471
|
-
&& config.queryKeyNormType === 'rmsnorm'
|
|
472
|
-
&& config.queryKeyNormAxis === 'head',
|
|
473
|
-
getNormWeightBuffer,
|
|
474
|
-
rmsNormEps,
|
|
475
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
476
|
-
skipKNorm: qkNormState.skipKNorm,
|
|
477
|
-
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
478
|
-
projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
|
|
479
|
-
},
|
|
480
|
-
qkNormRoPEFusion: {
|
|
481
|
-
enabled: qkNormRoPEFusionFlag
|
|
482
|
-
&& qkNormState.wantsQKNorm
|
|
483
|
-
&& config.queryKeyNormType === 'rmsnorm'
|
|
484
|
-
&& config.queryKeyNormAxis === 'head'
|
|
485
|
-
&& !disableRoPE
|
|
486
|
-
&& !!state.ropeFreqsCos
|
|
487
|
-
&& !!state.ropeFreqsSin,
|
|
488
|
-
getNormWeightBuffer,
|
|
489
|
-
rmsNormEps,
|
|
490
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
491
|
-
skipKNorm: qkNormState.skipKNorm,
|
|
492
|
-
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
493
|
-
projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
494
|
-
freqsCos: state.ropeFreqsCos,
|
|
495
|
-
freqsSin: state.ropeFreqsSin,
|
|
496
|
-
startPos: currentSeqLen,
|
|
497
|
-
rotaryDim: config.ropeRotaryDim,
|
|
498
|
-
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
499
|
-
interleaved: config.ropeInterleaved,
|
|
500
|
-
reusesSharedKV,
|
|
501
|
-
f16KVCacheWrite: directF16KVCacheWrite,
|
|
502
|
-
},
|
|
503
|
-
}));
|
|
504
|
-
if (!kvCacheWriteFused && (!kTensor || !vTensor)) {
|
|
505
|
-
throw new Error('Recorded attention projection returned missing K/V tensors without a fused KV cache write.');
|
|
506
|
-
}
|
|
507
|
-
// Deferred release of the norm weight buffer for fused path.
|
|
508
|
-
if (fusedNormWeightRec && fusedNormOwnedRec) {
|
|
509
|
-
releaseOrTrack(recorder, fusedNormWeightRec);
|
|
510
|
-
}
|
|
511
|
-
if (qTensor) {
|
|
512
|
-
await runProbes('q_proj', qTensor.buffer, {
|
|
513
|
-
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
514
|
-
probes: state.debugProbes, recorder,
|
|
515
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
516
|
-
});
|
|
517
|
-
}
|
|
518
|
-
if (kTensor) {
|
|
519
|
-
await runProbes('k_proj', kTensor.buffer, {
|
|
520
|
-
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
521
|
-
probes: state.debugProbes, recorder,
|
|
522
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
|
|
523
|
-
});
|
|
524
|
-
}
|
|
525
|
-
if (vTensor) {
|
|
526
|
-
await runProbes('v_proj', vTensor.buffer, {
|
|
527
|
-
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
528
|
-
probes: state.debugProbes, recorder,
|
|
529
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
|
|
530
|
-
});
|
|
531
|
-
}
|
|
532
|
-
if (qTensor && shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
533
|
-
enqueueRecordedTensorHealth(
|
|
534
|
-
recorder,
|
|
535
|
-
`L${layerIdx}.q_proj_HEALTH`,
|
|
536
|
-
qTensor,
|
|
537
|
-
qTensor.dtype,
|
|
538
|
-
numTokens * numHeads * headDim
|
|
539
|
-
);
|
|
540
|
-
enqueueRecordedTensorHealth(
|
|
541
|
-
recorder,
|
|
542
|
-
`L${layerIdx}.k_proj_HEALTH`,
|
|
543
|
-
kTensor,
|
|
544
|
-
kTensor?.dtype ?? null,
|
|
545
|
-
numTokens * numKVHeads * headDim
|
|
546
|
-
);
|
|
547
|
-
enqueueRecordedTensorHealth(
|
|
548
|
-
recorder,
|
|
549
|
-
`L${layerIdx}.v_proj_HEALTH`,
|
|
550
|
-
vTensor,
|
|
551
|
-
vTensor?.dtype ?? null,
|
|
552
|
-
numTokens * numKVHeads * headDim
|
|
553
|
-
);
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
// Optional Q/K normalization. The manifest owns both its numerical family
|
|
557
|
-
// and whether it applies per head or across the complete projection.
|
|
558
|
-
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
559
|
-
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
560
|
-
recorder,
|
|
561
|
-
qTensor,
|
|
562
|
-
kTensor,
|
|
563
|
-
layerWeights,
|
|
564
|
-
getNormWeightBuffer,
|
|
565
|
-
rmsNormEps,
|
|
566
|
-
numTokens,
|
|
567
|
-
numHeads,
|
|
568
|
-
numKVHeads,
|
|
569
|
-
headDim,
|
|
570
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
571
|
-
normalizationType: config.queryKeyNormType,
|
|
572
|
-
normalizationAxis: config.queryKeyNormAxis,
|
|
573
|
-
releaseTemporary: (buffer) => releaseOrTrack(recorder, buffer),
|
|
574
|
-
skipKNorm: qkNormState.skipKNorm,
|
|
575
|
-
retainKInput: valueAliasesKey,
|
|
576
|
-
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
577
|
-
}));
|
|
578
|
-
}
|
|
579
|
-
if (qkNormState.wantsQKNorm) {
|
|
580
|
-
await runProbes('q_norm', qTensor.buffer, {
|
|
581
|
-
layerIdx, numTokens, hiddenSize: numHeads * headDim,
|
|
582
|
-
probes: state.debugProbes, recorder,
|
|
583
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: qTensor.dtype,
|
|
584
|
-
});
|
|
585
|
-
if (kTensor) {
|
|
586
|
-
await runProbes('k_norm', kTensor.buffer, {
|
|
587
|
-
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
588
|
-
probes: state.debugProbes, recorder,
|
|
589
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: kTensor.dtype,
|
|
590
|
-
});
|
|
591
|
-
}
|
|
592
|
-
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
593
|
-
enqueueRecordedTensorHealth(
|
|
594
|
-
recorder,
|
|
595
|
-
`L${layerIdx}.q_norm_HEALTH`,
|
|
596
|
-
qTensor,
|
|
597
|
-
qTensor.dtype,
|
|
598
|
-
numTokens * numHeads * headDim
|
|
599
|
-
);
|
|
600
|
-
enqueueRecordedTensorHealth(
|
|
601
|
-
recorder,
|
|
602
|
-
`L${layerIdx}.k_norm_HEALTH`,
|
|
603
|
-
kTensor,
|
|
604
|
-
kTensor?.dtype ?? null,
|
|
605
|
-
numTokens * numKVHeads * headDim
|
|
606
|
-
);
|
|
607
|
-
}
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
if (!kvCacheWriteFused && config.valueNorm === true && !reusesSharedKV) {
|
|
611
|
-
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
612
|
-
vTensor = await applyAttentionValueNorm({
|
|
613
|
-
recorder,
|
|
614
|
-
vTensor,
|
|
615
|
-
rmsNormEps,
|
|
616
|
-
numTokens,
|
|
617
|
-
numKVHeads,
|
|
618
|
-
headDim,
|
|
619
|
-
releaseTemporary: (buffer) => {
|
|
620
|
-
if (!valueNormInputAliasesKey) {
|
|
621
|
-
releaseOrTrack(recorder, buffer);
|
|
622
|
-
}
|
|
623
|
-
},
|
|
624
|
-
});
|
|
625
|
-
await runProbes('v_norm', vTensor.buffer, {
|
|
626
|
-
layerIdx, numTokens, hiddenSize: numKVHeads * headDim,
|
|
627
|
-
probes: state.debugProbes, recorder,
|
|
628
|
-
operatorDiagnostics: state.operatorDiagnostics, dtype: vTensor.dtype,
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
if (normed !== attentionInput) releaseOrTrack(recorder, normed.buffer);
|
|
633
|
-
if (attentionInputTemp) recorder.trackTemporaryBuffer(attentionInput.buffer);
|
|
634
|
-
|
|
635
|
-
// 3. RoPE (modifies tensor in-place)
|
|
636
|
-
if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
637
|
-
const ropeOptions = {
|
|
638
|
-
headDim,
|
|
639
|
-
rotaryDim: config.ropeRotaryDim,
|
|
640
|
-
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
641
|
-
interleaved: config.ropeInterleaved,
|
|
642
|
-
startPos: currentSeqLen,
|
|
643
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
644
|
-
};
|
|
645
|
-
if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
|
|
646
|
-
await recordRoPEQK(recorder, qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
647
|
-
...ropeOptions,
|
|
648
|
-
numQHeads: numHeads,
|
|
649
|
-
numKVHeads,
|
|
650
|
-
});
|
|
651
|
-
} else {
|
|
652
|
-
await recordRoPE(recorder, qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
653
|
-
...ropeOptions,
|
|
654
|
-
numHeads,
|
|
655
|
-
});
|
|
656
|
-
if (!reusesSharedKV) {
|
|
657
|
-
await recordRoPE(recorder, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
658
|
-
...ropeOptions,
|
|
659
|
-
numHeads: numKVHeads,
|
|
660
|
-
});
|
|
661
|
-
}
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
665
|
-
enqueueRecordedTensorHealth(
|
|
666
|
-
recorder,
|
|
667
|
-
`L${layerIdx}.q_rope_HEALTH`,
|
|
668
|
-
qTensor,
|
|
669
|
-
qTensor.dtype,
|
|
670
|
-
numTokens * numHeads * headDim
|
|
671
|
-
);
|
|
672
|
-
enqueueRecordedTensorHealth(
|
|
673
|
-
recorder,
|
|
674
|
-
`L${layerIdx}.k_rope_HEALTH`,
|
|
675
|
-
kTensor,
|
|
676
|
-
kTensor?.dtype ?? null,
|
|
677
|
-
numTokens * numKVHeads * headDim
|
|
678
|
-
);
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
if (storeSharedKV && state.sharedAttentionState) {
|
|
682
|
-
state.sharedAttentionState.set(layerIdx, {
|
|
683
|
-
kTensor,
|
|
684
|
-
vTensor,
|
|
685
|
-
headDim,
|
|
686
|
-
numKVHeads,
|
|
687
|
-
});
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
// 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
|
|
691
|
-
if (!skipKVCacheWrites && !diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
692
|
-
if (kvCacheWriteFused) {
|
|
693
|
-
state.kvCache.recordF16UpdateAlreadyWrittenFromGPU(layerIdx, currentSeqLen, numTokens, tokenIds);
|
|
694
|
-
} else if (state.kvCache.kvDtype === 'f16') {
|
|
695
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
696
|
-
if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
697
|
-
assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
|
|
698
|
-
}
|
|
699
|
-
if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
700
|
-
assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
|
|
701
|
-
}
|
|
702
|
-
const canWriteDirectF32ToF16 = kTensor.dtype === 'f32'
|
|
703
|
-
&& vTensor.dtype === 'f32'
|
|
704
|
-
&& typeof state.kvCache.recordUpdateF32ToF16FromGPU === 'function';
|
|
705
|
-
if (canWriteDirectF32ToF16) {
|
|
706
|
-
await state.kvCache.recordUpdateF32ToF16FromGPU(
|
|
707
|
-
recorder,
|
|
708
|
-
layerIdx,
|
|
709
|
-
kTensor.buffer,
|
|
710
|
-
vTensor.buffer,
|
|
711
|
-
currentSeqLen,
|
|
712
|
-
numTokens,
|
|
713
|
-
tokenIds
|
|
714
|
-
);
|
|
715
|
-
} else {
|
|
716
|
-
const kCasted = kTensor.dtype === 'f16' ? kTensor : await recordCastF32ToF16(recorder, kTensor);
|
|
717
|
-
const vCasted = vTensor.dtype === 'f16' ? vTensor : await recordCastF32ToF16(recorder, vTensor);
|
|
718
|
-
|
|
719
|
-
await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
|
|
720
|
-
|
|
721
|
-
if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
|
|
722
|
-
if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
|
|
723
|
-
}
|
|
724
|
-
} else {
|
|
725
|
-
await state.kvCache.recordUpdateFromGPU(recorder, layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
// Resolve KV cache state and build dispatch parameters (shared with run.js)
|
|
730
|
-
let kvState;
|
|
731
|
-
if (diffusionGemmaDecoder) {
|
|
732
|
-
const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
|
|
733
|
-
let decoderKTensor = kTensor;
|
|
734
|
-
let decoderVTensor = vTensor;
|
|
735
|
-
if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
|
|
736
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
737
|
-
if (!hasExplicitF16KvContract) {
|
|
738
|
-
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
|
|
739
|
-
}
|
|
740
|
-
decoderKTensor = await recordCastF32ToF16(recorder, decoderKTensor);
|
|
741
|
-
decoderKTemp = decoderKTensor;
|
|
742
|
-
} else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
|
|
743
|
-
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
|
|
744
|
-
decoderKTensor = await recordCastF16ToF32(recorder, decoderKTensor);
|
|
745
|
-
decoderKTemp = decoderKTensor;
|
|
746
|
-
}
|
|
747
|
-
if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
|
|
748
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
749
|
-
if (!hasExplicitF16KvContract) {
|
|
750
|
-
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
|
|
751
|
-
}
|
|
752
|
-
decoderVTensor = await recordCastF32ToF16(recorder, decoderVTensor);
|
|
753
|
-
decoderVTemp = decoderVTensor;
|
|
754
|
-
} else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
|
|
755
|
-
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
|
|
756
|
-
decoderVTensor = await recordCastF16ToF32(recorder, decoderVTensor);
|
|
757
|
-
decoderVTemp = decoderVTensor;
|
|
758
|
-
}
|
|
759
|
-
kvState = await createDiffusionGemmaDecoderKVState({
|
|
760
|
-
state,
|
|
761
|
-
layerIdx,
|
|
762
|
-
kTensor: decoderKTensor,
|
|
763
|
-
vTensor: decoderVTensor,
|
|
764
|
-
currentSeqLen,
|
|
765
|
-
numTokens,
|
|
766
|
-
numKVHeads,
|
|
767
|
-
headDim,
|
|
768
|
-
layerType,
|
|
769
|
-
slidingWindow,
|
|
770
|
-
kvDtype: decoderKVDtype,
|
|
771
|
-
recorder,
|
|
772
|
-
});
|
|
773
|
-
if (decoderKTemp) {
|
|
774
|
-
recorder.trackTemporaryBuffer(decoderKTemp.buffer);
|
|
775
|
-
decoderKTemp = null;
|
|
776
|
-
}
|
|
777
|
-
if (decoderVTemp) {
|
|
778
|
-
recorder.trackTemporaryBuffer(decoderVTemp.buffer);
|
|
779
|
-
decoderVTemp = null;
|
|
780
|
-
}
|
|
781
|
-
decoderKVState = kvState;
|
|
782
|
-
} else {
|
|
783
|
-
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens, {
|
|
784
|
-
forceCurrentTensors: skipKVCacheWrites,
|
|
785
|
-
});
|
|
786
|
-
}
|
|
787
|
-
const dispatchConfig = {
|
|
788
|
-
layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
|
|
789
|
-
slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
|
|
790
|
-
layerType, layerTypes: config.layerTypes,
|
|
791
|
-
queryPreAttnScalar,
|
|
792
|
-
causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
|
|
793
|
-
activationDtype: attentionActivationDtype,
|
|
794
|
-
kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
|
|
795
|
-
};
|
|
796
|
-
const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
|
|
797
|
-
const {
|
|
798
|
-
effectiveSlidingWindow, attentionKernelVariant, attnScale,
|
|
799
|
-
cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
|
|
800
|
-
prefillFallbackNeedsCast, causalForAttention,
|
|
801
|
-
} = dispatchParams;
|
|
802
|
-
const attentionKernelPath = skipKVCacheWrites ? null : kernelPath;
|
|
803
|
-
|
|
804
|
-
// 5. Attention
|
|
805
|
-
|
|
806
|
-
recordAttentionInputs(state, buildAttentionInputsData(
|
|
807
|
-
dispatchConfig, input, normed, kvState, dispatchParams,
|
|
808
|
-
{ useF16Activations, matmulOutputDtype },
|
|
809
|
-
usedFusedQKV, qTensor, kTensor, vTensor,
|
|
810
|
-
));
|
|
811
|
-
const mergedSessionRec = getRuntimeConfig()?.inference?.session;
|
|
812
|
-
let attentionOutputGateFused = false;
|
|
813
|
-
|
|
814
|
-
const attentionKernelRunners = {
|
|
815
|
-
bdpa: async () => {
|
|
816
|
-
const basisKDtype = 'f16';
|
|
817
|
-
const basisVDtype = 'f16';
|
|
818
|
-
const numBasisVectors = Math.max(1, kvState.bdpaBasisCount);
|
|
819
|
-
const basisKTensor = createTensor(kvState.bdpaBasisK, basisKDtype, [numBasisVectors, numKVHeads * headDim], 'bdpa_basis_k');
|
|
820
|
-
const basisVTensor = createTensor(kvState.bdpaBasisV, basisVDtype, [numBasisVectors, numKVHeads * headDim], 'bdpa_basis_v');
|
|
821
|
-
|
|
822
|
-
let qForBDPA = qTensor;
|
|
823
|
-
if (qForBDPA.dtype !== 'f16') {
|
|
824
|
-
assertAttentionDtypeTransitionAllowed(state, qForBDPA.dtype, 'f16', 'BDPA attention would narrow Q implicitly.');
|
|
825
|
-
qForBDPA = await recordCastF32ToF16(recorder, qTensor);
|
|
826
|
-
recorder.trackTemporaryBuffer(qForBDPA.buffer);
|
|
827
|
-
}
|
|
828
|
-
|
|
829
|
-
return recordAttentionBDPA(recorder, qForBDPA, basisKTensor, basisVTensor, kvState.bdpaPagedK, kvState.bdpaPagedV, kvState.bdpaIndex, numHeads, headDim, {
|
|
830
|
-
seqLen: numTokens,
|
|
831
|
-
kvLen: kvState.kvLenForAttention,
|
|
832
|
-
numKVHeads,
|
|
833
|
-
causal: causalForAttention,
|
|
834
|
-
startPos: kvState.startPosForMask,
|
|
835
|
-
layerIdx,
|
|
836
|
-
slidingWindow: effectiveSlidingWindow,
|
|
837
|
-
attnSoftcap,
|
|
838
|
-
scale: attnScale,
|
|
839
|
-
ropeCos: state.ropeFreqsCos,
|
|
840
|
-
ropeSin: state.ropeFreqsSin,
|
|
841
|
-
});
|
|
842
|
-
},
|
|
843
|
-
tieredQuant: async () => {
|
|
844
|
-
let qForAttention = qTensor;
|
|
845
|
-
if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
|
|
846
|
-
assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Tiered quant attention would widen Q implicitly.');
|
|
847
|
-
qForAttention = await recordCastF16ToF32(recorder, qTensor);
|
|
848
|
-
recorder.trackTemporaryBuffer(qForAttention.buffer);
|
|
849
|
-
}
|
|
850
|
-
if (kvState.coldQuantMode === 'none') {
|
|
851
|
-
throw new Error('Tiered quant attention requires cold quant mode.');
|
|
852
|
-
}
|
|
853
|
-
if (!kvState.coldScalesK || !kvState.coldScalesV) {
|
|
854
|
-
throw new Error('Tiered quant attention requires cold scale buffers.');
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
|
|
858
|
-
const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
|
|
859
|
-
return recordAttentionTieredQuant(
|
|
860
|
-
recorder,
|
|
861
|
-
qForAttention,
|
|
862
|
-
cachedHotKTensor,
|
|
863
|
-
cachedHotVTensor,
|
|
864
|
-
kvState.cachedKCold,
|
|
865
|
-
kvState.cachedVCold,
|
|
866
|
-
kvState.coldScalesK,
|
|
867
|
-
kvState.coldScalesV,
|
|
868
|
-
numHeads,
|
|
869
|
-
headDim,
|
|
870
|
-
buildTieredQuantAttentionOptions(kvState, {
|
|
871
|
-
seqLen: numTokens,
|
|
872
|
-
numKVHeads,
|
|
873
|
-
causal: causalForAttention,
|
|
874
|
-
startPos: kvState.startPosForMask,
|
|
875
|
-
slidingWindow: effectiveSlidingWindow ?? 0,
|
|
876
|
-
attnSoftcap,
|
|
877
|
-
scale: attnScale,
|
|
878
|
-
})
|
|
879
|
-
);
|
|
880
|
-
},
|
|
881
|
-
contiguousQuant: async () => {
|
|
882
|
-
let qForAttention = qTensor;
|
|
883
|
-
if (qTensor.dtype !== 'f32') {
|
|
884
|
-
assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Contiguous quant attention would widen Q implicitly.');
|
|
885
|
-
qForAttention = await recordCastF16ToF32(recorder, qTensor);
|
|
886
|
-
recorder.trackTemporaryBuffer(qForAttention.buffer);
|
|
887
|
-
}
|
|
888
|
-
|
|
889
|
-
if (!kvState.coldScalesK || !kvState.coldScalesV) {
|
|
890
|
-
throw new Error('Contiguous quant attention requires scale buffers.');
|
|
891
|
-
}
|
|
892
|
-
if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
|
|
893
|
-
throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
return recordAttentionContiguousQuant(
|
|
897
|
-
recorder,
|
|
898
|
-
qForAttention,
|
|
899
|
-
kvState.cachedKCold,
|
|
900
|
-
kvState.cachedVCold,
|
|
901
|
-
kvState.coldScalesK,
|
|
902
|
-
kvState.coldScalesV,
|
|
903
|
-
numHeads,
|
|
904
|
-
headDim,
|
|
905
|
-
buildContiguousQuantAttentionOptions(kvState, {
|
|
906
|
-
seqLen: numTokens,
|
|
907
|
-
kvLen: kvState.kvLenForAttention,
|
|
908
|
-
numKVHeads,
|
|
909
|
-
causal: causalForAttention,
|
|
910
|
-
startPos: kvState.startPosForMask,
|
|
911
|
-
slidingWindow: effectiveSlidingWindow ?? 0,
|
|
912
|
-
attnSoftcap,
|
|
913
|
-
scale: attnScale,
|
|
914
|
-
})
|
|
915
|
-
);
|
|
916
|
-
},
|
|
917
|
-
tiered: async () => {
|
|
918
|
-
const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
|
|
919
|
-
const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
|
|
920
|
-
const cachedColdKTensor = createTensor(kvState.cachedKCold, cachedKDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_K_cold');
|
|
921
|
-
const cachedColdVTensor = createTensor(kvState.cachedVCold, cachedVDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_V_cold');
|
|
922
|
-
return recordAttentionTiered(recorder, qTensor, cachedHotKTensor, cachedHotVTensor, cachedColdKTensor, cachedColdVTensor, numHeads, headDim, {
|
|
923
|
-
seqLen: numTokens,
|
|
924
|
-
coldLen: kvState.coldLen,
|
|
925
|
-
hotLen: kvState.hotLen,
|
|
926
|
-
numKVHeads,
|
|
927
|
-
causal: causalForAttention,
|
|
928
|
-
startPos: kvState.startPosForMask,
|
|
929
|
-
slidingWindow: effectiveSlidingWindow ?? 0,
|
|
930
|
-
attnSoftcap,
|
|
931
|
-
scale: attnScale,
|
|
932
|
-
hotWindow: kvState.hotWindow,
|
|
933
|
-
hotStart: kvState.hotStart,
|
|
934
|
-
coldPageTable: kvState.coldPageTable,
|
|
935
|
-
coldPageSize: kvState.coldPageSize,
|
|
936
|
-
coldLayout: kvState.coldPageTable ? 2 : 0,
|
|
937
|
-
hotLayout: kvState.hotWindow > 0 ? 1 : 0,
|
|
938
|
-
});
|
|
939
|
-
},
|
|
940
|
-
contiguous: async () => {
|
|
941
|
-
// Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
|
|
942
|
-
let kForAttn = cachedKTensor;
|
|
943
|
-
let vForAttn = cachedVTensor;
|
|
944
|
-
if (prefillFallbackNeedsCast) {
|
|
945
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
946
|
-
if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
947
|
-
assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'Prefill fallback attention would narrow K implicitly.');
|
|
948
|
-
}
|
|
949
|
-
if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
950
|
-
assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'Prefill fallback attention would narrow V implicitly.');
|
|
951
|
-
}
|
|
952
|
-
const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
|
|
953
|
-
? await recordCastF32ToF16(recorder, kTensor) : kTensor;
|
|
954
|
-
const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
|
|
955
|
-
? await recordCastF32ToF16(recorder, vTensor) : vTensor;
|
|
956
|
-
kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
|
|
957
|
-
vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
|
|
958
|
-
if (kTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(kCasted.buffer);
|
|
959
|
-
if (vTensor.dtype !== 'f16') recorder.trackTemporaryBuffer(vCasted.buffer);
|
|
960
|
-
}
|
|
961
|
-
// Session precedence is runtime-over-manifest per config-style-guide
|
|
962
|
-
// §Category Rules. getRuntimeConfig() returns the merged session (manifest
|
|
963
|
-
// is the base layer, runtime fields win field-by-field via merge.js).
|
|
964
|
-
// Kernel enforces head_dim=256, f16 KV, contiguous layout; only applies
|
|
965
|
-
// when numTokens > 1 (prefill). Same flag semantics as the non-recorder
|
|
966
|
-
// path in ./run.js.
|
|
967
|
-
const useFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useFlashPrefillAttention === true && numTokens > 1;
|
|
968
|
-
const useOrtFlashPrefillRec = !diffusionGemmaDecoder && mergedSessionRec?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
969
|
-
const useOutputGateFusion = canUseAttentionOutputGateFusion({
|
|
970
|
-
session: mergedSessionRec,
|
|
971
|
-
qGateTensor,
|
|
972
|
-
numTokens,
|
|
973
|
-
numHeads,
|
|
974
|
-
headDim,
|
|
975
|
-
cachedKDtype,
|
|
976
|
-
cachedVDtype,
|
|
977
|
-
kernelPath: attentionKernelPath,
|
|
978
|
-
diffusionGemmaDecoder,
|
|
979
|
-
});
|
|
980
|
-
const result = await recordAttention(recorder, qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
981
|
-
seqLen: numTokens,
|
|
982
|
-
kvLen: kvState.kvLenForAttention,
|
|
983
|
-
numKVHeads,
|
|
984
|
-
causal: causalForAttention,
|
|
985
|
-
bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
|
|
986
|
-
bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
|
|
987
|
-
startPos: kvState.startPosForMask,
|
|
988
|
-
layerIdx,
|
|
989
|
-
slidingWindow: effectiveSlidingWindow,
|
|
990
|
-
attnSoftcap,
|
|
991
|
-
scale: attnScale,
|
|
992
|
-
kvStart: kvState.kvStart,
|
|
993
|
-
kvLayout: kvState.kvLayout,
|
|
994
|
-
kvPageTable: kvState.kvPageTable,
|
|
995
|
-
kvPageSize: kvState.kvPageSize,
|
|
996
|
-
kernelPath: attentionKernelPath,
|
|
997
|
-
useFlashPrefill: skipKVCacheWrites ? false : useFlashPrefillRec,
|
|
998
|
-
useOrtFlashPrefill: skipKVCacheWrites ? false : useOrtFlashPrefillRec,
|
|
999
|
-
outputGate: useOutputGateFusion ? qGateTensor : null,
|
|
1000
|
-
});
|
|
1001
|
-
attentionOutputGateFused = result?.outputGateFused === true;
|
|
1002
|
-
return result;
|
|
1003
|
-
},
|
|
1004
|
-
};
|
|
1005
|
-
const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
|
|
1006
|
-
if (!runAttentionKernel) {
|
|
1007
|
-
throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
|
|
1008
|
-
}
|
|
1009
|
-
|
|
1010
|
-
try {
|
|
1011
|
-
attnOutput = await runAttentionKernel();
|
|
1012
|
-
} finally {
|
|
1013
|
-
if (decoderKVState?.ownedBuffers) {
|
|
1014
|
-
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1015
|
-
recorder.trackTemporaryBuffer(buffer);
|
|
1016
|
-
}
|
|
1017
|
-
decoderKVState.ownedBuffers = null;
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
1021
|
-
layerIdx,
|
|
1022
|
-
numTokens,
|
|
1023
|
-
hiddenSize: numHeads * headDim,
|
|
1024
|
-
probes: state.debugProbes,
|
|
1025
|
-
recorder,
|
|
1026
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
1027
|
-
dtype: attnOutput.dtype,
|
|
1028
|
-
});
|
|
1029
|
-
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
1030
|
-
enqueueRecordedTensorHealth(
|
|
1031
|
-
recorder,
|
|
1032
|
-
`L${layerIdx}.attn_core_out_HEALTH`,
|
|
1033
|
-
attnOutput,
|
|
1034
|
-
attnOutput.dtype,
|
|
1035
|
-
numTokens * numHeads * headDim
|
|
1036
|
-
);
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
attnForProjection = attnOutput;
|
|
1040
|
-
if (qGateTensor && !attentionOutputGateFused) {
|
|
1041
|
-
// Mirror run.js gate dispatch. Qwen3_5/Qwen 3.6 full attention applies
|
|
1042
|
-
// sigmoid(gate) even when the HF config surfaces `output_gate_type=swish`.
|
|
1043
|
-
const gateActivation = 'sigmoid';
|
|
1044
|
-
attnForProjection = await recordSiLU(recorder, attnOutput, {
|
|
1045
|
-
size: numTokens * numHeads * headDim,
|
|
1046
|
-
gate: qGateTensor,
|
|
1047
|
-
useVec4: (numTokens * numHeads * headDim) % 4 === 0,
|
|
1048
|
-
gateActivation,
|
|
1049
|
-
inputActivation: 'identity',
|
|
1050
|
-
swigluLimit: null,
|
|
1051
|
-
});
|
|
1052
|
-
recorder.trackTemporaryBuffer(attnOutput.buffer);
|
|
1053
|
-
}
|
|
1054
|
-
|
|
1055
|
-
// 6. Output projection (with optional fused residual for decode)
|
|
1056
|
-
|
|
1057
|
-
output = null;
|
|
1058
|
-
let residualFused = false;
|
|
1059
|
-
let oProjInput = attnForProjection;
|
|
1060
|
-
oProjInputTemp = null;
|
|
1061
|
-
if (layerWeights.oProj && getWeightBuffer) {
|
|
1062
|
-
({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
|
|
1063
|
-
attnForProjection,
|
|
1064
|
-
oProjInputDtype,
|
|
1065
|
-
(tensor) => {
|
|
1066
|
-
assertAttentionDtypeTransitionAllowed(
|
|
1067
|
-
state,
|
|
1068
|
-
tensor.dtype,
|
|
1069
|
-
oProjInputDtype,
|
|
1070
|
-
'Attention output projection would change activations implicitly.',
|
|
1071
|
-
'step_precision'
|
|
1072
|
-
);
|
|
1073
|
-
return oProjInputDtype === 'f16'
|
|
1074
|
-
? recordCastF32ToF16(recorder, tensor)
|
|
1075
|
-
: recordCastF16ToF32(recorder, tensor);
|
|
1076
|
-
}
|
|
1077
|
-
));
|
|
1078
|
-
const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
|
|
1079
|
-
const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
|
|
1080
|
-
|
|
1081
|
-
// Use fused o_proj + residual for decode when possible
|
|
1082
|
-
// Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
|
|
1083
|
-
const oProjDtype = getWeightDtype(oProjBuf);
|
|
1084
|
-
const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
|
|
1085
|
-
allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
|
|
1086
|
-
hasResidual: Boolean(residualTensor),
|
|
1087
|
-
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1088
|
-
attnIsF32: oProjInput.dtype === 'f32',
|
|
1089
|
-
attnIsF16: oProjInput.dtype === 'f16',
|
|
1090
|
-
hasLoRA: Boolean(loraO),
|
|
1091
|
-
oProjIsF16: oProjDtype === 'f16',
|
|
1092
|
-
});
|
|
1093
|
-
const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
|
|
1094
|
-
phase,
|
|
1095
|
-
session: mergedSessionRec,
|
|
1096
|
-
hasResidual: Boolean(residualTensor),
|
|
1097
|
-
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1098
|
-
inputDtype: oProjInput.dtype,
|
|
1099
|
-
outputDtype: oProjOutputDtype,
|
|
1100
|
-
oProjDtype,
|
|
1101
|
-
hasLoRA: Boolean(loraO),
|
|
1102
|
-
});
|
|
1103
|
-
if (canUseFused && residualTensor) {
|
|
1104
|
-
// FUSED PATH: o_proj matmul + residual add in one dispatch
|
|
1105
|
-
output = await recordMatmulResidualFused(recorder, oProjInput, oProjBuf, residualTensor, {
|
|
1106
|
-
N: hiddenSize,
|
|
1107
|
-
K: numHeads * headDim,
|
|
1108
|
-
});
|
|
1109
|
-
residualFused = true;
|
|
1110
|
-
} else {
|
|
1111
|
-
// STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
|
|
1112
|
-
output = await recordMatmul(recorder, oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
|
|
1113
|
-
transposeB: 'auto',
|
|
1114
|
-
role: 'o_proj',
|
|
1115
|
-
layerIdx,
|
|
1116
|
-
kernelPath,
|
|
1117
|
-
outputDtype: oProjOutputDtype,
|
|
1118
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
1119
|
-
residualTensor: canUseWideTileResidual ? residualTensor : null,
|
|
1120
|
-
});
|
|
1121
|
-
residualFused = canUseWideTileResidual;
|
|
1122
|
-
}
|
|
1123
|
-
// Release temporary buffer if we created it (original was not already on GPU)
|
|
1124
|
-
if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
|
|
1125
|
-
releaseOrTrack(recorder, isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
|
|
1126
|
-
}
|
|
1127
|
-
} else {
|
|
1128
|
-
output = oProjInput;
|
|
1129
|
-
}
|
|
1130
|
-
|
|
1131
|
-
// Apply LoRA to output projection if present (only if not using fused path)
|
|
1132
|
-
if (!residualFused) {
|
|
1133
|
-
const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
|
|
1134
|
-
if (loraO && getWeightBuffer) {
|
|
1135
|
-
const combined = await applyLoRA(
|
|
1136
|
-
oProjInput,
|
|
1137
|
-
output,
|
|
1138
|
-
loraO,
|
|
1139
|
-
{ M: numTokens, N: hiddenSize, K: numHeads * headDim },
|
|
1140
|
-
getWeightBuffer,
|
|
1141
|
-
recorder,
|
|
1142
|
-
{ kernelPath }
|
|
1143
|
-
);
|
|
1144
|
-
if (combined.buffer !== output.buffer) {
|
|
1145
|
-
recorder.trackTemporaryBuffer(output.buffer);
|
|
1146
|
-
output = combined;
|
|
1147
|
-
}
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
|
|
1151
|
-
if (layerWeights.oProjBias) {
|
|
1152
|
-
const { tensor: oProjBiasTensor, owned } = getVectorTensor(
|
|
1153
|
-
layerWeights.oProjBias,
|
|
1154
|
-
`L${layerIdx}.o_proj_bias`,
|
|
1155
|
-
hiddenSize,
|
|
1156
|
-
{},
|
|
1157
|
-
debugFlags
|
|
1158
|
-
);
|
|
1159
|
-
output = await recordBiasAdd(
|
|
1160
|
-
recorder,
|
|
1161
|
-
output,
|
|
1162
|
-
oProjBiasTensor,
|
|
1163
|
-
numTokens,
|
|
1164
|
-
hiddenSize,
|
|
1165
|
-
{
|
|
1166
|
-
label: `L${layerIdx}.o_proj_bias`,
|
|
1167
|
-
layerIdx,
|
|
1168
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
1169
|
-
}
|
|
1170
|
-
);
|
|
1171
|
-
if (owned) releaseOrTrack(recorder, oProjBiasTensor.buffer);
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
finalOutput = output;
|
|
1175
|
-
if (shouldTraceRecordedHealth(layerIdx, debugFlags)) {
|
|
1176
|
-
enqueueRecordedTensorHealth(
|
|
1177
|
-
recorder,
|
|
1178
|
-
`L${layerIdx}.o_proj_HEALTH`,
|
|
1179
|
-
output,
|
|
1180
|
-
output.dtype,
|
|
1181
|
-
numTokens * hiddenSize
|
|
1182
|
-
);
|
|
1183
|
-
}
|
|
1184
|
-
|
|
1185
|
-
const buffersToTrack = [];
|
|
1186
|
-
if (output.buffer !== attnForProjection.buffer) {
|
|
1187
|
-
buffersToTrack.push(attnForProjection.buffer);
|
|
1188
|
-
}
|
|
1189
|
-
if (oProjInputTemp && oProjInputTemp.buffer !== attnForProjection.buffer) {
|
|
1190
|
-
buffersToTrack.push(oProjInputTemp.buffer);
|
|
1191
|
-
}
|
|
1192
|
-
if (output.dtype !== oProjOutputDtype) {
|
|
1193
|
-
assertAttentionDtypeTransitionAllowed(state, output.dtype, oProjOutputDtype, 'Attention output would change implicitly before leaving the layer.');
|
|
1194
|
-
const coercedOutput = oProjOutputDtype === 'f16'
|
|
1195
|
-
? await recordCastF32ToF16(recorder, output)
|
|
1196
|
-
: await recordCastF16ToF32(recorder, output);
|
|
1197
|
-
buffersToTrack.push(output.buffer);
|
|
1198
|
-
finalOutput = coercedOutput;
|
|
1199
|
-
}
|
|
1200
|
-
|
|
1201
|
-
// Track intermediate buffers for cleanup after submit (not release!)
|
|
1202
|
-
// These buffers are used by recorded operations that haven't executed yet.
|
|
1203
|
-
// Releasing them back to the pool would allow reuse before the encoder is submitted,
|
|
1204
|
-
// causing data corruption (especially for small decode buffers).
|
|
1205
|
-
if (qTensor) {
|
|
1206
|
-
recorder.trackTemporaryBuffer(qTensor.buffer);
|
|
1207
|
-
}
|
|
1208
|
-
if (qGateTensor) {
|
|
1209
|
-
recorder.trackTemporaryBuffer(qGateTensor.buffer);
|
|
1210
|
-
}
|
|
1211
|
-
if (!retainSharedKvBuffers) {
|
|
1212
|
-
if (kTensor) {
|
|
1213
|
-
recorder.trackTemporaryBuffer(kTensor.buffer);
|
|
1214
|
-
}
|
|
1215
|
-
if (vTensor?.buffer && vTensor.buffer !== kTensor?.buffer) {
|
|
1216
|
-
recorder.trackTemporaryBuffer(vTensor.buffer);
|
|
1217
|
-
}
|
|
1218
|
-
}
|
|
1219
|
-
for (const buffer of buffersToTrack) {
|
|
1220
|
-
recorder.trackTemporaryBuffer(buffer);
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
return { output: finalOutput, residualFused };
|
|
1224
|
-
} catch (error) {
|
|
1225
|
-
const tracked = new Set();
|
|
1226
|
-
const trackOnce = (buffer) => {
|
|
1227
|
-
if (!buffer || tracked.has(buffer)) return;
|
|
1228
|
-
tracked.add(buffer);
|
|
1229
|
-
recorder.trackTemporaryBuffer(buffer);
|
|
1230
|
-
};
|
|
1231
|
-
if (finalOutput?.buffer && finalOutput.buffer !== output?.buffer) {
|
|
1232
|
-
trackOnce(finalOutput.buffer);
|
|
1233
|
-
}
|
|
1234
|
-
if (output?.buffer && output.buffer !== attnForProjection?.buffer) {
|
|
1235
|
-
trackOnce(output.buffer);
|
|
1236
|
-
}
|
|
1237
|
-
if (oProjInputTemp?.buffer) {
|
|
1238
|
-
trackOnce(oProjInputTemp.buffer);
|
|
1239
|
-
}
|
|
1240
|
-
if (attnForProjection?.buffer && attnForProjection.buffer !== attnOutput?.buffer) {
|
|
1241
|
-
trackOnce(attnForProjection.buffer);
|
|
1242
|
-
}
|
|
1243
|
-
if (attnOutput?.buffer) {
|
|
1244
|
-
trackOnce(attnOutput.buffer);
|
|
1245
|
-
}
|
|
1246
|
-
if (qGateTensor?.buffer) {
|
|
1247
|
-
trackOnce(qGateTensor.buffer);
|
|
1248
|
-
}
|
|
1249
|
-
if (qTensor?.buffer) {
|
|
1250
|
-
trackOnce(qTensor.buffer);
|
|
1251
|
-
}
|
|
1252
|
-
if (kTensor?.buffer && !retainSharedKvBuffers) {
|
|
1253
|
-
trackOnce(kTensor.buffer);
|
|
1254
|
-
}
|
|
1255
|
-
if (vTensor?.buffer && !retainSharedKvBuffers) {
|
|
1256
|
-
trackOnce(vTensor.buffer);
|
|
1257
|
-
}
|
|
1258
|
-
if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {
|
|
1259
|
-
trackOnce(normed.buffer);
|
|
1260
|
-
}
|
|
1261
|
-
if (attentionInputTemp && attentionInput?.buffer) {
|
|
1262
|
-
trackOnce(attentionInput.buffer);
|
|
1263
|
-
}
|
|
1264
|
-
if (decoderKTemp?.buffer) {
|
|
1265
|
-
trackOnce(decoderKTemp.buffer);
|
|
1266
|
-
}
|
|
1267
|
-
if (decoderVTemp?.buffer) {
|
|
1268
|
-
trackOnce(decoderVTemp.buffer);
|
|
1269
|
-
}
|
|
1270
|
-
if (decoderKVState?.ownedBuffers) {
|
|
1271
|
-
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1272
|
-
trackOnce(buffer);
|
|
1273
|
-
}
|
|
1274
|
-
decoderKVState.ownedBuffers = null;
|
|
1275
|
-
}
|
|
1276
|
-
throw error;
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Recorded attention facade.
|
|
3
|
+
*
|
|
4
|
+
* Semantic decisions are resolved in plan.js. The recorded adapter binds the
|
|
5
|
+
* canonical interpreter to the caller's recorder.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export { recordLayerAttentionGPU } from './executor-recorded.js';
|