doppler-gpu 0.4.15 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +48 -1
- package/src/client/runtime/model-session.js +149 -30
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -1,1386 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
runRMSNorm,
|
|
10
|
-
runLayerNorm,
|
|
11
|
-
runBiasAdd,
|
|
12
|
-
runRoPE,
|
|
13
|
-
canUseRoPEQK,
|
|
14
|
-
runRoPEQK,
|
|
15
|
-
runAttention,
|
|
16
|
-
runAttentionBDPA,
|
|
17
|
-
runAttentionTieredQuant,
|
|
18
|
-
runAttentionTiered,
|
|
19
|
-
runAttentionContiguousQuant,
|
|
20
|
-
runSiLU,
|
|
21
|
-
castF16ToF32,
|
|
22
|
-
castF32ToF16,
|
|
23
|
-
runMatmulResidualFused,
|
|
24
|
-
shouldUseFusedMatmulResidual,
|
|
25
|
-
} from '../../../../gpu/kernel-selector.js';
|
|
26
|
-
import { createTensor } from '../../../../gpu/tensor.js';
|
|
27
|
-
import { isKernelDebugEnabled, dumpTokenVector, dumpKVCache, logKernelStep } from '../debug-utils.js';
|
|
28
|
-
import { applyLoRA } from '../lora-apply.js';
|
|
29
|
-
import { getLoRAModule } from '../lora.js';
|
|
30
|
-
import { kernelTrace, traceStep } from '../kernel-trace.js';
|
|
31
|
-
import { log, trace } from '../../../../debug/index.js';
|
|
32
|
-
import { selectRuleValue } from '../../../../rules/rule-registry.js';
|
|
33
|
-
import { runProbes } from '../probes.js';
|
|
34
|
-
import {
|
|
35
|
-
recordAttentionInputs,
|
|
36
|
-
shouldForceF32AttentionProjectionForRoPE,
|
|
37
|
-
resolveAttentionProjectionOutputDtype,
|
|
38
|
-
projectAttentionQKV,
|
|
39
|
-
applyAttentionQKNorm,
|
|
40
|
-
applyAttentionValueNorm,
|
|
41
|
-
hasAttentionProjectionDiagnostics,
|
|
42
|
-
hasAttentionStageDiagnostics,
|
|
43
|
-
resolveAttentionQKNormState,
|
|
44
|
-
} from './projections.js';
|
|
45
|
-
import { prepareAttentionProjectionInput } from './output-projection.js';
|
|
46
|
-
|
|
47
|
-
import {
|
|
48
|
-
shouldDebugLayer,
|
|
49
|
-
markStageLogged,
|
|
50
|
-
} from './types.js';
|
|
51
|
-
import {
|
|
52
|
-
getKernelPathMatmulPrecision,
|
|
53
|
-
getKernelPathMatmulVariant,
|
|
54
|
-
} from '../../../../config/kernel-path-loader.js';
|
|
55
|
-
import {
|
|
56
|
-
resolveKVCacheState,
|
|
57
|
-
createDiffusionGemmaDecoderKVState,
|
|
58
|
-
buildAttentionDispatchParams,
|
|
59
|
-
buildAttentionInputsData,
|
|
60
|
-
} from './dispatch-params.js';
|
|
61
|
-
import {
|
|
62
|
-
buildTieredQuantAttentionOptions,
|
|
63
|
-
buildContiguousQuantAttentionOptions,
|
|
64
|
-
} from './quant-options.js';
|
|
65
|
-
import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
|
|
66
|
-
import {
|
|
67
|
-
resolveAttentionPrecisionContract,
|
|
68
|
-
isAttentionKvDtypeExplicit,
|
|
69
|
-
} from './precision-contract.js';
|
|
70
|
-
import { canUseRmsNormWideTileProjectionFusion } from './rmsnorm-fusion-gate.js';
|
|
71
|
-
import { canUseAttentionOutputGateFusion } from './output-gate-fusion.js';
|
|
72
|
-
import { getVectorTensor } from '../weights.js';
|
|
73
|
-
|
|
74
|
-
const ATTENTION_DTYPE_LOGGED = new Set();
|
|
75
|
-
|
|
76
|
-
function isWideTileQ4KPhaseEnabled(session, phase) {
|
|
77
|
-
return phase === 'decode'
|
|
78
|
-
? session?.useWideTileQ4KDecode === true
|
|
79
|
-
: session?.useWideTileQ4KPrefill === true;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function canUseQ4KWideTileResidualFusion(options = {}) {
|
|
83
|
-
return options.hasResidual === true
|
|
84
|
-
&& options.oProjDtype === 'q4k'
|
|
85
|
-
&& options.inputDtype === 'f32'
|
|
86
|
-
&& options.outputDtype === 'f32'
|
|
87
|
-
&& options.residualMatches === true
|
|
88
|
-
&& options.hasLoRA !== true
|
|
89
|
-
&& getKernelCapabilities().hasF16 === true
|
|
90
|
-
&& getKernelCapabilities().hasSubgroups === true
|
|
91
|
-
&& options.session?.useWideTileResidualFusion === true
|
|
92
|
-
&& options.session?.retainQ4KMaterialization === true
|
|
93
|
-
&& isWideTileQ4KPhaseEnabled(options.session, options.phase);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function assertAttentionDtypeTransitionAllowed(state, fromDtype, toDtype, detail, transitionDeclaredBy = null) {
|
|
97
|
-
assertImplicitDtypeTransitionAllowed({
|
|
98
|
-
executionPolicies: state?.executionPolicies ?? null,
|
|
99
|
-
fromDtype,
|
|
100
|
-
toDtype,
|
|
101
|
-
op: 'attention',
|
|
102
|
-
detail,
|
|
103
|
-
transitionDeclaredBy,
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
export async function runLayerAttentionGPU(
|
|
108
|
-
input,
|
|
109
|
-
layerWeights,
|
|
110
|
-
config,
|
|
111
|
-
state,
|
|
112
|
-
debug = false,
|
|
113
|
-
debugFlags = {},
|
|
114
|
-
getWeightBuffer,
|
|
115
|
-
getNormWeightBuffer,
|
|
116
|
-
debugCheckBuffer,
|
|
117
|
-
lora
|
|
118
|
-
) {
|
|
119
|
-
const {
|
|
120
|
-
layerIdx,
|
|
121
|
-
numTokens,
|
|
122
|
-
isPrefill,
|
|
123
|
-
numHeads,
|
|
124
|
-
numKVHeads,
|
|
125
|
-
headDim,
|
|
126
|
-
hiddenSize,
|
|
127
|
-
rmsNormEps,
|
|
128
|
-
currentSeqLen,
|
|
129
|
-
slidingWindow,
|
|
130
|
-
layerType,
|
|
131
|
-
residualTensor,
|
|
132
|
-
attnSoftcap,
|
|
133
|
-
queryPreAttnScalar,
|
|
134
|
-
skipInputNorm = false,
|
|
135
|
-
tokenIds = null,
|
|
136
|
-
kernelPath = null,
|
|
137
|
-
disableRoPE = false,
|
|
138
|
-
multimodalBidirectionalSpan = null,
|
|
139
|
-
sharedKVSourceLayerIdx = null,
|
|
140
|
-
storeSharedKV = false,
|
|
141
|
-
diffusionGemmaDecoder = false,
|
|
142
|
-
precomputedInputNorm = null,
|
|
143
|
-
} = config;
|
|
144
|
-
|
|
145
|
-
const device = getDevice();
|
|
146
|
-
const phase = isPrefill ? 'prefill' : 'decode';
|
|
147
|
-
const attentionPrecisionContract = resolveAttentionPrecisionContract(config, state);
|
|
148
|
-
const attentionActivationDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
|
|
149
|
-
dtype: attentionPrecisionContract.resolvedActivationDtype ?? config.activationDtype,
|
|
150
|
-
});
|
|
151
|
-
const oProjPrecision = getKernelPathMatmulPrecision('o_proj', phase, layerIdx, kernelPath);
|
|
152
|
-
const oProjInputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
|
|
153
|
-
dtype: oProjPrecision?.inputDtype ?? attentionActivationDtype,
|
|
154
|
-
});
|
|
155
|
-
const oProjOutputDtype = selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', {
|
|
156
|
-
dtype: oProjPrecision?.outputDtype
|
|
157
|
-
?? attentionPrecisionContract.resolvedOutputDtype
|
|
158
|
-
?? config.activationDtype,
|
|
159
|
-
});
|
|
160
|
-
const wantsF16Output = oProjOutputDtype === 'f16';
|
|
161
|
-
const useF16Activations = attentionActivationDtype === 'f16';
|
|
162
|
-
const kvCacheFallback = selectRuleValue('inference', 'dtype', 'f16OrF32', { useF16: useF16Activations });
|
|
163
|
-
const kvCacheDtype = attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? kvCacheFallback;
|
|
164
|
-
const allowF16Attention = useF16Activations && kvCacheDtype === 'f16';
|
|
165
|
-
const attentionInputTransitionDeclaredBy = attentionPrecisionContract.explicitInputDtype
|
|
166
|
-
? 'step_precision'
|
|
167
|
-
: null;
|
|
168
|
-
let attentionInput = input;
|
|
169
|
-
let attentionInputTemp = false;
|
|
170
|
-
let normed = attentionInput;
|
|
171
|
-
let qTensor = null;
|
|
172
|
-
let qGateTensor = null;
|
|
173
|
-
let kTensor = null;
|
|
174
|
-
let vTensor = null;
|
|
175
|
-
let attnOutput = null;
|
|
176
|
-
let attnForProjection = null;
|
|
177
|
-
let output = null;
|
|
178
|
-
let finalOutput = null;
|
|
179
|
-
let oProjInputTemp = null;
|
|
180
|
-
let valueAliasesKey = false;
|
|
181
|
-
let decoderKVState = null;
|
|
182
|
-
let decoderKTemp = null;
|
|
183
|
-
let decoderVTemp = null;
|
|
184
|
-
if (!allowF16Attention && input.dtype !== attentionActivationDtype) {
|
|
185
|
-
assertAttentionDtypeTransitionAllowed(
|
|
186
|
-
state,
|
|
187
|
-
input.dtype,
|
|
188
|
-
attentionActivationDtype,
|
|
189
|
-
'The attention kernel selection would widen the input implicitly.',
|
|
190
|
-
attentionInputTransitionDeclaredBy
|
|
191
|
-
);
|
|
192
|
-
attentionInput = attentionActivationDtype === 'f16'
|
|
193
|
-
? await castF32ToF16(input)
|
|
194
|
-
: await castF16ToF32(input);
|
|
195
|
-
attentionInputTemp = true;
|
|
196
|
-
normed = attentionInput;
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
// Debug: attention input for configured layers
|
|
200
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'attn_input', debugFlags) && debugCheckBuffer) {
|
|
201
|
-
await debugCheckBuffer(attentionInput.buffer, `L${layerIdx} attention input (GPU)`, numTokens, hiddenSize);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
// Debug logging moved to debug-utils.ts (enable via setDebugConfig)
|
|
205
|
-
|
|
206
|
-
if (!layerWeights) {
|
|
207
|
-
const bytesPerElement = wantsF16Output ? 2 : 4;
|
|
208
|
-
const outputBuf = acquireBuffer(numTokens * hiddenSize * bytesPerElement, undefined, 'attn_output');
|
|
209
|
-
const output = createTensor(outputBuf, oProjOutputDtype, [numTokens, hiddenSize], 'attn_output');
|
|
210
|
-
return { output, residualFused: false };
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
const qSize = numTokens * numHeads * headDim;
|
|
214
|
-
const kvSize = numTokens * numKVHeads * headDim;
|
|
215
|
-
const sharedKVEntry = sharedKVSourceLayerIdx == null
|
|
216
|
-
? null
|
|
217
|
-
: (state.sharedAttentionState?.get(sharedKVSourceLayerIdx) ?? null);
|
|
218
|
-
if (sharedKVSourceLayerIdx != null && !sharedKVEntry) {
|
|
219
|
-
throw new Error(
|
|
220
|
-
`Layer ${layerIdx} requires shared K/V from layer ${sharedKVSourceLayerIdx}, ` +
|
|
221
|
-
'but no shared K/V state was stored for that source layer.'
|
|
222
|
-
);
|
|
223
|
-
}
|
|
224
|
-
if (sharedKVEntry && (
|
|
225
|
-
sharedKVEntry.headDim !== headDim
|
|
226
|
-
|| sharedKVEntry.numKVHeads !== numKVHeads
|
|
227
|
-
)) {
|
|
228
|
-
throw new Error(
|
|
229
|
-
`Layer ${layerIdx} shared K/V geometry mismatch. ` +
|
|
230
|
-
`Expected numKVHeads=${numKVHeads}, headDim=${headDim}; ` +
|
|
231
|
-
`got numKVHeads=${sharedKVEntry.numKVHeads}, headDim=${sharedKVEntry.headDim}.`
|
|
232
|
-
);
|
|
233
|
-
}
|
|
234
|
-
const reusesSharedKV = sharedKVEntry != null;
|
|
235
|
-
|
|
236
|
-
// 1. Input norm
|
|
237
|
-
// Opt-in fusion: when useFusedRmsnormWideTile is set AND preconditions match,
|
|
238
|
-
// defer the rmsnorm into each q/k/v_proj matmul via a dedicated WideTile
|
|
239
|
-
// variant. Saves one standalone rmsnorm dispatch per layer.
|
|
240
|
-
let fusedNormWeight = null;
|
|
241
|
-
let fusedNormEps = null;
|
|
242
|
-
let fusedNormOffset = false;
|
|
243
|
-
let fusedNormWeightOwned = false;
|
|
244
|
-
// state.runtimeConfig.inference.session is a stale snapshot ({}); the live
|
|
245
|
-
// module-level runtime config carries the merged profile/override session.
|
|
246
|
-
const rmsNormFusionFlag = getRuntimeConfig()?.inference?.session?.useFusedRmsnormWideTile === true;
|
|
247
|
-
const hasInputNormWeight = layerWeights?.inputNorm != null && getNormWeightBuffer != null;
|
|
248
|
-
const canSelectFusedRmsNormProjection = canUseRmsNormWideTileProjectionFusion(layerWeights, reusesSharedKV);
|
|
249
|
-
const canFuseInputNormIntoProjections = rmsNormFusionFlag
|
|
250
|
-
&& config.normalizationType === 'rmsnorm'
|
|
251
|
-
&& hasInputNormWeight
|
|
252
|
-
&& canSelectFusedRmsNormProjection
|
|
253
|
-
&& !skipInputNorm
|
|
254
|
-
&& isPrefill
|
|
255
|
-
&& numTokens > 1
|
|
256
|
-
&& attentionInput.dtype === 'f32';
|
|
257
|
-
|
|
258
|
-
try {
|
|
259
|
-
if (precomputedInputNorm) {
|
|
260
|
-
if (
|
|
261
|
-
isPrefill
|
|
262
|
-
|| numTokens !== 1
|
|
263
|
-
|| skipInputNorm
|
|
264
|
-
|| !layerWeights.inputNorm
|
|
265
|
-
|| !getNormWeightBuffer
|
|
266
|
-
|| precomputedInputNorm.dtype !== attentionInput.dtype
|
|
267
|
-
|| precomputedInputNorm.dtype !== 'f32'
|
|
268
|
-
) {
|
|
269
|
-
throw new Error(`Layer ${layerIdx} received an incompatible precomputed input norm tensor.`);
|
|
270
|
-
}
|
|
271
|
-
normed = precomputedInputNorm;
|
|
272
|
-
await runProbes('post_input_norm', normed.buffer, {
|
|
273
|
-
layerIdx,
|
|
274
|
-
numTokens,
|
|
275
|
-
hiddenSize,
|
|
276
|
-
probes: state.debugProbes,
|
|
277
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
278
|
-
dtype: normed.dtype,
|
|
279
|
-
});
|
|
280
|
-
} else if (canFuseInputNormIntoProjections) {
|
|
281
|
-
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
282
|
-
fusedNormWeight = normWeightBuf;
|
|
283
|
-
fusedNormEps = rmsNormEps;
|
|
284
|
-
fusedNormOffset = config.rmsNormWeightOffset === true;
|
|
285
|
-
fusedNormWeightOwned = !isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm);
|
|
286
|
-
// Keep normed = attentionInput (raw). Each q/k/v_proj matmul runs the
|
|
287
|
-
// norm internally via the fused kernel variant.
|
|
288
|
-
} else if (!skipInputNorm && layerWeights.inputNorm && getNormWeightBuffer) {
|
|
289
|
-
const normWeightBuf = getNormWeightBuffer(layerWeights.inputNorm, 'input_norm');
|
|
290
|
-
let normBiasTensor = null;
|
|
291
|
-
let normBiasOwned = false;
|
|
292
|
-
try {
|
|
293
|
-
// Debug: norm weights for configured layers
|
|
294
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'norm_weights', debugFlags) && debugCheckBuffer) {
|
|
295
|
-
await debugCheckBuffer(normWeightBuf, `L${layerIdx} input norm weights (GPU)`, 1, hiddenSize);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
if (config.normalizationType === 'layernorm') {
|
|
299
|
-
if (!layerWeights.inputNormBias) {
|
|
300
|
-
throw new Error(`Layer ${layerIdx} requires input_layernorm.bias for LayerNorm execution.`);
|
|
301
|
-
}
|
|
302
|
-
({ tensor: normBiasTensor, owned: normBiasOwned } = getVectorTensor(
|
|
303
|
-
layerWeights.inputNormBias,
|
|
304
|
-
`L${layerIdx}.input_norm_bias`,
|
|
305
|
-
hiddenSize,
|
|
306
|
-
{},
|
|
307
|
-
debugFlags
|
|
308
|
-
));
|
|
309
|
-
normed = await runLayerNorm(
|
|
310
|
-
attentionInput,
|
|
311
|
-
normWeightBuf,
|
|
312
|
-
normBiasTensor.buffer,
|
|
313
|
-
rmsNormEps,
|
|
314
|
-
{ batchSize: numTokens, hiddenSize }
|
|
315
|
-
);
|
|
316
|
-
} else {
|
|
317
|
-
normed = await runRMSNorm(attentionInput, normWeightBuf, rmsNormEps, {
|
|
318
|
-
batchSize: numTokens,
|
|
319
|
-
hiddenSize,
|
|
320
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
} finally {
|
|
324
|
-
if (normBiasOwned && normBiasTensor) releaseBuffer(normBiasTensor.buffer);
|
|
325
|
-
if (!isGpuBufferInstance(layerWeights.inputNorm) && !isWeightBuffer(layerWeights.inputNorm)) releaseBuffer(normWeightBuf);
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
// Trace input norm output
|
|
329
|
-
if (kernelTrace.enabled) {
|
|
330
|
-
await traceStep('rmsnorm', `L${layerIdx}.input_norm`, layerIdx, normed.buffer, [numTokens, hiddenSize]);
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
334
|
-
logKernelStep('rmsnorm', { layerIdx, label: 'input_norm', size: numTokens * hiddenSize });
|
|
335
|
-
await dumpTokenVector(normed.buffer, 'input_norm_out', {
|
|
336
|
-
layerIdx,
|
|
337
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
338
|
-
rowSize: hiddenSize,
|
|
339
|
-
dtype: normed.dtype,
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
await runProbes('post_input_norm', normed.buffer, {
|
|
344
|
-
layerIdx,
|
|
345
|
-
numTokens,
|
|
346
|
-
hiddenSize,
|
|
347
|
-
probes: state.debugProbes,
|
|
348
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
349
|
-
dtype: normed.dtype,
|
|
350
|
-
});
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
// Debug: Check normed input for L0 prefill
|
|
354
|
-
// Debug: normed input for configured layers
|
|
355
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'attn_normed', debugFlags) && debugCheckBuffer) {
|
|
356
|
-
await debugCheckBuffer(normed.buffer, `L${layerIdx} normed input (GPU)`, numTokens);
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
const debugLayers = debugFlags.debugLayers;
|
|
360
|
-
const shouldLogLayer = debugLayers === null ? layerIdx === 0 : shouldDebugLayer(layerIdx, debugLayers);
|
|
361
|
-
if (shouldLogLayer) {
|
|
362
|
-
const phase = selectRuleValue('kernels', 'attention', 'phase', { isDecode: !isPrefill });
|
|
363
|
-
const logKey = `L${layerIdx}_${phase}_dtypes`;
|
|
364
|
-
if (!ATTENTION_DTYPE_LOGGED.has(logKey)) {
|
|
365
|
-
ATTENTION_DTYPE_LOGGED.add(logKey);
|
|
366
|
-
trace.attn(layerIdx, `dtypes: activation=${attentionActivationDtype}, input=${input.dtype}, normed=${normed.dtype}`);
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
371
|
-
await dumpTokenVector(normed.buffer, 'attn_in', {
|
|
372
|
-
layerIdx,
|
|
373
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
374
|
-
rowSize: hiddenSize,
|
|
375
|
-
dtype: normed.dtype,
|
|
376
|
-
});
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
// 2. Q/K/V projections
|
|
380
|
-
const qProjVariant = getKernelPathMatmulVariant('q_proj', phase, layerIdx, kernelPath);
|
|
381
|
-
const kernelPathIsF16 = qProjVariant != null && qProjVariant.includes('f16') && !qProjVariant.includes('f32');
|
|
382
|
-
const matmulOutputDtype = resolveAttentionProjectionOutputDtype(attentionActivationDtype, {
|
|
383
|
-
forceF32: shouldForceF32AttentionProjectionForRoPE({
|
|
384
|
-
attentionInputDtype: attentionActivationDtype,
|
|
385
|
-
headDim,
|
|
386
|
-
rotaryDim: config.ropeRotaryDim,
|
|
387
|
-
interleaved: config.ropeInterleaved,
|
|
388
|
-
kernelPathIsF16,
|
|
389
|
-
}),
|
|
390
|
-
});
|
|
391
|
-
let usedFusedQKV = false;
|
|
392
|
-
let qkNormApplied = false;
|
|
393
|
-
let ropeApplied = false;
|
|
394
|
-
const qkNormState = resolveAttentionQKNormState({ config, layerWeights, layerIdx, reusesSharedKV });
|
|
395
|
-
const qkNormProjectionDiagnostics = hasAttentionProjectionDiagnostics(state)
|
|
396
|
-
|| kernelTrace.enabled
|
|
397
|
-
|| isKernelDebugEnabled(layerIdx)
|
|
398
|
-
|| (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
|
|
399
|
-
const qkNormRoPEDiagnostics = hasAttentionStageDiagnostics(
|
|
400
|
-
state,
|
|
401
|
-
['q_proj', 'k_proj', 'v_proj', 'q_norm', 'k_norm']
|
|
402
|
-
)
|
|
403
|
-
|| kernelTrace.enabled
|
|
404
|
-
|| isKernelDebugEnabled(layerIdx)
|
|
405
|
-
|| (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !!debugCheckBuffer);
|
|
406
|
-
const runtimeSession = getRuntimeConfig()?.inference?.session;
|
|
407
|
-
const qkNormFusionFlag = runtimeSession?.useFusedQKVSplitQKNorm === true;
|
|
408
|
-
const qkNormRoPEFusionFlag = runtimeSession?.useFusedQKVSplitQKNormRoPE === true;
|
|
409
|
-
({ qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV, valueAliasesKey, qkNormApplied, ropeApplied } = await projectAttentionQKV({
|
|
410
|
-
recorder: null,
|
|
411
|
-
normed,
|
|
412
|
-
layerWeights,
|
|
413
|
-
numTokens,
|
|
414
|
-
numHeads,
|
|
415
|
-
numKVHeads,
|
|
416
|
-
headDim,
|
|
417
|
-
hiddenSize,
|
|
418
|
-
layerIdx,
|
|
419
|
-
kernelPath,
|
|
420
|
-
matmulOutputDtype,
|
|
421
|
-
getWeightBuffer,
|
|
422
|
-
lora,
|
|
423
|
-
matmulDebug: state.runtimeConfig?.shared?.debug?.matmul ?? null,
|
|
424
|
-
attentionOutputGate: config.attentionOutputGate === true,
|
|
425
|
-
sharedKTensor: sharedKVEntry?.kTensor ?? null,
|
|
426
|
-
sharedVTensor: sharedKVEntry?.vTensor ?? null,
|
|
427
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
428
|
-
releaseTemporary: (buffer) => releaseBuffer(buffer),
|
|
429
|
-
onFusedQKV: layerIdx === 0 && isPrefill
|
|
430
|
-
? ({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize }) => {
|
|
431
|
-
trace.attn(layerIdx, `Using fused QKV path: ${qSizeFused}+${kSizeFused}+${vSizeFused}=${totalSize}`);
|
|
432
|
-
}
|
|
433
|
-
: null,
|
|
434
|
-
fusedNormWeight,
|
|
435
|
-
fusedNormEps,
|
|
436
|
-
fusedNormOffset,
|
|
437
|
-
qkNormFusion: {
|
|
438
|
-
enabled: qkNormFusionFlag
|
|
439
|
-
&& qkNormState.wantsQKNorm
|
|
440
|
-
&& config.queryKeyNormType === 'rmsnorm'
|
|
441
|
-
&& config.queryKeyNormAxis === 'head',
|
|
442
|
-
getNormWeightBuffer,
|
|
443
|
-
rmsNormEps,
|
|
444
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
445
|
-
skipKNorm: qkNormState.skipKNorm,
|
|
446
|
-
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
447
|
-
projectionDiagnosticsEnabled: qkNormProjectionDiagnostics,
|
|
448
|
-
},
|
|
449
|
-
qkNormRoPEFusion: {
|
|
450
|
-
enabled: qkNormRoPEFusionFlag
|
|
451
|
-
&& qkNormState.wantsQKNorm
|
|
452
|
-
&& config.queryKeyNormType === 'rmsnorm'
|
|
453
|
-
&& config.queryKeyNormAxis === 'head'
|
|
454
|
-
&& !disableRoPE
|
|
455
|
-
&& !!state.ropeFreqsCos
|
|
456
|
-
&& !!state.ropeFreqsSin,
|
|
457
|
-
getNormWeightBuffer,
|
|
458
|
-
rmsNormEps,
|
|
459
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
460
|
-
skipKNorm: qkNormState.skipKNorm,
|
|
461
|
-
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
462
|
-
projectionDiagnosticsEnabled: qkNormRoPEDiagnostics,
|
|
463
|
-
freqsCos: state.ropeFreqsCos,
|
|
464
|
-
freqsSin: state.ropeFreqsSin,
|
|
465
|
-
startPos: currentSeqLen,
|
|
466
|
-
rotaryDim: config.ropeRotaryDim,
|
|
467
|
-
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
468
|
-
interleaved: config.ropeInterleaved,
|
|
469
|
-
reusesSharedKV,
|
|
470
|
-
},
|
|
471
|
-
}));
|
|
472
|
-
// Release the in-flight norm weight buffer used by the fused projections.
|
|
473
|
-
if (fusedNormWeight && fusedNormWeightOwned) {
|
|
474
|
-
releaseBuffer(fusedNormWeight);
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
// Trace Q/K/V projections
|
|
478
|
-
if (kernelTrace.enabled) {
|
|
479
|
-
await traceStep('matmul', `L${layerIdx}.q_proj`, layerIdx, qTensor.buffer, [numTokens, numHeads * headDim]);
|
|
480
|
-
await traceStep('matmul', `L${layerIdx}.k_proj`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
|
|
481
|
-
await traceStep('matmul', `L${layerIdx}.v_proj`, layerIdx, vTensor.buffer, [numTokens, numKVHeads * headDim]);
|
|
482
|
-
}
|
|
483
|
-
await runProbes('q_proj', qTensor.buffer, {
|
|
484
|
-
layerIdx,
|
|
485
|
-
numTokens,
|
|
486
|
-
hiddenSize: numHeads * headDim,
|
|
487
|
-
probes: state.debugProbes,
|
|
488
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
489
|
-
dtype: qTensor.dtype,
|
|
490
|
-
});
|
|
491
|
-
if (!reusesSharedKV) {
|
|
492
|
-
await runProbes('k_proj', kTensor.buffer, {
|
|
493
|
-
layerIdx,
|
|
494
|
-
numTokens,
|
|
495
|
-
hiddenSize: numKVHeads * headDim,
|
|
496
|
-
probes: state.debugProbes,
|
|
497
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
498
|
-
dtype: kTensor.dtype,
|
|
499
|
-
});
|
|
500
|
-
await runProbes('v_proj', vTensor.buffer, {
|
|
501
|
-
layerIdx,
|
|
502
|
-
numTokens,
|
|
503
|
-
hiddenSize: numKVHeads * headDim,
|
|
504
|
-
probes: state.debugProbes,
|
|
505
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
506
|
-
dtype: vTensor.dtype,
|
|
507
|
-
});
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
// Kernel step debug: Q/K/V projections
|
|
511
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
512
|
-
logKernelStep('matmul', { layerIdx, label: 'Q_proj', M: numTokens, N: numHeads * headDim, K: hiddenSize });
|
|
513
|
-
await dumpTokenVector(qTensor.buffer, 'Q_proj', {
|
|
514
|
-
layerIdx,
|
|
515
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
516
|
-
rowSize: numHeads * headDim,
|
|
517
|
-
dtype: qTensor.dtype,
|
|
518
|
-
});
|
|
519
|
-
logKernelStep('matmul', { layerIdx, label: 'K_proj', M: numTokens, N: numKVHeads * headDim, K: hiddenSize });
|
|
520
|
-
await dumpTokenVector(kTensor.buffer, 'K_proj', {
|
|
521
|
-
layerIdx,
|
|
522
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
523
|
-
rowSize: numKVHeads * headDim,
|
|
524
|
-
dtype: kTensor.dtype,
|
|
525
|
-
});
|
|
526
|
-
logKernelStep('matmul', { layerIdx, label: 'V_proj', M: numTokens, N: numKVHeads * headDim, K: hiddenSize });
|
|
527
|
-
await dumpTokenVector(vTensor.buffer, 'V_proj', {
|
|
528
|
-
layerIdx,
|
|
529
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
530
|
-
rowSize: numKVHeads * headDim,
|
|
531
|
-
dtype: vTensor.dtype,
|
|
532
|
-
});
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
// Debug: Check Q/K/V after projections for L0 prefill
|
|
536
|
-
// Debug: Q/K/V projections for configured layers
|
|
537
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'qkv_proj', debugFlags) && debugCheckBuffer) {
|
|
538
|
-
await debugCheckBuffer(qTensor.buffer, `L${layerIdx} Q after proj (GPU)`, numTokens, numHeads * headDim);
|
|
539
|
-
await debugCheckBuffer(kTensor.buffer, `L${layerIdx} K after proj (GPU)`, numTokens, numKVHeads * headDim);
|
|
540
|
-
await debugCheckBuffer(vTensor.buffer, `L${layerIdx} V after proj (GPU)`, numTokens, numKVHeads * headDim);
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
// Optional Q/K normalization; the manifest owns the axis and norm family.
|
|
544
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
545
|
-
logKernelStep(
|
|
546
|
-
'qk_norm',
|
|
547
|
-
{
|
|
548
|
-
layerIdx,
|
|
549
|
-
label: `hasQ=${qkNormState.hasQNorm} hasK=${qkNormState.hasKNorm} ` +
|
|
550
|
-
`wants=${qkNormState.wantsQKNorm} unit=${qkNormState.allowUnitQKNorm}`,
|
|
551
|
-
}
|
|
552
|
-
);
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
if (qkNormState.wantsQKNorm && !qkNormApplied) {
|
|
556
|
-
({ qTensor, kTensor } = await applyAttentionQKNorm({
|
|
557
|
-
recorder: null,
|
|
558
|
-
qTensor,
|
|
559
|
-
kTensor,
|
|
560
|
-
layerWeights,
|
|
561
|
-
getNormWeightBuffer,
|
|
562
|
-
rmsNormEps,
|
|
563
|
-
numTokens,
|
|
564
|
-
numHeads,
|
|
565
|
-
numKVHeads,
|
|
566
|
-
headDim,
|
|
567
|
-
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
568
|
-
normalizationType: config.queryKeyNormType,
|
|
569
|
-
normalizationAxis: config.queryKeyNormAxis,
|
|
570
|
-
releaseTemporary: (buffer) => releaseBuffer(buffer),
|
|
571
|
-
skipKNorm: qkNormState.skipKNorm,
|
|
572
|
-
onQNormApplied: isKernelDebugEnabled(layerIdx)
|
|
573
|
-
? async (tensor) => {
|
|
574
|
-
await dumpTokenVector(tensor.buffer, 'Q_norm', {
|
|
575
|
-
layerIdx,
|
|
576
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
577
|
-
rowSize: numHeads * headDim,
|
|
578
|
-
dtype: tensor.dtype,
|
|
579
|
-
});
|
|
580
|
-
}
|
|
581
|
-
: null,
|
|
582
|
-
onKNormApplied: isKernelDebugEnabled(layerIdx)
|
|
583
|
-
? async (tensor) => {
|
|
584
|
-
await dumpTokenVector(tensor.buffer, 'K_norm', {
|
|
585
|
-
layerIdx,
|
|
586
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
587
|
-
rowSize: numKVHeads * headDim,
|
|
588
|
-
dtype: tensor.dtype,
|
|
589
|
-
});
|
|
590
|
-
}
|
|
591
|
-
: null,
|
|
592
|
-
retainKInput: valueAliasesKey,
|
|
593
|
-
allowUnitQKNorm: qkNormState.allowUnitQKNorm,
|
|
594
|
-
}));
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
if (config.valueNorm === true && !reusesSharedKV) {
|
|
598
|
-
const valueNormInputAliasesKey = vTensor.buffer === kTensor.buffer;
|
|
599
|
-
vTensor = await applyAttentionValueNorm({
|
|
600
|
-
recorder: null,
|
|
601
|
-
vTensor,
|
|
602
|
-
rmsNormEps,
|
|
603
|
-
numTokens,
|
|
604
|
-
numKVHeads,
|
|
605
|
-
headDim,
|
|
606
|
-
releaseTemporary: (buffer) => {
|
|
607
|
-
if (!valueNormInputAliasesKey) {
|
|
608
|
-
releaseBuffer(buffer);
|
|
609
|
-
}
|
|
610
|
-
},
|
|
611
|
-
});
|
|
612
|
-
await runProbes('v_norm', vTensor.buffer, {
|
|
613
|
-
layerIdx,
|
|
614
|
-
numTokens,
|
|
615
|
-
hiddenSize: numKVHeads * headDim,
|
|
616
|
-
probes: state.debugProbes,
|
|
617
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
618
|
-
dtype: vTensor.dtype,
|
|
619
|
-
});
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
if (qkNormState.wantsQKNorm) {
|
|
623
|
-
await runProbes('q_norm', qTensor.buffer, {
|
|
624
|
-
layerIdx,
|
|
625
|
-
numTokens,
|
|
626
|
-
hiddenSize: numHeads * headDim,
|
|
627
|
-
probes: state.debugProbes,
|
|
628
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
629
|
-
dtype: qTensor.dtype,
|
|
630
|
-
});
|
|
631
|
-
if (!reusesSharedKV) {
|
|
632
|
-
await runProbes('k_norm', kTensor.buffer, {
|
|
633
|
-
layerIdx,
|
|
634
|
-
numTokens,
|
|
635
|
-
hiddenSize: numKVHeads * headDim,
|
|
636
|
-
probes: state.debugProbes,
|
|
637
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
638
|
-
dtype: kTensor.dtype,
|
|
639
|
-
});
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
if (normed !== attentionInput) releaseBuffer(normed.buffer);
|
|
644
|
-
if (attentionInputTemp) releaseBuffer(attentionInput.buffer);
|
|
645
|
-
|
|
646
|
-
// 3. RoPE (modifies tensor in-place)
|
|
647
|
-
|
|
648
|
-
if (!ropeApplied && !disableRoPE && state.ropeFreqsCos && state.ropeFreqsSin) {
|
|
649
|
-
const ropeOptions = {
|
|
650
|
-
headDim,
|
|
651
|
-
rotaryDim: config.ropeRotaryDim,
|
|
652
|
-
pairSpanDim: config.ropeFrequencyBaseDim ?? config.ropeRotaryDim,
|
|
653
|
-
interleaved: config.ropeInterleaved,
|
|
654
|
-
startPos: currentSeqLen,
|
|
655
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
656
|
-
};
|
|
657
|
-
if (canUseRoPEQK(qTensor, kTensor, { reusesSharedKV })) {
|
|
658
|
-
await runRoPEQK(qTensor, kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
659
|
-
...ropeOptions,
|
|
660
|
-
numQHeads: numHeads,
|
|
661
|
-
numKVHeads,
|
|
662
|
-
});
|
|
663
|
-
} else {
|
|
664
|
-
await runRoPE(qTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
665
|
-
...ropeOptions,
|
|
666
|
-
numHeads,
|
|
667
|
-
});
|
|
668
|
-
if (!reusesSharedKV) {
|
|
669
|
-
await runRoPE(kTensor, state.ropeFreqsCos, state.ropeFreqsSin, numTokens, {
|
|
670
|
-
...ropeOptions,
|
|
671
|
-
numHeads: numKVHeads,
|
|
672
|
-
});
|
|
673
|
-
}
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
// Trace RoPE outputs
|
|
677
|
-
if (kernelTrace.enabled) {
|
|
678
|
-
await traceStep('rope', `L${layerIdx}.q_rope`, layerIdx, qTensor.buffer, [numTokens, numHeads * headDim]);
|
|
679
|
-
if (!reusesSharedKV) {
|
|
680
|
-
await traceStep('rope', `L${layerIdx}.k_rope`, layerIdx, kTensor.buffer, [numTokens, numKVHeads * headDim]);
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
await runProbes('q_rope', qTensor.buffer, {
|
|
685
|
-
layerIdx,
|
|
686
|
-
numTokens,
|
|
687
|
-
hiddenSize: numHeads * headDim,
|
|
688
|
-
probes: state.debugProbes,
|
|
689
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
690
|
-
dtype: qTensor.dtype,
|
|
691
|
-
});
|
|
692
|
-
await runProbes('k_rope', kTensor.buffer, {
|
|
693
|
-
layerIdx,
|
|
694
|
-
numTokens,
|
|
695
|
-
hiddenSize: numKVHeads * headDim,
|
|
696
|
-
probes: state.debugProbes,
|
|
697
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
698
|
-
dtype: kTensor.dtype,
|
|
699
|
-
});
|
|
700
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
701
|
-
logKernelStep('rope', { layerIdx, label: `startPos=${currentSeqLen}` });
|
|
702
|
-
await dumpTokenVector(qTensor.buffer, 'Q_rope', {
|
|
703
|
-
layerIdx,
|
|
704
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
705
|
-
rowSize: numHeads * headDim,
|
|
706
|
-
dtype: qTensor.dtype,
|
|
707
|
-
});
|
|
708
|
-
await dumpTokenVector(kTensor.buffer, 'K_rope', {
|
|
709
|
-
layerIdx,
|
|
710
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
711
|
-
rowSize: numKVHeads * headDim,
|
|
712
|
-
dtype: kTensor.dtype,
|
|
713
|
-
});
|
|
714
|
-
}
|
|
715
|
-
|
|
716
|
-
if (storeSharedKV && state.sharedAttentionState) {
|
|
717
|
-
state.sharedAttentionState.set(layerIdx, {
|
|
718
|
-
kTensor,
|
|
719
|
-
vTensor,
|
|
720
|
-
headDim,
|
|
721
|
-
numKVHeads,
|
|
722
|
-
});
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
// Debug: Check Q/K after RoPE for L0 prefill
|
|
726
|
-
// Debug: Q/K after RoPE for configured layers
|
|
727
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'qk_rope', debugFlags) && debugCheckBuffer) {
|
|
728
|
-
await debugCheckBuffer(qTensor.buffer, `L${layerIdx} Q after RoPE (GPU)`, numTokens, numHeads * headDim);
|
|
729
|
-
await debugCheckBuffer(kTensor.buffer, `L${layerIdx} K after RoPE (GPU)`, numTokens, numKVHeads * headDim);
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
// 4. Update KV cache (cache stores raw GPUBuffers for memory efficiency)
|
|
733
|
-
|
|
734
|
-
if (!diffusionGemmaDecoder && state.kvCache?.hasGPUCache?.()) {
|
|
735
|
-
if (state.kvCache.kvDtype === 'f16') {
|
|
736
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
737
|
-
if (kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
738
|
-
assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for KV cache storage.');
|
|
739
|
-
}
|
|
740
|
-
if (vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
741
|
-
assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for KV cache storage.');
|
|
742
|
-
}
|
|
743
|
-
const kCasted = kTensor.dtype === 'f16' ? kTensor : await castF32ToF16(kTensor);
|
|
744
|
-
const vCasted = vTensor.dtype === 'f16' ? vTensor : await castF32ToF16(vTensor);
|
|
745
|
-
|
|
746
|
-
await state.kvCache.updateFromGPU(layerIdx, kCasted.buffer, vCasted.buffer, currentSeqLen, numTokens, tokenIds);
|
|
747
|
-
|
|
748
|
-
if (kTensor.dtype !== 'f16') releaseBuffer(kCasted.buffer);
|
|
749
|
-
if (vTensor.dtype !== 'f16') releaseBuffer(vCasted.buffer);
|
|
750
|
-
} else {
|
|
751
|
-
await state.kvCache.updateFromGPU(layerIdx, kTensor.buffer, vTensor.buffer, currentSeqLen, numTokens, tokenIds);
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
// Kernel step debug: KV cache state after update
|
|
755
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
756
|
-
trace.kv(layerIdx, `KV cache updated: startPos=${currentSeqLen}, numTokens=${numTokens}`);
|
|
757
|
-
await dumpKVCache(( (state.kvCache)), layerIdx);
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
// Resolve KV cache state and build dispatch parameters (shared with record.js)
|
|
762
|
-
let kvState;
|
|
763
|
-
if (diffusionGemmaDecoder) {
|
|
764
|
-
const decoderKVDtype = kvCacheDtype ?? kTensor.dtype;
|
|
765
|
-
let decoderKTensor = kTensor;
|
|
766
|
-
let decoderVTensor = vTensor;
|
|
767
|
-
if (decoderKVDtype === 'f16' && decoderKTensor.dtype !== 'f16') {
|
|
768
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
769
|
-
if (!hasExplicitF16KvContract) {
|
|
770
|
-
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f16', 'DiffusionGemma decoder K would be narrowed implicitly.');
|
|
771
|
-
}
|
|
772
|
-
decoderKTensor = await castF32ToF16(decoderKTensor);
|
|
773
|
-
decoderKTemp = decoderKTensor;
|
|
774
|
-
} else if (decoderKVDtype === 'f32' && decoderKTensor.dtype !== 'f32') {
|
|
775
|
-
assertAttentionDtypeTransitionAllowed(state, decoderKTensor.dtype, 'f32', 'DiffusionGemma decoder K would be widened implicitly.');
|
|
776
|
-
decoderKTensor = await castF16ToF32(decoderKTensor);
|
|
777
|
-
decoderKTemp = decoderKTensor;
|
|
778
|
-
}
|
|
779
|
-
if (decoderKVDtype === 'f16' && decoderVTensor.dtype !== 'f16') {
|
|
780
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
781
|
-
if (!hasExplicitF16KvContract) {
|
|
782
|
-
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f16', 'DiffusionGemma decoder V would be narrowed implicitly.');
|
|
783
|
-
}
|
|
784
|
-
decoderVTensor = await castF32ToF16(decoderVTensor);
|
|
785
|
-
decoderVTemp = decoderVTensor;
|
|
786
|
-
} else if (decoderKVDtype === 'f32' && decoderVTensor.dtype !== 'f32') {
|
|
787
|
-
assertAttentionDtypeTransitionAllowed(state, decoderVTensor.dtype, 'f32', 'DiffusionGemma decoder V would be widened implicitly.');
|
|
788
|
-
decoderVTensor = await castF16ToF32(decoderVTensor);
|
|
789
|
-
decoderVTemp = decoderVTensor;
|
|
790
|
-
}
|
|
791
|
-
kvState = await createDiffusionGemmaDecoderKVState({
|
|
792
|
-
state,
|
|
793
|
-
layerIdx,
|
|
794
|
-
kTensor: decoderKTensor,
|
|
795
|
-
vTensor: decoderVTensor,
|
|
796
|
-
currentSeqLen,
|
|
797
|
-
numTokens,
|
|
798
|
-
numKVHeads,
|
|
799
|
-
headDim,
|
|
800
|
-
layerType,
|
|
801
|
-
slidingWindow,
|
|
802
|
-
kvDtype: decoderKVDtype,
|
|
803
|
-
});
|
|
804
|
-
if (decoderKTemp) {
|
|
805
|
-
releaseBuffer(decoderKTemp.buffer);
|
|
806
|
-
decoderKTemp = null;
|
|
807
|
-
}
|
|
808
|
-
if (decoderVTemp && decoderVTemp.buffer !== decoderKTensor.buffer) {
|
|
809
|
-
releaseBuffer(decoderVTemp.buffer);
|
|
810
|
-
}
|
|
811
|
-
decoderVTemp = null;
|
|
812
|
-
} else {
|
|
813
|
-
kvState = resolveKVCacheState(state, layerIdx, kTensor, vTensor, currentSeqLen, numTokens);
|
|
814
|
-
}
|
|
815
|
-
const dispatchConfig = {
|
|
816
|
-
layerIdx, numTokens, isPrefill, numHeads, numKVHeads, headDim, hiddenSize,
|
|
817
|
-
slidingWindow: diffusionGemmaDecoder ? null : slidingWindow,
|
|
818
|
-
layerType,
|
|
819
|
-
layerTypes: config.layerTypes,
|
|
820
|
-
queryPreAttnScalar,
|
|
821
|
-
causalAttention: diffusionGemmaDecoder ? false : config.causalAttention,
|
|
822
|
-
activationDtype: attentionActivationDtype,
|
|
823
|
-
kvCacheDtype: attentionPrecisionContract.resolvedKvCacheDtype ?? state.kvCache?.kvDtype ?? null,
|
|
824
|
-
};
|
|
825
|
-
const dispatchParams = buildAttentionDispatchParams(dispatchConfig, state, kTensor, vTensor, kvState);
|
|
826
|
-
const {
|
|
827
|
-
effectiveSlidingWindow, attentionKernelVariant, attnScale,
|
|
828
|
-
cachedKDtype, cachedVDtype, cachedKTensor, cachedVTensor,
|
|
829
|
-
prefillFallbackNeedsCast, causalForAttention,
|
|
830
|
-
} = dispatchParams;
|
|
831
|
-
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
832
|
-
let attentionOutputGateFused = false;
|
|
833
|
-
|
|
834
|
-
// 5. Attention (uses raw GPUBuffers)
|
|
835
|
-
|
|
836
|
-
// Debug: log scale on layer 0
|
|
837
|
-
if (layerIdx === 0 && isPrefill) {
|
|
838
|
-
trace.attn(layerIdx, `Attention scale=${attnScale.toFixed(6)}, queryPreAttnScalar=${queryPreAttnScalar ?? 'undefined'}, headDim=${headDim}`);
|
|
839
|
-
}
|
|
840
|
-
|
|
841
|
-
// Kernel step debug: KV cache state after resolve
|
|
842
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
843
|
-
trace.kv(layerIdx, `KV resolved: kvLen=${kvState.kvLenForAttention}, layout=${kvState.kvLayout}`);
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
recordAttentionInputs(state, buildAttentionInputsData(
|
|
847
|
-
dispatchConfig, input, normed, kvState, dispatchParams,
|
|
848
|
-
{ useF16Activations, matmulOutputDtype },
|
|
849
|
-
usedFusedQKV, qTensor, kTensor, vTensor,
|
|
850
|
-
));
|
|
851
|
-
|
|
852
|
-
const attentionKernelRunners = {
|
|
853
|
-
bdpa: async () => {
|
|
854
|
-
const BDPA_MAX_HEAD_DIM = 256;
|
|
855
|
-
const BDPA_MAX_KV_LEN = 2048;
|
|
856
|
-
if (numTokens !== 1) {
|
|
857
|
-
throw new Error(`BDPA attention supports decode-only seqLen=1; got seqLen=${numTokens}.`);
|
|
858
|
-
}
|
|
859
|
-
if (headDim > BDPA_MAX_HEAD_DIM) {
|
|
860
|
-
throw new Error(
|
|
861
|
-
`BDPA attention kernel supports headDim <= ${BDPA_MAX_HEAD_DIM}; got headDim=${headDim}.`
|
|
862
|
-
);
|
|
863
|
-
}
|
|
864
|
-
if (kvState.kvLenForAttention > BDPA_MAX_KV_LEN) {
|
|
865
|
-
throw new Error(
|
|
866
|
-
`BDPA attention kernel supports kvLen <= ${BDPA_MAX_KV_LEN}; got kvLen=${kvState.kvLenForAttention}.`
|
|
867
|
-
);
|
|
868
|
-
}
|
|
869
|
-
const basisKDtype = 'f16';
|
|
870
|
-
const basisVDtype = 'f16';
|
|
871
|
-
const basisCount = Math.max(1, kvState.bdpaBasisCount);
|
|
872
|
-
const basisKTensor = createTensor(kvState.bdpaBasisK, basisKDtype, [basisCount, numKVHeads * headDim], 'bdpa_basis_k');
|
|
873
|
-
const basisVTensor = createTensor(kvState.bdpaBasisV, basisVDtype, [basisCount, numKVHeads * headDim], 'bdpa_basis_v');
|
|
874
|
-
let qForBDPA = qTensor;
|
|
875
|
-
if (qForBDPA.dtype !== 'f16') {
|
|
876
|
-
assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f16', 'Q would be narrowed implicitly for BDPA.');
|
|
877
|
-
qForBDPA = await castF32ToF16(qTensor);
|
|
878
|
-
}
|
|
879
|
-
const output = await runAttentionBDPA(qForBDPA, basisKTensor, basisVTensor, kvState.bdpaPagedK, kvState.bdpaPagedV, kvState.bdpaIndex, numHeads, headDim, {
|
|
880
|
-
seqLen: numTokens,
|
|
881
|
-
kvLen: kvState.kvLenForAttention,
|
|
882
|
-
numKVHeads,
|
|
883
|
-
causal: causalForAttention,
|
|
884
|
-
startPos: kvState.startPosForMask,
|
|
885
|
-
layerIdx,
|
|
886
|
-
slidingWindow: effectiveSlidingWindow,
|
|
887
|
-
attnSoftcap,
|
|
888
|
-
scale: attnScale,
|
|
889
|
-
ropeCos: state.ropeFreqsCos,
|
|
890
|
-
ropeSin: state.ropeFreqsSin,
|
|
891
|
-
});
|
|
892
|
-
if (qForBDPA !== qTensor) {
|
|
893
|
-
releaseBuffer(qForBDPA.buffer);
|
|
894
|
-
}
|
|
895
|
-
return output;
|
|
896
|
-
},
|
|
897
|
-
tieredQuant: async () => {
|
|
898
|
-
let qForAttention = qTensor;
|
|
899
|
-
let qTemp = null;
|
|
900
|
-
if (kvState.coldQuantMode !== 'none' && qTensor.dtype !== 'f32') {
|
|
901
|
-
assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Q would be widened implicitly for tiered quant attention.');
|
|
902
|
-
qForAttention = await castF16ToF32(qTensor);
|
|
903
|
-
qTemp = qForAttention;
|
|
904
|
-
}
|
|
905
|
-
const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
|
|
906
|
-
const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
|
|
907
|
-
|
|
908
|
-
if (kvState.coldQuantMode === 'none') {
|
|
909
|
-
throw new Error('Tiered quant attention requires cold quant mode.');
|
|
910
|
-
}
|
|
911
|
-
if (!kvState.coldScalesK || !kvState.coldScalesV) {
|
|
912
|
-
throw new Error('Tiered quant attention requires cold scale buffers.');
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
const output = await runAttentionTieredQuant(
|
|
916
|
-
qForAttention,
|
|
917
|
-
cachedHotKTensor,
|
|
918
|
-
cachedHotVTensor,
|
|
919
|
-
kvState.cachedKCold,
|
|
920
|
-
kvState.cachedVCold,
|
|
921
|
-
kvState.coldScalesK,
|
|
922
|
-
kvState.coldScalesV,
|
|
923
|
-
numHeads,
|
|
924
|
-
headDim,
|
|
925
|
-
buildTieredQuantAttentionOptions(kvState, {
|
|
926
|
-
seqLen: numTokens,
|
|
927
|
-
numKVHeads,
|
|
928
|
-
causal: causalForAttention,
|
|
929
|
-
startPos: kvState.startPosForMask,
|
|
930
|
-
slidingWindow: effectiveSlidingWindow ?? 0,
|
|
931
|
-
attnSoftcap,
|
|
932
|
-
scale: attnScale,
|
|
933
|
-
})
|
|
934
|
-
);
|
|
935
|
-
|
|
936
|
-
if (qTemp) {
|
|
937
|
-
releaseBuffer(qTemp.buffer);
|
|
938
|
-
}
|
|
939
|
-
return output;
|
|
940
|
-
},
|
|
941
|
-
contiguousQuant: async () => {
|
|
942
|
-
let qForAttention = qTensor;
|
|
943
|
-
let qTemp = null;
|
|
944
|
-
if (qTensor.dtype !== 'f32') {
|
|
945
|
-
assertAttentionDtypeTransitionAllowed(state, qTensor.dtype, 'f32', 'Q would be widened implicitly for contiguous quant attention.');
|
|
946
|
-
qForAttention = await castF16ToF32(qTensor);
|
|
947
|
-
qTemp = qForAttention;
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
if (!kvState.coldScalesK || !kvState.coldScalesV) {
|
|
951
|
-
throw new Error('Contiguous quant attention requires scale buffers.');
|
|
952
|
-
}
|
|
953
|
-
if (!kvState.rotationMatrixBuffer || !kvState.codebookCentroidsBuffer) {
|
|
954
|
-
throw new Error('Contiguous quant attention requires TurboQuant shared buffers.');
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
const output = await runAttentionContiguousQuant(
|
|
958
|
-
qForAttention,
|
|
959
|
-
kvState.cachedKCold,
|
|
960
|
-
kvState.cachedVCold,
|
|
961
|
-
kvState.coldScalesK,
|
|
962
|
-
kvState.coldScalesV,
|
|
963
|
-
numHeads,
|
|
964
|
-
headDim,
|
|
965
|
-
buildContiguousQuantAttentionOptions(kvState, {
|
|
966
|
-
seqLen: numTokens,
|
|
967
|
-
kvLen: kvState.kvLenForAttention,
|
|
968
|
-
numKVHeads,
|
|
969
|
-
causal: causalForAttention,
|
|
970
|
-
startPos: kvState.startPosForMask,
|
|
971
|
-
slidingWindow: effectiveSlidingWindow ?? 0,
|
|
972
|
-
attnSoftcap,
|
|
973
|
-
scale: attnScale,
|
|
974
|
-
})
|
|
975
|
-
);
|
|
976
|
-
|
|
977
|
-
if (qTemp) {
|
|
978
|
-
releaseBuffer(qTemp.buffer);
|
|
979
|
-
}
|
|
980
|
-
return output;
|
|
981
|
-
},
|
|
982
|
-
tiered: async () => {
|
|
983
|
-
const qForAttention = qTensor;
|
|
984
|
-
const cachedHotKTensor = createTensor(kvState.cachedKHot, cachedKDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_K_hot');
|
|
985
|
-
const cachedHotVTensor = createTensor(kvState.cachedVHot, cachedVDtype, [kvState.hotLen, numKVHeads * headDim], 'cached_V_hot');
|
|
986
|
-
const cachedColdKTensor = createTensor(kvState.cachedKCold, cachedKDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_K_cold');
|
|
987
|
-
const cachedColdVTensor = createTensor(kvState.cachedVCold, cachedVDtype, [kvState.coldLen, numKVHeads * headDim], 'cached_V_cold');
|
|
988
|
-
return runAttentionTiered(qForAttention, cachedHotKTensor, cachedHotVTensor, cachedColdKTensor, cachedColdVTensor, numHeads, headDim, {
|
|
989
|
-
seqLen: numTokens,
|
|
990
|
-
coldLen: kvState.coldLen,
|
|
991
|
-
hotLen: kvState.hotLen,
|
|
992
|
-
numKVHeads,
|
|
993
|
-
causal: causalForAttention,
|
|
994
|
-
startPos: kvState.startPosForMask,
|
|
995
|
-
slidingWindow: effectiveSlidingWindow ?? 0,
|
|
996
|
-
attnSoftcap,
|
|
997
|
-
scale: attnScale,
|
|
998
|
-
hotWindow: kvState.hotWindow,
|
|
999
|
-
hotStart: kvState.hotStart,
|
|
1000
|
-
coldPageTable: kvState.coldPageTable,
|
|
1001
|
-
coldPageSize: kvState.coldPageSize,
|
|
1002
|
-
coldLayout: kvState.coldPageTable ? 2 : 0,
|
|
1003
|
-
hotLayout: kvState.hotWindow > 0 ? 1 : 0,
|
|
1004
|
-
});
|
|
1005
|
-
},
|
|
1006
|
-
contiguous: async () => {
|
|
1007
|
-
// Prefill fallback: quantized/tiered layouts use raw K/V for prefill, cast to f16 to match kernel path
|
|
1008
|
-
let kForAttn = cachedKTensor;
|
|
1009
|
-
let vForAttn = cachedVTensor;
|
|
1010
|
-
if (prefillFallbackNeedsCast) {
|
|
1011
|
-
const hasExplicitF16KvContract = isAttentionKvDtypeExplicit(attentionPrecisionContract, 'f16');
|
|
1012
|
-
if (cachedKDtype === 'f16' && kTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
1013
|
-
assertAttentionDtypeTransitionAllowed(state, kTensor.dtype, 'f16', 'K would be narrowed implicitly for prefill fallback.');
|
|
1014
|
-
}
|
|
1015
|
-
if (cachedVDtype === 'f16' && vTensor.dtype !== 'f16' && !hasExplicitF16KvContract) {
|
|
1016
|
-
assertAttentionDtypeTransitionAllowed(state, vTensor.dtype, 'f16', 'V would be narrowed implicitly for prefill fallback.');
|
|
1017
|
-
}
|
|
1018
|
-
const kCasted = cachedKDtype === 'f16' && kTensor.dtype !== 'f16'
|
|
1019
|
-
? await castF32ToF16(kTensor) : kTensor;
|
|
1020
|
-
const vCasted = cachedVDtype === 'f16' && vTensor.dtype !== 'f16'
|
|
1021
|
-
? await castF32ToF16(vTensor) : vTensor;
|
|
1022
|
-
kForAttn = createTensor(kCasted.buffer, kCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_K');
|
|
1023
|
-
vForAttn = createTensor(vCasted.buffer, vCasted.dtype, [kvState.kvLenForAttention, numKVHeads * headDim], 'cached_V');
|
|
1024
|
-
}
|
|
1025
|
-
// Session precedence is runtime-over-manifest per config-style-guide
|
|
1026
|
-
// §Category Rules: `resolvedSession = merge(manifest.session, runtime.session)`
|
|
1027
|
-
// with runtime fields winning. getRuntimeConfig() returns that merged
|
|
1028
|
-
// session (state.runtimeConfig.inference.session is a stale snapshot —
|
|
1029
|
-
// see comment above at the rmsNorm fusion check). The kernel itself
|
|
1030
|
-
// enforces remaining preconditions (head_dim=256, etc.).
|
|
1031
|
-
const useFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useFlashPrefillAttention === true && numTokens > 1;
|
|
1032
|
-
const useOrtFlashPrefill = !diffusionGemmaDecoder && mergedSession?.useOrtFlashPrefillAttention === true && numTokens > 1;
|
|
1033
|
-
const useOutputGateFusion = canUseAttentionOutputGateFusion({
|
|
1034
|
-
session: mergedSession,
|
|
1035
|
-
qGateTensor,
|
|
1036
|
-
numTokens,
|
|
1037
|
-
numHeads,
|
|
1038
|
-
headDim,
|
|
1039
|
-
cachedKDtype,
|
|
1040
|
-
cachedVDtype,
|
|
1041
|
-
kernelPath,
|
|
1042
|
-
diffusionGemmaDecoder,
|
|
1043
|
-
});
|
|
1044
|
-
const result = await runAttention(qTensor, kForAttn, vForAttn, null, numHeads, headDim, {
|
|
1045
|
-
seqLen: numTokens,
|
|
1046
|
-
kvLen: kvState.kvLenForAttention,
|
|
1047
|
-
numKVHeads,
|
|
1048
|
-
causal: causalForAttention,
|
|
1049
|
-
bidirectionalSpanStart: multimodalBidirectionalSpan?.start ?? 0,
|
|
1050
|
-
bidirectionalSpanLength: multimodalBidirectionalSpan?.length ?? 0,
|
|
1051
|
-
startPos: kvState.startPosForMask,
|
|
1052
|
-
layerIdx,
|
|
1053
|
-
slidingWindow: effectiveSlidingWindow,
|
|
1054
|
-
attnSoftcap,
|
|
1055
|
-
scale: attnScale,
|
|
1056
|
-
kvStart: kvState.kvStart,
|
|
1057
|
-
kvLayout: kvState.kvLayout,
|
|
1058
|
-
kvPageTable: kvState.kvPageTable,
|
|
1059
|
-
kvPageSize: kvState.kvPageSize,
|
|
1060
|
-
kernelPath,
|
|
1061
|
-
useFlashPrefill,
|
|
1062
|
-
useOrtFlashPrefill,
|
|
1063
|
-
outputGate: useOutputGateFusion ? qGateTensor : null,
|
|
1064
|
-
});
|
|
1065
|
-
attentionOutputGateFused = result?.outputGateFused === true;
|
|
1066
|
-
if (prefillFallbackNeedsCast) {
|
|
1067
|
-
if (kTensor.dtype !== 'f16') releaseBuffer(kForAttn.buffer);
|
|
1068
|
-
if (vTensor.dtype !== 'f16') releaseBuffer(vForAttn.buffer);
|
|
1069
|
-
}
|
|
1070
|
-
return result;
|
|
1071
|
-
},
|
|
1072
|
-
};
|
|
1073
|
-
|
|
1074
|
-
const runAttentionKernel = attentionKernelRunners[attentionKernelVariant];
|
|
1075
|
-
if (!runAttentionKernel) {
|
|
1076
|
-
throw new Error(`Unsupported attention kernel variant "${attentionKernelVariant}" at layer ${layerIdx}`);
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
try {
|
|
1080
|
-
attnOutput = await runAttentionKernel();
|
|
1081
|
-
} finally {
|
|
1082
|
-
if (decoderKVState?.ownedBuffers) {
|
|
1083
|
-
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1084
|
-
releaseBuffer(buffer);
|
|
1085
|
-
}
|
|
1086
|
-
decoderKVState.ownedBuffers = null;
|
|
1087
|
-
}
|
|
1088
|
-
}
|
|
1089
|
-
await runProbes('attn_core_out', attnOutput.buffer, {
|
|
1090
|
-
layerIdx,
|
|
1091
|
-
numTokens,
|
|
1092
|
-
hiddenSize: numHeads * headDim,
|
|
1093
|
-
probes: state.debugProbes,
|
|
1094
|
-
operatorDiagnostics: state.operatorDiagnostics,
|
|
1095
|
-
dtype: attnOutput.dtype,
|
|
1096
|
-
});
|
|
1097
|
-
|
|
1098
|
-
// Trace attention output
|
|
1099
|
-
if (kernelTrace.enabled) {
|
|
1100
|
-
await traceStep('attention', `L${layerIdx}.attention`, layerIdx, attnOutput.buffer, [numTokens, numHeads * headDim]);
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
// Kernel step debug: attention output
|
|
1104
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
1105
|
-
logKernelStep('attention', { layerIdx, label: `seqLen=${numTokens} kvLen=${kvState.kvLenForAttention}` });
|
|
1106
|
-
await dumpTokenVector(attnOutput.buffer, 'attn_out', {
|
|
1107
|
-
layerIdx,
|
|
1108
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
1109
|
-
rowSize: numHeads * headDim,
|
|
1110
|
-
dtype: attnOutput.dtype,
|
|
1111
|
-
});
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
// Debug: attention output for configured layers
|
|
1115
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'attn_out', debugFlags) && debugCheckBuffer) {
|
|
1116
|
-
await debugCheckBuffer(attnOutput.buffer, `L${layerIdx} attention output (before o_proj, GPU)`, numTokens, numHeads * headDim);
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
attnForProjection = attnOutput;
|
|
1120
|
-
if (qGateTensor && !attentionOutputGateFused) {
|
|
1121
|
-
// The Qwen3_5 reference (HF transformers) full-attention path
|
|
1122
|
-
// applies sigmoid(gate) — NOT silu(gate) — to the attention
|
|
1123
|
-
// output regardless of how the architecture's gate label is
|
|
1124
|
-
// surfaced in the HF config. See Qwen3_5Attention.forward in
|
|
1125
|
-
// transformers/models/qwen3_5/modeling_qwen3_5.py:
|
|
1126
|
-
// attn_output = attn_output * torch.sigmoid(gate)
|
|
1127
|
-
// The earlier `swish` -> `silu` mapping here was incorrect and
|
|
1128
|
-
// produced sign-flipped output for any token whose gate value
|
|
1129
|
-
// was negative (silu is sign-preserving via x * sigmoid(x);
|
|
1130
|
-
// sigmoid is unconditionally in (0, 1)). At Qwen 3.6 27B scale
|
|
1131
|
-
// this drove residual-stream divergence by L=3 (HF reference vs
|
|
1132
|
-
// upstream rel_l2≈1.37, mean per-token correlation ≈ -0.96 — a
|
|
1133
|
-
// sign-flip pattern). config.outputGateType is retained for
|
|
1134
|
-
// future architectures whose gate truly is silu, but Qwen 3.6's
|
|
1135
|
-
// `output_gate_type=swish` still routes here as sigmoid because
|
|
1136
|
-
// that is what the reference forward computes.
|
|
1137
|
-
const gateActivation = 'sigmoid';
|
|
1138
|
-
attnForProjection = await runSiLU(attnOutput, {
|
|
1139
|
-
size: numTokens * numHeads * headDim,
|
|
1140
|
-
gate: qGateTensor,
|
|
1141
|
-
useVec4: (numTokens * numHeads * headDim) % 4 === 0,
|
|
1142
|
-
gateActivation,
|
|
1143
|
-
inputActivation: 'identity',
|
|
1144
|
-
swigluLimit: null,
|
|
1145
|
-
});
|
|
1146
|
-
releaseBuffer(attnOutput.buffer);
|
|
1147
|
-
}
|
|
1148
|
-
|
|
1149
|
-
// 6. Output projection (with optional fused residual for decode)
|
|
1150
|
-
|
|
1151
|
-
output = null;
|
|
1152
|
-
let residualFused = false;
|
|
1153
|
-
let oProjInput = attnForProjection;
|
|
1154
|
-
oProjInputTemp = null;
|
|
1155
|
-
if (layerWeights.oProj && getWeightBuffer) {
|
|
1156
|
-
({ oProjInput, oProjInputTemp } = await prepareAttentionProjectionInput(
|
|
1157
|
-
attnForProjection,
|
|
1158
|
-
oProjInputDtype,
|
|
1159
|
-
async (tensor) => {
|
|
1160
|
-
assertAttentionDtypeTransitionAllowed(
|
|
1161
|
-
state,
|
|
1162
|
-
tensor.dtype,
|
|
1163
|
-
oProjInputDtype,
|
|
1164
|
-
'o_proj input would change dtype implicitly.',
|
|
1165
|
-
'step_precision'
|
|
1166
|
-
);
|
|
1167
|
-
return oProjInputDtype === 'f16'
|
|
1168
|
-
? castF32ToF16(tensor)
|
|
1169
|
-
: castF16ToF32(tensor);
|
|
1170
|
-
}
|
|
1171
|
-
));
|
|
1172
|
-
const oProjBuf = getWeightBuffer(layerWeights.oProj, 'o_proj');
|
|
1173
|
-
const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
|
|
1174
|
-
|
|
1175
|
-
// Use fused o_proj + residual for decode when possible
|
|
1176
|
-
// Note: dtype from WeightBuffer metadata (buffer-dtypes WeakMap removed)
|
|
1177
|
-
const oProjDtype = getWeightDtype(oProjBuf);
|
|
1178
|
-
const canUseFused = selectRuleValue('inference', 'attention', 'useFusedOProjResidual', {
|
|
1179
|
-
allowFusedResidual: shouldUseFusedMatmulResidual(numTokens),
|
|
1180
|
-
hasResidual: Boolean(residualTensor),
|
|
1181
|
-
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1182
|
-
attnIsF32: oProjInput.dtype === 'f32',
|
|
1183
|
-
attnIsF16: oProjInput.dtype === 'f16',
|
|
1184
|
-
hasLoRA: Boolean(loraO),
|
|
1185
|
-
oProjIsF16: oProjDtype === 'f16',
|
|
1186
|
-
}); // GEMV kernel expects f16 weights
|
|
1187
|
-
const canUseWideTileResidual = canUseQ4KWideTileResidualFusion({
|
|
1188
|
-
phase,
|
|
1189
|
-
session: mergedSession,
|
|
1190
|
-
hasResidual: Boolean(residualTensor),
|
|
1191
|
-
residualMatches: Boolean(residualTensor && residualTensor.dtype === oProjInput.dtype),
|
|
1192
|
-
inputDtype: oProjInput.dtype,
|
|
1193
|
-
outputDtype: oProjOutputDtype,
|
|
1194
|
-
oProjDtype,
|
|
1195
|
-
hasLoRA: Boolean(loraO),
|
|
1196
|
-
});
|
|
1197
|
-
if (canUseFused && residualTensor) {
|
|
1198
|
-
// FUSED PATH: o_proj matmul + residual add in one dispatch
|
|
1199
|
-
output = await runMatmulResidualFused(oProjInput, oProjBuf, residualTensor, {
|
|
1200
|
-
N: hiddenSize,
|
|
1201
|
-
K: numHeads * headDim,
|
|
1202
|
-
});
|
|
1203
|
-
residualFused = true;
|
|
1204
|
-
|
|
1205
|
-
if (layerIdx === 0 && !isPrefill) {
|
|
1206
|
-
trace.attn(layerIdx, `Using fused o_proj+residual path`);
|
|
1207
|
-
}
|
|
1208
|
-
} else {
|
|
1209
|
-
// STANDARD PATH: o_proj matmul only unless Q4K WideTile fuses residual in the epilogue
|
|
1210
|
-
output = await runMatmul(oProjInput, oProjBuf, numTokens, hiddenSize, numHeads * headDim, {
|
|
1211
|
-
transposeB: 'auto',
|
|
1212
|
-
role: 'o_proj',
|
|
1213
|
-
layerIdx,
|
|
1214
|
-
kernelPath,
|
|
1215
|
-
outputDtype: oProjOutputDtype,
|
|
1216
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
1217
|
-
residualTensor: canUseWideTileResidual ? residualTensor : null,
|
|
1218
|
-
});
|
|
1219
|
-
residualFused = canUseWideTileResidual;
|
|
1220
|
-
}
|
|
1221
|
-
// Release temporary buffer if we created it (original was not already on GPU)
|
|
1222
|
-
if (!isGpuBufferInstance(layerWeights.oProj) && !isWeightBuffer(layerWeights.oProj)) {
|
|
1223
|
-
releaseBuffer(isWeightBuffer(oProjBuf) ? oProjBuf.buffer : oProjBuf);
|
|
1224
|
-
}
|
|
1225
|
-
|
|
1226
|
-
// Trace output projection
|
|
1227
|
-
if (kernelTrace.enabled) {
|
|
1228
|
-
await traceStep('matmul', `L${layerIdx}.o_proj${residualFused ? '+residual' : ''}`, layerIdx, output.buffer, [numTokens, hiddenSize]);
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
// Kernel step debug: output projection
|
|
1232
|
-
if (isKernelDebugEnabled(layerIdx)) {
|
|
1233
|
-
logKernelStep('matmul', { layerIdx, label: residualFused ? 'O_proj+residual' : 'O_proj', M: numTokens, N: hiddenSize, K: numHeads * headDim });
|
|
1234
|
-
await dumpTokenVector(output.buffer, 'o_proj_out', {
|
|
1235
|
-
layerIdx,
|
|
1236
|
-
tokenIdx: Math.max(0, numTokens - 1),
|
|
1237
|
-
rowSize: hiddenSize,
|
|
1238
|
-
dtype: output.dtype,
|
|
1239
|
-
});
|
|
1240
|
-
}
|
|
1241
|
-
|
|
1242
|
-
} else {
|
|
1243
|
-
output = oProjInput;
|
|
1244
|
-
}
|
|
1245
|
-
|
|
1246
|
-
// Apply LoRA to output projection if present (only if not using fused path)
|
|
1247
|
-
if (!residualFused) {
|
|
1248
|
-
const loraO = getLoRAModule(lora, layerIdx, 'o_proj');
|
|
1249
|
-
if (loraO && getWeightBuffer) {
|
|
1250
|
-
const combined = await applyLoRA(
|
|
1251
|
-
oProjInput,
|
|
1252
|
-
output,
|
|
1253
|
-
loraO,
|
|
1254
|
-
{ M: numTokens, N: hiddenSize, K: numHeads * headDim },
|
|
1255
|
-
getWeightBuffer,
|
|
1256
|
-
undefined,
|
|
1257
|
-
{ kernelPath }
|
|
1258
|
-
);
|
|
1259
|
-
if (combined.buffer !== output.buffer) {
|
|
1260
|
-
releaseBuffer(output.buffer);
|
|
1261
|
-
output = combined;
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
if (layerWeights.oProjBias) {
|
|
1267
|
-
const { tensor: oProjBiasTensor, owned } = getVectorTensor(
|
|
1268
|
-
layerWeights.oProjBias,
|
|
1269
|
-
`L${layerIdx}.o_proj_bias`,
|
|
1270
|
-
hiddenSize,
|
|
1271
|
-
{},
|
|
1272
|
-
debugFlags
|
|
1273
|
-
);
|
|
1274
|
-
try {
|
|
1275
|
-
output = await runBiasAdd(output, oProjBiasTensor, numTokens, hiddenSize, {
|
|
1276
|
-
label: `L${layerIdx}.o_proj_bias`,
|
|
1277
|
-
layerIdx,
|
|
1278
|
-
executionPolicies: state.executionPolicies ?? null,
|
|
1279
|
-
});
|
|
1280
|
-
} finally {
|
|
1281
|
-
if (owned) releaseBuffer(oProjBiasTensor.buffer);
|
|
1282
|
-
}
|
|
1283
|
-
}
|
|
1284
|
-
|
|
1285
|
-
if (oProjInputTemp) {
|
|
1286
|
-
releaseBuffer(oProjInputTemp.buffer);
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
// Debug: o_proj output for configured layers
|
|
1290
|
-
if (isPrefill && shouldDebugLayer(layerIdx, debugFlags.debugLayers) && !markStageLogged(layerIdx, 'o_proj', debugFlags) && debugCheckBuffer) {
|
|
1291
|
-
await debugCheckBuffer(output.buffer, `L${layerIdx} attention output (after o_proj, GPU)`, numTokens, hiddenSize);
|
|
1292
|
-
}
|
|
1293
|
-
|
|
1294
|
-
finalOutput = output;
|
|
1295
|
-
|
|
1296
|
-
const buffersToRelease = [];
|
|
1297
|
-
if (output.buffer !== attnForProjection.buffer) {
|
|
1298
|
-
buffersToRelease.push(attnForProjection.buffer);
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
if (output.dtype !== oProjOutputDtype) {
|
|
1302
|
-
assertAttentionDtypeTransitionAllowed(
|
|
1303
|
-
state,
|
|
1304
|
-
output.dtype,
|
|
1305
|
-
oProjOutputDtype,
|
|
1306
|
-
'Attention output would change dtype implicitly at the end of the op.',
|
|
1307
|
-
'step_precision'
|
|
1308
|
-
);
|
|
1309
|
-
const coercedOutput = oProjOutputDtype === 'f16'
|
|
1310
|
-
? await castF32ToF16(output)
|
|
1311
|
-
: await castF16ToF32(output);
|
|
1312
|
-
buffersToRelease.push(output.buffer);
|
|
1313
|
-
finalOutput = coercedOutput;
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
// Cleanup
|
|
1317
|
-
releaseBuffer(qTensor.buffer);
|
|
1318
|
-
if (qGateTensor) {
|
|
1319
|
-
releaseBuffer(qGateTensor.buffer);
|
|
1320
|
-
}
|
|
1321
|
-
if (!reusesSharedKV && !storeSharedKV) {
|
|
1322
|
-
releaseBuffer(kTensor.buffer);
|
|
1323
|
-
if (vTensor.buffer !== kTensor.buffer) {
|
|
1324
|
-
releaseBuffer(vTensor.buffer);
|
|
1325
|
-
}
|
|
1326
|
-
}
|
|
1327
|
-
for (const buffer of buffersToRelease) {
|
|
1328
|
-
releaseBuffer(buffer);
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
return { output: finalOutput, residualFused };
|
|
1332
|
-
} catch (error) {
|
|
1333
|
-
const released = new Set();
|
|
1334
|
-
const releaseOnce = (buffer) => {
|
|
1335
|
-
if (!buffer || released.has(buffer)) return;
|
|
1336
|
-
released.add(buffer);
|
|
1337
|
-
releaseBuffer(buffer);
|
|
1338
|
-
};
|
|
1339
|
-
if (finalOutput?.buffer && finalOutput.buffer !== output?.buffer) {
|
|
1340
|
-
releaseOnce(finalOutput.buffer);
|
|
1341
|
-
}
|
|
1342
|
-
if (output?.buffer && output.buffer !== attnForProjection?.buffer) {
|
|
1343
|
-
releaseOnce(output.buffer);
|
|
1344
|
-
}
|
|
1345
|
-
if (oProjInputTemp?.buffer) {
|
|
1346
|
-
releaseOnce(oProjInputTemp.buffer);
|
|
1347
|
-
}
|
|
1348
|
-
if (attnForProjection?.buffer && attnForProjection.buffer !== attnOutput?.buffer) {
|
|
1349
|
-
releaseOnce(attnForProjection.buffer);
|
|
1350
|
-
}
|
|
1351
|
-
if (attnOutput?.buffer) {
|
|
1352
|
-
releaseOnce(attnOutput.buffer);
|
|
1353
|
-
}
|
|
1354
|
-
if (qGateTensor?.buffer) {
|
|
1355
|
-
releaseOnce(qGateTensor.buffer);
|
|
1356
|
-
}
|
|
1357
|
-
if (qTensor?.buffer) {
|
|
1358
|
-
releaseOnce(qTensor.buffer);
|
|
1359
|
-
}
|
|
1360
|
-
if (kTensor?.buffer && !reusesSharedKV && !storeSharedKV) {
|
|
1361
|
-
releaseOnce(kTensor.buffer);
|
|
1362
|
-
}
|
|
1363
|
-
if (vTensor?.buffer && !reusesSharedKV && !storeSharedKV) {
|
|
1364
|
-
releaseOnce(vTensor.buffer);
|
|
1365
|
-
}
|
|
1366
|
-
if (normed?.buffer && normed.buffer !== attentionInput?.buffer) {
|
|
1367
|
-
releaseOnce(normed.buffer);
|
|
1368
|
-
}
|
|
1369
|
-
if (attentionInputTemp && attentionInput?.buffer) {
|
|
1370
|
-
releaseOnce(attentionInput.buffer);
|
|
1371
|
-
}
|
|
1372
|
-
if (decoderKTemp?.buffer) {
|
|
1373
|
-
releaseOnce(decoderKTemp.buffer);
|
|
1374
|
-
}
|
|
1375
|
-
if (decoderVTemp?.buffer) {
|
|
1376
|
-
releaseOnce(decoderVTemp.buffer);
|
|
1377
|
-
}
|
|
1378
|
-
if (decoderKVState?.ownedBuffers) {
|
|
1379
|
-
for (const buffer of decoderKVState.ownedBuffers) {
|
|
1380
|
-
releaseOnce(buffer);
|
|
1381
|
-
}
|
|
1382
|
-
decoderKVState.ownedBuffers = null;
|
|
1383
|
-
}
|
|
1384
|
-
throw error;
|
|
1385
|
-
}
|
|
1386
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Immediate attention facade.
|
|
3
|
+
*
|
|
4
|
+
* Semantic decisions are resolved in plan.js. The immediate adapter records
|
|
5
|
+
* the canonical interpreter and submits it immediately.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export { runLayerAttentionGPU } from './executor-immediate.js';
|