doppler-gpu 0.4.15 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +25 -2
- package/src/cli/doppler-cli.js +253 -94
- package/src/cli/runtime-profiles.js +3 -7
- package/src/client/doppler-api.d.ts +28 -0
- package/src/client/doppler-api.js +2 -0
- package/src/client/inspection.d.ts +122 -0
- package/src/client/inspection.js +343 -0
- package/src/client/runtime/index.d.ts +54 -0
- package/src/client/runtime/index.js +190 -18
- package/src/client/runtime/model-manager.d.ts +0 -4
- package/src/client/runtime/model-manager.js +0 -28
- package/src/client/runtime/model-session.d.ts +48 -1
- package/src/client/runtime/model-session.js +149 -30
- package/src/client/runtime/model-source.d.ts +10 -1
- package/src/client/runtime/scoped-session.d.ts +117 -0
- package/src/client/runtime/scoped-session.js +257 -0
- package/src/client/runtime/types.d.ts +0 -1
- package/src/client/runtime/types.js +0 -1
- package/src/config/evidence/boundary-evidence-policy.json +54 -0
- package/src/config/evidence/source-intake-policy.json +122 -0
- package/src/config/inspection/observation-policies.json +80 -0
- package/src/config/param-validator.js +21 -36
- package/src/config/runtime/compute/f16-activations.json +1 -0
- package/src/config/runtime/compute/f16-batched.json +1 -0
- package/src/config/runtime/default.json +1 -0
- package/src/config/runtime/diagnostics/debug-logits.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
- package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
- package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
- package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
- package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
- package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
- package/src/config/runtime/model/gemma2-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
- package/src/config/runtime/model/gemma2-pipeline.json +1 -0
- package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
- package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
- package/src/config/runtime/optimization-candidates/registry.json +5 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
- package/src/config/runtime/profiles/default.json +1 -0
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
- package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/low-memory.json +1 -0
- package/src/config/runtime/profiles/production.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
- package/src/config/runtime/profiles/rerank-stability.json +1 -3
- package/src/config/runtime/profiles/rerank-throughput.json +1 -3
- package/src/config/runtime/profiles/simulation.json +1 -3
- package/src/config/runtime/profiles/throughput.json +1 -3
- package/src/config/runtime/profiles/trace-layers.json +1 -3
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
- package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
- package/src/config/runtime/profiles/turboquant.json +1 -0
- package/src/config/runtime/profiles/vector-stability.json +1 -3
- package/src/config/runtime/profiles/vector-throughput.json +1 -0
- package/src/config/runtime/profiles/verbose-trace.json +1 -3
- package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
- package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
- package/src/config/schema/benchmark.schema.d.ts +7 -0
- package/src/config/schema/benchmark.schema.js +5 -0
- package/src/config/schema/index.d.ts +8 -16
- package/src/config/schema/index.js +5 -8
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
- package/src/config/schema/kernel-thresholds.schema.js +0 -12
- package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
- package/src/config/schema/kernel-warmup.schema.js +0 -1
- package/src/config/schema/runtime-profile.schema.d.ts +24 -0
- package/src/config/schema/runtime-profile.schema.js +92 -0
- package/src/config/schema/shared-runtime.schema.d.ts +1 -4
- package/src/config/schema/shared-runtime.schema.js +0 -2
- package/src/config/schema/tooling.schema.d.ts +8 -5
- package/src/config/schema/tooling.schema.js +3 -2
- package/src/debug/tensor.d.ts +2 -0
- package/src/debug/tensor.js +21 -2
- package/src/gpu/command-recorder.d.ts +5 -0
- package/src/gpu/command-recorder.js +17 -0
- package/src/gpu/kernel-runtime.js +2 -10
- package/src/gpu/kernels/index.d.ts +0 -2
- package/src/gpu/kernels/index.js +0 -2
- package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
- package/src/gpu/kernels/kernel-prewarm.js +60 -0
- package/src/gpu/kernels/utils.d.ts +2 -4
- package/src/gpu/kernels/utils.js +2 -4
- package/src/index.d.ts +13 -1
- package/src/index.js +7 -1
- package/src/inference/browser-harness-model-helpers.js +4 -0
- package/src/inference/browser-harness-runtime-helpers.js +25 -10
- package/src/inference/browser-harness-text-helpers.js +4 -1
- package/src/inference/browser-harness.d.ts +4 -0
- package/src/inference/browser-harness.js +91 -4
- package/src/inference/observation-context.d.ts +22 -0
- package/src/inference/observation-context.js +41 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
- package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
- package/src/inference/pipelines/text/attention/index.js +7 -0
- package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
- package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
- package/src/inference/pipelines/text/attention/plan.js +328 -0
- package/src/inference/pipelines/text/attention/receipt.js +60 -0
- package/src/inference/pipelines/text/attention/record.js +8 -1278
- package/src/inference/pipelines/text/attention/run.js +8 -1386
- package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-steps.js +32 -15
- package/src/inference/pipelines/text/generator.js +10 -5
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
- package/src/inference/pipelines/text/layer.js +2 -0
- package/src/inference/pipelines/text/model-load.d.ts +0 -1
- package/src/inference/pipelines/text/model-load.js +2 -11
- package/src/inference/pipelines/text/probes.js +10 -0
- package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
- package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
- package/src/inference/pipelines/text/state.d.ts +6 -0
- package/src/inference/pipelines/text/state.js +3 -0
- package/src/inference/pipelines/text/types.d.ts +6 -0
- package/src/inference/pipelines/text.js +16 -1
- package/src/inference/refactor-receipt.d.ts +49 -0
- package/src/inference/refactor-receipt.js +188 -0
- package/src/inference/resource-scope.d.ts +33 -0
- package/src/inference/resource-scope.js +168 -0
- package/src/inference/test-harness.d.ts +3 -0
- package/src/inference/test-harness.js +1 -0
- package/src/storage/model-cache.d.ts +53 -0
- package/src/storage/model-cache.js +550 -0
- package/src/tooling/boundary-evidence.d.ts +51 -0
- package/src/tooling/boundary-evidence.js +408 -0
- package/src/tooling/browser-command-runner.js +4 -4
- package/src/tooling/command-context.d.ts +25 -0
- package/src/tooling/command-context.js +53 -0
- package/src/tooling/command-runner-shared.js +22 -5
- package/src/tooling/demo-receipts.d.ts +56 -0
- package/src/tooling/demo-receipts.js +113 -0
- package/src/tooling/diagnose-runner.js +2 -2
- package/src/tooling/execution-cost-ledger.d.ts +21 -0
- package/src/tooling/execution-cost-ledger.js +199 -0
- package/src/tooling/node-command-runner.js +2 -2
- package/src/tooling/opfs-cache.d.ts +11 -49
- package/src/tooling/opfs-cache.js +4 -510
- package/src/tooling/precision-replay-math.d.ts +42 -0
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-parity.d.ts +3 -0
- package/src/tooling/program-bundle-parity.js +8 -0
- package/src/tooling/program-bundle.d.ts +1 -52
- package/src/tooling/program-bundle.js +0 -139
- package/src/tooling/registered-variant-calibration.d.ts +24 -0
- package/src/tooling/registered-variant-calibration.js +286 -0
- package/src/tooling/runtime-input-composition.js +105 -53
- package/src/tooling/runtime-intent.js +41 -0
- package/src/tooling/runtime-optimization-index.d.ts +6 -0
- package/src/tooling/runtime-optimization-index.js +72 -0
- package/src/tooling/runtime-optimization.d.ts +47 -6
- package/src/tooling/runtime-optimization.js +559 -40
- package/src/tooling/source-intake.d.ts +49 -0
- package/src/tooling/source-intake.js +414 -0
- package/src/tooling-exports/evidence.d.ts +34 -0
- package/src/tooling-exports/evidence.js +34 -0
- package/src/tooling-exports/runtime.d.ts +7 -0
- package/src/tooling-exports/runtime.js +7 -0
- package/src/tooling-exports.d.ts +11 -0
- package/src/tooling-exports.js +6 -3
- package/src/tooling-exports.shared.d.ts +32 -0
- package/src/tooling-exports.shared.js +31 -0
- package/src/version.js +1 -1
- package/src/config/schema/tuner.schema.d.ts +0 -39
- package/src/config/schema/tuner.schema.js +0 -13
- package/src/gpu/kernel-tuner/benchmarks.js +0 -902
- package/src/gpu/kernel-tuner/cache.js +0 -145
- package/src/gpu/kernel-tuner/index.js +0 -38
- package/src/gpu/kernel-tuner/tuner.js +0 -247
- package/src/gpu/kernel-tuner.js +0 -12
- package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
- package/src/gpu/kernels/kernel-tuning.js +0 -158
- package/src/gpu/profiler.js +0 -382
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import { log } from '../../debug/index.js';
|
|
4
|
-
import { getRuntimeConfig } from '../../config/runtime.js';
|
|
5
|
-
import { getDeviceEpoch } from '../device.js';
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export function getTunerConfig() {
|
|
9
|
-
return getRuntimeConfig().shared.tuner;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
// Track the device epoch at which a cache was loaded. If the epoch has advanced,
|
|
13
|
-
// the cache is stale and should be rebuilt.
|
|
14
|
-
let cacheDeviceEpoch = -1;
|
|
15
|
-
|
|
16
|
-
function normalizeSignaturePart(value) {
|
|
17
|
-
return String(value ?? '')
|
|
18
|
-
.trim()
|
|
19
|
-
.replace(/[^a-zA-Z0-9]/g, '_');
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function hasAdapterIdentity(info) {
|
|
23
|
-
if (!info || typeof info !== 'object') {
|
|
24
|
-
return false;
|
|
25
|
-
}
|
|
26
|
-
return ['vendor', 'architecture', 'device'].some((key) => {
|
|
27
|
-
const value = String(info[key] ?? '').trim().toLowerCase();
|
|
28
|
-
return value.length > 0 && value !== 'unknown';
|
|
29
|
-
});
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function getFallbackSignature(capabilities) {
|
|
33
|
-
return [
|
|
34
|
-
capabilities?.hasF16 ? 'f16' : 'nof16',
|
|
35
|
-
capabilities?.hasSubgroups ? 'subgroups' : 'nosubgroups',
|
|
36
|
-
capabilities?.hasSubgroupsF16 ? 'subgroupsf16' : 'nosubgroupsf16',
|
|
37
|
-
capabilities?.hasTimestampQuery ? 'timestamp' : 'notimestamp',
|
|
38
|
-
`buf${Number.isFinite(capabilities?.maxBufferSize) ? capabilities.maxBufferSize : 'na'}`,
|
|
39
|
-
`wg${Number.isFinite(capabilities?.maxWorkgroupSize) ? capabilities.maxWorkgroupSize : 'na'}`,
|
|
40
|
-
`wgs${Number.isFinite(capabilities?.maxWorkgroupStorageSize) ? capabilities.maxWorkgroupStorageSize : 'na'}`,
|
|
41
|
-
].join('_');
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
function isValidDeviceInfo(value) {
|
|
45
|
-
if (value == null) {
|
|
46
|
-
return true;
|
|
47
|
-
}
|
|
48
|
-
return typeof value === 'object';
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function isValidTuneRecord(value) {
|
|
52
|
-
return !!value
|
|
53
|
-
&& typeof value === 'object'
|
|
54
|
-
&& Array.isArray(value.optimalWorkgroupSize)
|
|
55
|
-
&& value.optimalWorkgroupSize.length === 3
|
|
56
|
-
&& value.optimalWorkgroupSize.every((entry) => Number.isFinite(entry) && entry >= 0)
|
|
57
|
-
&& Number.isFinite(value.optimalTileSize)
|
|
58
|
-
&& value.optimalTileSize >= 0
|
|
59
|
-
&& Number.isFinite(value.throughput)
|
|
60
|
-
&& value.throughput >= 0
|
|
61
|
-
&& Number.isFinite(value.timeMs)
|
|
62
|
-
&& value.timeMs >= 0
|
|
63
|
-
&& isValidDeviceInfo(value.deviceInfo);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
export function getDeviceSignature(capabilities) {
|
|
68
|
-
const info = capabilities?.adapterInfo;
|
|
69
|
-
if (hasAdapterIdentity(info)) {
|
|
70
|
-
return [
|
|
71
|
-
normalizeSignaturePart(info.vendor),
|
|
72
|
-
normalizeSignaturePart(info.architecture),
|
|
73
|
-
normalizeSignaturePart(info.device),
|
|
74
|
-
].join('_');
|
|
75
|
-
}
|
|
76
|
-
return getFallbackSignature(capabilities);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
export function generateCacheKey(kernelName, inputSizes) {
|
|
81
|
-
return `${kernelName}_${JSON.stringify(inputSizes)}`;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
export function loadCache(capabilities) {
|
|
86
|
-
const currentEpoch = getDeviceEpoch();
|
|
87
|
-
if (cacheDeviceEpoch !== -1 && cacheDeviceEpoch !== currentEpoch) {
|
|
88
|
-
log.debug('KernelTuner', 'Device epoch changed (' + cacheDeviceEpoch + ' -> ' + currentEpoch + '), reloading tuner cache');
|
|
89
|
-
}
|
|
90
|
-
cacheDeviceEpoch = currentEpoch;
|
|
91
|
-
|
|
92
|
-
if (typeof localStorage === 'undefined') {
|
|
93
|
-
return new Map();
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
const signature = getDeviceSignature(capabilities);
|
|
97
|
-
const cacheKey = getTunerConfig().cacheKeyPrefix + signature;
|
|
98
|
-
|
|
99
|
-
try {
|
|
100
|
-
const cached = localStorage.getItem(cacheKey);
|
|
101
|
-
if (cached) {
|
|
102
|
-
const data = JSON.parse(cached);
|
|
103
|
-
if (!data || typeof data !== 'object' || Array.isArray(data)) {
|
|
104
|
-
throw new Error('Kernel tuner cache payload must be an object.');
|
|
105
|
-
}
|
|
106
|
-
const records = [];
|
|
107
|
-
for (const [key, value] of Object.entries(data)) {
|
|
108
|
-
if (!isValidTuneRecord(value)) {
|
|
109
|
-
throw new Error(`Kernel tuner cache record "${key}" is malformed.`);
|
|
110
|
-
}
|
|
111
|
-
records.push([key, value]);
|
|
112
|
-
}
|
|
113
|
-
return new Map(records);
|
|
114
|
-
}
|
|
115
|
-
} catch (e) {
|
|
116
|
-
log.warn('KernelTuner', `Failed to load cache: ${e}`);
|
|
117
|
-
localStorage.removeItem(cacheKey);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
return new Map();
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
export function saveCache(cache, capabilities) {
|
|
125
|
-
if (typeof localStorage === 'undefined') return;
|
|
126
|
-
|
|
127
|
-
const signature = getDeviceSignature(capabilities);
|
|
128
|
-
const cacheKey = getTunerConfig().cacheKeyPrefix + signature;
|
|
129
|
-
|
|
130
|
-
try {
|
|
131
|
-
const data = Object.fromEntries(cache);
|
|
132
|
-
localStorage.setItem(cacheKey, JSON.stringify(data));
|
|
133
|
-
} catch (e) {
|
|
134
|
-
log.warn('KernelTuner', `Failed to save cache: ${e}`);
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
export function clearCacheStorage(capabilities) {
|
|
140
|
-
if (typeof localStorage === 'undefined') return;
|
|
141
|
-
|
|
142
|
-
const signature = getDeviceSignature(capabilities);
|
|
143
|
-
localStorage.removeItem(getTunerConfig().cacheKeyPrefix + signature);
|
|
144
|
-
}
|
|
145
|
-
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// Cache utilities
|
|
4
|
-
export {
|
|
5
|
-
getTunerConfig,
|
|
6
|
-
getDeviceSignature,
|
|
7
|
-
generateCacheKey,
|
|
8
|
-
loadCache,
|
|
9
|
-
saveCache,
|
|
10
|
-
clearCacheStorage,
|
|
11
|
-
} from './cache.js';
|
|
12
|
-
|
|
13
|
-
// Benchmark functions
|
|
14
|
-
export {
|
|
15
|
-
benchmarkPipeline,
|
|
16
|
-
createComputePipeline,
|
|
17
|
-
tuneMatmul,
|
|
18
|
-
tuneAttention,
|
|
19
|
-
tuneSoftmax,
|
|
20
|
-
tuneRMSNorm,
|
|
21
|
-
tuneDequant,
|
|
22
|
-
tuneGeneric,
|
|
23
|
-
createMatmulShader,
|
|
24
|
-
createAttentionShader,
|
|
25
|
-
createSoftmaxShader,
|
|
26
|
-
createRMSNormShader,
|
|
27
|
-
createDequantShader,
|
|
28
|
-
} from './benchmarks.js';
|
|
29
|
-
|
|
30
|
-
// Main tuner class and utilities
|
|
31
|
-
export {
|
|
32
|
-
KernelTuner,
|
|
33
|
-
getKernelTuner,
|
|
34
|
-
tuneKernel,
|
|
35
|
-
} from './tuner.js';
|
|
36
|
-
|
|
37
|
-
// Default export
|
|
38
|
-
export { KernelTuner as default } from './tuner.js';
|
|
@@ -1,247 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import { getDevice, getKernelCapabilities, getDeviceLimits, getDeviceEpoch } from '../device.js';
|
|
4
|
-
import { getKernelThresholds } from '../../config/schema/index.js';
|
|
5
|
-
import { GPUProfiler } from '../profiler.js';
|
|
6
|
-
import {
|
|
7
|
-
getTunerConfig,
|
|
8
|
-
loadCache,
|
|
9
|
-
saveCache,
|
|
10
|
-
clearCacheStorage,
|
|
11
|
-
generateCacheKey,
|
|
12
|
-
} from './cache.js';
|
|
13
|
-
import {
|
|
14
|
-
tuneMatmul,
|
|
15
|
-
tuneAttention,
|
|
16
|
-
tuneSoftmax,
|
|
17
|
-
tuneRMSNorm,
|
|
18
|
-
tuneDequant,
|
|
19
|
-
tuneGeneric,
|
|
20
|
-
} from './benchmarks.js';
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
export class KernelTuner {
|
|
24
|
-
|
|
25
|
-
#device;
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
#profiler;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
#limits;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
#capabilities;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
#cache;
|
|
38
|
-
|
|
39
|
-
#deviceEpoch;
|
|
40
|
-
|
|
41
|
-
constructor() {
|
|
42
|
-
this.#device = null;
|
|
43
|
-
this.#profiler = null;
|
|
44
|
-
this.#limits = null;
|
|
45
|
-
this.#capabilities = null;
|
|
46
|
-
this.#cache = new Map();
|
|
47
|
-
this.#deviceEpoch = -1;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
async init() {
|
|
52
|
-
const device = getDevice();
|
|
53
|
-
if (!device) {
|
|
54
|
-
throw new Error('GPU device not initialized');
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
const deviceEpoch = getDeviceEpoch();
|
|
58
|
-
if (this.#device === device && this.#deviceEpoch === deviceEpoch && this.#profiler) {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
this.destroy();
|
|
63
|
-
|
|
64
|
-
this.#device = device;
|
|
65
|
-
this.#deviceEpoch = deviceEpoch;
|
|
66
|
-
this.#profiler = new GPUProfiler(this.#device);
|
|
67
|
-
this.#limits = getDeviceLimits();
|
|
68
|
-
this.#capabilities = getKernelCapabilities();
|
|
69
|
-
|
|
70
|
-
// Load cached results
|
|
71
|
-
this.#cache = loadCache(this.#capabilities);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
#generateWorkgroupCandidates() {
|
|
76
|
-
const { maxComputeWorkgroupSizeX, maxComputeWorkgroupSizeY, maxComputeInvocationsPerWorkgroup } = getKernelThresholds().tuner;
|
|
77
|
-
const maxX = this.#limits?.maxComputeWorkgroupSizeX ?? maxComputeWorkgroupSizeX;
|
|
78
|
-
const maxY = this.#limits?.maxComputeWorkgroupSizeY ?? maxComputeWorkgroupSizeY;
|
|
79
|
-
const maxInvocations = this.#limits?.maxComputeInvocationsPerWorkgroup ?? maxComputeInvocationsPerWorkgroup;
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
const candidates = [];
|
|
83
|
-
|
|
84
|
-
// 1D workgroups
|
|
85
|
-
for (const x of [64, 128, 256, 512]) {
|
|
86
|
-
if (x <= maxX && x <= maxInvocations) {
|
|
87
|
-
candidates.push([x, 1, 1]);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// 2D workgroups (for matrix operations)
|
|
92
|
-
for (const x of [8, 16, 32]) {
|
|
93
|
-
for (const y of [8, 16, 32]) {
|
|
94
|
-
if (x <= maxX && y <= maxY && x * y <= maxInvocations) {
|
|
95
|
-
candidates.push([x, y, 1]);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
return candidates;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
async tuneKernel(
|
|
105
|
-
kernelName,
|
|
106
|
-
inputSizes,
|
|
107
|
-
options = {}
|
|
108
|
-
) {
|
|
109
|
-
const {
|
|
110
|
-
warmup = getTunerConfig().defaultWarmupIterations,
|
|
111
|
-
iterations = getTunerConfig().defaultTimedIterations,
|
|
112
|
-
forceRetune = false,
|
|
113
|
-
} = options;
|
|
114
|
-
|
|
115
|
-
// Check cache
|
|
116
|
-
|
|
117
|
-
const cacheKey = generateCacheKey(kernelName, inputSizes);
|
|
118
|
-
if (!forceRetune && this.#cache.has(cacheKey)) {
|
|
119
|
-
return (this.#cache.get(cacheKey));
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// Get candidates to test
|
|
123
|
-
const candidates = this.#generateWorkgroupCandidates();
|
|
124
|
-
|
|
125
|
-
// Run tuning based on kernel type
|
|
126
|
-
|
|
127
|
-
let bestResult;
|
|
128
|
-
|
|
129
|
-
if (!this.#device) {
|
|
130
|
-
return tuneGeneric(this.#capabilities);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
switch (kernelName) {
|
|
134
|
-
case 'matmul':
|
|
135
|
-
bestResult = await tuneMatmul(
|
|
136
|
-
this.#device,
|
|
137
|
-
inputSizes,
|
|
138
|
-
candidates,
|
|
139
|
-
warmup,
|
|
140
|
-
iterations,
|
|
141
|
-
this.#capabilities
|
|
142
|
-
);
|
|
143
|
-
break;
|
|
144
|
-
case 'attention':
|
|
145
|
-
bestResult = await tuneAttention(
|
|
146
|
-
this.#device,
|
|
147
|
-
inputSizes,
|
|
148
|
-
candidates,
|
|
149
|
-
warmup,
|
|
150
|
-
iterations,
|
|
151
|
-
this.#capabilities
|
|
152
|
-
);
|
|
153
|
-
break;
|
|
154
|
-
case 'softmax':
|
|
155
|
-
bestResult = await tuneSoftmax(
|
|
156
|
-
this.#device,
|
|
157
|
-
inputSizes,
|
|
158
|
-
candidates,
|
|
159
|
-
warmup,
|
|
160
|
-
iterations,
|
|
161
|
-
this.#capabilities
|
|
162
|
-
);
|
|
163
|
-
break;
|
|
164
|
-
case 'rmsnorm':
|
|
165
|
-
bestResult = await tuneRMSNorm(
|
|
166
|
-
this.#device,
|
|
167
|
-
inputSizes,
|
|
168
|
-
candidates,
|
|
169
|
-
warmup,
|
|
170
|
-
iterations,
|
|
171
|
-
this.#capabilities
|
|
172
|
-
);
|
|
173
|
-
break;
|
|
174
|
-
case 'dequant':
|
|
175
|
-
bestResult = await tuneDequant(
|
|
176
|
-
this.#device,
|
|
177
|
-
inputSizes,
|
|
178
|
-
candidates,
|
|
179
|
-
warmup,
|
|
180
|
-
iterations,
|
|
181
|
-
this.#capabilities
|
|
182
|
-
);
|
|
183
|
-
break;
|
|
184
|
-
default:
|
|
185
|
-
bestResult = tuneGeneric(this.#capabilities);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
// Cache result
|
|
189
|
-
this.#cache.set(cacheKey, bestResult);
|
|
190
|
-
saveCache(this.#cache, this.#capabilities);
|
|
191
|
-
|
|
192
|
-
return bestResult;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
getCachedResult(kernelName, inputSizes) {
|
|
197
|
-
|
|
198
|
-
const cacheKey = generateCacheKey(kernelName, inputSizes);
|
|
199
|
-
return this.#cache.get(cacheKey) || null;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
clearCache() {
|
|
204
|
-
this.#cache.clear();
|
|
205
|
-
clearCacheStorage(this.#capabilities);
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
getAllCachedResults() {
|
|
210
|
-
return Object.fromEntries(this.#cache);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
destroy() {
|
|
215
|
-
if (this.#profiler) {
|
|
216
|
-
this.#profiler.destroy();
|
|
217
|
-
}
|
|
218
|
-
this.#profiler = null;
|
|
219
|
-
this.#device = null;
|
|
220
|
-
this.#limits = null;
|
|
221
|
-
this.#capabilities = null;
|
|
222
|
-
this.#cache = new Map();
|
|
223
|
-
this.#deviceEpoch = -1;
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Global tuner instance
|
|
228
|
-
|
|
229
|
-
let globalTuner = null;
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
export async function getKernelTuner() {
|
|
233
|
-
if (!globalTuner) {
|
|
234
|
-
globalTuner = new KernelTuner();
|
|
235
|
-
}
|
|
236
|
-
await globalTuner.init();
|
|
237
|
-
return globalTuner;
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
export async function tuneKernel(
|
|
242
|
-
kernelName,
|
|
243
|
-
inputSizes
|
|
244
|
-
) {
|
|
245
|
-
const tuner = await getKernelTuner();
|
|
246
|
-
return tuner.tuneKernel(kernelName, inputSizes);
|
|
247
|
-
}
|
package/src/gpu/kernel-tuner.js
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Kernel Tuning - Auto-tuning and kernel prewarming
|
|
3
|
-
*
|
|
4
|
-
* Provides utilities for tuning kernel workgroup sizes and
|
|
5
|
-
* prewarming kernel pipelines for optimal performance.
|
|
6
|
-
*
|
|
7
|
-
* @module gpu/kernels/kernel-tuning
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
// ============================================================================
|
|
11
|
-
// Workgroup Size Tuning
|
|
12
|
-
// ============================================================================
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Get tuned workgroup size for an operation
|
|
16
|
-
*/
|
|
17
|
-
export declare function getTunedWorkgroupSize(
|
|
18
|
-
operation: string,
|
|
19
|
-
inputSizes?: Record<string, number>
|
|
20
|
-
): Promise<[number, number, number]>;
|
|
21
|
-
|
|
22
|
-
// ============================================================================
|
|
23
|
-
// Auto-Tuning
|
|
24
|
-
// ============================================================================
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Run auto-tuning for all kernels with given model config
|
|
28
|
-
*/
|
|
29
|
-
export declare function autoTuneKernels(
|
|
30
|
-
modelConfig?: Record<string, number>
|
|
31
|
-
): Promise<Record<string, any>>;
|
|
32
|
-
|
|
33
|
-
// ============================================================================
|
|
34
|
-
// Pipeline Prewarming
|
|
35
|
-
// ============================================================================
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Prewarm all supported kernel pipelines
|
|
39
|
-
*/
|
|
40
|
-
export declare function prewarmKernels(
|
|
41
|
-
options?: { mode?: 'parallel' | 'sequential' }
|
|
42
|
-
): Promise<void>;
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import { getKernelCapabilities } from '../device.js';
|
|
4
|
-
import { getKernelTuner, getTunerConfig } from '../kernel-tuner.js';
|
|
5
|
-
import { KERNEL_CONFIGS } from './kernel-configs.js';
|
|
6
|
-
import { createPipeline, clearPipelineCaches } from './pipeline-cache.js';
|
|
7
|
-
import { clearShaderCaches } from './shader-cache.js';
|
|
8
|
-
import { hasRequiredFeatures } from './feature-check.js';
|
|
9
|
-
import { log } from '../../debug/index.js';
|
|
10
|
-
|
|
11
|
-
// ============================================================================
|
|
12
|
-
// Workgroup Size Tuning
|
|
13
|
-
// ============================================================================
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
export async function getTunedWorkgroupSize(
|
|
17
|
-
operation,
|
|
18
|
-
inputSizes = {}
|
|
19
|
-
) {
|
|
20
|
-
try {
|
|
21
|
-
const tuner = await getKernelTuner();
|
|
22
|
-
const result = tuner.getCachedResult(operation, inputSizes);
|
|
23
|
-
|
|
24
|
-
if (result) {
|
|
25
|
-
return result.optimalWorkgroupSize;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// Run tuning if not cached
|
|
29
|
-
const tuneResult = await tuner.tuneKernel(operation, inputSizes);
|
|
30
|
-
return tuneResult.optimalWorkgroupSize;
|
|
31
|
-
} catch (e) {
|
|
32
|
-
log.warn('KernelTuning', `Tuning failed for ${operation}, using defaults: ${e.message}`);
|
|
33
|
-
const { fallbackWorkgroupSizes } = getTunerConfig();
|
|
34
|
-
const fallback = fallbackWorkgroupSizes?.[operation] ?? fallbackWorkgroupSizes?.default;
|
|
35
|
-
if (!fallback) {
|
|
36
|
-
throw new Error(`KernelTuning: missing fallback workgroup size for "${operation}".`);
|
|
37
|
-
}
|
|
38
|
-
return fallback;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
// ============================================================================
|
|
43
|
-
// Auto-Tuning
|
|
44
|
-
// ============================================================================
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
export async function autoTuneKernels(
|
|
48
|
-
modelConfig = {}
|
|
49
|
-
) {
|
|
50
|
-
const {
|
|
51
|
-
hiddenSize = 4096,
|
|
52
|
-
intermediateSize = 14336,
|
|
53
|
-
numHeads = 32,
|
|
54
|
-
headDim = 128,
|
|
55
|
-
maxSeqLen = 4096,
|
|
56
|
-
vocabSize = 32000,
|
|
57
|
-
} = modelConfig;
|
|
58
|
-
|
|
59
|
-
const tuner = await getKernelTuner();
|
|
60
|
-
|
|
61
|
-
const results = {};
|
|
62
|
-
|
|
63
|
-
// Tune matmul for common sizes
|
|
64
|
-
results.matmul_hidden = await tuner.tuneKernel('matmul', {
|
|
65
|
-
M: 1, N: hiddenSize, K: hiddenSize,
|
|
66
|
-
});
|
|
67
|
-
results.matmul_ffn = await tuner.tuneKernel('matmul', {
|
|
68
|
-
M: 1, N: intermediateSize, K: hiddenSize,
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
// Tune attention
|
|
72
|
-
results.attention = await tuner.tuneKernel('attention', {
|
|
73
|
-
seqLen: 1, numHeads, headDim,
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
// Tune softmax (LM head output)
|
|
77
|
-
results.softmax = await tuner.tuneKernel('softmax', {
|
|
78
|
-
innerSize: vocabSize, outerSize: 1,
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
// Tune RMSNorm
|
|
82
|
-
results.rmsnorm = await tuner.tuneKernel('rmsnorm', {
|
|
83
|
-
hiddenSize, numTokens: 1,
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
// Tune dequant
|
|
87
|
-
results.dequant = await tuner.tuneKernel('dequant', {
|
|
88
|
-
numBlocks: 1000,
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
log.debug('KernelTuning', `Auto-tuning complete: ${JSON.stringify(results)}`);
|
|
92
|
-
return results;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// ============================================================================
|
|
96
|
-
// Pipeline Prewarming
|
|
97
|
-
// ============================================================================
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
export async function prewarmKernels(
|
|
101
|
-
options = {}
|
|
102
|
-
) {
|
|
103
|
-
const caps = getKernelCapabilities();
|
|
104
|
-
const mode = options.mode ?? 'parallel';
|
|
105
|
-
const entries = Object.entries(KERNEL_CONFIGS)
|
|
106
|
-
.sort(([a], [b]) => a.localeCompare(b))
|
|
107
|
-
.map(([operation, variants]) => [
|
|
108
|
-
operation,
|
|
109
|
-
Object.entries(variants).sort(([a], [b]) => a.localeCompare(b))
|
|
110
|
-
]);
|
|
111
|
-
|
|
112
|
-
try {
|
|
113
|
-
if (mode === 'sequential') {
|
|
114
|
-
let count = 0;
|
|
115
|
-
for (const [operation, variants] of entries) {
|
|
116
|
-
for (const [variant, cfg] of variants) {
|
|
117
|
-
if (cfg.requires && !hasRequiredFeatures(cfg.requires, caps)) {
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
|
-
try {
|
|
121
|
-
await createPipeline(operation, variant);
|
|
122
|
-
count += 1;
|
|
123
|
-
} catch (e) {
|
|
124
|
-
log.warn('KernelTuning', `Prewarm failed for ${operation}/${variant}: ${e.message}`);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
log.debug('KernelTuning', `Prewarmed ${count} kernel pipelines`);
|
|
129
|
-
return;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const jobs = [];
|
|
134
|
-
for (const [operation, variants] of entries) {
|
|
135
|
-
for (const [variant, cfg] of variants) {
|
|
136
|
-
if (cfg.requires && !hasRequiredFeatures(cfg.requires, caps)) {
|
|
137
|
-
continue;
|
|
138
|
-
}
|
|
139
|
-
jobs.push(
|
|
140
|
-
createPipeline(operation, variant)
|
|
141
|
-
.then(() => {}) // Ignore the pipeline result
|
|
142
|
-
.catch((e) => {
|
|
143
|
-
log.warn('KernelTuning', `Prewarm failed for ${operation}/${variant}: ${e.message}`);
|
|
144
|
-
})
|
|
145
|
-
);
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
await Promise.all(jobs);
|
|
150
|
-
log.debug('KernelTuning', `Prewarmed ${jobs.length} kernel pipelines`);
|
|
151
|
-
} catch (e) {
|
|
152
|
-
// Clean up partially compiled shaders and pipelines to avoid leaking GPU resources
|
|
153
|
-
log.warn('KernelTuning', `Prewarm aborted, cleaning up compiled pipelines: ${e.message}`);
|
|
154
|
-
clearPipelineCaches();
|
|
155
|
-
clearShaderCaches();
|
|
156
|
-
throw e;
|
|
157
|
-
}
|
|
158
|
-
}
|