doppler-gpu 0.4.8 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/models/catalog.json +21 -7
- package/package.json +8 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/version.js +1 -1
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Matrix Multiplication Backward Kernel - dX = dY @ W^T with packed F16 W.
|
|
2
|
+
//
|
|
3
|
+
// dY and dX remain F32. W is stored as consecutive IEEE-754 binary16 values,
|
|
4
|
+
// packed two values per u32 storage word. unpack2x16float keeps this path
|
|
5
|
+
// available without requiring native shader-f16 arithmetic.
|
|
6
|
+
|
|
7
|
+
const MAX_TILE_SIZE: u32 = 16u;
|
|
8
|
+
const TILE_AREA: u32 = MAX_TILE_SIZE * MAX_TILE_SIZE;
|
|
9
|
+
|
|
10
|
+
override TILE_SIZE: u32 = 16u;
|
|
11
|
+
|
|
12
|
+
struct Uniforms {
|
|
13
|
+
M: u32,
|
|
14
|
+
N: u32,
|
|
15
|
+
K: u32,
|
|
16
|
+
alpha: f32,
|
|
17
|
+
transpose_b: u32,
|
|
18
|
+
_pad0: u32,
|
|
19
|
+
_pad1: u32,
|
|
20
|
+
_pad2: u32,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
24
|
+
@group(0) @binding(1) var<storage, read> dY: array<f32>;
|
|
25
|
+
@group(0) @binding(2) var<storage, read> W: array<u32>;
|
|
26
|
+
@group(0) @binding(3) var<storage, read_write> dX: array<f32>;
|
|
27
|
+
|
|
28
|
+
var<workgroup> tileDY: array<f32, TILE_AREA>;
|
|
29
|
+
var<workgroup> tileW: array<f32, TILE_AREA>;
|
|
30
|
+
|
|
31
|
+
fn load_f16(index: u32) -> f32 {
|
|
32
|
+
let pair = unpack2x16float(W[index >> 1u]);
|
|
33
|
+
return select(pair.x, pair.y, (index & 1u) == 1u);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
@compute @workgroup_size(TILE_SIZE, TILE_SIZE, 1)
|
|
37
|
+
fn main(
|
|
38
|
+
@builtin(global_invocation_id) global_id: vec3<u32>,
|
|
39
|
+
@builtin(local_invocation_id) local_id: vec3<u32>
|
|
40
|
+
) {
|
|
41
|
+
let row = global_id.x;
|
|
42
|
+
let col = global_id.y;
|
|
43
|
+
let local_row = local_id.x;
|
|
44
|
+
let local_col = local_id.y;
|
|
45
|
+
|
|
46
|
+
var sum: f32 = 0.0;
|
|
47
|
+
let num_tiles = (u.N + TILE_SIZE - 1u) / TILE_SIZE;
|
|
48
|
+
|
|
49
|
+
for (var t: u32 = 0u; t < num_tiles; t = t + 1u) {
|
|
50
|
+
let tile_n = t * TILE_SIZE;
|
|
51
|
+
let tile_idx = local_row * TILE_SIZE + local_col;
|
|
52
|
+
let dy_col = tile_n + local_col;
|
|
53
|
+
if (row < u.M && dy_col < u.N) {
|
|
54
|
+
tileDY[tile_idx] = dY[row * u.N + dy_col];
|
|
55
|
+
} else {
|
|
56
|
+
tileDY[tile_idx] = 0.0;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
let wt_row = tile_n + local_row;
|
|
60
|
+
if (wt_row < u.N && col < u.K) {
|
|
61
|
+
if (u.transpose_b == 0u) {
|
|
62
|
+
tileW[tile_idx] = load_f16(col * u.N + wt_row);
|
|
63
|
+
} else {
|
|
64
|
+
tileW[tile_idx] = load_f16(wt_row * u.K + col);
|
|
65
|
+
}
|
|
66
|
+
} else {
|
|
67
|
+
tileW[tile_idx] = 0.0;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
workgroupBarrier();
|
|
71
|
+
|
|
72
|
+
for (var k: u32 = 0u; k < TILE_SIZE; k = k + 1u) {
|
|
73
|
+
sum += tileDY[local_row * TILE_SIZE + k] * tileW[k * TILE_SIZE + local_col];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
workgroupBarrier();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (row < u.M && col < u.K) {
|
|
80
|
+
dX[row * u.K + col] = sum * u.alpha;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -9,7 +9,13 @@ import { DEFAULT_HIGH_PRECISION_EPS } from '../../../config/schema/index.js';
|
|
|
9
9
|
|
|
10
10
|
export async function runRmsNormBackward(input, weight, gradOutput, options = {}) {
|
|
11
11
|
const device = getDevice();
|
|
12
|
-
const {
|
|
12
|
+
const {
|
|
13
|
+
numTokens,
|
|
14
|
+
hiddenSize,
|
|
15
|
+
eps = DEFAULT_HIGH_PRECISION_EPS,
|
|
16
|
+
rmsNormWeightOffset = false,
|
|
17
|
+
outputBuffer = null,
|
|
18
|
+
} = options;
|
|
13
19
|
|
|
14
20
|
if (!numTokens || !hiddenSize) {
|
|
15
21
|
throw new Error('rmsnorm backward requires numTokens and hiddenSize');
|
|
@@ -22,7 +28,12 @@ export async function runRmsNormBackward(input, weight, gradOutput, options = {}
|
|
|
22
28
|
const outputSize = numTokens * hiddenSize * 4;
|
|
23
29
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'rmsnorm_backward_output');
|
|
24
30
|
|
|
25
|
-
const pipeline = await createPipeline(
|
|
31
|
+
const pipeline = await createPipeline(
|
|
32
|
+
'rmsnorm_backward',
|
|
33
|
+
'default',
|
|
34
|
+
null,
|
|
35
|
+
{ RMS_NORM_OFFSET: rmsNormWeightOffset }
|
|
36
|
+
);
|
|
26
37
|
const uniformBuffer = createUniformBufferWithView(
|
|
27
38
|
'rmsnorm_backward_uniforms',
|
|
28
39
|
16,
|
|
@@ -57,7 +68,13 @@ export async function runRmsNormBackward(input, weight, gradOutput, options = {}
|
|
|
57
68
|
|
|
58
69
|
export async function recordRmsNormBackward(recorder, input, weight, gradOutput, options = {}) {
|
|
59
70
|
const device = recorder.device;
|
|
60
|
-
const {
|
|
71
|
+
const {
|
|
72
|
+
numTokens,
|
|
73
|
+
hiddenSize,
|
|
74
|
+
eps = DEFAULT_HIGH_PRECISION_EPS,
|
|
75
|
+
rmsNormWeightOffset = false,
|
|
76
|
+
outputBuffer = null,
|
|
77
|
+
} = options;
|
|
61
78
|
|
|
62
79
|
if (!numTokens || !hiddenSize) {
|
|
63
80
|
throw new Error('rmsnorm backward requires numTokens and hiddenSize');
|
|
@@ -70,7 +87,12 @@ export async function recordRmsNormBackward(recorder, input, weight, gradOutput,
|
|
|
70
87
|
const outputSize = numTokens * hiddenSize * 4;
|
|
71
88
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'rmsnorm_backward_output');
|
|
72
89
|
|
|
73
|
-
const pipeline = await createPipeline(
|
|
90
|
+
const pipeline = await createPipeline(
|
|
91
|
+
'rmsnorm_backward',
|
|
92
|
+
'default',
|
|
93
|
+
null,
|
|
94
|
+
{ RMS_NORM_OFFSET: rmsNormWeightOffset }
|
|
95
|
+
);
|
|
74
96
|
const uniformBuffer = createUniformBufferWithView(
|
|
75
97
|
'rmsnorm_backward_uniforms',
|
|
76
98
|
16,
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* Uses shared-memory reduction for per-token stats.
|
|
8
8
|
*/
|
|
9
9
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
10
|
+
override RMS_NORM_OFFSET: bool = false;
|
|
10
11
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
11
12
|
|
|
12
13
|
struct Uniforms {
|
|
@@ -44,7 +45,8 @@ fn main(
|
|
|
44
45
|
|
|
45
46
|
for (var i: u32 = local_id; i < hidden_size; i = i + WORKGROUP_SIZE) {
|
|
46
47
|
let x = input[base + i];
|
|
47
|
-
let
|
|
48
|
+
let effective_weight = select(weight[i], 1.0 + weight[i], RMS_NORM_OFFSET);
|
|
49
|
+
let g = grad_output[base + i] * effective_weight;
|
|
48
50
|
sum_sq = sum_sq + x * x;
|
|
49
51
|
sum_gx = sum_gx + g * x;
|
|
50
52
|
}
|
|
@@ -72,7 +74,8 @@ fn main(
|
|
|
72
74
|
|
|
73
75
|
for (var i: u32 = local_id; i < hidden_size; i = i + WORKGROUP_SIZE) {
|
|
74
76
|
let x = input[base + i];
|
|
75
|
-
let
|
|
77
|
+
let effective_weight = select(weight[i], 1.0 + weight[i], RMS_NORM_OFFSET);
|
|
78
|
+
let g = grad_output[base + i] * effective_weight;
|
|
76
79
|
output[base + i] = g * inv_rms - x * coeff;
|
|
77
80
|
}
|
|
78
|
-
}
|
|
81
|
+
}
|
|
@@ -56,6 +56,16 @@ export declare function createBackwardKernel(
|
|
|
56
56
|
record: (recorder: CommandRecorder, ...args: unknown[]) => Promise<Tensor>;
|
|
57
57
|
};
|
|
58
58
|
|
|
59
|
+
export declare function resolveMatmulBackwardDxVariant(
|
|
60
|
+
weight: { dtype?: string | null } | null
|
|
61
|
+
): 'default' | 'f16_weight';
|
|
62
|
+
|
|
63
|
+
export interface MatmulBackwardDxOptions {
|
|
64
|
+
alpha?: number;
|
|
65
|
+
transposeB?: boolean;
|
|
66
|
+
outputBuffer?: GPUBuffer | null;
|
|
67
|
+
}
|
|
68
|
+
|
|
59
69
|
/**
|
|
60
70
|
* Matmul-backward gradient w.r.t. input (dX = dY · Wᵀ). Submit + wait.
|
|
61
71
|
*/
|
|
@@ -65,7 +75,7 @@ export declare function runMatmulBackwardDx(
|
|
|
65
75
|
M: number,
|
|
66
76
|
K: number,
|
|
67
77
|
N: number,
|
|
68
|
-
options?:
|
|
78
|
+
options?: MatmulBackwardDxOptions
|
|
69
79
|
): Promise<Tensor>;
|
|
70
80
|
|
|
71
81
|
/** Record runMatmulBackwardDx onto a CommandRecorder. */
|
|
@@ -76,5 +86,5 @@ export declare function recordMatmulBackwardDx(
|
|
|
76
86
|
M: number,
|
|
77
87
|
K: number,
|
|
78
88
|
N: number,
|
|
79
|
-
options?:
|
|
89
|
+
options?: MatmulBackwardDxOptions
|
|
80
90
|
): Promise<Tensor>;
|
|
@@ -192,13 +192,20 @@ export function createBackwardKernel(opName, spec) {
|
|
|
192
192
|
return { run, record };
|
|
193
193
|
}
|
|
194
194
|
|
|
195
|
+
export function resolveMatmulBackwardDxVariant(weight) {
|
|
196
|
+
const dtype = String(weight?.dtype || 'f32').toLowerCase();
|
|
197
|
+
if (dtype === 'f16') return 'f16_weight';
|
|
198
|
+
if (dtype === 'f32') return 'default';
|
|
199
|
+
throw new Error(`matmul backward dX does not support weight dtype "${dtype}".`);
|
|
200
|
+
}
|
|
201
|
+
|
|
195
202
|
export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
|
|
196
203
|
const { alpha = 1.0, transposeB = false, outputBuffer = null } = options;
|
|
197
204
|
const device = getDevice();
|
|
198
205
|
const outputSize = M * K * 4;
|
|
199
206
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
|
|
200
207
|
|
|
201
|
-
const pipeline = await createPipeline('matmul_backward',
|
|
208
|
+
const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
|
|
202
209
|
|
|
203
210
|
const uniformBuffer = createUniformBufferWithView(
|
|
204
211
|
'matmul_backward_uniforms',
|
|
@@ -244,7 +251,7 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
|
|
|
244
251
|
const outputSize = M * K * 4;
|
|
245
252
|
const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
|
|
246
253
|
|
|
247
|
-
const pipeline = await createPipeline('matmul_backward',
|
|
254
|
+
const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
|
|
248
255
|
|
|
249
256
|
const uniformBuffer = createUniformBufferWithView(
|
|
250
257
|
'matmul_backward_uniforms',
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* Cast F16 to F32 Kernel
|
|
5
5
|
*
|
|
6
6
|
* Converts a buffer of f16 values to f32.
|
|
7
|
+
* Supports 2D dispatch for tensors larger than one workgroup dimension.
|
|
7
8
|
*/
|
|
8
9
|
|
|
9
10
|
enable f16;
|
|
@@ -22,8 +23,11 @@ struct Uniforms {
|
|
|
22
23
|
@group(0) @binding(2) var<storage, read_write> output: array<f32>;
|
|
23
24
|
|
|
24
25
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
25
|
-
fn main(
|
|
26
|
-
|
|
26
|
+
fn main(
|
|
27
|
+
@builtin(global_invocation_id) gid: vec3<u32>,
|
|
28
|
+
@builtin(num_workgroups) num_wg: vec3<u32>
|
|
29
|
+
) {
|
|
30
|
+
let idx = gid.x + gid.y * num_wg.x * WORKGROUP_SIZE;
|
|
27
31
|
if (idx >= u.count) {
|
|
28
32
|
return;
|
|
29
33
|
}
|
|
@@ -460,6 +460,11 @@ export {
|
|
|
460
460
|
recordSoftmaxBackward,
|
|
461
461
|
runRmsNormBackward,
|
|
462
462
|
recordRmsNormBackward,
|
|
463
|
+
runGatedRmsNormBackward,
|
|
464
|
+
recordGatedRmsNormBackward,
|
|
465
|
+
runCausalConv1dSiluBackward,
|
|
466
|
+
recordCausalConv1dSiluBackward,
|
|
467
|
+
runGatedDeltaRecurrentBackward,
|
|
463
468
|
runLayerNormBackward,
|
|
464
469
|
recordLayerNormBackward,
|
|
465
470
|
runAttentionBackward,
|
package/src/gpu/kernels/index.js
CHANGED
|
@@ -370,6 +370,11 @@ export {
|
|
|
370
370
|
recordSoftmaxBackward,
|
|
371
371
|
runRmsNormBackward,
|
|
372
372
|
recordRmsNormBackward,
|
|
373
|
+
runGatedRmsNormBackward,
|
|
374
|
+
recordGatedRmsNormBackward,
|
|
375
|
+
runCausalConv1dSiluBackward,
|
|
376
|
+
recordCausalConv1dSiluBackward,
|
|
377
|
+
runGatedDeltaRecurrentBackward,
|
|
373
378
|
runLayerNormBackward,
|
|
374
379
|
recordLayerNormBackward,
|
|
375
380
|
runAttentionBackward,
|
|
@@ -460,6 +460,11 @@ export function selectMatmulVariantAndFlags(mode, M, N, K, aDtype, bDtype, trans
|
|
|
460
460
|
variant: pathVariant,
|
|
461
461
|
reason: 'path_override_quantized_fallthrough',
|
|
462
462
|
});
|
|
463
|
+
} else if (bDtype === 'f16' && isQ4KFusedVariant(pathVariant)) {
|
|
464
|
+
logKernelSelectionOnce('matmul', {
|
|
465
|
+
variant: pathVariant,
|
|
466
|
+
reason: 'path_override_f16_fallthrough',
|
|
467
|
+
});
|
|
463
468
|
} else {
|
|
464
469
|
throw new Error(`[Matmul] Path variant "${pathVariant}" rejected for role=${options.role ?? '?'} layerIdx=${options.layerIdx ?? '?'} phase=${phase} M=${M} K=${K} aDtype=${aDtype} bDtype=${bDtype} outDtype=${requestedOutputDtype}`);
|
|
465
470
|
}
|
|
@@ -16,6 +16,7 @@ export interface SiLUOptions extends OutputBufferOptions {
|
|
|
16
16
|
size?: number | null;
|
|
17
17
|
gate?: Tensor | null;
|
|
18
18
|
gateActivation?: 'silu' | 'sigmoid';
|
|
19
|
+
/** Required when gate is present; ungated SiLU defaults to silu. */
|
|
19
20
|
inputActivation?: 'silu' | 'identity';
|
|
20
21
|
useVec4?: boolean;
|
|
21
22
|
biasOffset?: number;
|
package/src/gpu/kernels/silu.js
CHANGED
|
@@ -49,6 +49,19 @@ function resolveSwigluLimit(value, context) {
|
|
|
49
49
|
return value;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
function resolveInputActivation(value, hasGate) {
|
|
53
|
+
if (value === undefined) {
|
|
54
|
+
if (hasGate) {
|
|
55
|
+
throw new Error('Gated SiLU requires an explicit inputActivation (identity or silu).');
|
|
56
|
+
}
|
|
57
|
+
return 'silu';
|
|
58
|
+
}
|
|
59
|
+
if (value !== 'identity' && value !== 'silu') {
|
|
60
|
+
throw new Error(`Unsupported SiLU inputActivation: ${value}`);
|
|
61
|
+
}
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
|
|
52
65
|
|
|
53
66
|
function createSiLUBindGroupEntries(uniformBuffer, input, output, gate) {
|
|
54
67
|
const gateBuffer = gate?.buffer ?? input.buffer;
|
|
@@ -91,9 +104,10 @@ export async function runSiLU(
|
|
|
91
104
|
useVec4 = false,
|
|
92
105
|
swigluLimit,
|
|
93
106
|
gateActivation = 'silu',
|
|
94
|
-
inputActivation
|
|
107
|
+
inputActivation,
|
|
95
108
|
} = options;
|
|
96
109
|
const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
|
|
110
|
+
const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
|
|
97
111
|
|
|
98
112
|
const isF16 = canUseF16(input);
|
|
99
113
|
const bytesPerElement = dtypeBytes(input.dtype);
|
|
@@ -109,7 +123,7 @@ export async function runSiLU(
|
|
|
109
123
|
const constants = {
|
|
110
124
|
...(overrides || {}),
|
|
111
125
|
...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
|
|
112
|
-
...(
|
|
126
|
+
...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
|
|
113
127
|
};
|
|
114
128
|
const pipeline = await getPipelineFast('silu', variant, null, constants);
|
|
115
129
|
|
|
@@ -344,9 +358,10 @@ export async function recordSiLU(
|
|
|
344
358
|
outputBuffer = null,
|
|
345
359
|
swigluLimit,
|
|
346
360
|
gateActivation = 'silu',
|
|
347
|
-
inputActivation
|
|
361
|
+
inputActivation,
|
|
348
362
|
} = options;
|
|
349
363
|
const resolvedSwigluLimit = resolveSwigluLimit(swigluLimit, 'SiLU');
|
|
364
|
+
const resolvedInputActivation = resolveInputActivation(inputActivation, Boolean(gate));
|
|
350
365
|
|
|
351
366
|
const isF16 = canUseF16(input);
|
|
352
367
|
const bytesPerElement = dtypeBytes(input.dtype);
|
|
@@ -362,7 +377,7 @@ export async function recordSiLU(
|
|
|
362
377
|
const constants = {
|
|
363
378
|
...(overrides || {}),
|
|
364
379
|
...(gate && gateActivation === 'sigmoid' ? { GATE_USE_SIGMOID: true } : {}),
|
|
365
|
-
...(
|
|
380
|
+
...(resolvedInputActivation === 'identity' ? { INPUT_USE_IDENTITY: true } : {}),
|
|
366
381
|
};
|
|
367
382
|
const pipeline = await getPipelineFast('silu', variant, null, constants);
|
|
368
383
|
|
|
@@ -63,7 +63,7 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
function resolveEmbeddingScale(config, hiddenSize) {
|
|
66
|
+
export function resolveEmbeddingScale(config, hiddenSize) {
|
|
67
67
|
const embeddingScale = config.embeddingScale;
|
|
68
68
|
const scaleEmbeddings = config.scaleEmbeddings;
|
|
69
69
|
if (embeddingScale === undefined) {
|
|
@@ -530,10 +530,14 @@ function collectCoveredLayers(entries, targetLayers, predicate) {
|
|
|
530
530
|
return covered;
|
|
531
531
|
}
|
|
532
532
|
|
|
533
|
-
function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId) {
|
|
533
|
+
function assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype) {
|
|
534
534
|
if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
|
|
535
535
|
return;
|
|
536
536
|
}
|
|
537
|
+
const normalizedWeightDtype = String(weightDtype || '').trim().toLowerCase();
|
|
538
|
+
if (normalizedWeightDtype === 'f16' || normalizedWeightDtype === 'f32') {
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
537
541
|
|
|
538
542
|
const linearLayers = layerTypes
|
|
539
543
|
.map((layerType, layerIdx) => ({ layerType, layerIdx }))
|
|
@@ -730,6 +734,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
730
734
|
const modelId = options.modelId ?? 'model';
|
|
731
735
|
const numLayers = options.numLayers ?? 0;
|
|
732
736
|
const headDim = Number.isFinite(options.headDim) ? Math.floor(options.headDim) : null;
|
|
737
|
+
const weightDtype = options.weightDtype ?? null;
|
|
733
738
|
const hasCapabilityProof = options.capabilities && typeof options.capabilities === 'object';
|
|
734
739
|
const hasUseGPUOption = hasOwnProperty(options, 'useGPU');
|
|
735
740
|
const useGPU = hasUseGPUOption
|
|
@@ -861,7 +866,7 @@ export function compileExecutionV1(options = {}) {
|
|
|
861
866
|
|
|
862
867
|
}
|
|
863
868
|
|
|
864
|
-
assertHybridLinearProjectionIsolation(execution, layerTypes, modelId);
|
|
869
|
+
assertHybridLinearProjectionIsolation(execution, layerTypes, modelId, weightDtype);
|
|
865
870
|
|
|
866
871
|
// Transformed graphs have null digests for derived kernels — skip digest
|
|
867
872
|
// validation since the original graph was already validated above.
|
|
@@ -1054,6 +1059,7 @@ export function applyExecutionV1RuntimeConfig(options = {}) {
|
|
|
1054
1059
|
modelId: manifest.modelId ?? options.modelId,
|
|
1055
1060
|
numLayers: options.numLayers ?? manifest.architecture?.numLayers ?? 0,
|
|
1056
1061
|
headDim: options.headDim ?? manifest.architecture?.headDim ?? null,
|
|
1062
|
+
weightDtype: manifest.quantizationInfo?.weights ?? null,
|
|
1057
1063
|
capabilities: options.capabilities ?? null,
|
|
1058
1064
|
platform: options.platform ?? null,
|
|
1059
1065
|
runtimeSession: hasExplicitRuntimeOverrides
|
|
@@ -1257,6 +1257,7 @@ export async function runDenseFFNGPU(
|
|
|
1257
1257
|
const activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
1258
1258
|
size: numTokens * intermediateSize,
|
|
1259
1259
|
gate: gateOutput,
|
|
1260
|
+
inputActivation: 'identity',
|
|
1260
1261
|
swigluLimit,
|
|
1261
1262
|
label: `L${layerIdx}.ffn_activation`,
|
|
1262
1263
|
layerIdx,
|
|
@@ -1646,6 +1647,7 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1646
1647
|
activatedOutput = await dispatchActivation(hiddenActivation, upOutput, {
|
|
1647
1648
|
size: numTokens * intermediateSize,
|
|
1648
1649
|
gate: gateOutput,
|
|
1650
|
+
inputActivation: 'identity',
|
|
1649
1651
|
swigluLimit,
|
|
1650
1652
|
}, recorder);
|
|
1651
1653
|
|
|
@@ -2277,6 +2277,27 @@ export class PipelineGenerator {
|
|
|
2277
2277
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2278
2278
|
this.#state.stats.prefillProfileSteps = [];
|
|
2279
2279
|
const opts = resolvePrefillOptions(this.#state, options);
|
|
2280
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2281
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2282
|
+
if (diagnosticsEnabled) {
|
|
2283
|
+
const captureConfig = {
|
|
2284
|
+
...createDefaultCaptureConfig(),
|
|
2285
|
+
enabled: true,
|
|
2286
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2287
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2288
|
+
};
|
|
2289
|
+
validateCaptureConfig(captureConfig);
|
|
2290
|
+
this.#state.operatorDiagnostics = {
|
|
2291
|
+
enabled: true,
|
|
2292
|
+
captureConfig,
|
|
2293
|
+
emitter: new OperatorEventEmitter({
|
|
2294
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2295
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2296
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2297
|
+
}),
|
|
2298
|
+
tsirFixture: null,
|
|
2299
|
+
};
|
|
2300
|
+
}
|
|
2280
2301
|
try {
|
|
2281
2302
|
const { inputIds, logits, phase: prefillPhase } = await this._prefillPromptToLogits(prompt, opts, 'prefillWithLogits');
|
|
2282
2303
|
|
|
@@ -2298,6 +2319,14 @@ export class PipelineGenerator {
|
|
|
2298
2319
|
};
|
|
2299
2320
|
} finally {
|
|
2300
2321
|
this._closeFinitenessFallbackWindow(opts);
|
|
2322
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2323
|
+
? {
|
|
2324
|
+
enabled: true,
|
|
2325
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2326
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2327
|
+
}
|
|
2328
|
+
: null;
|
|
2329
|
+
this.#state.operatorDiagnostics = null;
|
|
2301
2330
|
}
|
|
2302
2331
|
}
|
|
2303
2332
|
|
|
@@ -46,12 +46,12 @@ export async function applyLoRA(input, baseOutput, lora, dims, getWeightBuffer,
|
|
|
46
46
|
let scaled = null;
|
|
47
47
|
try {
|
|
48
48
|
loraIntermediate = recorder
|
|
49
|
-
? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB:
|
|
50
|
-
: await runMatmul(input, aBuf, M, rank, K, { transposeB:
|
|
49
|
+
? await recordMatmul(recorder, input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype })
|
|
50
|
+
: await runMatmul(input, aBuf, M, rank, K, { transposeB: false, role: 'lora_a', kernelPath, outputDtype: intermediateDtype });
|
|
51
51
|
|
|
52
52
|
loraOutput = recorder
|
|
53
|
-
? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB:
|
|
54
|
-
: await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB:
|
|
53
|
+
? await recordMatmul(recorder, loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype })
|
|
54
|
+
: await runMatmul(loraIntermediate, bBuf, M, N, rank, { transposeB: false, role: 'lora_b', kernelPath, outputDtype });
|
|
55
55
|
|
|
56
56
|
scaled = recorder
|
|
57
57
|
? await recordScale(recorder, loraOutput, lora.scale, { outputBuffer: null })
|
|
@@ -122,6 +122,7 @@ async function runExpertCPU(layerIdx, expertIdx, input, config, expertWeights) {
|
|
|
122
122
|
activatedOutput = await activationFn(upOutput, {
|
|
123
123
|
size: numTokens * intermediateSize,
|
|
124
124
|
gate: gateOutput,
|
|
125
|
+
inputActivation: 'identity',
|
|
125
126
|
swigluLimit,
|
|
126
127
|
});
|
|
127
128
|
|